projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
scsi: pm8001: Remove unused variable 'value'
[linux-2.6-block.git] / drivers / scsi / smartpqi / smartpqi_init.c
CommitLineData
2cc37b15 1// SPDX-License-Identifier: GPL-2.0
6c223761
KB		 2/*
3 * driver for Microsemi PQI-based storage controllers
2a712681 4 * Copyright (c) 2019-2020 Microchip Technology Inc. and its subsidiaries
2f4c4b92 5 * Copyright (c) 2016-2018 Microsemi Corporation
6c223761
KB		 6 * Copyright (c) 2016 PMC-Sierra, Inc.
7 *
2f4c4b92 8 * Questions/Comments/Bugfixes to storagedev@microchip.com
6c223761
KB		 9 *
10 */
11
12#include <linux/module.h>
13#include <linux/kernel.h>
14#include <linux/pci.h>
15#include <linux/delay.h>
16#include <linux/interrupt.h>
17#include <linux/sched.h>
18#include <linux/rtc.h>
19#include <linux/bcd.h>
3c50976f 20#include <linux/reboot.h>
6c223761 21#include <linux/cciss_ioctl.h>
52198226 22#include <linux/blk-mq-pci.h>
6c223761
KB		 23#include <scsi/scsi_host.h>
24#include <scsi/scsi_cmnd.h>
25#include <scsi/scsi_device.h>
26#include <scsi/scsi_eh.h>
27#include <scsi/scsi_transport_sas.h>
28#include <asm/unaligned.h>
29#include "smartpqi.h"
30#include "smartpqi_sis.h"
31
32#if !defined(BUILD_TIMESTAMP)
33#define BUILD_TIMESTAMP
34#endif
35
ce60a2b8 36#define DRIVER_VERSION "1.2.16-010"
2d154f5f 37#define DRIVER_MAJOR 1
f7cb8ac6 38#define DRIVER_MINOR 2
ce60a2b8
DB		 39#define DRIVER_RELEASE 16
40#define DRIVER_REVISION 10
6c223761 41
2d154f5f
KB		 42#define DRIVER_NAME "Microsemi PQI Driver (v" \
43 DRIVER_VERSION BUILD_TIMESTAMP ")"
6c223761
KB		 44#define DRIVER_NAME_SHORT "smartpqi"
45
e1d213bd
KB		 46#define PQI_EXTRA_SGL_MEMORY (12 * sizeof(struct pqi_sg_descriptor))
47
6c223761
KB		 48MODULE_AUTHOR("Microsemi");
49MODULE_DESCRIPTION("Driver for Microsemi Smart Family Controller version "
50 DRIVER_VERSION);
51MODULE_SUPPORTED_DEVICE("Microsemi Smart Family Controllers");
52MODULE_VERSION(DRIVER_VERSION);
53MODULE_LICENSE("GPL");
54
6c223761 55static void pqi_take_ctrl_offline(struct pqi_ctrl_info *ctrl_info);
5f310425 56static void pqi_ctrl_offline_worker(struct work_struct *work);
376fb880 57static void pqi_retry_raid_bypass_requests(struct pqi_ctrl_info *ctrl_info);
6c223761
KB		 58static int pqi_scan_scsi_devices(struct pqi_ctrl_info *ctrl_info);
59static void pqi_scan_start(struct Scsi_Host *shost);
60static void pqi_start_io(struct pqi_ctrl_info *ctrl_info,
61 struct pqi_queue_group *queue_group, enum pqi_io_path path,
62 struct pqi_io_request *io_request);
63static int pqi_submit_raid_request_synchronous(struct pqi_ctrl_info *ctrl_info,
64 struct pqi_iu_header *request, unsigned int flags,
65 struct pqi_raid_error_info *error_info, unsigned long timeout_msecs);
66static int pqi_aio_submit_io(struct pqi_ctrl_info *ctrl_info,
67 struct scsi_cmnd *scmd, u32 aio_handle, u8 *cdb,
68 unsigned int cdb_length, struct pqi_queue_group *queue_group,
376fb880 69 struct pqi_encryption_info *encryption_info, bool raid_bypass);
4fd22c13
MR		 70static void pqi_ofa_ctrl_quiesce(struct pqi_ctrl_info *ctrl_info);
71static void pqi_ofa_ctrl_unquiesce(struct pqi_ctrl_info *ctrl_info);
72static int pqi_ofa_ctrl_restart(struct pqi_ctrl_info *ctrl_info);
73static void pqi_ofa_setup_host_buffer(struct pqi_ctrl_info *ctrl_info,
74 u32 bytes_requested);
75static void pqi_ofa_free_host_buffer(struct pqi_ctrl_info *ctrl_info);
76static int pqi_ofa_host_memory_update(struct pqi_ctrl_info *ctrl_info);
1e46731e
MR		 77static int pqi_device_wait_for_pending_io(struct pqi_ctrl_info *ctrl_info,
78 struct pqi_scsi_dev *device, unsigned long timeout_secs);
6c223761
KB		 79
80/* for flags argument to pqi_submit_raid_request_synchronous() */
81#define PQI_SYNC_FLAGS_INTERRUPTABLE 0x1
82
83static struct scsi_transport_template *pqi_sas_transport_template;
84
85static atomic_t pqi_controller_count = ATOMIC_INIT(0);
86
3c50976f
KB		 87enum pqi_lockup_action {
88 NONE,
89 REBOOT,
90 PANIC
91};
92
93static enum pqi_lockup_action pqi_lockup_action = NONE;
94
95static struct {
96 enum pqi_lockup_action action;
97 char *name;
98} pqi_lockup_actions[] = {
99 {
100 .action = NONE,
101 .name = "none",
102 },
103 {
104 .action = REBOOT,
105 .name = "reboot",
106 },
107 {
108 .action = PANIC,
109 .name = "panic",
110 },
111};
112
6a50d6ad
KB		 113static unsigned int pqi_supported_event_types[] = {
114 PQI_EVENT_TYPE_HOTPLUG,
115 PQI_EVENT_TYPE_HARDWARE,
116 PQI_EVENT_TYPE_PHYSICAL_DEVICE,
117 PQI_EVENT_TYPE_LOGICAL_DEVICE,
4fd22c13 118 PQI_EVENT_TYPE_OFA,
6a50d6ad
KB		 119 PQI_EVENT_TYPE_AIO_STATE_CHANGE,
120 PQI_EVENT_TYPE_AIO_CONFIG_CHANGE,
121};
122
6c223761
KB		 123static int pqi_disable_device_id_wildcards;
124module_param_named(disable_device_id_wildcards,
cbe0c7b1 125 pqi_disable_device_id_wildcards, int, 0644);
6c223761
KB		 126MODULE_PARM_DESC(disable_device_id_wildcards,
127 "Disable device ID wildcards.");
128
5a259e32
KB		 129static int pqi_disable_heartbeat;
130module_param_named(disable_heartbeat,
131 pqi_disable_heartbeat, int, 0644);
132MODULE_PARM_DESC(disable_heartbeat,
133 "Disable heartbeat.");
134
135static int pqi_disable_ctrl_shutdown;
136module_param_named(disable_ctrl_shutdown,
137 pqi_disable_ctrl_shutdown, int, 0644);
138MODULE_PARM_DESC(disable_ctrl_shutdown,
139 "Disable controller shutdown when controller locked up.");
140
3c50976f
KB		 141static char *pqi_lockup_action_param;
142module_param_named(lockup_action,
143 pqi_lockup_action_param, charp, 0644);
144MODULE_PARM_DESC(lockup_action, "Action to take when controller locked up.\n"
145 "\t\tSupported: none, reboot, panic\n"
146 "\t\tDefault: none");
147
5e6a9760
GW		 148static int pqi_expose_ld_first;
149module_param_named(expose_ld_first,
150 pqi_expose_ld_first, int, 0644);
151MODULE_PARM_DESC(expose_ld_first,
152 "Expose logical drives before physical drives.");
153
522bc026
DC		 154static int pqi_hide_vsep;
155module_param_named(hide_vsep,
156 pqi_hide_vsep, int, 0644);
157MODULE_PARM_DESC(hide_vsep,
158 "Hide the virtual SEP for direct attached drives.");
159
6c223761
KB		 160static char *raid_levels[] = {
161 "RAID-0",
162 "RAID-4",
163 "RAID-1(1+0)",
164 "RAID-5",
165 "RAID-5+1",
166 "RAID-ADG",
167 "RAID-1(ADM)",
168};
169
170static char *pqi_raid_level_to_string(u8 raid_level)
171{
172 if (raid_level < ARRAY_SIZE(raid_levels))
173 return raid_levels[raid_level];
174
a9f93392 175 return "RAID UNKNOWN";
6c223761
KB		 176}
177
178#define SA_RAID_0 0
179#define SA_RAID_4 1
180#define SA_RAID_1 2 /* also used for RAID 10 */
181#define SA_RAID_5 3 /* also used for RAID 50 */
182#define SA_RAID_51 4
183#define SA_RAID_6 5 /* also used for RAID 60 */
184#define SA_RAID_ADM 6 /* also used for RAID 1+0 ADM */
185#define SA_RAID_MAX SA_RAID_ADM
186#define SA_RAID_UNKNOWN 0xff
187
188static inline void pqi_scsi_done(struct scsi_cmnd *scmd)
189{
7561a7e4 190 pqi_prep_for_scsi_done(scmd);
6c223761
KB		 191 scmd->scsi_done(scmd);
192}
193
b6e2ef67 194static inline void pqi_disable_write_same(struct scsi_device *sdev)
6c223761 195{
b6e2ef67 196 sdev->no_write_same = 1;
6c223761
KB		 197}
198
6c223761 199static inline bool pqi_scsi3addr_equal(u8 *scsi3addr1, u8 *scsi3addr2)
6c223761 200{
6c223761 201 return memcmp(scsi3addr1, scsi3addr2, 8) == 0;
6c223761
KB		 202}
203
204static inline bool pqi_is_logical_device(struct pqi_scsi_dev *device)
205{
206 return !device->is_physical_device;
207}
208
bd10cf0b
KB		 209static inline bool pqi_is_external_raid_addr(u8 *scsi3addr)
210{
211 return scsi3addr[2] != 0;
212}
213
694c5d5b
KB		 214static inline bool pqi_ctrl_offline(struct pqi_ctrl_info *ctrl_info)
215{
216 return !ctrl_info->controller_online;
217}
218
6c223761
KB		 219static inline void pqi_check_ctrl_health(struct pqi_ctrl_info *ctrl_info)
220{
221 if (ctrl_info->controller_online)
222 if (!sis_is_firmware_running(ctrl_info))
223 pqi_take_ctrl_offline(ctrl_info);
224}
225
226static inline bool pqi_is_hba_lunid(u8 *scsi3addr)
227{
228 return pqi_scsi3addr_equal(scsi3addr, RAID_CTLR_LUNID);
229}
230
ff6abb73
KB		 231static inline enum pqi_ctrl_mode pqi_get_ctrl_mode(
232 struct pqi_ctrl_info *ctrl_info)
233{
234 return sis_read_driver_scratch(ctrl_info);
235}
236
237static inline void pqi_save_ctrl_mode(struct pqi_ctrl_info *ctrl_info,
238 enum pqi_ctrl_mode mode)
239{
240 sis_write_driver_scratch(ctrl_info, mode);
241}
242
694c5d5b
KB		 243static inline void pqi_ctrl_block_device_reset(struct pqi_ctrl_info *ctrl_info)
244{
245 ctrl_info->block_device_reset = true;
246}
247
248static inline bool pqi_device_reset_blocked(struct pqi_ctrl_info *ctrl_info)
249{
250 return ctrl_info->block_device_reset;
251}
252
253static inline bool pqi_ctrl_blocked(struct pqi_ctrl_info *ctrl_info)
254{
255 return ctrl_info->block_requests;
256}
257
7561a7e4
KB		 258static inline void pqi_ctrl_block_requests(struct pqi_ctrl_info *ctrl_info)
259{
260 ctrl_info->block_requests = true;
261 scsi_block_requests(ctrl_info->scsi_host);
262}
263
264static inline void pqi_ctrl_unblock_requests(struct pqi_ctrl_info *ctrl_info)
265{
266 ctrl_info->block_requests = false;
267 wake_up_all(&ctrl_info->block_requests_wait);
376fb880 268 pqi_retry_raid_bypass_requests(ctrl_info);
7561a7e4
KB		 269 scsi_unblock_requests(ctrl_info->scsi_host);
270}
271
7561a7e4
KB		 272static unsigned long pqi_wait_if_ctrl_blocked(struct pqi_ctrl_info *ctrl_info,
273 unsigned long timeout_msecs)
274{
275 unsigned long remaining_msecs;
276
277 if (!pqi_ctrl_blocked(ctrl_info))
278 return timeout_msecs;
279
280 atomic_inc(&ctrl_info->num_blocked_threads);
281
282 if (timeout_msecs == NO_TIMEOUT) {
283 wait_event(ctrl_info->block_requests_wait,
284 !pqi_ctrl_blocked(ctrl_info));
285 remaining_msecs = timeout_msecs;
286 } else {
287 unsigned long remaining_jiffies;
288
289 remaining_jiffies =
290 wait_event_timeout(ctrl_info->block_requests_wait,
291 !pqi_ctrl_blocked(ctrl_info),
292 msecs_to_jiffies(timeout_msecs));
293 remaining_msecs = jiffies_to_msecs(remaining_jiffies);
294 }
295
296 atomic_dec(&ctrl_info->num_blocked_threads);
297
298 return remaining_msecs;
299}
300
7561a7e4
KB		 301static inline void pqi_ctrl_wait_until_quiesced(struct pqi_ctrl_info *ctrl_info)
302{
303 while (atomic_read(&ctrl_info->num_busy_threads) >
304 atomic_read(&ctrl_info->num_blocked_threads))
305 usleep_range(1000, 2000);
306}
307
03b288cf
KB		 308static inline bool pqi_device_offline(struct pqi_scsi_dev *device)
309{
310 return device->device_offline;
311}
312
7561a7e4
KB		 313static inline void pqi_device_reset_start(struct pqi_scsi_dev *device)
314{
315 device->in_reset = true;
316}
317
318static inline void pqi_device_reset_done(struct pqi_scsi_dev *device)
319{
320 device->in_reset = false;
321}
322
323static inline bool pqi_device_in_reset(struct pqi_scsi_dev *device)
324{
325 return device->in_reset;
326}
6c223761 327
4fd22c13
MR		 328static inline void pqi_ctrl_ofa_start(struct pqi_ctrl_info *ctrl_info)
329{
330 ctrl_info->in_ofa = true;
331}
332
333static inline void pqi_ctrl_ofa_done(struct pqi_ctrl_info *ctrl_info)
334{
335 ctrl_info->in_ofa = false;
336}
337
338static inline bool pqi_ctrl_in_ofa(struct pqi_ctrl_info *ctrl_info)
339{
340 return ctrl_info->in_ofa;
341}
342
1e46731e
MR		 343static inline void pqi_device_remove_start(struct pqi_scsi_dev *device)
344{
345 device->in_remove = true;
346}
347
348static inline bool pqi_device_in_remove(struct pqi_ctrl_info *ctrl_info,
349 struct pqi_scsi_dev *device)
350{
12de280b 351 return device->in_remove && !ctrl_info->in_shutdown;
1e46731e
MR		 352}
353
0530736e
KB		 354static inline void pqi_ctrl_shutdown_start(struct pqi_ctrl_info *ctrl_info)
355{
356 ctrl_info->in_shutdown = true;
357}
358
359static inline bool pqi_ctrl_in_shutdown(struct pqi_ctrl_info *ctrl_info)
360{
361 return ctrl_info->in_shutdown;
362}
363
5f310425
KB		 364static inline void pqi_schedule_rescan_worker_with_delay(
365 struct pqi_ctrl_info *ctrl_info, unsigned long delay)
366{
367 if (pqi_ctrl_offline(ctrl_info))
368 return;
4fd22c13
MR		 369 if (pqi_ctrl_in_ofa(ctrl_info))
370 return;
5f310425
KB		 371
372 schedule_delayed_work(&ctrl_info->rescan_work, delay);
373}
374
6c223761
KB		 375static inline void pqi_schedule_rescan_worker(struct pqi_ctrl_info *ctrl_info)
376{
5f310425
KB		 377 pqi_schedule_rescan_worker_with_delay(ctrl_info, 0);
378}
379
4fd22c13 380#define PQI_RESCAN_WORK_DELAY (10 * PQI_HZ)
5f310425
KB		 381
382static inline void pqi_schedule_rescan_worker_delayed(
383 struct pqi_ctrl_info *ctrl_info)
384{
385 pqi_schedule_rescan_worker_with_delay(ctrl_info, PQI_RESCAN_WORK_DELAY);
6c223761
KB		 386}
387
061ef06a
KB		 388static inline void pqi_cancel_rescan_worker(struct pqi_ctrl_info *ctrl_info)
389{
390 cancel_delayed_work_sync(&ctrl_info->rescan_work);
391}
392
0530736e
KB		 393static inline void pqi_cancel_event_worker(struct pqi_ctrl_info *ctrl_info)
394{
395 cancel_work_sync(&ctrl_info->event_work);
396}
397
98f87667
KB		 398static inline u32 pqi_read_heartbeat_counter(struct pqi_ctrl_info *ctrl_info)
399{
400 if (!ctrl_info->heartbeat_counter)
401 return 0;
402
403 return readl(ctrl_info->heartbeat_counter);
404}
405
4fd22c13
MR		 406static inline u8 pqi_read_soft_reset_status(struct pqi_ctrl_info *ctrl_info)
407{
408 if (!ctrl_info->soft_reset_status)
409 return 0;
410
411 return readb(ctrl_info->soft_reset_status);
412}
413
414static inline void pqi_clear_soft_reset_status(struct pqi_ctrl_info *ctrl_info,
694c5d5b 415 u8 clear)
4fd22c13
MR		 416{
417 u8 status;
418
419 if (!ctrl_info->soft_reset_status)
420 return;
421
422 status = pqi_read_soft_reset_status(ctrl_info);
423 status &= ~clear;
424 writeb(status, ctrl_info->soft_reset_status);
425}
426
6c223761
KB		 427static int pqi_map_single(struct pci_dev *pci_dev,
428 struct pqi_sg_descriptor *sg_descriptor, void *buffer,
6917a9cc 429 size_t buffer_length, enum dma_data_direction data_direction)
6c223761
KB		 430{
431 dma_addr_t bus_address;
432
6917a9cc 433 if (!buffer || buffer_length == 0 || data_direction == DMA_NONE)
6c223761
KB		 434 return 0;
435
6917a9cc 436 bus_address = dma_map_single(&pci_dev->dev, buffer, buffer_length,
6c223761 437 data_direction);
6917a9cc 438 if (dma_mapping_error(&pci_dev->dev, bus_address))
6c223761
KB		 439 return -ENOMEM;
440
441 put_unaligned_le64((u64)bus_address, &sg_descriptor->address);
442 put_unaligned_le32(buffer_length, &sg_descriptor->length);
443 put_unaligned_le32(CISS_SG_LAST, &sg_descriptor->flags);
444
445 return 0;
446}
447
448static void pqi_pci_unmap(struct pci_dev *pci_dev,
449 struct pqi_sg_descriptor *descriptors, int num_descriptors,
6917a9cc 450 enum dma_data_direction data_direction)
6c223761
KB		 451{
452 int i;
453
6917a9cc 454 if (data_direction == DMA_NONE)
6c223761
KB		 455 return;
456
457 for (i = 0; i < num_descriptors; i++)
6917a9cc 458 dma_unmap_single(&pci_dev->dev,
6c223761
KB		 459 (dma_addr_t)get_unaligned_le64(&descriptors[i].address),
460 get_unaligned_le32(&descriptors[i].length),
461 data_direction);
462}
463
464static int pqi_build_raid_path_request(struct pqi_ctrl_info *ctrl_info,
465 struct pqi_raid_path_request *request, u8 cmd,
466 u8 *scsi3addr, void *buffer, size_t buffer_length,
6917a9cc 467 u16 vpd_page, enum dma_data_direction *dir)
6c223761
KB		 468{
469 u8 *cdb;
171c2865 470 size_t cdb_length = buffer_length;
6c223761
KB		 471
472 memset(request, 0, sizeof(*request));
473
474 request->header.iu_type = PQI_REQUEST_IU_RAID_PATH_IO;
475 put_unaligned_le16(offsetof(struct pqi_raid_path_request,
476 sg_descriptors[1]) - PQI_REQUEST_HEADER_LENGTH,
477 &request->header.iu_length);
478 put_unaligned_le32(buffer_length, &request->buffer_length);
479 memcpy(request->lun_number, scsi3addr, sizeof(request->lun_number));
480 request->task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
481 request->additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_0;
482
483 cdb = request->cdb;
484
485 switch (cmd) {
486 case INQUIRY:
487 request->data_direction = SOP_READ_FLAG;
488 cdb[0] = INQUIRY;
489 if (vpd_page & VPD_PAGE) {
490 cdb[1] = 0x1;
491 cdb[2] = (u8)vpd_page;
492 }
171c2865 493 cdb[4] = (u8)cdb_length;
6c223761
KB		 494 break;
495 case CISS_REPORT_LOG:
496 case CISS_REPORT_PHYS:
497 request->data_direction = SOP_READ_FLAG;
498 cdb[0] = cmd;
499 if (cmd == CISS_REPORT_PHYS)
694c5d5b 500 cdb[1] = CISS_REPORT_PHYS_FLAG_OTHER;
6c223761 501 else
694c5d5b 502 cdb[1] = CISS_REPORT_LOG_FLAG_UNIQUE_LUN_ID;
171c2865 503 put_unaligned_be32(cdb_length, &cdb[6]);
6c223761
KB		 504 break;
505 case CISS_GET_RAID_MAP:
506 request->data_direction = SOP_READ_FLAG;
507 cdb[0] = CISS_READ;
508 cdb[1] = CISS_GET_RAID_MAP;
171c2865 509 put_unaligned_be32(cdb_length, &cdb[6]);
6c223761 510 break;
58322fe0 511 case SA_FLUSH_CACHE:
6c223761
KB		 512 request->data_direction = SOP_WRITE_FLAG;
513 cdb[0] = BMIC_WRITE;
58322fe0 514 cdb[6] = BMIC_FLUSH_CACHE;
171c2865 515 put_unaligned_be16(cdb_length, &cdb[7]);
6c223761 516 break;
171c2865
DC		 517 case BMIC_SENSE_DIAG_OPTIONS:
518 cdb_length = 0;
df561f66 519 fallthrough;
6c223761
KB		 520 case BMIC_IDENTIFY_CONTROLLER:
521 case BMIC_IDENTIFY_PHYSICAL_DEVICE:
6d90615f 522 case BMIC_SENSE_SUBSYSTEM_INFORMATION:
6c223761
KB		 523 request->data_direction = SOP_READ_FLAG;
524 cdb[0] = BMIC_READ;
525 cdb[6] = cmd;
171c2865 526 put_unaligned_be16(cdb_length, &cdb[7]);
6c223761 527 break;
171c2865
DC		 528 case BMIC_SET_DIAG_OPTIONS:
529 cdb_length = 0;
df561f66 530 fallthrough;
6c223761
KB		 531 case BMIC_WRITE_HOST_WELLNESS:
532 request->data_direction = SOP_WRITE_FLAG;
533 cdb[0] = BMIC_WRITE;
534 cdb[6] = cmd;
171c2865 535 put_unaligned_be16(cdb_length, &cdb[7]);
6c223761 536 break;
3d46a59a
DB		 537 case BMIC_CSMI_PASSTHRU:
538 request->data_direction = SOP_BIDIRECTIONAL;
539 cdb[0] = BMIC_WRITE;
540 cdb[5] = CSMI_CC_SAS_SMP_PASSTHRU;
541 cdb[6] = cmd;
542 put_unaligned_be16(cdb_length, &cdb[7]);
6c223761
KB		 543 break;
544 default:
9e68cccc 545 dev_err(&ctrl_info->pci_dev->dev, "unknown command 0x%c\n", cmd);
6c223761
KB		 546 break;
547 }
548
549 switch (request->data_direction) {
550 case SOP_READ_FLAG:
6917a9cc 551 *dir = DMA_FROM_DEVICE;
6c223761
KB		 552 break;
553 case SOP_WRITE_FLAG:
6917a9cc 554 *dir = DMA_TO_DEVICE;
6c223761
KB		 555 break;
556 case SOP_NO_DIRECTION_FLAG:
6917a9cc 557 *dir = DMA_NONE;
6c223761
KB		 558 break;
559 default:
6917a9cc 560 *dir = DMA_BIDIRECTIONAL;
6c223761
KB		 561 break;
562 }
563
6c223761 564 return pqi_map_single(ctrl_info->pci_dev, &request->sg_descriptors[0],
6917a9cc 565 buffer, buffer_length, *dir);
6c223761
KB		 566}
567
376fb880
KB		 568static inline void pqi_reinit_io_request(struct pqi_io_request *io_request)
569{
570 io_request->scmd = NULL;
571 io_request->status = 0;
572 io_request->error_info = NULL;
573 io_request->raid_bypass = false;
574}
575
6c223761
KB		 576static struct pqi_io_request *pqi_alloc_io_request(
577 struct pqi_ctrl_info *ctrl_info)
578{
579 struct pqi_io_request *io_request;
580 u16 i = ctrl_info->next_io_request_slot; /* benignly racy */
581
582 while (1) {
583 io_request = &ctrl_info->io_request_pool[i];
584 if (atomic_inc_return(&io_request->refcount) == 1)
585 break;
586 atomic_dec(&io_request->refcount);
587 i = (i + 1) % ctrl_info->max_io_slots;
588 }
589
590 /* benignly racy */
591 ctrl_info->next_io_request_slot = (i + 1) % ctrl_info->max_io_slots;
592
376fb880 593 pqi_reinit_io_request(io_request);
6c223761
KB		 594
595 return io_request;
596}
597
598static void pqi_free_io_request(struct pqi_io_request *io_request)
599{
600 atomic_dec(&io_request->refcount);
601}
602
02133b68 603static int pqi_send_scsi_raid_request(struct pqi_ctrl_info *ctrl_info, u8 cmd,
694c5d5b
KB		 604 u8 *scsi3addr, void *buffer, size_t buffer_length, u16 vpd_page,
605 struct pqi_raid_error_info *error_info, unsigned long timeout_msecs)
6c223761
KB		 606{
607 int rc;
6c223761 608 struct pqi_raid_path_request request;
694c5d5b 609 enum dma_data_direction dir;
6c223761
KB		 610
611 rc = pqi_build_raid_path_request(ctrl_info, &request,
02133b68
DC		 612 cmd, scsi3addr, buffer,
613 buffer_length, vpd_page, &dir);
6c223761
KB		 614 if (rc)
615 return rc;
616
694c5d5b
KB		 617 rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0,
618 error_info, timeout_msecs);
6c223761 619
6917a9cc 620 pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1, dir);
694c5d5b 621
6c223761
KB		 622 return rc;
623}
624
694c5d5b 625/* helper functions for pqi_send_scsi_raid_request */
02133b68
DC		 626
627static inline int pqi_send_ctrl_raid_request(struct pqi_ctrl_info *ctrl_info,
694c5d5b 628 u8 cmd, void *buffer, size_t buffer_length)
6c223761 629{
02133b68 630 return pqi_send_scsi_raid_request(ctrl_info, cmd, RAID_CTLR_LUNID,
694c5d5b 631 buffer, buffer_length, 0, NULL, NO_TIMEOUT);
02133b68 632}
6c223761 633
02133b68 634static inline int pqi_send_ctrl_raid_with_error(struct pqi_ctrl_info *ctrl_info,
694c5d5b
KB		 635 u8 cmd, void *buffer, size_t buffer_length,
636 struct pqi_raid_error_info *error_info)
02133b68
DC		 637{
638 return pqi_send_scsi_raid_request(ctrl_info, cmd, RAID_CTLR_LUNID,
694c5d5b 639 buffer, buffer_length, 0, error_info, NO_TIMEOUT);
02133b68 640}
6c223761 641
02133b68 642static inline int pqi_identify_controller(struct pqi_ctrl_info *ctrl_info,
694c5d5b 643 struct bmic_identify_controller *buffer)
02133b68
DC		 644{
645 return pqi_send_ctrl_raid_request(ctrl_info, BMIC_IDENTIFY_CONTROLLER,
694c5d5b 646 buffer, sizeof(*buffer));
02133b68
DC		 647}
648
6d90615f 649static inline int pqi_sense_subsystem_info(struct pqi_ctrl_info *ctrl_info,
694c5d5b 650 struct bmic_sense_subsystem_info *sense_info)
6d90615f
MB		 651{
652 return pqi_send_ctrl_raid_request(ctrl_info,
694c5d5b
KB		 653 BMIC_SENSE_SUBSYSTEM_INFORMATION, sense_info,
654 sizeof(*sense_info));
6d90615f
MB		 655}
656
02133b68 657static inline int pqi_scsi_inquiry(struct pqi_ctrl_info *ctrl_info,
6c223761 658 u8 *scsi3addr, u16 vpd_page, void *buffer, size_t buffer_length)
02133b68
DC		 659{
660 return pqi_send_scsi_raid_request(ctrl_info, INQUIRY, scsi3addr,
661 buffer, buffer_length, vpd_page, NULL, NO_TIMEOUT);
6c223761
KB		 662}
663
664static int pqi_identify_physical_device(struct pqi_ctrl_info *ctrl_info,
665 struct pqi_scsi_dev *device,
694c5d5b 666 struct bmic_identify_physical_device *buffer, size_t buffer_length)
6c223761
KB		 667{
668 int rc;
6917a9cc 669 enum dma_data_direction dir;
6c223761
KB		 670 u16 bmic_device_index;
671 struct pqi_raid_path_request request;
672
673 rc = pqi_build_raid_path_request(ctrl_info, &request,
674 BMIC_IDENTIFY_PHYSICAL_DEVICE, RAID_CTLR_LUNID, buffer,
6917a9cc 675 buffer_length, 0, &dir);
6c223761
KB		 676 if (rc)
677 return rc;
678
679 bmic_device_index = CISS_GET_DRIVE_NUMBER(device->scsi3addr);
680 request.cdb[2] = (u8)bmic_device_index;
681 request.cdb[9] = (u8)(bmic_device_index >> 8);
682
683 rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header,
684 0, NULL, NO_TIMEOUT);
685
6917a9cc 686 pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1, dir);
694c5d5b 687
6c223761
KB		 688 return rc;
689}
690
58322fe0
KB		 691static int pqi_flush_cache(struct pqi_ctrl_info *ctrl_info,
692 enum bmic_flush_cache_shutdown_event shutdown_event)
6c223761
KB		 693{
694 int rc;
58322fe0 695 struct bmic_flush_cache *flush_cache;
6c223761
KB		 696
697 /*
698 * Don't bother trying to flush the cache if the controller is
699 * locked up.
700 */
701 if (pqi_ctrl_offline(ctrl_info))
702 return -ENXIO;
703
58322fe0
KB		 704 flush_cache = kzalloc(sizeof(*flush_cache), GFP_KERNEL);
705 if (!flush_cache)
6c223761
KB		 706 return -ENOMEM;
707
58322fe0
KB		 708 flush_cache->shutdown_event = shutdown_event;
709
02133b68
DC		 710 rc = pqi_send_ctrl_raid_request(ctrl_info, SA_FLUSH_CACHE, flush_cache,
711 sizeof(*flush_cache));
6c223761 712
58322fe0 713 kfree(flush_cache);
6c223761
KB		 714
715 return rc;
716}
717
3d46a59a
DB		 718int pqi_csmi_smp_passthru(struct pqi_ctrl_info *ctrl_info,
719 struct bmic_csmi_smp_passthru_buffer *buffer, size_t buffer_length,
720 struct pqi_raid_error_info *error_info)
721{
722 return pqi_send_ctrl_raid_with_error(ctrl_info, BMIC_CSMI_PASSTHRU,
723 buffer, buffer_length, error_info);
724}
171c2865 725
694c5d5b 726#define PQI_FETCH_PTRAID_DATA (1 << 31)
171c2865
DC		 727
728static int pqi_set_diag_rescan(struct pqi_ctrl_info *ctrl_info)
6c223761
KB		 729{
730 int rc;
171c2865 731 struct bmic_diag_options *diag;
6c223761 732
171c2865
DC		 733 diag = kzalloc(sizeof(*diag), GFP_KERNEL);
734 if (!diag)
735 return -ENOMEM;
736
02133b68 737 rc = pqi_send_ctrl_raid_request(ctrl_info, BMIC_SENSE_DIAG_OPTIONS,
694c5d5b 738 diag, sizeof(*diag));
6c223761 739 if (rc)
171c2865 740 goto out;
6c223761 741
171c2865
DC		 742 diag->options |= cpu_to_le32(PQI_FETCH_PTRAID_DATA);
743
694c5d5b
KB		 744 rc = pqi_send_ctrl_raid_request(ctrl_info, BMIC_SET_DIAG_OPTIONS, diag,
745 sizeof(*diag));
746
171c2865
DC		 747out:
748 kfree(diag);
6c223761 749
6c223761
KB		 750 return rc;
751}
752
02133b68 753static inline int pqi_write_host_wellness(struct pqi_ctrl_info *ctrl_info,
6c223761
KB		 754 void *buffer, size_t buffer_length)
755{
02133b68 756 return pqi_send_ctrl_raid_request(ctrl_info, BMIC_WRITE_HOST_WELLNESS,
694c5d5b 757 buffer, buffer_length);
6c223761
KB		 758}
759
760#pragma pack(1)
761
762struct bmic_host_wellness_driver_version {
763 u8 start_tag[4];
764 u8 driver_version_tag[2];
765 __le16 driver_version_length;
766 char driver_version[32];
b2346b50 767 u8 dont_write_tag[2];
6c223761
KB		 768 u8 end_tag[2];
769};
770
771#pragma pack()
772
773static int pqi_write_driver_version_to_host_wellness(
774 struct pqi_ctrl_info *ctrl_info)
775{
776 int rc;
777 struct bmic_host_wellness_driver_version *buffer;
778 size_t buffer_length;
779
780 buffer_length = sizeof(*buffer);
781
782 buffer = kmalloc(buffer_length, GFP_KERNEL);
783 if (!buffer)
784 return -ENOMEM;
785
786 buffer->start_tag[0] = '<';
787 buffer->start_tag[1] = 'H';
788 buffer->start_tag[2] = 'W';
789 buffer->start_tag[3] = '>';
790 buffer->driver_version_tag[0] = 'D';
791 buffer->driver_version_tag[1] = 'V';
792 put_unaligned_le16(sizeof(buffer->driver_version),
793 &buffer->driver_version_length);
061ef06a 794 strncpy(buffer->driver_version, "Linux " DRIVER_VERSION,
6c223761
KB		 795 sizeof(buffer->driver_version) - 1);
796 buffer->driver_version[sizeof(buffer->driver_version) - 1] = '\0';
b2346b50
MR		 797 buffer->dont_write_tag[0] = 'D';
798 buffer->dont_write_tag[1] = 'W';
6c223761
KB		 799 buffer->end_tag[0] = 'Z';
800 buffer->end_tag[1] = 'Z';
801
802 rc = pqi_write_host_wellness(ctrl_info, buffer, buffer_length);
803
804 kfree(buffer);
805
806 return rc;
807}
808
809#pragma pack(1)
810
811struct bmic_host_wellness_time {
812 u8 start_tag[4];
813 u8 time_tag[2];
814 __le16 time_length;
815 u8 time[8];
816 u8 dont_write_tag[2];
817 u8 end_tag[2];
818};
819
820#pragma pack()
821
822static int pqi_write_current_time_to_host_wellness(
823 struct pqi_ctrl_info *ctrl_info)
824{
825 int rc;
826 struct bmic_host_wellness_time *buffer;
827 size_t buffer_length;
828 time64_t local_time;
829 unsigned int year;
ed10858e 830 struct tm tm;
6c223761
KB		 831
832 buffer_length = sizeof(*buffer);
833
834 buffer = kmalloc(buffer_length, GFP_KERNEL);
835 if (!buffer)
836 return -ENOMEM;
837
838 buffer->start_tag[0] = '<';
839 buffer->start_tag[1] = 'H';
840 buffer->start_tag[2] = 'W';
841 buffer->start_tag[3] = '>';
842 buffer->time_tag[0] = 'T';
843 buffer->time_tag[1] = 'D';
844 put_unaligned_le16(sizeof(buffer->time),
845 &buffer->time_length);
846
ed10858e
AB		 847 local_time = ktime_get_real_seconds();
848 time64_to_tm(local_time, -sys_tz.tz_minuteswest * 60, &tm);
6c223761
KB		 849 year = tm.tm_year + 1900;
850
851 buffer->time[0] = bin2bcd(tm.tm_hour);
852 buffer->time[1] = bin2bcd(tm.tm_min);
853 buffer->time[2] = bin2bcd(tm.tm_sec);
854 buffer->time[3] = 0;
855 buffer->time[4] = bin2bcd(tm.tm_mon + 1);
856 buffer->time[5] = bin2bcd(tm.tm_mday);
857 buffer->time[6] = bin2bcd(year / 100);
858 buffer->time[7] = bin2bcd(year % 100);
859
860 buffer->dont_write_tag[0] = 'D';
861 buffer->dont_write_tag[1] = 'W';
862 buffer->end_tag[0] = 'Z';
863 buffer->end_tag[1] = 'Z';
864
865 rc = pqi_write_host_wellness(ctrl_info, buffer, buffer_length);
866
867 kfree(buffer);
868
869 return rc;
870}
871
4fd22c13 872#define PQI_UPDATE_TIME_WORK_INTERVAL (24UL * 60 * 60 * PQI_HZ)
6c223761
KB		 873
874static void pqi_update_time_worker(struct work_struct *work)
875{
876 int rc;
877 struct pqi_ctrl_info *ctrl_info;
878
879 ctrl_info = container_of(to_delayed_work(work), struct pqi_ctrl_info,
880 update_time_work);
881
5f310425
KB		 882 if (pqi_ctrl_offline(ctrl_info))
883 return;
884
6c223761
KB		 885 rc = pqi_write_current_time_to_host_wellness(ctrl_info);
886 if (rc)
887 dev_warn(&ctrl_info->pci_dev->dev,
888 "error updating time on controller\n");
889
890 schedule_delayed_work(&ctrl_info->update_time_work,
891 PQI_UPDATE_TIME_WORK_INTERVAL);
892}
893
894static inline void pqi_schedule_update_time_worker(
4fbebf1a 895 struct pqi_ctrl_info *ctrl_info)
6c223761 896{
4fbebf1a 897 schedule_delayed_work(&ctrl_info->update_time_work, 0);
061ef06a
KB		 898}
899
900static inline void pqi_cancel_update_time_worker(
901 struct pqi_ctrl_info *ctrl_info)
902{
061ef06a 903 cancel_delayed_work_sync(&ctrl_info->update_time_work);
6c223761
KB		 904}
905
02133b68 906static inline int pqi_report_luns(struct pqi_ctrl_info *ctrl_info, u8 cmd,
6c223761
KB		 907 void *buffer, size_t buffer_length)
908{
02133b68 909 return pqi_send_ctrl_raid_request(ctrl_info, cmd, buffer,
694c5d5b 910 buffer_length);
6c223761
KB		 911}
912
913static int pqi_report_phys_logical_luns(struct pqi_ctrl_info *ctrl_info, u8 cmd,
914 void **buffer)
915{
916 int rc;
917 size_t lun_list_length;
918 size_t lun_data_length;
919 size_t new_lun_list_length;
920 void *lun_data = NULL;
921 struct report_lun_header *report_lun_header;
922
923 report_lun_header = kmalloc(sizeof(*report_lun_header), GFP_KERNEL);
924 if (!report_lun_header) {
925 rc = -ENOMEM;
926 goto out;
927 }
928
929 rc = pqi_report_luns(ctrl_info, cmd, report_lun_header,
930 sizeof(*report_lun_header));
931 if (rc)
932 goto out;
933
934 lun_list_length = get_unaligned_be32(&report_lun_header->list_length);
935
936again:
937 lun_data_length = sizeof(struct report_lun_header) + lun_list_length;
938
939 lun_data = kmalloc(lun_data_length, GFP_KERNEL);
940 if (!lun_data) {
941 rc = -ENOMEM;
942 goto out;
943 }
944
945 if (lun_list_length == 0) {
946 memcpy(lun_data, report_lun_header, sizeof(*report_lun_header));
947 goto out;
948 }
949
950 rc = pqi_report_luns(ctrl_info, cmd, lun_data, lun_data_length);
951 if (rc)
952 goto out;
953
954 new_lun_list_length = get_unaligned_be32(
955 &((struct report_lun_header *)lun_data)->list_length);
956
957 if (new_lun_list_length > lun_list_length) {
958 lun_list_length = new_lun_list_length;
959 kfree(lun_data);
960 goto again;
961 }
962
963out:
964 kfree(report_lun_header);
965
966 if (rc) {
967 kfree(lun_data);
968 lun_data = NULL;
969 }
970
971 *buffer = lun_data;
972
973 return rc;
974}
975
976static inline int pqi_report_phys_luns(struct pqi_ctrl_info *ctrl_info,
977 void **buffer)
978{
979 return pqi_report_phys_logical_luns(ctrl_info, CISS_REPORT_PHYS,
980 buffer);
981}
982
983static inline int pqi_report_logical_luns(struct pqi_ctrl_info *ctrl_info,
984 void **buffer)
985{
986 return pqi_report_phys_logical_luns(ctrl_info, CISS_REPORT_LOG, buffer);
987}
988
989static int pqi_get_device_lists(struct pqi_ctrl_info *ctrl_info,
990 struct report_phys_lun_extended **physdev_list,
991 struct report_log_lun_extended **logdev_list)
992{
993 int rc;
994 size_t logdev_list_length;
995 size_t logdev_data_length;
996 struct report_log_lun_extended *internal_logdev_list;
997 struct report_log_lun_extended *logdev_data;
998 struct report_lun_header report_lun_header;
999
1000 rc = pqi_report_phys_luns(ctrl_info, (void **)physdev_list);
1001 if (rc)
1002 dev_err(&ctrl_info->pci_dev->dev,
1003 "report physical LUNs failed\n");
1004
1005 rc = pqi_report_logical_luns(ctrl_info, (void **)logdev_list);
1006 if (rc)
1007 dev_err(&ctrl_info->pci_dev->dev,
1008 "report logical LUNs failed\n");
1009
1010 /*
1011 * Tack the controller itself onto the end of the logical device list.
1012 */
1013
1014 logdev_data = *logdev_list;
1015
1016 if (logdev_data) {
1017 logdev_list_length =
1018 get_unaligned_be32(&logdev_data->header.list_length);
1019 } else {
1020 memset(&report_lun_header, 0, sizeof(report_lun_header));
1021 logdev_data =
1022 (struct report_log_lun_extended *)&report_lun_header;
1023 logdev_list_length = 0;
1024 }
1025
1026 logdev_data_length = sizeof(struct report_lun_header) +
1027 logdev_list_length;
1028
1029 internal_logdev_list = kmalloc(logdev_data_length +
1030 sizeof(struct report_log_lun_extended), GFP_KERNEL);
1031 if (!internal_logdev_list) {
1032 kfree(*logdev_list);
1033 *logdev_list = NULL;
1034 return -ENOMEM;
1035 }
1036
1037 memcpy(internal_logdev_list, logdev_data, logdev_data_length);
1038 memset((u8 *)internal_logdev_list + logdev_data_length, 0,
1039 sizeof(struct report_log_lun_extended_entry));
1040 put_unaligned_be32(logdev_list_length +
1041 sizeof(struct report_log_lun_extended_entry),
1042 &internal_logdev_list->header.list_length);
1043
1044 kfree(*logdev_list);
1045 *logdev_list = internal_logdev_list;
1046
1047 return 0;
1048}
1049
1050static inline void pqi_set_bus_target_lun(struct pqi_scsi_dev *device,
1051 int bus, int target, int lun)
1052{
1053 device->bus = bus;
1054 device->target = target;
1055 device->lun = lun;
1056}
1057
1058static void pqi_assign_bus_target_lun(struct pqi_scsi_dev *device)
1059{
1060 u8 *scsi3addr;
1061 u32 lunid;
bd10cf0b
KB		 1062 int bus;
1063 int target;
1064 int lun;
6c223761
KB		 1065
1066 scsi3addr = device->scsi3addr;
1067 lunid = get_unaligned_le32(scsi3addr);
1068
1069 if (pqi_is_hba_lunid(scsi3addr)) {
1070 /* The specified device is the controller. */
1071 pqi_set_bus_target_lun(device, PQI_HBA_BUS, 0, lunid & 0x3fff);
1072 device->target_lun_valid = true;
1073 return;
1074 }
1075
1076 if (pqi_is_logical_device(device)) {
bd10cf0b
KB		 1077 if (device->is_external_raid_device) {
1078 bus = PQI_EXTERNAL_RAID_VOLUME_BUS;
1079 target = (lunid >> 16) & 0x3fff;
1080 lun = lunid & 0xff;
1081 } else {
1082 bus = PQI_RAID_VOLUME_BUS;
1083 target = 0;
1084 lun = lunid & 0x3fff;
1085 }
1086 pqi_set_bus_target_lun(device, bus, target, lun);
6c223761
KB		 1087 device->target_lun_valid = true;
1088 return;
1089 }
1090
1091 /*
1092 * Defer target and LUN assignment for non-controller physical devices
1093 * because the SAS transport layer will make these assignments later.
1094 */
1095 pqi_set_bus_target_lun(device, PQI_PHYSICAL_DEVICE_BUS, 0, 0);
1096}
1097
1098static void pqi_get_raid_level(struct pqi_ctrl_info *ctrl_info,
1099 struct pqi_scsi_dev *device)
1100{
1101 int rc;
1102 u8 raid_level;
1103 u8 *buffer;
1104
1105 raid_level = SA_RAID_UNKNOWN;
1106
1107 buffer = kmalloc(64, GFP_KERNEL);
1108 if (buffer) {
1109 rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr,
1110 VPD_PAGE | CISS_VPD_LV_DEVICE_GEOMETRY, buffer, 64);
1111 if (rc == 0) {
1112 raid_level = buffer[8];
1113 if (raid_level > SA_RAID_MAX)
1114 raid_level = SA_RAID_UNKNOWN;
1115 }
1116 kfree(buffer);
1117 }
1118
1119 device->raid_level = raid_level;
1120}
1121
1122static int pqi_validate_raid_map(struct pqi_ctrl_info *ctrl_info,
1123 struct pqi_scsi_dev *device, struct raid_map *raid_map)
1124{
1125 char *err_msg;
1126 u32 raid_map_size;
1127 u32 r5or6_blocks_per_row;
6c223761
KB		 1128
1129 raid_map_size = get_unaligned_le32(&raid_map->structure_size);
1130
1131 if (raid_map_size < offsetof(struct raid_map, disk_data)) {
1132 err_msg = "RAID map too small";
1133 goto bad_raid_map;
1134 }
1135
6c223761
KB		 1136 if (device->raid_level == SA_RAID_1) {
1137 if (get_unaligned_le16(&raid_map->layout_map_count) != 2) {
1138 err_msg = "invalid RAID-1 map";
1139 goto bad_raid_map;
1140 }
1141 } else if (device->raid_level == SA_RAID_ADM) {
1142 if (get_unaligned_le16(&raid_map->layout_map_count) != 3) {
1143 err_msg = "invalid RAID-1(ADM) map";
1144 goto bad_raid_map;
1145 }
1146 } else if ((device->raid_level == SA_RAID_5 ||
1147 device->raid_level == SA_RAID_6) &&
1148 get_unaligned_le16(&raid_map->layout_map_count) > 1) {
1149 /* RAID 50/60 */
1150 r5or6_blocks_per_row =
1151 get_unaligned_le16(&raid_map->strip_size) *
1152 get_unaligned_le16(&raid_map->data_disks_per_row);
1153 if (r5or6_blocks_per_row == 0) {
1154 err_msg = "invalid RAID-5 or RAID-6 map";
1155 goto bad_raid_map;
1156 }
1157 }
1158
1159 return 0;
1160
1161bad_raid_map:
d87d5474 1162 dev_warn(&ctrl_info->pci_dev->dev,
38a7338a
KB		 1163 "logical device %08x%08x %s\n",
1164 *((u32 *)&device->scsi3addr),
1165 *((u32 *)&device->scsi3addr[4]), err_msg);
6c223761
KB		 1166
1167 return -EINVAL;
1168}
1169
1170static int pqi_get_raid_map(struct pqi_ctrl_info *ctrl_info,
1171 struct pqi_scsi_dev *device)
1172{
1173 int rc;
a91aaae0 1174 u32 raid_map_size;
6c223761
KB		 1175 struct raid_map *raid_map;
1176
1177 raid_map = kmalloc(sizeof(*raid_map), GFP_KERNEL);
1178 if (!raid_map)
1179 return -ENOMEM;
1180
a91aaae0
AK		 1181 rc = pqi_send_scsi_raid_request(ctrl_info, CISS_GET_RAID_MAP,
1182 device->scsi3addr, raid_map, sizeof(*raid_map),
1183 0, NULL, NO_TIMEOUT);
1184
6c223761
KB		 1185 if (rc)
1186 goto error;
1187
a91aaae0 1188 raid_map_size = get_unaligned_le32(&raid_map->structure_size);
6c223761 1189
a91aaae0 1190 if (raid_map_size > sizeof(*raid_map)) {
6c223761 1191
a91aaae0
AK		 1192 kfree(raid_map);
1193
1194 raid_map = kmalloc(raid_map_size, GFP_KERNEL);
1195 if (!raid_map)
1196 return -ENOMEM;
1197
1198 rc = pqi_send_scsi_raid_request(ctrl_info, CISS_GET_RAID_MAP,
1199 device->scsi3addr, raid_map, raid_map_size,
1200 0, NULL, NO_TIMEOUT);
1201 if (rc)
1202 goto error;
1203
1204 if (get_unaligned_le32(&raid_map->structure_size)
1205 != raid_map_size) {
1206 dev_warn(&ctrl_info->pci_dev->dev,
1207 "Requested %d bytes, received %d bytes",
1208 raid_map_size,
1209 get_unaligned_le32(&raid_map->structure_size));
1210 goto error;
1211 }
1212 }
6c223761
KB		 1213
1214 rc = pqi_validate_raid_map(ctrl_info, device, raid_map);
1215 if (rc)
1216 goto error;
1217
1218 device->raid_map = raid_map;
1219
1220 return 0;
1221
1222error:
1223 kfree(raid_map);
1224
1225 return rc;
1226}
1227
588a63fe 1228static void pqi_get_raid_bypass_status(struct pqi_ctrl_info *ctrl_info,
6c223761
KB		 1229 struct pqi_scsi_dev *device)
1230{
1231 int rc;
1232 u8 *buffer;
588a63fe 1233 u8 bypass_status;
6c223761
KB		 1234
1235 buffer = kmalloc(64, GFP_KERNEL);
1236 if (!buffer)
1237 return;
1238
1239 rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr,
588a63fe 1240 VPD_PAGE | CISS_VPD_LV_BYPASS_STATUS, buffer, 64);
6c223761
KB		 1241 if (rc)
1242 goto out;
1243
694c5d5b
KB		 1244#define RAID_BYPASS_STATUS 4
1245#define RAID_BYPASS_CONFIGURED 0x1
1246#define RAID_BYPASS_ENABLED 0x2
6c223761 1247
588a63fe
KB		 1248 bypass_status = buffer[RAID_BYPASS_STATUS];
1249 device->raid_bypass_configured =
1250 (bypass_status & RAID_BYPASS_CONFIGURED) != 0;
1251 if (device->raid_bypass_configured &&
1252 (bypass_status & RAID_BYPASS_ENABLED) &&
1253 pqi_get_raid_map(ctrl_info, device) == 0)
1254 device->raid_bypass_enabled = true;
6c223761
KB		 1255
1256out:
1257 kfree(buffer);
1258}
1259
1260/*
1261 * Use vendor-specific VPD to determine online/offline status of a volume.
1262 */
1263
1264static void pqi_get_volume_status(struct pqi_ctrl_info *ctrl_info,
1265 struct pqi_scsi_dev *device)
1266{
1267 int rc;
1268 size_t page_length;
1269 u8 volume_status = CISS_LV_STATUS_UNAVAILABLE;
1270 bool volume_offline = true;
1271 u32 volume_flags;
1272 struct ciss_vpd_logical_volume_status *vpd;
1273
1274 vpd = kmalloc(sizeof(*vpd), GFP_KERNEL);
1275 if (!vpd)
1276 goto no_buffer;
1277
1278 rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr,
1279 VPD_PAGE | CISS_VPD_LV_STATUS, vpd, sizeof(*vpd));
1280 if (rc)
1281 goto out;
1282
7ff44499
DC		 1283 if (vpd->page_code != CISS_VPD_LV_STATUS)
1284 goto out;
1285
6c223761
KB		 1286 page_length = offsetof(struct ciss_vpd_logical_volume_status,
1287 volume_status) + vpd->page_length;
1288 if (page_length < sizeof(*vpd))
1289 goto out;
1290
1291 volume_status = vpd->volume_status;
1292 volume_flags = get_unaligned_be32(&vpd->flags);
1293 volume_offline = (volume_flags & CISS_LV_FLAGS_NO_HOST_IO) != 0;
1294
1295out:
1296 kfree(vpd);
1297no_buffer:
1298 device->volume_status = volume_status;
1299 device->volume_offline = volume_offline;
1300}
1301
ce143793
KB		 1302static int pqi_get_physical_device_info(struct pqi_ctrl_info *ctrl_info,
1303 struct pqi_scsi_dev *device,
1304 struct bmic_identify_physical_device *id_phys)
1305{
1306 int rc;
26b390ab 1307
ce143793
KB		 1308 memset(id_phys, 0, sizeof(*id_phys));
1309
1310 rc = pqi_identify_physical_device(ctrl_info, device,
1311 id_phys, sizeof(*id_phys));
1312 if (rc) {
1313 device->queue_depth = PQI_PHYSICAL_DISK_DEFAULT_MAX_QUEUE_DEPTH;
1314 return rc;
1315 }
1316
1317 scsi_sanitize_inquiry_string(&id_phys->model[0], 8);
1318 scsi_sanitize_inquiry_string(&id_phys->model[8], 16);
1319
1320 memcpy(device->vendor, &id_phys->model[0], sizeof(device->vendor));
1321 memcpy(device->model, &id_phys->model[8], sizeof(device->model));
1322
1323 device->box_index = id_phys->box_index;
1324 device->phys_box_on_bus = id_phys->phys_box_on_bus;
1325 device->phy_connected_dev_type = id_phys->phy_connected_dev_type[0];
1326 device->queue_depth =
1327 get_unaligned_le16(&id_phys->current_queue_depth_limit);
1328 device->active_path_index = id_phys->active_path_number;
1329 device->path_map = id_phys->redundant_path_present_map;
1330 memcpy(&device->box,
1331 &id_phys->alternate_paths_phys_box_on_port,
1332 sizeof(device->box));
1333 memcpy(&device->phys_connector,
1334 &id_phys->alternate_paths_phys_connector,
1335 sizeof(device->phys_connector));
1336 device->bay = id_phys->phys_bay_in_box;
1337
1338 return 0;
1339}
1340
1341static int pqi_get_logical_device_info(struct pqi_ctrl_info *ctrl_info,
6c223761
KB		 1342 struct pqi_scsi_dev *device)
1343{
1344 int rc;
1345 u8 *buffer;
3d46a59a 1346
6c223761
KB		 1347 buffer = kmalloc(64, GFP_KERNEL);
1348 if (!buffer)
1349 return -ENOMEM;
1350
1351 /* Send an inquiry to the device to see what it is. */
ce143793
KB		 1352 rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr, 0, buffer, 64);
1353 if (rc)
1354 goto out;
6c223761
KB		 1355
1356 scsi_sanitize_inquiry_string(&buffer[8], 8);
1357 scsi_sanitize_inquiry_string(&buffer[16], 16);
1358
1359 device->devtype = buffer[0] & 0x1f;
cbe0c7b1
KB		 1360 memcpy(device->vendor, &buffer[8], sizeof(device->vendor));
1361 memcpy(device->model, &buffer[16], sizeof(device->model));
6c223761 1362
ce143793 1363 if (device->devtype == TYPE_DISK) {
bd10cf0b
KB		 1364 if (device->is_external_raid_device) {
1365 device->raid_level = SA_RAID_UNKNOWN;
1366 device->volume_status = CISS_LV_OK;
1367 device->volume_offline = false;
1368 } else {
1369 pqi_get_raid_level(ctrl_info, device);
588a63fe 1370 pqi_get_raid_bypass_status(ctrl_info, device);
bd10cf0b
KB		 1371 pqi_get_volume_status(ctrl_info, device);
1372 }
6c223761
KB		 1373 }
1374
1375out:
1376 kfree(buffer);
1377
1378 return rc;
1379}
1380
ce143793 1381static int pqi_get_device_info(struct pqi_ctrl_info *ctrl_info,
6c223761
KB		 1382 struct pqi_scsi_dev *device,
1383 struct bmic_identify_physical_device *id_phys)
1384{
1385 int rc;
1386
ce143793
KB		 1387 if (device->is_expander_smp_device)
1388 return 0;
6c223761 1389
ce143793
KB		 1390 if (pqi_is_logical_device(device))
1391 rc = pqi_get_logical_device_info(ctrl_info, device);
1392 else
1393 rc = pqi_get_physical_device_info(ctrl_info, device, id_phys);
694c5d5b 1394
ce143793 1395 return rc;
6c223761
KB		 1396}
1397
1398static void pqi_show_volume_status(struct pqi_ctrl_info *ctrl_info,
1399 struct pqi_scsi_dev *device)
1400{
1401 char *status;
1402 static const char unknown_state_str[] =
1403 "Volume is in an unknown state (%u)";
1404 char unknown_state_buffer[sizeof(unknown_state_str) + 10];
1405
1406 switch (device->volume_status) {
1407 case CISS_LV_OK:
1408 status = "Volume online";
1409 break;
1410 case CISS_LV_FAILED:
1411 status = "Volume failed";
1412 break;
1413 case CISS_LV_NOT_CONFIGURED:
1414 status = "Volume not configured";
1415 break;
1416 case CISS_LV_DEGRADED:
1417 status = "Volume degraded";
1418 break;
1419 case CISS_LV_READY_FOR_RECOVERY:
1420 status = "Volume ready for recovery operation";
1421 break;
1422 case CISS_LV_UNDERGOING_RECOVERY:
1423 status = "Volume undergoing recovery";
1424 break;
1425 case CISS_LV_WRONG_PHYSICAL_DRIVE_REPLACED:
1426 status = "Wrong physical drive was replaced";
1427 break;
1428 case CISS_LV_PHYSICAL_DRIVE_CONNECTION_PROBLEM:
1429 status = "A physical drive not properly connected";
1430 break;
1431 case CISS_LV_HARDWARE_OVERHEATING:
1432 status = "Hardware is overheating";
1433 break;
1434 case CISS_LV_HARDWARE_HAS_OVERHEATED:
1435 status = "Hardware has overheated";
1436 break;
1437 case CISS_LV_UNDERGOING_EXPANSION:
1438 status = "Volume undergoing expansion";
1439 break;
1440 case CISS_LV_NOT_AVAILABLE:
1441 status = "Volume waiting for transforming volume";
1442 break;
1443 case CISS_LV_QUEUED_FOR_EXPANSION:
1444 status = "Volume queued for expansion";
1445 break;
1446 case CISS_LV_DISABLED_SCSI_ID_CONFLICT:
1447 status = "Volume disabled due to SCSI ID conflict";
1448 break;
1449 case CISS_LV_EJECTED:
1450 status = "Volume has been ejected";
1451 break;
1452 case CISS_LV_UNDERGOING_ERASE:
1453 status = "Volume undergoing background erase";
1454 break;
1455 case CISS_LV_READY_FOR_PREDICTIVE_SPARE_REBUILD:
1456 status = "Volume ready for predictive spare rebuild";
1457 break;
1458 case CISS_LV_UNDERGOING_RPI:
1459 status = "Volume undergoing rapid parity initialization";
1460 break;
1461 case CISS_LV_PENDING_RPI:
1462 status = "Volume queued for rapid parity initialization";
1463 break;
1464 case CISS_LV_ENCRYPTED_NO_KEY:
1465 status = "Encrypted volume inaccessible - key not present";
1466 break;
1467 case CISS_LV_UNDERGOING_ENCRYPTION:
1468 status = "Volume undergoing encryption process";
1469 break;
1470 case CISS_LV_UNDERGOING_ENCRYPTION_REKEYING:
1471 status = "Volume undergoing encryption re-keying process";
1472 break;
1473 case CISS_LV_ENCRYPTED_IN_NON_ENCRYPTED_CONTROLLER:
d87d5474 1474 status = "Volume encrypted but encryption is disabled";
6c223761
KB		 1475 break;
1476 case CISS_LV_PENDING_ENCRYPTION:
1477 status = "Volume pending migration to encrypted state";
1478 break;
1479 case CISS_LV_PENDING_ENCRYPTION_REKEYING:
1480 status = "Volume pending encryption rekeying";
1481 break;
1482 case CISS_LV_NOT_SUPPORTED:
1483 status = "Volume not supported on this controller";
1484 break;
1485 case CISS_LV_STATUS_UNAVAILABLE:
1486 status = "Volume status not available";
1487 break;
1488 default:
1489 snprintf(unknown_state_buffer, sizeof(unknown_state_buffer),
1490 unknown_state_str, device->volume_status);
1491 status = unknown_state_buffer;
1492 break;
1493 }
1494
1495 dev_info(&ctrl_info->pci_dev->dev,
1496 "scsi %d:%d:%d:%d %s\n",
1497 ctrl_info->scsi_host->host_no,
1498 device->bus, device->target, device->lun, status);
1499}
1500
6c223761
KB		 1501static void pqi_rescan_worker(struct work_struct *work)
1502{
1503 struct pqi_ctrl_info *ctrl_info;
1504
1505 ctrl_info = container_of(to_delayed_work(work), struct pqi_ctrl_info,
1506 rescan_work);
1507
1508 pqi_scan_scsi_devices(ctrl_info);
1509}
1510
1511static int pqi_add_device(struct pqi_ctrl_info *ctrl_info,
1512 struct pqi_scsi_dev *device)
1513{
1514 int rc;
1515
1516 if (pqi_is_logical_device(device))
1517 rc = scsi_add_device(ctrl_info->scsi_host, device->bus,
1518 device->target, device->lun);
1519 else
1520 rc = pqi_add_sas_device(ctrl_info->sas_host, device);
1521
1522 return rc;
1523}
1524
1e46731e
MR		 1525#define PQI_PENDING_IO_TIMEOUT_SECS 20
1526
6c223761
KB		 1527static inline void pqi_remove_device(struct pqi_ctrl_info *ctrl_info,
1528 struct pqi_scsi_dev *device)
1529{
1e46731e
MR		 1530 int rc;
1531
1532 pqi_device_remove_start(device);
1533
4d15ad38 1534 rc = pqi_device_wait_for_pending_io(ctrl_info, device, PQI_PENDING_IO_TIMEOUT_SECS);
1e46731e
MR		 1535 if (rc)
1536 dev_err(&ctrl_info->pci_dev->dev,
4d15ad38 1537 "scsi %d:%d:%d:%d removing device with %d outstanding command(s)\n",
1e46731e
MR		 1538 ctrl_info->scsi_host->host_no, device->bus,
1539 device->target, device->lun,
1540 atomic_read(&device->scsi_cmds_outstanding));
1541
6c223761
KB		 1542 if (pqi_is_logical_device(device))
1543 scsi_remove_device(device->sdev);
1544 else
1545 pqi_remove_sas_device(device);
1546}
1547
1548/* Assumes the SCSI device list lock is held. */
1549
1550static struct pqi_scsi_dev *pqi_find_scsi_dev(struct pqi_ctrl_info *ctrl_info,
1551 int bus, int target, int lun)
1552{
1553 struct pqi_scsi_dev *device;
1554
4d15ad38
KB		 1555 list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry)
1556 if (device->bus == bus && device->target == target && device->lun == lun)
6c223761
KB		 1557 return device;
1558
1559 return NULL;
1560}
1561
1562static inline bool pqi_device_equal(struct pqi_scsi_dev *dev1,
1563 struct pqi_scsi_dev *dev2)
1564{
1565 if (dev1->is_physical_device != dev2->is_physical_device)
1566 return false;
1567
1568 if (dev1->is_physical_device)
1569 return dev1->wwid == dev2->wwid;
1570
1571 return memcmp(dev1->volume_id, dev2->volume_id,
1572 sizeof(dev1->volume_id)) == 0;
1573}
1574
1575enum pqi_find_result {
1576 DEVICE_NOT_FOUND,
1577 DEVICE_CHANGED,
1578 DEVICE_SAME,
1579};
1580
1581static enum pqi_find_result pqi_scsi_find_entry(struct pqi_ctrl_info *ctrl_info,
4d15ad38 1582 struct pqi_scsi_dev *device_to_find, struct pqi_scsi_dev **matching_device)
6c223761
KB		 1583{
1584 struct pqi_scsi_dev *device;
1585
4d15ad38
KB		 1586 list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry) {
1587 if (pqi_scsi3addr_equal(device_to_find->scsi3addr, device->scsi3addr)) {
6c223761
KB		 1588 *matching_device = device;
1589 if (pqi_device_equal(device_to_find, device)) {
1590 if (device_to_find->volume_offline)
1591 return DEVICE_CHANGED;
1592 return DEVICE_SAME;
1593 }
1594 return DEVICE_CHANGED;
1595 }
1596 }
1597
1598 return DEVICE_NOT_FOUND;
1599}
1600
3d46a59a
DB		 1601static inline const char *pqi_device_type(struct pqi_scsi_dev *device)
1602{
1603 if (device->is_expander_smp_device)
1604 return "Enclosure SMP ";
1605
1606 return scsi_device_type(device->devtype);
1607}
1608
6de783f6
KB		 1609#define PQI_DEV_INFO_BUFFER_LENGTH 128
1610
6c223761
KB		 1611static void pqi_dev_info(struct pqi_ctrl_info *ctrl_info,
1612 char *action, struct pqi_scsi_dev *device)
1613{
6de783f6
KB		 1614 ssize_t count;
1615 char buffer[PQI_DEV_INFO_BUFFER_LENGTH];
1616
1617 count = snprintf(buffer, PQI_DEV_INFO_BUFFER_LENGTH,
1618 "%d:%d:", ctrl_info->scsi_host->host_no, device->bus);
1619
1620 if (device->target_lun_valid)
181aea89 1621 count += scnprintf(buffer + count,
6de783f6
KB		 1622 PQI_DEV_INFO_BUFFER_LENGTH - count,
1623 "%d:%d",
1624 device->target,
1625 device->lun);
1626 else
181aea89 1627 count += scnprintf(buffer + count,
6de783f6
KB		 1628 PQI_DEV_INFO_BUFFER_LENGTH - count,
1629 "-:-");
1630
1631 if (pqi_is_logical_device(device))
181aea89 1632 count += scnprintf(buffer + count,
6de783f6
KB		 1633 PQI_DEV_INFO_BUFFER_LENGTH - count,
1634 " %08x%08x",
1635 *((u32 *)&device->scsi3addr),
1636 *((u32 *)&device->scsi3addr[4]));
1637 else
181aea89 1638 count += scnprintf(buffer + count,
6de783f6
KB		 1639 PQI_DEV_INFO_BUFFER_LENGTH - count,
1640 " %016llx", device->sas_address);
1641
181aea89 1642 count += scnprintf(buffer + count, PQI_DEV_INFO_BUFFER_LENGTH - count,
6de783f6 1643 " %s %.8s %.16s ",
3d46a59a 1644 pqi_device_type(device),
6c223761 1645 device->vendor,
6de783f6
KB		 1646 device->model);
1647
1648 if (pqi_is_logical_device(device)) {
1649 if (device->devtype == TYPE_DISK)
181aea89 1650 count += scnprintf(buffer + count,
6de783f6
KB		 1651 PQI_DEV_INFO_BUFFER_LENGTH - count,
1652 "SSDSmartPathCap%c En%c %-12s",
588a63fe
KB		 1653 device->raid_bypass_configured ? '+' : '-',
1654 device->raid_bypass_enabled ? '+' : '-',
6de783f6
KB		 1655 pqi_raid_level_to_string(device->raid_level));
1656 } else {
181aea89 1657 count += scnprintf(buffer + count,
6de783f6
KB		 1658 PQI_DEV_INFO_BUFFER_LENGTH - count,
1659 "AIO%c", device->aio_enabled ? '+' : '-');
1660 if (device->devtype == TYPE_DISK ||
1661 device->devtype == TYPE_ZBC)
181aea89 1662 count += scnprintf(buffer + count,
6de783f6
KB		 1663 PQI_DEV_INFO_BUFFER_LENGTH - count,
1664 " qd=%-6d", device->queue_depth);
1665 }
1666
1667 dev_info(&ctrl_info->pci_dev->dev, "%s %s\n", action, buffer);
6c223761
KB		 1668}
1669
1670/* Assumes the SCSI device list lock is held. */
1671
1672static void pqi_scsi_update_device(struct pqi_scsi_dev *existing_device,
1673 struct pqi_scsi_dev *new_device)
1674{
1675 existing_device->devtype = new_device->devtype;
1676 existing_device->device_type = new_device->device_type;
1677 existing_device->bus = new_device->bus;
1678 if (new_device->target_lun_valid) {
1679 existing_device->target = new_device->target;
1680 existing_device->lun = new_device->lun;
1681 existing_device->target_lun_valid = true;
1682 }
1683
244ca45e
MR		 1684 if ((existing_device->volume_status == CISS_LV_QUEUED_FOR_EXPANSION ||
1685 existing_device->volume_status == CISS_LV_UNDERGOING_EXPANSION) &&
1686 new_device->volume_status == CISS_LV_OK)
1687 existing_device->rescan = true;
1688
6c223761
KB		 1689 /* By definition, the scsi3addr and wwid fields are already the same. */
1690
1691 existing_device->is_physical_device = new_device->is_physical_device;
bd10cf0b
KB		 1692 existing_device->is_external_raid_device =
1693 new_device->is_external_raid_device;
3d46a59a
DB		 1694 existing_device->is_expander_smp_device =
1695 new_device->is_expander_smp_device;
6c223761
KB		 1696 existing_device->aio_enabled = new_device->aio_enabled;
1697 memcpy(existing_device->vendor, new_device->vendor,
1698 sizeof(existing_device->vendor));
1699 memcpy(existing_device->model, new_device->model,
1700 sizeof(existing_device->model));
1701 existing_device->sas_address = new_device->sas_address;
1702 existing_device->raid_level = new_device->raid_level;
1703 existing_device->queue_depth = new_device->queue_depth;
1704 existing_device->aio_handle = new_device->aio_handle;
1705 existing_device->volume_status = new_device->volume_status;
1706 existing_device->active_path_index = new_device->active_path_index;
1707 existing_device->path_map = new_device->path_map;
1708 existing_device->bay = new_device->bay;
2d2ad4bc
GW		 1709 existing_device->box_index = new_device->box_index;
1710 existing_device->phys_box_on_bus = new_device->phys_box_on_bus;
1711 existing_device->phy_connected_dev_type =
1712 new_device->phy_connected_dev_type;
6c223761
KB		 1713 memcpy(existing_device->box, new_device->box,
1714 sizeof(existing_device->box));
1715 memcpy(existing_device->phys_connector, new_device->phys_connector,
1716 sizeof(existing_device->phys_connector));
6c223761
KB		 1717 existing_device->offload_to_mirror = 0;
1718 kfree(existing_device->raid_map);
1719 existing_device->raid_map = new_device->raid_map;
588a63fe
KB		 1720 existing_device->raid_bypass_configured =
1721 new_device->raid_bypass_configured;
1722 existing_device->raid_bypass_enabled =
1723 new_device->raid_bypass_enabled;
a9a68101 1724 existing_device->device_offline = false;
6c223761
KB		 1725
1726 /* To prevent this from being freed later. */
1727 new_device->raid_map = NULL;
1728}
1729
1730static inline void pqi_free_device(struct pqi_scsi_dev *device)
1731{
1732 if (device) {
1733 kfree(device->raid_map);
1734 kfree(device);
1735 }
1736}
1737
1738/*
1739 * Called when exposing a new device to the OS fails in order to re-adjust
1740 * our internal SCSI device list to match the SCSI ML's view.
1741 */
1742
1743static inline void pqi_fixup_botched_add(struct pqi_ctrl_info *ctrl_info,
1744 struct pqi_scsi_dev *device)
1745{
1746 unsigned long flags;
1747
1748 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
1749 list_del(&device->scsi_device_list_entry);
1750 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
1751
1752 /* Allow the device structure to be freed later. */
1753 device->keep_device = false;
1754}
1755
3d46a59a
DB		 1756static inline bool pqi_is_device_added(struct pqi_scsi_dev *device)
1757{
1758 if (device->is_expander_smp_device)
1759 return device->sas_port != NULL;
1760
1761 return device->sdev != NULL;
1762}
1763
6c223761
KB		 1764static void pqi_update_device_list(struct pqi_ctrl_info *ctrl_info,
1765 struct pqi_scsi_dev *new_device_list[], unsigned int num_new_devices)
1766{
1767 int rc;
1768 unsigned int i;
1769 unsigned long flags;
1770 enum pqi_find_result find_result;
1771 struct pqi_scsi_dev *device;
1772 struct pqi_scsi_dev *next;
1773 struct pqi_scsi_dev *matching_device;
8a994a04
KB		 1774 LIST_HEAD(add_list);
1775 LIST_HEAD(delete_list);
6c223761
KB		 1776
1777 /*
1778 * The idea here is to do as little work as possible while holding the
1779 * spinlock. That's why we go to great pains to defer anything other
1780 * than updating the internal device list until after we release the
1781 * spinlock.
1782 */
1783
1784 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
1785
1786 /* Assume that all devices in the existing list have gone away. */
4d15ad38 1787 list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry)
6c223761
KB		 1788 device->device_gone = true;
1789
1790 for (i = 0; i < num_new_devices; i++) {
1791 device = new_device_list[i];
1792
1793 find_result = pqi_scsi_find_entry(ctrl_info, device,
694c5d5b 1794 &matching_device);
6c223761
KB		 1795
1796 switch (find_result) {
1797 case DEVICE_SAME:
1798 /*
1799 * The newly found device is already in the existing
1800 * device list.
1801 */
1802 device->new_device = false;
1803 matching_device->device_gone = false;
1804 pqi_scsi_update_device(matching_device, device);
1805 break;
1806 case DEVICE_NOT_FOUND:
1807 /*
1808 * The newly found device is NOT in the existing device
1809 * list.
1810 */
1811 device->new_device = true;
1812 break;
1813 case DEVICE_CHANGED:
1814 /*
1815 * The original device has gone away and we need to add
1816 * the new device.
1817 */
1818 device->new_device = true;
1819 break;
6c223761
KB		 1820 }
1821 }
1822
1823 /* Process all devices that have gone away. */
1824 list_for_each_entry_safe(device, next, &ctrl_info->scsi_device_list,
1825 scsi_device_list_entry) {
1826 if (device->device_gone) {
4d15ad38 1827 list_del_init(&device->scsi_device_list_entry);
6c223761
KB		 1828 list_add_tail(&device->delete_list_entry, &delete_list);
1829 }
1830 }
1831
1832 /* Process all new devices. */
1833 for (i = 0; i < num_new_devices; i++) {
1834 device = new_device_list[i];
1835 if (!device->new_device)
1836 continue;
1837 if (device->volume_offline)
1838 continue;
1839 list_add_tail(&device->scsi_device_list_entry,
1840 &ctrl_info->scsi_device_list);
1841 list_add_tail(&device->add_list_entry, &add_list);
1842 /* To prevent this device structure from being freed later. */
1843 device->keep_device = true;
1844 }
1845
6c223761
KB		 1846 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
1847
4fd22c13
MR		 1848 if (pqi_ctrl_in_ofa(ctrl_info))
1849 pqi_ctrl_ofa_done(ctrl_info);
1850
6c223761 1851 /* Remove all devices that have gone away. */
4d15ad38 1852 list_for_each_entry_safe(device, next, &delete_list, delete_list_entry) {
6c223761
KB		 1853 if (device->volume_offline) {
1854 pqi_dev_info(ctrl_info, "offline", device);
1855 pqi_show_volume_status(ctrl_info, device);
6c223761
KB		 1856 }
1857 list_del(&device->delete_list_entry);
4d15ad38
KB		 1858 if (pqi_is_device_added(device)) {
1859 pqi_remove_device(ctrl_info, device);
1860 } else {
1861 if (!device->volume_offline)
1862 pqi_dev_info(ctrl_info, "removed", device);
1863 pqi_free_device(device);
1864 }
6c223761
KB		 1865 }
1866
1867 /*
1868 * Notify the SCSI ML if the queue depth of any existing device has
1869 * changed.
1870 */
1871 list_for_each_entry(device, &ctrl_info->scsi_device_list,
1872 scsi_device_list_entry) {
244ca45e
MR		 1873 if (device->sdev) {
1874 if (device->queue_depth !=
1875 device->advertised_queue_depth) {
1876 device->advertised_queue_depth = device->queue_depth;
1877 scsi_change_queue_depth(device->sdev,
1878 device->advertised_queue_depth);
1879 }
1880 if (device->rescan) {
1881 scsi_rescan_device(&device->sdev->sdev_gendev);
1882 device->rescan = false;
1883 }
6c223761
KB		 1884 }
1885 }
1886
1887 /* Expose any new devices. */
1888 list_for_each_entry_safe(device, next, &add_list, add_list_entry) {
3d46a59a 1889 if (!pqi_is_device_added(device)) {
6c223761 1890 rc = pqi_add_device(ctrl_info, device);
ce143793
KB		 1891 if (rc == 0) {
1892 pqi_dev_info(ctrl_info, "added", device);
1893 } else {
6c223761
KB		 1894 dev_warn(&ctrl_info->pci_dev->dev,
1895 "scsi %d:%d:%d:%d addition failed, device not added\n",
1896 ctrl_info->scsi_host->host_no,
1897 device->bus, device->target,
1898 device->lun);
1899 pqi_fixup_botched_add(ctrl_info, device);
6c223761
KB		 1900 }
1901 }
6c223761
KB		 1902 }
1903}
1904
ce143793 1905static inline bool pqi_is_supported_device(struct pqi_scsi_dev *device)
6c223761 1906{
ce143793
KB		 1907 /*
1908 * Only support the HBA controller itself as a RAID
1909 * controller. If it's a RAID controller other than
1910 * the HBA itself (an external RAID controller, for
1911 * example), we don't support it.
1912 */
1913 if (device->device_type == SA_DEVICE_TYPE_CONTROLLER &&
1914 !pqi_is_hba_lunid(device->scsi3addr))
1915 return false;
6c223761 1916
ce143793 1917 return true;
6c223761
KB		 1918}
1919
94086f5b 1920static inline bool pqi_skip_device(u8 *scsi3addr)
6c223761 1921{
94086f5b
KB		 1922 /* Ignore all masked devices. */
1923 if (MASKED_DEVICE(scsi3addr))
6c223761 1924 return true;
6c223761
KB		 1925
1926 return false;
1927}
1928
522bc026
DC		 1929static inline void pqi_mask_device(u8 *scsi3addr)
1930{
1931 scsi3addr[3] |= 0xc0;
1932}
1933
3d46a59a
DB		 1934static inline bool pqi_is_device_with_sas_address(struct pqi_scsi_dev *device)
1935{
ce143793
KB		 1936 switch (device->device_type) {
1937 case SA_DEVICE_TYPE_SAS:
1938 case SA_DEVICE_TYPE_EXPANDER_SMP:
1939 case SA_DEVICE_TYPE_SES:
3d46a59a
DB		 1940 return true;
1941 }
1942
1943 return false;
1944}
1945
cd128244
DC		 1946static inline bool pqi_expose_device(struct pqi_scsi_dev *device)
1947{
1948 return !device->is_physical_device ||
1949 !pqi_skip_device(device->scsi3addr);
1950}
1951
6c223761
KB		 1952static int pqi_update_scsi_devices(struct pqi_ctrl_info *ctrl_info)
1953{
1954 int i;
1955 int rc;
8a994a04 1956 LIST_HEAD(new_device_list_head);
6c223761
KB		 1957 struct report_phys_lun_extended *physdev_list = NULL;
1958 struct report_log_lun_extended *logdev_list = NULL;
1959 struct report_phys_lun_extended_entry *phys_lun_ext_entry;
1960 struct report_log_lun_extended_entry *log_lun_ext_entry;
1961 struct bmic_identify_physical_device *id_phys = NULL;
1962 u32 num_physicals;
1963 u32 num_logicals;
1964 struct pqi_scsi_dev **new_device_list = NULL;
1965 struct pqi_scsi_dev *device;
1966 struct pqi_scsi_dev *next;
1967 unsigned int num_new_devices;
1968 unsigned int num_valid_devices;
1969 bool is_physical_device;
1970 u8 *scsi3addr;
5e6a9760
GW		 1971 unsigned int physical_index;
1972 unsigned int logical_index;
6c223761 1973 static char *out_of_memory_msg =
6de783f6 1974 "failed to allocate memory, device discovery stopped";
6c223761 1975
6c223761
KB		 1976 rc = pqi_get_device_lists(ctrl_info, &physdev_list, &logdev_list);
1977 if (rc)
1978 goto out;
1979
1980 if (physdev_list)
1981 num_physicals =
1982 get_unaligned_be32(&physdev_list->header.list_length)
1983 / sizeof(physdev_list->lun_entries[0]);
1984 else
1985 num_physicals = 0;
1986
1987 if (logdev_list)
1988 num_logicals =
1989 get_unaligned_be32(&logdev_list->header.list_length)
1990 / sizeof(logdev_list->lun_entries[0]);
1991 else
1992 num_logicals = 0;
1993
1994 if (num_physicals) {
1995 /*
1996 * We need this buffer for calls to pqi_get_physical_disk_info()
1997 * below. We allocate it here instead of inside
1998 * pqi_get_physical_disk_info() because it's a fairly large
1999 * buffer.
2000 */
2001 id_phys = kmalloc(sizeof(*id_phys), GFP_KERNEL);
2002 if (!id_phys) {
2003 dev_warn(&ctrl_info->pci_dev->dev, "%s\n",
2004 out_of_memory_msg);
2005 rc = -ENOMEM;
2006 goto out;
2007 }
522bc026 2008
694c5d5b 2009 if (pqi_hide_vsep) {
522bc026
DC		 2010 for (i = num_physicals - 1; i >= 0; i--) {
2011 phys_lun_ext_entry =
2012 &physdev_list->lun_entries[i];
2013 if (CISS_GET_DRIVE_NUMBER(
2014 phys_lun_ext_entry->lunid) ==
2015 PQI_VSEP_CISS_BTL) {
2016 pqi_mask_device(
2017 phys_lun_ext_entry->lunid);
2018 break;
2019 }
2020 }
2021 }
6c223761
KB		 2022 }
2023
2024 num_new_devices = num_physicals + num_logicals;
2025
6da2ec56
KC		 2026 new_device_list = kmalloc_array(num_new_devices,
2027 sizeof(*new_device_list),
2028 GFP_KERNEL);
6c223761
KB		 2029 if (!new_device_list) {
2030 dev_warn(&ctrl_info->pci_dev->dev, "%s\n", out_of_memory_msg);
2031 rc = -ENOMEM;
2032 goto out;
2033 }
2034
2035 for (i = 0; i < num_new_devices; i++) {
2036 device = kzalloc(sizeof(*device), GFP_KERNEL);
2037 if (!device) {
2038 dev_warn(&ctrl_info->pci_dev->dev, "%s\n",
2039 out_of_memory_msg);
2040 rc = -ENOMEM;
2041 goto out;
2042 }
2043 list_add_tail(&device->new_device_list_entry,
2044 &new_device_list_head);
2045 }
2046
2047 device = NULL;
2048 num_valid_devices = 0;
5e6a9760
GW		 2049 physical_index = 0;
2050 logical_index = 0;
6c223761
KB		 2051
2052 for (i = 0; i < num_new_devices; i++) {
2053
5e6a9760
GW		 2054 if ((!pqi_expose_ld_first && i < num_physicals) ||
2055 (pqi_expose_ld_first && i >= num_logicals)) {
6c223761 2056 is_physical_device = true;
5e6a9760
GW		 2057 phys_lun_ext_entry =
2058 &physdev_list->lun_entries[physical_index++];
6c223761
KB		 2059 log_lun_ext_entry = NULL;
2060 scsi3addr = phys_lun_ext_entry->lunid;
2061 } else {
2062 is_physical_device = false;
2063 phys_lun_ext_entry = NULL;
2064 log_lun_ext_entry =
5e6a9760 2065 &logdev_list->lun_entries[logical_index++];
6c223761
KB		 2066 scsi3addr = log_lun_ext_entry->lunid;
2067 }
2068
94086f5b 2069 if (is_physical_device && pqi_skip_device(scsi3addr))
6c223761
KB		 2070 continue;
2071
2072 if (device)
2073 device = list_next_entry(device, new_device_list_entry);
2074 else
2075 device = list_first_entry(&new_device_list_head,
2076 struct pqi_scsi_dev, new_device_list_entry);
2077
2078 memcpy(device->scsi3addr, scsi3addr, sizeof(device->scsi3addr));
2079 device->is_physical_device = is_physical_device;
3d46a59a 2080 if (is_physical_device) {
ce143793
KB		 2081 device->device_type = phys_lun_ext_entry->device_type;
2082 if (device->device_type == SA_DEVICE_TYPE_EXPANDER_SMP)
3d46a59a
DB		 2083 device->is_expander_smp_device = true;
2084 } else {
bd10cf0b
KB		 2085 device->is_external_raid_device =
2086 pqi_is_external_raid_addr(scsi3addr);
3d46a59a 2087 }
6c223761 2088
ce143793
KB		 2089 if (!pqi_is_supported_device(device))
2090 continue;
2091
6c223761 2092 /* Gather information about the device. */
ce143793 2093 rc = pqi_get_device_info(ctrl_info, device, id_phys);
6c223761
KB		 2094 if (rc == -ENOMEM) {
2095 dev_warn(&ctrl_info->pci_dev->dev, "%s\n",
2096 out_of_memory_msg);
2097 goto out;
2098 }
2099 if (rc) {
6de783f6
KB		 2100 if (device->is_physical_device)
2101 dev_warn(&ctrl_info->pci_dev->dev,
2102 "obtaining device info failed, skipping physical device %016llx\n",
2103 get_unaligned_be64(
2104 &phys_lun_ext_entry->wwid));
2105 else
2106 dev_warn(&ctrl_info->pci_dev->dev,
2107 "obtaining device info failed, skipping logical device %08x%08x\n",
2108 *((u32 *)&device->scsi3addr),
2109 *((u32 *)&device->scsi3addr[4]));
6c223761
KB		 2110 rc = 0;
2111 continue;
2112 }
2113
6c223761
KB		 2114 pqi_assign_bus_target_lun(device);
2115
6c223761
KB		 2116 if (device->is_physical_device) {
2117 device->wwid = phys_lun_ext_entry->wwid;
2118 if ((phys_lun_ext_entry->device_flags &
694c5d5b 2119 CISS_REPORT_PHYS_DEV_FLAG_AIO_ENABLED) &&
3d46a59a 2120 phys_lun_ext_entry->aio_handle) {
6c223761 2121 device->aio_enabled = true;
9e322310
CIK		 2122 device->aio_handle =
2123 phys_lun_ext_entry->aio_handle;
3d46a59a 2124 }
6c223761
KB		 2125 } else {
2126 memcpy(device->volume_id, log_lun_ext_entry->volume_id,
2127 sizeof(device->volume_id));
2128 }
2129
3d46a59a
DB		 2130 if (pqi_is_device_with_sas_address(device))
2131 device->sas_address = get_unaligned_be64(&device->wwid);
6c223761
KB		 2132
2133 new_device_list[num_valid_devices++] = device;
2134 }
2135
2136 pqi_update_device_list(ctrl_info, new_device_list, num_valid_devices);
2137
2138out:
2139 list_for_each_entry_safe(device, next, &new_device_list_head,
2140 new_device_list_entry) {
2141 if (device->keep_device)
2142 continue;
2143 list_del(&device->new_device_list_entry);
2144 pqi_free_device(device);
2145 }
2146
2147 kfree(new_device_list);
2148 kfree(physdev_list);
2149 kfree(logdev_list);
2150 kfree(id_phys);
2151
2152 return rc;
2153}
2154
6c223761
KB		 2155static int pqi_scan_scsi_devices(struct pqi_ctrl_info *ctrl_info)
2156{
530dd8a7 2157 int rc = 0;
6c223761
KB		 2158
2159 if (pqi_ctrl_offline(ctrl_info))
2160 return -ENXIO;
2161
530dd8a7 2162 if (!mutex_trylock(&ctrl_info->scan_mutex)) {
5f310425 2163 pqi_schedule_rescan_worker_delayed(ctrl_info);
9946a398 2164 rc = -EINPROGRESS;
530dd8a7
MR		 2165 } else {
2166 rc = pqi_update_scsi_devices(ctrl_info);
2167 if (rc)
2168 pqi_schedule_rescan_worker_delayed(ctrl_info);
2169 mutex_unlock(&ctrl_info->scan_mutex);
2170 }
6c223761
KB		 2171
2172 return rc;
2173}
2174
2175static void pqi_scan_start(struct Scsi_Host *shost)
2176{
4fd22c13
MR		 2177 struct pqi_ctrl_info *ctrl_info;
2178
2179 ctrl_info = shost_to_hba(shost);
2180 if (pqi_ctrl_in_ofa(ctrl_info))
2181 return;
2182
2183 pqi_scan_scsi_devices(ctrl_info);
6c223761
KB		 2184}
2185
2186/* Returns TRUE if scan is finished. */
2187
2188static int pqi_scan_finished(struct Scsi_Host *shost,
2189 unsigned long elapsed_time)
2190{
2191 struct pqi_ctrl_info *ctrl_info;
2192
2193 ctrl_info = shost_priv(shost);
2194
2195 return !mutex_is_locked(&ctrl_info->scan_mutex);
2196}
2197
061ef06a
KB		 2198static void pqi_wait_until_scan_finished(struct pqi_ctrl_info *ctrl_info)
2199{
2200 mutex_lock(&ctrl_info->scan_mutex);
2201 mutex_unlock(&ctrl_info->scan_mutex);
2202}
2203
2204static void pqi_wait_until_lun_reset_finished(struct pqi_ctrl_info *ctrl_info)
2205{
2206 mutex_lock(&ctrl_info->lun_reset_mutex);
2207 mutex_unlock(&ctrl_info->lun_reset_mutex);
2208}
2209
4fd22c13
MR		 2210static void pqi_wait_until_ofa_finished(struct pqi_ctrl_info *ctrl_info)
2211{
2212 mutex_lock(&ctrl_info->ofa_mutex);
2213 mutex_unlock(&ctrl_info->ofa_mutex);
2214}
2215
6c223761
KB		 2216static inline void pqi_set_encryption_info(
2217 struct pqi_encryption_info *encryption_info, struct raid_map *raid_map,
2218 u64 first_block)
2219{
2220 u32 volume_blk_size;
2221
2222 /*
2223 * Set the encryption tweak values based on logical block address.
2224 * If the block size is 512, the tweak value is equal to the LBA.
2225 * For other block sizes, tweak value is (LBA * block size) / 512.
2226 */
2227 volume_blk_size = get_unaligned_le32(&raid_map->volume_blk_size);
2228 if (volume_blk_size != 512)
2229 first_block = (first_block * volume_blk_size) / 512;
2230
2231 encryption_info->data_encryption_key_index =
2232 get_unaligned_le16(&raid_map->data_encryption_key_index);
2233 encryption_info->encrypt_tweak_lower = lower_32_bits(first_block);
2234 encryption_info->encrypt_tweak_upper = upper_32_bits(first_block);
2235}
2236
2237/*
588a63fe 2238 * Attempt to perform RAID bypass mapping for a logical volume I/O.
6c223761
KB		 2239 */
2240
2241#define PQI_RAID_BYPASS_INELIGIBLE 1
2242
2243static int pqi_raid_bypass_submit_scsi_cmd(struct pqi_ctrl_info *ctrl_info,
2244 struct pqi_scsi_dev *device, struct scsi_cmnd *scmd,
2245 struct pqi_queue_group *queue_group)
2246{
2247 struct raid_map *raid_map;
2248 bool is_write = false;
2249 u32 map_index;
2250 u64 first_block;
2251 u64 last_block;
2252 u32 block_cnt;
2253 u32 blocks_per_row;
2254 u64 first_row;
2255 u64 last_row;
2256 u32 first_row_offset;
2257 u32 last_row_offset;
2258 u32 first_column;
2259 u32 last_column;
2260 u64 r0_first_row;
2261 u64 r0_last_row;
2262 u32 r5or6_blocks_per_row;
2263 u64 r5or6_first_row;
2264 u64 r5or6_last_row;
2265 u32 r5or6_first_row_offset;
2266 u32 r5or6_last_row_offset;
2267 u32 r5or6_first_column;
2268 u32 r5or6_last_column;
2269 u16 data_disks_per_row;
2270 u32 total_disks_per_row;
2271 u16 layout_map_count;
2272 u32 stripesize;
2273 u16 strip_size;
2274 u32 first_group;
2275 u32 last_group;
2276 u32 current_group;
2277 u32 map_row;
2278 u32 aio_handle;
2279 u64 disk_block;
2280 u32 disk_block_cnt;
2281 u8 cdb[16];
2282 u8 cdb_length;
2283 int offload_to_mirror;
2284 struct pqi_encryption_info *encryption_info_ptr;
2285 struct pqi_encryption_info encryption_info;
2286#if BITS_PER_LONG == 32
2287 u64 tmpdiv;
2288#endif
2289
2290 /* Check for valid opcode, get LBA and block count. */
2291 switch (scmd->cmnd[0]) {
2292 case WRITE_6:
2293 is_write = true;
df561f66 2294 fallthrough;
6c223761 2295 case READ_6:
e018ef57
B		 2296 first_block = (u64)(((scmd->cmnd[1] & 0x1f) << 16) |
2297 (scmd->cmnd[2] << 8) | scmd->cmnd[3]);
6c223761
KB		 2298 block_cnt = (u32)scmd->cmnd[4];
2299 if (block_cnt == 0)
2300 block_cnt = 256;
2301 break;
2302 case WRITE_10:
2303 is_write = true;
df561f66 2304 fallthrough;
6c223761
KB		 2305 case READ_10:
2306 first_block = (u64)get_unaligned_be32(&scmd->cmnd[2]);
2307 block_cnt = (u32)get_unaligned_be16(&scmd->cmnd[7]);
2308 break;
2309 case WRITE_12:
2310 is_write = true;
df561f66 2311 fallthrough;
6c223761
KB		 2312 case READ_12:
2313 first_block = (u64)get_unaligned_be32(&scmd->cmnd[2]);
2314 block_cnt = get_unaligned_be32(&scmd->cmnd[6]);
2315 break;
2316 case WRITE_16:
2317 is_write = true;
df561f66 2318 fallthrough;
6c223761
KB		 2319 case READ_16:
2320 first_block = get_unaligned_be64(&scmd->cmnd[2]);
2321 block_cnt = get_unaligned_be32(&scmd->cmnd[10]);
2322 break;
2323 default:
2324 /* Process via normal I/O path. */
2325 return PQI_RAID_BYPASS_INELIGIBLE;
2326 }
2327
2328 /* Check for write to non-RAID-0. */
2329 if (is_write && device->raid_level != SA_RAID_0)
2330 return PQI_RAID_BYPASS_INELIGIBLE;
2331
2332 if (unlikely(block_cnt == 0))
2333 return PQI_RAID_BYPASS_INELIGIBLE;
2334
2335 last_block = first_block + block_cnt - 1;
2336 raid_map = device->raid_map;
2337
2338 /* Check for invalid block or wraparound. */
2339 if (last_block >= get_unaligned_le64(&raid_map->volume_blk_cnt) ||
2340 last_block < first_block)
2341 return PQI_RAID_BYPASS_INELIGIBLE;
2342
2343 data_disks_per_row = get_unaligned_le16(&raid_map->data_disks_per_row);
2344 strip_size = get_unaligned_le16(&raid_map->strip_size);
2345 layout_map_count = get_unaligned_le16(&raid_map->layout_map_count);
2346
2347 /* Calculate stripe information for the request. */
2348 blocks_per_row = data_disks_per_row * strip_size;
2349#if BITS_PER_LONG == 32
2350 tmpdiv = first_block;
2351 do_div(tmpdiv, blocks_per_row);
2352 first_row = tmpdiv;
2353 tmpdiv = last_block;
2354 do_div(tmpdiv, blocks_per_row);
2355 last_row = tmpdiv;
2356 first_row_offset = (u32)(first_block - (first_row * blocks_per_row));
2357 last_row_offset = (u32)(last_block - (last_row * blocks_per_row));
2358 tmpdiv = first_row_offset;
2359 do_div(tmpdiv, strip_size);
2360 first_column = tmpdiv;
2361 tmpdiv = last_row_offset;
2362 do_div(tmpdiv, strip_size);
2363 last_column = tmpdiv;
2364#else
2365 first_row = first_block / blocks_per_row;
2366 last_row = last_block / blocks_per_row;
2367 first_row_offset = (u32)(first_block - (first_row * blocks_per_row));
2368 last_row_offset = (u32)(last_block - (last_row * blocks_per_row));
2369 first_column = first_row_offset / strip_size;
2370 last_column = last_row_offset / strip_size;
2371#endif
2372
2373 /* If this isn't a single row/column then give to the controller. */
2374 if (first_row != last_row || first_column != last_column)
2375 return PQI_RAID_BYPASS_INELIGIBLE;
2376
2377 /* Proceeding with driver mapping. */
2378 total_disks_per_row = data_disks_per_row +
2379 get_unaligned_le16(&raid_map->metadata_disks_per_row);
2380 map_row = ((u32)(first_row >> raid_map->parity_rotation_shift)) %
2381 get_unaligned_le16(&raid_map->row_cnt);
2382 map_index = (map_row * total_disks_per_row) + first_column;
2383
2384 /* RAID 1 */
2385 if (device->raid_level == SA_RAID_1) {
2386 if (device->offload_to_mirror)
2387 map_index += data_disks_per_row;
2388 device->offload_to_mirror = !device->offload_to_mirror;
2389 } else if (device->raid_level == SA_RAID_ADM) {
2390 /* RAID ADM */
2391 /*
2392 * Handles N-way mirrors (R1-ADM) and R10 with # of drives
2393 * divisible by 3.
2394 */
2395 offload_to_mirror = device->offload_to_mirror;
2396 if (offload_to_mirror == 0) {
2397 /* use physical disk in the first mirrored group. */
2398 map_index %= data_disks_per_row;
2399 } else {
2400 do {
2401 /*
2402 * Determine mirror group that map_index
2403 * indicates.
2404 */
2405 current_group = map_index / data_disks_per_row;
2406
2407 if (offload_to_mirror != current_group) {
2408 if (current_group <
2409 layout_map_count - 1) {
2410 /*
2411 * Select raid index from
2412 * next group.
2413 */
2414 map_index += data_disks_per_row;
2415 current_group++;
2416 } else {
2417 /*
2418 * Select raid index from first
2419 * group.
2420 */
2421 map_index %= data_disks_per_row;
2422 current_group = 0;
2423 }
2424 }
2425 } while (offload_to_mirror != current_group);
2426 }
2427
2428 /* Set mirror group to use next time. */
2429 offload_to_mirror =
2430 (offload_to_mirror >= layout_map_count - 1) ?
2431 0 : offload_to_mirror + 1;
6c223761
KB		 2432 device->offload_to_mirror = offload_to_mirror;
2433 /*
2434 * Avoid direct use of device->offload_to_mirror within this
2435 * function since multiple threads might simultaneously
2436 * increment it beyond the range of device->layout_map_count -1.
2437 */
2438 } else if ((device->raid_level == SA_RAID_5 ||
2439 device->raid_level == SA_RAID_6) && layout_map_count > 1) {
2440 /* RAID 50/60 */
2441 /* Verify first and last block are in same RAID group */
2442 r5or6_blocks_per_row = strip_size * data_disks_per_row;
2443 stripesize = r5or6_blocks_per_row * layout_map_count;
2444#if BITS_PER_LONG == 32
2445 tmpdiv = first_block;
2446 first_group = do_div(tmpdiv, stripesize);
2447 tmpdiv = first_group;
2448 do_div(tmpdiv, r5or6_blocks_per_row);
2449 first_group = tmpdiv;
2450 tmpdiv = last_block;
2451 last_group = do_div(tmpdiv, stripesize);
2452 tmpdiv = last_group;
2453 do_div(tmpdiv, r5or6_blocks_per_row);
2454 last_group = tmpdiv;
2455#else
2456 first_group = (first_block % stripesize) / r5or6_blocks_per_row;
2457 last_group = (last_block % stripesize) / r5or6_blocks_per_row;
2458#endif
2459 if (first_group != last_group)
2460 return PQI_RAID_BYPASS_INELIGIBLE;
2461
2462 /* Verify request is in a single row of RAID 5/6 */
2463#if BITS_PER_LONG == 32
2464 tmpdiv = first_block;
2465 do_div(tmpdiv, stripesize);
2466 first_row = r5or6_first_row = r0_first_row = tmpdiv;
2467 tmpdiv = last_block;
2468 do_div(tmpdiv, stripesize);
2469 r5or6_last_row = r0_last_row = tmpdiv;
2470#else
2471 first_row = r5or6_first_row = r0_first_row =
2472 first_block / stripesize;
2473 r5or6_last_row = r0_last_row = last_block / stripesize;
2474#endif
2475 if (r5or6_first_row != r5or6_last_row)
2476 return PQI_RAID_BYPASS_INELIGIBLE;
2477
2478 /* Verify request is in a single column */
2479#if BITS_PER_LONG == 32
2480 tmpdiv = first_block;
2481 first_row_offset = do_div(tmpdiv, stripesize);
2482 tmpdiv = first_row_offset;
2483 first_row_offset = (u32)do_div(tmpdiv, r5or6_blocks_per_row);
2484 r5or6_first_row_offset = first_row_offset;
2485 tmpdiv = last_block;
2486 r5or6_last_row_offset = do_div(tmpdiv, stripesize);
2487 tmpdiv = r5or6_last_row_offset;
2488 r5or6_last_row_offset = do_div(tmpdiv, r5or6_blocks_per_row);
2489 tmpdiv = r5or6_first_row_offset;
2490 do_div(tmpdiv, strip_size);
2491 first_column = r5or6_first_column = tmpdiv;
2492 tmpdiv = r5or6_last_row_offset;
2493 do_div(tmpdiv, strip_size);
2494 r5or6_last_column = tmpdiv;
2495#else
2496 first_row_offset = r5or6_first_row_offset =
2497 (u32)((first_block % stripesize) %
2498 r5or6_blocks_per_row);
2499
2500 r5or6_last_row_offset =
2501 (u32)((last_block % stripesize) %
2502 r5or6_blocks_per_row);
2503
2504 first_column = r5or6_first_row_offset / strip_size;
2505 r5or6_first_column = first_column;
2506 r5or6_last_column = r5or6_last_row_offset / strip_size;
2507#endif
2508 if (r5or6_first_column != r5or6_last_column)
2509 return PQI_RAID_BYPASS_INELIGIBLE;
2510
2511 /* Request is eligible */
2512 map_row =
2513 ((u32)(first_row >> raid_map->parity_rotation_shift)) %
2514 get_unaligned_le16(&raid_map->row_cnt);
2515
2516 map_index = (first_group *
2517 (get_unaligned_le16(&raid_map->row_cnt) *
2518 total_disks_per_row)) +
2519 (map_row * total_disks_per_row) + first_column;
2520 }
2521
6c223761
KB		 2522 aio_handle = raid_map->disk_data[map_index].aio_handle;
2523 disk_block = get_unaligned_le64(&raid_map->disk_starting_blk) +
2524 first_row * strip_size +
2525 (first_row_offset - first_column * strip_size);
2526 disk_block_cnt = block_cnt;
2527
2528 /* Handle differing logical/physical block sizes. */
2529 if (raid_map->phys_blk_shift) {
2530 disk_block <<= raid_map->phys_blk_shift;
2531 disk_block_cnt <<= raid_map->phys_blk_shift;
2532 }
2533
2534 if (unlikely(disk_block_cnt > 0xffff))
2535 return PQI_RAID_BYPASS_INELIGIBLE;
2536
2537 /* Build the new CDB for the physical disk I/O. */
2538 if (disk_block > 0xffffffff) {
2539 cdb[0] = is_write ? WRITE_16 : READ_16;
2540 cdb[1] = 0;
2541 put_unaligned_be64(disk_block, &cdb[2]);
2542 put_unaligned_be32(disk_block_cnt, &cdb[10]);
2543 cdb[14] = 0;
2544 cdb[15] = 0;
2545 cdb_length = 16;
2546 } else {
2547 cdb[0] = is_write ? WRITE_10 : READ_10;
2548 cdb[1] = 0;
2549 put_unaligned_be32((u32)disk_block, &cdb[2]);
2550 cdb[6] = 0;
2551 put_unaligned_be16((u16)disk_block_cnt, &cdb[7]);
2552 cdb[9] = 0;
2553 cdb_length = 10;
2554 }
2555
2556 if (get_unaligned_le16(&raid_map->flags) &
2557 RAID_MAP_ENCRYPTION_ENABLED) {
2558 pqi_set_encryption_info(&encryption_info, raid_map,
2559 first_block);
2560 encryption_info_ptr = &encryption_info;
2561 } else {
2562 encryption_info_ptr = NULL;
2563 }
2564
2565 return pqi_aio_submit_io(ctrl_info, scmd, aio_handle,
376fb880 2566 cdb, cdb_length, queue_group, encryption_info_ptr, true);
6c223761
KB		 2567}
2568
2569#define PQI_STATUS_IDLE 0x0
2570
2571#define PQI_CREATE_ADMIN_QUEUE_PAIR 1
2572#define PQI_DELETE_ADMIN_QUEUE_PAIR 2
2573
2574#define PQI_DEVICE_STATE_POWER_ON_AND_RESET 0x0
2575#define PQI_DEVICE_STATE_STATUS_AVAILABLE 0x1
2576#define PQI_DEVICE_STATE_ALL_REGISTERS_READY 0x2
2577#define PQI_DEVICE_STATE_ADMIN_QUEUE_PAIR_READY 0x3
2578#define PQI_DEVICE_STATE_ERROR 0x4
2579
2580#define PQI_MODE_READY_TIMEOUT_SECS 30
2581#define PQI_MODE_READY_POLL_INTERVAL_MSECS 1
2582
2583static int pqi_wait_for_pqi_mode_ready(struct pqi_ctrl_info *ctrl_info)
2584{
2585 struct pqi_device_registers __iomem *pqi_registers;
2586 unsigned long timeout;
2587 u64 signature;
2588 u8 status;
2589
2590 pqi_registers = ctrl_info->pqi_registers;
4fd22c13 2591 timeout = (PQI_MODE_READY_TIMEOUT_SECS * PQI_HZ) + jiffies;
6c223761
KB		 2592
2593 while (1) {
2594 signature = readq(&pqi_registers->signature);
2595 if (memcmp(&signature, PQI_DEVICE_SIGNATURE,
2596 sizeof(signature)) == 0)
2597 break;
2598 if (time_after(jiffies, timeout)) {
2599 dev_err(&ctrl_info->pci_dev->dev,
2600 "timed out waiting for PQI signature\n");
2601 return -ETIMEDOUT;
2602 }
2603 msleep(PQI_MODE_READY_POLL_INTERVAL_MSECS);
2604 }
2605
2606 while (1) {
2607 status = readb(&pqi_registers->function_and_status_code);
2608 if (status == PQI_STATUS_IDLE)
2609 break;
2610 if (time_after(jiffies, timeout)) {
2611 dev_err(&ctrl_info->pci_dev->dev,
2612 "timed out waiting for PQI IDLE\n");
2613 return -ETIMEDOUT;
2614 }
2615 msleep(PQI_MODE_READY_POLL_INTERVAL_MSECS);
2616 }
2617
2618 while (1) {
2619 if (readl(&pqi_registers->device_status) ==
2620 PQI_DEVICE_STATE_ALL_REGISTERS_READY)
2621 break;
2622 if (time_after(jiffies, timeout)) {
2623 dev_err(&ctrl_info->pci_dev->dev,
2624 "timed out waiting for PQI all registers ready\n");
2625 return -ETIMEDOUT;
2626 }
2627 msleep(PQI_MODE_READY_POLL_INTERVAL_MSECS);
2628 }
2629
2630 return 0;
2631}
2632
2633static inline void pqi_aio_path_disabled(struct pqi_io_request *io_request)
2634{
2635 struct pqi_scsi_dev *device;
2636
2637 device = io_request->scmd->device->hostdata;
588a63fe 2638 device->raid_bypass_enabled = false;
376fb880 2639 device->aio_enabled = false;
6c223761
KB		 2640}
2641
d87d5474 2642static inline void pqi_take_device_offline(struct scsi_device *sdev, char *path)
6c223761
KB		 2643{
2644 struct pqi_ctrl_info *ctrl_info;
e58081a7 2645 struct pqi_scsi_dev *device;
6c223761 2646
03b288cf
KB		 2647 device = sdev->hostdata;
2648 if (device->device_offline)
2649 return;
2650
2651 device->device_offline = true;
03b288cf
KB		 2652 ctrl_info = shost_to_hba(sdev->host);
2653 pqi_schedule_rescan_worker(ctrl_info);
a9a68101 2654 dev_err(&ctrl_info->pci_dev->dev, "re-scanning %s scsi %d:%d:%d:%d\n",
03b288cf
KB		 2655 path, ctrl_info->scsi_host->host_no, device->bus,
2656 device->target, device->lun);
6c223761
KB		 2657}
2658
2659static void pqi_process_raid_io_error(struct pqi_io_request *io_request)
2660{
2661 u8 scsi_status;
2662 u8 host_byte;
2663 struct scsi_cmnd *scmd;
2664 struct pqi_raid_error_info *error_info;
2665 size_t sense_data_length;
2666 int residual_count;
2667 int xfer_count;
2668 struct scsi_sense_hdr sshdr;
2669
2670 scmd = io_request->scmd;
2671 if (!scmd)
2672 return;
2673
2674 error_info = io_request->error_info;
2675 scsi_status = error_info->status;
2676 host_byte = DID_OK;
2677
f5b63206
KB		 2678 switch (error_info->data_out_result) {
2679 case PQI_DATA_IN_OUT_GOOD:
2680 break;
2681 case PQI_DATA_IN_OUT_UNDERFLOW:
6c223761
KB		 2682 xfer_count =
2683 get_unaligned_le32(&error_info->data_out_transferred);
2684 residual_count = scsi_bufflen(scmd) - xfer_count;
2685 scsi_set_resid(scmd, residual_count);
2686 if (xfer_count < scmd->underflow)
2687 host_byte = DID_SOFT_ERROR;
f5b63206
KB		 2688 break;
2689 case PQI_DATA_IN_OUT_UNSOLICITED_ABORT:
2690 case PQI_DATA_IN_OUT_ABORTED:
2691 host_byte = DID_ABORT;
2692 break;
2693 case PQI_DATA_IN_OUT_TIMEOUT:
2694 host_byte = DID_TIME_OUT;
2695 break;
2696 case PQI_DATA_IN_OUT_BUFFER_OVERFLOW:
2697 case PQI_DATA_IN_OUT_PROTOCOL_ERROR:
2698 case PQI_DATA_IN_OUT_BUFFER_ERROR:
2699 case PQI_DATA_IN_OUT_BUFFER_OVERFLOW_DESCRIPTOR_AREA:
2700 case PQI_DATA_IN_OUT_BUFFER_OVERFLOW_BRIDGE:
2701 case PQI_DATA_IN_OUT_ERROR:
2702 case PQI_DATA_IN_OUT_HARDWARE_ERROR:
2703 case PQI_DATA_IN_OUT_PCIE_FABRIC_ERROR:
2704 case PQI_DATA_IN_OUT_PCIE_COMPLETION_TIMEOUT:
2705 case PQI_DATA_IN_OUT_PCIE_COMPLETER_ABORT_RECEIVED:
2706 case PQI_DATA_IN_OUT_PCIE_UNSUPPORTED_REQUEST_RECEIVED:
2707 case PQI_DATA_IN_OUT_PCIE_ECRC_CHECK_FAILED:
2708 case PQI_DATA_IN_OUT_PCIE_UNSUPPORTED_REQUEST:
2709 case PQI_DATA_IN_OUT_PCIE_ACS_VIOLATION:
2710 case PQI_DATA_IN_OUT_PCIE_TLP_PREFIX_BLOCKED:
2711 case PQI_DATA_IN_OUT_PCIE_POISONED_MEMORY_READ:
2712 default:
2713 host_byte = DID_ERROR;
2714 break;
6c223761
KB		 2715 }
2716
2717 sense_data_length = get_unaligned_le16(&error_info->sense_data_length);
2718 if (sense_data_length == 0)
2719 sense_data_length =
2720 get_unaligned_le16(&error_info->response_data_length);
2721 if (sense_data_length) {
2722 if (sense_data_length > sizeof(error_info->data))
2723 sense_data_length = sizeof(error_info->data);
2724
2725 if (scsi_status == SAM_STAT_CHECK_CONDITION &&
2726 scsi_normalize_sense(error_info->data,
2727 sense_data_length, &sshdr) &&
2728 sshdr.sense_key == HARDWARE_ERROR &&
8ef860ae 2729 sshdr.asc == 0x3e) {
441b7195
EV		 2730 struct pqi_ctrl_info *ctrl_info = shost_to_hba(scmd->device->host);
2731 struct pqi_scsi_dev *device = scmd->device->hostdata;
2732
8ef860ae
EV		 2733 switch (sshdr.ascq) {
2734 case 0x1: /* LOGICAL UNIT FAILURE */
2735 if (printk_ratelimit())
2736 scmd_printk(KERN_ERR, scmd, "received 'logical unit failure' from controller for scsi %d:%d:%d:%d\n",
2737 ctrl_info->scsi_host->host_no, device->bus, device->target, device->lun);
2738 pqi_take_device_offline(scmd->device, "RAID");
2739 host_byte = DID_NO_CONNECT;
2740 break;
2741
2742 default: /* See http://www.t10.org/lists/asc-num.htm#ASC_3E */
2743 if (printk_ratelimit())
2744 scmd_printk(KERN_ERR, scmd, "received unhandled error %d from controller for scsi %d:%d:%d:%d\n",
2745 sshdr.ascq, ctrl_info->scsi_host->host_no, device->bus, device->target, device->lun);
2746 break;
2747 }
6c223761
KB		 2748 }
2749
2750 if (sense_data_length > SCSI_SENSE_BUFFERSIZE)
2751 sense_data_length = SCSI_SENSE_BUFFERSIZE;
2752 memcpy(scmd->sense_buffer, error_info->data,
2753 sense_data_length);
2754 }
2755
2756 scmd->result = scsi_status;
2757 set_host_byte(scmd, host_byte);
2758}
2759
2760static void pqi_process_aio_io_error(struct pqi_io_request *io_request)
2761{
2762 u8 scsi_status;
2763 u8 host_byte;
2764 struct scsi_cmnd *scmd;
2765 struct pqi_aio_error_info *error_info;
2766 size_t sense_data_length;
2767 int residual_count;
2768 int xfer_count;
2769 bool device_offline;
2770
2771 scmd = io_request->scmd;
2772 error_info = io_request->error_info;
2773 host_byte = DID_OK;
2774 sense_data_length = 0;
2775 device_offline = false;
2776
2777 switch (error_info->service_response) {
2778 case PQI_AIO_SERV_RESPONSE_COMPLETE:
2779 scsi_status = error_info->status;
2780 break;
2781 case PQI_AIO_SERV_RESPONSE_FAILURE:
2782 switch (error_info->status) {
2783 case PQI_AIO_STATUS_IO_ABORTED:
2784 scsi_status = SAM_STAT_TASK_ABORTED;
2785 break;
2786 case PQI_AIO_STATUS_UNDERRUN:
2787 scsi_status = SAM_STAT_GOOD;
2788 residual_count = get_unaligned_le32(
2789 &error_info->residual_count);
2790 scsi_set_resid(scmd, residual_count);
2791 xfer_count = scsi_bufflen(scmd) - residual_count;
2792 if (xfer_count < scmd->underflow)
2793 host_byte = DID_SOFT_ERROR;
2794 break;
2795 case PQI_AIO_STATUS_OVERRUN:
2796 scsi_status = SAM_STAT_GOOD;
2797 break;
2798 case PQI_AIO_STATUS_AIO_PATH_DISABLED:
2799 pqi_aio_path_disabled(io_request);
2800 scsi_status = SAM_STAT_GOOD;
2801 io_request->status = -EAGAIN;
2802 break;
2803 case PQI_AIO_STATUS_NO_PATH_TO_DEVICE:
2804 case PQI_AIO_STATUS_INVALID_DEVICE:
376fb880
KB		 2805 if (!io_request->raid_bypass) {
2806 device_offline = true;
2807 pqi_take_device_offline(scmd->device, "AIO");
2808 host_byte = DID_NO_CONNECT;
2809 }
6c223761
KB		 2810 scsi_status = SAM_STAT_CHECK_CONDITION;
2811 break;
2812 case PQI_AIO_STATUS_IO_ERROR:
2813 default:
2814 scsi_status = SAM_STAT_CHECK_CONDITION;
2815 break;
2816 }
2817 break;
2818 case PQI_AIO_SERV_RESPONSE_TMF_COMPLETE:
2819 case PQI_AIO_SERV_RESPONSE_TMF_SUCCEEDED:
2820 scsi_status = SAM_STAT_GOOD;
2821 break;
2822 case PQI_AIO_SERV_RESPONSE_TMF_REJECTED:
2823 case PQI_AIO_SERV_RESPONSE_TMF_INCORRECT_LUN:
2824 default:
2825 scsi_status = SAM_STAT_CHECK_CONDITION;
2826 break;
2827 }
2828
2829 if (error_info->data_present) {
2830 sense_data_length =
2831 get_unaligned_le16(&error_info->data_length);
2832 if (sense_data_length) {
2833 if (sense_data_length > sizeof(error_info->data))
2834 sense_data_length = sizeof(error_info->data);
2835 if (sense_data_length > SCSI_SENSE_BUFFERSIZE)
2836 sense_data_length = SCSI_SENSE_BUFFERSIZE;
2837 memcpy(scmd->sense_buffer, error_info->data,
2838 sense_data_length);
2839 }
2840 }
2841
2842 if (device_offline && sense_data_length == 0)
2843 scsi_build_sense_buffer(0, scmd->sense_buffer, HARDWARE_ERROR,
2844 0x3e, 0x1);
2845
2846 scmd->result = scsi_status;
2847 set_host_byte(scmd, host_byte);
2848}
2849
2850static void pqi_process_io_error(unsigned int iu_type,
2851 struct pqi_io_request *io_request)
2852{
2853 switch (iu_type) {
2854 case PQI_RESPONSE_IU_RAID_PATH_IO_ERROR:
2855 pqi_process_raid_io_error(io_request);
2856 break;
2857 case PQI_RESPONSE_IU_AIO_PATH_IO_ERROR:
2858 pqi_process_aio_io_error(io_request);
2859 break;
2860 }
2861}
2862
2863static int pqi_interpret_task_management_response(
2864 struct pqi_task_management_response *response)
2865{
2866 int rc;
2867
2868 switch (response->response_code) {
b17f0486
KB		 2869 case SOP_TMF_COMPLETE:
2870 case SOP_TMF_FUNCTION_SUCCEEDED:
6c223761
KB		 2871 rc = 0;
2872 break;
3406384b
MR		 2873 case SOP_TMF_REJECTED:
2874 rc = -EAGAIN;
2875 break;
6c223761
KB		 2876 default:
2877 rc = -EIO;
2878 break;
2879 }
2880
2881 return rc;
2882}
2883
9e68cccc 2884static inline void pqi_invalid_response(struct pqi_ctrl_info *ctrl_info)
6c223761 2885{
9e68cccc
KB		 2886 pqi_take_ctrl_offline(ctrl_info);
2887}
2888
2889static int pqi_process_io_intr(struct pqi_ctrl_info *ctrl_info, struct pqi_queue_group *queue_group)
6c223761 2890{
9e68cccc 2891 int num_responses;
6c223761
KB		 2892 pqi_index_t oq_pi;
2893 pqi_index_t oq_ci;
2894 struct pqi_io_request *io_request;
2895 struct pqi_io_response *response;
2896 u16 request_id;
2897
2898 num_responses = 0;
2899 oq_ci = queue_group->oq_ci_copy;
2900
2901 while (1) {
dac12fbc 2902 oq_pi = readl(queue_group->oq_pi);
9e68cccc
KB		 2903 if (oq_pi >= ctrl_info->num_elements_per_oq) {
2904 pqi_invalid_response(ctrl_info);
2905 dev_err(&ctrl_info->pci_dev->dev,
2906 "I/O interrupt: producer index (%u) out of range (0-%u): consumer index: %u\n",
2907 oq_pi, ctrl_info->num_elements_per_oq - 1, oq_ci);
2908 return -1;
2909 }
6c223761
KB		 2910 if (oq_pi == oq_ci)
2911 break;
2912
2913 num_responses++;
2914 response = queue_group->oq_element_array +
2915 (oq_ci * PQI_OPERATIONAL_OQ_ELEMENT_LENGTH);
2916
2917 request_id = get_unaligned_le16(&response->request_id);
9e68cccc
KB		 2918 if (request_id >= ctrl_info->max_io_slots) {
2919 pqi_invalid_response(ctrl_info);
2920 dev_err(&ctrl_info->pci_dev->dev,
2921 "request ID in response (%u) out of range (0-%u): producer index: %u consumer index: %u\n",
2922 request_id, ctrl_info->max_io_slots - 1, oq_pi, oq_ci);
2923 return -1;
2924 }
6c223761
KB		 2925
2926 io_request = &ctrl_info->io_request_pool[request_id];
9e68cccc
KB		 2927 if (atomic_read(&io_request->refcount) == 0) {
2928 pqi_invalid_response(ctrl_info);
2929 dev_err(&ctrl_info->pci_dev->dev,
2930 "request ID in response (%u) does not match an outstanding I/O request: producer index: %u consumer index: %u\n",
2931 request_id, oq_pi, oq_ci);
2932 return -1;
2933 }
6c223761
KB		 2934
2935 switch (response->header.iu_type) {
2936 case PQI_RESPONSE_IU_RAID_PATH_IO_SUCCESS:
2937 case PQI_RESPONSE_IU_AIO_PATH_IO_SUCCESS:
2ba55c98
KB		 2938 if (io_request->scmd)
2939 io_request->scmd->result = 0;
df561f66 2940 fallthrough;
6c223761
KB		 2941 case PQI_RESPONSE_IU_GENERAL_MANAGEMENT:
2942 break;
b212c251
KB		 2943 case PQI_RESPONSE_IU_VENDOR_GENERAL:
2944 io_request->status =
2945 get_unaligned_le16(
2946 &((struct pqi_vendor_general_response *)
2947 response)->status);
2948 break;
6c223761
KB		 2949 case PQI_RESPONSE_IU_TASK_MANAGEMENT:
2950 io_request->status =
2951 pqi_interpret_task_management_response(
2952 (void *)response);
2953 break;
2954 case PQI_RESPONSE_IU_AIO_PATH_DISABLED:
2955 pqi_aio_path_disabled(io_request);
2956 io_request->status = -EAGAIN;
2957 break;
2958 case PQI_RESPONSE_IU_RAID_PATH_IO_ERROR:
2959 case PQI_RESPONSE_IU_AIO_PATH_IO_ERROR:
2960 io_request->error_info = ctrl_info->error_buffer +
2961 (get_unaligned_le16(&response->error_index) *
2962 PQI_ERROR_BUFFER_ELEMENT_LENGTH);
9e68cccc 2963 pqi_process_io_error(response->header.iu_type, io_request);
6c223761
KB		 2964 break;
2965 default:
9e68cccc 2966 pqi_invalid_response(ctrl_info);
6c223761 2967 dev_err(&ctrl_info->pci_dev->dev,
9e68cccc
KB		 2968 "unexpected IU type: 0x%x: producer index: %u consumer index: %u\n",
2969 response->header.iu_type, oq_pi, oq_ci);
2970 return -1;
6c223761
KB		 2971 }
2972
9e68cccc 2973 io_request->io_complete_callback(io_request, io_request->context);
6c223761
KB		 2974
2975 /*
2976 * Note that the I/O request structure CANNOT BE TOUCHED after
2977 * returning from the I/O completion callback!
2978 */
6c223761
KB		 2979 oq_ci = (oq_ci + 1) % ctrl_info->num_elements_per_oq;
2980 }
2981
2982 if (num_responses) {
2983 queue_group->oq_ci_copy = oq_ci;
2984 writel(oq_ci, queue_group->oq_ci);
2985 }
2986
2987 return num_responses;
2988}
2989
2990static inline unsigned int pqi_num_elements_free(unsigned int pi,
df7a1fcf 2991 unsigned int ci, unsigned int elements_in_queue)
6c223761
KB		 2992{
2993 unsigned int num_elements_used;
2994
2995 if (pi >= ci)
2996 num_elements_used = pi - ci;
2997 else
2998 num_elements_used = elements_in_queue - ci + pi;
2999
3000 return elements_in_queue - num_elements_used - 1;
3001}
3002
98f87667 3003static void pqi_send_event_ack(struct pqi_ctrl_info *ctrl_info,
6c223761
KB		 3004 struct pqi_event_acknowledge_request *iu, size_t iu_length)
3005{
3006 pqi_index_t iq_pi;
3007 pqi_index_t iq_ci;
3008 unsigned long flags;
3009 void *next_element;
6c223761
KB		 3010 struct pqi_queue_group *queue_group;
3011
3012 queue_group = &ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP];
3013 put_unaligned_le16(queue_group->oq_id, &iu->header.response_queue_id);
3014
6c223761
KB		 3015 while (1) {
3016 spin_lock_irqsave(&queue_group->submit_lock[RAID_PATH], flags);
3017
3018 iq_pi = queue_group->iq_pi_copy[RAID_PATH];
dac12fbc 3019 iq_ci = readl(queue_group->iq_ci[RAID_PATH]);
6c223761
KB		 3020
3021 if (pqi_num_elements_free(iq_pi, iq_ci,
3022 ctrl_info->num_elements_per_iq))
3023 break;
3024
3025 spin_unlock_irqrestore(
3026 &queue_group->submit_lock[RAID_PATH], flags);
3027
98f87667 3028 if (pqi_ctrl_offline(ctrl_info))
6c223761 3029 return;
6c223761
KB		 3030 }
3031
3032 next_element = queue_group->iq_element_array[RAID_PATH] +
3033 (iq_pi * PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
3034
3035 memcpy(next_element, iu, iu_length);
3036
3037 iq_pi = (iq_pi + 1) % ctrl_info->num_elements_per_iq;
6c223761
KB		 3038 queue_group->iq_pi_copy[RAID_PATH] = iq_pi;
3039
3040 /*
3041 * This write notifies the controller that an IU is available to be
3042 * processed.
3043 */
3044 writel(iq_pi, queue_group->iq_pi[RAID_PATH]);
3045
3046 spin_unlock_irqrestore(&queue_group->submit_lock[RAID_PATH], flags);
6c223761
KB		 3047}
3048
3049static void pqi_acknowledge_event(struct pqi_ctrl_info *ctrl_info,
3050 struct pqi_event *event)
3051{
3052 struct pqi_event_acknowledge_request request;
3053
3054 memset(&request, 0, sizeof(request));
3055
3056 request.header.iu_type = PQI_REQUEST_IU_ACKNOWLEDGE_VENDOR_EVENT;
3057 put_unaligned_le16(sizeof(request) - PQI_REQUEST_HEADER_LENGTH,
3058 &request.header.iu_length);
3059 request.event_type = event->event_type;
3060 request.event_id = event->event_id;
3061 request.additional_event_id = event->additional_event_id;
3062
98f87667 3063 pqi_send_event_ack(ctrl_info, &request, sizeof(request));
6c223761
KB		 3064}
3065
4fd22c13
MR		 3066#define PQI_SOFT_RESET_STATUS_TIMEOUT_SECS 30
3067#define PQI_SOFT_RESET_STATUS_POLL_INTERVAL_SECS 1
3068
3069static enum pqi_soft_reset_status pqi_poll_for_soft_reset_status(
3070 struct pqi_ctrl_info *ctrl_info)
6c223761 3071{
4fd22c13
MR		 3072 unsigned long timeout;
3073 u8 status;
6c223761 3074
4fd22c13 3075 timeout = (PQI_SOFT_RESET_STATUS_TIMEOUT_SECS * PQI_HZ) + jiffies;
6c223761 3076
4fd22c13
MR		 3077 while (1) {
3078 status = pqi_read_soft_reset_status(ctrl_info);
3079 if (status & PQI_SOFT_RESET_INITIATE)
3080 return RESET_INITIATE_DRIVER;
3081
3082 if (status & PQI_SOFT_RESET_ABORT)
3083 return RESET_ABORT;
3084
3085 if (time_after(jiffies, timeout)) {
3086 dev_err(&ctrl_info->pci_dev->dev,
3087 "timed out waiting for soft reset status\n");
3088 return RESET_TIMEDOUT;
3089 }
3090
3091 if (!sis_is_firmware_running(ctrl_info))
3092 return RESET_NORESPONSE;
3093
3094 ssleep(PQI_SOFT_RESET_STATUS_POLL_INTERVAL_SECS);
3095 }
3096}
3097
3098static void pqi_process_soft_reset(struct pqi_ctrl_info *ctrl_info,
694c5d5b 3099 enum pqi_soft_reset_status reset_status)
4fd22c13
MR		 3100{
3101 int rc;
3102
3103 switch (reset_status) {
3104 case RESET_INITIATE_DRIVER:
4fd22c13
MR		 3105 case RESET_TIMEDOUT:
3106 dev_info(&ctrl_info->pci_dev->dev,
3107 "resetting controller %u\n", ctrl_info->ctrl_id);
3108 sis_soft_reset(ctrl_info);
df561f66 3109 fallthrough;
4fd22c13
MR		 3110 case RESET_INITIATE_FIRMWARE:
3111 rc = pqi_ofa_ctrl_restart(ctrl_info);
3112 pqi_ofa_free_host_buffer(ctrl_info);
3113 dev_info(&ctrl_info->pci_dev->dev,
3114 "Online Firmware Activation for controller %u: %s\n",
3115 ctrl_info->ctrl_id, rc == 0 ? "SUCCESS" : "FAILED");
3116 break;
3117 case RESET_ABORT:
3118 pqi_ofa_ctrl_unquiesce(ctrl_info);
3119 dev_info(&ctrl_info->pci_dev->dev,
3120 "Online Firmware Activation for controller %u: %s\n",
3121 ctrl_info->ctrl_id, "ABORTED");
3122 break;
3123 case RESET_NORESPONSE:
3124 pqi_ofa_free_host_buffer(ctrl_info);
3125 pqi_take_ctrl_offline(ctrl_info);
3126 break;
3127 }
3128}
3129
3130static void pqi_ofa_process_event(struct pqi_ctrl_info *ctrl_info,
3131 struct pqi_event *event)
3132{
3133 u16 event_id;
3134 enum pqi_soft_reset_status status;
3135
3136 event_id = get_unaligned_le16(&event->event_id);
3137
3138 mutex_lock(&ctrl_info->ofa_mutex);
3139
3140 if (event_id == PQI_EVENT_OFA_QUIESCE) {
3141 dev_info(&ctrl_info->pci_dev->dev,
694c5d5b
KB		 3142 "Received Online Firmware Activation quiesce event for controller %u\n",
3143 ctrl_info->ctrl_id);
4fd22c13
MR		 3144 pqi_ofa_ctrl_quiesce(ctrl_info);
3145 pqi_acknowledge_event(ctrl_info, event);
3146 if (ctrl_info->soft_reset_handshake_supported) {
3147 status = pqi_poll_for_soft_reset_status(ctrl_info);
3148 pqi_process_soft_reset(ctrl_info, status);
3149 } else {
3150 pqi_process_soft_reset(ctrl_info,
3151 RESET_INITIATE_FIRMWARE);
3152 }
3153
3154 } else if (event_id == PQI_EVENT_OFA_MEMORY_ALLOCATION) {
3155 pqi_acknowledge_event(ctrl_info, event);
3156 pqi_ofa_setup_host_buffer(ctrl_info,
3157 le32_to_cpu(event->ofa_bytes_requested));
3158 pqi_ofa_host_memory_update(ctrl_info);
3159 } else if (event_id == PQI_EVENT_OFA_CANCELLED) {
3160 pqi_ofa_free_host_buffer(ctrl_info);
3161 pqi_acknowledge_event(ctrl_info, event);
3162 dev_info(&ctrl_info->pci_dev->dev,
694c5d5b
KB		 3163 "Online Firmware Activation(%u) cancel reason : %u\n",
3164 ctrl_info->ctrl_id, event->ofa_cancel_reason);
4fd22c13
MR		 3165 }
3166
3167 mutex_unlock(&ctrl_info->ofa_mutex);
3168}
3169
6c223761
KB		 3170static void pqi_event_worker(struct work_struct *work)
3171{
3172 unsigned int i;
3173 struct pqi_ctrl_info *ctrl_info;
6a50d6ad 3174 struct pqi_event *event;
6c223761
KB		 3175
3176 ctrl_info = container_of(work, struct pqi_ctrl_info, event_work);
3177
7561a7e4
KB		 3178 pqi_ctrl_busy(ctrl_info);
3179 pqi_wait_if_ctrl_blocked(ctrl_info, NO_TIMEOUT);
5f310425
KB		 3180 if (pqi_ctrl_offline(ctrl_info))
3181 goto out;
3182
3183 pqi_schedule_rescan_worker_delayed(ctrl_info);
7561a7e4 3184
6a50d6ad 3185 event = ctrl_info->events;
6c223761 3186 for (i = 0; i < PQI_NUM_SUPPORTED_EVENTS; i++) {
6a50d6ad
KB		 3187 if (event->pending) {
3188 event->pending = false;
4fd22c13
MR		 3189 if (event->event_type == PQI_EVENT_TYPE_OFA) {
3190 pqi_ctrl_unbusy(ctrl_info);
3191 pqi_ofa_process_event(ctrl_info, event);
3192 return;
3193 }
6a50d6ad 3194 pqi_acknowledge_event(ctrl_info, event);
6c223761 3195 }
6a50d6ad 3196 event++;
6c223761
KB		 3197 }
3198
5f310425 3199out:
7561a7e4 3200 pqi_ctrl_unbusy(ctrl_info);
6c223761
KB		 3201}
3202
4fd22c13 3203#define PQI_HEARTBEAT_TIMER_INTERVAL (10 * PQI_HZ)
6c223761 3204
74a0f573 3205static void pqi_heartbeat_timer_handler(struct timer_list *t)
6c223761
KB		 3206{
3207 int num_interrupts;
98f87667 3208 u32 heartbeat_count;
74a0f573
KC		 3209 struct pqi_ctrl_info *ctrl_info = from_timer(ctrl_info, t,
3210 heartbeat_timer);
6c223761 3211
98f87667
KB		 3212 pqi_check_ctrl_health(ctrl_info);
3213 if (pqi_ctrl_offline(ctrl_info))
061ef06a
KB		 3214 return;
3215
6c223761 3216 num_interrupts = atomic_read(&ctrl_info->num_interrupts);
98f87667 3217 heartbeat_count = pqi_read_heartbeat_counter(ctrl_info);
6c223761
KB		 3218
3219 if (num_interrupts == ctrl_info->previous_num_interrupts) {
98f87667
KB		 3220 if (heartbeat_count == ctrl_info->previous_heartbeat_count) {
3221 dev_err(&ctrl_info->pci_dev->dev,
3222 "no heartbeat detected - last heartbeat count: %u\n",
3223 heartbeat_count);
6c223761
KB		 3224 pqi_take_ctrl_offline(ctrl_info);
3225 return;
3226 }
6c223761 3227 } else {
98f87667 3228 ctrl_info->previous_num_interrupts = num_interrupts;
6c223761
KB		 3229 }
3230
98f87667 3231 ctrl_info->previous_heartbeat_count = heartbeat_count;
6c223761
KB		 3232 mod_timer(&ctrl_info->heartbeat_timer,
3233 jiffies + PQI_HEARTBEAT_TIMER_INTERVAL);
3234}
3235
3236static void pqi_start_heartbeat_timer(struct pqi_ctrl_info *ctrl_info)
3237{
98f87667
KB		 3238 if (!ctrl_info->heartbeat_counter)
3239 return;
3240
6c223761
KB		 3241 ctrl_info->previous_num_interrupts =
3242 atomic_read(&ctrl_info->num_interrupts);
98f87667
KB		 3243 ctrl_info->previous_heartbeat_count =
3244 pqi_read_heartbeat_counter(ctrl_info);
6c223761 3245
6c223761
KB		 3246 ctrl_info->heartbeat_timer.expires =
3247 jiffies + PQI_HEARTBEAT_TIMER_INTERVAL;
061ef06a 3248 add_timer(&ctrl_info->heartbeat_timer);
6c223761
KB		 3249}
3250
3251static inline void pqi_stop_heartbeat_timer(struct pqi_ctrl_info *ctrl_info)
3252{
98f87667 3253 del_timer_sync(&ctrl_info->heartbeat_timer);
6c223761
KB		 3254}
3255
6a50d6ad 3256static inline int pqi_event_type_to_event_index(unsigned int event_type)
6c223761
KB		 3257{
3258 int index;
3259
6a50d6ad
KB		 3260 for (index = 0; index < ARRAY_SIZE(pqi_supported_event_types); index++)
3261 if (event_type == pqi_supported_event_types[index])
3262 return index;
6c223761 3263
6a50d6ad
KB		 3264 return -1;
3265}
3266
3267static inline bool pqi_is_supported_event(unsigned int event_type)
3268{
3269 return pqi_event_type_to_event_index(event_type) != -1;
6c223761
KB		 3270}
3271
4fd22c13
MR		 3272static void pqi_ofa_capture_event_payload(struct pqi_event *event,
3273 struct pqi_event_response *response)
3274{
3275 u16 event_id;
3276
3277 event_id = get_unaligned_le16(&event->event_id);
3278
3279 if (event->event_type == PQI_EVENT_TYPE_OFA) {
3280 if (event_id == PQI_EVENT_OFA_MEMORY_ALLOCATION) {
3281 event->ofa_bytes_requested =
3282 response->data.ofa_memory_allocation.bytes_requested;
3283 } else if (event_id == PQI_EVENT_OFA_CANCELLED) {
3284 event->ofa_cancel_reason =
3285 response->data.ofa_cancelled.reason;
3286 }
3287 }
3288}
3289
9e68cccc 3290static int pqi_process_event_intr(struct pqi_ctrl_info *ctrl_info)
6c223761 3291{
9e68cccc 3292 int num_events;
6c223761
KB		 3293 pqi_index_t oq_pi;
3294 pqi_index_t oq_ci;
3295 struct pqi_event_queue *event_queue;
3296 struct pqi_event_response *response;
6a50d6ad 3297 struct pqi_event *event;
6c223761
KB		 3298 int event_index;
3299
3300 event_queue = &ctrl_info->event_queue;
3301 num_events = 0;
6c223761
KB		 3302 oq_ci = event_queue->oq_ci_copy;
3303
3304 while (1) {
dac12fbc 3305 oq_pi = readl(event_queue->oq_pi);
9e68cccc
KB		 3306 if (oq_pi >= PQI_NUM_EVENT_QUEUE_ELEMENTS) {
3307 pqi_invalid_response(ctrl_info);
3308 dev_err(&ctrl_info->pci_dev->dev,
3309 "event interrupt: producer index (%u) out of range (0-%u): consumer index: %u\n",
3310 oq_pi, PQI_NUM_EVENT_QUEUE_ELEMENTS - 1, oq_ci);
3311 return -1;
3312 }
3313
6c223761
KB		 3314 if (oq_pi == oq_ci)
3315 break;
3316
3317 num_events++;
9e68cccc 3318 response = event_queue->oq_element_array + (oq_ci * PQI_EVENT_OQ_ELEMENT_LENGTH);
6c223761
KB		 3319
3320 event_index =
3321 pqi_event_type_to_event_index(response->event_type);
3322
9e68cccc
KB		 3323 if (event_index >= 0 && response->request_acknowledge) {
3324 event = &ctrl_info->events[event_index];
3325 event->pending = true;
3326 event->event_type = response->event_type;
3327 event->event_id = response->event_id;
3328 event->additional_event_id = response->additional_event_id;
3329 if (event->event_type == PQI_EVENT_TYPE_OFA)
4fd22c13 3330 pqi_ofa_capture_event_payload(event, response);
6c223761
KB		 3331 }
3332
3333 oq_ci = (oq_ci + 1) % PQI_NUM_EVENT_QUEUE_ELEMENTS;
3334 }
3335
3336 if (num_events) {
3337 event_queue->oq_ci_copy = oq_ci;
3338 writel(oq_ci, event_queue->oq_ci);
98f87667 3339 schedule_work(&ctrl_info->event_work);
6c223761
KB		 3340 }
3341
3342 return num_events;
3343}
3344
061ef06a
KB		 3345#define PQI_LEGACY_INTX_MASK 0x1
3346
3347static inline void pqi_configure_legacy_intx(struct pqi_ctrl_info *ctrl_info,
694c5d5b 3348 bool enable_intx)
061ef06a
KB		 3349{
3350 u32 intx_mask;
3351 struct pqi_device_registers __iomem *pqi_registers;
3352 volatile void __iomem *register_addr;
3353
3354 pqi_registers = ctrl_info->pqi_registers;
3355
3356 if (enable_intx)
3357 register_addr = &pqi_registers->legacy_intx_mask_clear;
3358 else
3359 register_addr = &pqi_registers->legacy_intx_mask_set;
3360
3361 intx_mask = readl(register_addr);
3362 intx_mask |= PQI_LEGACY_INTX_MASK;
3363 writel(intx_mask, register_addr);
3364}
3365
3366static void pqi_change_irq_mode(struct pqi_ctrl_info *ctrl_info,
3367 enum pqi_irq_mode new_mode)
3368{
3369 switch (ctrl_info->irq_mode) {
3370 case IRQ_MODE_MSIX:
3371 switch (new_mode) {
3372 case IRQ_MODE_MSIX:
3373 break;
3374 case IRQ_MODE_INTX:
3375 pqi_configure_legacy_intx(ctrl_info, true);
061ef06a
KB		 3376 sis_enable_intx(ctrl_info);
3377 break;
3378 case IRQ_MODE_NONE:
061ef06a
KB		 3379 break;
3380 }
3381 break;
3382 case IRQ_MODE_INTX:
3383 switch (new_mode) {
3384 case IRQ_MODE_MSIX:
3385 pqi_configure_legacy_intx(ctrl_info, false);
061ef06a
KB		 3386 sis_enable_msix(ctrl_info);
3387 break;
3388 case IRQ_MODE_INTX:
3389 break;
3390 case IRQ_MODE_NONE:
3391 pqi_configure_legacy_intx(ctrl_info, false);
061ef06a
KB		 3392 break;
3393 }
3394 break;
3395 case IRQ_MODE_NONE:
3396 switch (new_mode) {
3397 case IRQ_MODE_MSIX:
3398 sis_enable_msix(ctrl_info);
3399 break;
3400 case IRQ_MODE_INTX:
3401 pqi_configure_legacy_intx(ctrl_info, true);
3402 sis_enable_intx(ctrl_info);
3403 break;
3404 case IRQ_MODE_NONE:
3405 break;
3406 }
3407 break;
3408 }
3409
3410 ctrl_info->irq_mode = new_mode;
3411}
3412
3413#define PQI_LEGACY_INTX_PENDING 0x1
3414
3415static inline bool pqi_is_valid_irq(struct pqi_ctrl_info *ctrl_info)
3416{
3417 bool valid_irq;
3418 u32 intx_status;
3419
3420 switch (ctrl_info->irq_mode) {
3421 case IRQ_MODE_MSIX:
3422 valid_irq = true;
3423 break;
3424 case IRQ_MODE_INTX:
3425 intx_status =
3426 readl(&ctrl_info->pqi_registers->legacy_intx_status);
3427 if (intx_status & PQI_LEGACY_INTX_PENDING)
3428 valid_irq = true;
3429 else
3430 valid_irq = false;
3431 break;
3432 case IRQ_MODE_NONE:
3433 default:
3434 valid_irq = false;
3435 break;
3436 }
3437
3438 return valid_irq;
3439}
3440
6c223761
KB		 3441static irqreturn_t pqi_irq_handler(int irq, void *data)
3442{
3443 struct pqi_ctrl_info *ctrl_info;
3444 struct pqi_queue_group *queue_group;
9e68cccc
KB		 3445 int num_io_responses_handled;
3446 int num_events_handled;
6c223761
KB		 3447
3448 queue_group = data;
3449 ctrl_info = queue_group->ctrl_info;
3450
061ef06a 3451 if (!pqi_is_valid_irq(ctrl_info))
6c223761
KB		 3452 return IRQ_NONE;
3453
9e68cccc
KB		 3454 num_io_responses_handled = pqi_process_io_intr(ctrl_info, queue_group);
3455 if (num_io_responses_handled < 0)
3456 goto out;
6c223761 3457
9e68cccc
KB		 3458 if (irq == ctrl_info->event_irq) {
3459 num_events_handled = pqi_process_event_intr(ctrl_info);
3460 if (num_events_handled < 0)
3461 goto out;
3462 } else {
3463 num_events_handled = 0;
3464 }
6c223761 3465
9e68cccc 3466 if (num_io_responses_handled + num_events_handled > 0)
6c223761
KB		 3467 atomic_inc(&ctrl_info->num_interrupts);
3468
3469 pqi_start_io(ctrl_info, queue_group, RAID_PATH, NULL);
3470 pqi_start_io(ctrl_info, queue_group, AIO_PATH, NULL);
3471
9e68cccc 3472out:
6c223761
KB		 3473 return IRQ_HANDLED;
3474}
3475
3476static int pqi_request_irqs(struct pqi_ctrl_info *ctrl_info)
3477{
d91d7820 3478 struct pci_dev *pci_dev = ctrl_info->pci_dev;
6c223761
KB		 3479 int i;
3480 int rc;
3481
d91d7820 3482 ctrl_info->event_irq = pci_irq_vector(pci_dev, 0);
6c223761
KB		 3483
3484 for (i = 0; i < ctrl_info->num_msix_vectors_enabled; i++) {
d91d7820 3485 rc = request_irq(pci_irq_vector(pci_dev, i), pqi_irq_handler, 0,
52198226 3486 DRIVER_NAME_SHORT, &ctrl_info->queue_groups[i]);
6c223761 3487 if (rc) {
d91d7820 3488 dev_err(&pci_dev->dev,
6c223761 3489 "irq %u init failed with error %d\n",
d91d7820 3490 pci_irq_vector(pci_dev, i), rc);
6c223761
KB		 3491 return rc;
3492 }
3493 ctrl_info->num_msix_vectors_initialized++;
3494 }
3495
3496 return 0;
3497}
3498
98bf061b
KB		 3499static void pqi_free_irqs(struct pqi_ctrl_info *ctrl_info)
3500{
3501 int i;
3502
3503 for (i = 0; i < ctrl_info->num_msix_vectors_initialized; i++)
3504 free_irq(pci_irq_vector(ctrl_info->pci_dev, i),
3505 &ctrl_info->queue_groups[i]);
3506
3507 ctrl_info->num_msix_vectors_initialized = 0;
3508}
3509
6c223761
KB		 3510static int pqi_enable_msix_interrupts(struct pqi_ctrl_info *ctrl_info)
3511{
98bf061b 3512 int num_vectors_enabled;
6c223761 3513
98bf061b 3514 num_vectors_enabled = pci_alloc_irq_vectors(ctrl_info->pci_dev,
52198226
CH		 3515 PQI_MIN_MSIX_VECTORS, ctrl_info->num_queue_groups,
3516 PCI_IRQ_MSIX | PCI_IRQ_AFFINITY);
98bf061b 3517 if (num_vectors_enabled < 0) {
6c223761 3518 dev_err(&ctrl_info->pci_dev->dev,
98bf061b
KB		 3519 "MSI-X init failed with error %d\n",
3520 num_vectors_enabled);
3521 return num_vectors_enabled;
6c223761
KB		 3522 }
3523
98bf061b 3524 ctrl_info->num_msix_vectors_enabled = num_vectors_enabled;
061ef06a 3525 ctrl_info->irq_mode = IRQ_MODE_MSIX;
6c223761
KB		 3526 return 0;
3527}
3528
98bf061b
KB		 3529static void pqi_disable_msix_interrupts(struct pqi_ctrl_info *ctrl_info)
3530{
3531 if (ctrl_info->num_msix_vectors_enabled) {
3532 pci_free_irq_vectors(ctrl_info->pci_dev);
3533 ctrl_info->num_msix_vectors_enabled = 0;
3534 }
3535}
3536
6c223761
KB		 3537static int pqi_alloc_operational_queues(struct pqi_ctrl_info *ctrl_info)
3538{
3539 unsigned int i;
3540 size_t alloc_length;
3541 size_t element_array_length_per_iq;
3542 size_t element_array_length_per_oq;
3543 void *element_array;
dac12fbc 3544 void __iomem *next_queue_index;
6c223761
KB		 3545 void *aligned_pointer;
3546 unsigned int num_inbound_queues;
3547 unsigned int num_outbound_queues;
3548 unsigned int num_queue_indexes;
3549 struct pqi_queue_group *queue_group;
3550
3551 element_array_length_per_iq =
3552 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH *
3553 ctrl_info->num_elements_per_iq;
3554 element_array_length_per_oq =
3555 PQI_OPERATIONAL_OQ_ELEMENT_LENGTH *
3556 ctrl_info->num_elements_per_oq;
3557 num_inbound_queues = ctrl_info->num_queue_groups * 2;
3558 num_outbound_queues = ctrl_info->num_queue_groups;
3559 num_queue_indexes = (ctrl_info->num_queue_groups * 3) + 1;
3560
3561 aligned_pointer = NULL;
3562
3563 for (i = 0; i < num_inbound_queues; i++) {
3564 aligned_pointer = PTR_ALIGN(aligned_pointer,
3565 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
3566 aligned_pointer += element_array_length_per_iq;
3567 }
3568
3569 for (i = 0; i < num_outbound_queues; i++) {
3570 aligned_pointer = PTR_ALIGN(aligned_pointer,
3571 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
3572 aligned_pointer += element_array_length_per_oq;
3573 }
3574
3575 aligned_pointer = PTR_ALIGN(aligned_pointer,
3576 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
3577 aligned_pointer += PQI_NUM_EVENT_QUEUE_ELEMENTS *
3578 PQI_EVENT_OQ_ELEMENT_LENGTH;
3579
3580 for (i = 0; i < num_queue_indexes; i++) {
3581 aligned_pointer = PTR_ALIGN(aligned_pointer,
3582 PQI_OPERATIONAL_INDEX_ALIGNMENT);
3583 aligned_pointer += sizeof(pqi_index_t);
3584 }
3585
3586 alloc_length = (size_t)aligned_pointer +
3587 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT;
3588
e1d213bd
KB		 3589 alloc_length += PQI_EXTRA_SGL_MEMORY;
3590
6c223761 3591 ctrl_info->queue_memory_base =
750afb08
LC		 3592 dma_alloc_coherent(&ctrl_info->pci_dev->dev, alloc_length,
3593 &ctrl_info->queue_memory_base_dma_handle,
3594 GFP_KERNEL);
6c223761 3595
d87d5474 3596 if (!ctrl_info->queue_memory_base)
6c223761 3597 return -ENOMEM;
6c223761
KB		 3598
3599 ctrl_info->queue_memory_length = alloc_length;
3600
3601 element_array = PTR_ALIGN(ctrl_info->queue_memory_base,
3602 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
3603
3604 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
3605 queue_group = &ctrl_info->queue_groups[i];
3606 queue_group->iq_element_array[RAID_PATH] = element_array;
3607 queue_group->iq_element_array_bus_addr[RAID_PATH] =
3608 ctrl_info->queue_memory_base_dma_handle +
3609 (element_array - ctrl_info->queue_memory_base);
3610 element_array += element_array_length_per_iq;
3611 element_array = PTR_ALIGN(element_array,
3612 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
3613 queue_group->iq_element_array[AIO_PATH] = element_array;
3614 queue_group->iq_element_array_bus_addr[AIO_PATH] =
3615 ctrl_info->queue_memory_base_dma_handle +
3616 (element_array - ctrl_info->queue_memory_base);
3617 element_array += element_array_length_per_iq;
3618 element_array = PTR_ALIGN(element_array,
3619 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
3620 }
3621
3622 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
3623 queue_group = &ctrl_info->queue_groups[i];
3624 queue_group->oq_element_array = element_array;
3625 queue_group->oq_element_array_bus_addr =
3626 ctrl_info->queue_memory_base_dma_handle +
3627 (element_array - ctrl_info->queue_memory_base);
3628 element_array += element_array_length_per_oq;
3629 element_array = PTR_ALIGN(element_array,
3630 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
3631 }
3632
3633 ctrl_info->event_queue.oq_element_array = element_array;
3634 ctrl_info->event_queue.oq_element_array_bus_addr =
3635 ctrl_info->queue_memory_base_dma_handle +
3636 (element_array - ctrl_info->queue_memory_base);
3637 element_array += PQI_NUM_EVENT_QUEUE_ELEMENTS *
3638 PQI_EVENT_OQ_ELEMENT_LENGTH;
3639
dac12fbc 3640 next_queue_index = (void __iomem *)PTR_ALIGN(element_array,
6c223761
KB		 3641 PQI_OPERATIONAL_INDEX_ALIGNMENT);
3642
3643 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
3644 queue_group = &ctrl_info->queue_groups[i];
3645 queue_group->iq_ci[RAID_PATH] = next_queue_index;
3646 queue_group->iq_ci_bus_addr[RAID_PATH] =
3647 ctrl_info->queue_memory_base_dma_handle +
dac12fbc
KB		 3648 (next_queue_index -
3649 (void __iomem *)ctrl_info->queue_memory_base);
6c223761
KB		 3650 next_queue_index += sizeof(pqi_index_t);
3651 next_queue_index = PTR_ALIGN(next_queue_index,
3652 PQI_OPERATIONAL_INDEX_ALIGNMENT);
3653 queue_group->iq_ci[AIO_PATH] = next_queue_index;
3654 queue_group->iq_ci_bus_addr[AIO_PATH] =
3655 ctrl_info->queue_memory_base_dma_handle +
dac12fbc
KB		 3656 (next_queue_index -
3657 (void __iomem *)ctrl_info->queue_memory_base);
6c223761
KB		 3658 next_queue_index += sizeof(pqi_index_t);
3659 next_queue_index = PTR_ALIGN(next_queue_index,
3660 PQI_OPERATIONAL_INDEX_ALIGNMENT);
3661 queue_group->oq_pi = next_queue_index;
3662 queue_group->oq_pi_bus_addr =
3663 ctrl_info->queue_memory_base_dma_handle +
dac12fbc
KB		 3664 (next_queue_index -
3665 (void __iomem *)ctrl_info->queue_memory_base);
6c223761
KB		 3666 next_queue_index += sizeof(pqi_index_t);
3667 next_queue_index = PTR_ALIGN(next_queue_index,
3668 PQI_OPERATIONAL_INDEX_ALIGNMENT);
3669 }
3670
3671 ctrl_info->event_queue.oq_pi = next_queue_index;
3672 ctrl_info->event_queue.oq_pi_bus_addr =
3673 ctrl_info->queue_memory_base_dma_handle +
dac12fbc
KB		 3674 (next_queue_index -
3675 (void __iomem *)ctrl_info->queue_memory_base);
6c223761
KB		 3676
3677 return 0;
3678}
3679
3680static void pqi_init_operational_queues(struct pqi_ctrl_info *ctrl_info)
3681{
3682 unsigned int i;
3683 u16 next_iq_id = PQI_MIN_OPERATIONAL_QUEUE_ID;
3684 u16 next_oq_id = PQI_MIN_OPERATIONAL_QUEUE_ID;
3685
3686 /*
3687 * Initialize the backpointers to the controller structure in
3688 * each operational queue group structure.
3689 */
3690 for (i = 0; i < ctrl_info->num_queue_groups; i++)
3691 ctrl_info->queue_groups[i].ctrl_info = ctrl_info;
3692
3693 /*
3694 * Assign IDs to all operational queues. Note that the IDs
3695 * assigned to operational IQs are independent of the IDs
3696 * assigned to operational OQs.
3697 */
3698 ctrl_info->event_queue.oq_id = next_oq_id++;
3699 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
3700 ctrl_info->queue_groups[i].iq_id[RAID_PATH] = next_iq_id++;
3701 ctrl_info->queue_groups[i].iq_id[AIO_PATH] = next_iq_id++;
3702 ctrl_info->queue_groups[i].oq_id = next_oq_id++;
3703 }
3704
3705 /*
3706 * Assign MSI-X table entry indexes to all queues. Note that the
3707 * interrupt for the event queue is shared with the first queue group.
3708 */
3709 ctrl_info->event_queue.int_msg_num = 0;
3710 for (i = 0; i < ctrl_info->num_queue_groups; i++)
3711 ctrl_info->queue_groups[i].int_msg_num = i;
3712
3713 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
3714 spin_lock_init(&ctrl_info->queue_groups[i].submit_lock[0]);
3715 spin_lock_init(&ctrl_info->queue_groups[i].submit_lock[1]);
3716 INIT_LIST_HEAD(&ctrl_info->queue_groups[i].request_list[0]);
3717 INIT_LIST_HEAD(&ctrl_info->queue_groups[i].request_list[1]);
3718 }
3719}
3720
3721static int pqi_alloc_admin_queues(struct pqi_ctrl_info *ctrl_info)
3722{
3723 size_t alloc_length;
3724 struct pqi_admin_queues_aligned *admin_queues_aligned;
3725 struct pqi_admin_queues *admin_queues;
3726
3727 alloc_length = sizeof(struct pqi_admin_queues_aligned) +
3728 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT;
3729
3730 ctrl_info->admin_queue_memory_base =
750afb08
LC		 3731 dma_alloc_coherent(&ctrl_info->pci_dev->dev, alloc_length,
3732 &ctrl_info->admin_queue_memory_base_dma_handle,
3733 GFP_KERNEL);
6c223761
KB		 3734
3735 if (!ctrl_info->admin_queue_memory_base)
3736 return -ENOMEM;
3737
3738 ctrl_info->admin_queue_memory_length = alloc_length;
3739
3740 admin_queues = &ctrl_info->admin_queues;
3741 admin_queues_aligned = PTR_ALIGN(ctrl_info->admin_queue_memory_base,
3742 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
3743 admin_queues->iq_element_array =
3744 &admin_queues_aligned->iq_element_array;
3745 admin_queues->oq_element_array =
3746 &admin_queues_aligned->oq_element_array;
3747 admin_queues->iq_ci = &admin_queues_aligned->iq_ci;
dac12fbc
KB		 3748 admin_queues->oq_pi =
3749 (pqi_index_t __iomem *)&admin_queues_aligned->oq_pi;
6c223761
KB		 3750
3751 admin_queues->iq_element_array_bus_addr =
3752 ctrl_info->admin_queue_memory_base_dma_handle +
3753 (admin_queues->iq_element_array -
3754 ctrl_info->admin_queue_memory_base);
3755 admin_queues->oq_element_array_bus_addr =
3756 ctrl_info->admin_queue_memory_base_dma_handle +
3757 (admin_queues->oq_element_array -
3758 ctrl_info->admin_queue_memory_base);
3759 admin_queues->iq_ci_bus_addr =
3760 ctrl_info->admin_queue_memory_base_dma_handle +
3761 ((void *)admin_queues->iq_ci -
3762 ctrl_info->admin_queue_memory_base);
3763 admin_queues->oq_pi_bus_addr =
3764 ctrl_info->admin_queue_memory_base_dma_handle +
dac12fbc
KB		 3765 ((void __iomem *)admin_queues->oq_pi -
3766 (void __iomem *)ctrl_info->admin_queue_memory_base);
6c223761
KB		 3767
3768 return 0;
3769}
3770
4fd22c13 3771#define PQI_ADMIN_QUEUE_CREATE_TIMEOUT_JIFFIES PQI_HZ
6c223761
KB		 3772#define PQI_ADMIN_QUEUE_CREATE_POLL_INTERVAL_MSECS 1
3773
3774static int pqi_create_admin_queues(struct pqi_ctrl_info *ctrl_info)
3775{
3776 struct pqi_device_registers __iomem *pqi_registers;
3777 struct pqi_admin_queues *admin_queues;
3778 unsigned long timeout;
3779 u8 status;
3780 u32 reg;
3781
3782 pqi_registers = ctrl_info->pqi_registers;
3783 admin_queues = &ctrl_info->admin_queues;
3784
3785 writeq((u64)admin_queues->iq_element_array_bus_addr,
3786 &pqi_registers->admin_iq_element_array_addr);
3787 writeq((u64)admin_queues->oq_element_array_bus_addr,
3788 &pqi_registers->admin_oq_element_array_addr);
3789 writeq((u64)admin_queues->iq_ci_bus_addr,
3790 &pqi_registers->admin_iq_ci_addr);
3791 writeq((u64)admin_queues->oq_pi_bus_addr,
3792 &pqi_registers->admin_oq_pi_addr);
3793
3794 reg = PQI_ADMIN_IQ_NUM_ELEMENTS |
e655d469 3795 (PQI_ADMIN_OQ_NUM_ELEMENTS << 8) |
6c223761
KB		 3796 (admin_queues->int_msg_num << 16);
3797 writel(reg, &pqi_registers->admin_iq_num_elements);
3798 writel(PQI_CREATE_ADMIN_QUEUE_PAIR,
3799 &pqi_registers->function_and_status_code);
3800
3801 timeout = PQI_ADMIN_QUEUE_CREATE_TIMEOUT_JIFFIES + jiffies;
3802 while (1) {
3803 status = readb(&pqi_registers->function_and_status_code);
3804 if (status == PQI_STATUS_IDLE)
3805 break;
3806 if (time_after(jiffies, timeout))
3807 return -ETIMEDOUT;
3808 msleep(PQI_ADMIN_QUEUE_CREATE_POLL_INTERVAL_MSECS);
3809 }
3810
3811 /*
3812 * The offset registers are not initialized to the correct
3813 * offsets until *after* the create admin queue pair command
3814 * completes successfully.
3815 */
3816 admin_queues->iq_pi = ctrl_info->iomem_base +
3817 PQI_DEVICE_REGISTERS_OFFSET +
3818 readq(&pqi_registers->admin_iq_pi_offset);
3819 admin_queues->oq_ci = ctrl_info->iomem_base +
3820 PQI_DEVICE_REGISTERS_OFFSET +
3821 readq(&pqi_registers->admin_oq_ci_offset);
3822
3823 return 0;
3824}
3825
3826static void pqi_submit_admin_request(struct pqi_ctrl_info *ctrl_info,
3827 struct pqi_general_admin_request *request)
3828{
3829 struct pqi_admin_queues *admin_queues;
3830 void *next_element;
3831 pqi_index_t iq_pi;
3832
3833 admin_queues = &ctrl_info->admin_queues;
3834 iq_pi = admin_queues->iq_pi_copy;
3835
3836 next_element = admin_queues->iq_element_array +
3837 (iq_pi * PQI_ADMIN_IQ_ELEMENT_LENGTH);
3838
3839 memcpy(next_element, request, sizeof(*request));
3840
3841 iq_pi = (iq_pi + 1) % PQI_ADMIN_IQ_NUM_ELEMENTS;
3842 admin_queues->iq_pi_copy = iq_pi;
3843
3844 /*
3845 * This write notifies the controller that an IU is available to be
3846 * processed.
3847 */
3848 writel(iq_pi, admin_queues->iq_pi);
3849}
3850
13bede67
KB		 3851#define PQI_ADMIN_REQUEST_TIMEOUT_SECS 60
3852
6c223761
KB		 3853static int pqi_poll_for_admin_response(struct pqi_ctrl_info *ctrl_info,
3854 struct pqi_general_admin_response *response)
3855{
3856 struct pqi_admin_queues *admin_queues;
3857 pqi_index_t oq_pi;
3858 pqi_index_t oq_ci;
3859 unsigned long timeout;
3860
3861 admin_queues = &ctrl_info->admin_queues;
3862 oq_ci = admin_queues->oq_ci_copy;
3863
4fd22c13 3864 timeout = (PQI_ADMIN_REQUEST_TIMEOUT_SECS * PQI_HZ) + jiffies;
6c223761
KB		 3865
3866 while (1) {
dac12fbc 3867 oq_pi = readl(admin_queues->oq_pi);
6c223761
KB		 3868 if (oq_pi != oq_ci)
3869 break;
3870 if (time_after(jiffies, timeout)) {
3871 dev_err(&ctrl_info->pci_dev->dev,
3872 "timed out waiting for admin response\n");
3873 return -ETIMEDOUT;
3874 }
13bede67
KB		 3875 if (!sis_is_firmware_running(ctrl_info))
3876 return -ENXIO;
6c223761
KB		 3877 usleep_range(1000, 2000);
3878 }
3879
3880 memcpy(response, admin_queues->oq_element_array +
3881 (oq_ci * PQI_ADMIN_OQ_ELEMENT_LENGTH), sizeof(*response));
3882
3883 oq_ci = (oq_ci + 1) % PQI_ADMIN_OQ_NUM_ELEMENTS;
3884 admin_queues->oq_ci_copy = oq_ci;
3885 writel(oq_ci, admin_queues->oq_ci);
3886
3887 return 0;
3888}
3889
3890static void pqi_start_io(struct pqi_ctrl_info *ctrl_info,
3891 struct pqi_queue_group *queue_group, enum pqi_io_path path,
3892 struct pqi_io_request *io_request)
3893{
3894 struct pqi_io_request *next;
3895 void *next_element;
3896 pqi_index_t iq_pi;
3897 pqi_index_t iq_ci;
3898 size_t iu_length;
3899 unsigned long flags;
3900 unsigned int num_elements_needed;
3901 unsigned int num_elements_to_end_of_queue;
3902 size_t copy_count;
3903 struct pqi_iu_header *request;
3904
3905 spin_lock_irqsave(&queue_group->submit_lock[path], flags);
3906
376fb880
KB		 3907 if (io_request) {
3908 io_request->queue_group = queue_group;
6c223761
KB		 3909 list_add_tail(&io_request->request_list_entry,
3910 &queue_group->request_list[path]);
376fb880 3911 }
6c223761
KB		 3912
3913 iq_pi = queue_group->iq_pi_copy[path];
3914
3915 list_for_each_entry_safe(io_request, next,
3916 &queue_group->request_list[path], request_list_entry) {
3917
3918 request = io_request->iu;
3919
3920 iu_length = get_unaligned_le16(&request->iu_length) +
3921 PQI_REQUEST_HEADER_LENGTH;
3922 num_elements_needed =
3923 DIV_ROUND_UP(iu_length,
3924 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
3925
dac12fbc 3926 iq_ci = readl(queue_group->iq_ci[path]);
6c223761
KB		 3927
3928 if (num_elements_needed > pqi_num_elements_free(iq_pi, iq_ci,
3929 ctrl_info->num_elements_per_iq))
3930 break;
3931
3932 put_unaligned_le16(queue_group->oq_id,
3933 &request->response_queue_id);
3934
3935 next_element = queue_group->iq_element_array[path] +
3936 (iq_pi * PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
3937
3938 num_elements_to_end_of_queue =
3939 ctrl_info->num_elements_per_iq - iq_pi;
3940
3941 if (num_elements_needed <= num_elements_to_end_of_queue) {
3942 memcpy(next_element, request, iu_length);
3943 } else {
3944 copy_count = num_elements_to_end_of_queue *
3945 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH;
3946 memcpy(next_element, request, copy_count);
3947 memcpy(queue_group->iq_element_array[path],
3948 (u8 *)request + copy_count,
3949 iu_length - copy_count);
3950 }
3951
3952 iq_pi = (iq_pi + num_elements_needed) %
3953 ctrl_info->num_elements_per_iq;
3954
3955 list_del(&io_request->request_list_entry);
3956 }
3957
3958 if (iq_pi != queue_group->iq_pi_copy[path]) {
3959 queue_group->iq_pi_copy[path] = iq_pi;
3960 /*
3961 * This write notifies the controller that one or more IUs are
3962 * available to be processed.
3963 */
3964 writel(iq_pi, queue_group->iq_pi[path]);
3965 }
3966
3967 spin_unlock_irqrestore(&queue_group->submit_lock[path], flags);
3968}
3969
1f37e992
KB		 3970#define PQI_WAIT_FOR_COMPLETION_IO_TIMEOUT_SECS 10
3971
3972static int pqi_wait_for_completion_io(struct pqi_ctrl_info *ctrl_info,
3973 struct completion *wait)
3974{
3975 int rc;
1f37e992
KB		 3976
3977 while (1) {
3978 if (wait_for_completion_io_timeout(wait,
4fd22c13 3979 PQI_WAIT_FOR_COMPLETION_IO_TIMEOUT_SECS * PQI_HZ)) {
1f37e992
KB		 3980 rc = 0;
3981 break;
3982 }
3983
3984 pqi_check_ctrl_health(ctrl_info);
3985 if (pqi_ctrl_offline(ctrl_info)) {
3986 rc = -ENXIO;
3987 break;
3988 }
1f37e992
KB		 3989 }
3990
3991 return rc;
3992}
3993
6c223761
KB		 3994static void pqi_raid_synchronous_complete(struct pqi_io_request *io_request,
3995 void *context)
3996{
3997 struct completion *waiting = context;
3998
3999 complete(waiting);
4000}
4001
694c5d5b
KB		 4002static int pqi_process_raid_io_error_synchronous(
4003 struct pqi_raid_error_info *error_info)
26b390ab
KB		 4004{
4005 int rc = -EIO;
4006
4007 switch (error_info->data_out_result) {
4008 case PQI_DATA_IN_OUT_GOOD:
4009 if (error_info->status == SAM_STAT_GOOD)
4010 rc = 0;
4011 break;
4012 case PQI_DATA_IN_OUT_UNDERFLOW:
4013 if (error_info->status == SAM_STAT_GOOD ||
4014 error_info->status == SAM_STAT_CHECK_CONDITION)
4015 rc = 0;
4016 break;
4017 case PQI_DATA_IN_OUT_ABORTED:
4018 rc = PQI_CMD_STATUS_ABORTED;
4019 break;
4020 }
4021
4022 return rc;
4023}
4024
6c223761
KB		 4025static int pqi_submit_raid_request_synchronous(struct pqi_ctrl_info *ctrl_info,
4026 struct pqi_iu_header *request, unsigned int flags,
4027 struct pqi_raid_error_info *error_info, unsigned long timeout_msecs)
4028{
957c5ab1 4029 int rc = 0;
6c223761
KB		 4030 struct pqi_io_request *io_request;
4031 unsigned long start_jiffies;
4032 unsigned long msecs_blocked;
4033 size_t iu_length;
957c5ab1 4034 DECLARE_COMPLETION_ONSTACK(wait);
6c223761
KB		 4035
4036 /*
4037 * Note that specifying PQI_SYNC_FLAGS_INTERRUPTABLE and a timeout value
4038 * are mutually exclusive.
4039 */
4040
4041 if (flags & PQI_SYNC_FLAGS_INTERRUPTABLE) {
4042 if (down_interruptible(&ctrl_info->sync_request_sem))
4043 return -ERESTARTSYS;
4044 } else {
4045 if (timeout_msecs == NO_TIMEOUT) {
4046 down(&ctrl_info->sync_request_sem);
4047 } else {
4048 start_jiffies = jiffies;
4049 if (down_timeout(&ctrl_info->sync_request_sem,
4050 msecs_to_jiffies(timeout_msecs)))
4051 return -ETIMEDOUT;
4052 msecs_blocked =
4053 jiffies_to_msecs(jiffies - start_jiffies);
cc8f5260
DC		 4054 if (msecs_blocked >= timeout_msecs) {
4055 rc = -ETIMEDOUT;
4056 goto out;
4057 }
6c223761
KB		 4058 timeout_msecs -= msecs_blocked;
4059 }
4060 }
4061
7561a7e4
KB		 4062 pqi_ctrl_busy(ctrl_info);
4063 timeout_msecs = pqi_wait_if_ctrl_blocked(ctrl_info, timeout_msecs);
4064 if (timeout_msecs == 0) {
957c5ab1 4065 pqi_ctrl_unbusy(ctrl_info);
7561a7e4
KB		 4066 rc = -ETIMEDOUT;
4067 goto out;
4068 }
4069
376fb880 4070 if (pqi_ctrl_offline(ctrl_info)) {
957c5ab1 4071 pqi_ctrl_unbusy(ctrl_info);
376fb880
KB		 4072 rc = -ENXIO;
4073 goto out;
4074 }
4075
0530736e
KB		 4076 atomic_inc(&ctrl_info->sync_cmds_outstanding);
4077
6c223761
KB		 4078 io_request = pqi_alloc_io_request(ctrl_info);
4079
4080 put_unaligned_le16(io_request->index,
4081 &(((struct pqi_raid_path_request *)request)->request_id));
4082
4083 if (request->iu_type == PQI_REQUEST_IU_RAID_PATH_IO)
4084 ((struct pqi_raid_path_request *)request)->error_index =
4085 ((struct pqi_raid_path_request *)request)->request_id;
4086
4087 iu_length = get_unaligned_le16(&request->iu_length) +
4088 PQI_REQUEST_HEADER_LENGTH;
4089 memcpy(io_request->iu, request, iu_length);
4090
957c5ab1
KB		 4091 io_request->io_complete_callback = pqi_raid_synchronous_complete;
4092 io_request->context = &wait;
4093
4094 pqi_start_io(ctrl_info,
4095 &ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP], RAID_PATH,
4096 io_request);
4097
4098 pqi_ctrl_unbusy(ctrl_info);
4099
4100 if (timeout_msecs == NO_TIMEOUT) {
4101 pqi_wait_for_completion_io(ctrl_info, &wait);
4102 } else {
4103 if (!wait_for_completion_io_timeout(&wait,
4104 msecs_to_jiffies(timeout_msecs))) {
4105 dev_warn(&ctrl_info->pci_dev->dev,
4106 "command timed out\n");
4107 rc = -ETIMEDOUT;
4108 }
4109 }
6c223761
KB		 4110
4111 if (error_info) {
4112 if (io_request->error_info)
4113 memcpy(error_info, io_request->error_info,
4114 sizeof(*error_info));
4115 else
4116 memset(error_info, 0, sizeof(*error_info));
4117 } else if (rc == 0 && io_request->error_info) {
26b390ab
KB		 4118 rc = pqi_process_raid_io_error_synchronous(
4119 io_request->error_info);
6c223761
KB		 4120 }
4121
4122 pqi_free_io_request(io_request);
4123
0530736e 4124 atomic_dec(&ctrl_info->sync_cmds_outstanding);
7561a7e4 4125out:
6c223761
KB		 4126 up(&ctrl_info->sync_request_sem);
4127
4128 return rc;
4129}
4130
4131static int pqi_validate_admin_response(
4132 struct pqi_general_admin_response *response, u8 expected_function_code)
4133{
4134 if (response->header.iu_type != PQI_RESPONSE_IU_GENERAL_ADMIN)
4135 return -EINVAL;
4136
4137 if (get_unaligned_le16(&response->header.iu_length) !=
4138 PQI_GENERAL_ADMIN_IU_LENGTH)
4139 return -EINVAL;
4140
4141 if (response->function_code != expected_function_code)
4142 return -EINVAL;
4143
4144 if (response->status != PQI_GENERAL_ADMIN_STATUS_SUCCESS)
4145 return -EINVAL;
4146
4147 return 0;
4148}
4149
4150static int pqi_submit_admin_request_synchronous(
4151 struct pqi_ctrl_info *ctrl_info,
4152 struct pqi_general_admin_request *request,
4153 struct pqi_general_admin_response *response)
4154{
4155 int rc;
4156
4157 pqi_submit_admin_request(ctrl_info, request);
4158
4159 rc = pqi_poll_for_admin_response(ctrl_info, response);
4160
4161 if (rc == 0)
4162 rc = pqi_validate_admin_response(response,
4163 request->function_code);
4164
4165 return rc;
4166}
4167
4168static int pqi_report_device_capability(struct pqi_ctrl_info *ctrl_info)
4169{
4170 int rc;
4171 struct pqi_general_admin_request request;
4172 struct pqi_general_admin_response response;
4173 struct pqi_device_capability *capability;
4174 struct pqi_iu_layer_descriptor *sop_iu_layer_descriptor;
4175
4176 capability = kmalloc(sizeof(*capability), GFP_KERNEL);
4177 if (!capability)
4178 return -ENOMEM;
4179
4180 memset(&request, 0, sizeof(request));
4181
4182 request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
4183 put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
4184 &request.header.iu_length);
4185 request.function_code =
4186 PQI_GENERAL_ADMIN_FUNCTION_REPORT_DEVICE_CAPABILITY;
4187 put_unaligned_le32(sizeof(*capability),
4188 &request.data.report_device_capability.buffer_length);
4189
4190 rc = pqi_map_single(ctrl_info->pci_dev,
4191 &request.data.report_device_capability.sg_descriptor,
4192 capability, sizeof(*capability),
6917a9cc 4193 DMA_FROM_DEVICE);
6c223761
KB		 4194 if (rc)
4195 goto out;
4196
4197 rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
4198 &response);
4199
4200 pqi_pci_unmap(ctrl_info->pci_dev,
4201 &request.data.report_device_capability.sg_descriptor, 1,
6917a9cc 4202 DMA_FROM_DEVICE);
6c223761
KB		 4203
4204 if (rc)
4205 goto out;
4206
4207 if (response.status != PQI_GENERAL_ADMIN_STATUS_SUCCESS) {
4208 rc = -EIO;
4209 goto out;
4210 }
4211
4212 ctrl_info->max_inbound_queues =
4213 get_unaligned_le16(&capability->max_inbound_queues);
4214 ctrl_info->max_elements_per_iq =
4215 get_unaligned_le16(&capability->max_elements_per_iq);
4216 ctrl_info->max_iq_element_length =
4217 get_unaligned_le16(&capability->max_iq_element_length)
4218 * 16;
4219 ctrl_info->max_outbound_queues =
4220 get_unaligned_le16(&capability->max_outbound_queues);
4221 ctrl_info->max_elements_per_oq =
4222 get_unaligned_le16(&capability->max_elements_per_oq);
4223 ctrl_info->max_oq_element_length =
4224 get_unaligned_le16(&capability->max_oq_element_length)
4225 * 16;
4226
4227 sop_iu_layer_descriptor =
4228 &capability->iu_layer_descriptors[PQI_PROTOCOL_SOP];
4229
4230 ctrl_info->max_inbound_iu_length_per_firmware =
4231 get_unaligned_le16(
4232 &sop_iu_layer_descriptor->max_inbound_iu_length);
4233 ctrl_info->inbound_spanning_supported =
4234 sop_iu_layer_descriptor->inbound_spanning_supported;
4235 ctrl_info->outbound_spanning_supported =
4236 sop_iu_layer_descriptor->outbound_spanning_supported;
4237
4238out:
4239 kfree(capability);
4240
4241 return rc;
4242}
4243
4244static int pqi_validate_device_capability(struct pqi_ctrl_info *ctrl_info)
4245{
4246 if (ctrl_info->max_iq_element_length <
4247 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH) {
4248 dev_err(&ctrl_info->pci_dev->dev,
4249 "max. inbound queue element length of %d is less than the required length of %d\n",
4250 ctrl_info->max_iq_element_length,
4251 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
4252 return -EINVAL;
4253 }
4254
4255 if (ctrl_info->max_oq_element_length <
4256 PQI_OPERATIONAL_OQ_ELEMENT_LENGTH) {
4257 dev_err(&ctrl_info->pci_dev->dev,
4258 "max. outbound queue element length of %d is less than the required length of %d\n",
4259 ctrl_info->max_oq_element_length,
4260 PQI_OPERATIONAL_OQ_ELEMENT_LENGTH);
4261 return -EINVAL;
4262 }
4263
4264 if (ctrl_info->max_inbound_iu_length_per_firmware <
4265 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH) {
4266 dev_err(&ctrl_info->pci_dev->dev,
4267 "max. inbound IU length of %u is less than the min. required length of %d\n",
4268 ctrl_info->max_inbound_iu_length_per_firmware,
4269 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
4270 return -EINVAL;
4271 }
4272
77668f41
KB		 4273 if (!ctrl_info->inbound_spanning_supported) {
4274 dev_err(&ctrl_info->pci_dev->dev,
4275 "the controller does not support inbound spanning\n");
4276 return -EINVAL;
4277 }
4278
4279 if (ctrl_info->outbound_spanning_supported) {
4280 dev_err(&ctrl_info->pci_dev->dev,
4281 "the controller supports outbound spanning but this driver does not\n");
4282 return -EINVAL;
4283 }
4284
6c223761
KB		 4285 return 0;
4286}
4287
6c223761
KB		 4288static int pqi_create_event_queue(struct pqi_ctrl_info *ctrl_info)
4289{
4290 int rc;
4291 struct pqi_event_queue *event_queue;
4292 struct pqi_general_admin_request request;
4293 struct pqi_general_admin_response response;
4294
4295 event_queue = &ctrl_info->event_queue;
4296
4297 /*
4298 * Create OQ (Outbound Queue - device to host queue) to dedicate
4299 * to events.
4300 */
4301 memset(&request, 0, sizeof(request));
4302 request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
4303 put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
4304 &request.header.iu_length);
4305 request.function_code = PQI_GENERAL_ADMIN_FUNCTION_CREATE_OQ;
4306 put_unaligned_le16(event_queue->oq_id,
4307 &request.data.create_operational_oq.queue_id);
4308 put_unaligned_le64((u64)event_queue->oq_element_array_bus_addr,
4309 &request.data.create_operational_oq.element_array_addr);
4310 put_unaligned_le64((u64)event_queue->oq_pi_bus_addr,
4311 &request.data.create_operational_oq.pi_addr);
4312 put_unaligned_le16(PQI_NUM_EVENT_QUEUE_ELEMENTS,
4313 &request.data.create_operational_oq.num_elements);
4314 put_unaligned_le16(PQI_EVENT_OQ_ELEMENT_LENGTH / 16,
4315 &request.data.create_operational_oq.element_length);
4316 request.data.create_operational_oq.queue_protocol = PQI_PROTOCOL_SOP;
4317 put_unaligned_le16(event_queue->int_msg_num,
4318 &request.data.create_operational_oq.int_msg_num);
4319
4320 rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
4321 &response);
4322 if (rc)
4323 return rc;
4324
4325 event_queue->oq_ci = ctrl_info->iomem_base +
4326 PQI_DEVICE_REGISTERS_OFFSET +
4327 get_unaligned_le64(
4328 &response.data.create_operational_oq.oq_ci_offset);
4329
4330 return 0;
4331}
4332
061ef06a
KB		 4333static int pqi_create_queue_group(struct pqi_ctrl_info *ctrl_info,
4334 unsigned int group_number)
6c223761 4335{
6c223761
KB		 4336 int rc;
4337 struct pqi_queue_group *queue_group;
4338 struct pqi_general_admin_request request;
4339 struct pqi_general_admin_response response;
4340
061ef06a 4341 queue_group = &ctrl_info->queue_groups[group_number];
6c223761
KB		 4342
4343 /*
4344 * Create IQ (Inbound Queue - host to device queue) for
4345 * RAID path.
4346 */
4347 memset(&request, 0, sizeof(request));
4348 request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
4349 put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
4350 &request.header.iu_length);
4351 request.function_code = PQI_GENERAL_ADMIN_FUNCTION_CREATE_IQ;
4352 put_unaligned_le16(queue_group->iq_id[RAID_PATH],
4353 &request.data.create_operational_iq.queue_id);
4354 put_unaligned_le64(
4355 (u64)queue_group->iq_element_array_bus_addr[RAID_PATH],
4356 &request.data.create_operational_iq.element_array_addr);
4357 put_unaligned_le64((u64)queue_group->iq_ci_bus_addr[RAID_PATH],
4358 &request.data.create_operational_iq.ci_addr);
4359 put_unaligned_le16(ctrl_info->num_elements_per_iq,
4360 &request.data.create_operational_iq.num_elements);
4361 put_unaligned_le16(PQI_OPERATIONAL_IQ_ELEMENT_LENGTH / 16,
4362 &request.data.create_operational_iq.element_length);
4363 request.data.create_operational_iq.queue_protocol = PQI_PROTOCOL_SOP;
4364
4365 rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
4366 &response);
4367 if (rc) {
4368 dev_err(&ctrl_info->pci_dev->dev,
4369 "error creating inbound RAID queue\n");
4370 return rc;
4371 }
4372
4373 queue_group->iq_pi[RAID_PATH] = ctrl_info->iomem_base +
4374 PQI_DEVICE_REGISTERS_OFFSET +
4375 get_unaligned_le64(
4376 &response.data.create_operational_iq.iq_pi_offset);
4377
4378 /*
4379 * Create IQ (Inbound Queue - host to device queue) for
4380 * Advanced I/O (AIO) path.
4381 */
4382 memset(&request, 0, sizeof(request));
4383 request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
4384 put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
4385 &request.header.iu_length);
4386 request.function_code = PQI_GENERAL_ADMIN_FUNCTION_CREATE_IQ;
4387 put_unaligned_le16(queue_group->iq_id[AIO_PATH],
4388 &request.data.create_operational_iq.queue_id);
4389 put_unaligned_le64((u64)queue_group->
4390 iq_element_array_bus_addr[AIO_PATH],
4391 &request.data.create_operational_iq.element_array_addr);
4392 put_unaligned_le64((u64)queue_group->iq_ci_bus_addr[AIO_PATH],
4393 &request.data.create_operational_iq.ci_addr);
4394 put_unaligned_le16(ctrl_info->num_elements_per_iq,
4395 &request.data.create_operational_iq.num_elements);
4396 put_unaligned_le16(PQI_OPERATIONAL_IQ_ELEMENT_LENGTH / 16,
4397 &request.data.create_operational_iq.element_length);
4398 request.data.create_operational_iq.queue_protocol = PQI_PROTOCOL_SOP;
4399
4400 rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
4401 &response);
4402 if (rc) {
4403 dev_err(&ctrl_info->pci_dev->dev,
4404 "error creating inbound AIO queue\n");
339faa81 4405 return rc;
6c223761
KB		 4406 }
4407
4408 queue_group->iq_pi[AIO_PATH] = ctrl_info->iomem_base +
4409 PQI_DEVICE_REGISTERS_OFFSET +
4410 get_unaligned_le64(
4411 &response.data.create_operational_iq.iq_pi_offset);
4412
4413 /*
4414 * Designate the 2nd IQ as the AIO path. By default, all IQs are
4415 * assumed to be for RAID path I/O unless we change the queue's
4416 * property.
4417 */
4418 memset(&request, 0, sizeof(request));
4419 request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
4420 put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
4421 &request.header.iu_length);
4422 request.function_code = PQI_GENERAL_ADMIN_FUNCTION_CHANGE_IQ_PROPERTY;
4423 put_unaligned_le16(queue_group->iq_id[AIO_PATH],
4424 &request.data.change_operational_iq_properties.queue_id);
4425 put_unaligned_le32(PQI_IQ_PROPERTY_IS_AIO_QUEUE,
4426 &request.data.change_operational_iq_properties.vendor_specific);
4427
4428 rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
4429 &response);
4430 if (rc) {
4431 dev_err(&ctrl_info->pci_dev->dev,
4432 "error changing queue property\n");
339faa81 4433 return rc;
6c223761
KB		 4434 }
4435
4436 /*
4437 * Create OQ (Outbound Queue - device to host queue).
4438 */
4439 memset(&request, 0, sizeof(request));
4440 request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
4441 put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
4442 &request.header.iu_length);
4443 request.function_code = PQI_GENERAL_ADMIN_FUNCTION_CREATE_OQ;
4444 put_unaligned_le16(queue_group->oq_id,
4445 &request.data.create_operational_oq.queue_id);
4446 put_unaligned_le64((u64)queue_group->oq_element_array_bus_addr,
4447 &request.data.create_operational_oq.element_array_addr);
4448 put_unaligned_le64((u64)queue_group->oq_pi_bus_addr,
4449 &request.data.create_operational_oq.pi_addr);
4450 put_unaligned_le16(ctrl_info->num_elements_per_oq,
4451 &request.data.create_operational_oq.num_elements);
4452 put_unaligned_le16(PQI_OPERATIONAL_OQ_ELEMENT_LENGTH / 16,
4453 &request.data.create_operational_oq.element_length);
4454 request.data.create_operational_oq.queue_protocol = PQI_PROTOCOL_SOP;
4455 put_unaligned_le16(queue_group->int_msg_num,
4456 &request.data.create_operational_oq.int_msg_num);
4457
4458 rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
4459 &response);
4460 if (rc) {
4461 dev_err(&ctrl_info->pci_dev->dev,
4462 "error creating outbound queue\n");
339faa81 4463 return rc;
6c223761
KB		 4464 }
4465
4466 queue_group->oq_ci = ctrl_info->iomem_base +
4467 PQI_DEVICE_REGISTERS_OFFSET +
4468 get_unaligned_le64(
4469 &response.data.create_operational_oq.oq_ci_offset);
4470
6c223761 4471 return 0;
6c223761
KB		 4472}
4473
4474static int pqi_create_queues(struct pqi_ctrl_info *ctrl_info)
4475{
4476 int rc;
4477 unsigned int i;
4478
4479 rc = pqi_create_event_queue(ctrl_info);
4480 if (rc) {
4481 dev_err(&ctrl_info->pci_dev->dev,
4482 "error creating event queue\n");
4483 return rc;
4484 }
4485
4486 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
061ef06a 4487 rc = pqi_create_queue_group(ctrl_info, i);
6c223761
KB		 4488 if (rc) {
4489 dev_err(&ctrl_info->pci_dev->dev,
4490 "error creating queue group number %u/%u\n",
4491 i, ctrl_info->num_queue_groups);
4492 return rc;
4493 }
4494 }
4495
4496 return 0;
4497}
4498
4499#define PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH \
4500 (offsetof(struct pqi_event_config, descriptors) + \
4501 (PQI_MAX_EVENT_DESCRIPTORS * sizeof(struct pqi_event_descriptor)))
4502
6a50d6ad
KB		 4503static int pqi_configure_events(struct pqi_ctrl_info *ctrl_info,
4504 bool enable_events)
6c223761
KB		 4505{
4506 int rc;
4507 unsigned int i;
4508 struct pqi_event_config *event_config;
6a50d6ad 4509 struct pqi_event_descriptor *event_descriptor;
6c223761
KB		 4510 struct pqi_general_management_request request;
4511
4512 event_config = kmalloc(PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH,
4513 GFP_KERNEL);
4514 if (!event_config)
4515 return -ENOMEM;
4516
4517 memset(&request, 0, sizeof(request));
4518
4519 request.header.iu_type = PQI_REQUEST_IU_REPORT_VENDOR_EVENT_CONFIG;
4520 put_unaligned_le16(offsetof(struct pqi_general_management_request,
4521 data.report_event_configuration.sg_descriptors[1]) -
4522 PQI_REQUEST_HEADER_LENGTH, &request.header.iu_length);
4523 put_unaligned_le32(PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH,
4524 &request.data.report_event_configuration.buffer_length);
4525
4526 rc = pqi_map_single(ctrl_info->pci_dev,
4527 request.data.report_event_configuration.sg_descriptors,
4528 event_config, PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH,
6917a9cc 4529 DMA_FROM_DEVICE);
6c223761
KB		 4530 if (rc)
4531 goto out;
4532
4533 rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header,
4534 0, NULL, NO_TIMEOUT);
4535
4536 pqi_pci_unmap(ctrl_info->pci_dev,
4537 request.data.report_event_configuration.sg_descriptors, 1,
6917a9cc 4538 DMA_FROM_DEVICE);
6c223761
KB		 4539
4540 if (rc)
4541 goto out;
4542
6a50d6ad
KB		 4543 for (i = 0; i < event_config->num_event_descriptors; i++) {
4544 event_descriptor = &event_config->descriptors[i];
4545 if (enable_events &&
4546 pqi_is_supported_event(event_descriptor->event_type))
4547 put_unaligned_le16(ctrl_info->event_queue.oq_id,
4548 &event_descriptor->oq_id);
4549 else
4550 put_unaligned_le16(0, &event_descriptor->oq_id);
4551 }
6c223761
KB		 4552
4553 memset(&request, 0, sizeof(request));
4554
4555 request.header.iu_type = PQI_REQUEST_IU_SET_VENDOR_EVENT_CONFIG;
4556 put_unaligned_le16(offsetof(struct pqi_general_management_request,
4557 data.report_event_configuration.sg_descriptors[1]) -
4558 PQI_REQUEST_HEADER_LENGTH, &request.header.iu_length);
4559 put_unaligned_le32(PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH,
4560 &request.data.report_event_configuration.buffer_length);
4561
4562 rc = pqi_map_single(ctrl_info->pci_dev,
4563 request.data.report_event_configuration.sg_descriptors,
4564 event_config, PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH,
6917a9cc 4565 DMA_TO_DEVICE);
6c223761
KB		 4566 if (rc)
4567 goto out;
4568
4569 rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0,
4570 NULL, NO_TIMEOUT);
4571
4572 pqi_pci_unmap(ctrl_info->pci_dev,
4573 request.data.report_event_configuration.sg_descriptors, 1,
6917a9cc 4574 DMA_TO_DEVICE);
6c223761
KB		 4575
4576out:
4577 kfree(event_config);
4578
4579 return rc;
4580}
4581
6a50d6ad
KB		 4582static inline int pqi_enable_events(struct pqi_ctrl_info *ctrl_info)
4583{
4584 return pqi_configure_events(ctrl_info, true);
4585}
4586
4587static inline int pqi_disable_events(struct pqi_ctrl_info *ctrl_info)
4588{
4589 return pqi_configure_events(ctrl_info, false);
4590}
4591
6c223761
KB		 4592static void pqi_free_all_io_requests(struct pqi_ctrl_info *ctrl_info)
4593{
4594 unsigned int i;
4595 struct device *dev;
4596 size_t sg_chain_buffer_length;
4597 struct pqi_io_request *io_request;
4598
4599 if (!ctrl_info->io_request_pool)
4600 return;
4601
4602 dev = &ctrl_info->pci_dev->dev;
4603 sg_chain_buffer_length = ctrl_info->sg_chain_buffer_length;
4604 io_request = ctrl_info->io_request_pool;
4605
4606 for (i = 0; i < ctrl_info->max_io_slots; i++) {
4607 kfree(io_request->iu);
4608 if (!io_request->sg_chain_buffer)
4609 break;
4610 dma_free_coherent(dev, sg_chain_buffer_length,
4611 io_request->sg_chain_buffer,
4612 io_request->sg_chain_buffer_dma_handle);
4613 io_request++;
4614 }
4615
4616 kfree(ctrl_info->io_request_pool);
4617 ctrl_info->io_request_pool = NULL;
4618}
4619
4620static inline int pqi_alloc_error_buffer(struct pqi_ctrl_info *ctrl_info)
4621{
6c223761 4622
694c5d5b
KB		 4623 ctrl_info->error_buffer = dma_alloc_coherent(&ctrl_info->pci_dev->dev,
4624 ctrl_info->error_buffer_length,
4625 &ctrl_info->error_buffer_dma_handle,
4626 GFP_KERNEL);
6c223761
KB		 4627 if (!ctrl_info->error_buffer)
4628 return -ENOMEM;
4629
4630 return 0;
4631}
4632
4633static int pqi_alloc_io_resources(struct pqi_ctrl_info *ctrl_info)
4634{
4635 unsigned int i;
4636 void *sg_chain_buffer;
4637 size_t sg_chain_buffer_length;
4638 dma_addr_t sg_chain_buffer_dma_handle;
4639 struct device *dev;
4640 struct pqi_io_request *io_request;
4641
6396bb22
KC		 4642 ctrl_info->io_request_pool =
4643 kcalloc(ctrl_info->max_io_slots,
4644 sizeof(ctrl_info->io_request_pool[0]), GFP_KERNEL);
6c223761
KB		 4645
4646 if (!ctrl_info->io_request_pool) {
4647 dev_err(&ctrl_info->pci_dev->dev,
4648 "failed to allocate I/O request pool\n");
4649 goto error;
4650 }
4651
4652 dev = &ctrl_info->pci_dev->dev;
4653 sg_chain_buffer_length = ctrl_info->sg_chain_buffer_length;
4654 io_request = ctrl_info->io_request_pool;
4655
4656 for (i = 0; i < ctrl_info->max_io_slots; i++) {
4657 io_request->iu =
4658 kmalloc(ctrl_info->max_inbound_iu_length, GFP_KERNEL);
4659
4660 if (!io_request->iu) {
4661 dev_err(&ctrl_info->pci_dev->dev,
4662 "failed to allocate IU buffers\n");
4663 goto error;
4664 }
4665
4666 sg_chain_buffer = dma_alloc_coherent(dev,
4667 sg_chain_buffer_length, &sg_chain_buffer_dma_handle,
4668 GFP_KERNEL);
4669
4670 if (!sg_chain_buffer) {
4671 dev_err(&ctrl_info->pci_dev->dev,
4672 "failed to allocate PQI scatter-gather chain buffers\n");
4673 goto error;
4674 }
4675
4676 io_request->index = i;
4677 io_request->sg_chain_buffer = sg_chain_buffer;
4678 io_request->sg_chain_buffer_dma_handle =
4679 sg_chain_buffer_dma_handle;
4680 io_request++;
4681 }
4682
4683 return 0;
4684
4685error:
4686 pqi_free_all_io_requests(ctrl_info);
4687
4688 return -ENOMEM;
4689}
4690
4691/*
4692 * Calculate required resources that are sized based on max. outstanding
4693 * requests and max. transfer size.
4694 */
4695
4696static void pqi_calculate_io_resources(struct pqi_ctrl_info *ctrl_info)
4697{
4698 u32 max_transfer_size;
4699 u32 max_sg_entries;
4700
4701 ctrl_info->scsi_ml_can_queue =
4702 ctrl_info->max_outstanding_requests - PQI_RESERVED_IO_SLOTS;
4703 ctrl_info->max_io_slots = ctrl_info->max_outstanding_requests;
4704
4705 ctrl_info->error_buffer_length =
4706 ctrl_info->max_io_slots * PQI_ERROR_BUFFER_ELEMENT_LENGTH;
4707
d727a776
KB		 4708 if (reset_devices)
4709 max_transfer_size = min(ctrl_info->max_transfer_size,
4710 PQI_MAX_TRANSFER_SIZE_KDUMP);
4711 else
4712 max_transfer_size = min(ctrl_info->max_transfer_size,
4713 PQI_MAX_TRANSFER_SIZE);
6c223761
KB		 4714
4715 max_sg_entries = max_transfer_size / PAGE_SIZE;
4716
4717 /* +1 to cover when the buffer is not page-aligned. */
4718 max_sg_entries++;
4719
4720 max_sg_entries = min(ctrl_info->max_sg_entries, max_sg_entries);
4721
4722 max_transfer_size = (max_sg_entries - 1) * PAGE_SIZE;
4723
4724 ctrl_info->sg_chain_buffer_length =
e1d213bd
KB		 4725 (max_sg_entries * sizeof(struct pqi_sg_descriptor)) +
4726 PQI_EXTRA_SGL_MEMORY;
6c223761
KB		 4727 ctrl_info->sg_tablesize = max_sg_entries;
4728 ctrl_info->max_sectors = max_transfer_size / 512;
4729}
4730
4731static void pqi_calculate_queue_resources(struct pqi_ctrl_info *ctrl_info)
4732{
6c223761
KB		 4733 int num_queue_groups;
4734 u16 num_elements_per_iq;
4735 u16 num_elements_per_oq;
4736
d727a776
KB		 4737 if (reset_devices) {
4738 num_queue_groups = 1;
4739 } else {
4740 int num_cpus;
4741 int max_queue_groups;
4742
4743 max_queue_groups = min(ctrl_info->max_inbound_queues / 2,
4744 ctrl_info->max_outbound_queues - 1);
4745 max_queue_groups = min(max_queue_groups, PQI_MAX_QUEUE_GROUPS);
6c223761 4746
d727a776
KB		 4747 num_cpus = num_online_cpus();
4748 num_queue_groups = min(num_cpus, ctrl_info->max_msix_vectors);
4749 num_queue_groups = min(num_queue_groups, max_queue_groups);
4750 }
6c223761
KB		 4751
4752 ctrl_info->num_queue_groups = num_queue_groups;
061ef06a 4753 ctrl_info->max_hw_queue_index = num_queue_groups - 1;
6c223761 4754
77668f41
KB		 4755 /*
4756 * Make sure that the max. inbound IU length is an even multiple
4757 * of our inbound element length.
4758 */
4759 ctrl_info->max_inbound_iu_length =
4760 (ctrl_info->max_inbound_iu_length_per_firmware /
4761 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH) *
4762 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH;
6c223761
KB		 4763
4764 num_elements_per_iq =
4765 (ctrl_info->max_inbound_iu_length /
4766 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
4767
4768 /* Add one because one element in each queue is unusable. */
4769 num_elements_per_iq++;
4770
4771 num_elements_per_iq = min(num_elements_per_iq,
4772 ctrl_info->max_elements_per_iq);
4773
4774 num_elements_per_oq = ((num_elements_per_iq - 1) * 2) + 1;
4775 num_elements_per_oq = min(num_elements_per_oq,
4776 ctrl_info->max_elements_per_oq);
4777
4778 ctrl_info->num_elements_per_iq = num_elements_per_iq;
4779 ctrl_info->num_elements_per_oq = num_elements_per_oq;
4780
4781 ctrl_info->max_sg_per_iu =
4782 ((ctrl_info->max_inbound_iu_length -
4783 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH) /
4784 sizeof(struct pqi_sg_descriptor)) +
4785 PQI_MAX_EMBEDDED_SG_DESCRIPTORS;
4786}
4787
4788static inline void pqi_set_sg_descriptor(
4789 struct pqi_sg_descriptor *sg_descriptor, struct scatterlist *sg)
4790{
4791 u64 address = (u64)sg_dma_address(sg);
4792 unsigned int length = sg_dma_len(sg);
4793
4794 put_unaligned_le64(address, &sg_descriptor->address);
4795 put_unaligned_le32(length, &sg_descriptor->length);
4796 put_unaligned_le32(0, &sg_descriptor->flags);
4797}
4798
4799static int pqi_build_raid_sg_list(struct pqi_ctrl_info *ctrl_info,
4800 struct pqi_raid_path_request *request, struct scsi_cmnd *scmd,
4801 struct pqi_io_request *io_request)
4802{
4803 int i;
4804 u16 iu_length;
4805 int sg_count;
4806 bool chained;
4807 unsigned int num_sg_in_iu;
4808 unsigned int max_sg_per_iu;
4809 struct scatterlist *sg;
4810 struct pqi_sg_descriptor *sg_descriptor;
4811
4812 sg_count = scsi_dma_map(scmd);
4813 if (sg_count < 0)
4814 return sg_count;
4815
4816 iu_length = offsetof(struct pqi_raid_path_request, sg_descriptors) -
4817 PQI_REQUEST_HEADER_LENGTH;
4818
4819 if (sg_count == 0)
4820 goto out;
4821
4822 sg = scsi_sglist(scmd);
4823 sg_descriptor = request->sg_descriptors;
4824 max_sg_per_iu = ctrl_info->max_sg_per_iu - 1;
4825 chained = false;
4826 num_sg_in_iu = 0;
4827 i = 0;
4828
4829 while (1) {
4830 pqi_set_sg_descriptor(sg_descriptor, sg);
4831 if (!chained)
4832 num_sg_in_iu++;
4833 i++;
4834 if (i == sg_count)
4835 break;
4836 sg_descriptor++;
4837 if (i == max_sg_per_iu) {
4838 put_unaligned_le64(
4839 (u64)io_request->sg_chain_buffer_dma_handle,
4840 &sg_descriptor->address);
4841 put_unaligned_le32((sg_count - num_sg_in_iu)
4842 * sizeof(*sg_descriptor),
4843 &sg_descriptor->length);
4844 put_unaligned_le32(CISS_SG_CHAIN,
4845 &sg_descriptor->flags);
4846 chained = true;
4847 num_sg_in_iu++;
4848 sg_descriptor = io_request->sg_chain_buffer;
4849 }
4850 sg = sg_next(sg);
4851 }
4852
4853 put_unaligned_le32(CISS_SG_LAST, &sg_descriptor->flags);
4854 request->partial = chained;
4855 iu_length += num_sg_in_iu * sizeof(*sg_descriptor);
4856
4857out:
4858 put_unaligned_le16(iu_length, &request->header.iu_length);
4859
4860 return 0;
4861}
4862
4863static int pqi_build_aio_sg_list(struct pqi_ctrl_info *ctrl_info,
4864 struct pqi_aio_path_request *request, struct scsi_cmnd *scmd,
4865 struct pqi_io_request *io_request)
4866{
4867 int i;
4868 u16 iu_length;
4869 int sg_count;
a60eec02
KB		 4870 bool chained;
4871 unsigned int num_sg_in_iu;
4872 unsigned int max_sg_per_iu;
6c223761
KB		 4873 struct scatterlist *sg;
4874 struct pqi_sg_descriptor *sg_descriptor;
4875
4876 sg_count = scsi_dma_map(scmd);
4877 if (sg_count < 0)
4878 return sg_count;
a60eec02
KB		 4879
4880 iu_length = offsetof(struct pqi_aio_path_request, sg_descriptors) -
4881 PQI_REQUEST_HEADER_LENGTH;
4882 num_sg_in_iu = 0;
4883
6c223761
KB		 4884 if (sg_count == 0)
4885 goto out;
4886
a60eec02
KB		 4887 sg = scsi_sglist(scmd);
4888 sg_descriptor = request->sg_descriptors;
4889 max_sg_per_iu = ctrl_info->max_sg_per_iu - 1;
4890 chained = false;
4891 i = 0;
4892
4893 while (1) {
4894 pqi_set_sg_descriptor(sg_descriptor, sg);
4895 if (!chained)
4896 num_sg_in_iu++;
4897 i++;
4898 if (i == sg_count)
4899 break;
4900 sg_descriptor++;
4901 if (i == max_sg_per_iu) {
4902 put_unaligned_le64(
4903 (u64)io_request->sg_chain_buffer_dma_handle,
4904 &sg_descriptor->address);
4905 put_unaligned_le32((sg_count - num_sg_in_iu)
4906 * sizeof(*sg_descriptor),
4907 &sg_descriptor->length);
4908 put_unaligned_le32(CISS_SG_CHAIN,
4909 &sg_descriptor->flags);
4910 chained = true;
4911 num_sg_in_iu++;
4912 sg_descriptor = io_request->sg_chain_buffer;
6c223761 4913 }
a60eec02 4914 sg = sg_next(sg);
6c223761
KB		 4915 }
4916
a60eec02
KB		 4917 put_unaligned_le32(CISS_SG_LAST, &sg_descriptor->flags);
4918 request->partial = chained;
6c223761 4919 iu_length += num_sg_in_iu * sizeof(*sg_descriptor);
a60eec02
KB		 4920
4921out:
6c223761
KB		 4922 put_unaligned_le16(iu_length, &request->header.iu_length);
4923 request->num_sg_descriptors = num_sg_in_iu;
4924
4925 return 0;
4926}
4927
4928static void pqi_raid_io_complete(struct pqi_io_request *io_request,
4929 void *context)
4930{
4931 struct scsi_cmnd *scmd;
4932
4933 scmd = io_request->scmd;
4934 pqi_free_io_request(io_request);
4935 scsi_dma_unmap(scmd);
4936 pqi_scsi_done(scmd);
4937}
4938
376fb880
KB		 4939static int pqi_raid_submit_scsi_cmd_with_io_request(
4940 struct pqi_ctrl_info *ctrl_info, struct pqi_io_request *io_request,
6c223761
KB		 4941 struct pqi_scsi_dev *device, struct scsi_cmnd *scmd,
4942 struct pqi_queue_group *queue_group)
4943{
4944 int rc;
4945 size_t cdb_length;
6c223761
KB		 4946 struct pqi_raid_path_request *request;
4947
6c223761
KB		 4948 io_request->io_complete_callback = pqi_raid_io_complete;
4949 io_request->scmd = scmd;
4950
6c223761
KB		 4951 request = io_request->iu;
4952 memset(request, 0,
4953 offsetof(struct pqi_raid_path_request, sg_descriptors));
4954
4955 request->header.iu_type = PQI_REQUEST_IU_RAID_PATH_IO;
4956 put_unaligned_le32(scsi_bufflen(scmd), &request->buffer_length);
4957 request->task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
4958 put_unaligned_le16(io_request->index, &request->request_id);
4959 request->error_index = request->request_id;
4960 memcpy(request->lun_number, device->scsi3addr,
4961 sizeof(request->lun_number));
4962
4963 cdb_length = min_t(size_t, scmd->cmd_len, sizeof(request->cdb));
4964 memcpy(request->cdb, scmd->cmnd, cdb_length);
4965
4966 switch (cdb_length) {
4967 case 6:
4968 case 10:
4969 case 12:
4970 case 16:
4971 /* No bytes in the Additional CDB bytes field */
4972 request->additional_cdb_bytes_usage =
4973 SOP_ADDITIONAL_CDB_BYTES_0;
4974 break;
4975 case 20:
4976 /* 4 bytes in the Additional cdb field */
4977 request->additional_cdb_bytes_usage =
4978 SOP_ADDITIONAL_CDB_BYTES_4;
4979 break;
4980 case 24:
4981 /* 8 bytes in the Additional cdb field */
4982 request->additional_cdb_bytes_usage =
4983 SOP_ADDITIONAL_CDB_BYTES_8;
4984 break;
4985 case 28:
4986 /* 12 bytes in the Additional cdb field */
4987 request->additional_cdb_bytes_usage =
4988 SOP_ADDITIONAL_CDB_BYTES_12;
4989 break;
4990 case 32:
4991 default:
4992 /* 16 bytes in the Additional cdb field */
4993 request->additional_cdb_bytes_usage =
4994 SOP_ADDITIONAL_CDB_BYTES_16;
4995 break;
4996 }
4997
4998 switch (scmd->sc_data_direction) {
4999 case DMA_TO_DEVICE:
5000 request->data_direction = SOP_READ_FLAG;
5001 break;
5002 case DMA_FROM_DEVICE:
5003 request->data_direction = SOP_WRITE_FLAG;
5004 break;
5005 case DMA_NONE:
5006 request->data_direction = SOP_NO_DIRECTION_FLAG;
5007 break;
5008 case DMA_BIDIRECTIONAL:
5009 request->data_direction = SOP_BIDIRECTIONAL;
5010 break;
5011 default:
5012 dev_err(&ctrl_info->pci_dev->dev,
5013 "unknown data direction: %d\n",
5014 scmd->sc_data_direction);
6c223761
KB		 5015 break;
5016 }
5017
5018 rc = pqi_build_raid_sg_list(ctrl_info, request, scmd, io_request);
5019 if (rc) {
5020 pqi_free_io_request(io_request);
5021 return SCSI_MLQUEUE_HOST_BUSY;
5022 }
5023
5024 pqi_start_io(ctrl_info, queue_group, RAID_PATH, io_request);
5025
5026 return 0;
5027}
5028
376fb880
KB		 5029static inline int pqi_raid_submit_scsi_cmd(struct pqi_ctrl_info *ctrl_info,
5030 struct pqi_scsi_dev *device, struct scsi_cmnd *scmd,
5031 struct pqi_queue_group *queue_group)
5032{
5033 struct pqi_io_request *io_request;
5034
5035 io_request = pqi_alloc_io_request(ctrl_info);
5036
5037 return pqi_raid_submit_scsi_cmd_with_io_request(ctrl_info, io_request,
5038 device, scmd, queue_group);
5039}
5040
5041static inline void pqi_schedule_bypass_retry(struct pqi_ctrl_info *ctrl_info)
5042{
5043 if (!pqi_ctrl_blocked(ctrl_info))
5044 schedule_work(&ctrl_info->raid_bypass_retry_work);
5045}
5046
5047static bool pqi_raid_bypass_retry_needed(struct pqi_io_request *io_request)
5048{
5049 struct scsi_cmnd *scmd;
03b288cf 5050 struct pqi_scsi_dev *device;
376fb880
KB		 5051 struct pqi_ctrl_info *ctrl_info;
5052
5053 if (!io_request->raid_bypass)
5054 return false;
5055
5056 scmd = io_request->scmd;
5057 if ((scmd->result & 0xff) == SAM_STAT_GOOD)
5058 return false;
5059 if (host_byte(scmd->result) == DID_NO_CONNECT)
5060 return false;
5061
03b288cf
KB		 5062 device = scmd->device->hostdata;
5063 if (pqi_device_offline(device))
5064 return false;
5065
376fb880
KB		 5066 ctrl_info = shost_to_hba(scmd->device->host);
5067 if (pqi_ctrl_offline(ctrl_info))
5068 return false;
5069
5070 return true;
5071}
5072
5073static inline void pqi_add_to_raid_bypass_retry_list(
5074 struct pqi_ctrl_info *ctrl_info,
5075 struct pqi_io_request *io_request, bool at_head)
5076{
5077 unsigned long flags;
5078
5079 spin_lock_irqsave(&ctrl_info->raid_bypass_retry_list_lock, flags);
5080 if (at_head)
5081 list_add(&io_request->request_list_entry,
5082 &ctrl_info->raid_bypass_retry_list);
5083 else
5084 list_add_tail(&io_request->request_list_entry,
5085 &ctrl_info->raid_bypass_retry_list);
5086 spin_unlock_irqrestore(&ctrl_info->raid_bypass_retry_list_lock, flags);
5087}
5088
5089static void pqi_queued_raid_bypass_complete(struct pqi_io_request *io_request,
5090 void *context)
5091{
5092 struct scsi_cmnd *scmd;
5093
5094 scmd = io_request->scmd;
5095 pqi_free_io_request(io_request);
5096 pqi_scsi_done(scmd);
5097}
5098
5099static void pqi_queue_raid_bypass_retry(struct pqi_io_request *io_request)
5100{
5101 struct scsi_cmnd *scmd;
5102 struct pqi_ctrl_info *ctrl_info;
5103
5104 io_request->io_complete_callback = pqi_queued_raid_bypass_complete;
5105 scmd = io_request->scmd;
5106 scmd->result = 0;
5107 ctrl_info = shost_to_hba(scmd->device->host);
5108
5109 pqi_add_to_raid_bypass_retry_list(ctrl_info, io_request, false);
5110 pqi_schedule_bypass_retry(ctrl_info);
5111}
5112
5113static int pqi_retry_raid_bypass(struct pqi_io_request *io_request)
5114{
5115 struct scsi_cmnd *scmd;
5116 struct pqi_scsi_dev *device;
5117 struct pqi_ctrl_info *ctrl_info;
5118 struct pqi_queue_group *queue_group;
5119
5120 scmd = io_request->scmd;
5121 device = scmd->device->hostdata;
5122 if (pqi_device_in_reset(device)) {
5123 pqi_free_io_request(io_request);
5124 set_host_byte(scmd, DID_RESET);
5125 pqi_scsi_done(scmd);
5126 return 0;
5127 }
5128
5129 ctrl_info = shost_to_hba(scmd->device->host);
5130 queue_group = io_request->queue_group;
5131
5132 pqi_reinit_io_request(io_request);
5133
5134 return pqi_raid_submit_scsi_cmd_with_io_request(ctrl_info, io_request,
5135 device, scmd, queue_group);
5136}
5137
5138static inline struct pqi_io_request *pqi_next_queued_raid_bypass_request(
5139 struct pqi_ctrl_info *ctrl_info)
5140{
5141 unsigned long flags;
5142 struct pqi_io_request *io_request;
5143
5144 spin_lock_irqsave(&ctrl_info->raid_bypass_retry_list_lock, flags);
5145 io_request = list_first_entry_or_null(
5146 &ctrl_info->raid_bypass_retry_list,
5147 struct pqi_io_request, request_list_entry);
5148 if (io_request)
5149 list_del(&io_request->request_list_entry);
5150 spin_unlock_irqrestore(&ctrl_info->raid_bypass_retry_list_lock, flags);
5151
5152 return io_request;
5153}
5154
5155static void pqi_retry_raid_bypass_requests(struct pqi_ctrl_info *ctrl_info)
5156{
5157 int rc;
5158 struct pqi_io_request *io_request;
5159
5160 pqi_ctrl_busy(ctrl_info);
5161
5162 while (1) {
5163 if (pqi_ctrl_blocked(ctrl_info))
5164 break;
5165 io_request = pqi_next_queued_raid_bypass_request(ctrl_info);
5166 if (!io_request)
5167 break;
5168 rc = pqi_retry_raid_bypass(io_request);
5169 if (rc) {
5170 pqi_add_to_raid_bypass_retry_list(ctrl_info, io_request,
5171 true);
5172 pqi_schedule_bypass_retry(ctrl_info);
5173 break;
5174 }
5175 }
5176
5177 pqi_ctrl_unbusy(ctrl_info);
5178}
5179
5180static void pqi_raid_bypass_retry_worker(struct work_struct *work)
5181{
5182 struct pqi_ctrl_info *ctrl_info;
5183
5184 ctrl_info = container_of(work, struct pqi_ctrl_info,
5185 raid_bypass_retry_work);
5186 pqi_retry_raid_bypass_requests(ctrl_info);
5187}
5188
5f310425
KB		 5189static void pqi_clear_all_queued_raid_bypass_retries(
5190 struct pqi_ctrl_info *ctrl_info)
376fb880
KB		 5191{
5192 unsigned long flags;
376fb880
KB		 5193
5194 spin_lock_irqsave(&ctrl_info->raid_bypass_retry_list_lock, flags);
5f310425 5195 INIT_LIST_HEAD(&ctrl_info->raid_bypass_retry_list);
376fb880
KB		 5196 spin_unlock_irqrestore(&ctrl_info->raid_bypass_retry_list_lock, flags);
5197}
5198
6c223761
KB		 5199static void pqi_aio_io_complete(struct pqi_io_request *io_request,
5200 void *context)
5201{
5202 struct scsi_cmnd *scmd;
5203
5204 scmd = io_request->scmd;
5205 scsi_dma_unmap(scmd);
5206 if (io_request->status == -EAGAIN)
5207 set_host_byte(scmd, DID_IMM_RETRY);
376fb880
KB		 5208 else if (pqi_raid_bypass_retry_needed(io_request)) {
5209 pqi_queue_raid_bypass_retry(io_request);
5210 return;
5211 }
6c223761
KB		 5212 pqi_free_io_request(io_request);
5213 pqi_scsi_done(scmd);
5214}
5215
5216static inline int pqi_aio_submit_scsi_cmd(struct pqi_ctrl_info *ctrl_info,
5217 struct pqi_scsi_dev *device, struct scsi_cmnd *scmd,
5218 struct pqi_queue_group *queue_group)
5219{
5220 return pqi_aio_submit_io(ctrl_info, scmd, device->aio_handle,
376fb880 5221 scmd->cmnd, scmd->cmd_len, queue_group, NULL, false);
6c223761
KB		 5222}
5223
5224static int pqi_aio_submit_io(struct pqi_ctrl_info *ctrl_info,
5225 struct scsi_cmnd *scmd, u32 aio_handle, u8 *cdb,
5226 unsigned int cdb_length, struct pqi_queue_group *queue_group,
376fb880 5227 struct pqi_encryption_info *encryption_info, bool raid_bypass)
6c223761
KB		 5228{
5229 int rc;
5230 struct pqi_io_request *io_request;
5231 struct pqi_aio_path_request *request;
5232
5233 io_request = pqi_alloc_io_request(ctrl_info);
5234 io_request->io_complete_callback = pqi_aio_io_complete;
5235 io_request->scmd = scmd;
376fb880 5236 io_request->raid_bypass = raid_bypass;
6c223761
KB		 5237
5238 request = io_request->iu;
5239 memset(request, 0,
5240 offsetof(struct pqi_raid_path_request, sg_descriptors));
5241
5242 request->header.iu_type = PQI_REQUEST_IU_AIO_PATH_IO;
5243 put_unaligned_le32(aio_handle, &request->nexus_id);
5244 put_unaligned_le32(scsi_bufflen(scmd), &request->buffer_length);
5245 request->task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
5246 put_unaligned_le16(io_request->index, &request->request_id);
5247 request->error_index = request->request_id;
5248 if (cdb_length > sizeof(request->cdb))
5249 cdb_length = sizeof(request->cdb);
5250 request->cdb_length = cdb_length;
5251 memcpy(request->cdb, cdb, cdb_length);
5252
5253 switch (scmd->sc_data_direction) {
5254 case DMA_TO_DEVICE:
5255 request->data_direction = SOP_READ_FLAG;
5256 break;
5257 case DMA_FROM_DEVICE:
5258 request->data_direction = SOP_WRITE_FLAG;
5259 break;
5260 case DMA_NONE:
5261 request->data_direction = SOP_NO_DIRECTION_FLAG;
5262 break;
5263 case DMA_BIDIRECTIONAL:
5264 request->data_direction = SOP_BIDIRECTIONAL;
5265 break;
5266 default:
5267 dev_err(&ctrl_info->pci_dev->dev,
5268 "unknown data direction: %d\n",
5269 scmd->sc_data_direction);
6c223761
KB		 5270 break;
5271 }
5272
5273 if (encryption_info) {
5274 request->encryption_enable = true;
5275 put_unaligned_le16(encryption_info->data_encryption_key_index,
5276 &request->data_encryption_key_index);
5277 put_unaligned_le32(encryption_info->encrypt_tweak_lower,
5278 &request->encrypt_tweak_lower);
5279 put_unaligned_le32(encryption_info->encrypt_tweak_upper,
5280 &request->encrypt_tweak_upper);
5281 }
5282
5283 rc = pqi_build_aio_sg_list(ctrl_info, request, scmd, io_request);
5284 if (rc) {
5285 pqi_free_io_request(io_request);
5286 return SCSI_MLQUEUE_HOST_BUSY;
5287 }
5288
5289 pqi_start_io(ctrl_info, queue_group, AIO_PATH, io_request);
5290
5291 return 0;
5292}
5293
061ef06a
KB		 5294static inline u16 pqi_get_hw_queue(struct pqi_ctrl_info *ctrl_info,
5295 struct scsi_cmnd *scmd)
5296{
5297 u16 hw_queue;
5298
5299 hw_queue = blk_mq_unique_tag_to_hwq(blk_mq_unique_tag(scmd->request));
5300 if (hw_queue > ctrl_info->max_hw_queue_index)
5301 hw_queue = 0;
5302
5303 return hw_queue;
5304}
5305
7561a7e4
KB		 5306/*
5307 * This function gets called just before we hand the completed SCSI request
5308 * back to the SML.
5309 */
5310
5311void pqi_prep_for_scsi_done(struct scsi_cmnd *scmd)
5312{
5313 struct pqi_scsi_dev *device;
5314
1e46731e
MR		 5315 if (!scmd->device) {
5316 set_host_byte(scmd, DID_NO_CONNECT);
5317 return;
5318 }
5319
7561a7e4 5320 device = scmd->device->hostdata;
1e46731e
MR		 5321 if (!device) {
5322 set_host_byte(scmd, DID_NO_CONNECT);
5323 return;
5324 }
5325
7561a7e4
KB		 5326 atomic_dec(&device->scsi_cmds_outstanding);
5327}
5328
6c223761 5329static int pqi_scsi_queue_command(struct Scsi_Host *shost,
7d81d2b8 5330 struct scsi_cmnd *scmd)
6c223761
KB		 5331{
5332 int rc;
5333 struct pqi_ctrl_info *ctrl_info;
5334 struct pqi_scsi_dev *device;
061ef06a 5335 u16 hw_queue;
6c223761
KB		 5336 struct pqi_queue_group *queue_group;
5337 bool raid_bypassed;
5338
5339 device = scmd->device->hostdata;
6c223761
KB		 5340 ctrl_info = shost_to_hba(shost);
5341
1e46731e
MR		 5342 if (!device) {
5343 set_host_byte(scmd, DID_NO_CONNECT);
5344 pqi_scsi_done(scmd);
5345 return 0;
5346 }
5347
7561a7e4
KB		 5348 atomic_inc(&device->scsi_cmds_outstanding);
5349
1e46731e
MR		 5350 if (pqi_ctrl_offline(ctrl_info) || pqi_device_in_remove(ctrl_info,
5351 device)) {
6c223761
KB		 5352 set_host_byte(scmd, DID_NO_CONNECT);
5353 pqi_scsi_done(scmd);
5354 return 0;
5355 }
5356
7561a7e4 5357 pqi_ctrl_busy(ctrl_info);
4fd22c13 5358 if (pqi_ctrl_blocked(ctrl_info) || pqi_device_in_reset(device) ||
0530736e 5359 pqi_ctrl_in_ofa(ctrl_info) || pqi_ctrl_in_shutdown(ctrl_info)) {
7561a7e4
KB		 5360 rc = SCSI_MLQUEUE_HOST_BUSY;
5361 goto out;
5362 }
5363
7d81d2b8
KB		 5364 /*
5365 * This is necessary because the SML doesn't zero out this field during
5366 * error recovery.
5367 */
5368 scmd->result = 0;
5369
061ef06a
KB		 5370 hw_queue = pqi_get_hw_queue(ctrl_info, scmd);
5371 queue_group = &ctrl_info->queue_groups[hw_queue];
6c223761
KB		 5372
5373 if (pqi_is_logical_device(device)) {
5374 raid_bypassed = false;
588a63fe 5375 if (device->raid_bypass_enabled &&
694c5d5b 5376 !blk_rq_is_passthrough(scmd->request)) {
6c223761
KB		 5377 rc = pqi_raid_bypass_submit_scsi_cmd(ctrl_info, device,
5378 scmd, queue_group);
8b664fef 5379 if (rc == 0 || rc == SCSI_MLQUEUE_HOST_BUSY) {
376fb880 5380 raid_bypassed = true;
8b664fef
KB		 5381 atomic_inc(&device->raid_bypass_cnt);
5382 }
6c223761
KB		 5383 }
5384 if (!raid_bypassed)
8b664fef 5385 rc = pqi_raid_submit_scsi_cmd(ctrl_info, device, scmd, queue_group);
6c223761
KB		 5386 } else {
5387 if (device->aio_enabled)
8b664fef 5388 rc = pqi_aio_submit_scsi_cmd(ctrl_info, device, scmd, queue_group);
6c223761 5389 else
8b664fef 5390 rc = pqi_raid_submit_scsi_cmd(ctrl_info, device, scmd, queue_group);
6c223761
KB		 5391 }
5392
7561a7e4
KB		 5393out:
5394 pqi_ctrl_unbusy(ctrl_info);
5395 if (rc)
5396 atomic_dec(&device->scsi_cmds_outstanding);
5397
6c223761
KB		 5398 return rc;
5399}
5400
7561a7e4
KB		 5401static int pqi_wait_until_queued_io_drained(struct pqi_ctrl_info *ctrl_info,
5402 struct pqi_queue_group *queue_group)
5403{
5404 unsigned int path;
5405 unsigned long flags;
5406 bool list_is_empty;
5407
5408 for (path = 0; path < 2; path++) {
5409 while (1) {
5410 spin_lock_irqsave(
5411 &queue_group->submit_lock[path], flags);
5412 list_is_empty =
5413 list_empty(&queue_group->request_list[path]);
5414 spin_unlock_irqrestore(
5415 &queue_group->submit_lock[path], flags);
5416 if (list_is_empty)
5417 break;
5418 pqi_check_ctrl_health(ctrl_info);
5419 if (pqi_ctrl_offline(ctrl_info))
5420 return -ENXIO;
5421 usleep_range(1000, 2000);
5422 }
5423 }
5424
5425 return 0;
5426}
5427
5428static int pqi_wait_until_inbound_queues_empty(struct pqi_ctrl_info *ctrl_info)
5429{
5430 int rc;
5431 unsigned int i;
5432 unsigned int path;
5433 struct pqi_queue_group *queue_group;
5434 pqi_index_t iq_pi;
5435 pqi_index_t iq_ci;
5436
5437 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
5438 queue_group = &ctrl_info->queue_groups[i];
5439
5440 rc = pqi_wait_until_queued_io_drained(ctrl_info, queue_group);
5441 if (rc)
5442 return rc;
5443
5444 for (path = 0; path < 2; path++) {
5445 iq_pi = queue_group->iq_pi_copy[path];
5446
5447 while (1) {
dac12fbc 5448 iq_ci = readl(queue_group->iq_ci[path]);
7561a7e4
KB		 5449 if (iq_ci == iq_pi)
5450 break;
5451 pqi_check_ctrl_health(ctrl_info);
5452 if (pqi_ctrl_offline(ctrl_info))
5453 return -ENXIO;
5454 usleep_range(1000, 2000);
5455 }
5456 }
5457 }
5458
5459 return 0;
5460}
5461
5462static void pqi_fail_io_queued_for_device(struct pqi_ctrl_info *ctrl_info,
5463 struct pqi_scsi_dev *device)
5464{
5465 unsigned int i;
5466 unsigned int path;
5467 struct pqi_queue_group *queue_group;
5468 unsigned long flags;
5469 struct pqi_io_request *io_request;
5470 struct pqi_io_request *next;
5471 struct scsi_cmnd *scmd;
5472 struct pqi_scsi_dev *scsi_device;
5473
5474 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
5475 queue_group = &ctrl_info->queue_groups[i];
5476
5477 for (path = 0; path < 2; path++) {
5478 spin_lock_irqsave(
5479 &queue_group->submit_lock[path], flags);
5480
5481 list_for_each_entry_safe(io_request, next,
5482 &queue_group->request_list[path],
5483 request_list_entry) {
5484 scmd = io_request->scmd;
5485 if (!scmd)
5486 continue;
5487
5488 scsi_device = scmd->device->hostdata;
5489 if (scsi_device != device)
5490 continue;
5491
5492 list_del(&io_request->request_list_entry);
5493 set_host_byte(scmd, DID_RESET);
5494 pqi_scsi_done(scmd);
5495 }
5496
5497 spin_unlock_irqrestore(
5498 &queue_group->submit_lock[path], flags);
5499 }
5500 }
5501}
5502
4fd22c13
MR		 5503static void pqi_fail_io_queued_for_all_devices(struct pqi_ctrl_info *ctrl_info)
5504{
5505 unsigned int i;
5506 unsigned int path;
5507 struct pqi_queue_group *queue_group;
5508 unsigned long flags;
5509 struct pqi_io_request *io_request;
5510 struct pqi_io_request *next;
5511 struct scsi_cmnd *scmd;
5512
5513 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
5514 queue_group = &ctrl_info->queue_groups[i];
5515
5516 for (path = 0; path < 2; path++) {
5517 spin_lock_irqsave(&queue_group->submit_lock[path],
5518 flags);
5519
5520 list_for_each_entry_safe(io_request, next,
5521 &queue_group->request_list[path],
5522 request_list_entry) {
5523
5524 scmd = io_request->scmd;
5525 if (!scmd)
5526 continue;
5527
5528 list_del(&io_request->request_list_entry);
5529 set_host_byte(scmd, DID_RESET);
5530 pqi_scsi_done(scmd);
5531 }
5532
5533 spin_unlock_irqrestore(
5534 &queue_group->submit_lock[path], flags);
5535 }
5536 }
5537}
5538
061ef06a 5539static int pqi_device_wait_for_pending_io(struct pqi_ctrl_info *ctrl_info,
1e46731e 5540 struct pqi_scsi_dev *device, unsigned long timeout_secs)
061ef06a 5541{
1e46731e
MR		 5542 unsigned long timeout;
5543
4fd22c13 5544 timeout = (timeout_secs * PQI_HZ) + jiffies;
1e46731e 5545
061ef06a
KB		 5546 while (atomic_read(&device->scsi_cmds_outstanding)) {
5547 pqi_check_ctrl_health(ctrl_info);
5548 if (pqi_ctrl_offline(ctrl_info))
5549 return -ENXIO;
1e46731e
MR		 5550 if (timeout_secs != NO_TIMEOUT) {
5551 if (time_after(jiffies, timeout)) {
5552 dev_err(&ctrl_info->pci_dev->dev,
5553 "timed out waiting for pending IO\n");
5554 return -ETIMEDOUT;
5555 }
5556 }
061ef06a
KB		 5557 usleep_range(1000, 2000);
5558 }
5559
5560 return 0;
5561}
5562
4fd22c13
MR		 5563static int pqi_ctrl_wait_for_pending_io(struct pqi_ctrl_info *ctrl_info,
5564 unsigned long timeout_secs)
061ef06a
KB		 5565{
5566 bool io_pending;
5567 unsigned long flags;
4fd22c13 5568 unsigned long timeout;
061ef06a
KB		 5569 struct pqi_scsi_dev *device;
5570
4fd22c13 5571 timeout = (timeout_secs * PQI_HZ) + jiffies;
061ef06a
KB		 5572 while (1) {
5573 io_pending = false;
5574
5575 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
5576 list_for_each_entry(device, &ctrl_info->scsi_device_list,
5577 scsi_device_list_entry) {
5578 if (atomic_read(&device->scsi_cmds_outstanding)) {
5579 io_pending = true;
5580 break;
5581 }
5582 }
5583 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock,
5584 flags);
5585
5586 if (!io_pending)
5587 break;
5588
5589 pqi_check_ctrl_health(ctrl_info);
5590 if (pqi_ctrl_offline(ctrl_info))
5591 return -ENXIO;
5592
4fd22c13
MR		 5593 if (timeout_secs != NO_TIMEOUT) {
5594 if (time_after(jiffies, timeout)) {
5595 dev_err(&ctrl_info->pci_dev->dev,
5596 "timed out waiting for pending IO\n");
5597 return -ETIMEDOUT;
5598 }
5599 }
061ef06a
KB		 5600 usleep_range(1000, 2000);
5601 }
5602
5603 return 0;
5604}
5605
0530736e
KB		 5606static int pqi_ctrl_wait_for_pending_sync_cmds(struct pqi_ctrl_info *ctrl_info)
5607{
5608 while (atomic_read(&ctrl_info->sync_cmds_outstanding)) {
5609 pqi_check_ctrl_health(ctrl_info);
5610 if (pqi_ctrl_offline(ctrl_info))
5611 return -ENXIO;
5612 usleep_range(1000, 2000);
5613 }
5614
5615 return 0;
5616}
5617
14bb215d
KB		 5618static void pqi_lun_reset_complete(struct pqi_io_request *io_request,
5619 void *context)
6c223761 5620{
14bb215d 5621 struct completion *waiting = context;
6c223761 5622
14bb215d
KB		 5623 complete(waiting);
5624}
6c223761 5625
bb9af08c 5626#define PQI_LUN_RESET_TIMEOUT_SECS 30
c2922f17 5627#define PQI_LUN_RESET_POLL_COMPLETION_SECS 10
14bb215d
KB		 5628
5629static int pqi_wait_for_lun_reset_completion(struct pqi_ctrl_info *ctrl_info,
5630 struct pqi_scsi_dev *device, struct completion *wait)
5631{
5632 int rc;
14bb215d
KB		 5633
5634 while (1) {
5635 if (wait_for_completion_io_timeout(wait,
c2922f17 5636 PQI_LUN_RESET_POLL_COMPLETION_SECS * PQI_HZ)) {
14bb215d
KB		 5637 rc = 0;
5638 break;
6c223761
KB		 5639 }
5640
14bb215d
KB		 5641 pqi_check_ctrl_health(ctrl_info);
5642 if (pqi_ctrl_offline(ctrl_info)) {
4e8415e3 5643 rc = -ENXIO;
14bb215d
KB		 5644 break;
5645 }
6c223761 5646 }
6c223761 5647
14bb215d 5648 return rc;
6c223761
KB		 5649}
5650
14bb215d 5651static int pqi_lun_reset(struct pqi_ctrl_info *ctrl_info,
6c223761
KB		 5652 struct pqi_scsi_dev *device)
5653{
5654 int rc;
5655 struct pqi_io_request *io_request;
5656 DECLARE_COMPLETION_ONSTACK(wait);
5657 struct pqi_task_management_request *request;
5658
6c223761 5659 io_request = pqi_alloc_io_request(ctrl_info);
14bb215d 5660 io_request->io_complete_callback = pqi_lun_reset_complete;
6c223761
KB		 5661 io_request->context = &wait;
5662
5663 request = io_request->iu;
5664 memset(request, 0, sizeof(*request));
5665
5666 request->header.iu_type = PQI_REQUEST_IU_TASK_MANAGEMENT;
5667 put_unaligned_le16(sizeof(*request) - PQI_REQUEST_HEADER_LENGTH,
5668 &request->header.iu_length);
5669 put_unaligned_le16(io_request->index, &request->request_id);
5670 memcpy(request->lun_number, device->scsi3addr,
5671 sizeof(request->lun_number));
5672 request->task_management_function = SOP_TASK_MANAGEMENT_LUN_RESET;
c2922f17
MB		 5673 if (ctrl_info->tmf_iu_timeout_supported)
5674 put_unaligned_le16(PQI_LUN_RESET_TIMEOUT_SECS,
5675 &request->timeout);
6c223761
KB		 5676
5677 pqi_start_io(ctrl_info,
5678 &ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP], RAID_PATH,
5679 io_request);
5680
14bb215d
KB		 5681 rc = pqi_wait_for_lun_reset_completion(ctrl_info, device, &wait);
5682 if (rc == 0)
6c223761 5683 rc = io_request->status;
6c223761
KB		 5684
5685 pqi_free_io_request(io_request);
6c223761
KB		 5686
5687 return rc;
5688}
5689
429fab70
KB		 5690/* Performs a reset at the LUN level. */
5691
3406384b
MR		 5692#define PQI_LUN_RESET_RETRIES 3
5693#define PQI_LUN_RESET_RETRY_INTERVAL_MSECS 10000
429fab70 5694#define PQI_LUN_RESET_PENDING_IO_TIMEOUT_SECS 120
6c223761 5695
4fd22c13 5696static int _pqi_device_reset(struct pqi_ctrl_info *ctrl_info,
6c223761
KB		 5697 struct pqi_scsi_dev *device)
5698{
5699 int rc;
3406384b 5700 unsigned int retries;
4fd22c13 5701 unsigned long timeout_secs;
6c223761 5702
3406384b
MR		 5703 for (retries = 0;;) {
5704 rc = pqi_lun_reset(ctrl_info, device);
c2922f17 5705 if (rc == 0 || ++retries > PQI_LUN_RESET_RETRIES)
3406384b
MR		 5706 break;
5707 msleep(PQI_LUN_RESET_RETRY_INTERVAL_MSECS);
5708 }
429fab70
KB		 5709
5710 timeout_secs = rc ? PQI_LUN_RESET_PENDING_IO_TIMEOUT_SECS : NO_TIMEOUT;
4fd22c13
MR		 5711
5712 rc |= pqi_device_wait_for_pending_io(ctrl_info, device, timeout_secs);
6c223761 5713
14bb215d 5714 return rc == 0 ? SUCCESS : FAILED;
6c223761
KB		 5715}
5716
4fd22c13
MR		 5717static int pqi_device_reset(struct pqi_ctrl_info *ctrl_info,
5718 struct pqi_scsi_dev *device)
5719{
5720 int rc;
5721
5722 mutex_lock(&ctrl_info->lun_reset_mutex);
5723
5724 pqi_ctrl_block_requests(ctrl_info);
5725 pqi_ctrl_wait_until_quiesced(ctrl_info);
5726 pqi_fail_io_queued_for_device(ctrl_info, device);
5727 rc = pqi_wait_until_inbound_queues_empty(ctrl_info);
5728 pqi_device_reset_start(device);
5729 pqi_ctrl_unblock_requests(ctrl_info);
5730
5731 if (rc)
5732 rc = FAILED;
5733 else
5734 rc = _pqi_device_reset(ctrl_info, device);
5735
5736 pqi_device_reset_done(device);
5737
5738 mutex_unlock(&ctrl_info->lun_reset_mutex);
429fab70 5739
4fd22c13
MR		 5740 return rc;
5741}
5742
6c223761
KB		 5743static int pqi_eh_device_reset_handler(struct scsi_cmnd *scmd)
5744{
5745 int rc;
7561a7e4 5746 struct Scsi_Host *shost;
6c223761
KB		 5747 struct pqi_ctrl_info *ctrl_info;
5748 struct pqi_scsi_dev *device;
5749
7561a7e4
KB		 5750 shost = scmd->device->host;
5751 ctrl_info = shost_to_hba(shost);
6c223761
KB		 5752 device = scmd->device->hostdata;
5753
5754 dev_err(&ctrl_info->pci_dev->dev,
5755 "resetting scsi %d:%d:%d:%d\n",
7561a7e4 5756 shost->host_no, device->bus, device->target, device->lun);
6c223761 5757
7561a7e4 5758 pqi_check_ctrl_health(ctrl_info);
0530736e
KB		 5759 if (pqi_ctrl_offline(ctrl_info) ||
5760 pqi_device_reset_blocked(ctrl_info)) {
7561a7e4
KB		 5761 rc = FAILED;
5762 goto out;
5763 }
6c223761 5764
4fd22c13 5765 pqi_wait_until_ofa_finished(ctrl_info);
7561a7e4 5766
0530736e 5767 atomic_inc(&ctrl_info->sync_cmds_outstanding);
4fd22c13 5768 rc = pqi_device_reset(ctrl_info, device);
0530736e 5769 atomic_dec(&ctrl_info->sync_cmds_outstanding);
429fab70 5770
7561a7e4 5771out:
6c223761
KB		 5772 dev_err(&ctrl_info->pci_dev->dev,
5773 "reset of scsi %d:%d:%d:%d: %s\n",
7561a7e4 5774 shost->host_no, device->bus, device->target, device->lun,
6c223761
KB		 5775 rc == SUCCESS ? "SUCCESS" : "FAILED");
5776
5777 return rc;
5778}
5779
5780static int pqi_slave_alloc(struct scsi_device *sdev)
5781{
5782 struct pqi_scsi_dev *device;
5783 unsigned long flags;
5784 struct pqi_ctrl_info *ctrl_info;
5785 struct scsi_target *starget;
5786 struct sas_rphy *rphy;
5787
5788 ctrl_info = shost_to_hba(sdev->host);
5789
5790 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
5791
5792 if (sdev_channel(sdev) == PQI_PHYSICAL_DEVICE_BUS) {
5793 starget = scsi_target(sdev);
5794 rphy = target_to_rphy(starget);
5795 device = pqi_find_device_by_sas_rphy(ctrl_info, rphy);
5796 if (device) {
5797 device->target = sdev_id(sdev);
5798 device->lun = sdev->lun;
5799 device->target_lun_valid = true;
5800 }
5801 } else {
5802 device = pqi_find_scsi_dev(ctrl_info, sdev_channel(sdev),
5803 sdev_id(sdev), sdev->lun);
5804 }
5805
94086f5b 5806 if (device) {
6c223761
KB		 5807 sdev->hostdata = device;
5808 device->sdev = sdev;
5809 if (device->queue_depth) {
5810 device->advertised_queue_depth = device->queue_depth;
5811 scsi_change_queue_depth(sdev,
5812 device->advertised_queue_depth);
5813 }
b6e2ef67
DC		 5814 if (pqi_is_logical_device(device))
5815 pqi_disable_write_same(sdev);
2b447f81
DC		 5816 else
5817 sdev->allow_restart = 1;
6c223761
KB		 5818 }
5819
5820 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
5821
5822 return 0;
5823}
5824
52198226
CH		 5825static int pqi_map_queues(struct Scsi_Host *shost)
5826{
5827 struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);
5828
79d3fa9e 5829 return blk_mq_pci_map_queues(&shost->tag_set.map[HCTX_TYPE_DEFAULT],
ed76e329 5830 ctrl_info->pci_dev, 0);
52198226
CH		 5831}
5832
ce143793
KB		 5833static int pqi_slave_configure(struct scsi_device *sdev)
5834{
5835 struct pqi_scsi_dev *device;
5836
5837 device = sdev->hostdata;
5838 device->devtype = sdev->type;
5839
5840 return 0;
5841}
5842
4d15ad38
KB		 5843static void pqi_slave_destroy(struct scsi_device *sdev)
5844{
5845 unsigned long flags;
5846 struct pqi_scsi_dev *device;
5847 struct pqi_ctrl_info *ctrl_info;
5848
5849 ctrl_info = shost_to_hba(sdev->host);
5850
5851 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
5852
5853 device = sdev->hostdata;
5854 if (device) {
5855 sdev->hostdata = NULL;
5856 if (!list_empty(&device->scsi_device_list_entry))
5857 list_del(&device->scsi_device_list_entry);
5858 }
5859
5860 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
5861
5862 if (device) {
5863 pqi_dev_info(ctrl_info, "removed", device);
5864 pqi_free_device(device);
5865 }
5866}
5867
8b664fef 5868static int pqi_getpciinfo_ioctl(struct pqi_ctrl_info *ctrl_info, void __user *arg)
6c223761
KB		 5869{
5870 struct pci_dev *pci_dev;
5871 u32 subsystem_vendor;
5872 u32 subsystem_device;
5873 cciss_pci_info_struct pciinfo;
5874
5875 if (!arg)
5876 return -EINVAL;
5877
5878 pci_dev = ctrl_info->pci_dev;
5879
5880 pciinfo.domain = pci_domain_nr(pci_dev->bus);
5881 pciinfo.bus = pci_dev->bus->number;
5882 pciinfo.dev_fn = pci_dev->devfn;
5883 subsystem_vendor = pci_dev->subsystem_vendor;
5884 subsystem_device = pci_dev->subsystem_device;
8b664fef 5885 pciinfo.board_id = ((subsystem_device << 16) & 0xffff0000) | subsystem_vendor;
6c223761
KB		 5886
5887 if (copy_to_user(arg, &pciinfo, sizeof(pciinfo)))
5888 return -EFAULT;
5889
5890 return 0;
5891}
5892
5893static int pqi_getdrivver_ioctl(void __user *arg)
5894{
5895 u32 version;
5896
5897 if (!arg)
5898 return -EINVAL;
5899
5900 version = (DRIVER_MAJOR << 28) | (DRIVER_MINOR << 24) |
5901 (DRIVER_RELEASE << 16) | DRIVER_REVISION;
5902
5903 if (copy_to_user(arg, &version, sizeof(version)))
5904 return -EFAULT;
5905
5906 return 0;
5907}
5908
5909struct ciss_error_info {
5910 u8 scsi_status;
5911 int command_status;
5912 size_t sense_data_length;
5913};
5914
5915static void pqi_error_info_to_ciss(struct pqi_raid_error_info *pqi_error_info,
5916 struct ciss_error_info *ciss_error_info)
5917{
5918 int ciss_cmd_status;
5919 size_t sense_data_length;
5920
5921 switch (pqi_error_info->data_out_result) {
5922 case PQI_DATA_IN_OUT_GOOD:
5923 ciss_cmd_status = CISS_CMD_STATUS_SUCCESS;
5924 break;
5925 case PQI_DATA_IN_OUT_UNDERFLOW:
5926 ciss_cmd_status = CISS_CMD_STATUS_DATA_UNDERRUN;
5927 break;
5928 case PQI_DATA_IN_OUT_BUFFER_OVERFLOW:
5929 ciss_cmd_status = CISS_CMD_STATUS_DATA_OVERRUN;
5930 break;
5931 case PQI_DATA_IN_OUT_PROTOCOL_ERROR:
5932 case PQI_DATA_IN_OUT_BUFFER_ERROR:
5933 case PQI_DATA_IN_OUT_BUFFER_OVERFLOW_DESCRIPTOR_AREA:
5934 case PQI_DATA_IN_OUT_BUFFER_OVERFLOW_BRIDGE:
5935 case PQI_DATA_IN_OUT_ERROR:
5936 ciss_cmd_status = CISS_CMD_STATUS_PROTOCOL_ERROR;
5937 break;
5938 case PQI_DATA_IN_OUT_HARDWARE_ERROR:
5939 case PQI_DATA_IN_OUT_PCIE_FABRIC_ERROR:
5940 case PQI_DATA_IN_OUT_PCIE_COMPLETION_TIMEOUT:
5941 case PQI_DATA_IN_OUT_PCIE_COMPLETER_ABORT_RECEIVED:
5942 case PQI_DATA_IN_OUT_PCIE_UNSUPPORTED_REQUEST_RECEIVED:
5943 case PQI_DATA_IN_OUT_PCIE_ECRC_CHECK_FAILED:
5944 case PQI_DATA_IN_OUT_PCIE_UNSUPPORTED_REQUEST:
5945 case PQI_DATA_IN_OUT_PCIE_ACS_VIOLATION:
5946 case PQI_DATA_IN_OUT_PCIE_TLP_PREFIX_BLOCKED:
5947 case PQI_DATA_IN_OUT_PCIE_POISONED_MEMORY_READ:
5948 ciss_cmd_status = CISS_CMD_STATUS_HARDWARE_ERROR;
5949 break;
5950 case PQI_DATA_IN_OUT_UNSOLICITED_ABORT:
5951 ciss_cmd_status = CISS_CMD_STATUS_UNSOLICITED_ABORT;
5952 break;
5953 case PQI_DATA_IN_OUT_ABORTED:
5954 ciss_cmd_status = CISS_CMD_STATUS_ABORTED;
5955 break;
5956 case PQI_DATA_IN_OUT_TIMEOUT:
5957 ciss_cmd_status = CISS_CMD_STATUS_TIMEOUT;
5958 break;
5959 default:
5960 ciss_cmd_status = CISS_CMD_STATUS_TARGET_STATUS;
5961 break;
5962 }
5963
5964 sense_data_length =
5965 get_unaligned_le16(&pqi_error_info->sense_data_length);
5966 if (sense_data_length == 0)
5967 sense_data_length =
5968 get_unaligned_le16(&pqi_error_info->response_data_length);
5969 if (sense_data_length)
5970 if (sense_data_length > sizeof(pqi_error_info->data))
5971 sense_data_length = sizeof(pqi_error_info->data);
5972
5973 ciss_error_info->scsi_status = pqi_error_info->status;
5974 ciss_error_info->command_status = ciss_cmd_status;
5975 ciss_error_info->sense_data_length = sense_data_length;
5976}
5977
5978static int pqi_passthru_ioctl(struct pqi_ctrl_info *ctrl_info, void __user *arg)
5979{
5980 int rc;
5981 char *kernel_buffer = NULL;
5982 u16 iu_length;
5983 size_t sense_data_length;
5984 IOCTL_Command_struct iocommand;
5985 struct pqi_raid_path_request request;
5986 struct pqi_raid_error_info pqi_error_info;
5987 struct ciss_error_info ciss_error_info;
5988
5989 if (pqi_ctrl_offline(ctrl_info))
5990 return -ENXIO;
5991 if (!arg)
5992 return -EINVAL;
5993 if (!capable(CAP_SYS_RAWIO))
5994 return -EPERM;
5995 if (copy_from_user(&iocommand, arg, sizeof(iocommand)))
5996 return -EFAULT;
5997 if (iocommand.buf_size < 1 &&
5998 iocommand.Request.Type.Direction != XFER_NONE)
5999 return -EINVAL;
6000 if (iocommand.Request.CDBLen > sizeof(request.cdb))
6001 return -EINVAL;
6002 if (iocommand.Request.Type.Type != TYPE_CMD)
6003 return -EINVAL;
6004
6005 switch (iocommand.Request.Type.Direction) {
6006 case XFER_NONE:
6007 case XFER_WRITE:
6008 case XFER_READ:
41555d54 6009 case XFER_READ | XFER_WRITE:
6c223761
KB		 6010 break;
6011 default:
6012 return -EINVAL;
6013 }
6014
6015 if (iocommand.buf_size > 0) {
6016 kernel_buffer = kmalloc(iocommand.buf_size, GFP_KERNEL);
6017 if (!kernel_buffer)
6018 return -ENOMEM;
6019 if (iocommand.Request.Type.Direction & XFER_WRITE) {
6020 if (copy_from_user(kernel_buffer, iocommand.buf,
6021 iocommand.buf_size)) {
6022 rc = -EFAULT;
6023 goto out;
6024 }
6025 } else {
6026 memset(kernel_buffer, 0, iocommand.buf_size);
6027 }
6028 }
6029
6030 memset(&request, 0, sizeof(request));
6031
6032 request.header.iu_type = PQI_REQUEST_IU_RAID_PATH_IO;
6033 iu_length = offsetof(struct pqi_raid_path_request, sg_descriptors) -
6034 PQI_REQUEST_HEADER_LENGTH;
6035 memcpy(request.lun_number, iocommand.LUN_info.LunAddrBytes,
6036 sizeof(request.lun_number));
6037 memcpy(request.cdb, iocommand.Request.CDB, iocommand.Request.CDBLen);
6038 request.additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_0;
6039
6040 switch (iocommand.Request.Type.Direction) {
6041 case XFER_NONE:
6042 request.data_direction = SOP_NO_DIRECTION_FLAG;
6043 break;
6044 case XFER_WRITE:
6045 request.data_direction = SOP_WRITE_FLAG;
6046 break;
6047 case XFER_READ:
6048 request.data_direction = SOP_READ_FLAG;
6049 break;
41555d54
KB		 6050 case XFER_READ | XFER_WRITE:
6051 request.data_direction = SOP_BIDIRECTIONAL;
6052 break;
6c223761
KB		 6053 }
6054
6055 request.task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
6056
6057 if (iocommand.buf_size > 0) {
6058 put_unaligned_le32(iocommand.buf_size, &request.buffer_length);
6059
6060 rc = pqi_map_single(ctrl_info->pci_dev,
6061 &request.sg_descriptors[0], kernel_buffer,
6917a9cc 6062 iocommand.buf_size, DMA_BIDIRECTIONAL);
6c223761
KB		 6063 if (rc)
6064 goto out;
6065
6066 iu_length += sizeof(request.sg_descriptors[0]);
6067 }
6068
6069 put_unaligned_le16(iu_length, &request.header.iu_length);
6070
21432010 6071 if (ctrl_info->raid_iu_timeout_supported)
6072 put_unaligned_le32(iocommand.Request.Timeout, &request.timeout);
6073
6c223761
KB		 6074 rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header,
6075 PQI_SYNC_FLAGS_INTERRUPTABLE, &pqi_error_info, NO_TIMEOUT);
6076
6077 if (iocommand.buf_size > 0)
6078 pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1,
6917a9cc 6079 DMA_BIDIRECTIONAL);
6c223761
KB		 6080
6081 memset(&iocommand.error_info, 0, sizeof(iocommand.error_info));
6082
6083 if (rc == 0) {
6084 pqi_error_info_to_ciss(&pqi_error_info, &ciss_error_info);
6085 iocommand.error_info.ScsiStatus = ciss_error_info.scsi_status;
6086 iocommand.error_info.CommandStatus =
6087 ciss_error_info.command_status;
6088 sense_data_length = ciss_error_info.sense_data_length;
6089 if (sense_data_length) {
6090 if (sense_data_length >
6091 sizeof(iocommand.error_info.SenseInfo))
6092 sense_data_length =
6093 sizeof(iocommand.error_info.SenseInfo);
6094 memcpy(iocommand.error_info.SenseInfo,
6095 pqi_error_info.data, sense_data_length);
6096 iocommand.error_info.SenseLen = sense_data_length;
6097 }
6098 }
6099
6100 if (copy_to_user(arg, &iocommand, sizeof(iocommand))) {
6101 rc = -EFAULT;
6102 goto out;
6103 }
6104
6105 if (rc == 0 && iocommand.buf_size > 0 &&
6106 (iocommand.Request.Type.Direction & XFER_READ)) {
6107 if (copy_to_user(iocommand.buf, kernel_buffer,
6108 iocommand.buf_size)) {
6109 rc = -EFAULT;
6110 }
6111 }
6112
6113out:
6114 kfree(kernel_buffer);
6115
6116 return rc;
6117}
6118
6f4e626f
NC		 6119static int pqi_ioctl(struct scsi_device *sdev, unsigned int cmd,
6120 void __user *arg)
6c223761
KB		 6121{
6122 int rc;
6123 struct pqi_ctrl_info *ctrl_info;
6124
6125 ctrl_info = shost_to_hba(sdev->host);
6126
694c5d5b 6127 if (pqi_ctrl_in_ofa(ctrl_info) || pqi_ctrl_in_shutdown(ctrl_info))
4fd22c13
MR		 6128 return -EBUSY;
6129
6c223761
KB		 6130 switch (cmd) {
6131 case CCISS_DEREGDISK:
6132 case CCISS_REGNEWDISK:
6133 case CCISS_REGNEWD:
6134 rc = pqi_scan_scsi_devices(ctrl_info);
6135 break;
6136 case CCISS_GETPCIINFO:
6137 rc = pqi_getpciinfo_ioctl(ctrl_info, arg);
6138 break;
6139 case CCISS_GETDRIVVER:
6140 rc = pqi_getdrivver_ioctl(arg);
6141 break;
6142 case CCISS_PASSTHRU:
6143 rc = pqi_passthru_ioctl(ctrl_info, arg);
6144 break;
6145 default:
6146 rc = -EINVAL;
6147 break;
6148 }
6149
6150 return rc;
6151}
6152
6d90615f 6153static ssize_t pqi_firmware_version_show(struct device *dev,
6c223761
KB		 6154 struct device_attribute *attr, char *buffer)
6155{
6c223761
KB		 6156 struct Scsi_Host *shost;
6157 struct pqi_ctrl_info *ctrl_info;
6158
6159 shost = class_to_shost(dev);
6160 ctrl_info = shost_to_hba(shost);
6161
6d90615f
MB		 6162 return snprintf(buffer, PAGE_SIZE, "%s\n", ctrl_info->firmware_version);
6163}
6164
6165static ssize_t pqi_driver_version_show(struct device *dev,
6166 struct device_attribute *attr, char *buffer)
6167{
694c5d5b
KB		 6168 return snprintf(buffer, PAGE_SIZE, "%s\n",
6169 DRIVER_VERSION BUILD_TIMESTAMP);
6d90615f 6170}
6c223761 6171
6d90615f
MB		 6172static ssize_t pqi_serial_number_show(struct device *dev,
6173 struct device_attribute *attr, char *buffer)
6174{
6175 struct Scsi_Host *shost;
6176 struct pqi_ctrl_info *ctrl_info;
6177
6178 shost = class_to_shost(dev);
6179 ctrl_info = shost_to_hba(shost);
6180
6181 return snprintf(buffer, PAGE_SIZE, "%s\n", ctrl_info->serial_number);
6182}
6183
6184static ssize_t pqi_model_show(struct device *dev,
6185 struct device_attribute *attr, char *buffer)
6186{
6187 struct Scsi_Host *shost;
6188 struct pqi_ctrl_info *ctrl_info;
6189
6190 shost = class_to_shost(dev);
6191 ctrl_info = shost_to_hba(shost);
6192
6193 return snprintf(buffer, PAGE_SIZE, "%s\n", ctrl_info->model);
6194}
6195
6196static ssize_t pqi_vendor_show(struct device *dev,
6197 struct device_attribute *attr, char *buffer)
6198{
6199 struct Scsi_Host *shost;
6200 struct pqi_ctrl_info *ctrl_info;
6201
6202 shost = class_to_shost(dev);
6203 ctrl_info = shost_to_hba(shost);
6204
6205 return snprintf(buffer, PAGE_SIZE, "%s\n", ctrl_info->vendor);
6c223761
KB		 6206}
6207
6208static ssize_t pqi_host_rescan_store(struct device *dev,
6209 struct device_attribute *attr, const char *buffer, size_t count)
6210{
6211 struct Scsi_Host *shost = class_to_shost(dev);
6212
6213 pqi_scan_start(shost);
6214
6215 return count;
6216}
6217
3c50976f
KB		 6218static ssize_t pqi_lockup_action_show(struct device *dev,
6219 struct device_attribute *attr, char *buffer)
6220{
6221 int count = 0;
6222 unsigned int i;
6223
6224 for (i = 0; i < ARRAY_SIZE(pqi_lockup_actions); i++) {
6225 if (pqi_lockup_actions[i].action == pqi_lockup_action)
181aea89 6226 count += scnprintf(buffer + count, PAGE_SIZE - count,
3c50976f
KB		 6227 "[%s] ", pqi_lockup_actions[i].name);
6228 else
181aea89 6229 count += scnprintf(buffer + count, PAGE_SIZE - count,
3c50976f
KB		 6230 "%s ", pqi_lockup_actions[i].name);
6231 }
6232
181aea89 6233 count += scnprintf(buffer + count, PAGE_SIZE - count, "\n");
3c50976f
KB		 6234
6235 return count;
6236}
6237
6238static ssize_t pqi_lockup_action_store(struct device *dev,
6239 struct device_attribute *attr, const char *buffer, size_t count)
6240{
6241 unsigned int i;
6242 char *action_name;
6243 char action_name_buffer[32];
6244
6245 strlcpy(action_name_buffer, buffer, sizeof(action_name_buffer));
6246 action_name = strstrip(action_name_buffer);
6247
6248 for (i = 0; i < ARRAY_SIZE(pqi_lockup_actions); i++) {
6249 if (strcmp(action_name, pqi_lockup_actions[i].name) == 0) {
6250 pqi_lockup_action = pqi_lockup_actions[i].action;
6251 return count;
6252 }
6253 }
6254
6255 return -EINVAL;
6256}
6257
6d90615f
MB		 6258static DEVICE_ATTR(driver_version, 0444, pqi_driver_version_show, NULL);
6259static DEVICE_ATTR(firmware_version, 0444, pqi_firmware_version_show, NULL);
6260static DEVICE_ATTR(model, 0444, pqi_model_show, NULL);
6261static DEVICE_ATTR(serial_number, 0444, pqi_serial_number_show, NULL);
6262static DEVICE_ATTR(vendor, 0444, pqi_vendor_show, NULL);
cbe0c7b1 6263static DEVICE_ATTR(rescan, 0200, NULL, pqi_host_rescan_store);
3c50976f
KB		 6264static DEVICE_ATTR(lockup_action, 0644,
6265 pqi_lockup_action_show, pqi_lockup_action_store);
6c223761
KB		 6266
6267static struct device_attribute *pqi_shost_attrs[] = {
6d90615f
MB		 6268 &dev_attr_driver_version,
6269 &dev_attr_firmware_version,
6270 &dev_attr_model,
6271 &dev_attr_serial_number,
6272 &dev_attr_vendor,
6c223761 6273 &dev_attr_rescan,
3c50976f 6274 &dev_attr_lockup_action,
6c223761
KB		 6275 NULL
6276};
6277
cd128244
DC		 6278static ssize_t pqi_unique_id_show(struct device *dev,
6279 struct device_attribute *attr, char *buffer)
6280{
6281 struct pqi_ctrl_info *ctrl_info;
6282 struct scsi_device *sdev;
6283 struct pqi_scsi_dev *device;
6284 unsigned long flags;
5b083b30 6285 u8 unique_id[16];
cd128244
DC		 6286
6287 sdev = to_scsi_device(dev);
6288 ctrl_info = shost_to_hba(sdev->host);
6289
6290 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
6291
6292 device = sdev->hostdata;
6293 if (!device) {
8b664fef 6294 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
cd128244
DC		 6295 return -ENODEV;
6296 }
5b083b30
KB		 6297
6298 if (device->is_physical_device) {
6299 memset(unique_id, 0, 8);
6300 memcpy(unique_id + 8, &device->wwid, sizeof(device->wwid));
6301 } else {
6302 memcpy(unique_id, device->volume_id, sizeof(device->volume_id));
6303 }
cd128244
DC		 6304
6305 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
6306
5995b236
MB		 6307 return snprintf(buffer, PAGE_SIZE,
6308 "%02X%02X%02X%02X%02X%02X%02X%02X%02X%02X%02X%02X%02X%02X%02X%02X\n",
5b083b30
KB		 6309 unique_id[0], unique_id[1], unique_id[2], unique_id[3],
6310 unique_id[4], unique_id[5], unique_id[6], unique_id[7],
6311 unique_id[8], unique_id[9], unique_id[10], unique_id[11],
6312 unique_id[12], unique_id[13], unique_id[14], unique_id[15]);
cd128244
DC		 6313}
6314
6315static ssize_t pqi_lunid_show(struct device *dev,
6316 struct device_attribute *attr, char *buffer)
6317{
6318 struct pqi_ctrl_info *ctrl_info;
6319 struct scsi_device *sdev;
6320 struct pqi_scsi_dev *device;
6321 unsigned long flags;
6322 u8 lunid[8];
6323
6324 sdev = to_scsi_device(dev);
6325 ctrl_info = shost_to_hba(sdev->host);
6326
6327 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
6328
6329 device = sdev->hostdata;
6330 if (!device) {
8b664fef 6331 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
cd128244
DC		 6332 return -ENODEV;
6333 }
694c5d5b 6334
cd128244
DC		 6335 memcpy(lunid, device->scsi3addr, sizeof(lunid));
6336
6337 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
6338
6339 return snprintf(buffer, PAGE_SIZE, "0x%8phN\n", lunid);
6340}
6341
694c5d5b
KB		 6342#define MAX_PATHS 8
6343
cd128244
DC		 6344static ssize_t pqi_path_info_show(struct device *dev,
6345 struct device_attribute *attr, char *buf)
6346{
6347 struct pqi_ctrl_info *ctrl_info;
6348 struct scsi_device *sdev;
6349 struct pqi_scsi_dev *device;
6350 unsigned long flags;
6351 int i;
6352 int output_len = 0;
6353 u8 box;
6354 u8 bay;
694c5d5b 6355 u8 path_map_index;
cd128244 6356 char *active;
694c5d5b 6357 u8 phys_connector[2];
cd128244
DC		 6358
6359 sdev = to_scsi_device(dev);
6360 ctrl_info = shost_to_hba(sdev->host);
6361
6362 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
6363
6364 device = sdev->hostdata;
6365 if (!device) {
8b664fef 6366 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
cd128244
DC		 6367 return -ENODEV;
6368 }
6369
6370 bay = device->bay;
6371 for (i = 0; i < MAX_PATHS; i++) {
694c5d5b 6372 path_map_index = 1 << i;
cd128244
DC		 6373 if (i == device->active_path_index)
6374 active = "Active";
6375 else if (device->path_map & path_map_index)
6376 active = "Inactive";
6377 else
6378 continue;
6379
6380 output_len += scnprintf(buf + output_len,
6381 PAGE_SIZE - output_len,
6382 "[%d:%d:%d:%d] %20.20s ",
6383 ctrl_info->scsi_host->host_no,
6384 device->bus, device->target,
6385 device->lun,
6386 scsi_device_type(device->devtype));
6387
6388 if (device->devtype == TYPE_RAID ||
6389 pqi_is_logical_device(device))
6390 goto end_buffer;
6391
6392 memcpy(&phys_connector, &device->phys_connector[i],
6393 sizeof(phys_connector));
6394 if (phys_connector[0] < '0')
6395 phys_connector[0] = '0';
6396 if (phys_connector[1] < '0')
6397 phys_connector[1] = '0';
6398
6399 output_len += scnprintf(buf + output_len,
6400 PAGE_SIZE - output_len,
6401 "PORT: %.2s ", phys_connector);
6402
6403 box = device->box[i];
6404 if (box != 0 && box != 0xFF)
6405 output_len += scnprintf(buf + output_len,
6406 PAGE_SIZE - output_len,
6407 "BOX: %hhu ", box);
6408
6409 if ((device->devtype == TYPE_DISK ||
6410 device->devtype == TYPE_ZBC) &&
6411 pqi_expose_device(device))
6412 output_len += scnprintf(buf + output_len,
6413 PAGE_SIZE - output_len,
6414 "BAY: %hhu ", bay);
6415
6416end_buffer:
6417 output_len += scnprintf(buf + output_len,
6418 PAGE_SIZE - output_len,
6419 "%s\n", active);
6420 }
6421
6422 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
694c5d5b 6423
cd128244
DC		 6424 return output_len;
6425}
6426
6c223761
KB		 6427static ssize_t pqi_sas_address_show(struct device *dev,
6428 struct device_attribute *attr, char *buffer)
6429{
6430 struct pqi_ctrl_info *ctrl_info;
6431 struct scsi_device *sdev;
6432 struct pqi_scsi_dev *device;
6433 unsigned long flags;
6434 u64 sas_address;
6435
6436 sdev = to_scsi_device(dev);
6437 ctrl_info = shost_to_hba(sdev->host);
6438
6439 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
6440
6441 device = sdev->hostdata;
8b664fef
KB		 6442 if (!device || !pqi_is_device_with_sas_address(device)) {
6443 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
6c223761
KB		 6444 return -ENODEV;
6445 }
694c5d5b 6446
6c223761
KB		 6447 sas_address = device->sas_address;
6448
6449 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
6450
6451 return snprintf(buffer, PAGE_SIZE, "0x%016llx\n", sas_address);
6452}
6453
6454static ssize_t pqi_ssd_smart_path_enabled_show(struct device *dev,
6455 struct device_attribute *attr, char *buffer)
6456{
6457 struct pqi_ctrl_info *ctrl_info;
6458 struct scsi_device *sdev;
6459 struct pqi_scsi_dev *device;
6460 unsigned long flags;
6461
6462 sdev = to_scsi_device(dev);
6463 ctrl_info = shost_to_hba(sdev->host);
6464
6465 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
6466
6467 device = sdev->hostdata;
8b664fef
KB		 6468 if (!device) {
6469 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
6470 return -ENODEV;
6471 }
6472
588a63fe 6473 buffer[0] = device->raid_bypass_enabled ? '1' : '0';
6c223761
KB		 6474 buffer[1] = '\n';
6475 buffer[2] = '\0';
6476
6477 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
6478
6479 return 2;
6480}
6481
a9f93392
KB		 6482static ssize_t pqi_raid_level_show(struct device *dev,
6483 struct device_attribute *attr, char *buffer)
6484{
6485 struct pqi_ctrl_info *ctrl_info;
6486 struct scsi_device *sdev;
6487 struct pqi_scsi_dev *device;
6488 unsigned long flags;
6489 char *raid_level;
6490
6491 sdev = to_scsi_device(dev);
6492 ctrl_info = shost_to_hba(sdev->host);
6493
6494 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
6495
6496 device = sdev->hostdata;
8b664fef
KB		 6497 if (!device) {
6498 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
6499 return -ENODEV;
6500 }
a9f93392
KB		 6501
6502 if (pqi_is_logical_device(device))
6503 raid_level = pqi_raid_level_to_string(device->raid_level);
6504 else
6505 raid_level = "N/A";
6506
6507 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
6508
6509 return snprintf(buffer, PAGE_SIZE, "%s\n", raid_level);
6510}
6511
8b664fef
KB		 6512static ssize_t pqi_raid_bypass_cnt_show(struct device *dev,
6513 struct device_attribute *attr, char *buffer)
6514{
6515 struct pqi_ctrl_info *ctrl_info;
6516 struct scsi_device *sdev;
6517 struct pqi_scsi_dev *device;
6518 unsigned long flags;
6519 int raid_bypass_cnt;
6520
6521 sdev = to_scsi_device(dev);
6522 ctrl_info = shost_to_hba(sdev->host);
6523
6524 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
6525
6526 device = sdev->hostdata;
6527 if (!device) {
6528 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
6529 return -ENODEV;
6530 }
6531
6532 raid_bypass_cnt = atomic_read(&device->raid_bypass_cnt);
6533
6534 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
6535
6536 return snprintf(buffer, PAGE_SIZE, "0x%x\n", raid_bypass_cnt);
6537}
6538
cd128244
DC		 6539static DEVICE_ATTR(lunid, 0444, pqi_lunid_show, NULL);
6540static DEVICE_ATTR(unique_id, 0444, pqi_unique_id_show, NULL);
6541static DEVICE_ATTR(path_info, 0444, pqi_path_info_show, NULL);
cbe0c7b1 6542static DEVICE_ATTR(sas_address, 0444, pqi_sas_address_show, NULL);
8b664fef 6543static DEVICE_ATTR(ssd_smart_path_enabled, 0444, pqi_ssd_smart_path_enabled_show, NULL);
a9f93392 6544static DEVICE_ATTR(raid_level, 0444, pqi_raid_level_show, NULL);
8b664fef 6545static DEVICE_ATTR(raid_bypass_cnt, 0444, pqi_raid_bypass_cnt_show, NULL);
6c223761
KB		 6546
6547static struct device_attribute *pqi_sdev_attrs[] = {
cd128244
DC		 6548 &dev_attr_lunid,
6549 &dev_attr_unique_id,
6550 &dev_attr_path_info,
6c223761
KB		 6551 &dev_attr_sas_address,
6552 &dev_attr_ssd_smart_path_enabled,
a9f93392 6553 &dev_attr_raid_level,
8b664fef 6554 &dev_attr_raid_bypass_cnt,
6c223761
KB		 6555 NULL
6556};
6557
6558static struct scsi_host_template pqi_driver_template = {
6559 .module = THIS_MODULE,
6560 .name = DRIVER_NAME_SHORT,
6561 .proc_name = DRIVER_NAME_SHORT,
6562 .queuecommand = pqi_scsi_queue_command,
6563 .scan_start = pqi_scan_start,
6564 .scan_finished = pqi_scan_finished,
6565 .this_id = -1,
6c223761
KB		 6566 .eh_device_reset_handler = pqi_eh_device_reset_handler,
6567 .ioctl = pqi_ioctl,
6568 .slave_alloc = pqi_slave_alloc,
ce143793 6569 .slave_configure = pqi_slave_configure,
4d15ad38 6570 .slave_destroy = pqi_slave_destroy,
52198226 6571 .map_queues = pqi_map_queues,
6c223761
KB		 6572 .sdev_attrs = pqi_sdev_attrs,
6573 .shost_attrs = pqi_shost_attrs,
6574};
6575
6576static int pqi_register_scsi(struct pqi_ctrl_info *ctrl_info)
6577{
6578 int rc;
6579 struct Scsi_Host *shost;
6580
6581 shost = scsi_host_alloc(&pqi_driver_template, sizeof(ctrl_info));
6582 if (!shost) {
6583 dev_err(&ctrl_info->pci_dev->dev,
6584 "scsi_host_alloc failed for controller %u\n",
6585 ctrl_info->ctrl_id);
6586 return -ENOMEM;
6587 }
6588
6589 shost->io_port = 0;
6590 shost->n_io_port = 0;
6591 shost->this_id = -1;
6592 shost->max_channel = PQI_MAX_BUS;
6593 shost->max_cmd_len = MAX_COMMAND_SIZE;
6594 shost->max_lun = ~0;
6595 shost->max_id = ~0;
6596 shost->max_sectors = ctrl_info->max_sectors;
6597 shost->can_queue = ctrl_info->scsi_ml_can_queue;
6598 shost->cmd_per_lun = shost->can_queue;
6599 shost->sg_tablesize = ctrl_info->sg_tablesize;
6600 shost->transportt = pqi_sas_transport_template;
52198226 6601 shost->irq = pci_irq_vector(ctrl_info->pci_dev, 0);
6c223761
KB		 6602 shost->unique_id = shost->irq;
6603 shost->nr_hw_queues = ctrl_info->num_queue_groups;
6604 shost->hostdata[0] = (unsigned long)ctrl_info;
6605
6606 rc = scsi_add_host(shost, &ctrl_info->pci_dev->dev);
6607 if (rc) {
6608 dev_err(&ctrl_info->pci_dev->dev,
6609 "scsi_add_host failed for controller %u\n",
6610 ctrl_info->ctrl_id);
6611 goto free_host;
6612 }
6613
6614 rc = pqi_add_sas_host(shost, ctrl_info);
6615 if (rc) {
6616 dev_err(&ctrl_info->pci_dev->dev,
6617 "add SAS host failed for controller %u\n",
6618 ctrl_info->ctrl_id);
6619 goto remove_host;
6620 }
6621
6622 ctrl_info->scsi_host = shost;
6623
6624 return 0;
6625
6626remove_host:
6627 scsi_remove_host(shost);
6628free_host:
6629 scsi_host_put(shost);
6630
6631 return rc;
6632}
6633
6634static void pqi_unregister_scsi(struct pqi_ctrl_info *ctrl_info)
6635{
6636 struct Scsi_Host *shost;
6637
6638 pqi_delete_sas_host(ctrl_info);
6639
6640 shost = ctrl_info->scsi_host;
6641 if (!shost)
6642 return;
6643
6644 scsi_remove_host(shost);
6645 scsi_host_put(shost);
6646}
6647
336b6819
KB		 6648static int pqi_wait_for_pqi_reset_completion(struct pqi_ctrl_info *ctrl_info)
6649{
6650 int rc = 0;
6651 struct pqi_device_registers __iomem *pqi_registers;
6652 unsigned long timeout;
6653 unsigned int timeout_msecs;
6654 union pqi_reset_register reset_reg;
6c223761 6655
336b6819
KB		 6656 pqi_registers = ctrl_info->pqi_registers;
6657 timeout_msecs = readw(&pqi_registers->max_reset_timeout) * 100;
6658 timeout = msecs_to_jiffies(timeout_msecs) + jiffies;
6659
6660 while (1) {
6661 msleep(PQI_RESET_POLL_INTERVAL_MSECS);
6662 reset_reg.all_bits = readl(&pqi_registers->device_reset);
6663 if (reset_reg.bits.reset_action == PQI_RESET_ACTION_COMPLETED)
6664 break;
6665 pqi_check_ctrl_health(ctrl_info);
6666 if (pqi_ctrl_offline(ctrl_info)) {
6667 rc = -ENXIO;
6668 break;
6669 }
6670 if (time_after(jiffies, timeout)) {
6671 rc = -ETIMEDOUT;
6672 break;
6673 }
6674 }
6675
6676 return rc;
6677}
6c223761
KB		 6678
6679static int pqi_reset(struct pqi_ctrl_info *ctrl_info)
6680{
6681 int rc;
336b6819
KB		 6682 union pqi_reset_register reset_reg;
6683
6684 if (ctrl_info->pqi_reset_quiesce_supported) {
6685 rc = sis_pqi_reset_quiesce(ctrl_info);
6686 if (rc) {
6687 dev_err(&ctrl_info->pci_dev->dev,
6688 "PQI reset failed during quiesce with error %d\n",
6689 rc);
6690 return rc;
6691 }
6692 }
6c223761 6693
336b6819
KB		 6694 reset_reg.all_bits = 0;
6695 reset_reg.bits.reset_type = PQI_RESET_TYPE_HARD_RESET;
6696 reset_reg.bits.reset_action = PQI_RESET_ACTION_RESET;
6c223761 6697
336b6819 6698 writel(reset_reg.all_bits, &ctrl_info->pqi_registers->device_reset);
6c223761 6699
336b6819 6700 rc = pqi_wait_for_pqi_reset_completion(ctrl_info);
6c223761
KB		 6701 if (rc)
6702 dev_err(&ctrl_info->pci_dev->dev,
336b6819 6703 "PQI reset failed with error %d\n", rc);
6c223761
KB		 6704
6705 return rc;
6706}
6707
6d90615f
MB		 6708static int pqi_get_ctrl_serial_number(struct pqi_ctrl_info *ctrl_info)
6709{
6710 int rc;
6711 struct bmic_sense_subsystem_info *sense_info;
6712
6713 sense_info = kzalloc(sizeof(*sense_info), GFP_KERNEL);
6714 if (!sense_info)
6715 return -ENOMEM;
6716
6717 rc = pqi_sense_subsystem_info(ctrl_info, sense_info);
6718 if (rc)
6719 goto out;
6720
6721 memcpy(ctrl_info->serial_number, sense_info->ctrl_serial_number,
6722 sizeof(sense_info->ctrl_serial_number));
6723 ctrl_info->serial_number[sizeof(sense_info->ctrl_serial_number)] = '\0';
6724
6725out:
6726 kfree(sense_info);
6727
6728 return rc;
6729}
6730
6731static int pqi_get_ctrl_product_details(struct pqi_ctrl_info *ctrl_info)
6c223761
KB		 6732{
6733 int rc;
6734 struct bmic_identify_controller *identify;
6735
6736 identify = kmalloc(sizeof(*identify), GFP_KERNEL);
6737 if (!identify)
6738 return -ENOMEM;
6739
6740 rc = pqi_identify_controller(ctrl_info, identify);
6741 if (rc)
6742 goto out;
6743
6744 memcpy(ctrl_info->firmware_version, identify->firmware_version,
6745 sizeof(identify->firmware_version));
6746 ctrl_info->firmware_version[sizeof(identify->firmware_version)] = '\0';
6747 snprintf(ctrl_info->firmware_version +
6748 strlen(ctrl_info->firmware_version),
6749 sizeof(ctrl_info->firmware_version),
6750 "-%u", get_unaligned_le16(&identify->firmware_build_number));
6751
6d90615f
MB		 6752 memcpy(ctrl_info->model, identify->product_id,
6753 sizeof(identify->product_id));
6754 ctrl_info->model[sizeof(identify->product_id)] = '\0';
6755
6756 memcpy(ctrl_info->vendor, identify->vendor_id,
6757 sizeof(identify->vendor_id));
6758 ctrl_info->vendor[sizeof(identify->vendor_id)] = '\0';
6759
6c223761
KB		 6760out:
6761 kfree(identify);
6762
6763 return rc;
6764}
6765
b212c251
KB		 6766struct pqi_config_table_section_info {
6767 struct pqi_ctrl_info *ctrl_info;
6768 void *section;
6769 u32 section_offset;
6770 void __iomem *section_iomem_addr;
6771};
6772
6773static inline bool pqi_is_firmware_feature_supported(
6774 struct pqi_config_table_firmware_features *firmware_features,
6775 unsigned int bit_position)
98f87667 6776{
b212c251 6777 unsigned int byte_index;
98f87667 6778
b212c251 6779 byte_index = bit_position / BITS_PER_BYTE;
98f87667 6780
b212c251
KB		 6781 if (byte_index >= le16_to_cpu(firmware_features->num_elements))
6782 return false;
98f87667 6783
b212c251
KB		 6784 return firmware_features->features_supported[byte_index] &
6785 (1 << (bit_position % BITS_PER_BYTE)) ? true : false;
6786}
6787
6788static inline bool pqi_is_firmware_feature_enabled(
6789 struct pqi_config_table_firmware_features *firmware_features,
6790 void __iomem *firmware_features_iomem_addr,
6791 unsigned int bit_position)
6792{
6793 unsigned int byte_index;
6794 u8 __iomem *features_enabled_iomem_addr;
6795
6796 byte_index = (bit_position / BITS_PER_BYTE) +
6797 (le16_to_cpu(firmware_features->num_elements) * 2);
6798
6799 features_enabled_iomem_addr = firmware_features_iomem_addr +
6800 offsetof(struct pqi_config_table_firmware_features,
6801 features_supported) + byte_index;
6802
6803 return *((__force u8 *)features_enabled_iomem_addr) &
6804 (1 << (bit_position % BITS_PER_BYTE)) ? true : false;
6805}
6806
6807static inline void pqi_request_firmware_feature(
6808 struct pqi_config_table_firmware_features *firmware_features,
6809 unsigned int bit_position)
6810{
6811 unsigned int byte_index;
6812
6813 byte_index = (bit_position / BITS_PER_BYTE) +
6814 le16_to_cpu(firmware_features->num_elements);
6815
6816 firmware_features->features_supported[byte_index] |=
6817 (1 << (bit_position % BITS_PER_BYTE));
6818}
6819
6820static int pqi_config_table_update(struct pqi_ctrl_info *ctrl_info,
6821 u16 first_section, u16 last_section)
6822{
6823 struct pqi_vendor_general_request request;
6824
6825 memset(&request, 0, sizeof(request));
6826
6827 request.header.iu_type = PQI_REQUEST_IU_VENDOR_GENERAL;
6828 put_unaligned_le16(sizeof(request) - PQI_REQUEST_HEADER_LENGTH,
6829 &request.header.iu_length);
6830 put_unaligned_le16(PQI_VENDOR_GENERAL_CONFIG_TABLE_UPDATE,
6831 &request.function_code);
6832 put_unaligned_le16(first_section,
6833 &request.data.config_table_update.first_section);
6834 put_unaligned_le16(last_section,
6835 &request.data.config_table_update.last_section);
6836
6837 return pqi_submit_raid_request_synchronous(ctrl_info, &request.header,
6838 0, NULL, NO_TIMEOUT);
6839}
6840
6841static int pqi_enable_firmware_features(struct pqi_ctrl_info *ctrl_info,
6842 struct pqi_config_table_firmware_features *firmware_features,
6843 void __iomem *firmware_features_iomem_addr)
6844{
6845 void *features_requested;
6846 void __iomem *features_requested_iomem_addr;
6847
6848 features_requested = firmware_features->features_supported +
6849 le16_to_cpu(firmware_features->num_elements);
6850
6851 features_requested_iomem_addr = firmware_features_iomem_addr +
6852 (features_requested - (void *)firmware_features);
6853
6854 memcpy_toio(features_requested_iomem_addr, features_requested,
6855 le16_to_cpu(firmware_features->num_elements));
6856
6857 return pqi_config_table_update(ctrl_info,
6858 PQI_CONFIG_TABLE_SECTION_FIRMWARE_FEATURES,
6859 PQI_CONFIG_TABLE_SECTION_FIRMWARE_FEATURES);
6860}
6861
6862struct pqi_firmware_feature {
6863 char *feature_name;
6864 unsigned int feature_bit;
6865 bool supported;
6866 bool enabled;
6867 void (*feature_status)(struct pqi_ctrl_info *ctrl_info,
6868 struct pqi_firmware_feature *firmware_feature);
6869};
6870
6871static void pqi_firmware_feature_status(struct pqi_ctrl_info *ctrl_info,
6872 struct pqi_firmware_feature *firmware_feature)
6873{
6874 if (!firmware_feature->supported) {
6875 dev_info(&ctrl_info->pci_dev->dev, "%s not supported by controller\n",
6876 firmware_feature->feature_name);
6877 return;
6878 }
6879
6880 if (firmware_feature->enabled) {
6881 dev_info(&ctrl_info->pci_dev->dev,
6882 "%s enabled\n", firmware_feature->feature_name);
6883 return;
6884 }
6885
6886 dev_err(&ctrl_info->pci_dev->dev, "failed to enable %s\n",
6887 firmware_feature->feature_name);
6888}
6889
21432010 6890static void pqi_ctrl_update_feature_flags(struct pqi_ctrl_info *ctrl_info,
6891 struct pqi_firmware_feature *firmware_feature)
6892{
6893 switch (firmware_feature->feature_bit) {
6894 case PQI_FIRMWARE_FEATURE_SOFT_RESET_HANDSHAKE:
6895 ctrl_info->soft_reset_handshake_supported =
6896 firmware_feature->enabled;
6897 break;
6898 case PQI_FIRMWARE_FEATURE_RAID_IU_TIMEOUT:
6899 ctrl_info->raid_iu_timeout_supported =
6900 firmware_feature->enabled;
6901 break;
c2922f17
MB		 6902 case PQI_FIRMWARE_FEATURE_TMF_IU_TIMEOUT:
6903 ctrl_info->tmf_iu_timeout_supported =
6904 firmware_feature->enabled;
6905 break;
21432010 6906 }
6907
6908 pqi_firmware_feature_status(ctrl_info, firmware_feature);
6909}
6910
b212c251
KB		 6911static inline void pqi_firmware_feature_update(struct pqi_ctrl_info *ctrl_info,
6912 struct pqi_firmware_feature *firmware_feature)
6913{
6914 if (firmware_feature->feature_status)
6915 firmware_feature->feature_status(ctrl_info, firmware_feature);
6916}
6917
6918static DEFINE_MUTEX(pqi_firmware_features_mutex);
6919
6920static struct pqi_firmware_feature pqi_firmware_features[] = {
6921 {
6922 .feature_name = "Online Firmware Activation",
6923 .feature_bit = PQI_FIRMWARE_FEATURE_OFA,
6924 .feature_status = pqi_firmware_feature_status,
6925 },
6926 {
6927 .feature_name = "Serial Management Protocol",
6928 .feature_bit = PQI_FIRMWARE_FEATURE_SMP,
6929 .feature_status = pqi_firmware_feature_status,
6930 },
4fd22c13
MR		 6931 {
6932 .feature_name = "New Soft Reset Handshake",
6933 .feature_bit = PQI_FIRMWARE_FEATURE_SOFT_RESET_HANDSHAKE,
21432010 6934 .feature_status = pqi_ctrl_update_feature_flags,
6935 },
6936 {
6937 .feature_name = "RAID IU Timeout",
6938 .feature_bit = PQI_FIRMWARE_FEATURE_RAID_IU_TIMEOUT,
6939 .feature_status = pqi_ctrl_update_feature_flags,
4fd22c13 6940 },
c2922f17
MB		 6941 {
6942 .feature_name = "TMF IU Timeout",
6943 .feature_bit = PQI_FIRMWARE_FEATURE_TMF_IU_TIMEOUT,
6944 .feature_status = pqi_ctrl_update_feature_flags,
6945 },
b212c251
KB		 6946};
6947
6948static void pqi_process_firmware_features(
6949 struct pqi_config_table_section_info *section_info)
6950{
6951 int rc;
6952 struct pqi_ctrl_info *ctrl_info;
6953 struct pqi_config_table_firmware_features *firmware_features;
6954 void __iomem *firmware_features_iomem_addr;
6955 unsigned int i;
6956 unsigned int num_features_supported;
6957
6958 ctrl_info = section_info->ctrl_info;
6959 firmware_features = section_info->section;
6960 firmware_features_iomem_addr = section_info->section_iomem_addr;
6961
6962 for (i = 0, num_features_supported = 0;
6963 i < ARRAY_SIZE(pqi_firmware_features); i++) {
6964 if (pqi_is_firmware_feature_supported(firmware_features,
6965 pqi_firmware_features[i].feature_bit)) {
6966 pqi_firmware_features[i].supported = true;
6967 num_features_supported++;
6968 } else {
6969 pqi_firmware_feature_update(ctrl_info,
6970 &pqi_firmware_features[i]);
6971 }
6972 }
6973
6974 if (num_features_supported == 0)
6975 return;
6976
6977 for (i = 0; i < ARRAY_SIZE(pqi_firmware_features); i++) {
6978 if (!pqi_firmware_features[i].supported)
6979 continue;
6980 pqi_request_firmware_feature(firmware_features,
6981 pqi_firmware_features[i].feature_bit);
6982 }
6983
6984 rc = pqi_enable_firmware_features(ctrl_info, firmware_features,
6985 firmware_features_iomem_addr);
6986 if (rc) {
6987 dev_err(&ctrl_info->pci_dev->dev,
6988 "failed to enable firmware features in PQI configuration table\n");
6989 for (i = 0; i < ARRAY_SIZE(pqi_firmware_features); i++) {
6990 if (!pqi_firmware_features[i].supported)
6991 continue;
6992 pqi_firmware_feature_update(ctrl_info,
6993 &pqi_firmware_features[i]);
6994 }
6995 return;
6996 }
6997
6998 for (i = 0; i < ARRAY_SIZE(pqi_firmware_features); i++) {
6999 if (!pqi_firmware_features[i].supported)
7000 continue;
7001 if (pqi_is_firmware_feature_enabled(firmware_features,
7002 firmware_features_iomem_addr,
4fd22c13 7003 pqi_firmware_features[i].feature_bit)) {
b212c251 7004 pqi_firmware_features[i].enabled = true;
4fd22c13 7005 }
b212c251
KB		 7006 pqi_firmware_feature_update(ctrl_info,
7007 &pqi_firmware_features[i]);
7008 }
7009}
7010
7011static void pqi_init_firmware_features(void)
7012{
7013 unsigned int i;
7014
7015 for (i = 0; i < ARRAY_SIZE(pqi_firmware_features); i++) {
7016 pqi_firmware_features[i].supported = false;
7017 pqi_firmware_features[i].enabled = false;
7018 }
7019}
7020
7021static void pqi_process_firmware_features_section(
7022 struct pqi_config_table_section_info *section_info)
7023{
7024 mutex_lock(&pqi_firmware_features_mutex);
7025 pqi_init_firmware_features();
7026 pqi_process_firmware_features(section_info);
7027 mutex_unlock(&pqi_firmware_features_mutex);
7028}
7029
98f87667
KB		 7030static int pqi_process_config_table(struct pqi_ctrl_info *ctrl_info)
7031{
7032 u32 table_length;
7033 u32 section_offset;
7034 void __iomem *table_iomem_addr;
7035 struct pqi_config_table *config_table;
7036 struct pqi_config_table_section_header *section;
b212c251 7037 struct pqi_config_table_section_info section_info;
98f87667
KB		 7038
7039 table_length = ctrl_info->config_table_length;
b212c251
KB		 7040 if (table_length == 0)
7041 return 0;
98f87667
KB		 7042
7043 config_table = kmalloc(table_length, GFP_KERNEL);
7044 if (!config_table) {
7045 dev_err(&ctrl_info->pci_dev->dev,
d87d5474 7046 "failed to allocate memory for PQI configuration table\n");
98f87667
KB		 7047 return -ENOMEM;
7048 }
7049
7050 /*
7051 * Copy the config table contents from I/O memory space into the
7052 * temporary buffer.
7053 */
7054 table_iomem_addr = ctrl_info->iomem_base +
7055 ctrl_info->config_table_offset;
7056 memcpy_fromio(config_table, table_iomem_addr, table_length);
7057
b212c251 7058 section_info.ctrl_info = ctrl_info;
98f87667
KB		 7059 section_offset =
7060 get_unaligned_le32(&config_table->first_section_offset);
7061
7062 while (section_offset) {
7063 section = (void *)config_table + section_offset;
7064
b212c251
KB		 7065 section_info.section = section;
7066 section_info.section_offset = section_offset;
7067 section_info.section_iomem_addr =
7068 table_iomem_addr + section_offset;
7069
98f87667 7070 switch (get_unaligned_le16(&section->section_id)) {
b212c251
KB		 7071 case PQI_CONFIG_TABLE_SECTION_FIRMWARE_FEATURES:
7072 pqi_process_firmware_features_section(&section_info);
7073 break;
98f87667 7074 case PQI_CONFIG_TABLE_SECTION_HEARTBEAT:
5a259e32
KB		 7075 if (pqi_disable_heartbeat)
7076 dev_warn(&ctrl_info->pci_dev->dev,
7077 "heartbeat disabled by module parameter\n");
7078 else
7079 ctrl_info->heartbeat_counter =
7080 table_iomem_addr +
7081 section_offset +
7082 offsetof(
7083 struct pqi_config_table_heartbeat,
7084 heartbeat_counter);
98f87667 7085 break;
4fd22c13
MR		 7086 case PQI_CONFIG_TABLE_SECTION_SOFT_RESET:
7087 ctrl_info->soft_reset_status =
7088 table_iomem_addr +
7089 section_offset +
7090 offsetof(struct pqi_config_table_soft_reset,
7091 soft_reset_status);
7092 break;
98f87667
KB		 7093 }
7094
7095 section_offset =
7096 get_unaligned_le16(&section->next_section_offset);
7097 }
7098
7099 kfree(config_table);
7100
7101 return 0;
7102}
7103
162d7753
KB		 7104/* Switches the controller from PQI mode back into SIS mode. */
7105
7106static int pqi_revert_to_sis_mode(struct pqi_ctrl_info *ctrl_info)
7107{
7108 int rc;
7109
061ef06a 7110 pqi_change_irq_mode(ctrl_info, IRQ_MODE_NONE);
162d7753
KB		 7111 rc = pqi_reset(ctrl_info);
7112 if (rc)
7113 return rc;
4f078e24
KB		 7114 rc = sis_reenable_sis_mode(ctrl_info);
7115 if (rc) {
7116 dev_err(&ctrl_info->pci_dev->dev,
7117 "re-enabling SIS mode failed with error %d\n", rc);
7118 return rc;
7119 }
162d7753
KB		 7120 pqi_save_ctrl_mode(ctrl_info, SIS_MODE);
7121
7122 return 0;
7123}
7124
7125/*
7126 * If the controller isn't already in SIS mode, this function forces it into
7127 * SIS mode.
7128 */
7129
7130static int pqi_force_sis_mode(struct pqi_ctrl_info *ctrl_info)
ff6abb73
KB		 7131{
7132 if (!sis_is_firmware_running(ctrl_info))
7133 return -ENXIO;
7134
162d7753
KB		 7135 if (pqi_get_ctrl_mode(ctrl_info) == SIS_MODE)
7136 return 0;
7137
7138 if (sis_is_kernel_up(ctrl_info)) {
7139 pqi_save_ctrl_mode(ctrl_info, SIS_MODE);
7140 return 0;
ff6abb73
KB		 7141 }
7142
162d7753 7143 return pqi_revert_to_sis_mode(ctrl_info);
ff6abb73
KB		 7144}
7145
0530736e
KB		 7146#define PQI_POST_RESET_DELAY_B4_MSGU_READY 5000
7147
6c223761
KB		 7148static int pqi_ctrl_init(struct pqi_ctrl_info *ctrl_info)
7149{
7150 int rc;
7151
0530736e
KB		 7152 if (reset_devices) {
7153 sis_soft_reset(ctrl_info);
7154 msleep(PQI_POST_RESET_DELAY_B4_MSGU_READY);
7155 } else {
7156 rc = pqi_force_sis_mode(ctrl_info);
7157 if (rc)
7158 return rc;
7159 }
6c223761
KB		 7160
7161 /*
7162 * Wait until the controller is ready to start accepting SIS
7163 * commands.
7164 */
7165 rc = sis_wait_for_ctrl_ready(ctrl_info);
8845fdfa 7166 if (rc)
6c223761 7167 return rc;
6c223761
KB		 7168
7169 /*
7170 * Get the controller properties. This allows us to determine
7171 * whether or not it supports PQI mode.
7172 */
7173 rc = sis_get_ctrl_properties(ctrl_info);
7174 if (rc) {
7175 dev_err(&ctrl_info->pci_dev->dev,
7176 "error obtaining controller properties\n");
7177 return rc;
7178 }
7179
7180 rc = sis_get_pqi_capabilities(ctrl_info);
7181 if (rc) {
7182 dev_err(&ctrl_info->pci_dev->dev,
7183 "error obtaining controller capabilities\n");
7184 return rc;
7185 }
7186
d727a776
KB		 7187 if (reset_devices) {
7188 if (ctrl_info->max_outstanding_requests >
7189 PQI_MAX_OUTSTANDING_REQUESTS_KDUMP)
7190 ctrl_info->max_outstanding_requests =
7191 PQI_MAX_OUTSTANDING_REQUESTS_KDUMP;
7192 } else {
7193 if (ctrl_info->max_outstanding_requests >
7194 PQI_MAX_OUTSTANDING_REQUESTS)
7195 ctrl_info->max_outstanding_requests =
7196 PQI_MAX_OUTSTANDING_REQUESTS;
7197 }
6c223761
KB		 7198
7199 pqi_calculate_io_resources(ctrl_info);
7200
7201 rc = pqi_alloc_error_buffer(ctrl_info);
7202 if (rc) {
7203 dev_err(&ctrl_info->pci_dev->dev,
7204 "failed to allocate PQI error buffer\n");
7205 return rc;
7206 }
7207
7208 /*
7209 * If the function we are about to call succeeds, the
7210 * controller will transition from legacy SIS mode
7211 * into PQI mode.
7212 */
7213 rc = sis_init_base_struct_addr(ctrl_info);
7214 if (rc) {
7215 dev_err(&ctrl_info->pci_dev->dev,
7216 "error initializing PQI mode\n");
7217 return rc;
7218 }
7219
7220 /* Wait for the controller to complete the SIS -> PQI transition. */
7221 rc = pqi_wait_for_pqi_mode_ready(ctrl_info);
7222 if (rc) {
7223 dev_err(&ctrl_info->pci_dev->dev,
7224 "transition to PQI mode failed\n");
7225 return rc;
7226 }
7227
7228 /* From here on, we are running in PQI mode. */
7229 ctrl_info->pqi_mode_enabled = true;
ff6abb73 7230 pqi_save_ctrl_mode(ctrl_info, PQI_MODE);
6c223761
KB		 7231
7232 rc = pqi_alloc_admin_queues(ctrl_info);
7233 if (rc) {
7234 dev_err(&ctrl_info->pci_dev->dev,
d87d5474 7235 "failed to allocate admin queues\n");
6c223761
KB		 7236 return rc;
7237 }
7238
7239 rc = pqi_create_admin_queues(ctrl_info);
7240 if (rc) {
7241 dev_err(&ctrl_info->pci_dev->dev,
7242 "error creating admin queues\n");
7243 return rc;
7244 }
7245
7246 rc = pqi_report_device_capability(ctrl_info);
7247 if (rc) {
7248 dev_err(&ctrl_info->pci_dev->dev,
7249 "obtaining device capability failed\n");
7250 return rc;
7251 }
7252
7253 rc = pqi_validate_device_capability(ctrl_info);
7254 if (rc)
7255 return rc;
7256
7257 pqi_calculate_queue_resources(ctrl_info);
7258
7259 rc = pqi_enable_msix_interrupts(ctrl_info);
7260 if (rc)
7261 return rc;
7262
7263 if (ctrl_info->num_msix_vectors_enabled < ctrl_info->num_queue_groups) {
7264 ctrl_info->max_msix_vectors =
7265 ctrl_info->num_msix_vectors_enabled;
7266 pqi_calculate_queue_resources(ctrl_info);
7267 }
7268
7269 rc = pqi_alloc_io_resources(ctrl_info);
7270 if (rc)
7271 return rc;
7272
7273 rc = pqi_alloc_operational_queues(ctrl_info);
d87d5474
KB		 7274 if (rc) {
7275 dev_err(&ctrl_info->pci_dev->dev,
7276 "failed to allocate operational queues\n");
6c223761 7277 return rc;
d87d5474 7278 }
6c223761
KB		 7279
7280 pqi_init_operational_queues(ctrl_info);
7281
7282 rc = pqi_request_irqs(ctrl_info);
7283 if (rc)
7284 return rc;
7285
6c223761
KB		 7286 rc = pqi_create_queues(ctrl_info);
7287 if (rc)
7288 return rc;
7289
061ef06a
KB		 7290 pqi_change_irq_mode(ctrl_info, IRQ_MODE_MSIX);
7291
7292 ctrl_info->controller_online = true;
b212c251
KB		 7293
7294 rc = pqi_process_config_table(ctrl_info);
7295 if (rc)
7296 return rc;
7297
061ef06a 7298 pqi_start_heartbeat_timer(ctrl_info);
6c223761 7299
6a50d6ad 7300 rc = pqi_enable_events(ctrl_info);
6c223761
KB		 7301 if (rc) {
7302 dev_err(&ctrl_info->pci_dev->dev,
6a50d6ad 7303 "error enabling events\n");
6c223761
KB		 7304 return rc;
7305 }
7306
6c223761
KB		 7307 /* Register with the SCSI subsystem. */
7308 rc = pqi_register_scsi(ctrl_info);
7309 if (rc)
7310 return rc;
7311
6d90615f
MB		 7312 rc = pqi_get_ctrl_product_details(ctrl_info);
7313 if (rc) {
7314 dev_err(&ctrl_info->pci_dev->dev,
7315 "error obtaining product details\n");
7316 return rc;
7317 }
7318
7319 rc = pqi_get_ctrl_serial_number(ctrl_info);
6c223761
KB		 7320 if (rc) {
7321 dev_err(&ctrl_info->pci_dev->dev,
6d90615f 7322 "error obtaining ctrl serial number\n");
6c223761
KB		 7323 return rc;
7324 }
7325
171c2865
DC		 7326 rc = pqi_set_diag_rescan(ctrl_info);
7327 if (rc) {
7328 dev_err(&ctrl_info->pci_dev->dev,
7329 "error enabling multi-lun rescan\n");
7330 return rc;
7331 }
7332
6c223761
KB		 7333 rc = pqi_write_driver_version_to_host_wellness(ctrl_info);
7334 if (rc) {
7335 dev_err(&ctrl_info->pci_dev->dev,
7336 "error updating host wellness\n");
7337 return rc;
7338 }
7339
7340 pqi_schedule_update_time_worker(ctrl_info);
7341
7342 pqi_scan_scsi_devices(ctrl_info);
7343
7344 return 0;
7345}
7346
061ef06a
KB		 7347static void pqi_reinit_queues(struct pqi_ctrl_info *ctrl_info)
7348{
7349 unsigned int i;
7350 struct pqi_admin_queues *admin_queues;
7351 struct pqi_event_queue *event_queue;
7352
7353 admin_queues = &ctrl_info->admin_queues;
7354 admin_queues->iq_pi_copy = 0;
7355 admin_queues->oq_ci_copy = 0;
dac12fbc 7356 writel(0, admin_queues->oq_pi);
061ef06a
KB		 7357
7358 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
7359 ctrl_info->queue_groups[i].iq_pi_copy[RAID_PATH] = 0;
7360 ctrl_info->queue_groups[i].iq_pi_copy[AIO_PATH] = 0;
7361 ctrl_info->queue_groups[i].oq_ci_copy = 0;
7362
dac12fbc
KB		 7363 writel(0, ctrl_info->queue_groups[i].iq_ci[RAID_PATH]);
7364 writel(0, ctrl_info->queue_groups[i].iq_ci[AIO_PATH]);
7365 writel(0, ctrl_info->queue_groups[i].oq_pi);
061ef06a
KB		 7366 }
7367
7368 event_queue = &ctrl_info->event_queue;
dac12fbc 7369 writel(0, event_queue->oq_pi);
061ef06a
KB		 7370 event_queue->oq_ci_copy = 0;
7371}
7372
7373static int pqi_ctrl_init_resume(struct pqi_ctrl_info *ctrl_info)
7374{
7375 int rc;
7376
7377 rc = pqi_force_sis_mode(ctrl_info);
7378 if (rc)
7379 return rc;
7380
7381 /*
7382 * Wait until the controller is ready to start accepting SIS
7383 * commands.
7384 */
7385 rc = sis_wait_for_ctrl_ready_resume(ctrl_info);
7386 if (rc)
7387 return rc;
7388
4fd22c13
MR		 7389 /*
7390 * Get the controller properties. This allows us to determine
7391 * whether or not it supports PQI mode.
7392 */
7393 rc = sis_get_ctrl_properties(ctrl_info);
7394 if (rc) {
7395 dev_err(&ctrl_info->pci_dev->dev,
7396 "error obtaining controller properties\n");
7397 return rc;
7398 }
7399
7400 rc = sis_get_pqi_capabilities(ctrl_info);
7401 if (rc) {
7402 dev_err(&ctrl_info->pci_dev->dev,
7403 "error obtaining controller capabilities\n");
7404 return rc;
7405 }
7406
061ef06a
KB		 7407 /*
7408 * If the function we are about to call succeeds, the
7409 * controller will transition from legacy SIS mode
7410 * into PQI mode.
7411 */
7412 rc = sis_init_base_struct_addr(ctrl_info);
7413 if (rc) {
7414 dev_err(&ctrl_info->pci_dev->dev,
7415 "error initializing PQI mode\n");
7416 return rc;
7417 }
7418
7419 /* Wait for the controller to complete the SIS -> PQI transition. */
7420 rc = pqi_wait_for_pqi_mode_ready(ctrl_info);
7421 if (rc) {
7422 dev_err(&ctrl_info->pci_dev->dev,
7423 "transition to PQI mode failed\n");
7424 return rc;
7425 }
7426
7427 /* From here on, we are running in PQI mode. */
7428 ctrl_info->pqi_mode_enabled = true;
7429 pqi_save_ctrl_mode(ctrl_info, PQI_MODE);
7430
7431 pqi_reinit_queues(ctrl_info);
7432
7433 rc = pqi_create_admin_queues(ctrl_info);
7434 if (rc) {
7435 dev_err(&ctrl_info->pci_dev->dev,
7436 "error creating admin queues\n");
7437 return rc;
7438 }
7439
7440 rc = pqi_create_queues(ctrl_info);
7441 if (rc)
7442 return rc;
7443
7444 pqi_change_irq_mode(ctrl_info, IRQ_MODE_MSIX);
7445
7446 ctrl_info->controller_online = true;
061ef06a
KB		 7447 pqi_ctrl_unblock_requests(ctrl_info);
7448
4fd22c13
MR		 7449 rc = pqi_process_config_table(ctrl_info);
7450 if (rc)
7451 return rc;
7452
7453 pqi_start_heartbeat_timer(ctrl_info);
7454
061ef06a
KB		 7455 rc = pqi_enable_events(ctrl_info);
7456 if (rc) {
7457 dev_err(&ctrl_info->pci_dev->dev,
d87d5474 7458 "error enabling events\n");
061ef06a
KB		 7459 return rc;
7460 }
7461
6d90615f 7462 rc = pqi_get_ctrl_product_details(ctrl_info);
4fd22c13
MR		 7463 if (rc) {
7464 dev_err(&ctrl_info->pci_dev->dev,
694c5d5b 7465 "error obtaining product details\n");
4fd22c13
MR		 7466 return rc;
7467 }
7468
171c2865
DC		 7469 rc = pqi_set_diag_rescan(ctrl_info);
7470 if (rc) {
7471 dev_err(&ctrl_info->pci_dev->dev,
7472 "error enabling multi-lun rescan\n");
7473 return rc;
7474 }
7475
061ef06a
KB		 7476 rc = pqi_write_driver_version_to_host_wellness(ctrl_info);
7477 if (rc) {
7478 dev_err(&ctrl_info->pci_dev->dev,
7479 "error updating host wellness\n");
7480 return rc;
7481 }
7482
7483 pqi_schedule_update_time_worker(ctrl_info);
7484
7485 pqi_scan_scsi_devices(ctrl_info);
7486
7487 return 0;
7488}
7489
a81ed5f3
KB		 7490static inline int pqi_set_pcie_completion_timeout(struct pci_dev *pci_dev,
7491 u16 timeout)
7492{
d20df83b
BOS		 7493 int rc;
7494
7495 rc = pcie_capability_clear_and_set_word(pci_dev, PCI_EXP_DEVCTL2,
a81ed5f3 7496 PCI_EXP_DEVCTL2_COMP_TIMEOUT, timeout);
d20df83b
BOS		 7497
7498 return pcibios_err_to_errno(rc);
a81ed5f3
KB		 7499}
7500
6c223761
KB		 7501static int pqi_pci_init(struct pqi_ctrl_info *ctrl_info)
7502{
7503 int rc;
7504 u64 mask;
7505
7506 rc = pci_enable_device(ctrl_info->pci_dev);
7507 if (rc) {
7508 dev_err(&ctrl_info->pci_dev->dev,
7509 "failed to enable PCI device\n");
7510 return rc;
7511 }
7512
7513 if (sizeof(dma_addr_t) > 4)
7514 mask = DMA_BIT_MASK(64);
7515 else
7516 mask = DMA_BIT_MASK(32);
7517
1d94f06e 7518 rc = dma_set_mask_and_coherent(&ctrl_info->pci_dev->dev, mask);
6c223761
KB		 7519 if (rc) {
7520 dev_err(&ctrl_info->pci_dev->dev, "failed to set DMA mask\n");
7521 goto disable_device;
7522 }
7523
7524 rc = pci_request_regions(ctrl_info->pci_dev, DRIVER_NAME_SHORT);
7525 if (rc) {
7526 dev_err(&ctrl_info->pci_dev->dev,
7527 "failed to obtain PCI resources\n");
7528 goto disable_device;
7529 }
7530
4bdc0d67 7531 ctrl_info->iomem_base = ioremap(pci_resource_start(
6c223761
KB		 7532 ctrl_info->pci_dev, 0),
7533 sizeof(struct pqi_ctrl_registers));
7534 if (!ctrl_info->iomem_base) {
7535 dev_err(&ctrl_info->pci_dev->dev,
7536 "failed to map memory for controller registers\n");
7537 rc = -ENOMEM;
7538 goto release_regions;
7539 }
7540
a81ed5f3
KB		 7541#define PCI_EXP_COMP_TIMEOUT_65_TO_210_MS 0x6
7542
7543 /* Increase the PCIe completion timeout. */
7544 rc = pqi_set_pcie_completion_timeout(ctrl_info->pci_dev,
7545 PCI_EXP_COMP_TIMEOUT_65_TO_210_MS);
7546 if (rc) {
7547 dev_err(&ctrl_info->pci_dev->dev,
7548 "failed to set PCIe completion timeout\n");
7549 goto release_regions;
7550 }
7551
6c223761
KB		 7552 /* Enable bus mastering. */
7553 pci_set_master(ctrl_info->pci_dev);
7554
cbe0c7b1
KB		 7555 ctrl_info->registers = ctrl_info->iomem_base;
7556 ctrl_info->pqi_registers = &ctrl_info->registers->pqi_registers;
7557
6c223761
KB		 7558 pci_set_drvdata(ctrl_info->pci_dev, ctrl_info);
7559
7560 return 0;
7561
7562release_regions:
7563 pci_release_regions(ctrl_info->pci_dev);
7564disable_device:
7565 pci_disable_device(ctrl_info->pci_dev);
7566
7567 return rc;
7568}
7569
7570static void pqi_cleanup_pci_init(struct pqi_ctrl_info *ctrl_info)
7571{
7572 iounmap(ctrl_info->iomem_base);
7573 pci_release_regions(ctrl_info->pci_dev);
cbe0c7b1
KB		 7574 if (pci_is_enabled(ctrl_info->pci_dev))
7575 pci_disable_device(ctrl_info->pci_dev);
6c223761
KB		 7576 pci_set_drvdata(ctrl_info->pci_dev, NULL);
7577}
7578
7579static struct pqi_ctrl_info *pqi_alloc_ctrl_info(int numa_node)
7580{
7581 struct pqi_ctrl_info *ctrl_info;
7582
7583 ctrl_info = kzalloc_node(sizeof(struct pqi_ctrl_info),
7584 GFP_KERNEL, numa_node);
7585 if (!ctrl_info)
7586 return NULL;
7587
7588 mutex_init(&ctrl_info->scan_mutex);
7561a7e4 7589 mutex_init(&ctrl_info->lun_reset_mutex);
4fd22c13 7590 mutex_init(&ctrl_info->ofa_mutex);
6c223761
KB		 7591
7592 INIT_LIST_HEAD(&ctrl_info->scsi_device_list);
7593 spin_lock_init(&ctrl_info->scsi_device_list_lock);
7594
7595 INIT_WORK(&ctrl_info->event_work, pqi_event_worker);
7596 atomic_set(&ctrl_info->num_interrupts, 0);
0530736e 7597 atomic_set(&ctrl_info->sync_cmds_outstanding, 0);
6c223761
KB		 7598
7599 INIT_DELAYED_WORK(&ctrl_info->rescan_work, pqi_rescan_worker);
7600 INIT_DELAYED_WORK(&ctrl_info->update_time_work, pqi_update_time_worker);
7601
74a0f573 7602 timer_setup(&ctrl_info->heartbeat_timer, pqi_heartbeat_timer_handler, 0);
5f310425 7603 INIT_WORK(&ctrl_info->ctrl_offline_work, pqi_ctrl_offline_worker);
98f87667 7604
6c223761
KB		 7605 sema_init(&ctrl_info->sync_request_sem,
7606 PQI_RESERVED_IO_SLOTS_SYNCHRONOUS_REQUESTS);
7561a7e4 7607 init_waitqueue_head(&ctrl_info->block_requests_wait);
6c223761 7608
376fb880
KB		 7609 INIT_LIST_HEAD(&ctrl_info->raid_bypass_retry_list);
7610 spin_lock_init(&ctrl_info->raid_bypass_retry_list_lock);
7611 INIT_WORK(&ctrl_info->raid_bypass_retry_work,
7612 pqi_raid_bypass_retry_worker);
7613
6c223761 7614 ctrl_info->ctrl_id = atomic_inc_return(&pqi_controller_count) - 1;
061ef06a 7615 ctrl_info->irq_mode = IRQ_MODE_NONE;
6c223761
KB		 7616 ctrl_info->max_msix_vectors = PQI_MAX_MSIX_VECTORS;
7617
7618 return ctrl_info;
7619}
7620
7621static inline void pqi_free_ctrl_info(struct pqi_ctrl_info *ctrl_info)
7622{
7623 kfree(ctrl_info);
7624}
7625
7626static void pqi_free_interrupts(struct pqi_ctrl_info *ctrl_info)
7627{
98bf061b
KB		 7628 pqi_free_irqs(ctrl_info);
7629 pqi_disable_msix_interrupts(ctrl_info);
6c223761
KB		 7630}
7631
7632static void pqi_free_ctrl_resources(struct pqi_ctrl_info *ctrl_info)
7633{
7634 pqi_stop_heartbeat_timer(ctrl_info);
7635 pqi_free_interrupts(ctrl_info);
7636 if (ctrl_info->queue_memory_base)
7637 dma_free_coherent(&ctrl_info->pci_dev->dev,
7638 ctrl_info->queue_memory_length,
7639 ctrl_info->queue_memory_base,
7640 ctrl_info->queue_memory_base_dma_handle);
7641 if (ctrl_info->admin_queue_memory_base)
7642 dma_free_coherent(&ctrl_info->pci_dev->dev,
7643 ctrl_info->admin_queue_memory_length,
7644 ctrl_info->admin_queue_memory_base,
7645 ctrl_info->admin_queue_memory_base_dma_handle);
7646 pqi_free_all_io_requests(ctrl_info);
7647 if (ctrl_info->error_buffer)
7648 dma_free_coherent(&ctrl_info->pci_dev->dev,
7649 ctrl_info->error_buffer_length,
7650 ctrl_info->error_buffer,
7651 ctrl_info->error_buffer_dma_handle);
7652 if (ctrl_info->iomem_base)
7653 pqi_cleanup_pci_init(ctrl_info);
7654 pqi_free_ctrl_info(ctrl_info);
7655}
7656
7657static void pqi_remove_ctrl(struct pqi_ctrl_info *ctrl_info)
7658{
061ef06a
KB		 7659 pqi_cancel_rescan_worker(ctrl_info);
7660 pqi_cancel_update_time_worker(ctrl_info);
e57a1f9b 7661 pqi_unregister_scsi(ctrl_info);
162d7753
KB		 7662 if (ctrl_info->pqi_mode_enabled)
7663 pqi_revert_to_sis_mode(ctrl_info);
6c223761
KB		 7664 pqi_free_ctrl_resources(ctrl_info);
7665}
7666
4fd22c13
MR		 7667static void pqi_ofa_ctrl_quiesce(struct pqi_ctrl_info *ctrl_info)
7668{
7669 pqi_cancel_update_time_worker(ctrl_info);
7670 pqi_cancel_rescan_worker(ctrl_info);
7671 pqi_wait_until_lun_reset_finished(ctrl_info);
7672 pqi_wait_until_scan_finished(ctrl_info);
7673 pqi_ctrl_ofa_start(ctrl_info);
7674 pqi_ctrl_block_requests(ctrl_info);
7675 pqi_ctrl_wait_until_quiesced(ctrl_info);
7676 pqi_ctrl_wait_for_pending_io(ctrl_info, PQI_PENDING_IO_TIMEOUT_SECS);
7677 pqi_fail_io_queued_for_all_devices(ctrl_info);
7678 pqi_wait_until_inbound_queues_empty(ctrl_info);
7679 pqi_stop_heartbeat_timer(ctrl_info);
7680 ctrl_info->pqi_mode_enabled = false;
7681 pqi_save_ctrl_mode(ctrl_info, SIS_MODE);
7682}
7683
7684static void pqi_ofa_ctrl_unquiesce(struct pqi_ctrl_info *ctrl_info)
7685{
7686 pqi_ofa_free_host_buffer(ctrl_info);
7687 ctrl_info->pqi_mode_enabled = true;
7688 pqi_save_ctrl_mode(ctrl_info, PQI_MODE);
7689 ctrl_info->controller_online = true;
7690 pqi_ctrl_unblock_requests(ctrl_info);
7691 pqi_start_heartbeat_timer(ctrl_info);
7692 pqi_schedule_update_time_worker(ctrl_info);
7693 pqi_clear_soft_reset_status(ctrl_info,
7694 PQI_SOFT_RESET_ABORT);
7695 pqi_scan_scsi_devices(ctrl_info);
7696}
7697
7698static int pqi_ofa_alloc_mem(struct pqi_ctrl_info *ctrl_info,
7699 u32 total_size, u32 chunk_size)
7700{
7701 u32 sg_count;
7702 u32 size;
7703 int i;
7704 struct pqi_sg_descriptor *mem_descriptor = NULL;
7705 struct device *dev;
7706 struct pqi_ofa_memory *ofap;
7707
7708 dev = &ctrl_info->pci_dev->dev;
7709
7710 sg_count = (total_size + chunk_size - 1);
e52c9e07 7711 sg_count /= chunk_size;
4fd22c13
MR		 7712
7713 ofap = ctrl_info->pqi_ofa_mem_virt_addr;
7714
7715 if (sg_count*chunk_size < total_size)
7716 goto out;
7717
7718 ctrl_info->pqi_ofa_chunk_virt_addr =
7719 kcalloc(sg_count, sizeof(void *), GFP_KERNEL);
7720 if (!ctrl_info->pqi_ofa_chunk_virt_addr)
7721 goto out;
7722
7723 for (size = 0, i = 0; size < total_size; size += chunk_size, i++) {
7724 dma_addr_t dma_handle;
7725
7726 ctrl_info->pqi_ofa_chunk_virt_addr[i] =
750afb08
LC		 7727 dma_alloc_coherent(dev, chunk_size, &dma_handle,
7728 GFP_KERNEL);
4fd22c13
MR		 7729
7730 if (!ctrl_info->pqi_ofa_chunk_virt_addr[i])
7731 break;
7732
7733 mem_descriptor = &ofap->sg_descriptor[i];
7734 put_unaligned_le64 ((u64) dma_handle, &mem_descriptor->address);
7735 put_unaligned_le32 (chunk_size, &mem_descriptor->length);
7736 }
7737
7738 if (!size || size < total_size)
7739 goto out_free_chunks;
7740
7741 put_unaligned_le32(CISS_SG_LAST, &mem_descriptor->flags);
7742 put_unaligned_le16(sg_count, &ofap->num_memory_descriptors);
7743 put_unaligned_le32(size, &ofap->bytes_allocated);
7744
7745 return 0;
7746
7747out_free_chunks:
7748 while (--i >= 0) {
7749 mem_descriptor = &ofap->sg_descriptor[i];
7750 dma_free_coherent(dev, chunk_size,
7751 ctrl_info->pqi_ofa_chunk_virt_addr[i],
7752 get_unaligned_le64(&mem_descriptor->address));
7753 }
7754 kfree(ctrl_info->pqi_ofa_chunk_virt_addr);
7755
7756out:
7757 put_unaligned_le32 (0, &ofap->bytes_allocated);
7758 return -ENOMEM;
7759}
7760
7761static int pqi_ofa_alloc_host_buffer(struct pqi_ctrl_info *ctrl_info)
7762{
7763 u32 total_size;
7764 u32 min_chunk_size;
7765 u32 chunk_sz;
7766
7767 total_size = le32_to_cpu(
7768 ctrl_info->pqi_ofa_mem_virt_addr->bytes_allocated);
7769 min_chunk_size = total_size / PQI_OFA_MAX_SG_DESCRIPTORS;
7770
7771 for (chunk_sz = total_size; chunk_sz >= min_chunk_size; chunk_sz /= 2)
7772 if (!pqi_ofa_alloc_mem(ctrl_info, total_size, chunk_sz))
7773 return 0;
7774
7775 return -ENOMEM;
7776}
7777
7778static void pqi_ofa_setup_host_buffer(struct pqi_ctrl_info *ctrl_info,
7779 u32 bytes_requested)
7780{
7781 struct pqi_ofa_memory *pqi_ofa_memory;
7782 struct device *dev;
7783
7784 dev = &ctrl_info->pci_dev->dev;
750afb08
LC		 7785 pqi_ofa_memory = dma_alloc_coherent(dev,
7786 PQI_OFA_MEMORY_DESCRIPTOR_LENGTH,
7787 &ctrl_info->pqi_ofa_mem_dma_handle,
7788 GFP_KERNEL);
4fd22c13
MR		 7789
7790 if (!pqi_ofa_memory)
7791 return;
7792
7793 put_unaligned_le16(PQI_OFA_VERSION, &pqi_ofa_memory->version);
7794 memcpy(&pqi_ofa_memory->signature, PQI_OFA_SIGNATURE,
7795 sizeof(pqi_ofa_memory->signature));
7796 pqi_ofa_memory->bytes_allocated = cpu_to_le32(bytes_requested);
7797
7798 ctrl_info->pqi_ofa_mem_virt_addr = pqi_ofa_memory;
7799
7800 if (pqi_ofa_alloc_host_buffer(ctrl_info) < 0) {
7801 dev_err(dev, "Failed to allocate host buffer of size = %u",
7802 bytes_requested);
7803 }
694c5d5b
KB		 7804
7805 return;
4fd22c13
MR		 7806}
7807
7808static void pqi_ofa_free_host_buffer(struct pqi_ctrl_info *ctrl_info)
7809{
7810 int i;
7811 struct pqi_sg_descriptor *mem_descriptor;
7812 struct pqi_ofa_memory *ofap;
7813
7814 ofap = ctrl_info->pqi_ofa_mem_virt_addr;
7815
7816 if (!ofap)
7817 return;
7818
7819 if (!ofap->bytes_allocated)
7820 goto out;
7821
7822 mem_descriptor = ofap->sg_descriptor;
7823
7824 for (i = 0; i < get_unaligned_le16(&ofap->num_memory_descriptors);
7825 i++) {
7826 dma_free_coherent(&ctrl_info->pci_dev->dev,
7827 get_unaligned_le32(&mem_descriptor[i].length),
7828 ctrl_info->pqi_ofa_chunk_virt_addr[i],
7829 get_unaligned_le64(&mem_descriptor[i].address));
7830 }
7831 kfree(ctrl_info->pqi_ofa_chunk_virt_addr);
7832
7833out:
7834 dma_free_coherent(&ctrl_info->pci_dev->dev,
7835 PQI_OFA_MEMORY_DESCRIPTOR_LENGTH, ofap,
7836 ctrl_info->pqi_ofa_mem_dma_handle);
7837 ctrl_info->pqi_ofa_mem_virt_addr = NULL;
7838}
7839
7840static int pqi_ofa_host_memory_update(struct pqi_ctrl_info *ctrl_info)
7841{
7842 struct pqi_vendor_general_request request;
7843 size_t size;
7844 struct pqi_ofa_memory *ofap;
7845
7846 memset(&request, 0, sizeof(request));
7847
7848 ofap = ctrl_info->pqi_ofa_mem_virt_addr;
7849
7850 request.header.iu_type = PQI_REQUEST_IU_VENDOR_GENERAL;
7851 put_unaligned_le16(sizeof(request) - PQI_REQUEST_HEADER_LENGTH,
7852 &request.header.iu_length);
7853 put_unaligned_le16(PQI_VENDOR_GENERAL_HOST_MEMORY_UPDATE,
7854 &request.function_code);
7855
7856 if (ofap) {
7857 size = offsetof(struct pqi_ofa_memory, sg_descriptor) +
7858 get_unaligned_le16(&ofap->num_memory_descriptors) *
7859 sizeof(struct pqi_sg_descriptor);
7860
7861 put_unaligned_le64((u64)ctrl_info->pqi_ofa_mem_dma_handle,
7862 &request.data.ofa_memory_allocation.buffer_address);
7863 put_unaligned_le32(size,
7864 &request.data.ofa_memory_allocation.buffer_length);
7865
7866 }
7867
7868 return pqi_submit_raid_request_synchronous(ctrl_info, &request.header,
7869 0, NULL, NO_TIMEOUT);
7870}
7871
4fd22c13
MR		 7872static int pqi_ofa_ctrl_restart(struct pqi_ctrl_info *ctrl_info)
7873{
7874 msleep(PQI_POST_RESET_DELAY_B4_MSGU_READY);
7875 return pqi_ctrl_init_resume(ctrl_info);
7876}
7877
3c50976f
KB		 7878static void pqi_perform_lockup_action(void)
7879{
7880 switch (pqi_lockup_action) {
7881 case PANIC:
7882 panic("FATAL: Smart Family Controller lockup detected");
7883 break;
7884 case REBOOT:
7885 emergency_restart();
7886 break;
7887 case NONE:
7888 default:
7889 break;
7890 }
7891}
7892
5f310425
KB		 7893static struct pqi_raid_error_info pqi_ctrl_offline_raid_error_info = {
7894 .data_out_result = PQI_DATA_IN_OUT_HARDWARE_ERROR,
7895 .status = SAM_STAT_CHECK_CONDITION,
7896};
7897
7898static void pqi_fail_all_outstanding_requests(struct pqi_ctrl_info *ctrl_info)
376fb880
KB		 7899{
7900 unsigned int i;
376fb880 7901 struct pqi_io_request *io_request;
376fb880
KB		 7902 struct scsi_cmnd *scmd;
7903
5f310425
KB		 7904 for (i = 0; i < ctrl_info->max_io_slots; i++) {
7905 io_request = &ctrl_info->io_request_pool[i];
7906 if (atomic_read(&io_request->refcount) == 0)
7907 continue;
376fb880 7908
5f310425
KB		 7909 scmd = io_request->scmd;
7910 if (scmd) {
7911 set_host_byte(scmd, DID_NO_CONNECT);
7912 } else {
7913 io_request->status = -ENXIO;
7914 io_request->error_info =
7915 &pqi_ctrl_offline_raid_error_info;
376fb880 7916 }
5f310425
KB		 7917
7918 io_request->io_complete_callback(io_request,
7919 io_request->context);
376fb880
KB		 7920 }
7921}
7922
5f310425 7923static void pqi_take_ctrl_offline_deferred(struct pqi_ctrl_info *ctrl_info)
376fb880 7924{
5f310425
KB		 7925 pqi_perform_lockup_action();
7926 pqi_stop_heartbeat_timer(ctrl_info);
7927 pqi_free_interrupts(ctrl_info);
7928 pqi_cancel_rescan_worker(ctrl_info);
7929 pqi_cancel_update_time_worker(ctrl_info);
7930 pqi_ctrl_wait_until_quiesced(ctrl_info);
7931 pqi_fail_all_outstanding_requests(ctrl_info);
7932 pqi_clear_all_queued_raid_bypass_retries(ctrl_info);
7933 pqi_ctrl_unblock_requests(ctrl_info);
7934}
7935
7936static void pqi_ctrl_offline_worker(struct work_struct *work)
7937{
7938 struct pqi_ctrl_info *ctrl_info;
7939
7940 ctrl_info = container_of(work, struct pqi_ctrl_info, ctrl_offline_work);
7941 pqi_take_ctrl_offline_deferred(ctrl_info);
376fb880
KB		 7942}
7943
7944static void pqi_take_ctrl_offline(struct pqi_ctrl_info *ctrl_info)
7945{
5f310425
KB		 7946 if (!ctrl_info->controller_online)
7947 return;
7948
376fb880 7949 ctrl_info->controller_online = false;
5f310425
KB		 7950 ctrl_info->pqi_mode_enabled = false;
7951 pqi_ctrl_block_requests(ctrl_info);
5a259e32
KB		 7952 if (!pqi_disable_ctrl_shutdown)
7953 sis_shutdown_ctrl(ctrl_info);
376fb880
KB		 7954 pci_disable_device(ctrl_info->pci_dev);
7955 dev_err(&ctrl_info->pci_dev->dev, "controller offline\n");
5f310425 7956 schedule_work(&ctrl_info->ctrl_offline_work);
376fb880
KB		 7957}
7958
d91d7820 7959static void pqi_print_ctrl_info(struct pci_dev *pci_dev,
6c223761
KB		 7960 const struct pci_device_id *id)
7961{
7962 char *ctrl_description;
7963
37b36847 7964 if (id->driver_data)
6c223761 7965 ctrl_description = (char *)id->driver_data;
37b36847
KB		 7966 else
7967 ctrl_description = "Microsemi Smart Family Controller";
6c223761 7968
d91d7820 7969 dev_info(&pci_dev->dev, "%s found\n", ctrl_description);
6c223761
KB		 7970}
7971
d91d7820
KB		 7972static int pqi_pci_probe(struct pci_dev *pci_dev,
7973 const struct pci_device_id *id)
6c223761
KB		 7974{
7975 int rc;
62dc51fb 7976 int node, cp_node;
6c223761
KB		 7977 struct pqi_ctrl_info *ctrl_info;
7978
d91d7820 7979 pqi_print_ctrl_info(pci_dev, id);
6c223761
KB		 7980
7981 if (pqi_disable_device_id_wildcards &&
7982 id->subvendor == PCI_ANY_ID &&
7983 id->subdevice == PCI_ANY_ID) {
d91d7820 7984 dev_warn(&pci_dev->dev,
6c223761
KB		 7985 "controller not probed because device ID wildcards are disabled\n");
7986 return -ENODEV;
7987 }
7988
7989 if (id->subvendor == PCI_ANY_ID || id->subdevice == PCI_ANY_ID)
d91d7820 7990 dev_warn(&pci_dev->dev,
6c223761
KB		 7991 "controller device ID matched using wildcards\n");
7992
d91d7820 7993 node = dev_to_node(&pci_dev->dev);
62dc51fb
SB		 7994 if (node == NUMA_NO_NODE) {
7995 cp_node = cpu_to_node(0);
7996 if (cp_node == NUMA_NO_NODE)
7997 cp_node = 0;
7998 set_dev_node(&pci_dev->dev, cp_node);
7999 }
6c223761
KB		 8000
8001 ctrl_info = pqi_alloc_ctrl_info(node);
8002 if (!ctrl_info) {
d91d7820 8003 dev_err(&pci_dev->dev,
6c223761
KB		 8004 "failed to allocate controller info block\n");
8005 return -ENOMEM;
8006 }
8007
d91d7820 8008 ctrl_info->pci_dev = pci_dev;
6c223761
KB		 8009
8010 rc = pqi_pci_init(ctrl_info);
8011 if (rc)
8012 goto error;
8013
8014 rc = pqi_ctrl_init(ctrl_info);
8015 if (rc)
8016 goto error;
8017
8018 return 0;
8019
8020error:
8021 pqi_remove_ctrl(ctrl_info);
8022
8023 return rc;
8024}
8025
d91d7820 8026static void pqi_pci_remove(struct pci_dev *pci_dev)
6c223761
KB		 8027{
8028 struct pqi_ctrl_info *ctrl_info;
8029
d91d7820 8030 ctrl_info = pci_get_drvdata(pci_dev);
6c223761
KB		 8031 if (!ctrl_info)
8032 return;
8033
1e46731e
MR		 8034 ctrl_info->in_shutdown = true;
8035
6c223761
KB		 8036 pqi_remove_ctrl(ctrl_info);
8037}
8038
0530736e
KB		 8039static void pqi_crash_if_pending_command(struct pqi_ctrl_info *ctrl_info)
8040{
8041 unsigned int i;
8042 struct pqi_io_request *io_request;
8043 struct scsi_cmnd *scmd;
8044
8045 for (i = 0; i < ctrl_info->max_io_slots; i++) {
8046 io_request = &ctrl_info->io_request_pool[i];
8047 if (atomic_read(&io_request->refcount) == 0)
8048 continue;
8049 scmd = io_request->scmd;
8050 WARN_ON(scmd != NULL); /* IO command from SML */
8051 WARN_ON(scmd == NULL); /* Non-IO cmd or driver initiated*/
8052 }
8053}
8054
d91d7820 8055static void pqi_shutdown(struct pci_dev *pci_dev)
6c223761
KB		 8056{
8057 int rc;
8058 struct pqi_ctrl_info *ctrl_info;
8059
d91d7820 8060 ctrl_info = pci_get_drvdata(pci_dev);
0530736e
KB		 8061 if (!ctrl_info) {
8062 dev_err(&pci_dev->dev,
8063 "cache could not be flushed\n");
8064 return;
8065 }
8066
8067 pqi_disable_events(ctrl_info);
8068 pqi_wait_until_ofa_finished(ctrl_info);
8069 pqi_cancel_update_time_worker(ctrl_info);
8070 pqi_cancel_rescan_worker(ctrl_info);
8071 pqi_cancel_event_worker(ctrl_info);
8072
8073 pqi_ctrl_shutdown_start(ctrl_info);
8074 pqi_ctrl_wait_until_quiesced(ctrl_info);
8075
8076 rc = pqi_ctrl_wait_for_pending_io(ctrl_info, NO_TIMEOUT);
8077 if (rc) {
8078 dev_err(&pci_dev->dev,
8079 "wait for pending I/O failed\n");
8080 return;
8081 }
8082
8083 pqi_ctrl_block_device_reset(ctrl_info);
8084 pqi_wait_until_lun_reset_finished(ctrl_info);
6c223761
KB		 8085
8086 /*
8087 * Write all data in the controller's battery-backed cache to
8088 * storage.
8089 */
58322fe0 8090 rc = pqi_flush_cache(ctrl_info, SHUTDOWN);
0530736e
KB		 8091 if (rc)
8092 dev_err(&pci_dev->dev,
8093 "unable to flush controller cache\n");
8094
8095 pqi_ctrl_block_requests(ctrl_info);
8096
8097 rc = pqi_ctrl_wait_for_pending_sync_cmds(ctrl_info);
8098 if (rc) {
8099 dev_err(&pci_dev->dev,
8100 "wait for pending sync cmds failed\n");
6c223761 8101 return;
0530736e
KB		 8102 }
8103
8104 pqi_crash_if_pending_command(ctrl_info);
8105 pqi_reset(ctrl_info);
6c223761
KB		 8106}
8107
3c50976f
KB		 8108static void pqi_process_lockup_action_param(void)
8109{
8110 unsigned int i;
8111
8112 if (!pqi_lockup_action_param)
8113 return;
8114
8115 for (i = 0; i < ARRAY_SIZE(pqi_lockup_actions); i++) {
8116 if (strcmp(pqi_lockup_action_param,
8117 pqi_lockup_actions[i].name) == 0) {
8118 pqi_lockup_action = pqi_lockup_actions[i].action;
8119 return;
8120 }
8121 }
8122
8123 pr_warn("%s: invalid lockup action setting \"%s\" - supported settings: none, reboot, panic\n",
8124 DRIVER_NAME_SHORT, pqi_lockup_action_param);
8125}
8126
8127static void pqi_process_module_params(void)
8128{
8129 pqi_process_lockup_action_param();
8130}
8131
5c146686 8132static __maybe_unused int pqi_suspend(struct pci_dev *pci_dev, pm_message_t state)
061ef06a
KB		 8133{
8134 struct pqi_ctrl_info *ctrl_info;
8135
8136 ctrl_info = pci_get_drvdata(pci_dev);
8137
8138 pqi_disable_events(ctrl_info);
8139 pqi_cancel_update_time_worker(ctrl_info);
8140 pqi_cancel_rescan_worker(ctrl_info);
8141 pqi_wait_until_scan_finished(ctrl_info);
8142 pqi_wait_until_lun_reset_finished(ctrl_info);
4fd22c13 8143 pqi_wait_until_ofa_finished(ctrl_info);
58322fe0 8144 pqi_flush_cache(ctrl_info, SUSPEND);
061ef06a
KB		 8145 pqi_ctrl_block_requests(ctrl_info);
8146 pqi_ctrl_wait_until_quiesced(ctrl_info);
8147 pqi_wait_until_inbound_queues_empty(ctrl_info);
4fd22c13 8148 pqi_ctrl_wait_for_pending_io(ctrl_info, NO_TIMEOUT);
061ef06a
KB		 8149 pqi_stop_heartbeat_timer(ctrl_info);
8150
8151 if (state.event == PM_EVENT_FREEZE)
8152 return 0;
8153
8154 pci_save_state(pci_dev);
8155 pci_set_power_state(pci_dev, pci_choose_state(pci_dev, state));
8156
8157 ctrl_info->controller_online = false;
8158 ctrl_info->pqi_mode_enabled = false;
8159
8160 return 0;
8161}
8162
5c146686 8163static __maybe_unused int pqi_resume(struct pci_dev *pci_dev)
061ef06a
KB		 8164{
8165 int rc;
8166 struct pqi_ctrl_info *ctrl_info;
8167
8168 ctrl_info = pci_get_drvdata(pci_dev);
8169
8170 if (pci_dev->current_state != PCI_D0) {
8171 ctrl_info->max_hw_queue_index = 0;
8172 pqi_free_interrupts(ctrl_info);
8173 pqi_change_irq_mode(ctrl_info, IRQ_MODE_INTX);
8174 rc = request_irq(pci_irq_vector(pci_dev, 0), pqi_irq_handler,
8175 IRQF_SHARED, DRIVER_NAME_SHORT,
8176 &ctrl_info->queue_groups[0]);
8177 if (rc) {
8178 dev_err(&ctrl_info->pci_dev->dev,
8179 "irq %u init failed with error %d\n",
8180 pci_dev->irq, rc);
8181 return rc;
8182 }
8183 pqi_start_heartbeat_timer(ctrl_info);
8184 pqi_ctrl_unblock_requests(ctrl_info);
8185 return 0;
8186 }
8187
8188 pci_set_power_state(pci_dev, PCI_D0);
8189 pci_restore_state(pci_dev);
8190
8191 return pqi_ctrl_init_resume(ctrl_info);
8192}
8193
6c223761
KB		 8194/* Define the PCI IDs for the controllers that we support. */
8195static const struct pci_device_id pqi_pci_id_table[] = {
b0f9408b
KB		 8196 {
8197 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8198 0x105b, 0x1211)
8199 },
8200 {
8201 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8202 0x105b, 0x1321)
8203 },
7eddabff
KB		 8204 {
8205 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8206 0x152d, 0x8a22)
8207 },
8208 {
8209 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8210 0x152d, 0x8a23)
8211 },
8212 {
8213 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8214 0x152d, 0x8a24)
8215 },
8216 {
8217 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8218 0x152d, 0x8a36)
8219 },
8220 {
8221 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8222 0x152d, 0x8a37)
8223 },
0595a0b4
AK		 8224 {
8225 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8226 0x193d, 0x1104)
8227 },
8228 {
8229 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8230 0x193d, 0x1105)
8231 },
8232 {
8233 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8234 0x193d, 0x1106)
8235 },
8236 {
8237 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8238 0x193d, 0x1107)
8239 },
b0f9408b
KB		 8240 {
8241 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8242 0x193d, 0x8460)
8243 },
8244 {
8245 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8246 0x193d, 0x8461)
8247 },
84a77fef
MB		 8248 {
8249 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8250 0x193d, 0xc460)
8251 },
8252 {
8253 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8254 0x193d, 0xc461)
8255 },
b0f9408b
KB		 8256 {
8257 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8258 0x193d, 0xf460)
8259 },
8260 {
8261 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8262 0x193d, 0xf461)
8263 },
8264 {
8265 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8266 0x1bd4, 0x0045)
8267 },
8268 {
8269 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8270 0x1bd4, 0x0046)
8271 },
8272 {
8273 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8274 0x1bd4, 0x0047)
8275 },
8276 {
8277 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8278 0x1bd4, 0x0048)
8279 },
9f8d05fa
KB		 8280 {
8281 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8282 0x1bd4, 0x004a)
8283 },
8284 {
8285 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8286 0x1bd4, 0x004b)
8287 },
8288 {
8289 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8290 0x1bd4, 0x004c)
8291 },
63a7956a
GW		 8292 {
8293 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8294 0x1bd4, 0x004f)
8295 },
c1b10475
AK		 8296 {
8297 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8298 0x19e5, 0xd227)
8299 },
8300 {
8301 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8302 0x19e5, 0xd228)
8303 },
8304 {
8305 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8306 0x19e5, 0xd229)
8307 },
8308 {
8309 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8310 0x19e5, 0xd22a)
8311 },
8312 {
8313 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8314 0x19e5, 0xd22b)
8315 },
8316 {
8317 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8318 0x19e5, 0xd22c)
8319 },
6c223761
KB		 8320 {
8321 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8322 PCI_VENDOR_ID_ADAPTEC2, 0x0110)
8323 },
8324 {
8325 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
55790064 8326 PCI_VENDOR_ID_ADAPTEC2, 0x0608)
6c223761
KB		 8327 },
8328 {
8329 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8330 PCI_VENDOR_ID_ADAPTEC2, 0x0800)
6c223761
KB		 8331 },
8332 {
8333 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8334 PCI_VENDOR_ID_ADAPTEC2, 0x0801)
6c223761
KB		 8335 },
8336 {
8337 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8338 PCI_VENDOR_ID_ADAPTEC2, 0x0802)
6c223761
KB		 8339 },
8340 {
8341 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8342 PCI_VENDOR_ID_ADAPTEC2, 0x0803)
6c223761
KB		 8343 },
8344 {
8345 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8346 PCI_VENDOR_ID_ADAPTEC2, 0x0804)
6c223761
KB		 8347 },
8348 {
8349 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8350 PCI_VENDOR_ID_ADAPTEC2, 0x0805)
6c223761
KB		 8351 },
8352 {
8353 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8354 PCI_VENDOR_ID_ADAPTEC2, 0x0806)
6c223761 8355 },
55790064
KB		 8356 {
8357 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8358 PCI_VENDOR_ID_ADAPTEC2, 0x0807)
8359 },
63a7956a
GW		 8360 {
8361 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8362 PCI_VENDOR_ID_ADAPTEC2, 0x0808)
8363 },
8364 {
8365 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8366 PCI_VENDOR_ID_ADAPTEC2, 0x0809)
8367 },
3af06083
MR		 8368 {
8369 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8370 PCI_VENDOR_ID_ADAPTEC2, 0x080a)
8371 },
6c223761
KB		 8372 {
8373 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8374 PCI_VENDOR_ID_ADAPTEC2, 0x0900)
6c223761
KB		 8375 },
8376 {
8377 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8378 PCI_VENDOR_ID_ADAPTEC2, 0x0901)
6c223761
KB		 8379 },
8380 {
8381 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8382 PCI_VENDOR_ID_ADAPTEC2, 0x0902)
6c223761
KB		 8383 },
8384 {
8385 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8386 PCI_VENDOR_ID_ADAPTEC2, 0x0903)
6c223761
KB		 8387 },
8388 {
8389 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8390 PCI_VENDOR_ID_ADAPTEC2, 0x0904)
6c223761
KB		 8391 },
8392 {
8393 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8394 PCI_VENDOR_ID_ADAPTEC2, 0x0905)
6c223761
KB		 8395 },
8396 {
8397 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8398 PCI_VENDOR_ID_ADAPTEC2, 0x0906)
6c223761
KB		 8399 },
8400 {
8401 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8402 PCI_VENDOR_ID_ADAPTEC2, 0x0907)
6c223761
KB		 8403 },
8404 {
8405 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8406 PCI_VENDOR_ID_ADAPTEC2, 0x0908)
6c223761 8407 },
55790064
KB		 8408 {
8409 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8410 PCI_VENDOR_ID_ADAPTEC2, 0x090a)
8411 },
6c223761
KB		 8412 {
8413 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8414 PCI_VENDOR_ID_ADAPTEC2, 0x1200)
6c223761
KB		 8415 },
8416 {
8417 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8418 PCI_VENDOR_ID_ADAPTEC2, 0x1201)
6c223761
KB		 8419 },
8420 {
8421 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8422 PCI_VENDOR_ID_ADAPTEC2, 0x1202)
6c223761
KB		 8423 },
8424 {
8425 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8426 PCI_VENDOR_ID_ADAPTEC2, 0x1280)
6c223761
KB		 8427 },
8428 {
8429 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8430 PCI_VENDOR_ID_ADAPTEC2, 0x1281)
6c223761 8431 },
b0f9408b
KB		 8432 {
8433 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8434 PCI_VENDOR_ID_ADAPTEC2, 0x1282)
8435 },
6c223761
KB		 8436 {
8437 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8438 PCI_VENDOR_ID_ADAPTEC2, 0x1300)
6c223761
KB		 8439 },
8440 {
8441 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8442 PCI_VENDOR_ID_ADAPTEC2, 0x1301)
6c223761 8443 },
bd809e8d
KB		 8444 {
8445 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8446 PCI_VENDOR_ID_ADAPTEC2, 0x1302)
8447 },
8448 {
8449 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8450 PCI_VENDOR_ID_ADAPTEC2, 0x1303)
8451 },
6c223761
KB		 8452 {
8453 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff
KB		 8454 PCI_VENDOR_ID_ADAPTEC2, 0x1380)
8455 },
9f8d05fa
KB		 8456 {
8457 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8458 PCI_VENDOR_ID_ADVANTECH, 0x8312)
8459 },
55790064
KB		 8460 {
8461 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8462 PCI_VENDOR_ID_DELL, 0x1fe0)
8463 },
7eddabff
KB		 8464 {
8465 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8466 PCI_VENDOR_ID_HP, 0x0600)
8467 },
8468 {
8469 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8470 PCI_VENDOR_ID_HP, 0x0601)
8471 },
8472 {
8473 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8474 PCI_VENDOR_ID_HP, 0x0602)
8475 },
8476 {
8477 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8478 PCI_VENDOR_ID_HP, 0x0603)
8479 },
8480 {
8481 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
55790064 8482 PCI_VENDOR_ID_HP, 0x0609)
7eddabff
KB		 8483 },
8484 {
8485 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8486 PCI_VENDOR_ID_HP, 0x0650)
8487 },
8488 {
8489 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8490 PCI_VENDOR_ID_HP, 0x0651)
8491 },
8492 {
8493 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8494 PCI_VENDOR_ID_HP, 0x0652)
8495 },
8496 {
8497 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8498 PCI_VENDOR_ID_HP, 0x0653)
8499 },
8500 {
8501 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8502 PCI_VENDOR_ID_HP, 0x0654)
8503 },
8504 {
8505 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8506 PCI_VENDOR_ID_HP, 0x0655)
8507 },
7eddabff
KB		 8508 {
8509 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8510 PCI_VENDOR_ID_HP, 0x0700)
8511 },
8512 {
8513 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8514 PCI_VENDOR_ID_HP, 0x0701)
6c223761
KB		 8515 },
8516 {
8517 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8518 PCI_VENDOR_ID_HP, 0x1001)
8519 },
8520 {
8521 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8522 PCI_VENDOR_ID_HP, 0x1100)
8523 },
8524 {
8525 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8526 PCI_VENDOR_ID_HP, 0x1101)
8527 },
8bdb3b9c
GW		 8528 {
8529 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8530 0x1d8d, 0x0800)
8531 },
8532 {
8533 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8534 0x1d8d, 0x0908)
8535 },
8536 {
8537 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8538 0x1d8d, 0x0806)
8539 },
8540 {
8541 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8542 0x1d8d, 0x0916)
8543 },
71ecc60d
GW		 8544 {
8545 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8546 PCI_VENDOR_ID_GIGABYTE, 0x1000)
8547 },
6c223761
KB		 8548 {
8549 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8550 PCI_ANY_ID, PCI_ANY_ID)
8551 },
8552 { 0 }
8553};
8554
8555MODULE_DEVICE_TABLE(pci, pqi_pci_id_table);
8556
8557static struct pci_driver pqi_pci_driver = {
8558 .name = DRIVER_NAME_SHORT,
8559 .id_table = pqi_pci_id_table,
8560 .probe = pqi_pci_probe,
8561 .remove = pqi_pci_remove,
8562 .shutdown = pqi_shutdown,
061ef06a
KB		 8563#if defined(CONFIG_PM)
8564 .suspend = pqi_suspend,
8565 .resume = pqi_resume,
8566#endif
6c223761
KB		 8567};
8568
8569static int __init pqi_init(void)
8570{
8571 int rc;
8572
8573 pr_info(DRIVER_NAME "\n");
8574
8b664fef 8575 pqi_sas_transport_template = sas_attach_transport(&pqi_sas_transport_functions);
6c223761
KB		 8576 if (!pqi_sas_transport_template)
8577 return -ENODEV;
8578
3c50976f
KB		 8579 pqi_process_module_params();
8580
6c223761
KB		 8581 rc = pci_register_driver(&pqi_pci_driver);
8582 if (rc)
8583 sas_release_transport(pqi_sas_transport_template);
8584
8585 return rc;
8586}
8587
8588static void __exit pqi_cleanup(void)
8589{
8590 pci_unregister_driver(&pqi_pci_driver);
8591 sas_release_transport(pqi_sas_transport_template);
8592}
8593
8594module_init(pqi_init);
8595module_exit(pqi_cleanup);
8596
8597static void __attribute__((unused)) verify_structures(void)
8598{
8599 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
8600 sis_host_to_ctrl_doorbell) != 0x20);
8601 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
8602 sis_interrupt_mask) != 0x34);
8603 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
8604 sis_ctrl_to_host_doorbell) != 0x9c);
8605 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
8606 sis_ctrl_to_host_doorbell_clear) != 0xa0);
ff6abb73
KB		 8607 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
8608 sis_driver_scratch) != 0xb0);
6c223761
KB		 8609 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
8610 sis_firmware_status) != 0xbc);
8611 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
8612 sis_mailbox) != 0x1000);
8613 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
8614 pqi_registers) != 0x4000);
8615
8616 BUILD_BUG_ON(offsetof(struct pqi_iu_header,
8617 iu_type) != 0x0);
8618 BUILD_BUG_ON(offsetof(struct pqi_iu_header,
8619 iu_length) != 0x2);
8620 BUILD_BUG_ON(offsetof(struct pqi_iu_header,
8621 response_queue_id) != 0x4);
8622 BUILD_BUG_ON(offsetof(struct pqi_iu_header,
8623 work_area) != 0x6);
8624 BUILD_BUG_ON(sizeof(struct pqi_iu_header) != 0x8);
8625
8626 BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
8627 status) != 0x0);
8628 BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
8629 service_response) != 0x1);
8630 BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
8631 data_present) != 0x2);
8632 BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
8633 reserved) != 0x3);
8634 BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
8635 residual_count) != 0x4);
8636 BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
8637 data_length) != 0x8);
8638 BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
8639 reserved1) != 0xa);
8640 BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
8641 data) != 0xc);
8642 BUILD_BUG_ON(sizeof(struct pqi_aio_error_info) != 0x10c);
8643
8644 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
8645 data_in_result) != 0x0);
8646 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
8647 data_out_result) != 0x1);
8648 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
8649 reserved) != 0x2);
8650 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
8651 status) != 0x5);
8652 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
8653 status_qualifier) != 0x6);
8654 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
8655 sense_data_length) != 0x8);
8656 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
8657 response_data_length) != 0xa);
8658 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
8659 data_in_transferred) != 0xc);
8660 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
8661 data_out_transferred) != 0x10);
8662 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
8663 data) != 0x14);
8664 BUILD_BUG_ON(sizeof(struct pqi_raid_error_info) != 0x114);
8665
8666 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
8667 signature) != 0x0);
8668 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
8669 function_and_status_code) != 0x8);
8670 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
8671 max_admin_iq_elements) != 0x10);
8672 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
8673 max_admin_oq_elements) != 0x11);
8674 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
8675 admin_iq_element_length) != 0x12);
8676 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
8677 admin_oq_element_length) != 0x13);
8678 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
8679 max_reset_timeout) != 0x14);
8680 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
8681 legacy_intx_status) != 0x18);
8682 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
8683 legacy_intx_mask_set) != 0x1c);
8684 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
8685 legacy_intx_mask_clear) != 0x20);
8686 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
8687 device_status) != 0x40);
8688 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
8689 admin_iq_pi_offset) != 0x48);
8690 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
8691 admin_oq_ci_offset) != 0x50);
8692 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
8693 admin_iq_element_array_addr) != 0x58);
8694 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
8695 admin_oq_element_array_addr) != 0x60);
8696 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
8697 admin_iq_ci_addr) != 0x68);
8698 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
8699 admin_oq_pi_addr) != 0x70);
8700 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
8701 admin_iq_num_elements) != 0x78);
8702 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
8703 admin_oq_num_elements) != 0x79);
8704 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
8705 admin_queue_int_msg_num) != 0x7a);
8706 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
8707 device_error) != 0x80);
8708 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
8709 error_details) != 0x88);
8710 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
8711 device_reset) != 0x90);
8712 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
8713 power_action) != 0x94);
8714 BUILD_BUG_ON(sizeof(struct pqi_device_registers) != 0x100);
8715
8716 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
8717 header.iu_type) != 0);
8718 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
8719 header.iu_length) != 2);
8720 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
8721 header.work_area) != 6);
8722 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
8723 request_id) != 8);
8724 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
8725 function_code) != 10);
8726 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
8727 data.report_device_capability.buffer_length) != 44);
8728 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
8729 data.report_device_capability.sg_descriptor) != 48);
8730 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
8731 data.create_operational_iq.queue_id) != 12);
8732 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
8733 data.create_operational_iq.element_array_addr) != 16);
8734 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
8735 data.create_operational_iq.ci_addr) != 24);
8736 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
8737 data.create_operational_iq.num_elements) != 32);
8738 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
8739 data.create_operational_iq.element_length) != 34);
8740 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
8741 data.create_operational_iq.queue_protocol) != 36);
8742 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
8743 data.create_operational_oq.queue_id) != 12);
8744 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
8745 data.create_operational_oq.element_array_addr) != 16);
8746 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
8747 data.create_operational_oq.pi_addr) != 24);
8748 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
8749 data.create_operational_oq.num_elements) != 32);
8750 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
8751 data.create_operational_oq.element_length) != 34);
8752 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
8753 data.create_operational_oq.queue_protocol) != 36);
8754 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
8755 data.create_operational_oq.int_msg_num) != 40);
8756 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
8757 data.create_operational_oq.coalescing_count) != 42);
8758 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
8759 data.create_operational_oq.min_coalescing_time) != 44);
8760 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
8761 data.create_operational_oq.max_coalescing_time) != 48);
8762 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
8763 data.delete_operational_queue.queue_id) != 12);
8764 BUILD_BUG_ON(sizeof(struct pqi_general_admin_request) != 64);
c593642c 8765 BUILD_BUG_ON(sizeof_field(struct pqi_general_admin_request,
6c223761 8766 data.create_operational_iq) != 64 - 11);
c593642c 8767 BUILD_BUG_ON(sizeof_field(struct pqi_general_admin_request,
6c223761 8768 data.create_operational_oq) != 64 - 11);
c593642c 8769 BUILD_BUG_ON(sizeof_field(struct pqi_general_admin_request,
6c223761
KB		 8770 data.delete_operational_queue) != 64 - 11);
8771
8772 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
8773 header.iu_type) != 0);
8774 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
8775 header.iu_length) != 2);
8776 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
8777 header.work_area) != 6);
8778 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
8779 request_id) != 8);
8780 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
8781 function_code) != 10);
8782 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
8783 status) != 11);
8784 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
8785 data.create_operational_iq.status_descriptor) != 12);
8786 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
8787 data.create_operational_iq.iq_pi_offset) != 16);
8788 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
8789 data.create_operational_oq.status_descriptor) != 12);
8790 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
8791 data.create_operational_oq.oq_ci_offset) != 16);
8792 BUILD_BUG_ON(sizeof(struct pqi_general_admin_response) != 64);
8793
8794 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
8795 header.iu_type) != 0);
8796 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
8797 header.iu_length) != 2);
8798 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
8799 header.response_queue_id) != 4);
8800 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
8801 header.work_area) != 6);
8802 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
8803 request_id) != 8);
8804 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
8805 nexus_id) != 10);
8806 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
8807 buffer_length) != 12);
8808 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
8809 lun_number) != 16);
8810 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
8811 protocol_specific) != 24);
8812 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
8813 error_index) != 27);
8814 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
8815 cdb) != 32);
21432010 8816 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
8817 timeout) != 60);
6c223761
KB		 8818 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
8819 sg_descriptors) != 64);
8820 BUILD_BUG_ON(sizeof(struct pqi_raid_path_request) !=
8821 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
8822
8823 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
8824 header.iu_type) != 0);
8825 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
8826 header.iu_length) != 2);
8827 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
8828 header.response_queue_id) != 4);
8829 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
8830 header.work_area) != 6);
8831 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
8832 request_id) != 8);
8833 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
8834 nexus_id) != 12);
8835 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
8836 buffer_length) != 16);
8837 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
8838 data_encryption_key_index) != 22);
8839 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
8840 encrypt_tweak_lower) != 24);
8841 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
8842 encrypt_tweak_upper) != 28);
8843 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
8844 cdb) != 32);
8845 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
8846 error_index) != 48);
8847 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
8848 num_sg_descriptors) != 50);
8849 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
8850 cdb_length) != 51);
8851 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
8852 lun_number) != 52);
8853 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
8854 sg_descriptors) != 64);
8855 BUILD_BUG_ON(sizeof(struct pqi_aio_path_request) !=
8856 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
8857
8858 BUILD_BUG_ON(offsetof(struct pqi_io_response,
8859 header.iu_type) != 0);
8860 BUILD_BUG_ON(offsetof(struct pqi_io_response,
8861 header.iu_length) != 2);
8862 BUILD_BUG_ON(offsetof(struct pqi_io_response,
8863 request_id) != 8);
8864 BUILD_BUG_ON(offsetof(struct pqi_io_response,
8865 error_index) != 10);
8866
8867 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
8868 header.iu_type) != 0);
8869 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
8870 header.iu_length) != 2);
8871 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
8872 header.response_queue_id) != 4);
8873 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
8874 request_id) != 8);
8875 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
8876 data.report_event_configuration.buffer_length) != 12);
8877 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
8878 data.report_event_configuration.sg_descriptors) != 16);
8879 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
8880 data.set_event_configuration.global_event_oq_id) != 10);
8881 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
8882 data.set_event_configuration.buffer_length) != 12);
8883 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
8884 data.set_event_configuration.sg_descriptors) != 16);
8885
8886 BUILD_BUG_ON(offsetof(struct pqi_iu_layer_descriptor,
8887 max_inbound_iu_length) != 6);
8888 BUILD_BUG_ON(offsetof(struct pqi_iu_layer_descriptor,
8889 max_outbound_iu_length) != 14);
8890 BUILD_BUG_ON(sizeof(struct pqi_iu_layer_descriptor) != 16);
8891
8892 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
8893 data_length) != 0);
8894 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
8895 iq_arbitration_priority_support_bitmask) != 8);
8896 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
8897 maximum_aw_a) != 9);
8898 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
8899 maximum_aw_b) != 10);
8900 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
8901 maximum_aw_c) != 11);
8902 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
8903 max_inbound_queues) != 16);
8904 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
8905 max_elements_per_iq) != 18);
8906 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
8907 max_iq_element_length) != 24);
8908 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
8909 min_iq_element_length) != 26);
8910 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
8911 max_outbound_queues) != 30);
8912 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
8913 max_elements_per_oq) != 32);
8914 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
8915 intr_coalescing_time_granularity) != 34);
8916 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
8917 max_oq_element_length) != 36);
8918 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
8919 min_oq_element_length) != 38);
8920 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
8921 iu_layer_descriptors) != 64);
8922 BUILD_BUG_ON(sizeof(struct pqi_device_capability) != 576);
8923
8924 BUILD_BUG_ON(offsetof(struct pqi_event_descriptor,
8925 event_type) != 0);
8926 BUILD_BUG_ON(offsetof(struct pqi_event_descriptor,
8927 oq_id) != 2);
8928 BUILD_BUG_ON(sizeof(struct pqi_event_descriptor) != 4);
8929
8930 BUILD_BUG_ON(offsetof(struct pqi_event_config,
8931 num_event_descriptors) != 2);
8932 BUILD_BUG_ON(offsetof(struct pqi_event_config,
8933 descriptors) != 4);
8934
061ef06a
KB		 8935 BUILD_BUG_ON(PQI_NUM_SUPPORTED_EVENTS !=
8936 ARRAY_SIZE(pqi_supported_event_types));
8937
6c223761
KB		 8938 BUILD_BUG_ON(offsetof(struct pqi_event_response,
8939 header.iu_type) != 0);
8940 BUILD_BUG_ON(offsetof(struct pqi_event_response,
8941 header.iu_length) != 2);
8942 BUILD_BUG_ON(offsetof(struct pqi_event_response,
8943 event_type) != 8);
8944 BUILD_BUG_ON(offsetof(struct pqi_event_response,
8945 event_id) != 10);
8946 BUILD_BUG_ON(offsetof(struct pqi_event_response,
8947 additional_event_id) != 12);
8948 BUILD_BUG_ON(offsetof(struct pqi_event_response,
8949 data) != 16);
8950 BUILD_BUG_ON(sizeof(struct pqi_event_response) != 32);
8951
8952 BUILD_BUG_ON(offsetof(struct pqi_event_acknowledge_request,
8953 header.iu_type) != 0);
8954 BUILD_BUG_ON(offsetof(struct pqi_event_acknowledge_request,
8955 header.iu_length) != 2);
8956 BUILD_BUG_ON(offsetof(struct pqi_event_acknowledge_request,
8957 event_type) != 8);
8958 BUILD_BUG_ON(offsetof(struct pqi_event_acknowledge_request,
8959 event_id) != 10);
8960 BUILD_BUG_ON(offsetof(struct pqi_event_acknowledge_request,
8961 additional_event_id) != 12);
8962 BUILD_BUG_ON(sizeof(struct pqi_event_acknowledge_request) != 16);
8963
8964 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
8965 header.iu_type) != 0);
8966 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
8967 header.iu_length) != 2);
8968 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
8969 request_id) != 8);
8970 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
8971 nexus_id) != 10);
8972 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
c2922f17
MB		 8973 timeout) != 14);
8974 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
6c223761
KB		 8975 lun_number) != 16);
8976 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
8977 protocol_specific) != 24);
8978 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
8979 outbound_queue_id_to_manage) != 26);
8980 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
8981 request_id_to_manage) != 28);
8982 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
8983 task_management_function) != 30);
8984 BUILD_BUG_ON(sizeof(struct pqi_task_management_request) != 32);
8985
8986 BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
8987 header.iu_type) != 0);
8988 BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
8989 header.iu_length) != 2);
8990 BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
8991 request_id) != 8);
8992 BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
8993 nexus_id) != 10);
8994 BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
8995 additional_response_info) != 12);
8996 BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
8997 response_code) != 15);
8998 BUILD_BUG_ON(sizeof(struct pqi_task_management_response) != 16);
8999
9000 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
9001 configured_logical_drive_count) != 0);
9002 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
9003 configuration_signature) != 1);
9004 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
9005 firmware_version) != 5);
9006 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
9007 extended_logical_unit_count) != 154);
9008 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
9009 firmware_build_number) != 190);
9010 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
9011 controller_mode) != 292);
9012
1be42f46
KB		 9013 BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
9014 phys_bay_in_box) != 115);
9015 BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
9016 device_type) != 120);
9017 BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
9018 redundant_path_present_map) != 1736);
9019 BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
9020 active_path_number) != 1738);
9021 BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
9022 alternate_paths_phys_connector) != 1739);
9023 BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
9024 alternate_paths_phys_box_on_port) != 1755);
9025 BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
9026 current_queue_depth_limit) != 1796);
9027 BUILD_BUG_ON(sizeof(struct bmic_identify_physical_device) != 2560);
9028
6c223761
KB		 9029 BUILD_BUG_ON(PQI_ADMIN_IQ_NUM_ELEMENTS > 255);
9030 BUILD_BUG_ON(PQI_ADMIN_OQ_NUM_ELEMENTS > 255);
9031 BUILD_BUG_ON(PQI_ADMIN_IQ_ELEMENT_LENGTH %
9032 PQI_QUEUE_ELEMENT_LENGTH_ALIGNMENT != 0);
9033 BUILD_BUG_ON(PQI_ADMIN_OQ_ELEMENT_LENGTH %
9034 PQI_QUEUE_ELEMENT_LENGTH_ALIGNMENT != 0);
9035 BUILD_BUG_ON(PQI_OPERATIONAL_IQ_ELEMENT_LENGTH > 1048560);
9036 BUILD_BUG_ON(PQI_OPERATIONAL_IQ_ELEMENT_LENGTH %
9037 PQI_QUEUE_ELEMENT_LENGTH_ALIGNMENT != 0);
9038 BUILD_BUG_ON(PQI_OPERATIONAL_OQ_ELEMENT_LENGTH > 1048560);
9039 BUILD_BUG_ON(PQI_OPERATIONAL_OQ_ELEMENT_LENGTH %
9040 PQI_QUEUE_ELEMENT_LENGTH_ALIGNMENT != 0);
9041
9042 BUILD_BUG_ON(PQI_RESERVED_IO_SLOTS >= PQI_MAX_OUTSTANDING_REQUESTS);
d727a776
KB		 9043 BUILD_BUG_ON(PQI_RESERVED_IO_SLOTS >=
9044 PQI_MAX_OUTSTANDING_REQUESTS_KDUMP);
6c223761 9045}
Linux 6.x block layer and io_uring tree(s)