projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
scsi: core: Stop using DRIVER_ERROR
[linux-2.6-block.git] / drivers / scsi / smartpqi / smartpqi_init.c
CommitLineData
2cc37b15 1// SPDX-License-Identifier: GPL-2.0
6c223761
KB		 2/*
3 * driver for Microsemi PQI-based storage controllers
2a712681 4 * Copyright (c) 2019-2020 Microchip Technology Inc. and its subsidiaries
2f4c4b92 5 * Copyright (c) 2016-2018 Microsemi Corporation
6c223761
KB		 6 * Copyright (c) 2016 PMC-Sierra, Inc.
7 *
2f4c4b92 8 * Questions/Comments/Bugfixes to storagedev@microchip.com
6c223761
KB		 9 *
10 */
11
12#include <linux/module.h>
13#include <linux/kernel.h>
14#include <linux/pci.h>
15#include <linux/delay.h>
16#include <linux/interrupt.h>
17#include <linux/sched.h>
18#include <linux/rtc.h>
19#include <linux/bcd.h>
3c50976f 20#include <linux/reboot.h>
6c223761 21#include <linux/cciss_ioctl.h>
52198226 22#include <linux/blk-mq-pci.h>
6c223761
KB		 23#include <scsi/scsi_host.h>
24#include <scsi/scsi_cmnd.h>
25#include <scsi/scsi_device.h>
26#include <scsi/scsi_eh.h>
27#include <scsi/scsi_transport_sas.h>
28#include <asm/unaligned.h>
29#include "smartpqi.h"
30#include "smartpqi_sis.h"
31
32#if !defined(BUILD_TIMESTAMP)
33#define BUILD_TIMESTAMP
34#endif
35
d56030f8
DB		 36#define DRIVER_VERSION "2.1.8-045"
37#define DRIVER_MAJOR 2
38#define DRIVER_MINOR 1
39#define DRIVER_RELEASE 8
40#define DRIVER_REVISION 45
6c223761 41
2d154f5f
KB		 42#define DRIVER_NAME "Microsemi PQI Driver (v" \
43 DRIVER_VERSION BUILD_TIMESTAMP ")"
6c223761
KB		 44#define DRIVER_NAME_SHORT "smartpqi"
45
e1d213bd
KB		 46#define PQI_EXTRA_SGL_MEMORY (12 * sizeof(struct pqi_sg_descriptor))
47
2790cd4d
KB		 48#define PQI_POST_RESET_DELAY_SECS 5
49#define PQI_POST_OFA_RESET_DELAY_UPON_TIMEOUT_SECS 10
50
6c223761
KB		 51MODULE_AUTHOR("Microsemi");
52MODULE_DESCRIPTION("Driver for Microsemi Smart Family Controller version "
53 DRIVER_VERSION);
6c223761
KB		 54MODULE_VERSION(DRIVER_VERSION);
55MODULE_LICENSE("GPL");
56
6c223761 57static void pqi_take_ctrl_offline(struct pqi_ctrl_info *ctrl_info);
5f310425 58static void pqi_ctrl_offline_worker(struct work_struct *work);
6c223761
KB		 59static int pqi_scan_scsi_devices(struct pqi_ctrl_info *ctrl_info);
60static void pqi_scan_start(struct Scsi_Host *shost);
61static void pqi_start_io(struct pqi_ctrl_info *ctrl_info,
62 struct pqi_queue_group *queue_group, enum pqi_io_path path,
63 struct pqi_io_request *io_request);
64static int pqi_submit_raid_request_synchronous(struct pqi_ctrl_info *ctrl_info,
65 struct pqi_iu_header *request, unsigned int flags,
ae0c189d 66 struct pqi_raid_error_info *error_info);
6c223761
KB		 67static int pqi_aio_submit_io(struct pqi_ctrl_info *ctrl_info,
68 struct scsi_cmnd *scmd, u32 aio_handle, u8 *cdb,
69 unsigned int cdb_length, struct pqi_queue_group *queue_group,
376fb880 70 struct pqi_encryption_info *encryption_info, bool raid_bypass);
7a012c23
DB		 71static int pqi_aio_submit_r1_write_io(struct pqi_ctrl_info *ctrl_info,
72 struct scsi_cmnd *scmd, struct pqi_queue_group *queue_group,
73 struct pqi_encryption_info *encryption_info, struct pqi_scsi_dev *device,
74 struct pqi_scsi_dev_raid_map_data *rmd);
6702d2c4
DB		 75static int pqi_aio_submit_r56_write_io(struct pqi_ctrl_info *ctrl_info,
76 struct scsi_cmnd *scmd, struct pqi_queue_group *queue_group,
77 struct pqi_encryption_info *encryption_info, struct pqi_scsi_dev *device,
78 struct pqi_scsi_dev_raid_map_data *rmd);
4fd22c13
MR		 79static void pqi_ofa_ctrl_quiesce(struct pqi_ctrl_info *ctrl_info);
80static void pqi_ofa_ctrl_unquiesce(struct pqi_ctrl_info *ctrl_info);
2790cd4d
KB		 81static int pqi_ofa_ctrl_restart(struct pqi_ctrl_info *ctrl_info, unsigned int delay_secs);
82static void pqi_ofa_setup_host_buffer(struct pqi_ctrl_info *ctrl_info);
4fd22c13
MR		 83static void pqi_ofa_free_host_buffer(struct pqi_ctrl_info *ctrl_info);
84static int pqi_ofa_host_memory_update(struct pqi_ctrl_info *ctrl_info);
1e46731e 85static int pqi_device_wait_for_pending_io(struct pqi_ctrl_info *ctrl_info,
18ff5f08 86 struct pqi_scsi_dev *device, unsigned long timeout_msecs);
6c223761
KB		 87
88/* for flags argument to pqi_submit_raid_request_synchronous() */
89#define PQI_SYNC_FLAGS_INTERRUPTABLE 0x1
90
91static struct scsi_transport_template *pqi_sas_transport_template;
92
93static atomic_t pqi_controller_count = ATOMIC_INIT(0);
94
3c50976f
KB		 95enum pqi_lockup_action {
96 NONE,
97 REBOOT,
98 PANIC
99};
100
101static enum pqi_lockup_action pqi_lockup_action = NONE;
102
103static struct {
104 enum pqi_lockup_action action;
105 char *name;
106} pqi_lockup_actions[] = {
107 {
108 .action = NONE,
109 .name = "none",
110 },
111 {
112 .action = REBOOT,
113 .name = "reboot",
114 },
115 {
116 .action = PANIC,
117 .name = "panic",
118 },
119};
120
6a50d6ad
KB		 121static unsigned int pqi_supported_event_types[] = {
122 PQI_EVENT_TYPE_HOTPLUG,
123 PQI_EVENT_TYPE_HARDWARE,
124 PQI_EVENT_TYPE_PHYSICAL_DEVICE,
125 PQI_EVENT_TYPE_LOGICAL_DEVICE,
4fd22c13 126 PQI_EVENT_TYPE_OFA,
6a50d6ad
KB		 127 PQI_EVENT_TYPE_AIO_STATE_CHANGE,
128 PQI_EVENT_TYPE_AIO_CONFIG_CHANGE,
129};
130
6c223761
KB		 131static int pqi_disable_device_id_wildcards;
132module_param_named(disable_device_id_wildcards,
cbe0c7b1 133 pqi_disable_device_id_wildcards, int, 0644);
6c223761
KB		 134MODULE_PARM_DESC(disable_device_id_wildcards,
135 "Disable device ID wildcards.");
136
5a259e32
KB		 137static int pqi_disable_heartbeat;
138module_param_named(disable_heartbeat,
139 pqi_disable_heartbeat, int, 0644);
140MODULE_PARM_DESC(disable_heartbeat,
141 "Disable heartbeat.");
142
143static int pqi_disable_ctrl_shutdown;
144module_param_named(disable_ctrl_shutdown,
145 pqi_disable_ctrl_shutdown, int, 0644);
146MODULE_PARM_DESC(disable_ctrl_shutdown,
147 "Disable controller shutdown when controller locked up.");
148
3c50976f
KB		 149static char *pqi_lockup_action_param;
150module_param_named(lockup_action,
151 pqi_lockup_action_param, charp, 0644);
152MODULE_PARM_DESC(lockup_action, "Action to take when controller locked up.\n"
153 "\t\tSupported: none, reboot, panic\n"
154 "\t\tDefault: none");
155
5e6a9760
GW		 156static int pqi_expose_ld_first;
157module_param_named(expose_ld_first,
158 pqi_expose_ld_first, int, 0644);
583891c9 159MODULE_PARM_DESC(expose_ld_first, "Expose logical drives before physical drives.");
5e6a9760 160
522bc026
DC		 161static int pqi_hide_vsep;
162module_param_named(hide_vsep,
163 pqi_hide_vsep, int, 0644);
583891c9 164MODULE_PARM_DESC(hide_vsep, "Hide the virtual SEP for direct attached drives.");
522bc026 165
6c223761
KB		 166static char *raid_levels[] = {
167 "RAID-0",
168 "RAID-4",
169 "RAID-1(1+0)",
170 "RAID-5",
171 "RAID-5+1",
7a012c23
DB		 172 "RAID-6",
173 "RAID-1(Triple)",
6c223761
KB		 174};
175
176static char *pqi_raid_level_to_string(u8 raid_level)
177{
178 if (raid_level < ARRAY_SIZE(raid_levels))
179 return raid_levels[raid_level];
180
a9f93392 181 return "RAID UNKNOWN";
6c223761
KB		 182}
183
184#define SA_RAID_0 0
185#define SA_RAID_4 1
186#define SA_RAID_1 2 /* also used for RAID 10 */
187#define SA_RAID_5 3 /* also used for RAID 50 */
188#define SA_RAID_51 4
189#define SA_RAID_6 5 /* also used for RAID 60 */
7a012c23
DB		 190#define SA_RAID_TRIPLE 6 /* also used for RAID 1+0 Triple */
191#define SA_RAID_MAX SA_RAID_TRIPLE
6c223761
KB		 192#define SA_RAID_UNKNOWN 0xff
193
194static inline void pqi_scsi_done(struct scsi_cmnd *scmd)
195{
7561a7e4 196 pqi_prep_for_scsi_done(scmd);
6c223761
KB		 197 scmd->scsi_done(scmd);
198}
199
b6e2ef67 200static inline void pqi_disable_write_same(struct scsi_device *sdev)
6c223761 201{
b6e2ef67 202 sdev->no_write_same = 1;
6c223761
KB		 203}
204
6c223761 205static inline bool pqi_scsi3addr_equal(u8 *scsi3addr1, u8 *scsi3addr2)
6c223761 206{
6c223761 207 return memcmp(scsi3addr1, scsi3addr2, 8) == 0;
6c223761
KB		 208}
209
210static inline bool pqi_is_logical_device(struct pqi_scsi_dev *device)
211{
212 return !device->is_physical_device;
213}
214
bd10cf0b
KB		 215static inline bool pqi_is_external_raid_addr(u8 *scsi3addr)
216{
217 return scsi3addr[2] != 0;
218}
219
694c5d5b
KB		 220static inline bool pqi_ctrl_offline(struct pqi_ctrl_info *ctrl_info)
221{
222 return !ctrl_info->controller_online;
223}
224
6c223761
KB		 225static inline void pqi_check_ctrl_health(struct pqi_ctrl_info *ctrl_info)
226{
227 if (ctrl_info->controller_online)
228 if (!sis_is_firmware_running(ctrl_info))
229 pqi_take_ctrl_offline(ctrl_info);
230}
231
232static inline bool pqi_is_hba_lunid(u8 *scsi3addr)
233{
234 return pqi_scsi3addr_equal(scsi3addr, RAID_CTLR_LUNID);
235}
236
583891c9 237static inline enum pqi_ctrl_mode pqi_get_ctrl_mode(struct pqi_ctrl_info *ctrl_info)
ff6abb73
KB		 238{
239 return sis_read_driver_scratch(ctrl_info);
240}
241
242static inline void pqi_save_ctrl_mode(struct pqi_ctrl_info *ctrl_info,
243 enum pqi_ctrl_mode mode)
244{
245 sis_write_driver_scratch(ctrl_info, mode);
246}
247
9fa82023
KB		 248static inline void pqi_ctrl_block_scan(struct pqi_ctrl_info *ctrl_info)
249{
250 ctrl_info->scan_blocked = true;
251 mutex_lock(&ctrl_info->scan_mutex);
252}
253
254static inline void pqi_ctrl_unblock_scan(struct pqi_ctrl_info *ctrl_info)
255{
256 ctrl_info->scan_blocked = false;
257 mutex_unlock(&ctrl_info->scan_mutex);
258}
259
260static inline bool pqi_ctrl_scan_blocked(struct pqi_ctrl_info *ctrl_info)
261{
262 return ctrl_info->scan_blocked;
263}
264
694c5d5b
KB		 265static inline void pqi_ctrl_block_device_reset(struct pqi_ctrl_info *ctrl_info)
266{
37f33181 267 mutex_lock(&ctrl_info->lun_reset_mutex);
694c5d5b
KB		 268}
269
37f33181 270static inline void pqi_ctrl_unblock_device_reset(struct pqi_ctrl_info *ctrl_info)
694c5d5b 271{
37f33181 272 mutex_unlock(&ctrl_info->lun_reset_mutex);
694c5d5b
KB		 273}
274
9fa82023
KB		 275static inline void pqi_scsi_block_requests(struct pqi_ctrl_info *ctrl_info)
276{
277 struct Scsi_Host *shost;
278 unsigned int num_loops;
279 int msecs_sleep;
280
281 shost = ctrl_info->scsi_host;
282
283 scsi_block_requests(shost);
284
285 num_loops = 0;
286 msecs_sleep = 20;
287 while (scsi_host_busy(shost)) {
288 num_loops++;
289 if (num_loops == 10)
290 msecs_sleep = 500;
291 msleep(msecs_sleep);
292 }
293}
294
295static inline void pqi_scsi_unblock_requests(struct pqi_ctrl_info *ctrl_info)
296{
297 scsi_unblock_requests(ctrl_info->scsi_host);
298}
299
300static inline void pqi_ctrl_busy(struct pqi_ctrl_info *ctrl_info)
301{
302 atomic_inc(&ctrl_info->num_busy_threads);
694c5d5b
KB		 303}
304
9fa82023 305static inline void pqi_ctrl_unbusy(struct pqi_ctrl_info *ctrl_info)
694c5d5b 306{
9fa82023 307 atomic_dec(&ctrl_info->num_busy_threads);
694c5d5b
KB		 308}
309
310static inline bool pqi_ctrl_blocked(struct pqi_ctrl_info *ctrl_info)
311{
312 return ctrl_info->block_requests;
313}
314
7561a7e4
KB		 315static inline void pqi_ctrl_block_requests(struct pqi_ctrl_info *ctrl_info)
316{
317 ctrl_info->block_requests = true;
7561a7e4
KB		 318}
319
320static inline void pqi_ctrl_unblock_requests(struct pqi_ctrl_info *ctrl_info)
321{
322 ctrl_info->block_requests = false;
323 wake_up_all(&ctrl_info->block_requests_wait);
7561a7e4
KB		 324}
325
ae0c189d 326static void pqi_wait_if_ctrl_blocked(struct pqi_ctrl_info *ctrl_info)
7561a7e4 327{
7561a7e4 328 if (!pqi_ctrl_blocked(ctrl_info))
ae0c189d 329 return;
7561a7e4
KB		 330
331 atomic_inc(&ctrl_info->num_blocked_threads);
ae0c189d
KB		 332 wait_event(ctrl_info->block_requests_wait,
333 !pqi_ctrl_blocked(ctrl_info));
7561a7e4 334 atomic_dec(&ctrl_info->num_blocked_threads);
7561a7e4
KB		 335}
336
18ff5f08
KB		 337#define PQI_QUIESCE_WARNING_TIMEOUT_SECS 10
338
7561a7e4
KB		 339static inline void pqi_ctrl_wait_until_quiesced(struct pqi_ctrl_info *ctrl_info)
340{
18ff5f08
KB		 341 unsigned long start_jiffies;
342 unsigned long warning_timeout;
343 bool displayed_warning;
344
345 displayed_warning = false;
346 start_jiffies = jiffies;
347 warning_timeout = (PQI_QUIESCE_WARNING_TIMEOUT_SECS * PQI_HZ) + start_jiffies;
348
7561a7e4 349 while (atomic_read(&ctrl_info->num_busy_threads) >
18ff5f08
KB		 350 atomic_read(&ctrl_info->num_blocked_threads)) {
351 if (time_after(jiffies, warning_timeout)) {
352 dev_warn(&ctrl_info->pci_dev->dev,
353 "waiting %u seconds for driver activity to quiesce\n",
354 jiffies_to_msecs(jiffies - start_jiffies) / 1000);
355 displayed_warning = true;
356 warning_timeout = (PQI_QUIESCE_WARNING_TIMEOUT_SECS * PQI_HZ) + jiffies;
357 }
7561a7e4 358 usleep_range(1000, 2000);
18ff5f08
KB		 359 }
360
361 if (displayed_warning)
362 dev_warn(&ctrl_info->pci_dev->dev,
363 "driver activity quiesced after waiting for %u seconds\n",
364 jiffies_to_msecs(jiffies - start_jiffies) / 1000);
7561a7e4
KB		 365}
366
03b288cf
KB		 367static inline bool pqi_device_offline(struct pqi_scsi_dev *device)
368{
369 return device->device_offline;
370}
371
2790cd4d 372static inline void pqi_ctrl_ofa_start(struct pqi_ctrl_info *ctrl_info)
7561a7e4 373{
2790cd4d 374 mutex_lock(&ctrl_info->ofa_mutex);
7561a7e4 375}
6c223761 376
2790cd4d 377static inline void pqi_ctrl_ofa_done(struct pqi_ctrl_info *ctrl_info)
4fd22c13 378{
2790cd4d 379 mutex_unlock(&ctrl_info->ofa_mutex);
4fd22c13
MR		 380}
381
2790cd4d 382static inline void pqi_wait_until_ofa_finished(struct pqi_ctrl_info *ctrl_info)
4fd22c13 383{
2790cd4d
KB		 384 mutex_lock(&ctrl_info->ofa_mutex);
385 mutex_unlock(&ctrl_info->ofa_mutex);
4fd22c13
MR		 386}
387
2790cd4d 388static inline bool pqi_ofa_in_progress(struct pqi_ctrl_info *ctrl_info)
4fd22c13 389{
2790cd4d 390 return mutex_is_locked(&ctrl_info->ofa_mutex);
4fd22c13
MR		 391}
392
1e46731e
MR		 393static inline void pqi_device_remove_start(struct pqi_scsi_dev *device)
394{
395 device->in_remove = true;
396}
397
1bdf6e93 398static inline bool pqi_device_in_remove(struct pqi_scsi_dev *device)
1e46731e 399{
1bdf6e93 400 return device->in_remove;
1e46731e
MR		 401}
402
2790cd4d 403static inline int pqi_event_type_to_event_index(unsigned int event_type)
0530736e 404{
2790cd4d
KB		 405 int index;
406
407 for (index = 0; index < ARRAY_SIZE(pqi_supported_event_types); index++)
408 if (event_type == pqi_supported_event_types[index])
409 return index;
410
411 return -1;
0530736e
KB		 412}
413
2790cd4d 414static inline bool pqi_is_supported_event(unsigned int event_type)
0530736e 415{
2790cd4d 416 return pqi_event_type_to_event_index(event_type) != -1;
0530736e
KB		 417}
418
583891c9
KB		 419static inline void pqi_schedule_rescan_worker_with_delay(struct pqi_ctrl_info *ctrl_info,
420 unsigned long delay)
5f310425
KB		 421{
422 if (pqi_ctrl_offline(ctrl_info))
423 return;
424
425 schedule_delayed_work(&ctrl_info->rescan_work, delay);
426}
427
6c223761
KB		 428static inline void pqi_schedule_rescan_worker(struct pqi_ctrl_info *ctrl_info)
429{
5f310425
KB		 430 pqi_schedule_rescan_worker_with_delay(ctrl_info, 0);
431}
432
4fd22c13 433#define PQI_RESCAN_WORK_DELAY (10 * PQI_HZ)
5f310425 434
583891c9 435static inline void pqi_schedule_rescan_worker_delayed(struct pqi_ctrl_info *ctrl_info)
5f310425
KB		 436{
437 pqi_schedule_rescan_worker_with_delay(ctrl_info, PQI_RESCAN_WORK_DELAY);
6c223761
KB		 438}
439
061ef06a
KB		 440static inline void pqi_cancel_rescan_worker(struct pqi_ctrl_info *ctrl_info)
441{
442 cancel_delayed_work_sync(&ctrl_info->rescan_work);
443}
444
98f87667
KB		 445static inline u32 pqi_read_heartbeat_counter(struct pqi_ctrl_info *ctrl_info)
446{
447 if (!ctrl_info->heartbeat_counter)
448 return 0;
449
450 return readl(ctrl_info->heartbeat_counter);
451}
452
4fd22c13
MR		 453static inline u8 pqi_read_soft_reset_status(struct pqi_ctrl_info *ctrl_info)
454{
4fd22c13
MR		 455 return readb(ctrl_info->soft_reset_status);
456}
457
4ccc354b 458static inline void pqi_clear_soft_reset_status(struct pqi_ctrl_info *ctrl_info)
4fd22c13
MR		 459{
460 u8 status;
461
4fd22c13 462 status = pqi_read_soft_reset_status(ctrl_info);
4ccc354b 463 status &= ~PQI_SOFT_RESET_ABORT;
4fd22c13
MR		 464 writeb(status, ctrl_info->soft_reset_status);
465}
466
6c223761
KB		 467static int pqi_map_single(struct pci_dev *pci_dev,
468 struct pqi_sg_descriptor *sg_descriptor, void *buffer,
6917a9cc 469 size_t buffer_length, enum dma_data_direction data_direction)
6c223761
KB		 470{
471 dma_addr_t bus_address;
472
6917a9cc 473 if (!buffer || buffer_length == 0 || data_direction == DMA_NONE)
6c223761
KB		 474 return 0;
475
6917a9cc 476 bus_address = dma_map_single(&pci_dev->dev, buffer, buffer_length,
6c223761 477 data_direction);
6917a9cc 478 if (dma_mapping_error(&pci_dev->dev, bus_address))
6c223761
KB		 479 return -ENOMEM;
480
481 put_unaligned_le64((u64)bus_address, &sg_descriptor->address);
482 put_unaligned_le32(buffer_length, &sg_descriptor->length);
483 put_unaligned_le32(CISS_SG_LAST, &sg_descriptor->flags);
484
485 return 0;
486}
487
488static void pqi_pci_unmap(struct pci_dev *pci_dev,
489 struct pqi_sg_descriptor *descriptors, int num_descriptors,
6917a9cc 490 enum dma_data_direction data_direction)
6c223761
KB		 491{
492 int i;
493
6917a9cc 494 if (data_direction == DMA_NONE)
6c223761
KB		 495 return;
496
497 for (i = 0; i < num_descriptors; i++)
6917a9cc 498 dma_unmap_single(&pci_dev->dev,
6c223761
KB		 499 (dma_addr_t)get_unaligned_le64(&descriptors[i].address),
500 get_unaligned_le32(&descriptors[i].length),
501 data_direction);
502}
503
504static int pqi_build_raid_path_request(struct pqi_ctrl_info *ctrl_info,
505 struct pqi_raid_path_request *request, u8 cmd,
506 u8 *scsi3addr, void *buffer, size_t buffer_length,
6917a9cc 507 u16 vpd_page, enum dma_data_direction *dir)
6c223761
KB		 508{
509 u8 *cdb;
171c2865 510 size_t cdb_length = buffer_length;
6c223761
KB		 511
512 memset(request, 0, sizeof(*request));
513
514 request->header.iu_type = PQI_REQUEST_IU_RAID_PATH_IO;
515 put_unaligned_le16(offsetof(struct pqi_raid_path_request,
516 sg_descriptors[1]) - PQI_REQUEST_HEADER_LENGTH,
517 &request->header.iu_length);
518 put_unaligned_le32(buffer_length, &request->buffer_length);
519 memcpy(request->lun_number, scsi3addr, sizeof(request->lun_number));
520 request->task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
521 request->additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_0;
522
523 cdb = request->cdb;
524
525 switch (cmd) {
526 case INQUIRY:
527 request->data_direction = SOP_READ_FLAG;
528 cdb[0] = INQUIRY;
529 if (vpd_page & VPD_PAGE) {
530 cdb[1] = 0x1;
531 cdb[2] = (u8)vpd_page;
532 }
171c2865 533 cdb[4] = (u8)cdb_length;
6c223761
KB		 534 break;
535 case CISS_REPORT_LOG:
536 case CISS_REPORT_PHYS:
537 request->data_direction = SOP_READ_FLAG;
538 cdb[0] = cmd;
539 if (cmd == CISS_REPORT_PHYS)
694c5d5b 540 cdb[1] = CISS_REPORT_PHYS_FLAG_OTHER;
6c223761 541 else
f6cc2a77 542 cdb[1] = ctrl_info->ciss_report_log_flags;
171c2865 543 put_unaligned_be32(cdb_length, &cdb[6]);
6c223761
KB		 544 break;
545 case CISS_GET_RAID_MAP:
546 request->data_direction = SOP_READ_FLAG;
547 cdb[0] = CISS_READ;
548 cdb[1] = CISS_GET_RAID_MAP;
171c2865 549 put_unaligned_be32(cdb_length, &cdb[6]);
6c223761 550 break;
58322fe0 551 case SA_FLUSH_CACHE:
ae0c189d 552 request->header.driver_flags = PQI_DRIVER_NONBLOCKABLE_REQUEST;
6c223761
KB		 553 request->data_direction = SOP_WRITE_FLAG;
554 cdb[0] = BMIC_WRITE;
58322fe0 555 cdb[6] = BMIC_FLUSH_CACHE;
171c2865 556 put_unaligned_be16(cdb_length, &cdb[7]);
6c223761 557 break;
171c2865
DC		 558 case BMIC_SENSE_DIAG_OPTIONS:
559 cdb_length = 0;
df561f66 560 fallthrough;
6c223761
KB		 561 case BMIC_IDENTIFY_CONTROLLER:
562 case BMIC_IDENTIFY_PHYSICAL_DEVICE:
6d90615f 563 case BMIC_SENSE_SUBSYSTEM_INFORMATION:
f6cc2a77 564 case BMIC_SENSE_FEATURE:
6c223761
KB		 565 request->data_direction = SOP_READ_FLAG;
566 cdb[0] = BMIC_READ;
567 cdb[6] = cmd;
171c2865 568 put_unaligned_be16(cdb_length, &cdb[7]);
6c223761 569 break;
171c2865
DC		 570 case BMIC_SET_DIAG_OPTIONS:
571 cdb_length = 0;
df561f66 572 fallthrough;
6c223761
KB		 573 case BMIC_WRITE_HOST_WELLNESS:
574 request->data_direction = SOP_WRITE_FLAG;
575 cdb[0] = BMIC_WRITE;
576 cdb[6] = cmd;
171c2865 577 put_unaligned_be16(cdb_length, &cdb[7]);
6c223761 578 break;
3d46a59a
DB		 579 case BMIC_CSMI_PASSTHRU:
580 request->data_direction = SOP_BIDIRECTIONAL;
581 cdb[0] = BMIC_WRITE;
582 cdb[5] = CSMI_CC_SAS_SMP_PASSTHRU;
583 cdb[6] = cmd;
584 put_unaligned_be16(cdb_length, &cdb[7]);
6c223761
KB		 585 break;
586 default:
9e68cccc 587 dev_err(&ctrl_info->pci_dev->dev, "unknown command 0x%c\n", cmd);
6c223761
KB		 588 break;
589 }
590
591 switch (request->data_direction) {
592 case SOP_READ_FLAG:
6917a9cc 593 *dir = DMA_FROM_DEVICE;
6c223761
KB		 594 break;
595 case SOP_WRITE_FLAG:
6917a9cc 596 *dir = DMA_TO_DEVICE;
6c223761
KB		 597 break;
598 case SOP_NO_DIRECTION_FLAG:
6917a9cc 599 *dir = DMA_NONE;
6c223761
KB		 600 break;
601 default:
6917a9cc 602 *dir = DMA_BIDIRECTIONAL;
6c223761
KB		 603 break;
604 }
605
6c223761 606 return pqi_map_single(ctrl_info->pci_dev, &request->sg_descriptors[0],
6917a9cc 607 buffer, buffer_length, *dir);
6c223761
KB		 608}
609
376fb880
KB		 610static inline void pqi_reinit_io_request(struct pqi_io_request *io_request)
611{
612 io_request->scmd = NULL;
613 io_request->status = 0;
614 io_request->error_info = NULL;
615 io_request->raid_bypass = false;
616}
617
6c223761
KB		 618static struct pqi_io_request *pqi_alloc_io_request(
619 struct pqi_ctrl_info *ctrl_info)
620{
621 struct pqi_io_request *io_request;
622 u16 i = ctrl_info->next_io_request_slot; /* benignly racy */
623
624 while (1) {
625 io_request = &ctrl_info->io_request_pool[i];
626 if (atomic_inc_return(&io_request->refcount) == 1)
627 break;
628 atomic_dec(&io_request->refcount);
629 i = (i + 1) % ctrl_info->max_io_slots;
630 }
631
632 /* benignly racy */
633 ctrl_info->next_io_request_slot = (i + 1) % ctrl_info->max_io_slots;
634
376fb880 635 pqi_reinit_io_request(io_request);
6c223761
KB		 636
637 return io_request;
638}
639
640static void pqi_free_io_request(struct pqi_io_request *io_request)
641{
642 atomic_dec(&io_request->refcount);
643}
644
02133b68 645static int pqi_send_scsi_raid_request(struct pqi_ctrl_info *ctrl_info, u8 cmd,
694c5d5b 646 u8 *scsi3addr, void *buffer, size_t buffer_length, u16 vpd_page,
ae0c189d 647 struct pqi_raid_error_info *error_info)
6c223761
KB		 648{
649 int rc;
6c223761 650 struct pqi_raid_path_request request;
694c5d5b 651 enum dma_data_direction dir;
6c223761 652
583891c9
KB		 653 rc = pqi_build_raid_path_request(ctrl_info, &request, cmd, scsi3addr,
654 buffer, buffer_length, vpd_page, &dir);
6c223761
KB		 655 if (rc)
656 return rc;
657
ae0c189d 658 rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0, error_info);
6c223761 659
6917a9cc 660 pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1, dir);
694c5d5b 661
6c223761
KB		 662 return rc;
663}
664
694c5d5b 665/* helper functions for pqi_send_scsi_raid_request */
02133b68
DC		 666
667static inline int pqi_send_ctrl_raid_request(struct pqi_ctrl_info *ctrl_info,
694c5d5b 668 u8 cmd, void *buffer, size_t buffer_length)
6c223761 669{
02133b68 670 return pqi_send_scsi_raid_request(ctrl_info, cmd, RAID_CTLR_LUNID,
ae0c189d 671 buffer, buffer_length, 0, NULL);
02133b68 672}
6c223761 673
02133b68 674static inline int pqi_send_ctrl_raid_with_error(struct pqi_ctrl_info *ctrl_info,
694c5d5b
KB		 675 u8 cmd, void *buffer, size_t buffer_length,
676 struct pqi_raid_error_info *error_info)
02133b68
DC		 677{
678 return pqi_send_scsi_raid_request(ctrl_info, cmd, RAID_CTLR_LUNID,
ae0c189d 679 buffer, buffer_length, 0, error_info);
02133b68 680}
6c223761 681
02133b68 682static inline int pqi_identify_controller(struct pqi_ctrl_info *ctrl_info,
694c5d5b 683 struct bmic_identify_controller *buffer)
02133b68
DC		 684{
685 return pqi_send_ctrl_raid_request(ctrl_info, BMIC_IDENTIFY_CONTROLLER,
694c5d5b 686 buffer, sizeof(*buffer));
02133b68
DC		 687}
688
6d90615f 689static inline int pqi_sense_subsystem_info(struct pqi_ctrl_info *ctrl_info,
694c5d5b 690 struct bmic_sense_subsystem_info *sense_info)
6d90615f
MB		 691{
692 return pqi_send_ctrl_raid_request(ctrl_info,
694c5d5b
KB		 693 BMIC_SENSE_SUBSYSTEM_INFORMATION, sense_info,
694 sizeof(*sense_info));
6d90615f
MB		 695}
696
02133b68 697static inline int pqi_scsi_inquiry(struct pqi_ctrl_info *ctrl_info,
6c223761 698 u8 *scsi3addr, u16 vpd_page, void *buffer, size_t buffer_length)
02133b68
DC		 699{
700 return pqi_send_scsi_raid_request(ctrl_info, INQUIRY, scsi3addr,
ae0c189d 701 buffer, buffer_length, vpd_page, NULL);
6c223761
KB		 702}
703
704static int pqi_identify_physical_device(struct pqi_ctrl_info *ctrl_info,
705 struct pqi_scsi_dev *device,
694c5d5b 706 struct bmic_identify_physical_device *buffer, size_t buffer_length)
6c223761
KB		 707{
708 int rc;
6917a9cc 709 enum dma_data_direction dir;
6c223761
KB		 710 u16 bmic_device_index;
711 struct pqi_raid_path_request request;
712
713 rc = pqi_build_raid_path_request(ctrl_info, &request,
714 BMIC_IDENTIFY_PHYSICAL_DEVICE, RAID_CTLR_LUNID, buffer,
6917a9cc 715 buffer_length, 0, &dir);
6c223761
KB		 716 if (rc)
717 return rc;
718
719 bmic_device_index = CISS_GET_DRIVE_NUMBER(device->scsi3addr);
720 request.cdb[2] = (u8)bmic_device_index;
721 request.cdb[9] = (u8)(bmic_device_index >> 8);
722
ae0c189d 723 rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0, NULL);
6c223761 724
6917a9cc 725 pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1, dir);
694c5d5b 726
6c223761
KB		 727 return rc;
728}
729
f6cc2a77
KB		 730static inline u32 pqi_aio_limit_to_bytes(__le16 *limit)
731{
732 u32 bytes;
733
734 bytes = get_unaligned_le16(limit);
735 if (bytes == 0)
736 bytes = ~0;
737 else
738 bytes *= 1024;
739
740 return bytes;
741}
742
743#pragma pack(1)
744
745struct bmic_sense_feature_buffer {
746 struct bmic_sense_feature_buffer_header header;
747 struct bmic_sense_feature_io_page_aio_subpage aio_subpage;
748};
749
750#pragma pack()
751
752#define MINIMUM_AIO_SUBPAGE_BUFFER_LENGTH \
753 offsetofend(struct bmic_sense_feature_buffer, \
754 aio_subpage.max_write_raid_1_10_3drive)
755
756#define MINIMUM_AIO_SUBPAGE_LENGTH \
757 (offsetofend(struct bmic_sense_feature_io_page_aio_subpage, \
758 max_write_raid_1_10_3drive) - \
759 sizeof_field(struct bmic_sense_feature_io_page_aio_subpage, header))
760
761static int pqi_get_advanced_raid_bypass_config(struct pqi_ctrl_info *ctrl_info)
762{
763 int rc;
764 enum dma_data_direction dir;
765 struct pqi_raid_path_request request;
766 struct bmic_sense_feature_buffer *buffer;
767
768 buffer = kmalloc(sizeof(*buffer), GFP_KERNEL);
769 if (!buffer)
770 return -ENOMEM;
771
583891c9
KB		 772 rc = pqi_build_raid_path_request(ctrl_info, &request, BMIC_SENSE_FEATURE, RAID_CTLR_LUNID,
773 buffer, sizeof(*buffer), 0, &dir);
f6cc2a77
KB		 774 if (rc)
775 goto error;
776
777 request.cdb[2] = BMIC_SENSE_FEATURE_IO_PAGE;
778 request.cdb[3] = BMIC_SENSE_FEATURE_IO_PAGE_AIO_SUBPAGE;
779
ae0c189d 780 rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0, NULL);
6c223761 781
6917a9cc 782 pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1, dir);
694c5d5b 783
f6cc2a77
KB		 784 if (rc)
785 goto error;
786
787 if (buffer->header.page_code != BMIC_SENSE_FEATURE_IO_PAGE ||
788 buffer->header.subpage_code !=
789 BMIC_SENSE_FEATURE_IO_PAGE_AIO_SUBPAGE ||
790 get_unaligned_le16(&buffer->header.buffer_length) <
791 MINIMUM_AIO_SUBPAGE_BUFFER_LENGTH ||
792 buffer->aio_subpage.header.page_code !=
793 BMIC_SENSE_FEATURE_IO_PAGE ||
794 buffer->aio_subpage.header.subpage_code !=
795 BMIC_SENSE_FEATURE_IO_PAGE_AIO_SUBPAGE ||
796 get_unaligned_le16(&buffer->aio_subpage.header.page_length) <
797 MINIMUM_AIO_SUBPAGE_LENGTH) {
798 goto error;
799 }
800
801 ctrl_info->max_transfer_encrypted_sas_sata =
802 pqi_aio_limit_to_bytes(
803 &buffer->aio_subpage.max_transfer_encrypted_sas_sata);
804
805 ctrl_info->max_transfer_encrypted_nvme =
806 pqi_aio_limit_to_bytes(
807 &buffer->aio_subpage.max_transfer_encrypted_nvme);
808
809 ctrl_info->max_write_raid_5_6 =
810 pqi_aio_limit_to_bytes(
811 &buffer->aio_subpage.max_write_raid_5_6);
812
813 ctrl_info->max_write_raid_1_10_2drive =
814 pqi_aio_limit_to_bytes(
815 &buffer->aio_subpage.max_write_raid_1_10_2drive);
816
817 ctrl_info->max_write_raid_1_10_3drive =
818 pqi_aio_limit_to_bytes(
819 &buffer->aio_subpage.max_write_raid_1_10_3drive);
820
821error:
822 kfree(buffer);
823
6c223761
KB		 824 return rc;
825}
826
58322fe0
KB		 827static int pqi_flush_cache(struct pqi_ctrl_info *ctrl_info,
828 enum bmic_flush_cache_shutdown_event shutdown_event)
6c223761
KB		 829{
830 int rc;
58322fe0 831 struct bmic_flush_cache *flush_cache;
6c223761 832
58322fe0
KB		 833 flush_cache = kzalloc(sizeof(*flush_cache), GFP_KERNEL);
834 if (!flush_cache)
6c223761
KB		 835 return -ENOMEM;
836
58322fe0
KB		 837 flush_cache->shutdown_event = shutdown_event;
838
02133b68
DC		 839 rc = pqi_send_ctrl_raid_request(ctrl_info, SA_FLUSH_CACHE, flush_cache,
840 sizeof(*flush_cache));
6c223761 841
58322fe0 842 kfree(flush_cache);
6c223761
KB		 843
844 return rc;
845}
846
3d46a59a
DB		 847int pqi_csmi_smp_passthru(struct pqi_ctrl_info *ctrl_info,
848 struct bmic_csmi_smp_passthru_buffer *buffer, size_t buffer_length,
849 struct pqi_raid_error_info *error_info)
850{
851 return pqi_send_ctrl_raid_with_error(ctrl_info, BMIC_CSMI_PASSTHRU,
852 buffer, buffer_length, error_info);
853}
171c2865 854
694c5d5b 855#define PQI_FETCH_PTRAID_DATA (1 << 31)
171c2865
DC		 856
857static int pqi_set_diag_rescan(struct pqi_ctrl_info *ctrl_info)
6c223761
KB		 858{
859 int rc;
171c2865 860 struct bmic_diag_options *diag;
6c223761 861
171c2865
DC		 862 diag = kzalloc(sizeof(*diag), GFP_KERNEL);
863 if (!diag)
864 return -ENOMEM;
865
02133b68 866 rc = pqi_send_ctrl_raid_request(ctrl_info, BMIC_SENSE_DIAG_OPTIONS,
694c5d5b 867 diag, sizeof(*diag));
6c223761 868 if (rc)
171c2865 869 goto out;
6c223761 870
171c2865
DC		 871 diag->options |= cpu_to_le32(PQI_FETCH_PTRAID_DATA);
872
694c5d5b
KB		 873 rc = pqi_send_ctrl_raid_request(ctrl_info, BMIC_SET_DIAG_OPTIONS, diag,
874 sizeof(*diag));
875
171c2865
DC		 876out:
877 kfree(diag);
6c223761 878
6c223761
KB		 879 return rc;
880}
881
02133b68 882static inline int pqi_write_host_wellness(struct pqi_ctrl_info *ctrl_info,
6c223761
KB		 883 void *buffer, size_t buffer_length)
884{
02133b68 885 return pqi_send_ctrl_raid_request(ctrl_info, BMIC_WRITE_HOST_WELLNESS,
694c5d5b 886 buffer, buffer_length);
6c223761
KB		 887}
888
889#pragma pack(1)
890
891struct bmic_host_wellness_driver_version {
892 u8 start_tag[4];
893 u8 driver_version_tag[2];
894 __le16 driver_version_length;
895 char driver_version[32];
b2346b50 896 u8 dont_write_tag[2];
6c223761
KB		 897 u8 end_tag[2];
898};
899
900#pragma pack()
901
902static int pqi_write_driver_version_to_host_wellness(
903 struct pqi_ctrl_info *ctrl_info)
904{
905 int rc;
906 struct bmic_host_wellness_driver_version *buffer;
907 size_t buffer_length;
908
909 buffer_length = sizeof(*buffer);
910
911 buffer = kmalloc(buffer_length, GFP_KERNEL);
912 if (!buffer)
913 return -ENOMEM;
914
915 buffer->start_tag[0] = '<';
916 buffer->start_tag[1] = 'H';
917 buffer->start_tag[2] = 'W';
918 buffer->start_tag[3] = '>';
919 buffer->driver_version_tag[0] = 'D';
920 buffer->driver_version_tag[1] = 'V';
921 put_unaligned_le16(sizeof(buffer->driver_version),
922 &buffer->driver_version_length);
061ef06a 923 strncpy(buffer->driver_version, "Linux " DRIVER_VERSION,
6c223761
KB		 924 sizeof(buffer->driver_version) - 1);
925 buffer->driver_version[sizeof(buffer->driver_version) - 1] = '\0';
b2346b50
MR		 926 buffer->dont_write_tag[0] = 'D';
927 buffer->dont_write_tag[1] = 'W';
6c223761
KB		 928 buffer->end_tag[0] = 'Z';
929 buffer->end_tag[1] = 'Z';
930
931 rc = pqi_write_host_wellness(ctrl_info, buffer, buffer_length);
932
933 kfree(buffer);
934
935 return rc;
936}
937
938#pragma pack(1)
939
940struct bmic_host_wellness_time {
941 u8 start_tag[4];
942 u8 time_tag[2];
943 __le16 time_length;
944 u8 time[8];
945 u8 dont_write_tag[2];
946 u8 end_tag[2];
947};
948
949#pragma pack()
950
951static int pqi_write_current_time_to_host_wellness(
952 struct pqi_ctrl_info *ctrl_info)
953{
954 int rc;
955 struct bmic_host_wellness_time *buffer;
956 size_t buffer_length;
957 time64_t local_time;
958 unsigned int year;
ed10858e 959 struct tm tm;
6c223761
KB		 960
961 buffer_length = sizeof(*buffer);
962
963 buffer = kmalloc(buffer_length, GFP_KERNEL);
964 if (!buffer)
965 return -ENOMEM;
966
967 buffer->start_tag[0] = '<';
968 buffer->start_tag[1] = 'H';
969 buffer->start_tag[2] = 'W';
970 buffer->start_tag[3] = '>';
971 buffer->time_tag[0] = 'T';
972 buffer->time_tag[1] = 'D';
973 put_unaligned_le16(sizeof(buffer->time),
974 &buffer->time_length);
975
ed10858e
AB		 976 local_time = ktime_get_real_seconds();
977 time64_to_tm(local_time, -sys_tz.tz_minuteswest * 60, &tm);
6c223761
KB		 978 year = tm.tm_year + 1900;
979
980 buffer->time[0] = bin2bcd(tm.tm_hour);
981 buffer->time[1] = bin2bcd(tm.tm_min);
982 buffer->time[2] = bin2bcd(tm.tm_sec);
983 buffer->time[3] = 0;
984 buffer->time[4] = bin2bcd(tm.tm_mon + 1);
985 buffer->time[5] = bin2bcd(tm.tm_mday);
986 buffer->time[6] = bin2bcd(year / 100);
987 buffer->time[7] = bin2bcd(year % 100);
988
989 buffer->dont_write_tag[0] = 'D';
990 buffer->dont_write_tag[1] = 'W';
991 buffer->end_tag[0] = 'Z';
992 buffer->end_tag[1] = 'Z';
993
994 rc = pqi_write_host_wellness(ctrl_info, buffer, buffer_length);
995
996 kfree(buffer);
997
998 return rc;
999}
1000
4fd22c13 1001#define PQI_UPDATE_TIME_WORK_INTERVAL (24UL * 60 * 60 * PQI_HZ)
6c223761
KB		 1002
1003static void pqi_update_time_worker(struct work_struct *work)
1004{
1005 int rc;
1006 struct pqi_ctrl_info *ctrl_info;
1007
1008 ctrl_info = container_of(to_delayed_work(work), struct pqi_ctrl_info,
1009 update_time_work);
1010
6c223761
KB		 1011 rc = pqi_write_current_time_to_host_wellness(ctrl_info);
1012 if (rc)
1013 dev_warn(&ctrl_info->pci_dev->dev,
1014 "error updating time on controller\n");
1015
1016 schedule_delayed_work(&ctrl_info->update_time_work,
1017 PQI_UPDATE_TIME_WORK_INTERVAL);
1018}
1019
583891c9 1020static inline void pqi_schedule_update_time_worker(struct pqi_ctrl_info *ctrl_info)
6c223761 1021{
4fbebf1a 1022 schedule_delayed_work(&ctrl_info->update_time_work, 0);
061ef06a
KB		 1023}
1024
583891c9 1025static inline void pqi_cancel_update_time_worker(struct pqi_ctrl_info *ctrl_info)
061ef06a 1026{
061ef06a 1027 cancel_delayed_work_sync(&ctrl_info->update_time_work);
6c223761
KB		 1028}
1029
583891c9
KB		 1030static inline int pqi_report_luns(struct pqi_ctrl_info *ctrl_info, u8 cmd, void *buffer,
1031 size_t buffer_length)
6c223761 1032{
583891c9 1033 return pqi_send_ctrl_raid_request(ctrl_info, cmd, buffer, buffer_length);
6c223761
KB		 1034}
1035
583891c9 1036static int pqi_report_phys_logical_luns(struct pqi_ctrl_info *ctrl_info, u8 cmd, void **buffer)
6c223761
KB		 1037{
1038 int rc;
1039 size_t lun_list_length;
1040 size_t lun_data_length;
1041 size_t new_lun_list_length;
1042 void *lun_data = NULL;
1043 struct report_lun_header *report_lun_header;
1044
1045 report_lun_header = kmalloc(sizeof(*report_lun_header), GFP_KERNEL);
1046 if (!report_lun_header) {
1047 rc = -ENOMEM;
1048 goto out;
1049 }
1050
583891c9 1051 rc = pqi_report_luns(ctrl_info, cmd, report_lun_header, sizeof(*report_lun_header));
6c223761
KB		 1052 if (rc)
1053 goto out;
1054
1055 lun_list_length = get_unaligned_be32(&report_lun_header->list_length);
1056
1057again:
1058 lun_data_length = sizeof(struct report_lun_header) + lun_list_length;
1059
1060 lun_data = kmalloc(lun_data_length, GFP_KERNEL);
1061 if (!lun_data) {
1062 rc = -ENOMEM;
1063 goto out;
1064 }
1065
1066 if (lun_list_length == 0) {
1067 memcpy(lun_data, report_lun_header, sizeof(*report_lun_header));
1068 goto out;
1069 }
1070
1071 rc = pqi_report_luns(ctrl_info, cmd, lun_data, lun_data_length);
1072 if (rc)
1073 goto out;
1074
583891c9
KB		 1075 new_lun_list_length =
1076 get_unaligned_be32(&((struct report_lun_header *)lun_data)->list_length);
6c223761
KB		 1077
1078 if (new_lun_list_length > lun_list_length) {
1079 lun_list_length = new_lun_list_length;
1080 kfree(lun_data);
1081 goto again;
1082 }
1083
1084out:
1085 kfree(report_lun_header);
1086
1087 if (rc) {
1088 kfree(lun_data);
1089 lun_data = NULL;
1090 }
1091
1092 *buffer = lun_data;
1093
1094 return rc;
1095}
1096
583891c9 1097static inline int pqi_report_phys_luns(struct pqi_ctrl_info *ctrl_info, void **buffer)
6c223761 1098{
583891c9 1099 return pqi_report_phys_logical_luns(ctrl_info, CISS_REPORT_PHYS, buffer);
6c223761
KB		 1100}
1101
583891c9 1102static inline int pqi_report_logical_luns(struct pqi_ctrl_info *ctrl_info, void **buffer)
6c223761
KB		 1103{
1104 return pqi_report_phys_logical_luns(ctrl_info, CISS_REPORT_LOG, buffer);
1105}
1106
1107static int pqi_get_device_lists(struct pqi_ctrl_info *ctrl_info,
1108 struct report_phys_lun_extended **physdev_list,
1109 struct report_log_lun_extended **logdev_list)
1110{
1111 int rc;
1112 size_t logdev_list_length;
1113 size_t logdev_data_length;
1114 struct report_log_lun_extended *internal_logdev_list;
1115 struct report_log_lun_extended *logdev_data;
1116 struct report_lun_header report_lun_header;
1117
1118 rc = pqi_report_phys_luns(ctrl_info, (void **)physdev_list);
1119 if (rc)
1120 dev_err(&ctrl_info->pci_dev->dev,
1121 "report physical LUNs failed\n");
1122
1123 rc = pqi_report_logical_luns(ctrl_info, (void **)logdev_list);
1124 if (rc)
1125 dev_err(&ctrl_info->pci_dev->dev,
1126 "report logical LUNs failed\n");
1127
1128 /*
1129 * Tack the controller itself onto the end of the logical device list.
1130 */
1131
1132 logdev_data = *logdev_list;
1133
1134 if (logdev_data) {
1135 logdev_list_length =
1136 get_unaligned_be32(&logdev_data->header.list_length);
1137 } else {
1138 memset(&report_lun_header, 0, sizeof(report_lun_header));
1139 logdev_data =
1140 (struct report_log_lun_extended *)&report_lun_header;
1141 logdev_list_length = 0;
1142 }
1143
1144 logdev_data_length = sizeof(struct report_lun_header) +
1145 logdev_list_length;
1146
1147 internal_logdev_list = kmalloc(logdev_data_length +
1148 sizeof(struct report_log_lun_extended), GFP_KERNEL);
1149 if (!internal_logdev_list) {
1150 kfree(*logdev_list);
1151 *logdev_list = NULL;
1152 return -ENOMEM;
1153 }
1154
1155 memcpy(internal_logdev_list, logdev_data, logdev_data_length);
1156 memset((u8 *)internal_logdev_list + logdev_data_length, 0,
1157 sizeof(struct report_log_lun_extended_entry));
1158 put_unaligned_be32(logdev_list_length +
1159 sizeof(struct report_log_lun_extended_entry),
1160 &internal_logdev_list->header.list_length);
1161
1162 kfree(*logdev_list);
1163 *logdev_list = internal_logdev_list;
1164
1165 return 0;
1166}
1167
1168static inline void pqi_set_bus_target_lun(struct pqi_scsi_dev *device,
1169 int bus, int target, int lun)
1170{
1171 device->bus = bus;
1172 device->target = target;
1173 device->lun = lun;
1174}
1175
1176static void pqi_assign_bus_target_lun(struct pqi_scsi_dev *device)
1177{
1178 u8 *scsi3addr;
1179 u32 lunid;
bd10cf0b
KB		 1180 int bus;
1181 int target;
1182 int lun;
6c223761
KB		 1183
1184 scsi3addr = device->scsi3addr;
1185 lunid = get_unaligned_le32(scsi3addr);
1186
1187 if (pqi_is_hba_lunid(scsi3addr)) {
1188 /* The specified device is the controller. */
1189 pqi_set_bus_target_lun(device, PQI_HBA_BUS, 0, lunid & 0x3fff);
1190 device->target_lun_valid = true;
1191 return;
1192 }
1193
1194 if (pqi_is_logical_device(device)) {
bd10cf0b
KB		 1195 if (device->is_external_raid_device) {
1196 bus = PQI_EXTERNAL_RAID_VOLUME_BUS;
1197 target = (lunid >> 16) & 0x3fff;
1198 lun = lunid & 0xff;
1199 } else {
1200 bus = PQI_RAID_VOLUME_BUS;
1201 target = 0;
1202 lun = lunid & 0x3fff;
1203 }
1204 pqi_set_bus_target_lun(device, bus, target, lun);
6c223761
KB		 1205 device->target_lun_valid = true;
1206 return;
1207 }
1208
1209 /*
1210 * Defer target and LUN assignment for non-controller physical devices
1211 * because the SAS transport layer will make these assignments later.
1212 */
1213 pqi_set_bus_target_lun(device, PQI_PHYSICAL_DEVICE_BUS, 0, 0);
1214}
1215
1216static void pqi_get_raid_level(struct pqi_ctrl_info *ctrl_info,
1217 struct pqi_scsi_dev *device)
1218{
1219 int rc;
1220 u8 raid_level;
1221 u8 *buffer;
1222
1223 raid_level = SA_RAID_UNKNOWN;
1224
1225 buffer = kmalloc(64, GFP_KERNEL);
1226 if (buffer) {
1227 rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr,
1228 VPD_PAGE | CISS_VPD_LV_DEVICE_GEOMETRY, buffer, 64);
1229 if (rc == 0) {
1230 raid_level = buffer[8];
1231 if (raid_level > SA_RAID_MAX)
1232 raid_level = SA_RAID_UNKNOWN;
1233 }
1234 kfree(buffer);
1235 }
1236
1237 device->raid_level = raid_level;
1238}
1239
1240static int pqi_validate_raid_map(struct pqi_ctrl_info *ctrl_info,
1241 struct pqi_scsi_dev *device, struct raid_map *raid_map)
1242{
1243 char *err_msg;
1244 u32 raid_map_size;
1245 u32 r5or6_blocks_per_row;
6c223761
KB		 1246
1247 raid_map_size = get_unaligned_le32(&raid_map->structure_size);
1248
1249 if (raid_map_size < offsetof(struct raid_map, disk_data)) {
1250 err_msg = "RAID map too small";
1251 goto bad_raid_map;
1252 }
1253
6c223761
KB		 1254 if (device->raid_level == SA_RAID_1) {
1255 if (get_unaligned_le16(&raid_map->layout_map_count) != 2) {
1256 err_msg = "invalid RAID-1 map";
1257 goto bad_raid_map;
1258 }
7a012c23 1259 } else if (device->raid_level == SA_RAID_TRIPLE) {
6c223761 1260 if (get_unaligned_le16(&raid_map->layout_map_count) != 3) {
7a012c23 1261 err_msg = "invalid RAID-1(Triple) map";
6c223761
KB		 1262 goto bad_raid_map;
1263 }
1264 } else if ((device->raid_level == SA_RAID_5 ||
1265 device->raid_level == SA_RAID_6) &&
1266 get_unaligned_le16(&raid_map->layout_map_count) > 1) {
1267 /* RAID 50/60 */
1268 r5or6_blocks_per_row =
1269 get_unaligned_le16(&raid_map->strip_size) *
1270 get_unaligned_le16(&raid_map->data_disks_per_row);
1271 if (r5or6_blocks_per_row == 0) {
1272 err_msg = "invalid RAID-5 or RAID-6 map";
1273 goto bad_raid_map;
1274 }
1275 }
1276
1277 return 0;
1278
1279bad_raid_map:
d87d5474 1280 dev_warn(&ctrl_info->pci_dev->dev,
38a7338a
KB		 1281 "logical device %08x%08x %s\n",
1282 *((u32 *)&device->scsi3addr),
1283 *((u32 *)&device->scsi3addr[4]), err_msg);
6c223761
KB		 1284
1285 return -EINVAL;
1286}
1287
1288static int pqi_get_raid_map(struct pqi_ctrl_info *ctrl_info,
1289 struct pqi_scsi_dev *device)
1290{
1291 int rc;
a91aaae0 1292 u32 raid_map_size;
6c223761
KB		 1293 struct raid_map *raid_map;
1294
1295 raid_map = kmalloc(sizeof(*raid_map), GFP_KERNEL);
1296 if (!raid_map)
1297 return -ENOMEM;
1298
a91aaae0 1299 rc = pqi_send_scsi_raid_request(ctrl_info, CISS_GET_RAID_MAP,
ae0c189d 1300 device->scsi3addr, raid_map, sizeof(*raid_map), 0, NULL);
6c223761
KB		 1301 if (rc)
1302 goto error;
1303
a91aaae0 1304 raid_map_size = get_unaligned_le32(&raid_map->structure_size);
6c223761 1305
a91aaae0 1306 if (raid_map_size > sizeof(*raid_map)) {
6c223761 1307
a91aaae0
AK		 1308 kfree(raid_map);
1309
1310 raid_map = kmalloc(raid_map_size, GFP_KERNEL);
1311 if (!raid_map)
1312 return -ENOMEM;
1313
1314 rc = pqi_send_scsi_raid_request(ctrl_info, CISS_GET_RAID_MAP,
ae0c189d 1315 device->scsi3addr, raid_map, raid_map_size, 0, NULL);
a91aaae0
AK		 1316 if (rc)
1317 goto error;
1318
1319 if (get_unaligned_le32(&raid_map->structure_size)
1320 != raid_map_size) {
1321 dev_warn(&ctrl_info->pci_dev->dev,
583891c9 1322 "requested %u bytes, received %u bytes\n",
a91aaae0
AK		 1323 raid_map_size,
1324 get_unaligned_le32(&raid_map->structure_size));
1325 goto error;
1326 }
1327 }
6c223761
KB		 1328
1329 rc = pqi_validate_raid_map(ctrl_info, device, raid_map);
1330 if (rc)
1331 goto error;
1332
1333 device->raid_map = raid_map;
1334
1335 return 0;
1336
1337error:
1338 kfree(raid_map);
1339
1340 return rc;
1341}
1342
f6cc2a77
KB		 1343static void pqi_set_max_transfer_encrypted(struct pqi_ctrl_info *ctrl_info,
1344 struct pqi_scsi_dev *device)
1345{
1346 if (!ctrl_info->lv_drive_type_mix_valid) {
1347 device->max_transfer_encrypted = ~0;
1348 return;
1349 }
1350
1351 switch (LV_GET_DRIVE_TYPE_MIX(device->scsi3addr)) {
1352 case LV_DRIVE_TYPE_MIX_SAS_HDD_ONLY:
1353 case LV_DRIVE_TYPE_MIX_SATA_HDD_ONLY:
1354 case LV_DRIVE_TYPE_MIX_SAS_OR_SATA_SSD_ONLY:
1355 case LV_DRIVE_TYPE_MIX_SAS_SSD_ONLY:
1356 case LV_DRIVE_TYPE_MIX_SATA_SSD_ONLY:
1357 case LV_DRIVE_TYPE_MIX_SAS_ONLY:
1358 case LV_DRIVE_TYPE_MIX_SATA_ONLY:
1359 device->max_transfer_encrypted =
1360 ctrl_info->max_transfer_encrypted_sas_sata;
1361 break;
1362 case LV_DRIVE_TYPE_MIX_NVME_ONLY:
1363 device->max_transfer_encrypted =
1364 ctrl_info->max_transfer_encrypted_nvme;
1365 break;
1366 case LV_DRIVE_TYPE_MIX_UNKNOWN:
1367 case LV_DRIVE_TYPE_MIX_NO_RESTRICTION:
1368 default:
1369 device->max_transfer_encrypted =
1370 min(ctrl_info->max_transfer_encrypted_sas_sata,
1371 ctrl_info->max_transfer_encrypted_nvme);
1372 break;
1373 }
1374}
1375
588a63fe 1376static void pqi_get_raid_bypass_status(struct pqi_ctrl_info *ctrl_info,
6c223761
KB		 1377 struct pqi_scsi_dev *device)
1378{
1379 int rc;
1380 u8 *buffer;
588a63fe 1381 u8 bypass_status;
6c223761
KB		 1382
1383 buffer = kmalloc(64, GFP_KERNEL);
1384 if (!buffer)
1385 return;
1386
1387 rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr,
588a63fe 1388 VPD_PAGE | CISS_VPD_LV_BYPASS_STATUS, buffer, 64);
6c223761
KB		 1389 if (rc)
1390 goto out;
1391
694c5d5b
KB		 1392#define RAID_BYPASS_STATUS 4
1393#define RAID_BYPASS_CONFIGURED 0x1
1394#define RAID_BYPASS_ENABLED 0x2
6c223761 1395
588a63fe
KB		 1396 bypass_status = buffer[RAID_BYPASS_STATUS];
1397 device->raid_bypass_configured =
1398 (bypass_status & RAID_BYPASS_CONFIGURED) != 0;
1399 if (device->raid_bypass_configured &&
1400 (bypass_status & RAID_BYPASS_ENABLED) &&
f6cc2a77 1401 pqi_get_raid_map(ctrl_info, device) == 0) {
588a63fe 1402 device->raid_bypass_enabled = true;
f6cc2a77
KB		 1403 if (get_unaligned_le16(&device->raid_map->flags) &
1404 RAID_MAP_ENCRYPTION_ENABLED)
1405 pqi_set_max_transfer_encrypted(ctrl_info, device);
1406 }
6c223761
KB		 1407
1408out:
1409 kfree(buffer);
1410}
1411
1412/*
1413 * Use vendor-specific VPD to determine online/offline status of a volume.
1414 */
1415
1416static void pqi_get_volume_status(struct pqi_ctrl_info *ctrl_info,
1417 struct pqi_scsi_dev *device)
1418{
1419 int rc;
1420 size_t page_length;
1421 u8 volume_status = CISS_LV_STATUS_UNAVAILABLE;
1422 bool volume_offline = true;
1423 u32 volume_flags;
1424 struct ciss_vpd_logical_volume_status *vpd;
1425
1426 vpd = kmalloc(sizeof(*vpd), GFP_KERNEL);
1427 if (!vpd)
1428 goto no_buffer;
1429
1430 rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr,
1431 VPD_PAGE | CISS_VPD_LV_STATUS, vpd, sizeof(*vpd));
1432 if (rc)
1433 goto out;
1434
7ff44499
DC		 1435 if (vpd->page_code != CISS_VPD_LV_STATUS)
1436 goto out;
1437
6c223761
KB		 1438 page_length = offsetof(struct ciss_vpd_logical_volume_status,
1439 volume_status) + vpd->page_length;
1440 if (page_length < sizeof(*vpd))
1441 goto out;
1442
1443 volume_status = vpd->volume_status;
1444 volume_flags = get_unaligned_be32(&vpd->flags);
1445 volume_offline = (volume_flags & CISS_LV_FLAGS_NO_HOST_IO) != 0;
1446
1447out:
1448 kfree(vpd);
1449no_buffer:
1450 device->volume_status = volume_status;
1451 device->volume_offline = volume_offline;
1452}
1453
ec504b23
MB		 1454#define PQI_DEVICE_PHY_MAP_SUPPORTED 0x10
1455
ce143793
KB		 1456static int pqi_get_physical_device_info(struct pqi_ctrl_info *ctrl_info,
1457 struct pqi_scsi_dev *device,
1458 struct bmic_identify_physical_device *id_phys)
1459{
1460 int rc;
26b390ab 1461
ce143793
KB		 1462 memset(id_phys, 0, sizeof(*id_phys));
1463
1464 rc = pqi_identify_physical_device(ctrl_info, device,
1465 id_phys, sizeof(*id_phys));
1466 if (rc) {
1467 device->queue_depth = PQI_PHYSICAL_DISK_DEFAULT_MAX_QUEUE_DEPTH;
1468 return rc;
1469 }
1470
1471 scsi_sanitize_inquiry_string(&id_phys->model[0], 8);
1472 scsi_sanitize_inquiry_string(&id_phys->model[8], 16);
1473
1474 memcpy(device->vendor, &id_phys->model[0], sizeof(device->vendor));
1475 memcpy(device->model, &id_phys->model[8], sizeof(device->model));
1476
1477 device->box_index = id_phys->box_index;
1478 device->phys_box_on_bus = id_phys->phys_box_on_bus;
1479 device->phy_connected_dev_type = id_phys->phy_connected_dev_type[0];
1480 device->queue_depth =
1481 get_unaligned_le16(&id_phys->current_queue_depth_limit);
1482 device->active_path_index = id_phys->active_path_number;
1483 device->path_map = id_phys->redundant_path_present_map;
1484 memcpy(&device->box,
1485 &id_phys->alternate_paths_phys_box_on_port,
1486 sizeof(device->box));
1487 memcpy(&device->phys_connector,
1488 &id_phys->alternate_paths_phys_connector,
1489 sizeof(device->phys_connector));
1490 device->bay = id_phys->phys_bay_in_box;
1491
7a84a821
KB		 1492 memcpy(&device->page_83_identifier, &id_phys->page_83_identifier,
1493 sizeof(device->page_83_identifier));
1494
ec504b23
MB		 1495 if ((id_phys->even_more_flags & PQI_DEVICE_PHY_MAP_SUPPORTED) &&
1496 id_phys->phy_count)
1497 device->phy_id =
1498 id_phys->phy_to_phy_map[device->active_path_index];
1499 else
1500 device->phy_id = 0xFF;
1501
ce143793
KB		 1502 return 0;
1503}
1504
1505static int pqi_get_logical_device_info(struct pqi_ctrl_info *ctrl_info,
6c223761
KB		 1506 struct pqi_scsi_dev *device)
1507{
1508 int rc;
1509 u8 *buffer;
3d46a59a 1510
6c223761
KB		 1511 buffer = kmalloc(64, GFP_KERNEL);
1512 if (!buffer)
1513 return -ENOMEM;
1514
1515 /* Send an inquiry to the device to see what it is. */
ce143793
KB		 1516 rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr, 0, buffer, 64);
1517 if (rc)
1518 goto out;
6c223761
KB		 1519
1520 scsi_sanitize_inquiry_string(&buffer[8], 8);
1521 scsi_sanitize_inquiry_string(&buffer[16], 16);
1522
1523 device->devtype = buffer[0] & 0x1f;
cbe0c7b1
KB		 1524 memcpy(device->vendor, &buffer[8], sizeof(device->vendor));
1525 memcpy(device->model, &buffer[16], sizeof(device->model));
6c223761 1526
ce143793 1527 if (device->devtype == TYPE_DISK) {
bd10cf0b
KB		 1528 if (device->is_external_raid_device) {
1529 device->raid_level = SA_RAID_UNKNOWN;
1530 device->volume_status = CISS_LV_OK;
1531 device->volume_offline = false;
1532 } else {
1533 pqi_get_raid_level(ctrl_info, device);
588a63fe 1534 pqi_get_raid_bypass_status(ctrl_info, device);
bd10cf0b
KB		 1535 pqi_get_volume_status(ctrl_info, device);
1536 }
6c223761
KB		 1537 }
1538
1539out:
1540 kfree(buffer);
1541
1542 return rc;
1543}
1544
ce143793 1545static int pqi_get_device_info(struct pqi_ctrl_info *ctrl_info,
6c223761
KB		 1546 struct pqi_scsi_dev *device,
1547 struct bmic_identify_physical_device *id_phys)
1548{
1549 int rc;
1550
ce143793
KB		 1551 if (device->is_expander_smp_device)
1552 return 0;
6c223761 1553
ce143793
KB		 1554 if (pqi_is_logical_device(device))
1555 rc = pqi_get_logical_device_info(ctrl_info, device);
1556 else
1557 rc = pqi_get_physical_device_info(ctrl_info, device, id_phys);
694c5d5b 1558
ce143793 1559 return rc;
6c223761
KB		 1560}
1561
1562static void pqi_show_volume_status(struct pqi_ctrl_info *ctrl_info,
1563 struct pqi_scsi_dev *device)
1564{
1565 char *status;
1566 static const char unknown_state_str[] =
1567 "Volume is in an unknown state (%u)";
1568 char unknown_state_buffer[sizeof(unknown_state_str) + 10];
1569
1570 switch (device->volume_status) {
1571 case CISS_LV_OK:
1572 status = "Volume online";
1573 break;
1574 case CISS_LV_FAILED:
1575 status = "Volume failed";
1576 break;
1577 case CISS_LV_NOT_CONFIGURED:
1578 status = "Volume not configured";
1579 break;
1580 case CISS_LV_DEGRADED:
1581 status = "Volume degraded";
1582 break;
1583 case CISS_LV_READY_FOR_RECOVERY:
1584 status = "Volume ready for recovery operation";
1585 break;
1586 case CISS_LV_UNDERGOING_RECOVERY:
1587 status = "Volume undergoing recovery";
1588 break;
1589 case CISS_LV_WRONG_PHYSICAL_DRIVE_REPLACED:
1590 status = "Wrong physical drive was replaced";
1591 break;
1592 case CISS_LV_PHYSICAL_DRIVE_CONNECTION_PROBLEM:
1593 status = "A physical drive not properly connected";
1594 break;
1595 case CISS_LV_HARDWARE_OVERHEATING:
1596 status = "Hardware is overheating";
1597 break;
1598 case CISS_LV_HARDWARE_HAS_OVERHEATED:
1599 status = "Hardware has overheated";
1600 break;
1601 case CISS_LV_UNDERGOING_EXPANSION:
1602 status = "Volume undergoing expansion";
1603 break;
1604 case CISS_LV_NOT_AVAILABLE:
1605 status = "Volume waiting for transforming volume";
1606 break;
1607 case CISS_LV_QUEUED_FOR_EXPANSION:
1608 status = "Volume queued for expansion";
1609 break;
1610 case CISS_LV_DISABLED_SCSI_ID_CONFLICT:
1611 status = "Volume disabled due to SCSI ID conflict";
1612 break;
1613 case CISS_LV_EJECTED:
1614 status = "Volume has been ejected";
1615 break;
1616 case CISS_LV_UNDERGOING_ERASE:
1617 status = "Volume undergoing background erase";
1618 break;
1619 case CISS_LV_READY_FOR_PREDICTIVE_SPARE_REBUILD:
1620 status = "Volume ready for predictive spare rebuild";
1621 break;
1622 case CISS_LV_UNDERGOING_RPI:
1623 status = "Volume undergoing rapid parity initialization";
1624 break;
1625 case CISS_LV_PENDING_RPI:
1626 status = "Volume queued for rapid parity initialization";
1627 break;
1628 case CISS_LV_ENCRYPTED_NO_KEY:
1629 status = "Encrypted volume inaccessible - key not present";
1630 break;
1631 case CISS_LV_UNDERGOING_ENCRYPTION:
1632 status = "Volume undergoing encryption process";
1633 break;
1634 case CISS_LV_UNDERGOING_ENCRYPTION_REKEYING:
1635 status = "Volume undergoing encryption re-keying process";
1636 break;
1637 case CISS_LV_ENCRYPTED_IN_NON_ENCRYPTED_CONTROLLER:
d87d5474 1638 status = "Volume encrypted but encryption is disabled";
6c223761
KB		 1639 break;
1640 case CISS_LV_PENDING_ENCRYPTION:
1641 status = "Volume pending migration to encrypted state";
1642 break;
1643 case CISS_LV_PENDING_ENCRYPTION_REKEYING:
1644 status = "Volume pending encryption rekeying";
1645 break;
1646 case CISS_LV_NOT_SUPPORTED:
1647 status = "Volume not supported on this controller";
1648 break;
1649 case CISS_LV_STATUS_UNAVAILABLE:
1650 status = "Volume status not available";
1651 break;
1652 default:
1653 snprintf(unknown_state_buffer, sizeof(unknown_state_buffer),
1654 unknown_state_str, device->volume_status);
1655 status = unknown_state_buffer;
1656 break;
1657 }
1658
1659 dev_info(&ctrl_info->pci_dev->dev,
1660 "scsi %d:%d:%d:%d %s\n",
1661 ctrl_info->scsi_host->host_no,
1662 device->bus, device->target, device->lun, status);
1663}
1664
6c223761
KB		 1665static void pqi_rescan_worker(struct work_struct *work)
1666{
1667 struct pqi_ctrl_info *ctrl_info;
1668
1669 ctrl_info = container_of(to_delayed_work(work), struct pqi_ctrl_info,
1670 rescan_work);
1671
1672 pqi_scan_scsi_devices(ctrl_info);
1673}
1674
1675static int pqi_add_device(struct pqi_ctrl_info *ctrl_info,
1676 struct pqi_scsi_dev *device)
1677{
1678 int rc;
1679
1680 if (pqi_is_logical_device(device))
1681 rc = scsi_add_device(ctrl_info->scsi_host, device->bus,
1682 device->target, device->lun);
1683 else
1684 rc = pqi_add_sas_device(ctrl_info->sas_host, device);
1685
1686 return rc;
1687}
1688
18ff5f08 1689#define PQI_REMOVE_DEVICE_PENDING_IO_TIMEOUT_MSECS (20 * 1000)
1e46731e 1690
583891c9 1691static inline void pqi_remove_device(struct pqi_ctrl_info *ctrl_info, struct pqi_scsi_dev *device)
6c223761 1692{
1e46731e
MR		 1693 int rc;
1694
1695 pqi_device_remove_start(device);
1696
18ff5f08
KB		 1697 rc = pqi_device_wait_for_pending_io(ctrl_info, device,
1698 PQI_REMOVE_DEVICE_PENDING_IO_TIMEOUT_MSECS);
1e46731e
MR		 1699 if (rc)
1700 dev_err(&ctrl_info->pci_dev->dev,
4d15ad38 1701 "scsi %d:%d:%d:%d removing device with %d outstanding command(s)\n",
1e46731e
MR		 1702 ctrl_info->scsi_host->host_no, device->bus,
1703 device->target, device->lun,
1704 atomic_read(&device->scsi_cmds_outstanding));
1705
6c223761
KB		 1706 if (pqi_is_logical_device(device))
1707 scsi_remove_device(device->sdev);
1708 else
1709 pqi_remove_sas_device(device);
1710}
1711
1712/* Assumes the SCSI device list lock is held. */
1713
1714static struct pqi_scsi_dev *pqi_find_scsi_dev(struct pqi_ctrl_info *ctrl_info,
1715 int bus, int target, int lun)
1716{
1717 struct pqi_scsi_dev *device;
1718
4d15ad38
KB		 1719 list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry)
1720 if (device->bus == bus && device->target == target && device->lun == lun)
6c223761
KB		 1721 return device;
1722
1723 return NULL;
1724}
1725
583891c9 1726static inline bool pqi_device_equal(struct pqi_scsi_dev *dev1, struct pqi_scsi_dev *dev2)
6c223761
KB		 1727{
1728 if (dev1->is_physical_device != dev2->is_physical_device)
1729 return false;
1730
1731 if (dev1->is_physical_device)
1732 return dev1->wwid == dev2->wwid;
1733
583891c9 1734 return memcmp(dev1->volume_id, dev2->volume_id, sizeof(dev1->volume_id)) == 0;
6c223761
KB		 1735}
1736
1737enum pqi_find_result {
1738 DEVICE_NOT_FOUND,
1739 DEVICE_CHANGED,
1740 DEVICE_SAME,
1741};
1742
1743static enum pqi_find_result pqi_scsi_find_entry(struct pqi_ctrl_info *ctrl_info,
4d15ad38 1744 struct pqi_scsi_dev *device_to_find, struct pqi_scsi_dev **matching_device)
6c223761
KB		 1745{
1746 struct pqi_scsi_dev *device;
1747
4d15ad38
KB		 1748 list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry) {
1749 if (pqi_scsi3addr_equal(device_to_find->scsi3addr, device->scsi3addr)) {
6c223761
KB		 1750 *matching_device = device;
1751 if (pqi_device_equal(device_to_find, device)) {
1752 if (device_to_find->volume_offline)
1753 return DEVICE_CHANGED;
1754 return DEVICE_SAME;
1755 }
1756 return DEVICE_CHANGED;
1757 }
1758 }
1759
1760 return DEVICE_NOT_FOUND;
1761}
1762
3d46a59a
DB		 1763static inline const char *pqi_device_type(struct pqi_scsi_dev *device)
1764{
1765 if (device->is_expander_smp_device)
1766 return "Enclosure SMP ";
1767
1768 return scsi_device_type(device->devtype);
1769}
1770
6de783f6
KB		 1771#define PQI_DEV_INFO_BUFFER_LENGTH 128
1772
6c223761
KB		 1773static void pqi_dev_info(struct pqi_ctrl_info *ctrl_info,
1774 char *action, struct pqi_scsi_dev *device)
1775{
6de783f6
KB		 1776 ssize_t count;
1777 char buffer[PQI_DEV_INFO_BUFFER_LENGTH];
1778
a4256252 1779 count = scnprintf(buffer, PQI_DEV_INFO_BUFFER_LENGTH,
6de783f6
KB		 1780 "%d:%d:", ctrl_info->scsi_host->host_no, device->bus);
1781
1782 if (device->target_lun_valid)
181aea89 1783 count += scnprintf(buffer + count,
6de783f6
KB		 1784 PQI_DEV_INFO_BUFFER_LENGTH - count,
1785 "%d:%d",
1786 device->target,
1787 device->lun);
1788 else
181aea89 1789 count += scnprintf(buffer + count,
6de783f6
KB		 1790 PQI_DEV_INFO_BUFFER_LENGTH - count,
1791 "-:-");
1792
1793 if (pqi_is_logical_device(device))
181aea89 1794 count += scnprintf(buffer + count,
6de783f6
KB		 1795 PQI_DEV_INFO_BUFFER_LENGTH - count,
1796 " %08x%08x",
1797 *((u32 *)&device->scsi3addr),
1798 *((u32 *)&device->scsi3addr[4]));
1799 else
181aea89 1800 count += scnprintf(buffer + count,
6de783f6
KB		 1801 PQI_DEV_INFO_BUFFER_LENGTH - count,
1802 " %016llx", device->sas_address);
1803
181aea89 1804 count += scnprintf(buffer + count, PQI_DEV_INFO_BUFFER_LENGTH - count,
6de783f6 1805 " %s %.8s %.16s ",
3d46a59a 1806 pqi_device_type(device),
6c223761 1807 device->vendor,
6de783f6
KB		 1808 device->model);
1809
1810 if (pqi_is_logical_device(device)) {
1811 if (device->devtype == TYPE_DISK)
181aea89 1812 count += scnprintf(buffer + count,
6de783f6
KB		 1813 PQI_DEV_INFO_BUFFER_LENGTH - count,
1814 "SSDSmartPathCap%c En%c %-12s",
588a63fe
KB		 1815 device->raid_bypass_configured ? '+' : '-',
1816 device->raid_bypass_enabled ? '+' : '-',
6de783f6
KB		 1817 pqi_raid_level_to_string(device->raid_level));
1818 } else {
181aea89 1819 count += scnprintf(buffer + count,
6de783f6
KB		 1820 PQI_DEV_INFO_BUFFER_LENGTH - count,
1821 "AIO%c", device->aio_enabled ? '+' : '-');
1822 if (device->devtype == TYPE_DISK ||
1823 device->devtype == TYPE_ZBC)
181aea89 1824 count += scnprintf(buffer + count,
6de783f6
KB		 1825 PQI_DEV_INFO_BUFFER_LENGTH - count,
1826 " qd=%-6d", device->queue_depth);
1827 }
1828
1829 dev_info(&ctrl_info->pci_dev->dev, "%s %s\n", action, buffer);
6c223761
KB		 1830}
1831
1832/* Assumes the SCSI device list lock is held. */
1833
1834static void pqi_scsi_update_device(struct pqi_scsi_dev *existing_device,
1835 struct pqi_scsi_dev *new_device)
1836{
6c223761
KB		 1837 existing_device->device_type = new_device->device_type;
1838 existing_device->bus = new_device->bus;
1839 if (new_device->target_lun_valid) {
1840 existing_device->target = new_device->target;
1841 existing_device->lun = new_device->lun;
1842 existing_device->target_lun_valid = true;
1843 }
1844
244ca45e
MR		 1845 if ((existing_device->volume_status == CISS_LV_QUEUED_FOR_EXPANSION ||
1846 existing_device->volume_status == CISS_LV_UNDERGOING_EXPANSION) &&
1847 new_device->volume_status == CISS_LV_OK)
1848 existing_device->rescan = true;
1849
6c223761
KB		 1850 /* By definition, the scsi3addr and wwid fields are already the same. */
1851
1852 existing_device->is_physical_device = new_device->is_physical_device;
bd10cf0b
KB		 1853 existing_device->is_external_raid_device =
1854 new_device->is_external_raid_device;
3d46a59a
DB		 1855 existing_device->is_expander_smp_device =
1856 new_device->is_expander_smp_device;
6c223761
KB		 1857 existing_device->aio_enabled = new_device->aio_enabled;
1858 memcpy(existing_device->vendor, new_device->vendor,
1859 sizeof(existing_device->vendor));
1860 memcpy(existing_device->model, new_device->model,
1861 sizeof(existing_device->model));
1862 existing_device->sas_address = new_device->sas_address;
1863 existing_device->raid_level = new_device->raid_level;
1864 existing_device->queue_depth = new_device->queue_depth;
1865 existing_device->aio_handle = new_device->aio_handle;
1866 existing_device->volume_status = new_device->volume_status;
1867 existing_device->active_path_index = new_device->active_path_index;
ec504b23 1868 existing_device->phy_id = new_device->phy_id;
6c223761
KB		 1869 existing_device->path_map = new_device->path_map;
1870 existing_device->bay = new_device->bay;
2d2ad4bc
GW		 1871 existing_device->box_index = new_device->box_index;
1872 existing_device->phys_box_on_bus = new_device->phys_box_on_bus;
583891c9 1873 existing_device->phy_connected_dev_type = new_device->phy_connected_dev_type;
6c223761
KB		 1874 memcpy(existing_device->box, new_device->box,
1875 sizeof(existing_device->box));
1876 memcpy(existing_device->phys_connector, new_device->phys_connector,
1877 sizeof(existing_device->phys_connector));
7a012c23 1878 existing_device->next_bypass_group = 0;
6c223761
KB		 1879 kfree(existing_device->raid_map);
1880 existing_device->raid_map = new_device->raid_map;
588a63fe
KB		 1881 existing_device->raid_bypass_configured =
1882 new_device->raid_bypass_configured;
1883 existing_device->raid_bypass_enabled =
1884 new_device->raid_bypass_enabled;
a9a68101 1885 existing_device->device_offline = false;
6c223761
KB		 1886
1887 /* To prevent this from being freed later. */
1888 new_device->raid_map = NULL;
1889}
1890
1891static inline void pqi_free_device(struct pqi_scsi_dev *device)
1892{
1893 if (device) {
1894 kfree(device->raid_map);
1895 kfree(device);
1896 }
1897}
1898
1899/*
1900 * Called when exposing a new device to the OS fails in order to re-adjust
1901 * our internal SCSI device list to match the SCSI ML's view.
1902 */
1903
1904static inline void pqi_fixup_botched_add(struct pqi_ctrl_info *ctrl_info,
1905 struct pqi_scsi_dev *device)
1906{
1907 unsigned long flags;
1908
1909 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
1910 list_del(&device->scsi_device_list_entry);
1911 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
1912
1913 /* Allow the device structure to be freed later. */
1914 device->keep_device = false;
1915}
1916
3d46a59a
DB		 1917static inline bool pqi_is_device_added(struct pqi_scsi_dev *device)
1918{
1919 if (device->is_expander_smp_device)
1920 return device->sas_port != NULL;
1921
1922 return device->sdev != NULL;
1923}
1924
6c223761
KB		 1925static void pqi_update_device_list(struct pqi_ctrl_info *ctrl_info,
1926 struct pqi_scsi_dev *new_device_list[], unsigned int num_new_devices)
1927{
1928 int rc;
1929 unsigned int i;
1930 unsigned long flags;
1931 enum pqi_find_result find_result;
1932 struct pqi_scsi_dev *device;
1933 struct pqi_scsi_dev *next;
1934 struct pqi_scsi_dev *matching_device;
8a994a04
KB		 1935 LIST_HEAD(add_list);
1936 LIST_HEAD(delete_list);
6c223761
KB		 1937
1938 /*
1939 * The idea here is to do as little work as possible while holding the
1940 * spinlock. That's why we go to great pains to defer anything other
1941 * than updating the internal device list until after we release the
1942 * spinlock.
1943 */
1944
1945 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
1946
1947 /* Assume that all devices in the existing list have gone away. */
4d15ad38 1948 list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry)
6c223761
KB		 1949 device->device_gone = true;
1950
1951 for (i = 0; i < num_new_devices; i++) {
1952 device = new_device_list[i];
1953
1954 find_result = pqi_scsi_find_entry(ctrl_info, device,
694c5d5b 1955 &matching_device);
6c223761
KB		 1956
1957 switch (find_result) {
1958 case DEVICE_SAME:
1959 /*
1960 * The newly found device is already in the existing
1961 * device list.
1962 */
1963 device->new_device = false;
1964 matching_device->device_gone = false;
1965 pqi_scsi_update_device(matching_device, device);
1966 break;
1967 case DEVICE_NOT_FOUND:
1968 /*
1969 * The newly found device is NOT in the existing device
1970 * list.
1971 */
1972 device->new_device = true;
1973 break;
1974 case DEVICE_CHANGED:
1975 /*
1976 * The original device has gone away and we need to add
1977 * the new device.
1978 */
1979 device->new_device = true;
1980 break;
6c223761
KB		 1981 }
1982 }
1983
1984 /* Process all devices that have gone away. */
1985 list_for_each_entry_safe(device, next, &ctrl_info->scsi_device_list,
1986 scsi_device_list_entry) {
1987 if (device->device_gone) {
4d15ad38 1988 list_del_init(&device->scsi_device_list_entry);
6c223761
KB		 1989 list_add_tail(&device->delete_list_entry, &delete_list);
1990 }
1991 }
1992
1993 /* Process all new devices. */
1994 for (i = 0; i < num_new_devices; i++) {
1995 device = new_device_list[i];
1996 if (!device->new_device)
1997 continue;
1998 if (device->volume_offline)
1999 continue;
2000 list_add_tail(&device->scsi_device_list_entry,
2001 &ctrl_info->scsi_device_list);
2002 list_add_tail(&device->add_list_entry, &add_list);
2003 /* To prevent this device structure from being freed later. */
2004 device->keep_device = true;
2005 }
2006
6c223761
KB		 2007 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
2008
2790cd4d
KB		 2009 /*
2010 * If OFA is in progress and there are devices that need to be deleted,
2011 * allow any pending reset operations to continue and unblock any SCSI
2012 * requests before removal.
2013 */
2014 if (pqi_ofa_in_progress(ctrl_info)) {
2015 list_for_each_entry_safe(device, next, &delete_list, delete_list_entry)
2016 if (pqi_is_device_added(device))
2017 pqi_device_remove_start(device);
2018 pqi_ctrl_unblock_device_reset(ctrl_info);
2019 pqi_scsi_unblock_requests(ctrl_info);
2020 }
4fd22c13 2021
6c223761 2022 /* Remove all devices that have gone away. */
4d15ad38 2023 list_for_each_entry_safe(device, next, &delete_list, delete_list_entry) {
6c223761
KB		 2024 if (device->volume_offline) {
2025 pqi_dev_info(ctrl_info, "offline", device);
2026 pqi_show_volume_status(ctrl_info, device);
6c223761
KB		 2027 }
2028 list_del(&device->delete_list_entry);
4d15ad38
KB		 2029 if (pqi_is_device_added(device)) {
2030 pqi_remove_device(ctrl_info, device);
2031 } else {
2032 if (!device->volume_offline)
2033 pqi_dev_info(ctrl_info, "removed", device);
2034 pqi_free_device(device);
2035 }
6c223761
KB		 2036 }
2037
2038 /*
2039 * Notify the SCSI ML if the queue depth of any existing device has
2040 * changed.
2041 */
583891c9
KB		 2042 list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry) {
2043 if (device->sdev && device->queue_depth != device->advertised_queue_depth) {
2044 device->advertised_queue_depth = device->queue_depth;
2045 scsi_change_queue_depth(device->sdev, device->advertised_queue_depth);
244ca45e
MR		 2046 if (device->rescan) {
2047 scsi_rescan_device(&device->sdev->sdev_gendev);
2048 device->rescan = false;
2049 }
6c223761
KB		 2050 }
2051 }
2052
2053 /* Expose any new devices. */
2054 list_for_each_entry_safe(device, next, &add_list, add_list_entry) {
3d46a59a 2055 if (!pqi_is_device_added(device)) {
6c223761 2056 rc = pqi_add_device(ctrl_info, device);
ce143793
KB		 2057 if (rc == 0) {
2058 pqi_dev_info(ctrl_info, "added", device);
2059 } else {
6c223761
KB		 2060 dev_warn(&ctrl_info->pci_dev->dev,
2061 "scsi %d:%d:%d:%d addition failed, device not added\n",
2062 ctrl_info->scsi_host->host_no,
2063 device->bus, device->target,
2064 device->lun);
2065 pqi_fixup_botched_add(ctrl_info, device);
6c223761
KB		 2066 }
2067 }
6c223761
KB		 2068 }
2069}
2070
ce143793 2071static inline bool pqi_is_supported_device(struct pqi_scsi_dev *device)
6c223761 2072{
ce143793
KB		 2073 /*
2074 * Only support the HBA controller itself as a RAID
2075 * controller. If it's a RAID controller other than
2076 * the HBA itself (an external RAID controller, for
2077 * example), we don't support it.
2078 */
2079 if (device->device_type == SA_DEVICE_TYPE_CONTROLLER &&
2080 !pqi_is_hba_lunid(device->scsi3addr))
583891c9 2081 return false;
6c223761 2082
ce143793 2083 return true;
6c223761
KB		 2084}
2085
94086f5b 2086static inline bool pqi_skip_device(u8 *scsi3addr)
6c223761 2087{
94086f5b
KB		 2088 /* Ignore all masked devices. */
2089 if (MASKED_DEVICE(scsi3addr))
6c223761 2090 return true;
6c223761
KB		 2091
2092 return false;
2093}
2094
522bc026
DC		 2095static inline void pqi_mask_device(u8 *scsi3addr)
2096{
2097 scsi3addr[3] |= 0xc0;
2098}
2099
3d46a59a
DB		 2100static inline bool pqi_is_device_with_sas_address(struct pqi_scsi_dev *device)
2101{
ce143793
KB		 2102 switch (device->device_type) {
2103 case SA_DEVICE_TYPE_SAS:
2104 case SA_DEVICE_TYPE_EXPANDER_SMP:
2105 case SA_DEVICE_TYPE_SES:
3d46a59a
DB		 2106 return true;
2107 }
2108
2109 return false;
2110}
2111
cd128244
DC		 2112static inline bool pqi_expose_device(struct pqi_scsi_dev *device)
2113{
583891c9 2114 return !device->is_physical_device || !pqi_skip_device(device->scsi3addr);
cd128244
DC		 2115}
2116
7a84a821
KB		 2117static inline void pqi_set_physical_device_wwid(struct pqi_ctrl_info *ctrl_info,
2118 struct pqi_scsi_dev *device, struct report_phys_lun_extended_entry *phys_lun_ext_entry)
2119{
2120 if (ctrl_info->unique_wwid_in_report_phys_lun_supported ||
2121 pqi_is_device_with_sas_address(device))
2122 device->wwid = phys_lun_ext_entry->wwid;
2123 else
2124 device->wwid = cpu_to_be64(get_unaligned_be64(&device->page_83_identifier));
cd128244
DC		 2125}
2126
6c223761
KB		 2127static int pqi_update_scsi_devices(struct pqi_ctrl_info *ctrl_info)
2128{
2129 int i;
2130 int rc;
8a994a04 2131 LIST_HEAD(new_device_list_head);
6c223761
KB		 2132 struct report_phys_lun_extended *physdev_list = NULL;
2133 struct report_log_lun_extended *logdev_list = NULL;
2134 struct report_phys_lun_extended_entry *phys_lun_ext_entry;
2135 struct report_log_lun_extended_entry *log_lun_ext_entry;
2136 struct bmic_identify_physical_device *id_phys = NULL;
2137 u32 num_physicals;
2138 u32 num_logicals;
2139 struct pqi_scsi_dev **new_device_list = NULL;
2140 struct pqi_scsi_dev *device;
2141 struct pqi_scsi_dev *next;
2142 unsigned int num_new_devices;
2143 unsigned int num_valid_devices;
2144 bool is_physical_device;
2145 u8 *scsi3addr;
5e6a9760
GW		 2146 unsigned int physical_index;
2147 unsigned int logical_index;
6c223761 2148 static char *out_of_memory_msg =
6de783f6 2149 "failed to allocate memory, device discovery stopped";
6c223761 2150
6c223761
KB		 2151 rc = pqi_get_device_lists(ctrl_info, &physdev_list, &logdev_list);
2152 if (rc)
2153 goto out;
2154
2155 if (physdev_list)
2156 num_physicals =
2157 get_unaligned_be32(&physdev_list->header.list_length)
2158 / sizeof(physdev_list->lun_entries[0]);
2159 else
2160 num_physicals = 0;
2161
2162 if (logdev_list)
2163 num_logicals =
2164 get_unaligned_be32(&logdev_list->header.list_length)
2165 / sizeof(logdev_list->lun_entries[0]);
2166 else
2167 num_logicals = 0;
2168
2169 if (num_physicals) {
2170 /*
2171 * We need this buffer for calls to pqi_get_physical_disk_info()
2172 * below. We allocate it here instead of inside
2173 * pqi_get_physical_disk_info() because it's a fairly large
2174 * buffer.
2175 */
2176 id_phys = kmalloc(sizeof(*id_phys), GFP_KERNEL);
2177 if (!id_phys) {
2178 dev_warn(&ctrl_info->pci_dev->dev, "%s\n",
2179 out_of_memory_msg);
2180 rc = -ENOMEM;
2181 goto out;
2182 }
522bc026 2183
694c5d5b 2184 if (pqi_hide_vsep) {
522bc026
DC		 2185 for (i = num_physicals - 1; i >= 0; i--) {
2186 phys_lun_ext_entry =
2187 &physdev_list->lun_entries[i];
583891c9
KB		 2188 if (CISS_GET_DRIVE_NUMBER(phys_lun_ext_entry->lunid) == PQI_VSEP_CISS_BTL) {
2189 pqi_mask_device(phys_lun_ext_entry->lunid);
522bc026
DC		 2190 break;
2191 }
2192 }
2193 }
6c223761
KB		 2194 }
2195
f6cc2a77
KB		 2196 if (num_logicals &&
2197 (logdev_list->header.flags & CISS_REPORT_LOG_FLAG_DRIVE_TYPE_MIX))
2198 ctrl_info->lv_drive_type_mix_valid = true;
2199
6c223761
KB		 2200 num_new_devices = num_physicals + num_logicals;
2201
6da2ec56
KC		 2202 new_device_list = kmalloc_array(num_new_devices,
2203 sizeof(*new_device_list),
2204 GFP_KERNEL);
6c223761
KB		 2205 if (!new_device_list) {
2206 dev_warn(&ctrl_info->pci_dev->dev, "%s\n", out_of_memory_msg);
2207 rc = -ENOMEM;
2208 goto out;
2209 }
2210
2211 for (i = 0; i < num_new_devices; i++) {
2212 device = kzalloc(sizeof(*device), GFP_KERNEL);
2213 if (!device) {
2214 dev_warn(&ctrl_info->pci_dev->dev, "%s\n",
2215 out_of_memory_msg);
2216 rc = -ENOMEM;
2217 goto out;
2218 }
2219 list_add_tail(&device->new_device_list_entry,
2220 &new_device_list_head);
2221 }
2222
2223 device = NULL;
2224 num_valid_devices = 0;
5e6a9760
GW		 2225 physical_index = 0;
2226 logical_index = 0;
6c223761
KB		 2227
2228 for (i = 0; i < num_new_devices; i++) {
2229
5e6a9760
GW		 2230 if ((!pqi_expose_ld_first && i < num_physicals) ||
2231 (pqi_expose_ld_first && i >= num_logicals)) {
6c223761 2232 is_physical_device = true;
5e6a9760
GW		 2233 phys_lun_ext_entry =
2234 &physdev_list->lun_entries[physical_index++];
6c223761
KB		 2235 log_lun_ext_entry = NULL;
2236 scsi3addr = phys_lun_ext_entry->lunid;
2237 } else {
2238 is_physical_device = false;
2239 phys_lun_ext_entry = NULL;
2240 log_lun_ext_entry =
5e6a9760 2241 &logdev_list->lun_entries[logical_index++];
6c223761
KB		 2242 scsi3addr = log_lun_ext_entry->lunid;
2243 }
2244
94086f5b 2245 if (is_physical_device && pqi_skip_device(scsi3addr))
6c223761
KB		 2246 continue;
2247
2248 if (device)
2249 device = list_next_entry(device, new_device_list_entry);
2250 else
2251 device = list_first_entry(&new_device_list_head,
2252 struct pqi_scsi_dev, new_device_list_entry);
2253
2254 memcpy(device->scsi3addr, scsi3addr, sizeof(device->scsi3addr));
2255 device->is_physical_device = is_physical_device;
3d46a59a 2256 if (is_physical_device) {
ce143793
KB		 2257 device->device_type = phys_lun_ext_entry->device_type;
2258 if (device->device_type == SA_DEVICE_TYPE_EXPANDER_SMP)
3d46a59a
DB		 2259 device->is_expander_smp_device = true;
2260 } else {
bd10cf0b
KB		 2261 device->is_external_raid_device =
2262 pqi_is_external_raid_addr(scsi3addr);
3d46a59a 2263 }
6c223761 2264
ce143793
KB		 2265 if (!pqi_is_supported_device(device))
2266 continue;
2267
6c223761 2268 /* Gather information about the device. */
ce143793 2269 rc = pqi_get_device_info(ctrl_info, device, id_phys);
6c223761
KB		 2270 if (rc == -ENOMEM) {
2271 dev_warn(&ctrl_info->pci_dev->dev, "%s\n",
2272 out_of_memory_msg);
2273 goto out;
2274 }
2275 if (rc) {
6de783f6
KB		 2276 if (device->is_physical_device)
2277 dev_warn(&ctrl_info->pci_dev->dev,
2278 "obtaining device info failed, skipping physical device %016llx\n",
583891c9 2279 get_unaligned_be64(&phys_lun_ext_entry->wwid));
6de783f6
KB		 2280 else
2281 dev_warn(&ctrl_info->pci_dev->dev,
2282 "obtaining device info failed, skipping logical device %08x%08x\n",
2283 *((u32 *)&device->scsi3addr),
2284 *((u32 *)&device->scsi3addr[4]));
6c223761
KB		 2285 rc = 0;
2286 continue;
2287 }
2288
6c223761
KB		 2289 pqi_assign_bus_target_lun(device);
2290
6c223761 2291 if (device->is_physical_device) {
7a84a821 2292 pqi_set_physical_device_wwid(ctrl_info, device, phys_lun_ext_entry);
6c223761 2293 if ((phys_lun_ext_entry->device_flags &
694c5d5b 2294 CISS_REPORT_PHYS_DEV_FLAG_AIO_ENABLED) &&
3d46a59a 2295 phys_lun_ext_entry->aio_handle) {
583891c9
KB		 2296 device->aio_enabled = true;
2297 device->aio_handle =
2298 phys_lun_ext_entry->aio_handle;
3d46a59a 2299 }
6c223761
KB		 2300 } else {
2301 memcpy(device->volume_id, log_lun_ext_entry->volume_id,
2302 sizeof(device->volume_id));
2303 }
2304
3d46a59a
DB		 2305 if (pqi_is_device_with_sas_address(device))
2306 device->sas_address = get_unaligned_be64(&device->wwid);
6c223761
KB		 2307
2308 new_device_list[num_valid_devices++] = device;
2309 }
2310
2311 pqi_update_device_list(ctrl_info, new_device_list, num_valid_devices);
2312
2313out:
2314 list_for_each_entry_safe(device, next, &new_device_list_head,
2315 new_device_list_entry) {
2316 if (device->keep_device)
2317 continue;
2318 list_del(&device->new_device_list_entry);
2319 pqi_free_device(device);
2320 }
2321
2322 kfree(new_device_list);
2323 kfree(physdev_list);
2324 kfree(logdev_list);
2325 kfree(id_phys);
2326
2327 return rc;
2328}
2329
6c223761
KB		 2330static int pqi_scan_scsi_devices(struct pqi_ctrl_info *ctrl_info)
2331{
66f1c2b4
KB		 2332 int rc;
2333 int mutex_acquired;
6c223761
KB		 2334
2335 if (pqi_ctrl_offline(ctrl_info))
2336 return -ENXIO;
2337
66f1c2b4
KB		 2338 mutex_acquired = mutex_trylock(&ctrl_info->scan_mutex);
2339
2340 if (!mutex_acquired) {
2341 if (pqi_ctrl_scan_blocked(ctrl_info))
2342 return -EBUSY;
5f310425 2343 pqi_schedule_rescan_worker_delayed(ctrl_info);
66f1c2b4 2344 return -EINPROGRESS;
530dd8a7 2345 }
6c223761 2346
66f1c2b4
KB		 2347 rc = pqi_update_scsi_devices(ctrl_info);
2348 if (rc && !pqi_ctrl_scan_blocked(ctrl_info))
2349 pqi_schedule_rescan_worker_delayed(ctrl_info);
2350
2351 mutex_unlock(&ctrl_info->scan_mutex);
2352
6c223761
KB		 2353 return rc;
2354}
2355
2356static void pqi_scan_start(struct Scsi_Host *shost)
2357{
4fd22c13
MR		 2358 struct pqi_ctrl_info *ctrl_info;
2359
2360 ctrl_info = shost_to_hba(shost);
4fd22c13
MR		 2361
2362 pqi_scan_scsi_devices(ctrl_info);
6c223761
KB		 2363}
2364
2365/* Returns TRUE if scan is finished. */
2366
2367static int pqi_scan_finished(struct Scsi_Host *shost,
2368 unsigned long elapsed_time)
2369{
2370 struct pqi_ctrl_info *ctrl_info;
2371
2372 ctrl_info = shost_priv(shost);
2373
2374 return !mutex_is_locked(&ctrl_info->scan_mutex);
2375}
2376
583891c9
KB		 2377static inline void pqi_set_encryption_info(struct pqi_encryption_info *encryption_info,
2378 struct raid_map *raid_map, u64 first_block)
6c223761
KB		 2379{
2380 u32 volume_blk_size;
2381
2382 /*
2383 * Set the encryption tweak values based on logical block address.
2384 * If the block size is 512, the tweak value is equal to the LBA.
2385 * For other block sizes, tweak value is (LBA * block size) / 512.
2386 */
2387 volume_blk_size = get_unaligned_le32(&raid_map->volume_blk_size);
2388 if (volume_blk_size != 512)
2389 first_block = (first_block * volume_blk_size) / 512;
2390
2391 encryption_info->data_encryption_key_index =
2392 get_unaligned_le16(&raid_map->data_encryption_key_index);
2393 encryption_info->encrypt_tweak_lower = lower_32_bits(first_block);
2394 encryption_info->encrypt_tweak_upper = upper_32_bits(first_block);
2395}
2396
2397/*
588a63fe 2398 * Attempt to perform RAID bypass mapping for a logical volume I/O.
6c223761
KB		 2399 */
2400
6702d2c4
DB		 2401static bool pqi_aio_raid_level_supported(struct pqi_ctrl_info *ctrl_info,
2402 struct pqi_scsi_dev_raid_map_data *rmd)
281a817f
DB		 2403{
2404 bool is_supported = true;
2405
2406 switch (rmd->raid_level) {
2407 case SA_RAID_0:
2408 break;
2409 case SA_RAID_1:
f6cc2a77
KB		 2410 if (rmd->is_write && (!ctrl_info->enable_r1_writes ||
2411 rmd->data_length > ctrl_info->max_write_raid_1_10_2drive))
2412 is_supported = false;
2413 break;
7a012c23 2414 case SA_RAID_TRIPLE:
f6cc2a77
KB		 2415 if (rmd->is_write && (!ctrl_info->enable_r1_writes ||
2416 rmd->data_length > ctrl_info->max_write_raid_1_10_3drive))
281a817f
DB		 2417 is_supported = false;
2418 break;
2419 case SA_RAID_5:
f6cc2a77
KB		 2420 if (rmd->is_write && (!ctrl_info->enable_r5_writes ||
2421 rmd->data_length > ctrl_info->max_write_raid_5_6))
6702d2c4
DB		 2422 is_supported = false;
2423 break;
281a817f 2424 case SA_RAID_6:
f6cc2a77
KB		 2425 if (rmd->is_write && (!ctrl_info->enable_r6_writes ||
2426 rmd->data_length > ctrl_info->max_write_raid_5_6))
281a817f
DB		 2427 is_supported = false;
2428 break;
281a817f
DB		 2429 default:
2430 is_supported = false;
f6cc2a77 2431 break;
281a817f
DB		 2432 }
2433
2434 return is_supported;
2435}
2436
6c223761
KB		 2437#define PQI_RAID_BYPASS_INELIGIBLE 1
2438
281a817f 2439static int pqi_get_aio_lba_and_block_count(struct scsi_cmnd *scmd,
583891c9 2440 struct pqi_scsi_dev_raid_map_data *rmd)
6c223761 2441{
6c223761
KB		 2442 /* Check for valid opcode, get LBA and block count. */
2443 switch (scmd->cmnd[0]) {
2444 case WRITE_6:
281a817f 2445 rmd->is_write = true;
df561f66 2446 fallthrough;
6c223761 2447 case READ_6:
281a817f 2448 rmd->first_block = (u64)(((scmd->cmnd[1] & 0x1f) << 16) |
e018ef57 2449 (scmd->cmnd[2] << 8) | scmd->cmnd[3]);
281a817f
DB		 2450 rmd->block_cnt = (u32)scmd->cmnd[4];
2451 if (rmd->block_cnt == 0)
2452 rmd->block_cnt = 256;
6c223761
KB		 2453 break;
2454 case WRITE_10:
281a817f 2455 rmd->is_write = true;
df561f66 2456 fallthrough;
6c223761 2457 case READ_10:
281a817f
DB		 2458 rmd->first_block = (u64)get_unaligned_be32(&scmd->cmnd[2]);
2459 rmd->block_cnt = (u32)get_unaligned_be16(&scmd->cmnd[7]);
6c223761
KB		 2460 break;
2461 case WRITE_12:
281a817f 2462 rmd->is_write = true;
df561f66 2463 fallthrough;
6c223761 2464 case READ_12:
281a817f
DB		 2465 rmd->first_block = (u64)get_unaligned_be32(&scmd->cmnd[2]);
2466 rmd->block_cnt = get_unaligned_be32(&scmd->cmnd[6]);
6c223761
KB		 2467 break;
2468 case WRITE_16:
281a817f 2469 rmd->is_write = true;
df561f66 2470 fallthrough;
6c223761 2471 case READ_16:
281a817f
DB		 2472 rmd->first_block = get_unaligned_be64(&scmd->cmnd[2]);
2473 rmd->block_cnt = get_unaligned_be32(&scmd->cmnd[10]);
6c223761
KB		 2474 break;
2475 default:
2476 /* Process via normal I/O path. */
2477 return PQI_RAID_BYPASS_INELIGIBLE;
2478 }
2479
281a817f 2480 put_unaligned_le32(scsi_bufflen(scmd), &rmd->data_length);
6c223761 2481
281a817f
DB		 2482 return 0;
2483}
6c223761 2484
281a817f 2485static int pci_get_aio_common_raid_map_values(struct pqi_ctrl_info *ctrl_info,
583891c9 2486 struct pqi_scsi_dev_raid_map_data *rmd, struct raid_map *raid_map)
281a817f
DB		 2487{
2488#if BITS_PER_LONG == 32
2489 u64 tmpdiv;
2490#endif
2491
2492 rmd->last_block = rmd->first_block + rmd->block_cnt - 1;
6c223761
KB		 2493
2494 /* Check for invalid block or wraparound. */
281a817f
DB		 2495 if (rmd->last_block >=
2496 get_unaligned_le64(&raid_map->volume_blk_cnt) ||
2497 rmd->last_block < rmd->first_block)
6c223761
KB		 2498 return PQI_RAID_BYPASS_INELIGIBLE;
2499
281a817f 2500 rmd->data_disks_per_row =
583891c9 2501 get_unaligned_le16(&raid_map->data_disks_per_row);
281a817f
DB		 2502 rmd->strip_size = get_unaligned_le16(&raid_map->strip_size);
2503 rmd->layout_map_count = get_unaligned_le16(&raid_map->layout_map_count);
6c223761
KB		 2504
2505 /* Calculate stripe information for the request. */
281a817f 2506 rmd->blocks_per_row = rmd->data_disks_per_row * rmd->strip_size;
667298ce
DB		 2507 if (rmd->blocks_per_row == 0) /* Used as a divisor in many calculations */
2508 return PQI_RAID_BYPASS_INELIGIBLE;
6c223761 2509#if BITS_PER_LONG == 32
281a817f
DB		 2510 tmpdiv = rmd->first_block;
2511 do_div(tmpdiv, rmd->blocks_per_row);
2512 rmd->first_row = tmpdiv;
2513 tmpdiv = rmd->last_block;
2514 do_div(tmpdiv, rmd->blocks_per_row);
2515 rmd->last_row = tmpdiv;
2516 rmd->first_row_offset = (u32)(rmd->first_block - (rmd->first_row * rmd->blocks_per_row));
2517 rmd->last_row_offset = (u32)(rmd->last_block - (rmd->last_row * rmd->blocks_per_row));
2518 tmpdiv = rmd->first_row_offset;
2519 do_div(tmpdiv, rmd->strip_size);
2520 rmd->first_column = tmpdiv;
2521 tmpdiv = rmd->last_row_offset;
2522 do_div(tmpdiv, rmd->strip_size);
2523 rmd->last_column = tmpdiv;
6c223761 2524#else
281a817f
DB		 2525 rmd->first_row = rmd->first_block / rmd->blocks_per_row;
2526 rmd->last_row = rmd->last_block / rmd->blocks_per_row;
2527 rmd->first_row_offset = (u32)(rmd->first_block -
583891c9 2528 (rmd->first_row * rmd->blocks_per_row));
281a817f 2529 rmd->last_row_offset = (u32)(rmd->last_block - (rmd->last_row *
583891c9 2530 rmd->blocks_per_row));
281a817f
DB		 2531 rmd->first_column = rmd->first_row_offset / rmd->strip_size;
2532 rmd->last_column = rmd->last_row_offset / rmd->strip_size;
6c223761
KB		 2533#endif
2534
2535 /* If this isn't a single row/column then give to the controller. */
281a817f 2536 if (rmd->first_row != rmd->last_row ||
583891c9 2537 rmd->first_column != rmd->last_column)
6c223761
KB		 2538 return PQI_RAID_BYPASS_INELIGIBLE;
2539
2540 /* Proceeding with driver mapping. */
281a817f 2541 rmd->total_disks_per_row = rmd->data_disks_per_row +
6c223761 2542 get_unaligned_le16(&raid_map->metadata_disks_per_row);
281a817f
DB		 2543 rmd->map_row = ((u32)(rmd->first_row >>
2544 raid_map->parity_rotation_shift)) %
6c223761 2545 get_unaligned_le16(&raid_map->row_cnt);
281a817f 2546 rmd->map_index = (rmd->map_row * rmd->total_disks_per_row) +
583891c9 2547 rmd->first_column;
6c223761 2548
281a817f
DB		 2549 return 0;
2550}
6c223761 2551
281a817f 2552static int pqi_calc_aio_r5_or_r6(struct pqi_scsi_dev_raid_map_data *rmd,
583891c9 2553 struct raid_map *raid_map)
281a817f 2554{
6c223761 2555#if BITS_PER_LONG == 32
281a817f 2556 u64 tmpdiv;
6c223761 2557#endif
6c223761 2558
667298ce
DB		 2559 if (rmd->blocks_per_row == 0) /* Used as a divisor in many calculations */
2560 return PQI_RAID_BYPASS_INELIGIBLE;
2561
281a817f 2562 /* RAID 50/60 */
583891c9 2563 /* Verify first and last block are in same RAID group. */
281a817f 2564 rmd->stripesize = rmd->blocks_per_row * rmd->layout_map_count;
6c223761 2565#if BITS_PER_LONG == 32
281a817f
DB		 2566 tmpdiv = rmd->first_block;
2567 rmd->first_group = do_div(tmpdiv, rmd->stripesize);
2568 tmpdiv = rmd->first_group;
2569 do_div(tmpdiv, rmd->blocks_per_row);
2570 rmd->first_group = tmpdiv;
2571 tmpdiv = rmd->last_block;
2572 rmd->last_group = do_div(tmpdiv, rmd->stripesize);
2573 tmpdiv = rmd->last_group;
2574 do_div(tmpdiv, rmd->blocks_per_row);
2575 rmd->last_group = tmpdiv;
6c223761 2576#else
281a817f
DB		 2577 rmd->first_group = (rmd->first_block % rmd->stripesize) / rmd->blocks_per_row;
2578 rmd->last_group = (rmd->last_block % rmd->stripesize) / rmd->blocks_per_row;
6c223761 2579#endif
281a817f
DB		 2580 if (rmd->first_group != rmd->last_group)
2581 return PQI_RAID_BYPASS_INELIGIBLE;
6c223761 2582
583891c9 2583 /* Verify request is in a single row of RAID 5/6. */
6c223761 2584#if BITS_PER_LONG == 32
281a817f
DB		 2585 tmpdiv = rmd->first_block;
2586 do_div(tmpdiv, rmd->stripesize);
2587 rmd->first_row = tmpdiv;
2588 rmd->r5or6_first_row = tmpdiv;
2589 tmpdiv = rmd->last_block;
2590 do_div(tmpdiv, rmd->stripesize);
2591 rmd->r5or6_last_row = tmpdiv;
6c223761 2592#else
281a817f
DB		 2593 rmd->first_row = rmd->r5or6_first_row =
2594 rmd->first_block / rmd->stripesize;
2595 rmd->r5or6_last_row = rmd->last_block / rmd->stripesize;
6c223761 2596#endif
281a817f
DB		 2597 if (rmd->r5or6_first_row != rmd->r5or6_last_row)
2598 return PQI_RAID_BYPASS_INELIGIBLE;
6c223761 2599
583891c9 2600 /* Verify request is in a single column. */
6c223761 2601#if BITS_PER_LONG == 32
281a817f
DB		 2602 tmpdiv = rmd->first_block;
2603 rmd->first_row_offset = do_div(tmpdiv, rmd->stripesize);
2604 tmpdiv = rmd->first_row_offset;
2605 rmd->first_row_offset = (u32)do_div(tmpdiv, rmd->blocks_per_row);
2606 rmd->r5or6_first_row_offset = rmd->first_row_offset;
2607 tmpdiv = rmd->last_block;
2608 rmd->r5or6_last_row_offset = do_div(tmpdiv, rmd->stripesize);
2609 tmpdiv = rmd->r5or6_last_row_offset;
2610 rmd->r5or6_last_row_offset = do_div(tmpdiv, rmd->blocks_per_row);
2611 tmpdiv = rmd->r5or6_first_row_offset;
2612 do_div(tmpdiv, rmd->strip_size);
2613 rmd->first_column = rmd->r5or6_first_column = tmpdiv;
2614 tmpdiv = rmd->r5or6_last_row_offset;
2615 do_div(tmpdiv, rmd->strip_size);
2616 rmd->r5or6_last_column = tmpdiv;
6c223761 2617#else
281a817f 2618 rmd->first_row_offset = rmd->r5or6_first_row_offset =
583891c9
KB		 2619 (u32)((rmd->first_block % rmd->stripesize) %
2620 rmd->blocks_per_row);
281a817f
DB		 2621
2622 rmd->r5or6_last_row_offset =
2623 (u32)((rmd->last_block % rmd->stripesize) %
2624 rmd->blocks_per_row);
2625
2626 rmd->first_column =
583891c9 2627 rmd->r5or6_first_row_offset / rmd->strip_size;
281a817f
DB		 2628 rmd->r5or6_first_column = rmd->first_column;
2629 rmd->r5or6_last_column = rmd->r5or6_last_row_offset / rmd->strip_size;
2630#endif
2631 if (rmd->r5or6_first_column != rmd->r5or6_last_column)
2632 return PQI_RAID_BYPASS_INELIGIBLE;
2633
583891c9 2634 /* Request is eligible. */
281a817f
DB		 2635 rmd->map_row =
2636 ((u32)(rmd->first_row >> raid_map->parity_rotation_shift)) %
2637 get_unaligned_le16(&raid_map->row_cnt);
6c223761 2638
281a817f
DB		 2639 rmd->map_index = (rmd->first_group *
2640 (get_unaligned_le16(&raid_map->row_cnt) *
2641 rmd->total_disks_per_row)) +
2642 (rmd->map_row * rmd->total_disks_per_row) + rmd->first_column;
6c223761 2643
6702d2c4
DB		 2644 if (rmd->is_write) {
2645 u32 index;
6c223761 2646
6702d2c4
DB		 2647 /*
2648 * p_parity_it_nexus and q_parity_it_nexus are pointers to the
2649 * parity entries inside the device's raid_map.
2650 *
2651 * A device's RAID map is bounded by: number of RAID disks squared.
2652 *
2653 * The devices RAID map size is checked during device
2654 * initialization.
2655 */
2656 index = DIV_ROUND_UP(rmd->map_index + 1, rmd->total_disks_per_row);
2657 index *= rmd->total_disks_per_row;
2658 index -= get_unaligned_le16(&raid_map->metadata_disks_per_row);
2659
2660 rmd->p_parity_it_nexus = raid_map->disk_data[index].aio_handle;
2661 if (rmd->raid_level == SA_RAID_6) {
2662 rmd->q_parity_it_nexus = raid_map->disk_data[index + 1].aio_handle;
2663 rmd->xor_mult = raid_map->disk_data[rmd->map_index].xor_mult[1];
2664 }
6702d2c4
DB		 2665#if BITS_PER_LONG == 32
2666 tmpdiv = rmd->first_block;
2667 do_div(tmpdiv, rmd->blocks_per_row);
2668 rmd->row = tmpdiv;
2669#else
2670 rmd->row = rmd->first_block / rmd->blocks_per_row;
6c223761 2671#endif
6702d2c4
DB		 2672 }
2673
281a817f
DB		 2674 return 0;
2675}
2676
2677static void pqi_set_aio_cdb(struct pqi_scsi_dev_raid_map_data *rmd)
2678{
2679 /* Build the new CDB for the physical disk I/O. */
2680 if (rmd->disk_block > 0xffffffff) {
2681 rmd->cdb[0] = rmd->is_write ? WRITE_16 : READ_16;
2682 rmd->cdb[1] = 0;
2683 put_unaligned_be64(rmd->disk_block, &rmd->cdb[2]);
2684 put_unaligned_be32(rmd->disk_block_cnt, &rmd->cdb[10]);
2685 rmd->cdb[14] = 0;
2686 rmd->cdb[15] = 0;
2687 rmd->cdb_length = 16;
2688 } else {
2689 rmd->cdb[0] = rmd->is_write ? WRITE_10 : READ_10;
2690 rmd->cdb[1] = 0;
2691 put_unaligned_be32((u32)rmd->disk_block, &rmd->cdb[2]);
2692 rmd->cdb[6] = 0;
2693 put_unaligned_be16((u16)rmd->disk_block_cnt, &rmd->cdb[7]);
2694 rmd->cdb[9] = 0;
2695 rmd->cdb_length = 10;
2696 }
2697}
2698
7a012c23 2699static void pqi_calc_aio_r1_nexus(struct raid_map *raid_map,
583891c9 2700 struct pqi_scsi_dev_raid_map_data *rmd)
7a012c23
DB		 2701{
2702 u32 index;
2703 u32 group;
2704
2705 group = rmd->map_index / rmd->data_disks_per_row;
2706
2707 index = rmd->map_index - (group * rmd->data_disks_per_row);
2708 rmd->it_nexus[0] = raid_map->disk_data[index].aio_handle;
2709 index += rmd->data_disks_per_row;
2710 rmd->it_nexus[1] = raid_map->disk_data[index].aio_handle;
2711 if (rmd->layout_map_count > 2) {
2712 index += rmd->data_disks_per_row;
2713 rmd->it_nexus[2] = raid_map->disk_data[index].aio_handle;
2714 }
2715
2716 rmd->num_it_nexus_entries = rmd->layout_map_count;
2717}
2718
281a817f
DB		 2719static int pqi_raid_bypass_submit_scsi_cmd(struct pqi_ctrl_info *ctrl_info,
2720 struct pqi_scsi_dev *device, struct scsi_cmnd *scmd,
2721 struct pqi_queue_group *queue_group)
2722{
281a817f 2723 int rc;
7a012c23
DB		 2724 struct raid_map *raid_map;
2725 u32 group;
2726 u32 next_bypass_group;
281a817f
DB		 2727 struct pqi_encryption_info *encryption_info_ptr;
2728 struct pqi_encryption_info encryption_info;
583891c9 2729 struct pqi_scsi_dev_raid_map_data rmd = { 0 };
281a817f
DB		 2730
2731 rc = pqi_get_aio_lba_and_block_count(scmd, &rmd);
2732 if (rc)
2733 return PQI_RAID_BYPASS_INELIGIBLE;
2734
2735 rmd.raid_level = device->raid_level;
6c223761 2736
6702d2c4 2737 if (!pqi_aio_raid_level_supported(ctrl_info, &rmd))
281a817f
DB		 2738 return PQI_RAID_BYPASS_INELIGIBLE;
2739
2740 if (unlikely(rmd.block_cnt == 0))
2741 return PQI_RAID_BYPASS_INELIGIBLE;
2742
2743 raid_map = device->raid_map;
6c223761 2744
281a817f
DB		 2745 rc = pci_get_aio_common_raid_map_values(ctrl_info, &rmd, raid_map);
2746 if (rc)
2747 return PQI_RAID_BYPASS_INELIGIBLE;
6c223761 2748
7a012c23
DB		 2749 if (device->raid_level == SA_RAID_1 ||
2750 device->raid_level == SA_RAID_TRIPLE) {
2751 if (rmd.is_write) {
2752 pqi_calc_aio_r1_nexus(raid_map, &rmd);
2753 } else {
2754 group = device->next_bypass_group;
2755 next_bypass_group = group + 1;
2756 if (next_bypass_group >= rmd.layout_map_count)
2757 next_bypass_group = 0;
2758 device->next_bypass_group = next_bypass_group;
2759 rmd.map_index += group * rmd.data_disks_per_row;
2760 }
281a817f 2761 } else if ((device->raid_level == SA_RAID_5 ||
6702d2c4
DB		 2762 device->raid_level == SA_RAID_6) &&
2763 (rmd.layout_map_count > 1 || rmd.is_write)) {
281a817f
DB		 2764 rc = pqi_calc_aio_r5_or_r6(&rmd, raid_map);
2765 if (rc)
2766 return PQI_RAID_BYPASS_INELIGIBLE;
6c223761
KB		 2767 }
2768
281a817f
DB		 2769 if (unlikely(rmd.map_index >= RAID_MAP_MAX_ENTRIES))
2770 return PQI_RAID_BYPASS_INELIGIBLE;
2771
2772 rmd.aio_handle = raid_map->disk_data[rmd.map_index].aio_handle;
2773 rmd.disk_block = get_unaligned_le64(&raid_map->disk_starting_blk) +
2774 rmd.first_row * rmd.strip_size +
2775 (rmd.first_row_offset - rmd.first_column * rmd.strip_size);
2776 rmd.disk_block_cnt = rmd.block_cnt;
6c223761
KB		 2777
2778 /* Handle differing logical/physical block sizes. */
2779 if (raid_map->phys_blk_shift) {
281a817f
DB		 2780 rmd.disk_block <<= raid_map->phys_blk_shift;
2781 rmd.disk_block_cnt <<= raid_map->phys_blk_shift;
6c223761
KB		 2782 }
2783
281a817f 2784 if (unlikely(rmd.disk_block_cnt > 0xffff))
6c223761
KB		 2785 return PQI_RAID_BYPASS_INELIGIBLE;
2786
281a817f 2787 pqi_set_aio_cdb(&rmd);
6c223761 2788
583891c9 2789 if (get_unaligned_le16(&raid_map->flags) & RAID_MAP_ENCRYPTION_ENABLED) {
f6cc2a77
KB		 2790 if (rmd.data_length > device->max_transfer_encrypted)
2791 return PQI_RAID_BYPASS_INELIGIBLE;
583891c9 2792 pqi_set_encryption_info(&encryption_info, raid_map, rmd.first_block);
6c223761
KB		 2793 encryption_info_ptr = &encryption_info;
2794 } else {
2795 encryption_info_ptr = NULL;
2796 }
2797
6702d2c4
DB		 2798 if (rmd.is_write) {
2799 switch (device->raid_level) {
7a012c23
DB		 2800 case SA_RAID_1:
2801 case SA_RAID_TRIPLE:
2802 return pqi_aio_submit_r1_write_io(ctrl_info, scmd, queue_group,
2803 encryption_info_ptr, device, &rmd);
6702d2c4
DB		 2804 case SA_RAID_5:
2805 case SA_RAID_6:
2806 return pqi_aio_submit_r56_write_io(ctrl_info, scmd, queue_group,
583891c9 2807 encryption_info_ptr, device, &rmd);
6702d2c4 2808 }
6702d2c4
DB		 2809 }
2810
f6cc2a77
KB		 2811 return pqi_aio_submit_io(ctrl_info, scmd, rmd.aio_handle,
2812 rmd.cdb, rmd.cdb_length, queue_group,
2813 encryption_info_ptr, true);
6c223761
KB		 2814}
2815
2816#define PQI_STATUS_IDLE 0x0
2817
2818#define PQI_CREATE_ADMIN_QUEUE_PAIR 1
2819#define PQI_DELETE_ADMIN_QUEUE_PAIR 2
2820
2821#define PQI_DEVICE_STATE_POWER_ON_AND_RESET 0x0
2822#define PQI_DEVICE_STATE_STATUS_AVAILABLE 0x1
2823#define PQI_DEVICE_STATE_ALL_REGISTERS_READY 0x2
2824#define PQI_DEVICE_STATE_ADMIN_QUEUE_PAIR_READY 0x3
2825#define PQI_DEVICE_STATE_ERROR 0x4
2826
2827#define PQI_MODE_READY_TIMEOUT_SECS 30
2828#define PQI_MODE_READY_POLL_INTERVAL_MSECS 1
2829
2830static int pqi_wait_for_pqi_mode_ready(struct pqi_ctrl_info *ctrl_info)
2831{
2832 struct pqi_device_registers __iomem *pqi_registers;
2833 unsigned long timeout;
2834 u64 signature;
2835 u8 status;
2836
2837 pqi_registers = ctrl_info->pqi_registers;
4fd22c13 2838 timeout = (PQI_MODE_READY_TIMEOUT_SECS * PQI_HZ) + jiffies;
6c223761
KB		 2839
2840 while (1) {
2841 signature = readq(&pqi_registers->signature);
2842 if (memcmp(&signature, PQI_DEVICE_SIGNATURE,
2843 sizeof(signature)) == 0)
2844 break;
2845 if (time_after(jiffies, timeout)) {
2846 dev_err(&ctrl_info->pci_dev->dev,
2847 "timed out waiting for PQI signature\n");
2848 return -ETIMEDOUT;
2849 }
2850 msleep(PQI_MODE_READY_POLL_INTERVAL_MSECS);
2851 }
2852
2853 while (1) {
2854 status = readb(&pqi_registers->function_and_status_code);
2855 if (status == PQI_STATUS_IDLE)
2856 break;
2857 if (time_after(jiffies, timeout)) {
2858 dev_err(&ctrl_info->pci_dev->dev,
2859 "timed out waiting for PQI IDLE\n");
2860 return -ETIMEDOUT;
2861 }
2862 msleep(PQI_MODE_READY_POLL_INTERVAL_MSECS);
2863 }
2864
2865 while (1) {
2866 if (readl(&pqi_registers->device_status) ==
2867 PQI_DEVICE_STATE_ALL_REGISTERS_READY)
2868 break;
2869 if (time_after(jiffies, timeout)) {
2870 dev_err(&ctrl_info->pci_dev->dev,
2871 "timed out waiting for PQI all registers ready\n");
2872 return -ETIMEDOUT;
2873 }
2874 msleep(PQI_MODE_READY_POLL_INTERVAL_MSECS);
2875 }
2876
2877 return 0;
2878}
2879
2880static inline void pqi_aio_path_disabled(struct pqi_io_request *io_request)
2881{
2882 struct pqi_scsi_dev *device;
2883
2884 device = io_request->scmd->device->hostdata;
588a63fe 2885 device->raid_bypass_enabled = false;
376fb880 2886 device->aio_enabled = false;
6c223761
KB		 2887}
2888
d87d5474 2889static inline void pqi_take_device_offline(struct scsi_device *sdev, char *path)
6c223761
KB		 2890{
2891 struct pqi_ctrl_info *ctrl_info;
e58081a7 2892 struct pqi_scsi_dev *device;
6c223761 2893
03b288cf
KB		 2894 device = sdev->hostdata;
2895 if (device->device_offline)
2896 return;
2897
2898 device->device_offline = true;
03b288cf
KB		 2899 ctrl_info = shost_to_hba(sdev->host);
2900 pqi_schedule_rescan_worker(ctrl_info);
a9a68101 2901 dev_err(&ctrl_info->pci_dev->dev, "re-scanning %s scsi %d:%d:%d:%d\n",
03b288cf
KB		 2902 path, ctrl_info->scsi_host->host_no, device->bus,
2903 device->target, device->lun);
6c223761
KB		 2904}
2905
2906static void pqi_process_raid_io_error(struct pqi_io_request *io_request)
2907{
2908 u8 scsi_status;
2909 u8 host_byte;
2910 struct scsi_cmnd *scmd;
2911 struct pqi_raid_error_info *error_info;
2912 size_t sense_data_length;
2913 int residual_count;
2914 int xfer_count;
2915 struct scsi_sense_hdr sshdr;
2916
2917 scmd = io_request->scmd;
2918 if (!scmd)
2919 return;
2920
2921 error_info = io_request->error_info;
2922 scsi_status = error_info->status;
2923 host_byte = DID_OK;
2924
f5b63206
KB		 2925 switch (error_info->data_out_result) {
2926 case PQI_DATA_IN_OUT_GOOD:
2927 break;
2928 case PQI_DATA_IN_OUT_UNDERFLOW:
6c223761
KB		 2929 xfer_count =
2930 get_unaligned_le32(&error_info->data_out_transferred);
2931 residual_count = scsi_bufflen(scmd) - xfer_count;
2932 scsi_set_resid(scmd, residual_count);
2933 if (xfer_count < scmd->underflow)
2934 host_byte = DID_SOFT_ERROR;
f5b63206
KB		 2935 break;
2936 case PQI_DATA_IN_OUT_UNSOLICITED_ABORT:
2937 case PQI_DATA_IN_OUT_ABORTED:
2938 host_byte = DID_ABORT;
2939 break;
2940 case PQI_DATA_IN_OUT_TIMEOUT:
2941 host_byte = DID_TIME_OUT;
2942 break;
2943 case PQI_DATA_IN_OUT_BUFFER_OVERFLOW:
2944 case PQI_DATA_IN_OUT_PROTOCOL_ERROR:
2945 case PQI_DATA_IN_OUT_BUFFER_ERROR:
2946 case PQI_DATA_IN_OUT_BUFFER_OVERFLOW_DESCRIPTOR_AREA:
2947 case PQI_DATA_IN_OUT_BUFFER_OVERFLOW_BRIDGE:
2948 case PQI_DATA_IN_OUT_ERROR:
2949 case PQI_DATA_IN_OUT_HARDWARE_ERROR:
2950 case PQI_DATA_IN_OUT_PCIE_FABRIC_ERROR:
2951 case PQI_DATA_IN_OUT_PCIE_COMPLETION_TIMEOUT:
2952 case PQI_DATA_IN_OUT_PCIE_COMPLETER_ABORT_RECEIVED:
2953 case PQI_DATA_IN_OUT_PCIE_UNSUPPORTED_REQUEST_RECEIVED:
2954 case PQI_DATA_IN_OUT_PCIE_ECRC_CHECK_FAILED:
2955 case PQI_DATA_IN_OUT_PCIE_UNSUPPORTED_REQUEST:
2956 case PQI_DATA_IN_OUT_PCIE_ACS_VIOLATION:
2957 case PQI_DATA_IN_OUT_PCIE_TLP_PREFIX_BLOCKED:
2958 case PQI_DATA_IN_OUT_PCIE_POISONED_MEMORY_READ:
2959 default:
2960 host_byte = DID_ERROR;
2961 break;
6c223761
KB		 2962 }
2963
2964 sense_data_length = get_unaligned_le16(&error_info->sense_data_length);
2965 if (sense_data_length == 0)
2966 sense_data_length =
2967 get_unaligned_le16(&error_info->response_data_length);
2968 if (sense_data_length) {
2969 if (sense_data_length > sizeof(error_info->data))
2970 sense_data_length = sizeof(error_info->data);
2971
2972 if (scsi_status == SAM_STAT_CHECK_CONDITION &&
2973 scsi_normalize_sense(error_info->data,
2974 sense_data_length, &sshdr) &&
2975 sshdr.sense_key == HARDWARE_ERROR &&
8ef860ae 2976 sshdr.asc == 0x3e) {
441b7195
EV		 2977 struct pqi_ctrl_info *ctrl_info = shost_to_hba(scmd->device->host);
2978 struct pqi_scsi_dev *device = scmd->device->hostdata;
2979
8ef860ae
EV		 2980 switch (sshdr.ascq) {
2981 case 0x1: /* LOGICAL UNIT FAILURE */
2982 if (printk_ratelimit())
2983 scmd_printk(KERN_ERR, scmd, "received 'logical unit failure' from controller for scsi %d:%d:%d:%d\n",
2984 ctrl_info->scsi_host->host_no, device->bus, device->target, device->lun);
2985 pqi_take_device_offline(scmd->device, "RAID");
2986 host_byte = DID_NO_CONNECT;
2987 break;
2988
2989 default: /* See http://www.t10.org/lists/asc-num.htm#ASC_3E */
2990 if (printk_ratelimit())
2991 scmd_printk(KERN_ERR, scmd, "received unhandled error %d from controller for scsi %d:%d:%d:%d\n",
2992 sshdr.ascq, ctrl_info->scsi_host->host_no, device->bus, device->target, device->lun);
2993 break;
2994 }
6c223761
KB		 2995 }
2996
2997 if (sense_data_length > SCSI_SENSE_BUFFERSIZE)
2998 sense_data_length = SCSI_SENSE_BUFFERSIZE;
2999 memcpy(scmd->sense_buffer, error_info->data,
3000 sense_data_length);
3001 }
3002
3003 scmd->result = scsi_status;
3004 set_host_byte(scmd, host_byte);
3005}
3006
3007static void pqi_process_aio_io_error(struct pqi_io_request *io_request)
3008{
3009 u8 scsi_status;
3010 u8 host_byte;
3011 struct scsi_cmnd *scmd;
3012 struct pqi_aio_error_info *error_info;
3013 size_t sense_data_length;
3014 int residual_count;
3015 int xfer_count;
3016 bool device_offline;
3017
3018 scmd = io_request->scmd;
3019 error_info = io_request->error_info;
3020 host_byte = DID_OK;
3021 sense_data_length = 0;
3022 device_offline = false;
3023
3024 switch (error_info->service_response) {
3025 case PQI_AIO_SERV_RESPONSE_COMPLETE:
3026 scsi_status = error_info->status;
3027 break;
3028 case PQI_AIO_SERV_RESPONSE_FAILURE:
3029 switch (error_info->status) {
3030 case PQI_AIO_STATUS_IO_ABORTED:
3031 scsi_status = SAM_STAT_TASK_ABORTED;
3032 break;
3033 case PQI_AIO_STATUS_UNDERRUN:
3034 scsi_status = SAM_STAT_GOOD;
3035 residual_count = get_unaligned_le32(
3036 &error_info->residual_count);
3037 scsi_set_resid(scmd, residual_count);
3038 xfer_count = scsi_bufflen(scmd) - residual_count;
3039 if (xfer_count < scmd->underflow)
3040 host_byte = DID_SOFT_ERROR;
3041 break;
3042 case PQI_AIO_STATUS_OVERRUN:
3043 scsi_status = SAM_STAT_GOOD;
3044 break;
3045 case PQI_AIO_STATUS_AIO_PATH_DISABLED:
3046 pqi_aio_path_disabled(io_request);
3047 scsi_status = SAM_STAT_GOOD;
3048 io_request->status = -EAGAIN;
3049 break;
3050 case PQI_AIO_STATUS_NO_PATH_TO_DEVICE:
3051 case PQI_AIO_STATUS_INVALID_DEVICE:
376fb880
KB		 3052 if (!io_request->raid_bypass) {
3053 device_offline = true;
3054 pqi_take_device_offline(scmd->device, "AIO");
3055 host_byte = DID_NO_CONNECT;
3056 }
6c223761
KB		 3057 scsi_status = SAM_STAT_CHECK_CONDITION;
3058 break;
3059 case PQI_AIO_STATUS_IO_ERROR:
3060 default:
3061 scsi_status = SAM_STAT_CHECK_CONDITION;
3062 break;
3063 }
3064 break;
3065 case PQI_AIO_SERV_RESPONSE_TMF_COMPLETE:
3066 case PQI_AIO_SERV_RESPONSE_TMF_SUCCEEDED:
3067 scsi_status = SAM_STAT_GOOD;
3068 break;
3069 case PQI_AIO_SERV_RESPONSE_TMF_REJECTED:
3070 case PQI_AIO_SERV_RESPONSE_TMF_INCORRECT_LUN:
3071 default:
3072 scsi_status = SAM_STAT_CHECK_CONDITION;
3073 break;
3074 }
3075
3076 if (error_info->data_present) {
3077 sense_data_length =
3078 get_unaligned_le16(&error_info->data_length);
3079 if (sense_data_length) {
3080 if (sense_data_length > sizeof(error_info->data))
3081 sense_data_length = sizeof(error_info->data);
3082 if (sense_data_length > SCSI_SENSE_BUFFERSIZE)
3083 sense_data_length = SCSI_SENSE_BUFFERSIZE;
3084 memcpy(scmd->sense_buffer, error_info->data,
3085 sense_data_length);
3086 }
3087 }
3088
3089 if (device_offline && sense_data_length == 0)
3090 scsi_build_sense_buffer(0, scmd->sense_buffer, HARDWARE_ERROR,
3091 0x3e, 0x1);
3092
3093 scmd->result = scsi_status;
3094 set_host_byte(scmd, host_byte);
3095}
3096
3097static void pqi_process_io_error(unsigned int iu_type,
3098 struct pqi_io_request *io_request)
3099{
3100 switch (iu_type) {
3101 case PQI_RESPONSE_IU_RAID_PATH_IO_ERROR:
3102 pqi_process_raid_io_error(io_request);
3103 break;
3104 case PQI_RESPONSE_IU_AIO_PATH_IO_ERROR:
3105 pqi_process_aio_io_error(io_request);
3106 break;
3107 }
3108}
3109
18ff5f08 3110static int pqi_interpret_task_management_response(struct pqi_ctrl_info *ctrl_info,
6c223761
KB		 3111 struct pqi_task_management_response *response)
3112{
3113 int rc;
3114
3115 switch (response->response_code) {
b17f0486
KB		 3116 case SOP_TMF_COMPLETE:
3117 case SOP_TMF_FUNCTION_SUCCEEDED:
6c223761
KB		 3118 rc = 0;
3119 break;
3406384b
MR		 3120 case SOP_TMF_REJECTED:
3121 rc = -EAGAIN;
3122 break;
6c223761
KB		 3123 default:
3124 rc = -EIO;
3125 break;
3126 }
3127
18ff5f08
KB		 3128 if (rc)
3129 dev_err(&ctrl_info->pci_dev->dev,
3130 "Task Management Function error: %d (response code: %u)\n", rc, response->response_code);
3131
6c223761
KB		 3132 return rc;
3133}
3134
9e68cccc 3135static inline void pqi_invalid_response(struct pqi_ctrl_info *ctrl_info)
6c223761 3136{
9e68cccc
KB		 3137 pqi_take_ctrl_offline(ctrl_info);
3138}
3139
3140static int pqi_process_io_intr(struct pqi_ctrl_info *ctrl_info, struct pqi_queue_group *queue_group)
6c223761 3141{
9e68cccc 3142 int num_responses;
6c223761
KB		 3143 pqi_index_t oq_pi;
3144 pqi_index_t oq_ci;
3145 struct pqi_io_request *io_request;
3146 struct pqi_io_response *response;
3147 u16 request_id;
3148
3149 num_responses = 0;
3150 oq_ci = queue_group->oq_ci_copy;
3151
3152 while (1) {
dac12fbc 3153 oq_pi = readl(queue_group->oq_pi);
9e68cccc
KB		 3154 if (oq_pi >= ctrl_info->num_elements_per_oq) {
3155 pqi_invalid_response(ctrl_info);
3156 dev_err(&ctrl_info->pci_dev->dev,
3157 "I/O interrupt: producer index (%u) out of range (0-%u): consumer index: %u\n",
3158 oq_pi, ctrl_info->num_elements_per_oq - 1, oq_ci);
3159 return -1;
3160 }
6c223761
KB		 3161 if (oq_pi == oq_ci)
3162 break;
3163
3164 num_responses++;
3165 response = queue_group->oq_element_array +
3166 (oq_ci * PQI_OPERATIONAL_OQ_ELEMENT_LENGTH);
3167
3168 request_id = get_unaligned_le16(&response->request_id);
9e68cccc
KB		 3169 if (request_id >= ctrl_info->max_io_slots) {
3170 pqi_invalid_response(ctrl_info);
3171 dev_err(&ctrl_info->pci_dev->dev,
3172 "request ID in response (%u) out of range (0-%u): producer index: %u consumer index: %u\n",
3173 request_id, ctrl_info->max_io_slots - 1, oq_pi, oq_ci);
3174 return -1;
3175 }
6c223761
KB		 3176
3177 io_request = &ctrl_info->io_request_pool[request_id];
9e68cccc
KB		 3178 if (atomic_read(&io_request->refcount) == 0) {
3179 pqi_invalid_response(ctrl_info);
3180 dev_err(&ctrl_info->pci_dev->dev,
3181 "request ID in response (%u) does not match an outstanding I/O request: producer index: %u consumer index: %u\n",
3182 request_id, oq_pi, oq_ci);
3183 return -1;
3184 }
6c223761
KB		 3185
3186 switch (response->header.iu_type) {
3187 case PQI_RESPONSE_IU_RAID_PATH_IO_SUCCESS:
3188 case PQI_RESPONSE_IU_AIO_PATH_IO_SUCCESS:
2ba55c98
KB		 3189 if (io_request->scmd)
3190 io_request->scmd->result = 0;
df561f66 3191 fallthrough;
6c223761
KB		 3192 case PQI_RESPONSE_IU_GENERAL_MANAGEMENT:
3193 break;
b212c251
KB		 3194 case PQI_RESPONSE_IU_VENDOR_GENERAL:
3195 io_request->status =
3196 get_unaligned_le16(
583891c9 3197 &((struct pqi_vendor_general_response *)response)->status);
b212c251 3198 break;
6c223761 3199 case PQI_RESPONSE_IU_TASK_MANAGEMENT:
18ff5f08
KB		 3200 io_request->status = pqi_interpret_task_management_response(ctrl_info,
3201 (void *)response);
6c223761
KB		 3202 break;
3203 case PQI_RESPONSE_IU_AIO_PATH_DISABLED:
3204 pqi_aio_path_disabled(io_request);
3205 io_request->status = -EAGAIN;
3206 break;
3207 case PQI_RESPONSE_IU_RAID_PATH_IO_ERROR:
3208 case PQI_RESPONSE_IU_AIO_PATH_IO_ERROR:
3209 io_request->error_info = ctrl_info->error_buffer +
3210 (get_unaligned_le16(&response->error_index) *
3211 PQI_ERROR_BUFFER_ELEMENT_LENGTH);
9e68cccc 3212 pqi_process_io_error(response->header.iu_type, io_request);
6c223761
KB		 3213 break;
3214 default:
9e68cccc 3215 pqi_invalid_response(ctrl_info);
6c223761 3216 dev_err(&ctrl_info->pci_dev->dev,
9e68cccc
KB		 3217 "unexpected IU type: 0x%x: producer index: %u consumer index: %u\n",
3218 response->header.iu_type, oq_pi, oq_ci);
3219 return -1;
6c223761
KB		 3220 }
3221
9e68cccc 3222 io_request->io_complete_callback(io_request, io_request->context);
6c223761
KB		 3223
3224 /*
3225 * Note that the I/O request structure CANNOT BE TOUCHED after
3226 * returning from the I/O completion callback!
3227 */
6c223761
KB		 3228 oq_ci = (oq_ci + 1) % ctrl_info->num_elements_per_oq;
3229 }
3230
3231 if (num_responses) {
3232 queue_group->oq_ci_copy = oq_ci;
3233 writel(oq_ci, queue_group->oq_ci);
3234 }
3235
3236 return num_responses;
3237}
3238
3239static inline unsigned int pqi_num_elements_free(unsigned int pi,
df7a1fcf 3240 unsigned int ci, unsigned int elements_in_queue)
6c223761
KB		 3241{
3242 unsigned int num_elements_used;
3243
3244 if (pi >= ci)
3245 num_elements_used = pi - ci;
3246 else
3247 num_elements_used = elements_in_queue - ci + pi;
3248
3249 return elements_in_queue - num_elements_used - 1;
3250}
3251
98f87667 3252static void pqi_send_event_ack(struct pqi_ctrl_info *ctrl_info,
6c223761
KB		 3253 struct pqi_event_acknowledge_request *iu, size_t iu_length)
3254{
3255 pqi_index_t iq_pi;
3256 pqi_index_t iq_ci;
3257 unsigned long flags;
3258 void *next_element;
6c223761
KB		 3259 struct pqi_queue_group *queue_group;
3260
3261 queue_group = &ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP];
3262 put_unaligned_le16(queue_group->oq_id, &iu->header.response_queue_id);
3263
6c223761
KB		 3264 while (1) {
3265 spin_lock_irqsave(&queue_group->submit_lock[RAID_PATH], flags);
3266
3267 iq_pi = queue_group->iq_pi_copy[RAID_PATH];
dac12fbc 3268 iq_ci = readl(queue_group->iq_ci[RAID_PATH]);
6c223761
KB		 3269
3270 if (pqi_num_elements_free(iq_pi, iq_ci,
3271 ctrl_info->num_elements_per_iq))
3272 break;
3273
3274 spin_unlock_irqrestore(
3275 &queue_group->submit_lock[RAID_PATH], flags);
3276
98f87667 3277 if (pqi_ctrl_offline(ctrl_info))
6c223761 3278 return;
6c223761
KB		 3279 }
3280
3281 next_element = queue_group->iq_element_array[RAID_PATH] +
3282 (iq_pi * PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
3283
3284 memcpy(next_element, iu, iu_length);
3285
3286 iq_pi = (iq_pi + 1) % ctrl_info->num_elements_per_iq;
6c223761
KB		 3287 queue_group->iq_pi_copy[RAID_PATH] = iq_pi;
3288
3289 /*
3290 * This write notifies the controller that an IU is available to be
3291 * processed.
3292 */
3293 writel(iq_pi, queue_group->iq_pi[RAID_PATH]);
3294
3295 spin_unlock_irqrestore(&queue_group->submit_lock[RAID_PATH], flags);
6c223761
KB		 3296}
3297
3298static void pqi_acknowledge_event(struct pqi_ctrl_info *ctrl_info,
3299 struct pqi_event *event)
3300{
3301 struct pqi_event_acknowledge_request request;
3302
3303 memset(&request, 0, sizeof(request));
3304
3305 request.header.iu_type = PQI_REQUEST_IU_ACKNOWLEDGE_VENDOR_EVENT;
3306 put_unaligned_le16(sizeof(request) - PQI_REQUEST_HEADER_LENGTH,
3307 &request.header.iu_length);
3308 request.event_type = event->event_type;
06b41e0d
KB		 3309 put_unaligned_le16(event->event_id, &request.event_id);
3310 put_unaligned_le32(event->additional_event_id, &request.additional_event_id);
6c223761 3311
98f87667 3312 pqi_send_event_ack(ctrl_info, &request, sizeof(request));
6c223761
KB		 3313}
3314
4fd22c13
MR		 3315#define PQI_SOFT_RESET_STATUS_TIMEOUT_SECS 30
3316#define PQI_SOFT_RESET_STATUS_POLL_INTERVAL_SECS 1
3317
3318static enum pqi_soft_reset_status pqi_poll_for_soft_reset_status(
3319 struct pqi_ctrl_info *ctrl_info)
6c223761 3320{
4fd22c13 3321 u8 status;
583891c9 3322 unsigned long timeout;
6c223761 3323
4fd22c13 3324 timeout = (PQI_SOFT_RESET_STATUS_TIMEOUT_SECS * PQI_HZ) + jiffies;
6c223761 3325
4fd22c13
MR		 3326 while (1) {
3327 status = pqi_read_soft_reset_status(ctrl_info);
3328 if (status & PQI_SOFT_RESET_INITIATE)
3329 return RESET_INITIATE_DRIVER;
3330
3331 if (status & PQI_SOFT_RESET_ABORT)
3332 return RESET_ABORT;
3333
4ccc354b
KB		 3334 if (!sis_is_firmware_running(ctrl_info))
3335 return RESET_NORESPONSE;
3336
4fd22c13 3337 if (time_after(jiffies, timeout)) {
4ccc354b 3338 dev_warn(&ctrl_info->pci_dev->dev,
4fd22c13
MR		 3339 "timed out waiting for soft reset status\n");
3340 return RESET_TIMEDOUT;
3341 }
3342
4fd22c13
MR		 3343 ssleep(PQI_SOFT_RESET_STATUS_POLL_INTERVAL_SECS);
3344 }
3345}
3346
4ccc354b 3347static void pqi_process_soft_reset(struct pqi_ctrl_info *ctrl_info)
4fd22c13
MR		 3348{
3349 int rc;
2790cd4d 3350 unsigned int delay_secs;
4ccc354b
KB		 3351 enum pqi_soft_reset_status reset_status;
3352
3353 if (ctrl_info->soft_reset_handshake_supported)
3354 reset_status = pqi_poll_for_soft_reset_status(ctrl_info);
3355 else
3356 reset_status = RESET_INITIATE_FIRMWARE;
4fd22c13 3357
2790cd4d 3358 delay_secs = PQI_POST_RESET_DELAY_SECS;
4fd22c13
MR		 3359
3360 switch (reset_status) {
4fd22c13 3361 case RESET_TIMEDOUT:
2790cd4d 3362 delay_secs = PQI_POST_OFA_RESET_DELAY_UPON_TIMEOUT_SECS;
4ccc354b
KB		 3363 fallthrough;
3364 case RESET_INITIATE_DRIVER:
4fd22c13 3365 dev_info(&ctrl_info->pci_dev->dev,
4ccc354b 3366 "Online Firmware Activation: resetting controller\n");
4fd22c13 3367 sis_soft_reset(ctrl_info);
df561f66 3368 fallthrough;
4fd22c13 3369 case RESET_INITIATE_FIRMWARE:
4ccc354b
KB		 3370 ctrl_info->pqi_mode_enabled = false;
3371 pqi_save_ctrl_mode(ctrl_info, SIS_MODE);
2790cd4d 3372 rc = pqi_ofa_ctrl_restart(ctrl_info, delay_secs);
4fd22c13 3373 pqi_ofa_free_host_buffer(ctrl_info);
4ccc354b 3374 pqi_ctrl_ofa_done(ctrl_info);
4fd22c13 3375 dev_info(&ctrl_info->pci_dev->dev,
4ccc354b
KB		 3376 "Online Firmware Activation: %s\n",
3377 rc == 0 ? "SUCCESS" : "FAILED");
4fd22c13
MR		 3378 break;
3379 case RESET_ABORT:
4fd22c13 3380 dev_info(&ctrl_info->pci_dev->dev,
4ccc354b
KB		 3381 "Online Firmware Activation ABORTED\n");
3382 if (ctrl_info->soft_reset_handshake_supported)
3383 pqi_clear_soft_reset_status(ctrl_info);
3384 pqi_ofa_free_host_buffer(ctrl_info);
3385 pqi_ctrl_ofa_done(ctrl_info);
3386 pqi_ofa_ctrl_unquiesce(ctrl_info);
4fd22c13
MR		 3387 break;
3388 case RESET_NORESPONSE:
4ccc354b
KB		 3389 fallthrough;
3390 default:
3391 dev_err(&ctrl_info->pci_dev->dev,
3392 "unexpected Online Firmware Activation reset status: 0x%x\n",
3393 reset_status);
4fd22c13 3394 pqi_ofa_free_host_buffer(ctrl_info);
4ccc354b
KB		 3395 pqi_ctrl_ofa_done(ctrl_info);
3396 pqi_ofa_ctrl_unquiesce(ctrl_info);
4fd22c13
MR		 3397 pqi_take_ctrl_offline(ctrl_info);
3398 break;
3399 }
3400}
3401
2790cd4d 3402static void pqi_ofa_memory_alloc_worker(struct work_struct *work)
4fd22c13 3403{
2790cd4d 3404 struct pqi_ctrl_info *ctrl_info;
4fd22c13 3405
2790cd4d 3406 ctrl_info = container_of(work, struct pqi_ctrl_info, ofa_memory_alloc_work);
4fd22c13 3407
2790cd4d
KB		 3408 pqi_ctrl_ofa_start(ctrl_info);
3409 pqi_ofa_setup_host_buffer(ctrl_info);
3410 pqi_ofa_host_memory_update(ctrl_info);
3411}
4fd22c13 3412
2790cd4d
KB		 3413static void pqi_ofa_quiesce_worker(struct work_struct *work)
3414{
3415 struct pqi_ctrl_info *ctrl_info;
3416 struct pqi_event *event;
4fd22c13 3417
2790cd4d
KB		 3418 ctrl_info = container_of(work, struct pqi_ctrl_info, ofa_quiesce_work);
3419
3420 event = &ctrl_info->events[pqi_event_type_to_event_index(PQI_EVENT_TYPE_OFA)];
3421
3422 pqi_ofa_ctrl_quiesce(ctrl_info);
3423 pqi_acknowledge_event(ctrl_info, event);
3424 pqi_process_soft_reset(ctrl_info);
3425}
4fd22c13 3426
2790cd4d
KB		 3427static bool pqi_ofa_process_event(struct pqi_ctrl_info *ctrl_info,
3428 struct pqi_event *event)
3429{
3430 bool ack_event;
3431
3432 ack_event = true;
3433
3434 switch (event->event_id) {
3435 case PQI_EVENT_OFA_MEMORY_ALLOCATION:
4fd22c13 3436 dev_info(&ctrl_info->pci_dev->dev,
2790cd4d
KB		 3437 "received Online Firmware Activation memory allocation request\n");
3438 schedule_work(&ctrl_info->ofa_memory_alloc_work);
3439 break;
3440 case PQI_EVENT_OFA_QUIESCE:
4fd22c13 3441 dev_info(&ctrl_info->pci_dev->dev,
2790cd4d
KB		 3442 "received Online Firmware Activation quiesce request\n");
3443 schedule_work(&ctrl_info->ofa_quiesce_work);
3444 ack_event = false;
3445 break;
3446 case PQI_EVENT_OFA_CANCELED:
4fd22c13 3447 dev_info(&ctrl_info->pci_dev->dev,
2790cd4d
KB		 3448 "received Online Firmware Activation cancel request: reason: %u\n",
3449 ctrl_info->ofa_cancel_reason);
3450 pqi_ofa_free_host_buffer(ctrl_info);
3451 pqi_ctrl_ofa_done(ctrl_info);
3452 break;
3453 default:
3454 dev_err(&ctrl_info->pci_dev->dev,
3455 "received unknown Online Firmware Activation request: event ID: %u\n",
3456 event->event_id);
3457 break;
4fd22c13
MR		 3458 }
3459
2790cd4d 3460 return ack_event;
4fd22c13
MR		 3461}
3462
6c223761
KB		 3463static void pqi_event_worker(struct work_struct *work)
3464{
3465 unsigned int i;
2790cd4d 3466 bool rescan_needed;
6c223761 3467 struct pqi_ctrl_info *ctrl_info;
6a50d6ad 3468 struct pqi_event *event;
2790cd4d 3469 bool ack_event;
6c223761
KB		 3470
3471 ctrl_info = container_of(work, struct pqi_ctrl_info, event_work);
3472
7561a7e4 3473 pqi_ctrl_busy(ctrl_info);
ae0c189d 3474 pqi_wait_if_ctrl_blocked(ctrl_info);
5f310425
KB		 3475 if (pqi_ctrl_offline(ctrl_info))
3476 goto out;
3477
2790cd4d 3478 rescan_needed = false;
6a50d6ad 3479 event = ctrl_info->events;
6c223761 3480 for (i = 0; i < PQI_NUM_SUPPORTED_EVENTS; i++) {
6a50d6ad
KB		 3481 if (event->pending) {
3482 event->pending = false;
4fd22c13 3483 if (event->event_type == PQI_EVENT_TYPE_OFA) {
2790cd4d
KB		 3484 ack_event = pqi_ofa_process_event(ctrl_info, event);
3485 } else {
3486 ack_event = true;
3487 rescan_needed = true;
4fd22c13 3488 }
2790cd4d
KB		 3489 if (ack_event)
3490 pqi_acknowledge_event(ctrl_info, event);
6c223761 3491 }
6a50d6ad 3492 event++;
6c223761
KB		 3493 }
3494
2790cd4d
KB		 3495 if (rescan_needed)
3496 pqi_schedule_rescan_worker_delayed(ctrl_info);
3497
5f310425 3498out:
7561a7e4 3499 pqi_ctrl_unbusy(ctrl_info);
6c223761
KB		 3500}
3501
4fd22c13 3502#define PQI_HEARTBEAT_TIMER_INTERVAL (10 * PQI_HZ)
6c223761 3503
74a0f573 3504static void pqi_heartbeat_timer_handler(struct timer_list *t)
6c223761
KB		 3505{
3506 int num_interrupts;
98f87667 3507 u32 heartbeat_count;
583891c9 3508 struct pqi_ctrl_info *ctrl_info = from_timer(ctrl_info, t, heartbeat_timer);
6c223761 3509
98f87667
KB		 3510 pqi_check_ctrl_health(ctrl_info);
3511 if (pqi_ctrl_offline(ctrl_info))
061ef06a
KB		 3512 return;
3513
6c223761 3514 num_interrupts = atomic_read(&ctrl_info->num_interrupts);
98f87667 3515 heartbeat_count = pqi_read_heartbeat_counter(ctrl_info);
6c223761
KB		 3516
3517 if (num_interrupts == ctrl_info->previous_num_interrupts) {
98f87667
KB		 3518 if (heartbeat_count == ctrl_info->previous_heartbeat_count) {
3519 dev_err(&ctrl_info->pci_dev->dev,
3520 "no heartbeat detected - last heartbeat count: %u\n",
3521 heartbeat_count);
6c223761
KB		 3522 pqi_take_ctrl_offline(ctrl_info);
3523 return;
3524 }
6c223761 3525 } else {
98f87667 3526 ctrl_info->previous_num_interrupts = num_interrupts;
6c223761
KB		 3527 }
3528
98f87667 3529 ctrl_info->previous_heartbeat_count = heartbeat_count;
6c223761
KB		 3530 mod_timer(&ctrl_info->heartbeat_timer,
3531 jiffies + PQI_HEARTBEAT_TIMER_INTERVAL);
3532}
3533
3534static void pqi_start_heartbeat_timer(struct pqi_ctrl_info *ctrl_info)
3535{
98f87667
KB		 3536 if (!ctrl_info->heartbeat_counter)
3537 return;
3538
6c223761
KB		 3539 ctrl_info->previous_num_interrupts =
3540 atomic_read(&ctrl_info->num_interrupts);
98f87667
KB		 3541 ctrl_info->previous_heartbeat_count =
3542 pqi_read_heartbeat_counter(ctrl_info);
6c223761 3543
6c223761
KB		 3544 ctrl_info->heartbeat_timer.expires =
3545 jiffies + PQI_HEARTBEAT_TIMER_INTERVAL;
061ef06a 3546 add_timer(&ctrl_info->heartbeat_timer);
6c223761
KB		 3547}
3548
3549static inline void pqi_stop_heartbeat_timer(struct pqi_ctrl_info *ctrl_info)
3550{
98f87667 3551 del_timer_sync(&ctrl_info->heartbeat_timer);
6c223761
KB		 3552}
3553
2790cd4d
KB		 3554static void pqi_ofa_capture_event_payload(struct pqi_ctrl_info *ctrl_info,
3555 struct pqi_event *event, struct pqi_event_response *response)
4fd22c13 3556{
2790cd4d
KB		 3557 switch (event->event_id) {
3558 case PQI_EVENT_OFA_MEMORY_ALLOCATION:
3559 ctrl_info->ofa_bytes_requested =
3560 get_unaligned_le32(&response->data.ofa_memory_allocation.bytes_requested);
3561 break;
3562 case PQI_EVENT_OFA_CANCELED:
3563 ctrl_info->ofa_cancel_reason =
3564 get_unaligned_le16(&response->data.ofa_cancelled.reason);
3565 break;
4fd22c13
MR		 3566 }
3567}
3568
9e68cccc 3569static int pqi_process_event_intr(struct pqi_ctrl_info *ctrl_info)
6c223761 3570{
9e68cccc 3571 int num_events;
6c223761
KB		 3572 pqi_index_t oq_pi;
3573 pqi_index_t oq_ci;
3574 struct pqi_event_queue *event_queue;
3575 struct pqi_event_response *response;
6a50d6ad 3576 struct pqi_event *event;
6c223761
KB		 3577 int event_index;
3578
3579 event_queue = &ctrl_info->event_queue;
3580 num_events = 0;
6c223761
KB		 3581 oq_ci = event_queue->oq_ci_copy;
3582
3583 while (1) {
dac12fbc 3584 oq_pi = readl(event_queue->oq_pi);
9e68cccc
KB		 3585 if (oq_pi >= PQI_NUM_EVENT_QUEUE_ELEMENTS) {
3586 pqi_invalid_response(ctrl_info);
3587 dev_err(&ctrl_info->pci_dev->dev,
3588 "event interrupt: producer index (%u) out of range (0-%u): consumer index: %u\n",
3589 oq_pi, PQI_NUM_EVENT_QUEUE_ELEMENTS - 1, oq_ci);
3590 return -1;
3591 }
3592
6c223761
KB		 3593 if (oq_pi == oq_ci)
3594 break;
3595
3596 num_events++;
9e68cccc 3597 response = event_queue->oq_element_array + (oq_ci * PQI_EVENT_OQ_ELEMENT_LENGTH);
6c223761 3598
583891c9 3599 event_index = pqi_event_type_to_event_index(response->event_type);
6c223761 3600
9e68cccc
KB		 3601 if (event_index >= 0 && response->request_acknowledge) {
3602 event = &ctrl_info->events[event_index];
3603 event->pending = true;
3604 event->event_type = response->event_type;
06b41e0d
KB		 3605 event->event_id = get_unaligned_le16(&response->event_id);
3606 event->additional_event_id =
3607 get_unaligned_le32(&response->additional_event_id);
9e68cccc 3608 if (event->event_type == PQI_EVENT_TYPE_OFA)
2790cd4d 3609 pqi_ofa_capture_event_payload(ctrl_info, event, response);
6c223761
KB		 3610 }
3611
3612 oq_ci = (oq_ci + 1) % PQI_NUM_EVENT_QUEUE_ELEMENTS;
3613 }
3614
3615 if (num_events) {
3616 event_queue->oq_ci_copy = oq_ci;
3617 writel(oq_ci, event_queue->oq_ci);
98f87667 3618 schedule_work(&ctrl_info->event_work);
6c223761
KB		 3619 }
3620
3621 return num_events;
3622}
3623
061ef06a
KB		 3624#define PQI_LEGACY_INTX_MASK 0x1
3625
583891c9 3626static inline void pqi_configure_legacy_intx(struct pqi_ctrl_info *ctrl_info, bool enable_intx)
061ef06a
KB		 3627{
3628 u32 intx_mask;
3629 struct pqi_device_registers __iomem *pqi_registers;
3630 volatile void __iomem *register_addr;
3631
3632 pqi_registers = ctrl_info->pqi_registers;
3633
3634 if (enable_intx)
3635 register_addr = &pqi_registers->legacy_intx_mask_clear;
3636 else
3637 register_addr = &pqi_registers->legacy_intx_mask_set;
3638
3639 intx_mask = readl(register_addr);
3640 intx_mask |= PQI_LEGACY_INTX_MASK;
3641 writel(intx_mask, register_addr);
3642}
3643
3644static void pqi_change_irq_mode(struct pqi_ctrl_info *ctrl_info,
3645 enum pqi_irq_mode new_mode)
3646{
3647 switch (ctrl_info->irq_mode) {
3648 case IRQ_MODE_MSIX:
3649 switch (new_mode) {
3650 case IRQ_MODE_MSIX:
3651 break;
3652 case IRQ_MODE_INTX:
3653 pqi_configure_legacy_intx(ctrl_info, true);
061ef06a
KB		 3654 sis_enable_intx(ctrl_info);
3655 break;
3656 case IRQ_MODE_NONE:
061ef06a
KB		 3657 break;
3658 }
3659 break;
3660 case IRQ_MODE_INTX:
3661 switch (new_mode) {
3662 case IRQ_MODE_MSIX:
3663 pqi_configure_legacy_intx(ctrl_info, false);
061ef06a
KB		 3664 sis_enable_msix(ctrl_info);
3665 break;
3666 case IRQ_MODE_INTX:
3667 break;
3668 case IRQ_MODE_NONE:
3669 pqi_configure_legacy_intx(ctrl_info, false);
061ef06a
KB		 3670 break;
3671 }
3672 break;
3673 case IRQ_MODE_NONE:
3674 switch (new_mode) {
3675 case IRQ_MODE_MSIX:
3676 sis_enable_msix(ctrl_info);
3677 break;
3678 case IRQ_MODE_INTX:
3679 pqi_configure_legacy_intx(ctrl_info, true);
3680 sis_enable_intx(ctrl_info);
3681 break;
3682 case IRQ_MODE_NONE:
3683 break;
3684 }
3685 break;
3686 }
3687
3688 ctrl_info->irq_mode = new_mode;
3689}
3690
3691#define PQI_LEGACY_INTX_PENDING 0x1
3692
3693static inline bool pqi_is_valid_irq(struct pqi_ctrl_info *ctrl_info)
3694{
3695 bool valid_irq;
3696 u32 intx_status;
3697
3698 switch (ctrl_info->irq_mode) {
3699 case IRQ_MODE_MSIX:
3700 valid_irq = true;
3701 break;
3702 case IRQ_MODE_INTX:
583891c9 3703 intx_status = readl(&ctrl_info->pqi_registers->legacy_intx_status);
061ef06a
KB		 3704 if (intx_status & PQI_LEGACY_INTX_PENDING)
3705 valid_irq = true;
3706 else
3707 valid_irq = false;
3708 break;
3709 case IRQ_MODE_NONE:
3710 default:
3711 valid_irq = false;
3712 break;
3713 }
3714
3715 return valid_irq;
3716}
3717
6c223761
KB		 3718static irqreturn_t pqi_irq_handler(int irq, void *data)
3719{
3720 struct pqi_ctrl_info *ctrl_info;
3721 struct pqi_queue_group *queue_group;
9e68cccc
KB		 3722 int num_io_responses_handled;
3723 int num_events_handled;
6c223761
KB		 3724
3725 queue_group = data;
3726 ctrl_info = queue_group->ctrl_info;
3727
061ef06a 3728 if (!pqi_is_valid_irq(ctrl_info))
6c223761
KB		 3729 return IRQ_NONE;
3730
9e68cccc
KB		 3731 num_io_responses_handled = pqi_process_io_intr(ctrl_info, queue_group);
3732 if (num_io_responses_handled < 0)
3733 goto out;
6c223761 3734
9e68cccc
KB		 3735 if (irq == ctrl_info->event_irq) {
3736 num_events_handled = pqi_process_event_intr(ctrl_info);
3737 if (num_events_handled < 0)
3738 goto out;
3739 } else {
3740 num_events_handled = 0;
3741 }
6c223761 3742
9e68cccc 3743 if (num_io_responses_handled + num_events_handled > 0)
6c223761
KB		 3744 atomic_inc(&ctrl_info->num_interrupts);
3745
3746 pqi_start_io(ctrl_info, queue_group, RAID_PATH, NULL);
3747 pqi_start_io(ctrl_info, queue_group, AIO_PATH, NULL);
3748
9e68cccc 3749out:
6c223761
KB		 3750 return IRQ_HANDLED;
3751}
3752
3753static int pqi_request_irqs(struct pqi_ctrl_info *ctrl_info)
3754{
d91d7820 3755 struct pci_dev *pci_dev = ctrl_info->pci_dev;
6c223761
KB		 3756 int i;
3757 int rc;
3758
d91d7820 3759 ctrl_info->event_irq = pci_irq_vector(pci_dev, 0);
6c223761
KB		 3760
3761 for (i = 0; i < ctrl_info->num_msix_vectors_enabled; i++) {
d91d7820 3762 rc = request_irq(pci_irq_vector(pci_dev, i), pqi_irq_handler, 0,
52198226 3763 DRIVER_NAME_SHORT, &ctrl_info->queue_groups[i]);
6c223761 3764 if (rc) {
d91d7820 3765 dev_err(&pci_dev->dev,
6c223761 3766 "irq %u init failed with error %d\n",
d91d7820 3767 pci_irq_vector(pci_dev, i), rc);
6c223761
KB		 3768 return rc;
3769 }
3770 ctrl_info->num_msix_vectors_initialized++;
3771 }
3772
3773 return 0;
3774}
3775
98bf061b
KB		 3776static void pqi_free_irqs(struct pqi_ctrl_info *ctrl_info)
3777{
3778 int i;
3779
3780 for (i = 0; i < ctrl_info->num_msix_vectors_initialized; i++)
3781 free_irq(pci_irq_vector(ctrl_info->pci_dev, i),
3782 &ctrl_info->queue_groups[i]);
3783
3784 ctrl_info->num_msix_vectors_initialized = 0;
3785}
3786
6c223761
KB		 3787static int pqi_enable_msix_interrupts(struct pqi_ctrl_info *ctrl_info)
3788{
98bf061b 3789 int num_vectors_enabled;
6c223761 3790
98bf061b 3791 num_vectors_enabled = pci_alloc_irq_vectors(ctrl_info->pci_dev,
52198226
CH		 3792 PQI_MIN_MSIX_VECTORS, ctrl_info->num_queue_groups,
3793 PCI_IRQ_MSIX | PCI_IRQ_AFFINITY);
98bf061b 3794 if (num_vectors_enabled < 0) {
6c223761 3795 dev_err(&ctrl_info->pci_dev->dev,
98bf061b
KB		 3796 "MSI-X init failed with error %d\n",
3797 num_vectors_enabled);
3798 return num_vectors_enabled;
6c223761
KB		 3799 }
3800
98bf061b 3801 ctrl_info->num_msix_vectors_enabled = num_vectors_enabled;
061ef06a 3802 ctrl_info->irq_mode = IRQ_MODE_MSIX;
6c223761
KB		 3803 return 0;
3804}
3805
98bf061b
KB		 3806static void pqi_disable_msix_interrupts(struct pqi_ctrl_info *ctrl_info)
3807{
3808 if (ctrl_info->num_msix_vectors_enabled) {
3809 pci_free_irq_vectors(ctrl_info->pci_dev);
3810 ctrl_info->num_msix_vectors_enabled = 0;
3811 }
3812}
3813
6c223761
KB		 3814static int pqi_alloc_operational_queues(struct pqi_ctrl_info *ctrl_info)
3815{
3816 unsigned int i;
3817 size_t alloc_length;
3818 size_t element_array_length_per_iq;
3819 size_t element_array_length_per_oq;
3820 void *element_array;
dac12fbc 3821 void __iomem *next_queue_index;
6c223761
KB		 3822 void *aligned_pointer;
3823 unsigned int num_inbound_queues;
3824 unsigned int num_outbound_queues;
3825 unsigned int num_queue_indexes;
3826 struct pqi_queue_group *queue_group;
3827
3828 element_array_length_per_iq =
3829 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH *
3830 ctrl_info->num_elements_per_iq;
3831 element_array_length_per_oq =
3832 PQI_OPERATIONAL_OQ_ELEMENT_LENGTH *
3833 ctrl_info->num_elements_per_oq;
3834 num_inbound_queues = ctrl_info->num_queue_groups * 2;
3835 num_outbound_queues = ctrl_info->num_queue_groups;
3836 num_queue_indexes = (ctrl_info->num_queue_groups * 3) + 1;
3837
3838 aligned_pointer = NULL;
3839
3840 for (i = 0; i < num_inbound_queues; i++) {
3841 aligned_pointer = PTR_ALIGN(aligned_pointer,
3842 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
3843 aligned_pointer += element_array_length_per_iq;
3844 }
3845
3846 for (i = 0; i < num_outbound_queues; i++) {
3847 aligned_pointer = PTR_ALIGN(aligned_pointer,
3848 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
3849 aligned_pointer += element_array_length_per_oq;
3850 }
3851
3852 aligned_pointer = PTR_ALIGN(aligned_pointer,
3853 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
3854 aligned_pointer += PQI_NUM_EVENT_QUEUE_ELEMENTS *
3855 PQI_EVENT_OQ_ELEMENT_LENGTH;
3856
3857 for (i = 0; i < num_queue_indexes; i++) {
3858 aligned_pointer = PTR_ALIGN(aligned_pointer,
3859 PQI_OPERATIONAL_INDEX_ALIGNMENT);
3860 aligned_pointer += sizeof(pqi_index_t);
3861 }
3862
3863 alloc_length = (size_t)aligned_pointer +
3864 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT;
3865
e1d213bd
KB		 3866 alloc_length += PQI_EXTRA_SGL_MEMORY;
3867
6c223761 3868 ctrl_info->queue_memory_base =
750afb08
LC		 3869 dma_alloc_coherent(&ctrl_info->pci_dev->dev, alloc_length,
3870 &ctrl_info->queue_memory_base_dma_handle,
3871 GFP_KERNEL);
6c223761 3872
d87d5474 3873 if (!ctrl_info->queue_memory_base)
6c223761 3874 return -ENOMEM;
6c223761
KB		 3875
3876 ctrl_info->queue_memory_length = alloc_length;
3877
3878 element_array = PTR_ALIGN(ctrl_info->queue_memory_base,
3879 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
3880
3881 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
3882 queue_group = &ctrl_info->queue_groups[i];
3883 queue_group->iq_element_array[RAID_PATH] = element_array;
3884 queue_group->iq_element_array_bus_addr[RAID_PATH] =
3885 ctrl_info->queue_memory_base_dma_handle +
3886 (element_array - ctrl_info->queue_memory_base);
3887 element_array += element_array_length_per_iq;
3888 element_array = PTR_ALIGN(element_array,
3889 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
3890 queue_group->iq_element_array[AIO_PATH] = element_array;
3891 queue_group->iq_element_array_bus_addr[AIO_PATH] =
3892 ctrl_info->queue_memory_base_dma_handle +
3893 (element_array - ctrl_info->queue_memory_base);
3894 element_array += element_array_length_per_iq;
3895 element_array = PTR_ALIGN(element_array,
3896 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
3897 }
3898
3899 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
3900 queue_group = &ctrl_info->queue_groups[i];
3901 queue_group->oq_element_array = element_array;
3902 queue_group->oq_element_array_bus_addr =
3903 ctrl_info->queue_memory_base_dma_handle +
3904 (element_array - ctrl_info->queue_memory_base);
3905 element_array += element_array_length_per_oq;
3906 element_array = PTR_ALIGN(element_array,
3907 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
3908 }
3909
3910 ctrl_info->event_queue.oq_element_array = element_array;
3911 ctrl_info->event_queue.oq_element_array_bus_addr =
3912 ctrl_info->queue_memory_base_dma_handle +
3913 (element_array - ctrl_info->queue_memory_base);
3914 element_array += PQI_NUM_EVENT_QUEUE_ELEMENTS *
3915 PQI_EVENT_OQ_ELEMENT_LENGTH;
3916
dac12fbc 3917 next_queue_index = (void __iomem *)PTR_ALIGN(element_array,
6c223761
KB		 3918 PQI_OPERATIONAL_INDEX_ALIGNMENT);
3919
3920 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
3921 queue_group = &ctrl_info->queue_groups[i];
3922 queue_group->iq_ci[RAID_PATH] = next_queue_index;
3923 queue_group->iq_ci_bus_addr[RAID_PATH] =
3924 ctrl_info->queue_memory_base_dma_handle +
dac12fbc
KB		 3925 (next_queue_index -
3926 (void __iomem *)ctrl_info->queue_memory_base);
6c223761
KB		 3927 next_queue_index += sizeof(pqi_index_t);
3928 next_queue_index = PTR_ALIGN(next_queue_index,
3929 PQI_OPERATIONAL_INDEX_ALIGNMENT);
3930 queue_group->iq_ci[AIO_PATH] = next_queue_index;
3931 queue_group->iq_ci_bus_addr[AIO_PATH] =
3932 ctrl_info->queue_memory_base_dma_handle +
dac12fbc
KB		 3933 (next_queue_index -
3934 (void __iomem *)ctrl_info->queue_memory_base);
6c223761
KB		 3935 next_queue_index += sizeof(pqi_index_t);
3936 next_queue_index = PTR_ALIGN(next_queue_index,
3937 PQI_OPERATIONAL_INDEX_ALIGNMENT);
3938 queue_group->oq_pi = next_queue_index;
3939 queue_group->oq_pi_bus_addr =
3940 ctrl_info->queue_memory_base_dma_handle +
dac12fbc
KB		 3941 (next_queue_index -
3942 (void __iomem *)ctrl_info->queue_memory_base);
6c223761
KB		 3943 next_queue_index += sizeof(pqi_index_t);
3944 next_queue_index = PTR_ALIGN(next_queue_index,
3945 PQI_OPERATIONAL_INDEX_ALIGNMENT);
3946 }
3947
3948 ctrl_info->event_queue.oq_pi = next_queue_index;
3949 ctrl_info->event_queue.oq_pi_bus_addr =
3950 ctrl_info->queue_memory_base_dma_handle +
dac12fbc
KB		 3951 (next_queue_index -
3952 (void __iomem *)ctrl_info->queue_memory_base);
6c223761
KB		 3953
3954 return 0;
3955}
3956
3957static void pqi_init_operational_queues(struct pqi_ctrl_info *ctrl_info)
3958{
3959 unsigned int i;
3960 u16 next_iq_id = PQI_MIN_OPERATIONAL_QUEUE_ID;
3961 u16 next_oq_id = PQI_MIN_OPERATIONAL_QUEUE_ID;
3962
3963 /*
3964 * Initialize the backpointers to the controller structure in
3965 * each operational queue group structure.
3966 */
3967 for (i = 0; i < ctrl_info->num_queue_groups; i++)
3968 ctrl_info->queue_groups[i].ctrl_info = ctrl_info;
3969
3970 /*
3971 * Assign IDs to all operational queues. Note that the IDs
3972 * assigned to operational IQs are independent of the IDs
3973 * assigned to operational OQs.
3974 */
3975 ctrl_info->event_queue.oq_id = next_oq_id++;
3976 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
3977 ctrl_info->queue_groups[i].iq_id[RAID_PATH] = next_iq_id++;
3978 ctrl_info->queue_groups[i].iq_id[AIO_PATH] = next_iq_id++;
3979 ctrl_info->queue_groups[i].oq_id = next_oq_id++;
3980 }
3981
3982 /*
3983 * Assign MSI-X table entry indexes to all queues. Note that the
3984 * interrupt for the event queue is shared with the first queue group.
3985 */
3986 ctrl_info->event_queue.int_msg_num = 0;
3987 for (i = 0; i < ctrl_info->num_queue_groups; i++)
3988 ctrl_info->queue_groups[i].int_msg_num = i;
3989
3990 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
3991 spin_lock_init(&ctrl_info->queue_groups[i].submit_lock[0]);
3992 spin_lock_init(&ctrl_info->queue_groups[i].submit_lock[1]);
3993 INIT_LIST_HEAD(&ctrl_info->queue_groups[i].request_list[0]);
3994 INIT_LIST_HEAD(&ctrl_info->queue_groups[i].request_list[1]);
3995 }
3996}
3997
3998static int pqi_alloc_admin_queues(struct pqi_ctrl_info *ctrl_info)
3999{
4000 size_t alloc_length;
4001 struct pqi_admin_queues_aligned *admin_queues_aligned;
4002 struct pqi_admin_queues *admin_queues;
4003
4004 alloc_length = sizeof(struct pqi_admin_queues_aligned) +
4005 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT;
4006
4007 ctrl_info->admin_queue_memory_base =
750afb08
LC		 4008 dma_alloc_coherent(&ctrl_info->pci_dev->dev, alloc_length,
4009 &ctrl_info->admin_queue_memory_base_dma_handle,
4010 GFP_KERNEL);
6c223761
KB		 4011
4012 if (!ctrl_info->admin_queue_memory_base)
4013 return -ENOMEM;
4014
4015 ctrl_info->admin_queue_memory_length = alloc_length;
4016
4017 admin_queues = &ctrl_info->admin_queues;
4018 admin_queues_aligned = PTR_ALIGN(ctrl_info->admin_queue_memory_base,
4019 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
4020 admin_queues->iq_element_array =
4021 &admin_queues_aligned->iq_element_array;
4022 admin_queues->oq_element_array =
4023 &admin_queues_aligned->oq_element_array;
583891c9
KB		 4024 admin_queues->iq_ci =
4025 (pqi_index_t __iomem *)&admin_queues_aligned->iq_ci;
dac12fbc
KB		 4026 admin_queues->oq_pi =
4027 (pqi_index_t __iomem *)&admin_queues_aligned->oq_pi;
6c223761
KB		 4028
4029 admin_queues->iq_element_array_bus_addr =
4030 ctrl_info->admin_queue_memory_base_dma_handle +
4031 (admin_queues->iq_element_array -
4032 ctrl_info->admin_queue_memory_base);
4033 admin_queues->oq_element_array_bus_addr =
4034 ctrl_info->admin_queue_memory_base_dma_handle +
4035 (admin_queues->oq_element_array -
4036 ctrl_info->admin_queue_memory_base);
4037 admin_queues->iq_ci_bus_addr =
4038 ctrl_info->admin_queue_memory_base_dma_handle +
583891c9
KB		 4039 ((void __iomem *)admin_queues->iq_ci -
4040 (void __iomem *)ctrl_info->admin_queue_memory_base);
6c223761
KB		 4041 admin_queues->oq_pi_bus_addr =
4042 ctrl_info->admin_queue_memory_base_dma_handle +
dac12fbc
KB		 4043 ((void __iomem *)admin_queues->oq_pi -
4044 (void __iomem *)ctrl_info->admin_queue_memory_base);
6c223761
KB		 4045
4046 return 0;
4047}
4048
4fd22c13 4049#define PQI_ADMIN_QUEUE_CREATE_TIMEOUT_JIFFIES PQI_HZ
6c223761
KB		 4050#define PQI_ADMIN_QUEUE_CREATE_POLL_INTERVAL_MSECS 1
4051
4052static int pqi_create_admin_queues(struct pqi_ctrl_info *ctrl_info)
4053{
4054 struct pqi_device_registers __iomem *pqi_registers;
4055 struct pqi_admin_queues *admin_queues;
4056 unsigned long timeout;
4057 u8 status;
4058 u32 reg;
4059
4060 pqi_registers = ctrl_info->pqi_registers;
4061 admin_queues = &ctrl_info->admin_queues;
4062
4063 writeq((u64)admin_queues->iq_element_array_bus_addr,
4064 &pqi_registers->admin_iq_element_array_addr);
4065 writeq((u64)admin_queues->oq_element_array_bus_addr,
4066 &pqi_registers->admin_oq_element_array_addr);
4067 writeq((u64)admin_queues->iq_ci_bus_addr,
4068 &pqi_registers->admin_iq_ci_addr);
4069 writeq((u64)admin_queues->oq_pi_bus_addr,
4070 &pqi_registers->admin_oq_pi_addr);
4071
4072 reg = PQI_ADMIN_IQ_NUM_ELEMENTS |
e655d469 4073 (PQI_ADMIN_OQ_NUM_ELEMENTS << 8) |
6c223761
KB		 4074 (admin_queues->int_msg_num << 16);
4075 writel(reg, &pqi_registers->admin_iq_num_elements);
583891c9 4076
6c223761
KB		 4077 writel(PQI_CREATE_ADMIN_QUEUE_PAIR,
4078 &pqi_registers->function_and_status_code);
4079
4080 timeout = PQI_ADMIN_QUEUE_CREATE_TIMEOUT_JIFFIES + jiffies;
4081 while (1) {
4082 status = readb(&pqi_registers->function_and_status_code);
4083 if (status == PQI_STATUS_IDLE)
4084 break;
4085 if (time_after(jiffies, timeout))
4086 return -ETIMEDOUT;
4087 msleep(PQI_ADMIN_QUEUE_CREATE_POLL_INTERVAL_MSECS);
4088 }
4089
4090 /*
4091 * The offset registers are not initialized to the correct
4092 * offsets until *after* the create admin queue pair command
4093 * completes successfully.
4094 */
4095 admin_queues->iq_pi = ctrl_info->iomem_base +
4096 PQI_DEVICE_REGISTERS_OFFSET +
4097 readq(&pqi_registers->admin_iq_pi_offset);
4098 admin_queues->oq_ci = ctrl_info->iomem_base +
4099 PQI_DEVICE_REGISTERS_OFFSET +
4100 readq(&pqi_registers->admin_oq_ci_offset);
4101
4102 return 0;
4103}
4104
4105static void pqi_submit_admin_request(struct pqi_ctrl_info *ctrl_info,
4106 struct pqi_general_admin_request *request)
4107{
4108 struct pqi_admin_queues *admin_queues;
4109 void *next_element;
4110 pqi_index_t iq_pi;
4111
4112 admin_queues = &ctrl_info->admin_queues;
4113 iq_pi = admin_queues->iq_pi_copy;
4114
4115 next_element = admin_queues->iq_element_array +
4116 (iq_pi * PQI_ADMIN_IQ_ELEMENT_LENGTH);
4117
4118 memcpy(next_element, request, sizeof(*request));
4119
4120 iq_pi = (iq_pi + 1) % PQI_ADMIN_IQ_NUM_ELEMENTS;
4121 admin_queues->iq_pi_copy = iq_pi;
4122
4123 /*
4124 * This write notifies the controller that an IU is available to be
4125 * processed.
4126 */
4127 writel(iq_pi, admin_queues->iq_pi);
4128}
4129
13bede67
KB		 4130#define PQI_ADMIN_REQUEST_TIMEOUT_SECS 60
4131
6c223761
KB		 4132static int pqi_poll_for_admin_response(struct pqi_ctrl_info *ctrl_info,
4133 struct pqi_general_admin_response *response)
4134{
4135 struct pqi_admin_queues *admin_queues;
4136 pqi_index_t oq_pi;
4137 pqi_index_t oq_ci;
4138 unsigned long timeout;
4139
4140 admin_queues = &ctrl_info->admin_queues;
4141 oq_ci = admin_queues->oq_ci_copy;
4142
4fd22c13 4143 timeout = (PQI_ADMIN_REQUEST_TIMEOUT_SECS * PQI_HZ) + jiffies;
6c223761
KB		 4144
4145 while (1) {
dac12fbc 4146 oq_pi = readl(admin_queues->oq_pi);
6c223761
KB		 4147 if (oq_pi != oq_ci)
4148 break;
4149 if (time_after(jiffies, timeout)) {
4150 dev_err(&ctrl_info->pci_dev->dev,
4151 "timed out waiting for admin response\n");
4152 return -ETIMEDOUT;
4153 }
13bede67
KB		 4154 if (!sis_is_firmware_running(ctrl_info))
4155 return -ENXIO;
6c223761
KB		 4156 usleep_range(1000, 2000);
4157 }
4158
4159 memcpy(response, admin_queues->oq_element_array +
4160 (oq_ci * PQI_ADMIN_OQ_ELEMENT_LENGTH), sizeof(*response));
4161
4162 oq_ci = (oq_ci + 1) % PQI_ADMIN_OQ_NUM_ELEMENTS;
4163 admin_queues->oq_ci_copy = oq_ci;
4164 writel(oq_ci, admin_queues->oq_ci);
4165
4166 return 0;
4167}
4168
4169static void pqi_start_io(struct pqi_ctrl_info *ctrl_info,
4170 struct pqi_queue_group *queue_group, enum pqi_io_path path,
4171 struct pqi_io_request *io_request)
4172{
4173 struct pqi_io_request *next;
4174 void *next_element;
4175 pqi_index_t iq_pi;
4176 pqi_index_t iq_ci;
4177 size_t iu_length;
4178 unsigned long flags;
4179 unsigned int num_elements_needed;
4180 unsigned int num_elements_to_end_of_queue;
4181 size_t copy_count;
4182 struct pqi_iu_header *request;
4183
4184 spin_lock_irqsave(&queue_group->submit_lock[path], flags);
4185
376fb880
KB		 4186 if (io_request) {
4187 io_request->queue_group = queue_group;
6c223761
KB		 4188 list_add_tail(&io_request->request_list_entry,
4189 &queue_group->request_list[path]);
376fb880 4190 }
6c223761
KB		 4191
4192 iq_pi = queue_group->iq_pi_copy[path];
4193
4194 list_for_each_entry_safe(io_request, next,
4195 &queue_group->request_list[path], request_list_entry) {
4196
4197 request = io_request->iu;
4198
4199 iu_length = get_unaligned_le16(&request->iu_length) +
4200 PQI_REQUEST_HEADER_LENGTH;
4201 num_elements_needed =
4202 DIV_ROUND_UP(iu_length,
4203 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
4204
dac12fbc 4205 iq_ci = readl(queue_group->iq_ci[path]);
6c223761
KB		 4206
4207 if (num_elements_needed > pqi_num_elements_free(iq_pi, iq_ci,
4208 ctrl_info->num_elements_per_iq))
4209 break;
4210
4211 put_unaligned_le16(queue_group->oq_id,
4212 &request->response_queue_id);
4213
4214 next_element = queue_group->iq_element_array[path] +
4215 (iq_pi * PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
4216
4217 num_elements_to_end_of_queue =
4218 ctrl_info->num_elements_per_iq - iq_pi;
4219
4220 if (num_elements_needed <= num_elements_to_end_of_queue) {
4221 memcpy(next_element, request, iu_length);
4222 } else {
4223 copy_count = num_elements_to_end_of_queue *
4224 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH;
4225 memcpy(next_element, request, copy_count);
4226 memcpy(queue_group->iq_element_array[path],
4227 (u8 *)request + copy_count,
4228 iu_length - copy_count);
4229 }
4230
4231 iq_pi = (iq_pi + num_elements_needed) %
4232 ctrl_info->num_elements_per_iq;
4233
4234 list_del(&io_request->request_list_entry);
4235 }
4236
4237 if (iq_pi != queue_group->iq_pi_copy[path]) {
4238 queue_group->iq_pi_copy[path] = iq_pi;
4239 /*
4240 * This write notifies the controller that one or more IUs are
4241 * available to be processed.
4242 */
4243 writel(iq_pi, queue_group->iq_pi[path]);
4244 }
4245
4246 spin_unlock_irqrestore(&queue_group->submit_lock[path], flags);
4247}
4248
1f37e992
KB		 4249#define PQI_WAIT_FOR_COMPLETION_IO_TIMEOUT_SECS 10
4250
4251static int pqi_wait_for_completion_io(struct pqi_ctrl_info *ctrl_info,
4252 struct completion *wait)
4253{
4254 int rc;
1f37e992
KB		 4255
4256 while (1) {
4257 if (wait_for_completion_io_timeout(wait,
4fd22c13 4258 PQI_WAIT_FOR_COMPLETION_IO_TIMEOUT_SECS * PQI_HZ)) {
1f37e992
KB		 4259 rc = 0;
4260 break;
4261 }
4262
4263 pqi_check_ctrl_health(ctrl_info);
4264 if (pqi_ctrl_offline(ctrl_info)) {
4265 rc = -ENXIO;
4266 break;
4267 }
1f37e992
KB		 4268 }
4269
4270 return rc;
4271}
4272
6c223761
KB		 4273static void pqi_raid_synchronous_complete(struct pqi_io_request *io_request,
4274 void *context)
4275{
4276 struct completion *waiting = context;
4277
4278 complete(waiting);
4279}
4280
694c5d5b
KB		 4281static int pqi_process_raid_io_error_synchronous(
4282 struct pqi_raid_error_info *error_info)
26b390ab
KB		 4283{
4284 int rc = -EIO;
4285
4286 switch (error_info->data_out_result) {
4287 case PQI_DATA_IN_OUT_GOOD:
4288 if (error_info->status == SAM_STAT_GOOD)
4289 rc = 0;
4290 break;
4291 case PQI_DATA_IN_OUT_UNDERFLOW:
4292 if (error_info->status == SAM_STAT_GOOD ||
4293 error_info->status == SAM_STAT_CHECK_CONDITION)
4294 rc = 0;
4295 break;
4296 case PQI_DATA_IN_OUT_ABORTED:
4297 rc = PQI_CMD_STATUS_ABORTED;
4298 break;
4299 }
4300
4301 return rc;
4302}
4303
ae0c189d
KB		 4304static inline bool pqi_is_blockable_request(struct pqi_iu_header *request)
4305{
4306 return (request->driver_flags & PQI_DRIVER_NONBLOCKABLE_REQUEST) == 0;
4307}
4308
6c223761
KB		 4309static int pqi_submit_raid_request_synchronous(struct pqi_ctrl_info *ctrl_info,
4310 struct pqi_iu_header *request, unsigned int flags,
ae0c189d 4311 struct pqi_raid_error_info *error_info)
6c223761 4312{
957c5ab1 4313 int rc = 0;
6c223761 4314 struct pqi_io_request *io_request;
6c223761 4315 size_t iu_length;
957c5ab1 4316 DECLARE_COMPLETION_ONSTACK(wait);
6c223761 4317
6c223761
KB		 4318 if (flags & PQI_SYNC_FLAGS_INTERRUPTABLE) {
4319 if (down_interruptible(&ctrl_info->sync_request_sem))
4320 return -ERESTARTSYS;
4321 } else {
ae0c189d 4322 down(&ctrl_info->sync_request_sem);
6c223761
KB		 4323 }
4324
7561a7e4 4325 pqi_ctrl_busy(ctrl_info);
ae0c189d
KB		 4326 /*
4327 * Wait for other admin queue updates such as;
4328 * config table changes, OFA memory updates, ...
4329 */
4330 if (pqi_is_blockable_request(request))
4331 pqi_wait_if_ctrl_blocked(ctrl_info);
7561a7e4 4332
376fb880
KB		 4333 if (pqi_ctrl_offline(ctrl_info)) {
4334 rc = -ENXIO;
4335 goto out;
4336 }
4337
6c223761
KB		 4338 io_request = pqi_alloc_io_request(ctrl_info);
4339
4340 put_unaligned_le16(io_request->index,
4341 &(((struct pqi_raid_path_request *)request)->request_id));
4342
4343 if (request->iu_type == PQI_REQUEST_IU_RAID_PATH_IO)
4344 ((struct pqi_raid_path_request *)request)->error_index =
4345 ((struct pqi_raid_path_request *)request)->request_id;
4346
4347 iu_length = get_unaligned_le16(&request->iu_length) +
4348 PQI_REQUEST_HEADER_LENGTH;
4349 memcpy(io_request->iu, request, iu_length);
4350
957c5ab1
KB		 4351 io_request->io_complete_callback = pqi_raid_synchronous_complete;
4352 io_request->context = &wait;
4353
583891c9 4354 pqi_start_io(ctrl_info, &ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP], RAID_PATH,
957c5ab1
KB		 4355 io_request);
4356
ae0c189d 4357 pqi_wait_for_completion_io(ctrl_info, &wait);
6c223761
KB		 4358
4359 if (error_info) {
4360 if (io_request->error_info)
583891c9 4361 memcpy(error_info, io_request->error_info, sizeof(*error_info));
6c223761
KB		 4362 else
4363 memset(error_info, 0, sizeof(*error_info));
4364 } else if (rc == 0 && io_request->error_info) {
583891c9 4365 rc = pqi_process_raid_io_error_synchronous(io_request->error_info);
6c223761
KB		 4366 }
4367
4368 pqi_free_io_request(io_request);
4369
7561a7e4 4370out:
ae0c189d 4371 pqi_ctrl_unbusy(ctrl_info);
6c223761
KB		 4372 up(&ctrl_info->sync_request_sem);
4373
4374 return rc;
4375}
4376
4377static int pqi_validate_admin_response(
4378 struct pqi_general_admin_response *response, u8 expected_function_code)
4379{
4380 if (response->header.iu_type != PQI_RESPONSE_IU_GENERAL_ADMIN)
4381 return -EINVAL;
4382
4383 if (get_unaligned_le16(&response->header.iu_length) !=
4384 PQI_GENERAL_ADMIN_IU_LENGTH)
4385 return -EINVAL;
4386
4387 if (response->function_code != expected_function_code)
4388 return -EINVAL;
4389
4390 if (response->status != PQI_GENERAL_ADMIN_STATUS_SUCCESS)
4391 return -EINVAL;
4392
4393 return 0;
4394}
4395
4396static int pqi_submit_admin_request_synchronous(
4397 struct pqi_ctrl_info *ctrl_info,
4398 struct pqi_general_admin_request *request,
4399 struct pqi_general_admin_response *response)
4400{
4401 int rc;
4402
4403 pqi_submit_admin_request(ctrl_info, request);
4404
4405 rc = pqi_poll_for_admin_response(ctrl_info, response);
4406
4407 if (rc == 0)
ae0c189d 4408 rc = pqi_validate_admin_response(response, request->function_code);
6c223761
KB		 4409
4410 return rc;
4411}
4412
4413static int pqi_report_device_capability(struct pqi_ctrl_info *ctrl_info)
4414{
4415 int rc;
4416 struct pqi_general_admin_request request;
4417 struct pqi_general_admin_response response;
4418 struct pqi_device_capability *capability;
4419 struct pqi_iu_layer_descriptor *sop_iu_layer_descriptor;
4420
4421 capability = kmalloc(sizeof(*capability), GFP_KERNEL);
4422 if (!capability)
4423 return -ENOMEM;
4424
4425 memset(&request, 0, sizeof(request));
4426
4427 request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
4428 put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
4429 &request.header.iu_length);
4430 request.function_code =
4431 PQI_GENERAL_ADMIN_FUNCTION_REPORT_DEVICE_CAPABILITY;
4432 put_unaligned_le32(sizeof(*capability),
4433 &request.data.report_device_capability.buffer_length);
4434
4435 rc = pqi_map_single(ctrl_info->pci_dev,
4436 &request.data.report_device_capability.sg_descriptor,
4437 capability, sizeof(*capability),
6917a9cc 4438 DMA_FROM_DEVICE);
6c223761
KB		 4439 if (rc)
4440 goto out;
4441
583891c9 4442 rc = pqi_submit_admin_request_synchronous(ctrl_info, &request, &response);
6c223761
KB		 4443
4444 pqi_pci_unmap(ctrl_info->pci_dev,
4445 &request.data.report_device_capability.sg_descriptor, 1,
6917a9cc 4446 DMA_FROM_DEVICE);
6c223761
KB		 4447
4448 if (rc)
4449 goto out;
4450
4451 if (response.status != PQI_GENERAL_ADMIN_STATUS_SUCCESS) {
4452 rc = -EIO;
4453 goto out;
4454 }
4455
4456 ctrl_info->max_inbound_queues =
4457 get_unaligned_le16(&capability->max_inbound_queues);
4458 ctrl_info->max_elements_per_iq =
4459 get_unaligned_le16(&capability->max_elements_per_iq);
4460 ctrl_info->max_iq_element_length =
4461 get_unaligned_le16(&capability->max_iq_element_length)
4462 * 16;
4463 ctrl_info->max_outbound_queues =
4464 get_unaligned_le16(&capability->max_outbound_queues);
4465 ctrl_info->max_elements_per_oq =
4466 get_unaligned_le16(&capability->max_elements_per_oq);
4467 ctrl_info->max_oq_element_length =
4468 get_unaligned_le16(&capability->max_oq_element_length)
4469 * 16;
4470
4471 sop_iu_layer_descriptor =
4472 &capability->iu_layer_descriptors[PQI_PROTOCOL_SOP];
4473
4474 ctrl_info->max_inbound_iu_length_per_firmware =
4475 get_unaligned_le16(
4476 &sop_iu_layer_descriptor->max_inbound_iu_length);
4477 ctrl_info->inbound_spanning_supported =
4478 sop_iu_layer_descriptor->inbound_spanning_supported;
4479 ctrl_info->outbound_spanning_supported =
4480 sop_iu_layer_descriptor->outbound_spanning_supported;
4481
4482out:
4483 kfree(capability);
4484
4485 return rc;
4486}
4487
4488static int pqi_validate_device_capability(struct pqi_ctrl_info *ctrl_info)
4489{
4490 if (ctrl_info->max_iq_element_length <
4491 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH) {
4492 dev_err(&ctrl_info->pci_dev->dev,
4493 "max. inbound queue element length of %d is less than the required length of %d\n",
4494 ctrl_info->max_iq_element_length,
4495 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
4496 return -EINVAL;
4497 }
4498
4499 if (ctrl_info->max_oq_element_length <
4500 PQI_OPERATIONAL_OQ_ELEMENT_LENGTH) {
4501 dev_err(&ctrl_info->pci_dev->dev,
4502 "max. outbound queue element length of %d is less than the required length of %d\n",
4503 ctrl_info->max_oq_element_length,
4504 PQI_OPERATIONAL_OQ_ELEMENT_LENGTH);
4505 return -EINVAL;
4506 }
4507
4508 if (ctrl_info->max_inbound_iu_length_per_firmware <
4509 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH) {
4510 dev_err(&ctrl_info->pci_dev->dev,
4511 "max. inbound IU length of %u is less than the min. required length of %d\n",
4512 ctrl_info->max_inbound_iu_length_per_firmware,
4513 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
4514 return -EINVAL;
4515 }
4516
77668f41
KB		 4517 if (!ctrl_info->inbound_spanning_supported) {
4518 dev_err(&ctrl_info->pci_dev->dev,
4519 "the controller does not support inbound spanning\n");
4520 return -EINVAL;
4521 }
4522
4523 if (ctrl_info->outbound_spanning_supported) {
4524 dev_err(&ctrl_info->pci_dev->dev,
4525 "the controller supports outbound spanning but this driver does not\n");
4526 return -EINVAL;
4527 }
4528
6c223761
KB		 4529 return 0;
4530}
4531
6c223761
KB		 4532static int pqi_create_event_queue(struct pqi_ctrl_info *ctrl_info)
4533{
4534 int rc;
4535 struct pqi_event_queue *event_queue;
4536 struct pqi_general_admin_request request;
4537 struct pqi_general_admin_response response;
4538
4539 event_queue = &ctrl_info->event_queue;
4540
4541 /*
4542 * Create OQ (Outbound Queue - device to host queue) to dedicate
4543 * to events.
4544 */
4545 memset(&request, 0, sizeof(request));
4546 request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
4547 put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
4548 &request.header.iu_length);
4549 request.function_code = PQI_GENERAL_ADMIN_FUNCTION_CREATE_OQ;
4550 put_unaligned_le16(event_queue->oq_id,
4551 &request.data.create_operational_oq.queue_id);
4552 put_unaligned_le64((u64)event_queue->oq_element_array_bus_addr,
4553 &request.data.create_operational_oq.element_array_addr);
4554 put_unaligned_le64((u64)event_queue->oq_pi_bus_addr,
4555 &request.data.create_operational_oq.pi_addr);
4556 put_unaligned_le16(PQI_NUM_EVENT_QUEUE_ELEMENTS,
4557 &request.data.create_operational_oq.num_elements);
4558 put_unaligned_le16(PQI_EVENT_OQ_ELEMENT_LENGTH / 16,
4559 &request.data.create_operational_oq.element_length);
4560 request.data.create_operational_oq.queue_protocol = PQI_PROTOCOL_SOP;
4561 put_unaligned_le16(event_queue->int_msg_num,
4562 &request.data.create_operational_oq.int_msg_num);
4563
4564 rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
4565 &response);
4566 if (rc)
4567 return rc;
4568
4569 event_queue->oq_ci = ctrl_info->iomem_base +
4570 PQI_DEVICE_REGISTERS_OFFSET +
4571 get_unaligned_le64(
4572 &response.data.create_operational_oq.oq_ci_offset);
4573
4574 return 0;
4575}
4576
061ef06a
KB		 4577static int pqi_create_queue_group(struct pqi_ctrl_info *ctrl_info,
4578 unsigned int group_number)
6c223761 4579{
6c223761
KB		 4580 int rc;
4581 struct pqi_queue_group *queue_group;
4582 struct pqi_general_admin_request request;
4583 struct pqi_general_admin_response response;
4584
061ef06a 4585 queue_group = &ctrl_info->queue_groups[group_number];
6c223761
KB		 4586
4587 /*
4588 * Create IQ (Inbound Queue - host to device queue) for
4589 * RAID path.
4590 */
4591 memset(&request, 0, sizeof(request));
4592 request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
4593 put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
4594 &request.header.iu_length);
4595 request.function_code = PQI_GENERAL_ADMIN_FUNCTION_CREATE_IQ;
4596 put_unaligned_le16(queue_group->iq_id[RAID_PATH],
4597 &request.data.create_operational_iq.queue_id);
4598 put_unaligned_le64(
4599 (u64)queue_group->iq_element_array_bus_addr[RAID_PATH],
4600 &request.data.create_operational_iq.element_array_addr);
4601 put_unaligned_le64((u64)queue_group->iq_ci_bus_addr[RAID_PATH],
4602 &request.data.create_operational_iq.ci_addr);
4603 put_unaligned_le16(ctrl_info->num_elements_per_iq,
4604 &request.data.create_operational_iq.num_elements);
4605 put_unaligned_le16(PQI_OPERATIONAL_IQ_ELEMENT_LENGTH / 16,
4606 &request.data.create_operational_iq.element_length);
4607 request.data.create_operational_iq.queue_protocol = PQI_PROTOCOL_SOP;
4608
4609 rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
4610 &response);
4611 if (rc) {
4612 dev_err(&ctrl_info->pci_dev->dev,
4613 "error creating inbound RAID queue\n");
4614 return rc;
4615 }
4616
4617 queue_group->iq_pi[RAID_PATH] = ctrl_info->iomem_base +
4618 PQI_DEVICE_REGISTERS_OFFSET +
4619 get_unaligned_le64(
4620 &response.data.create_operational_iq.iq_pi_offset);
4621
4622 /*
4623 * Create IQ (Inbound Queue - host to device queue) for
4624 * Advanced I/O (AIO) path.
4625 */
4626 memset(&request, 0, sizeof(request));
4627 request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
4628 put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
4629 &request.header.iu_length);
4630 request.function_code = PQI_GENERAL_ADMIN_FUNCTION_CREATE_IQ;
4631 put_unaligned_le16(queue_group->iq_id[AIO_PATH],
4632 &request.data.create_operational_iq.queue_id);
4633 put_unaligned_le64((u64)queue_group->
4634 iq_element_array_bus_addr[AIO_PATH],
4635 &request.data.create_operational_iq.element_array_addr);
4636 put_unaligned_le64((u64)queue_group->iq_ci_bus_addr[AIO_PATH],
4637 &request.data.create_operational_iq.ci_addr);
4638 put_unaligned_le16(ctrl_info->num_elements_per_iq,
4639 &request.data.create_operational_iq.num_elements);
4640 put_unaligned_le16(PQI_OPERATIONAL_IQ_ELEMENT_LENGTH / 16,
4641 &request.data.create_operational_iq.element_length);
4642 request.data.create_operational_iq.queue_protocol = PQI_PROTOCOL_SOP;
4643
4644 rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
4645 &response);
4646 if (rc) {
4647 dev_err(&ctrl_info->pci_dev->dev,
4648 "error creating inbound AIO queue\n");
339faa81 4649 return rc;
6c223761
KB		 4650 }
4651
4652 queue_group->iq_pi[AIO_PATH] = ctrl_info->iomem_base +
4653 PQI_DEVICE_REGISTERS_OFFSET +
4654 get_unaligned_le64(
4655 &response.data.create_operational_iq.iq_pi_offset);
4656
4657 /*
4658 * Designate the 2nd IQ as the AIO path. By default, all IQs are
4659 * assumed to be for RAID path I/O unless we change the queue's
4660 * property.
4661 */
4662 memset(&request, 0, sizeof(request));
4663 request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
4664 put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
4665 &request.header.iu_length);
4666 request.function_code = PQI_GENERAL_ADMIN_FUNCTION_CHANGE_IQ_PROPERTY;
4667 put_unaligned_le16(queue_group->iq_id[AIO_PATH],
4668 &request.data.change_operational_iq_properties.queue_id);
4669 put_unaligned_le32(PQI_IQ_PROPERTY_IS_AIO_QUEUE,
4670 &request.data.change_operational_iq_properties.vendor_specific);
4671
4672 rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
4673 &response);
4674 if (rc) {
4675 dev_err(&ctrl_info->pci_dev->dev,
4676 "error changing queue property\n");
339faa81 4677 return rc;
6c223761
KB		 4678 }
4679
4680 /*
4681 * Create OQ (Outbound Queue - device to host queue).
4682 */
4683 memset(&request, 0, sizeof(request));
4684 request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
4685 put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
4686 &request.header.iu_length);
4687 request.function_code = PQI_GENERAL_ADMIN_FUNCTION_CREATE_OQ;
4688 put_unaligned_le16(queue_group->oq_id,
4689 &request.data.create_operational_oq.queue_id);
4690 put_unaligned_le64((u64)queue_group->oq_element_array_bus_addr,
4691 &request.data.create_operational_oq.element_array_addr);
4692 put_unaligned_le64((u64)queue_group->oq_pi_bus_addr,
4693 &request.data.create_operational_oq.pi_addr);
4694 put_unaligned_le16(ctrl_info->num_elements_per_oq,
4695 &request.data.create_operational_oq.num_elements);
4696 put_unaligned_le16(PQI_OPERATIONAL_OQ_ELEMENT_LENGTH / 16,
4697 &request.data.create_operational_oq.element_length);
4698 request.data.create_operational_oq.queue_protocol = PQI_PROTOCOL_SOP;
4699 put_unaligned_le16(queue_group->int_msg_num,
4700 &request.data.create_operational_oq.int_msg_num);
4701
4702 rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
4703 &response);
4704 if (rc) {
4705 dev_err(&ctrl_info->pci_dev->dev,
4706 "error creating outbound queue\n");
339faa81 4707 return rc;
6c223761
KB		 4708 }
4709
4710 queue_group->oq_ci = ctrl_info->iomem_base +
4711 PQI_DEVICE_REGISTERS_OFFSET +
4712 get_unaligned_le64(
4713 &response.data.create_operational_oq.oq_ci_offset);
4714
6c223761 4715 return 0;
6c223761
KB		 4716}
4717
4718static int pqi_create_queues(struct pqi_ctrl_info *ctrl_info)
4719{
4720 int rc;
4721 unsigned int i;
4722
4723 rc = pqi_create_event_queue(ctrl_info);
4724 if (rc) {
4725 dev_err(&ctrl_info->pci_dev->dev,
4726 "error creating event queue\n");
4727 return rc;
4728 }
4729
4730 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
061ef06a 4731 rc = pqi_create_queue_group(ctrl_info, i);
6c223761
KB		 4732 if (rc) {
4733 dev_err(&ctrl_info->pci_dev->dev,
4734 "error creating queue group number %u/%u\n",
4735 i, ctrl_info->num_queue_groups);
4736 return rc;
4737 }
4738 }
4739
4740 return 0;
4741}
4742
4743#define PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH \
4744 (offsetof(struct pqi_event_config, descriptors) + \
4745 (PQI_MAX_EVENT_DESCRIPTORS * sizeof(struct pqi_event_descriptor)))
4746
6a50d6ad
KB		 4747static int pqi_configure_events(struct pqi_ctrl_info *ctrl_info,
4748 bool enable_events)
6c223761
KB		 4749{
4750 int rc;
4751 unsigned int i;
4752 struct pqi_event_config *event_config;
6a50d6ad 4753 struct pqi_event_descriptor *event_descriptor;
6c223761
KB		 4754 struct pqi_general_management_request request;
4755
4756 event_config = kmalloc(PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH,
4757 GFP_KERNEL);
4758 if (!event_config)
4759 return -ENOMEM;
4760
4761 memset(&request, 0, sizeof(request));
4762
4763 request.header.iu_type = PQI_REQUEST_IU_REPORT_VENDOR_EVENT_CONFIG;
4764 put_unaligned_le16(offsetof(struct pqi_general_management_request,
4765 data.report_event_configuration.sg_descriptors[1]) -
4766 PQI_REQUEST_HEADER_LENGTH, &request.header.iu_length);
4767 put_unaligned_le32(PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH,
4768 &request.data.report_event_configuration.buffer_length);
4769
4770 rc = pqi_map_single(ctrl_info->pci_dev,
4771 request.data.report_event_configuration.sg_descriptors,
4772 event_config, PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH,
6917a9cc 4773 DMA_FROM_DEVICE);
6c223761
KB		 4774 if (rc)
4775 goto out;
4776
ae0c189d 4777 rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0, NULL);
6c223761
KB		 4778
4779 pqi_pci_unmap(ctrl_info->pci_dev,
4780 request.data.report_event_configuration.sg_descriptors, 1,
6917a9cc 4781 DMA_FROM_DEVICE);
6c223761
KB		 4782
4783 if (rc)
4784 goto out;
4785
6a50d6ad
KB		 4786 for (i = 0; i < event_config->num_event_descriptors; i++) {
4787 event_descriptor = &event_config->descriptors[i];
4788 if (enable_events &&
4789 pqi_is_supported_event(event_descriptor->event_type))
583891c9 4790 put_unaligned_le16(ctrl_info->event_queue.oq_id,
6a50d6ad
KB		 4791 &event_descriptor->oq_id);
4792 else
4793 put_unaligned_le16(0, &event_descriptor->oq_id);
4794 }
6c223761
KB		 4795
4796 memset(&request, 0, sizeof(request));
4797
4798 request.header.iu_type = PQI_REQUEST_IU_SET_VENDOR_EVENT_CONFIG;
4799 put_unaligned_le16(offsetof(struct pqi_general_management_request,
4800 data.report_event_configuration.sg_descriptors[1]) -
4801 PQI_REQUEST_HEADER_LENGTH, &request.header.iu_length);
4802 put_unaligned_le32(PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH,
4803 &request.data.report_event_configuration.buffer_length);
4804
4805 rc = pqi_map_single(ctrl_info->pci_dev,
4806 request.data.report_event_configuration.sg_descriptors,
4807 event_config, PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH,
6917a9cc 4808 DMA_TO_DEVICE);
6c223761
KB		 4809 if (rc)
4810 goto out;
4811
ae0c189d 4812 rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0, NULL);
6c223761
KB		 4813
4814 pqi_pci_unmap(ctrl_info->pci_dev,
4815 request.data.report_event_configuration.sg_descriptors, 1,
6917a9cc 4816 DMA_TO_DEVICE);
6c223761
KB		 4817
4818out:
4819 kfree(event_config);
4820
4821 return rc;
4822}
4823
6a50d6ad
KB		 4824static inline int pqi_enable_events(struct pqi_ctrl_info *ctrl_info)
4825{
4826 return pqi_configure_events(ctrl_info, true);
4827}
4828
6c223761
KB		 4829static void pqi_free_all_io_requests(struct pqi_ctrl_info *ctrl_info)
4830{
4831 unsigned int i;
4832 struct device *dev;
4833 size_t sg_chain_buffer_length;
4834 struct pqi_io_request *io_request;
4835
4836 if (!ctrl_info->io_request_pool)
4837 return;
4838
4839 dev = &ctrl_info->pci_dev->dev;
4840 sg_chain_buffer_length = ctrl_info->sg_chain_buffer_length;
4841 io_request = ctrl_info->io_request_pool;
4842
4843 for (i = 0; i < ctrl_info->max_io_slots; i++) {
4844 kfree(io_request->iu);
4845 if (!io_request->sg_chain_buffer)
4846 break;
4847 dma_free_coherent(dev, sg_chain_buffer_length,
4848 io_request->sg_chain_buffer,
4849 io_request->sg_chain_buffer_dma_handle);
4850 io_request++;
4851 }
4852
4853 kfree(ctrl_info->io_request_pool);
4854 ctrl_info->io_request_pool = NULL;
4855}
4856
4857static inline int pqi_alloc_error_buffer(struct pqi_ctrl_info *ctrl_info)
4858{
694c5d5b
KB		 4859 ctrl_info->error_buffer = dma_alloc_coherent(&ctrl_info->pci_dev->dev,
4860 ctrl_info->error_buffer_length,
4861 &ctrl_info->error_buffer_dma_handle,
4862 GFP_KERNEL);
6c223761
KB		 4863 if (!ctrl_info->error_buffer)
4864 return -ENOMEM;
4865
4866 return 0;
4867}
4868
4869static int pqi_alloc_io_resources(struct pqi_ctrl_info *ctrl_info)
4870{
4871 unsigned int i;
4872 void *sg_chain_buffer;
4873 size_t sg_chain_buffer_length;
4874 dma_addr_t sg_chain_buffer_dma_handle;
4875 struct device *dev;
4876 struct pqi_io_request *io_request;
4877
583891c9
KB		 4878 ctrl_info->io_request_pool = kcalloc(ctrl_info->max_io_slots,
4879 sizeof(ctrl_info->io_request_pool[0]), GFP_KERNEL);
6c223761
KB		 4880
4881 if (!ctrl_info->io_request_pool) {
4882 dev_err(&ctrl_info->pci_dev->dev,
4883 "failed to allocate I/O request pool\n");
4884 goto error;
4885 }
4886
4887 dev = &ctrl_info->pci_dev->dev;
4888 sg_chain_buffer_length = ctrl_info->sg_chain_buffer_length;
4889 io_request = ctrl_info->io_request_pool;
4890
4891 for (i = 0; i < ctrl_info->max_io_slots; i++) {
583891c9 4892 io_request->iu = kmalloc(ctrl_info->max_inbound_iu_length, GFP_KERNEL);
6c223761
KB		 4893
4894 if (!io_request->iu) {
4895 dev_err(&ctrl_info->pci_dev->dev,
4896 "failed to allocate IU buffers\n");
4897 goto error;
4898 }
4899
4900 sg_chain_buffer = dma_alloc_coherent(dev,
4901 sg_chain_buffer_length, &sg_chain_buffer_dma_handle,
4902 GFP_KERNEL);
4903
4904 if (!sg_chain_buffer) {
4905 dev_err(&ctrl_info->pci_dev->dev,
4906 "failed to allocate PQI scatter-gather chain buffers\n");
4907 goto error;
4908 }
4909
4910 io_request->index = i;
4911 io_request->sg_chain_buffer = sg_chain_buffer;
583891c9 4912 io_request->sg_chain_buffer_dma_handle = sg_chain_buffer_dma_handle;
6c223761
KB		 4913 io_request++;
4914 }
4915
4916 return 0;
4917
4918error:
4919 pqi_free_all_io_requests(ctrl_info);
4920
4921 return -ENOMEM;
4922}
4923
4924/*
4925 * Calculate required resources that are sized based on max. outstanding
4926 * requests and max. transfer size.
4927 */
4928
4929static void pqi_calculate_io_resources(struct pqi_ctrl_info *ctrl_info)
4930{
4931 u32 max_transfer_size;
4932 u32 max_sg_entries;
4933
4934 ctrl_info->scsi_ml_can_queue =
4935 ctrl_info->max_outstanding_requests - PQI_RESERVED_IO_SLOTS;
4936 ctrl_info->max_io_slots = ctrl_info->max_outstanding_requests;
4937
4938 ctrl_info->error_buffer_length =
4939 ctrl_info->max_io_slots * PQI_ERROR_BUFFER_ELEMENT_LENGTH;
4940
d727a776
KB		 4941 if (reset_devices)
4942 max_transfer_size = min(ctrl_info->max_transfer_size,
4943 PQI_MAX_TRANSFER_SIZE_KDUMP);
4944 else
4945 max_transfer_size = min(ctrl_info->max_transfer_size,
4946 PQI_MAX_TRANSFER_SIZE);
6c223761
KB		 4947
4948 max_sg_entries = max_transfer_size / PAGE_SIZE;
4949
4950 /* +1 to cover when the buffer is not page-aligned. */
4951 max_sg_entries++;
4952
4953 max_sg_entries = min(ctrl_info->max_sg_entries, max_sg_entries);
4954
4955 max_transfer_size = (max_sg_entries - 1) * PAGE_SIZE;
4956
4957 ctrl_info->sg_chain_buffer_length =
e1d213bd
KB		 4958 (max_sg_entries * sizeof(struct pqi_sg_descriptor)) +
4959 PQI_EXTRA_SGL_MEMORY;
6c223761
KB		 4960 ctrl_info->sg_tablesize = max_sg_entries;
4961 ctrl_info->max_sectors = max_transfer_size / 512;
4962}
4963
4964static void pqi_calculate_queue_resources(struct pqi_ctrl_info *ctrl_info)
4965{
6c223761
KB		 4966 int num_queue_groups;
4967 u16 num_elements_per_iq;
4968 u16 num_elements_per_oq;
4969
d727a776
KB		 4970 if (reset_devices) {
4971 num_queue_groups = 1;
4972 } else {
4973 int num_cpus;
4974 int max_queue_groups;
4975
4976 max_queue_groups = min(ctrl_info->max_inbound_queues / 2,
4977 ctrl_info->max_outbound_queues - 1);
4978 max_queue_groups = min(max_queue_groups, PQI_MAX_QUEUE_GROUPS);
6c223761 4979
d727a776
KB		 4980 num_cpus = num_online_cpus();
4981 num_queue_groups = min(num_cpus, ctrl_info->max_msix_vectors);
4982 num_queue_groups = min(num_queue_groups, max_queue_groups);
4983 }
6c223761
KB		 4984
4985 ctrl_info->num_queue_groups = num_queue_groups;
061ef06a 4986 ctrl_info->max_hw_queue_index = num_queue_groups - 1;
6c223761 4987
77668f41
KB		 4988 /*
4989 * Make sure that the max. inbound IU length is an even multiple
4990 * of our inbound element length.
4991 */
4992 ctrl_info->max_inbound_iu_length =
4993 (ctrl_info->max_inbound_iu_length_per_firmware /
4994 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH) *
4995 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH;
6c223761
KB		 4996
4997 num_elements_per_iq =
4998 (ctrl_info->max_inbound_iu_length /
4999 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
5000
5001 /* Add one because one element in each queue is unusable. */
5002 num_elements_per_iq++;
5003
5004 num_elements_per_iq = min(num_elements_per_iq,
5005 ctrl_info->max_elements_per_iq);
5006
5007 num_elements_per_oq = ((num_elements_per_iq - 1) * 2) + 1;
5008 num_elements_per_oq = min(num_elements_per_oq,
5009 ctrl_info->max_elements_per_oq);
5010
5011 ctrl_info->num_elements_per_iq = num_elements_per_iq;
5012 ctrl_info->num_elements_per_oq = num_elements_per_oq;
5013
5014 ctrl_info->max_sg_per_iu =
5015 ((ctrl_info->max_inbound_iu_length -
5016 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH) /
5017 sizeof(struct pqi_sg_descriptor)) +
5018 PQI_MAX_EMBEDDED_SG_DESCRIPTORS;
6702d2c4
DB		 5019
5020 ctrl_info->max_sg_per_r56_iu =
5021 ((ctrl_info->max_inbound_iu_length -
5022 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH) /
5023 sizeof(struct pqi_sg_descriptor)) +
5024 PQI_MAX_EMBEDDED_R56_SG_DESCRIPTORS;
6c223761
KB		 5025}
5026
583891c9
KB		 5027static inline void pqi_set_sg_descriptor(struct pqi_sg_descriptor *sg_descriptor,
5028 struct scatterlist *sg)
6c223761
KB		 5029{
5030 u64 address = (u64)sg_dma_address(sg);
5031 unsigned int length = sg_dma_len(sg);
5032
5033 put_unaligned_le64(address, &sg_descriptor->address);
5034 put_unaligned_le32(length, &sg_descriptor->length);
5035 put_unaligned_le32(0, &sg_descriptor->flags);
5036}
5037
1a22bc4b
DB		 5038static unsigned int pqi_build_sg_list(struct pqi_sg_descriptor *sg_descriptor,
5039 struct scatterlist *sg, int sg_count, struct pqi_io_request *io_request,
5040 int max_sg_per_iu, bool *chained)
6c223761
KB		 5041{
5042 int i;
6c223761 5043 unsigned int num_sg_in_iu;
6c223761 5044
1a22bc4b 5045 *chained = false;
6c223761 5046 i = 0;
1a22bc4b
DB		 5047 num_sg_in_iu = 0;
5048 max_sg_per_iu--; /* Subtract 1 to leave room for chain marker. */
6c223761
KB		 5049
5050 while (1) {
5051 pqi_set_sg_descriptor(sg_descriptor, sg);
1a22bc4b 5052 if (!*chained)
6c223761
KB		 5053 num_sg_in_iu++;
5054 i++;
5055 if (i == sg_count)
5056 break;
5057 sg_descriptor++;
5058 if (i == max_sg_per_iu) {
1a22bc4b 5059 put_unaligned_le64((u64)io_request->sg_chain_buffer_dma_handle,
6c223761 5060 &sg_descriptor->address);
1a22bc4b 5061 put_unaligned_le32((sg_count - num_sg_in_iu) * sizeof(*sg_descriptor),
6c223761 5062 &sg_descriptor->length);
1a22bc4b
DB		 5063 put_unaligned_le32(CISS_SG_CHAIN, &sg_descriptor->flags);
5064 *chained = true;
6c223761
KB		 5065 num_sg_in_iu++;
5066 sg_descriptor = io_request->sg_chain_buffer;
5067 }
5068 sg = sg_next(sg);
5069 }
5070
5071 put_unaligned_le32(CISS_SG_LAST, &sg_descriptor->flags);
6c223761 5072
1a22bc4b 5073 return num_sg_in_iu;
6c223761
KB		 5074}
5075
6c223761
KB		 5076static int pqi_build_raid_sg_list(struct pqi_ctrl_info *ctrl_info,
5077 struct pqi_raid_path_request *request, struct scsi_cmnd *scmd,
6c223761
KB		 5078 struct pqi_io_request *io_request)
5079{
6c223761
KB		 5080 u16 iu_length;
5081 int sg_count;
a60eec02
KB		 5082 bool chained;
5083 unsigned int num_sg_in_iu;
6c223761
KB		 5084 struct scatterlist *sg;
5085 struct pqi_sg_descriptor *sg_descriptor;
5086
5087 sg_count = scsi_dma_map(scmd);
5088 if (sg_count < 0)
5089 return sg_count;
a60eec02 5090
6c223761 5091 iu_length = offsetof(struct pqi_raid_path_request, sg_descriptors) -
a60eec02 5092 PQI_REQUEST_HEADER_LENGTH;
a60eec02 5093
6c223761
KB		 5094 if (sg_count == 0)
5095 goto out;
5096
a60eec02
KB		 5097 sg = scsi_sglist(scmd);
5098 sg_descriptor = request->sg_descriptors;
a60eec02 5099
1a22bc4b
DB		 5100 num_sg_in_iu = pqi_build_sg_list(sg_descriptor, sg, sg_count, io_request,
5101 ctrl_info->max_sg_per_iu, &chained);
6c223761 5102
a60eec02 5103 request->partial = chained;
6c223761 5104 iu_length += num_sg_in_iu * sizeof(*sg_descriptor);
a60eec02
KB		 5105
5106out:
6c223761 5107 put_unaligned_le16(iu_length, &request->header.iu_length);
6c223761
KB		 5108
5109 return 0;
5110}
5111
7a012c23
DB		 5112static int pqi_build_aio_r1_sg_list(struct pqi_ctrl_info *ctrl_info,
5113 struct pqi_aio_r1_path_request *request, struct scsi_cmnd *scmd,
5114 struct pqi_io_request *io_request)
6c223761 5115{
7a012c23
DB		 5116 u16 iu_length;
5117 int sg_count;
5118 bool chained;
5119 unsigned int num_sg_in_iu;
5120 struct scatterlist *sg;
5121 struct pqi_sg_descriptor *sg_descriptor;
5122
5123 sg_count = scsi_dma_map(scmd);
5124 if (sg_count < 0)
5125 return sg_count;
5126
5127 iu_length = offsetof(struct pqi_aio_r1_path_request, sg_descriptors) -
5128 PQI_REQUEST_HEADER_LENGTH;
5129 num_sg_in_iu = 0;
5130
5131 if (sg_count == 0)
5132 goto out;
5133
5134 sg = scsi_sglist(scmd);
5135 sg_descriptor = request->sg_descriptors;
5136
5137 num_sg_in_iu = pqi_build_sg_list(sg_descriptor, sg, sg_count, io_request,
5138 ctrl_info->max_sg_per_iu, &chained);
5139
5140 request->partial = chained;
5141 iu_length += num_sg_in_iu * sizeof(*sg_descriptor);
5142
5143out:
5144 put_unaligned_le16(iu_length, &request->header.iu_length);
5145 request->num_sg_descriptors = num_sg_in_iu;
5146
5147 return 0;
5148}
5149
6702d2c4
DB		 5150static int pqi_build_aio_r56_sg_list(struct pqi_ctrl_info *ctrl_info,
5151 struct pqi_aio_r56_path_request *request, struct scsi_cmnd *scmd,
5152 struct pqi_io_request *io_request)
5153{
5154 u16 iu_length;
5155 int sg_count;
5156 bool chained;
5157 unsigned int num_sg_in_iu;
5158 struct scatterlist *sg;
5159 struct pqi_sg_descriptor *sg_descriptor;
5160
5161 sg_count = scsi_dma_map(scmd);
5162 if (sg_count < 0)
5163 return sg_count;
5164
5165 iu_length = offsetof(struct pqi_aio_r56_path_request, sg_descriptors) -
5166 PQI_REQUEST_HEADER_LENGTH;
5167 num_sg_in_iu = 0;
5168
5169 if (sg_count != 0) {
5170 sg = scsi_sglist(scmd);
5171 sg_descriptor = request->sg_descriptors;
5172
5173 num_sg_in_iu = pqi_build_sg_list(sg_descriptor, sg, sg_count, io_request,
5174 ctrl_info->max_sg_per_r56_iu, &chained);
5175
5176 request->partial = chained;
5177 iu_length += num_sg_in_iu * sizeof(*sg_descriptor);
5178 }
5179
5180 put_unaligned_le16(iu_length, &request->header.iu_length);
5181 request->num_sg_descriptors = num_sg_in_iu;
5182
5183 return 0;
5184}
5185
6c223761
KB		 5186static int pqi_build_aio_sg_list(struct pqi_ctrl_info *ctrl_info,
5187 struct pqi_aio_path_request *request, struct scsi_cmnd *scmd,
5188 struct pqi_io_request *io_request)
5189{
6c223761
KB		 5190 u16 iu_length;
5191 int sg_count;
a60eec02
KB		 5192 bool chained;
5193 unsigned int num_sg_in_iu;
6c223761
KB		 5194 struct scatterlist *sg;
5195 struct pqi_sg_descriptor *sg_descriptor;
5196
5197 sg_count = scsi_dma_map(scmd);
5198 if (sg_count < 0)
5199 return sg_count;
a60eec02
KB		 5200
5201 iu_length = offsetof(struct pqi_aio_path_request, sg_descriptors) -
5202 PQI_REQUEST_HEADER_LENGTH;
5203 num_sg_in_iu = 0;
5204
6c223761
KB		 5205 if (sg_count == 0)
5206 goto out;
5207
a60eec02
KB		 5208 sg = scsi_sglist(scmd);
5209 sg_descriptor = request->sg_descriptors;
a60eec02 5210
1a22bc4b
DB		 5211 num_sg_in_iu = pqi_build_sg_list(sg_descriptor, sg, sg_count, io_request,
5212 ctrl_info->max_sg_per_iu, &chained);
6c223761 5213
a60eec02 5214 request->partial = chained;
6c223761 5215 iu_length += num_sg_in_iu * sizeof(*sg_descriptor);
a60eec02
KB		 5216
5217out:
6c223761
KB		 5218 put_unaligned_le16(iu_length, &request->header.iu_length);
5219 request->num_sg_descriptors = num_sg_in_iu;
5220
5221 return 0;
5222}
5223
5224static void pqi_raid_io_complete(struct pqi_io_request *io_request,
5225 void *context)
5226{
5227 struct scsi_cmnd *scmd;
5228
5229 scmd = io_request->scmd;
5230 pqi_free_io_request(io_request);
5231 scsi_dma_unmap(scmd);
5232 pqi_scsi_done(scmd);
5233}
5234
376fb880
KB		 5235static int pqi_raid_submit_scsi_cmd_with_io_request(
5236 struct pqi_ctrl_info *ctrl_info, struct pqi_io_request *io_request,
6c223761
KB		 5237 struct pqi_scsi_dev *device, struct scsi_cmnd *scmd,
5238 struct pqi_queue_group *queue_group)
5239{
5240 int rc;
5241 size_t cdb_length;
6c223761
KB		 5242 struct pqi_raid_path_request *request;
5243
6c223761
KB		 5244 io_request->io_complete_callback = pqi_raid_io_complete;
5245 io_request->scmd = scmd;
5246
6c223761 5247 request = io_request->iu;
583891c9 5248 memset(request, 0, offsetof(struct pqi_raid_path_request, sg_descriptors));
6c223761
KB		 5249
5250 request->header.iu_type = PQI_REQUEST_IU_RAID_PATH_IO;
5251 put_unaligned_le32(scsi_bufflen(scmd), &request->buffer_length);
5252 request->task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
5253 put_unaligned_le16(io_request->index, &request->request_id);
5254 request->error_index = request->request_id;
583891c9 5255 memcpy(request->lun_number, device->scsi3addr, sizeof(request->lun_number));
6c223761
KB		 5256
5257 cdb_length = min_t(size_t, scmd->cmd_len, sizeof(request->cdb));
5258 memcpy(request->cdb, scmd->cmnd, cdb_length);
5259
5260 switch (cdb_length) {
5261 case 6:
5262 case 10:
5263 case 12:
5264 case 16:
583891c9 5265 request->additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_0;
6c223761
KB		 5266 break;
5267 case 20:
583891c9 5268 request->additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_4;
6c223761
KB		 5269 break;
5270 case 24:
583891c9 5271 request->additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_8;
6c223761
KB		 5272 break;
5273 case 28:
583891c9 5274 request->additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_12;
6c223761
KB		 5275 break;
5276 case 32:
5277 default:
583891c9 5278 request->additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_16;
6c223761
KB		 5279 break;
5280 }
5281
5282 switch (scmd->sc_data_direction) {
5283 case DMA_TO_DEVICE:
5284 request->data_direction = SOP_READ_FLAG;
5285 break;
5286 case DMA_FROM_DEVICE:
5287 request->data_direction = SOP_WRITE_FLAG;
5288 break;
5289 case DMA_NONE:
5290 request->data_direction = SOP_NO_DIRECTION_FLAG;
5291 break;
5292 case DMA_BIDIRECTIONAL:
5293 request->data_direction = SOP_BIDIRECTIONAL;
5294 break;
5295 default:
5296 dev_err(&ctrl_info->pci_dev->dev,
5297 "unknown data direction: %d\n",
5298 scmd->sc_data_direction);
6c223761
KB		 5299 break;
5300 }
5301
5302 rc = pqi_build_raid_sg_list(ctrl_info, request, scmd, io_request);
5303 if (rc) {
5304 pqi_free_io_request(io_request);
5305 return SCSI_MLQUEUE_HOST_BUSY;
5306 }
5307
5308 pqi_start_io(ctrl_info, queue_group, RAID_PATH, io_request);
5309
5310 return 0;
5311}
5312
376fb880
KB		 5313static inline int pqi_raid_submit_scsi_cmd(struct pqi_ctrl_info *ctrl_info,
5314 struct pqi_scsi_dev *device, struct scsi_cmnd *scmd,
5315 struct pqi_queue_group *queue_group)
5316{
5317 struct pqi_io_request *io_request;
5318
5319 io_request = pqi_alloc_io_request(ctrl_info);
5320
5321 return pqi_raid_submit_scsi_cmd_with_io_request(ctrl_info, io_request,
5322 device, scmd, queue_group);
5323}
5324
376fb880
KB		 5325static bool pqi_raid_bypass_retry_needed(struct pqi_io_request *io_request)
5326{
5327 struct scsi_cmnd *scmd;
03b288cf 5328 struct pqi_scsi_dev *device;
376fb880
KB		 5329 struct pqi_ctrl_info *ctrl_info;
5330
5331 if (!io_request->raid_bypass)
5332 return false;
5333
5334 scmd = io_request->scmd;
5335 if ((scmd->result & 0xff) == SAM_STAT_GOOD)
5336 return false;
5337 if (host_byte(scmd->result) == DID_NO_CONNECT)
5338 return false;
5339
03b288cf 5340 device = scmd->device->hostdata;
5be9db06 5341 if (pqi_device_offline(device) || pqi_device_in_remove(device))
03b288cf
KB		 5342 return false;
5343
376fb880
KB		 5344 ctrl_info = shost_to_hba(scmd->device->host);
5345 if (pqi_ctrl_offline(ctrl_info))
5346 return false;
5347
5348 return true;
5349}
5350
6c223761
KB		 5351static void pqi_aio_io_complete(struct pqi_io_request *io_request,
5352 void *context)
5353{
5354 struct scsi_cmnd *scmd;
5355
5356 scmd = io_request->scmd;
5357 scsi_dma_unmap(scmd);
5be9db06 5358 if (io_request->status == -EAGAIN || pqi_raid_bypass_retry_needed(io_request)) {
6c223761 5359 set_host_byte(scmd, DID_IMM_RETRY);
5be9db06 5360 scmd->SCp.this_residual++;
376fb880 5361 }
5be9db06 5362
6c223761
KB		 5363 pqi_free_io_request(io_request);
5364 pqi_scsi_done(scmd);
5365}
5366
5367static inline int pqi_aio_submit_scsi_cmd(struct pqi_ctrl_info *ctrl_info,
5368 struct pqi_scsi_dev *device, struct scsi_cmnd *scmd,
5369 struct pqi_queue_group *queue_group)
5370{
5371 return pqi_aio_submit_io(ctrl_info, scmd, device->aio_handle,
376fb880 5372 scmd->cmnd, scmd->cmd_len, queue_group, NULL, false);
6c223761
KB		 5373}
5374
5375static int pqi_aio_submit_io(struct pqi_ctrl_info *ctrl_info,
5376 struct scsi_cmnd *scmd, u32 aio_handle, u8 *cdb,
5377 unsigned int cdb_length, struct pqi_queue_group *queue_group,
376fb880 5378 struct pqi_encryption_info *encryption_info, bool raid_bypass)
6c223761
KB		 5379{
5380 int rc;
5381 struct pqi_io_request *io_request;
5382 struct pqi_aio_path_request *request;
5383
5384 io_request = pqi_alloc_io_request(ctrl_info);
5385 io_request->io_complete_callback = pqi_aio_io_complete;
5386 io_request->scmd = scmd;
376fb880 5387 io_request->raid_bypass = raid_bypass;
6c223761
KB		 5388
5389 request = io_request->iu;
583891c9 5390 memset(request, 0, offsetof(struct pqi_raid_path_request, sg_descriptors));
6c223761
KB		 5391
5392 request->header.iu_type = PQI_REQUEST_IU_AIO_PATH_IO;
5393 put_unaligned_le32(aio_handle, &request->nexus_id);
5394 put_unaligned_le32(scsi_bufflen(scmd), &request->buffer_length);
5395 request->task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
5396 put_unaligned_le16(io_request->index, &request->request_id);
5397 request->error_index = request->request_id;
5398 if (cdb_length > sizeof(request->cdb))
5399 cdb_length = sizeof(request->cdb);
5400 request->cdb_length = cdb_length;
5401 memcpy(request->cdb, cdb, cdb_length);
5402
5403 switch (scmd->sc_data_direction) {
5404 case DMA_TO_DEVICE:
5405 request->data_direction = SOP_READ_FLAG;
5406 break;
5407 case DMA_FROM_DEVICE:
5408 request->data_direction = SOP_WRITE_FLAG;
5409 break;
5410 case DMA_NONE:
5411 request->data_direction = SOP_NO_DIRECTION_FLAG;
5412 break;
5413 case DMA_BIDIRECTIONAL:
5414 request->data_direction = SOP_BIDIRECTIONAL;
5415 break;
5416 default:
5417 dev_err(&ctrl_info->pci_dev->dev,
5418 "unknown data direction: %d\n",
5419 scmd->sc_data_direction);
6c223761
KB		 5420 break;
5421 }
5422
5423 if (encryption_info) {
5424 request->encryption_enable = true;
5425 put_unaligned_le16(encryption_info->data_encryption_key_index,
5426 &request->data_encryption_key_index);
5427 put_unaligned_le32(encryption_info->encrypt_tweak_lower,
5428 &request->encrypt_tweak_lower);
5429 put_unaligned_le32(encryption_info->encrypt_tweak_upper,
5430 &request->encrypt_tweak_upper);
5431 }
5432
5433 rc = pqi_build_aio_sg_list(ctrl_info, request, scmd, io_request);
5434 if (rc) {
5435 pqi_free_io_request(io_request);
5436 return SCSI_MLQUEUE_HOST_BUSY;
5437 }
5438
5439 pqi_start_io(ctrl_info, queue_group, AIO_PATH, io_request);
5440
5441 return 0;
5442}
5443
7a012c23
DB		 5444static int pqi_aio_submit_r1_write_io(struct pqi_ctrl_info *ctrl_info,
5445 struct scsi_cmnd *scmd, struct pqi_queue_group *queue_group,
5446 struct pqi_encryption_info *encryption_info, struct pqi_scsi_dev *device,
5447 struct pqi_scsi_dev_raid_map_data *rmd)
7a012c23
DB		 5448{
5449 int rc;
5450 struct pqi_io_request *io_request;
5451 struct pqi_aio_r1_path_request *r1_request;
5452
5453 io_request = pqi_alloc_io_request(ctrl_info);
5454 io_request->io_complete_callback = pqi_aio_io_complete;
5455 io_request->scmd = scmd;
5456 io_request->raid_bypass = true;
5457
5458 r1_request = io_request->iu;
5459 memset(r1_request, 0, offsetof(struct pqi_aio_r1_path_request, sg_descriptors));
5460
5461 r1_request->header.iu_type = PQI_REQUEST_IU_AIO_PATH_RAID1_IO;
7a012c23
DB		 5462 put_unaligned_le16(*(u16 *)device->scsi3addr & 0x3fff, &r1_request->volume_id);
5463 r1_request->num_drives = rmd->num_it_nexus_entries;
5464 put_unaligned_le32(rmd->it_nexus[0], &r1_request->it_nexus_1);
5465 put_unaligned_le32(rmd->it_nexus[1], &r1_request->it_nexus_2);
5466 if (rmd->num_it_nexus_entries == 3)
5467 put_unaligned_le32(rmd->it_nexus[2], &r1_request->it_nexus_3);
5468
5469 put_unaligned_le32(scsi_bufflen(scmd), &r1_request->data_length);
5470 r1_request->task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
5471 put_unaligned_le16(io_request->index, &r1_request->request_id);
5472 r1_request->error_index = r1_request->request_id;
5473 if (rmd->cdb_length > sizeof(r1_request->cdb))
5474 rmd->cdb_length = sizeof(r1_request->cdb);
5475 r1_request->cdb_length = rmd->cdb_length;
5476 memcpy(r1_request->cdb, rmd->cdb, rmd->cdb_length);
5477
5478 /* The direction is always write. */
5479 r1_request->data_direction = SOP_READ_FLAG;
5480
5481 if (encryption_info) {
5482 r1_request->encryption_enable = true;
5483 put_unaligned_le16(encryption_info->data_encryption_key_index,
5484 &r1_request->data_encryption_key_index);
5485 put_unaligned_le32(encryption_info->encrypt_tweak_lower,
5486 &r1_request->encrypt_tweak_lower);
5487 put_unaligned_le32(encryption_info->encrypt_tweak_upper,
5488 &r1_request->encrypt_tweak_upper);
5489 }
5490
5491 rc = pqi_build_aio_r1_sg_list(ctrl_info, r1_request, scmd, io_request);
5492 if (rc) {
5493 pqi_free_io_request(io_request);
5494 return SCSI_MLQUEUE_HOST_BUSY;
5495 }
5496
5497 pqi_start_io(ctrl_info, queue_group, AIO_PATH, io_request);
5498
5499 return 0;
5500}
5501
6702d2c4
DB		 5502static int pqi_aio_submit_r56_write_io(struct pqi_ctrl_info *ctrl_info,
5503 struct scsi_cmnd *scmd, struct pqi_queue_group *queue_group,
5504 struct pqi_encryption_info *encryption_info, struct pqi_scsi_dev *device,
5505 struct pqi_scsi_dev_raid_map_data *rmd)
5506{
5507 int rc;
5508 struct pqi_io_request *io_request;
5509 struct pqi_aio_r56_path_request *r56_request;
5510
5511 io_request = pqi_alloc_io_request(ctrl_info);
5512 io_request->io_complete_callback = pqi_aio_io_complete;
5513 io_request->scmd = scmd;
5514 io_request->raid_bypass = true;
5515
5516 r56_request = io_request->iu;
5517 memset(r56_request, 0, offsetof(struct pqi_aio_r56_path_request, sg_descriptors));
5518
5519 if (device->raid_level == SA_RAID_5 || device->raid_level == SA_RAID_51)
5520 r56_request->header.iu_type = PQI_REQUEST_IU_AIO_PATH_RAID5_IO;
5521 else
5522 r56_request->header.iu_type = PQI_REQUEST_IU_AIO_PATH_RAID6_IO;
5523
5524 put_unaligned_le16(*(u16 *)device->scsi3addr & 0x3fff, &r56_request->volume_id);
5525 put_unaligned_le32(rmd->aio_handle, &r56_request->data_it_nexus);
5526 put_unaligned_le32(rmd->p_parity_it_nexus, &r56_request->p_parity_it_nexus);
5527 if (rmd->raid_level == SA_RAID_6) {
5528 put_unaligned_le32(rmd->q_parity_it_nexus, &r56_request->q_parity_it_nexus);
5529 r56_request->xor_multiplier = rmd->xor_mult;
5530 }
5531 put_unaligned_le32(scsi_bufflen(scmd), &r56_request->data_length);
5532 r56_request->task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
5533 put_unaligned_le64(rmd->row, &r56_request->row);
5534
5535 put_unaligned_le16(io_request->index, &r56_request->request_id);
5536 r56_request->error_index = r56_request->request_id;
5537
5538 if (rmd->cdb_length > sizeof(r56_request->cdb))
5539 rmd->cdb_length = sizeof(r56_request->cdb);
5540 r56_request->cdb_length = rmd->cdb_length;
5541 memcpy(r56_request->cdb, rmd->cdb, rmd->cdb_length);
5542
5543 /* The direction is always write. */
5544 r56_request->data_direction = SOP_READ_FLAG;
5545
5546 if (encryption_info) {
5547 r56_request->encryption_enable = true;
5548 put_unaligned_le16(encryption_info->data_encryption_key_index,
5549 &r56_request->data_encryption_key_index);
5550 put_unaligned_le32(encryption_info->encrypt_tweak_lower,
5551 &r56_request->encrypt_tweak_lower);
5552 put_unaligned_le32(encryption_info->encrypt_tweak_upper,
5553 &r56_request->encrypt_tweak_upper);
5554 }
5555
5556 rc = pqi_build_aio_r56_sg_list(ctrl_info, r56_request, scmd, io_request);
5557 if (rc) {
5558 pqi_free_io_request(io_request);
5559 return SCSI_MLQUEUE_HOST_BUSY;
5560 }
5561
5562 pqi_start_io(ctrl_info, queue_group, AIO_PATH, io_request);
5563
5564 return 0;
5565}
5566
061ef06a
KB		 5567static inline u16 pqi_get_hw_queue(struct pqi_ctrl_info *ctrl_info,
5568 struct scsi_cmnd *scmd)
5569{
5570 u16 hw_queue;
5571
5572 hw_queue = blk_mq_unique_tag_to_hwq(blk_mq_unique_tag(scmd->request));
5573 if (hw_queue > ctrl_info->max_hw_queue_index)
5574 hw_queue = 0;
5575
5576 return hw_queue;
5577}
5578
5be9db06
KB		 5579static inline bool pqi_is_bypass_eligible_request(struct scsi_cmnd *scmd)
5580{
5581 if (blk_rq_is_passthrough(scmd->request))
5582 return false;
5583
5584 return scmd->SCp.this_residual == 0;
5585}
5586
7561a7e4
KB		 5587/*
5588 * This function gets called just before we hand the completed SCSI request
5589 * back to the SML.
5590 */
5591
5592void pqi_prep_for_scsi_done(struct scsi_cmnd *scmd)
5593{
5594 struct pqi_scsi_dev *device;
5595
1e46731e
MR		 5596 if (!scmd->device) {
5597 set_host_byte(scmd, DID_NO_CONNECT);
5598 return;
5599 }
5600
7561a7e4 5601 device = scmd->device->hostdata;
1e46731e
MR		 5602 if (!device) {
5603 set_host_byte(scmd, DID_NO_CONNECT);
5604 return;
5605 }
5606
7561a7e4
KB		 5607 atomic_dec(&device->scsi_cmds_outstanding);
5608}
5609
c7ffedb3 5610static bool pqi_is_parity_write_stream(struct pqi_ctrl_info *ctrl_info,
7d81d2b8 5611 struct scsi_cmnd *scmd)
c7ffedb3
DB		 5612{
5613 u32 oldest_jiffies;
5614 u8 lru_index;
5615 int i;
5616 int rc;
5617 struct pqi_scsi_dev *device;
5618 struct pqi_stream_data *pqi_stream_data;
5619 struct pqi_scsi_dev_raid_map_data rmd;
5620
5621 if (!ctrl_info->enable_stream_detection)
5622 return false;
5623
5624 rc = pqi_get_aio_lba_and_block_count(scmd, &rmd);
5625 if (rc)
5626 return false;
5627
5628 /* Check writes only. */
5629 if (!rmd.is_write)
5630 return false;
5631
5632 device = scmd->device->hostdata;
5633
5634 /* Check for RAID 5/6 streams. */
5635 if (device->raid_level != SA_RAID_5 && device->raid_level != SA_RAID_6)
5636 return false;
5637
5638 /*
5639 * If controller does not support AIO RAID{5,6} writes, need to send
5640 * requests down non-AIO path.
5641 */
5642 if ((device->raid_level == SA_RAID_5 && !ctrl_info->enable_r5_writes) ||
5643 (device->raid_level == SA_RAID_6 && !ctrl_info->enable_r6_writes))
5644 return true;
5645
5646 lru_index = 0;
5647 oldest_jiffies = INT_MAX;
5648 for (i = 0; i < NUM_STREAMS_PER_LUN; i++) {
5649 pqi_stream_data = &device->stream_data[i];
5650 /*
5651 * Check for adjacent request or request is within
5652 * the previous request.
5653 */
5654 if ((pqi_stream_data->next_lba &&
5655 rmd.first_block >= pqi_stream_data->next_lba) &&
5656 rmd.first_block <= pqi_stream_data->next_lba +
5657 rmd.block_cnt) {
5658 pqi_stream_data->next_lba = rmd.first_block +
5659 rmd.block_cnt;
5660 pqi_stream_data->last_accessed = jiffies;
5661 return true;
5662 }
5663
5664 /* unused entry */
5665 if (pqi_stream_data->last_accessed == 0) {
5666 lru_index = i;
5667 break;
5668 }
5669
5670 /* Find entry with oldest last accessed time. */
5671 if (pqi_stream_data->last_accessed <= oldest_jiffies) {
5672 oldest_jiffies = pqi_stream_data->last_accessed;
5673 lru_index = i;
5674 }
5675 }
5676
5677 /* Set LRU entry. */
5678 pqi_stream_data = &device->stream_data[lru_index];
5679 pqi_stream_data->last_accessed = jiffies;
5680 pqi_stream_data->next_lba = rmd.first_block + rmd.block_cnt;
5681
5682 return false;
5683}
5684
5685static int pqi_scsi_queue_command(struct Scsi_Host *shost, struct scsi_cmnd *scmd)
6c223761
KB		 5686{
5687 int rc;
5688 struct pqi_ctrl_info *ctrl_info;
5689 struct pqi_scsi_dev *device;
061ef06a 5690 u16 hw_queue;
6c223761
KB		 5691 struct pqi_queue_group *queue_group;
5692 bool raid_bypassed;
5693
5694 device = scmd->device->hostdata;
6c223761 5695
1e46731e
MR		 5696 if (!device) {
5697 set_host_byte(scmd, DID_NO_CONNECT);
5698 pqi_scsi_done(scmd);
5699 return 0;
5700 }
5701
7561a7e4
KB		 5702 atomic_inc(&device->scsi_cmds_outstanding);
5703
583891c9
KB		 5704 ctrl_info = shost_to_hba(shost);
5705
1bdf6e93 5706 if (pqi_ctrl_offline(ctrl_info) || pqi_device_in_remove(device)) {
6c223761
KB		 5707 set_host_byte(scmd, DID_NO_CONNECT);
5708 pqi_scsi_done(scmd);
5709 return 0;
5710 }
5711
5be9db06 5712 if (pqi_ctrl_blocked(ctrl_info)) {
7561a7e4
KB		 5713 rc = SCSI_MLQUEUE_HOST_BUSY;
5714 goto out;
5715 }
5716
7d81d2b8
KB		 5717 /*
5718 * This is necessary because the SML doesn't zero out this field during
5719 * error recovery.
5720 */
5721 scmd->result = 0;
5722
061ef06a
KB		 5723 hw_queue = pqi_get_hw_queue(ctrl_info, scmd);
5724 queue_group = &ctrl_info->queue_groups[hw_queue];
6c223761
KB		 5725
5726 if (pqi_is_logical_device(device)) {
5727 raid_bypassed = false;
588a63fe 5728 if (device->raid_bypass_enabled &&
5be9db06
KB		 5729 pqi_is_bypass_eligible_request(scmd) &&
5730 !pqi_is_parity_write_stream(ctrl_info, scmd)) {
5731 rc = pqi_raid_bypass_submit_scsi_cmd(ctrl_info, device, scmd, queue_group);
8b664fef 5732 if (rc == 0 || rc == SCSI_MLQUEUE_HOST_BUSY) {
376fb880 5733 raid_bypassed = true;
8b664fef
KB		 5734 atomic_inc(&device->raid_bypass_cnt);
5735 }
6c223761
KB		 5736 }
5737 if (!raid_bypassed)
8b664fef 5738 rc = pqi_raid_submit_scsi_cmd(ctrl_info, device, scmd, queue_group);
6c223761
KB		 5739 } else {
5740 if (device->aio_enabled)
8b664fef 5741 rc = pqi_aio_submit_scsi_cmd(ctrl_info, device, scmd, queue_group);
6c223761 5742 else
8b664fef 5743 rc = pqi_raid_submit_scsi_cmd(ctrl_info, device, scmd, queue_group);
6c223761
KB		 5744 }
5745
7561a7e4 5746out:
7561a7e4
KB		 5747 if (rc)
5748 atomic_dec(&device->scsi_cmds_outstanding);
5749
6c223761
KB		 5750 return rc;
5751}
5752
7561a7e4
KB		 5753static int pqi_wait_until_queued_io_drained(struct pqi_ctrl_info *ctrl_info,
5754 struct pqi_queue_group *queue_group)
5755{
5756 unsigned int path;
5757 unsigned long flags;
5758 bool list_is_empty;
5759
5760 for (path = 0; path < 2; path++) {
5761 while (1) {
5762 spin_lock_irqsave(
5763 &queue_group->submit_lock[path], flags);
5764 list_is_empty =
5765 list_empty(&queue_group->request_list[path]);
5766 spin_unlock_irqrestore(
5767 &queue_group->submit_lock[path], flags);
5768 if (list_is_empty)
5769 break;
5770 pqi_check_ctrl_health(ctrl_info);
5771 if (pqi_ctrl_offline(ctrl_info))
5772 return -ENXIO;
5773 usleep_range(1000, 2000);
5774 }
5775 }
5776
5777 return 0;
5778}
5779
5780static int pqi_wait_until_inbound_queues_empty(struct pqi_ctrl_info *ctrl_info)
5781{
5782 int rc;
5783 unsigned int i;
5784 unsigned int path;
5785 struct pqi_queue_group *queue_group;
5786 pqi_index_t iq_pi;
5787 pqi_index_t iq_ci;
5788
5789 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
5790 queue_group = &ctrl_info->queue_groups[i];
5791
5792 rc = pqi_wait_until_queued_io_drained(ctrl_info, queue_group);
5793 if (rc)
5794 return rc;
5795
5796 for (path = 0; path < 2; path++) {
5797 iq_pi = queue_group->iq_pi_copy[path];
5798
5799 while (1) {
dac12fbc 5800 iq_ci = readl(queue_group->iq_ci[path]);
7561a7e4
KB		 5801 if (iq_ci == iq_pi)
5802 break;
5803 pqi_check_ctrl_health(ctrl_info);
5804 if (pqi_ctrl_offline(ctrl_info))
5805 return -ENXIO;
5806 usleep_range(1000, 2000);
5807 }
5808 }
5809 }
5810
5811 return 0;
5812}
5813
5814static void pqi_fail_io_queued_for_device(struct pqi_ctrl_info *ctrl_info,
5815 struct pqi_scsi_dev *device)
5816{
5817 unsigned int i;
5818 unsigned int path;
5819 struct pqi_queue_group *queue_group;
5820 unsigned long flags;
5821 struct pqi_io_request *io_request;
5822 struct pqi_io_request *next;
5823 struct scsi_cmnd *scmd;
5824 struct pqi_scsi_dev *scsi_device;
5825
5826 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
5827 queue_group = &ctrl_info->queue_groups[i];
5828
5829 for (path = 0; path < 2; path++) {
5830 spin_lock_irqsave(
5831 &queue_group->submit_lock[path], flags);
5832
5833 list_for_each_entry_safe(io_request, next,
5834 &queue_group->request_list[path],
5835 request_list_entry) {
583891c9 5836
7561a7e4
KB		 5837 scmd = io_request->scmd;
5838 if (!scmd)
5839 continue;
5840
5841 scsi_device = scmd->device->hostdata;
5842 if (scsi_device != device)
5843 continue;
5844
5845 list_del(&io_request->request_list_entry);
5846 set_host_byte(scmd, DID_RESET);
b622a601
MB		 5847 pqi_free_io_request(io_request);
5848 scsi_dma_unmap(scmd);
7561a7e4
KB		 5849 pqi_scsi_done(scmd);
5850 }
5851
5852 spin_unlock_irqrestore(
5853 &queue_group->submit_lock[path], flags);
5854 }
5855 }
5856}
5857
18ff5f08 5858#define PQI_PENDING_IO_WARNING_TIMEOUT_SECS 10
4fd22c13 5859
061ef06a 5860static int pqi_device_wait_for_pending_io(struct pqi_ctrl_info *ctrl_info,
18ff5f08 5861 struct pqi_scsi_dev *device, unsigned long timeout_msecs)
061ef06a 5862{
18ff5f08
KB		 5863 int cmds_outstanding;
5864 unsigned long start_jiffies;
5865 unsigned long warning_timeout;
5866 unsigned long msecs_waiting;
1e46731e 5867
18ff5f08
KB		 5868 start_jiffies = jiffies;
5869 warning_timeout = (PQI_PENDING_IO_WARNING_TIMEOUT_SECS * PQI_HZ) + start_jiffies;
1e46731e 5870
18ff5f08 5871 while ((cmds_outstanding = atomic_read(&device->scsi_cmds_outstanding)) > 0) {
061ef06a
KB		 5872 pqi_check_ctrl_health(ctrl_info);
5873 if (pqi_ctrl_offline(ctrl_info))
5874 return -ENXIO;
18ff5f08
KB		 5875 msecs_waiting = jiffies_to_msecs(jiffies - start_jiffies);
5876 if (msecs_waiting > timeout_msecs) {
5877 dev_err(&ctrl_info->pci_dev->dev,
5878 "scsi %d:%d:%d:%d: timed out after %lu seconds waiting for %d outstanding command(s)\n",
5879 ctrl_info->scsi_host->host_no, device->bus, device->target,
5880 device->lun, msecs_waiting / 1000, cmds_outstanding);
5881 return -ETIMEDOUT;
061ef06a 5882 }
18ff5f08
KB		 5883 if (time_after(jiffies, warning_timeout)) {
5884 dev_warn(&ctrl_info->pci_dev->dev,
5885 "scsi %d:%d:%d:%d: waiting %lu seconds for %d outstanding command(s)\n",
5886 ctrl_info->scsi_host->host_no, device->bus, device->target,
5887 device->lun, msecs_waiting / 1000, cmds_outstanding);
5888 warning_timeout = (PQI_PENDING_IO_WARNING_TIMEOUT_SECS * PQI_HZ) + jiffies;
4fd22c13 5889 }
061ef06a
KB		 5890 usleep_range(1000, 2000);
5891 }
5892
5893 return 0;
5894}
5895
14bb215d
KB		 5896static void pqi_lun_reset_complete(struct pqi_io_request *io_request,
5897 void *context)
6c223761 5898{
14bb215d 5899 struct completion *waiting = context;
6c223761 5900
14bb215d
KB		 5901 complete(waiting);
5902}
6c223761 5903
c2922f17 5904#define PQI_LUN_RESET_POLL_COMPLETION_SECS 10
14bb215d
KB		 5905
5906static int pqi_wait_for_lun_reset_completion(struct pqi_ctrl_info *ctrl_info,
5907 struct pqi_scsi_dev *device, struct completion *wait)
5908{
5909 int rc;
18ff5f08
KB		 5910 unsigned int wait_secs;
5911
5912 wait_secs = 0;
14bb215d
KB		 5913
5914 while (1) {
5915 if (wait_for_completion_io_timeout(wait,
c2922f17 5916 PQI_LUN_RESET_POLL_COMPLETION_SECS * PQI_HZ)) {
14bb215d
KB		 5917 rc = 0;
5918 break;
6c223761
KB		 5919 }
5920
14bb215d
KB		 5921 pqi_check_ctrl_health(ctrl_info);
5922 if (pqi_ctrl_offline(ctrl_info)) {
4e8415e3 5923 rc = -ENXIO;
14bb215d
KB		 5924 break;
5925 }
18ff5f08
KB		 5926
5927 wait_secs += PQI_LUN_RESET_POLL_COMPLETION_SECS;
5928
5929 dev_warn(&ctrl_info->pci_dev->dev,
5930 "scsi %d:%d:%d:%d: waiting %u seconds for LUN reset to complete\n",
5931 ctrl_info->scsi_host->host_no, device->bus, device->target, device->lun,
5932 wait_secs);
6c223761 5933 }
6c223761 5934
14bb215d 5935 return rc;
6c223761
KB		 5936}
5937
18ff5f08
KB		 5938#define PQI_LUN_RESET_FIRMWARE_TIMEOUT_SECS 30
5939
5940static int pqi_lun_reset(struct pqi_ctrl_info *ctrl_info, struct pqi_scsi_dev *device)
6c223761
KB		 5941{
5942 int rc;
5943 struct pqi_io_request *io_request;
5944 DECLARE_COMPLETION_ONSTACK(wait);
5945 struct pqi_task_management_request *request;
5946
6c223761 5947 io_request = pqi_alloc_io_request(ctrl_info);
14bb215d 5948 io_request->io_complete_callback = pqi_lun_reset_complete;
6c223761
KB		 5949 io_request->context = &wait;
5950
5951 request = io_request->iu;
5952 memset(request, 0, sizeof(*request));
5953
5954 request->header.iu_type = PQI_REQUEST_IU_TASK_MANAGEMENT;
5955 put_unaligned_le16(sizeof(*request) - PQI_REQUEST_HEADER_LENGTH,
5956 &request->header.iu_length);
5957 put_unaligned_le16(io_request->index, &request->request_id);
5958 memcpy(request->lun_number, device->scsi3addr,
5959 sizeof(request->lun_number));
5960 request->task_management_function = SOP_TASK_MANAGEMENT_LUN_RESET;
c2922f17 5961 if (ctrl_info->tmf_iu_timeout_supported)
18ff5f08 5962 put_unaligned_le16(PQI_LUN_RESET_FIRMWARE_TIMEOUT_SECS, &request->timeout);
6c223761 5963
583891c9 5964 pqi_start_io(ctrl_info, &ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP], RAID_PATH,
6c223761
KB		 5965 io_request);
5966
14bb215d
KB		 5967 rc = pqi_wait_for_lun_reset_completion(ctrl_info, device, &wait);
5968 if (rc == 0)
6c223761 5969 rc = io_request->status;
6c223761
KB		 5970
5971 pqi_free_io_request(io_request);
6c223761
KB		 5972
5973 return rc;
5974}
5975
18ff5f08
KB		 5976#define PQI_LUN_RESET_RETRIES 3
5977#define PQI_LUN_RESET_RETRY_INTERVAL_MSECS (10 * 1000)
5978#define PQI_LUN_RESET_PENDING_IO_TIMEOUT_MSECS (10 * 60 * 1000)
5979#define PQI_LUN_RESET_FAILED_PENDING_IO_TIMEOUT_MSECS (2 * 60 * 1000)
6c223761 5980
18ff5f08 5981static int pqi_lun_reset_with_retries(struct pqi_ctrl_info *ctrl_info, struct pqi_scsi_dev *device)
6c223761 5982{
18ff5f08
KB		 5983 int reset_rc;
5984 int wait_rc;
3406384b 5985 unsigned int retries;
18ff5f08 5986 unsigned long timeout_msecs;
6c223761 5987
3406384b 5988 for (retries = 0;;) {
18ff5f08
KB		 5989 reset_rc = pqi_lun_reset(ctrl_info, device);
5990 if (reset_rc == 0 || ++retries > PQI_LUN_RESET_RETRIES)
3406384b
MR		 5991 break;
5992 msleep(PQI_LUN_RESET_RETRY_INTERVAL_MSECS);
5993 }
429fab70 5994
18ff5f08
KB		 5995 timeout_msecs = reset_rc ? PQI_LUN_RESET_FAILED_PENDING_IO_TIMEOUT_MSECS :
5996 PQI_LUN_RESET_PENDING_IO_TIMEOUT_MSECS;
4fd22c13 5997
18ff5f08
KB		 5998 wait_rc = pqi_device_wait_for_pending_io(ctrl_info, device, timeout_msecs);
5999 if (wait_rc && reset_rc == 0)
6000 reset_rc = wait_rc;
6c223761 6001
18ff5f08 6002 return reset_rc == 0 ? SUCCESS : FAILED;
6c223761
KB		 6003}
6004
4fd22c13
MR		 6005static int pqi_device_reset(struct pqi_ctrl_info *ctrl_info,
6006 struct pqi_scsi_dev *device)
6007{
6008 int rc;
6009
4fd22c13
MR		 6010 pqi_ctrl_block_requests(ctrl_info);
6011 pqi_ctrl_wait_until_quiesced(ctrl_info);
6012 pqi_fail_io_queued_for_device(ctrl_info, device);
6013 rc = pqi_wait_until_inbound_queues_empty(ctrl_info);
4fd22c13
MR		 6014 if (rc)
6015 rc = FAILED;
6016 else
37f33181
KB		 6017 rc = pqi_lun_reset_with_retries(ctrl_info, device);
6018 pqi_ctrl_unblock_requests(ctrl_info);
429fab70 6019
4fd22c13
MR		 6020 return rc;
6021}
6022
6c223761
KB		 6023static int pqi_eh_device_reset_handler(struct scsi_cmnd *scmd)
6024{
6025 int rc;
7561a7e4 6026 struct Scsi_Host *shost;
6c223761
KB		 6027 struct pqi_ctrl_info *ctrl_info;
6028 struct pqi_scsi_dev *device;
6029
7561a7e4
KB		 6030 shost = scmd->device->host;
6031 ctrl_info = shost_to_hba(shost);
6c223761
KB		 6032 device = scmd->device->hostdata;
6033
37f33181
KB		 6034 mutex_lock(&ctrl_info->lun_reset_mutex);
6035
6c223761
KB		 6036 dev_err(&ctrl_info->pci_dev->dev,
6037 "resetting scsi %d:%d:%d:%d\n",
7561a7e4 6038 shost->host_no, device->bus, device->target, device->lun);
6c223761 6039
7561a7e4 6040 pqi_check_ctrl_health(ctrl_info);
37f33181 6041 if (pqi_ctrl_offline(ctrl_info))
7561a7e4 6042 rc = FAILED;
37f33181
KB		 6043 else
6044 rc = pqi_device_reset(ctrl_info, device);
429fab70 6045
6c223761
KB		 6046 dev_err(&ctrl_info->pci_dev->dev,
6047 "reset of scsi %d:%d:%d:%d: %s\n",
7561a7e4 6048 shost->host_no, device->bus, device->target, device->lun,
6c223761
KB		 6049 rc == SUCCESS ? "SUCCESS" : "FAILED");
6050
37f33181
KB		 6051 mutex_unlock(&ctrl_info->lun_reset_mutex);
6052
6c223761
KB		 6053 return rc;
6054}
6055
6056static int pqi_slave_alloc(struct scsi_device *sdev)
6057{
6058 struct pqi_scsi_dev *device;
6059 unsigned long flags;
6060 struct pqi_ctrl_info *ctrl_info;
6061 struct scsi_target *starget;
6062 struct sas_rphy *rphy;
6063
6064 ctrl_info = shost_to_hba(sdev->host);
6065
6066 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
6067
6068 if (sdev_channel(sdev) == PQI_PHYSICAL_DEVICE_BUS) {
6069 starget = scsi_target(sdev);
6070 rphy = target_to_rphy(starget);
6071 device = pqi_find_device_by_sas_rphy(ctrl_info, rphy);
6072 if (device) {
6073 device->target = sdev_id(sdev);
6074 device->lun = sdev->lun;
6075 device->target_lun_valid = true;
6076 }
6077 } else {
6078 device = pqi_find_scsi_dev(ctrl_info, sdev_channel(sdev),
6079 sdev_id(sdev), sdev->lun);
6080 }
6081
94086f5b 6082 if (device) {
6c223761
KB		 6083 sdev->hostdata = device;
6084 device->sdev = sdev;
6085 if (device->queue_depth) {
6086 device->advertised_queue_depth = device->queue_depth;
6087 scsi_change_queue_depth(sdev,
6088 device->advertised_queue_depth);
6089 }
99a12b48 6090 if (pqi_is_logical_device(device)) {
b6e2ef67 6091 pqi_disable_write_same(sdev);
99a12b48 6092 } else {
2b447f81 6093 sdev->allow_restart = 1;
99a12b48
KB		 6094 if (device->device_type == SA_DEVICE_TYPE_NVME)
6095 pqi_disable_write_same(sdev);
6096 }
6c223761
KB		 6097 }
6098
6099 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
6100
6101 return 0;
6102}
6103
52198226
CH		 6104static int pqi_map_queues(struct Scsi_Host *shost)
6105{
6106 struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);
6107
79d3fa9e 6108 return blk_mq_pci_map_queues(&shost->tag_set.map[HCTX_TYPE_DEFAULT],
ed76e329 6109 ctrl_info->pci_dev, 0);
52198226
CH		 6110}
6111
ce143793
KB		 6112static int pqi_slave_configure(struct scsi_device *sdev)
6113{
6114 struct pqi_scsi_dev *device;
6115
6116 device = sdev->hostdata;
6117 device->devtype = sdev->type;
6118
6119 return 0;
6120}
6121
4d15ad38
KB		 6122static void pqi_slave_destroy(struct scsi_device *sdev)
6123{
6124 unsigned long flags;
6125 struct pqi_scsi_dev *device;
6126 struct pqi_ctrl_info *ctrl_info;
6127
6128 ctrl_info = shost_to_hba(sdev->host);
6129
6130 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
6131
6132 device = sdev->hostdata;
6133 if (device) {
6134 sdev->hostdata = NULL;
6135 if (!list_empty(&device->scsi_device_list_entry))
6136 list_del(&device->scsi_device_list_entry);
6137 }
6138
6139 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
6140
6141 if (device) {
6142 pqi_dev_info(ctrl_info, "removed", device);
6143 pqi_free_device(device);
6144 }
6145}
6146
8b664fef 6147static int pqi_getpciinfo_ioctl(struct pqi_ctrl_info *ctrl_info, void __user *arg)
6c223761
KB		 6148{
6149 struct pci_dev *pci_dev;
6150 u32 subsystem_vendor;
6151 u32 subsystem_device;
6152 cciss_pci_info_struct pciinfo;
6153
6154 if (!arg)
6155 return -EINVAL;
6156
6157 pci_dev = ctrl_info->pci_dev;
6158
6159 pciinfo.domain = pci_domain_nr(pci_dev->bus);
6160 pciinfo.bus = pci_dev->bus->number;
6161 pciinfo.dev_fn = pci_dev->devfn;
6162 subsystem_vendor = pci_dev->subsystem_vendor;
6163 subsystem_device = pci_dev->subsystem_device;
8b664fef 6164 pciinfo.board_id = ((subsystem_device << 16) & 0xffff0000) | subsystem_vendor;
6c223761
KB		 6165
6166 if (copy_to_user(arg, &pciinfo, sizeof(pciinfo)))
6167 return -EFAULT;
6168
6169 return 0;
6170}
6171
6172static int pqi_getdrivver_ioctl(void __user *arg)
6173{
6174 u32 version;
6175
6176 if (!arg)
6177 return -EINVAL;
6178
6179 version = (DRIVER_MAJOR << 28) | (DRIVER_MINOR << 24) |
6180 (DRIVER_RELEASE << 16) | DRIVER_REVISION;
6181
6182 if (copy_to_user(arg, &version, sizeof(version)))
6183 return -EFAULT;
6184
6185 return 0;
6186}
6187
6188struct ciss_error_info {
6189 u8 scsi_status;
6190 int command_status;
6191 size_t sense_data_length;
6192};
6193
6194static void pqi_error_info_to_ciss(struct pqi_raid_error_info *pqi_error_info,
6195 struct ciss_error_info *ciss_error_info)
6196{
6197 int ciss_cmd_status;
6198 size_t sense_data_length;
6199
6200 switch (pqi_error_info->data_out_result) {
6201 case PQI_DATA_IN_OUT_GOOD:
6202 ciss_cmd_status = CISS_CMD_STATUS_SUCCESS;
6203 break;
6204 case PQI_DATA_IN_OUT_UNDERFLOW:
6205 ciss_cmd_status = CISS_CMD_STATUS_DATA_UNDERRUN;
6206 break;
6207 case PQI_DATA_IN_OUT_BUFFER_OVERFLOW:
6208 ciss_cmd_status = CISS_CMD_STATUS_DATA_OVERRUN;
6209 break;
6210 case PQI_DATA_IN_OUT_PROTOCOL_ERROR:
6211 case PQI_DATA_IN_OUT_BUFFER_ERROR:
6212 case PQI_DATA_IN_OUT_BUFFER_OVERFLOW_DESCRIPTOR_AREA:
6213 case PQI_DATA_IN_OUT_BUFFER_OVERFLOW_BRIDGE:
6214 case PQI_DATA_IN_OUT_ERROR:
6215 ciss_cmd_status = CISS_CMD_STATUS_PROTOCOL_ERROR;
6216 break;
6217 case PQI_DATA_IN_OUT_HARDWARE_ERROR:
6218 case PQI_DATA_IN_OUT_PCIE_FABRIC_ERROR:
6219 case PQI_DATA_IN_OUT_PCIE_COMPLETION_TIMEOUT:
6220 case PQI_DATA_IN_OUT_PCIE_COMPLETER_ABORT_RECEIVED:
6221 case PQI_DATA_IN_OUT_PCIE_UNSUPPORTED_REQUEST_RECEIVED:
6222 case PQI_DATA_IN_OUT_PCIE_ECRC_CHECK_FAILED:
6223 case PQI_DATA_IN_OUT_PCIE_UNSUPPORTED_REQUEST:
6224 case PQI_DATA_IN_OUT_PCIE_ACS_VIOLATION:
6225 case PQI_DATA_IN_OUT_PCIE_TLP_PREFIX_BLOCKED:
6226 case PQI_DATA_IN_OUT_PCIE_POISONED_MEMORY_READ:
6227 ciss_cmd_status = CISS_CMD_STATUS_HARDWARE_ERROR;
6228 break;
6229 case PQI_DATA_IN_OUT_UNSOLICITED_ABORT:
6230 ciss_cmd_status = CISS_CMD_STATUS_UNSOLICITED_ABORT;
6231 break;
6232 case PQI_DATA_IN_OUT_ABORTED:
6233 ciss_cmd_status = CISS_CMD_STATUS_ABORTED;
6234 break;
6235 case PQI_DATA_IN_OUT_TIMEOUT:
6236 ciss_cmd_status = CISS_CMD_STATUS_TIMEOUT;
6237 break;
6238 default:
6239 ciss_cmd_status = CISS_CMD_STATUS_TARGET_STATUS;
6240 break;
6241 }
6242
6243 sense_data_length =
6244 get_unaligned_le16(&pqi_error_info->sense_data_length);
6245 if (sense_data_length == 0)
6246 sense_data_length =
6247 get_unaligned_le16(&pqi_error_info->response_data_length);
6248 if (sense_data_length)
6249 if (sense_data_length > sizeof(pqi_error_info->data))
6250 sense_data_length = sizeof(pqi_error_info->data);
6251
6252 ciss_error_info->scsi_status = pqi_error_info->status;
6253 ciss_error_info->command_status = ciss_cmd_status;
6254 ciss_error_info->sense_data_length = sense_data_length;
6255}
6256
6257static int pqi_passthru_ioctl(struct pqi_ctrl_info *ctrl_info, void __user *arg)
6258{
6259 int rc;
6260 char *kernel_buffer = NULL;
6261 u16 iu_length;
6262 size_t sense_data_length;
6263 IOCTL_Command_struct iocommand;
6264 struct pqi_raid_path_request request;
6265 struct pqi_raid_error_info pqi_error_info;
6266 struct ciss_error_info ciss_error_info;
6267
6268 if (pqi_ctrl_offline(ctrl_info))
6269 return -ENXIO;
2790cd4d
KB		 6270 if (pqi_ofa_in_progress(ctrl_info) && pqi_ctrl_blocked(ctrl_info))
6271 return -EBUSY;
6c223761
KB		 6272 if (!arg)
6273 return -EINVAL;
6274 if (!capable(CAP_SYS_RAWIO))
6275 return -EPERM;
6276 if (copy_from_user(&iocommand, arg, sizeof(iocommand)))
6277 return -EFAULT;
6278 if (iocommand.buf_size < 1 &&
6279 iocommand.Request.Type.Direction != XFER_NONE)
6280 return -EINVAL;
6281 if (iocommand.Request.CDBLen > sizeof(request.cdb))
6282 return -EINVAL;
6283 if (iocommand.Request.Type.Type != TYPE_CMD)
6284 return -EINVAL;
6285
6286 switch (iocommand.Request.Type.Direction) {
6287 case XFER_NONE:
6288 case XFER_WRITE:
6289 case XFER_READ:
41555d54 6290 case XFER_READ | XFER_WRITE:
6c223761
KB		 6291 break;
6292 default:
6293 return -EINVAL;
6294 }
6295
6296 if (iocommand.buf_size > 0) {
6297 kernel_buffer = kmalloc(iocommand.buf_size, GFP_KERNEL);
6298 if (!kernel_buffer)
6299 return -ENOMEM;
6300 if (iocommand.Request.Type.Direction & XFER_WRITE) {
6301 if (copy_from_user(kernel_buffer, iocommand.buf,
6302 iocommand.buf_size)) {
6303 rc = -EFAULT;
6304 goto out;
6305 }
6306 } else {
6307 memset(kernel_buffer, 0, iocommand.buf_size);
6308 }
6309 }
6310
6311 memset(&request, 0, sizeof(request));
6312
6313 request.header.iu_type = PQI_REQUEST_IU_RAID_PATH_IO;
6314 iu_length = offsetof(struct pqi_raid_path_request, sg_descriptors) -
6315 PQI_REQUEST_HEADER_LENGTH;
6316 memcpy(request.lun_number, iocommand.LUN_info.LunAddrBytes,
6317 sizeof(request.lun_number));
6318 memcpy(request.cdb, iocommand.Request.CDB, iocommand.Request.CDBLen);
6319 request.additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_0;
6320
6321 switch (iocommand.Request.Type.Direction) {
6322 case XFER_NONE:
6323 request.data_direction = SOP_NO_DIRECTION_FLAG;
6324 break;
6325 case XFER_WRITE:
6326 request.data_direction = SOP_WRITE_FLAG;
6327 break;
6328 case XFER_READ:
6329 request.data_direction = SOP_READ_FLAG;
6330 break;
41555d54
KB		 6331 case XFER_READ | XFER_WRITE:
6332 request.data_direction = SOP_BIDIRECTIONAL;
6333 break;
6c223761
KB		 6334 }
6335
6336 request.task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
6337
6338 if (iocommand.buf_size > 0) {
6339 put_unaligned_le32(iocommand.buf_size, &request.buffer_length);
6340
6341 rc = pqi_map_single(ctrl_info->pci_dev,
6342 &request.sg_descriptors[0], kernel_buffer,
6917a9cc 6343 iocommand.buf_size, DMA_BIDIRECTIONAL);
6c223761
KB		 6344 if (rc)
6345 goto out;
6346
6347 iu_length += sizeof(request.sg_descriptors[0]);
6348 }
6349
6350 put_unaligned_le16(iu_length, &request.header.iu_length);
6351
21432010 6352 if (ctrl_info->raid_iu_timeout_supported)
6353 put_unaligned_le32(iocommand.Request.Timeout, &request.timeout);
6354
6c223761 6355 rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header,
ae0c189d 6356 PQI_SYNC_FLAGS_INTERRUPTABLE, &pqi_error_info);
6c223761
KB		 6357
6358 if (iocommand.buf_size > 0)
6359 pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1,
6917a9cc 6360 DMA_BIDIRECTIONAL);
6c223761
KB		 6361
6362 memset(&iocommand.error_info, 0, sizeof(iocommand.error_info));
6363
6364 if (rc == 0) {
6365 pqi_error_info_to_ciss(&pqi_error_info, &ciss_error_info);
6366 iocommand.error_info.ScsiStatus = ciss_error_info.scsi_status;
6367 iocommand.error_info.CommandStatus =
6368 ciss_error_info.command_status;
6369 sense_data_length = ciss_error_info.sense_data_length;
6370 if (sense_data_length) {
6371 if (sense_data_length >
6372 sizeof(iocommand.error_info.SenseInfo))
6373 sense_data_length =
6374 sizeof(iocommand.error_info.SenseInfo);
6375 memcpy(iocommand.error_info.SenseInfo,
6376 pqi_error_info.data, sense_data_length);
6377 iocommand.error_info.SenseLen = sense_data_length;
6378 }
6379 }
6380
6381 if (copy_to_user(arg, &iocommand, sizeof(iocommand))) {
6382 rc = -EFAULT;
6383 goto out;
6384 }
6385
6386 if (rc == 0 && iocommand.buf_size > 0 &&
6387 (iocommand.Request.Type.Direction & XFER_READ)) {
6388 if (copy_to_user(iocommand.buf, kernel_buffer,
6389 iocommand.buf_size)) {
6390 rc = -EFAULT;
6391 }
6392 }
6393
6394out:
6395 kfree(kernel_buffer);
6396
6397 return rc;
6398}
6399
6f4e626f
NC		 6400static int pqi_ioctl(struct scsi_device *sdev, unsigned int cmd,
6401 void __user *arg)
6c223761
KB		 6402{
6403 int rc;
6404 struct pqi_ctrl_info *ctrl_info;
6405
6406 ctrl_info = shost_to_hba(sdev->host);
6407
6408 switch (cmd) {
6409 case CCISS_DEREGDISK:
6410 case CCISS_REGNEWDISK:
6411 case CCISS_REGNEWD:
6412 rc = pqi_scan_scsi_devices(ctrl_info);
6413 break;
6414 case CCISS_GETPCIINFO:
6415 rc = pqi_getpciinfo_ioctl(ctrl_info, arg);
6416 break;
6417 case CCISS_GETDRIVVER:
6418 rc = pqi_getdrivver_ioctl(arg);
6419 break;
6420 case CCISS_PASSTHRU:
6421 rc = pqi_passthru_ioctl(ctrl_info, arg);
6422 break;
6423 default:
6424 rc = -EINVAL;
6425 break;
6426 }
6427
6428 return rc;
6429}
6430
6d90615f 6431static ssize_t pqi_firmware_version_show(struct device *dev,
6c223761
KB		 6432 struct device_attribute *attr, char *buffer)
6433{
6c223761
KB		 6434 struct Scsi_Host *shost;
6435 struct pqi_ctrl_info *ctrl_info;
6436
6437 shost = class_to_shost(dev);
6438 ctrl_info = shost_to_hba(shost);
6439
a4256252 6440 return scnprintf(buffer, PAGE_SIZE, "%s\n", ctrl_info->firmware_version);
6d90615f
MB		 6441}
6442
6443static ssize_t pqi_driver_version_show(struct device *dev,
6444 struct device_attribute *attr, char *buffer)
6445{
a4256252 6446 return scnprintf(buffer, PAGE_SIZE, "%s\n", DRIVER_VERSION BUILD_TIMESTAMP);
6d90615f 6447}
6c223761 6448
6d90615f
MB		 6449static ssize_t pqi_serial_number_show(struct device *dev,
6450 struct device_attribute *attr, char *buffer)
6451{
6452 struct Scsi_Host *shost;
6453 struct pqi_ctrl_info *ctrl_info;
6454
6455 shost = class_to_shost(dev);
6456 ctrl_info = shost_to_hba(shost);
6457
a4256252 6458 return scnprintf(buffer, PAGE_SIZE, "%s\n", ctrl_info->serial_number);
6d90615f
MB		 6459}
6460
6461static ssize_t pqi_model_show(struct device *dev,
6462 struct device_attribute *attr, char *buffer)
6463{
6464 struct Scsi_Host *shost;
6465 struct pqi_ctrl_info *ctrl_info;
6466
6467 shost = class_to_shost(dev);
6468 ctrl_info = shost_to_hba(shost);
6469
a4256252 6470 return scnprintf(buffer, PAGE_SIZE, "%s\n", ctrl_info->model);
6d90615f
MB		 6471}
6472
6473static ssize_t pqi_vendor_show(struct device *dev,
6474 struct device_attribute *attr, char *buffer)
6475{
6476 struct Scsi_Host *shost;
6477 struct pqi_ctrl_info *ctrl_info;
6478
6479 shost = class_to_shost(dev);
6480 ctrl_info = shost_to_hba(shost);
6481
a4256252 6482 return scnprintf(buffer, PAGE_SIZE, "%s\n", ctrl_info->vendor);
6c223761
KB		 6483}
6484
6485static ssize_t pqi_host_rescan_store(struct device *dev,
6486 struct device_attribute *attr, const char *buffer, size_t count)
6487{
6488 struct Scsi_Host *shost = class_to_shost(dev);
6489
6490 pqi_scan_start(shost);
6491
6492 return count;
6493}
6494
3c50976f
KB		 6495static ssize_t pqi_lockup_action_show(struct device *dev,
6496 struct device_attribute *attr, char *buffer)
6497{
6498 int count = 0;
6499 unsigned int i;
6500
6501 for (i = 0; i < ARRAY_SIZE(pqi_lockup_actions); i++) {
6502 if (pqi_lockup_actions[i].action == pqi_lockup_action)
181aea89 6503 count += scnprintf(buffer + count, PAGE_SIZE - count,
3c50976f
KB		 6504 "[%s] ", pqi_lockup_actions[i].name);
6505 else
181aea89 6506 count += scnprintf(buffer + count, PAGE_SIZE - count,
3c50976f
KB		 6507 "%s ", pqi_lockup_actions[i].name);
6508 }
6509
181aea89 6510 count += scnprintf(buffer + count, PAGE_SIZE - count, "\n");
3c50976f
KB		 6511
6512 return count;
6513}
6514
6515static ssize_t pqi_lockup_action_store(struct device *dev,
6516 struct device_attribute *attr, const char *buffer, size_t count)
6517{
6518 unsigned int i;
6519 char *action_name;
6520 char action_name_buffer[32];
6521
6522 strlcpy(action_name_buffer, buffer, sizeof(action_name_buffer));
6523 action_name = strstrip(action_name_buffer);
6524
6525 for (i = 0; i < ARRAY_SIZE(pqi_lockup_actions); i++) {
6526 if (strcmp(action_name, pqi_lockup_actions[i].name) == 0) {
6527 pqi_lockup_action = pqi_lockup_actions[i].action;
6528 return count;
6529 }
6530 }
6531
6532 return -EINVAL;
6533}
6534
5be746d7
DB		 6535static ssize_t pqi_host_enable_stream_detection_show(struct device *dev,
6536 struct device_attribute *attr, char *buffer)
6537{
6538 struct Scsi_Host *shost = class_to_shost(dev);
6539 struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);
6540
6541 return scnprintf(buffer, 10, "%x\n",
6542 ctrl_info->enable_stream_detection);
6543}
6544
6545static ssize_t pqi_host_enable_stream_detection_store(struct device *dev,
6546 struct device_attribute *attr, const char *buffer, size_t count)
6547{
6548 struct Scsi_Host *shost = class_to_shost(dev);
6549 struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);
6550 u8 set_stream_detection = 0;
6551
6552 if (kstrtou8(buffer, 0, &set_stream_detection))
6553 return -EINVAL;
6554
6555 if (set_stream_detection > 0)
6556 set_stream_detection = 1;
6557
6558 ctrl_info->enable_stream_detection = set_stream_detection;
6559
6560 return count;
6561}
6562
6702d2c4
DB		 6563static ssize_t pqi_host_enable_r5_writes_show(struct device *dev,
6564 struct device_attribute *attr, char *buffer)
6565{
6566 struct Scsi_Host *shost = class_to_shost(dev);
6567 struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);
6568
6569 return scnprintf(buffer, 10, "%x\n", ctrl_info->enable_r5_writes);
6570}
6571
6572static ssize_t pqi_host_enable_r5_writes_store(struct device *dev,
6573 struct device_attribute *attr, const char *buffer, size_t count)
6574{
6575 struct Scsi_Host *shost = class_to_shost(dev);
6576 struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);
6577 u8 set_r5_writes = 0;
6578
6579 if (kstrtou8(buffer, 0, &set_r5_writes))
6580 return -EINVAL;
6581
6582 if (set_r5_writes > 0)
6583 set_r5_writes = 1;
6584
6585 ctrl_info->enable_r5_writes = set_r5_writes;
6586
6587 return count;
6588}
6589
6590static ssize_t pqi_host_enable_r6_writes_show(struct device *dev,
6591 struct device_attribute *attr, char *buffer)
6592{
6593 struct Scsi_Host *shost = class_to_shost(dev);
6594 struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);
6595
6596 return scnprintf(buffer, 10, "%x\n", ctrl_info->enable_r6_writes);
6597}
6598
6599static ssize_t pqi_host_enable_r6_writes_store(struct device *dev,
6600 struct device_attribute *attr, const char *buffer, size_t count)
6601{
6602 struct Scsi_Host *shost = class_to_shost(dev);
6603 struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);
6604 u8 set_r6_writes = 0;
6605
6606 if (kstrtou8(buffer, 0, &set_r6_writes))
6607 return -EINVAL;
6608
6609 if (set_r6_writes > 0)
6610 set_r6_writes = 1;
6611
6612 ctrl_info->enable_r6_writes = set_r6_writes;
6613
6614 return count;
6615}
6616
6d90615f
MB		 6617static DEVICE_ATTR(driver_version, 0444, pqi_driver_version_show, NULL);
6618static DEVICE_ATTR(firmware_version, 0444, pqi_firmware_version_show, NULL);
6619static DEVICE_ATTR(model, 0444, pqi_model_show, NULL);
6620static DEVICE_ATTR(serial_number, 0444, pqi_serial_number_show, NULL);
6621static DEVICE_ATTR(vendor, 0444, pqi_vendor_show, NULL);
cbe0c7b1 6622static DEVICE_ATTR(rescan, 0200, NULL, pqi_host_rescan_store);
583891c9
KB		 6623static DEVICE_ATTR(lockup_action, 0644, pqi_lockup_action_show,
6624 pqi_lockup_action_store);
5be746d7
DB		 6625static DEVICE_ATTR(enable_stream_detection, 0644,
6626 pqi_host_enable_stream_detection_show,
6627 pqi_host_enable_stream_detection_store);
6702d2c4
DB		 6628static DEVICE_ATTR(enable_r5_writes, 0644,
6629 pqi_host_enable_r5_writes_show, pqi_host_enable_r5_writes_store);
6630static DEVICE_ATTR(enable_r6_writes, 0644,
6631 pqi_host_enable_r6_writes_show, pqi_host_enable_r6_writes_store);
6c223761
KB		 6632
6633static struct device_attribute *pqi_shost_attrs[] = {
6d90615f
MB		 6634 &dev_attr_driver_version,
6635 &dev_attr_firmware_version,
6636 &dev_attr_model,
6637 &dev_attr_serial_number,
6638 &dev_attr_vendor,
6c223761 6639 &dev_attr_rescan,
3c50976f 6640 &dev_attr_lockup_action,
5be746d7 6641 &dev_attr_enable_stream_detection,
6702d2c4
DB		 6642 &dev_attr_enable_r5_writes,
6643 &dev_attr_enable_r6_writes,
6c223761
KB		 6644 NULL
6645};
6646
cd128244
DC		 6647static ssize_t pqi_unique_id_show(struct device *dev,
6648 struct device_attribute *attr, char *buffer)
6649{
6650 struct pqi_ctrl_info *ctrl_info;
6651 struct scsi_device *sdev;
6652 struct pqi_scsi_dev *device;
6653 unsigned long flags;
5b083b30 6654 u8 unique_id[16];
cd128244
DC		 6655
6656 sdev = to_scsi_device(dev);
6657 ctrl_info = shost_to_hba(sdev->host);
6658
6659 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
6660
6661 device = sdev->hostdata;
6662 if (!device) {
8b664fef 6663 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
cd128244
DC		 6664 return -ENODEV;
6665 }
5b083b30
KB		 6666
6667 if (device->is_physical_device) {
6668 memset(unique_id, 0, 8);
6669 memcpy(unique_id + 8, &device->wwid, sizeof(device->wwid));
6670 } else {
6671 memcpy(unique_id, device->volume_id, sizeof(device->volume_id));
6672 }
cd128244
DC		 6673
6674 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
6675
a4256252 6676 return scnprintf(buffer, PAGE_SIZE,
583891c9
KB		 6677 "%02X%02X%02X%02X%02X%02X%02X%02X"
6678 "%02X%02X%02X%02X%02X%02X%02X%02X\n",
5b083b30
KB		 6679 unique_id[0], unique_id[1], unique_id[2], unique_id[3],
6680 unique_id[4], unique_id[5], unique_id[6], unique_id[7],
6681 unique_id[8], unique_id[9], unique_id[10], unique_id[11],
6682 unique_id[12], unique_id[13], unique_id[14], unique_id[15]);
cd128244
DC		 6683}
6684
6685static ssize_t pqi_lunid_show(struct device *dev,
6686 struct device_attribute *attr, char *buffer)
6687{
6688 struct pqi_ctrl_info *ctrl_info;
6689 struct scsi_device *sdev;
6690 struct pqi_scsi_dev *device;
6691 unsigned long flags;
6692 u8 lunid[8];
6693
6694 sdev = to_scsi_device(dev);
6695 ctrl_info = shost_to_hba(sdev->host);
6696
6697 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
6698
6699 device = sdev->hostdata;
6700 if (!device) {
8b664fef 6701 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
cd128244
DC		 6702 return -ENODEV;
6703 }
694c5d5b 6704
cd128244
DC		 6705 memcpy(lunid, device->scsi3addr, sizeof(lunid));
6706
6707 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
6708
a4256252 6709 return scnprintf(buffer, PAGE_SIZE, "0x%8phN\n", lunid);
cd128244
DC		 6710}
6711
694c5d5b
KB		 6712#define MAX_PATHS 8
6713
cd128244
DC		 6714static ssize_t pqi_path_info_show(struct device *dev,
6715 struct device_attribute *attr, char *buf)
6716{
6717 struct pqi_ctrl_info *ctrl_info;
6718 struct scsi_device *sdev;
6719 struct pqi_scsi_dev *device;
6720 unsigned long flags;
6721 int i;
6722 int output_len = 0;
6723 u8 box;
6724 u8 bay;
694c5d5b 6725 u8 path_map_index;
cd128244 6726 char *active;
694c5d5b 6727 u8 phys_connector[2];
cd128244
DC		 6728
6729 sdev = to_scsi_device(dev);
6730 ctrl_info = shost_to_hba(sdev->host);
6731
6732 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
6733
6734 device = sdev->hostdata;
6735 if (!device) {
8b664fef 6736 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
cd128244
DC		 6737 return -ENODEV;
6738 }
6739
6740 bay = device->bay;
6741 for (i = 0; i < MAX_PATHS; i++) {
694c5d5b 6742 path_map_index = 1 << i;
cd128244
DC		 6743 if (i == device->active_path_index)
6744 active = "Active";
6745 else if (device->path_map & path_map_index)
6746 active = "Inactive";
6747 else
6748 continue;
6749
6750 output_len += scnprintf(buf + output_len,
6751 PAGE_SIZE - output_len,
6752 "[%d:%d:%d:%d] %20.20s ",
6753 ctrl_info->scsi_host->host_no,
6754 device->bus, device->target,
6755 device->lun,
6756 scsi_device_type(device->devtype));
6757
6758 if (device->devtype == TYPE_RAID ||
6759 pqi_is_logical_device(device))
6760 goto end_buffer;
6761
6762 memcpy(&phys_connector, &device->phys_connector[i],
6763 sizeof(phys_connector));
6764 if (phys_connector[0] < '0')
6765 phys_connector[0] = '0';
6766 if (phys_connector[1] < '0')
6767 phys_connector[1] = '0';
6768
6769 output_len += scnprintf(buf + output_len,
6770 PAGE_SIZE - output_len,
6771 "PORT: %.2s ", phys_connector);
6772
6773 box = device->box[i];
6774 if (box != 0 && box != 0xFF)
6775 output_len += scnprintf(buf + output_len,
6776 PAGE_SIZE - output_len,
6777 "BOX: %hhu ", box);
6778
6779 if ((device->devtype == TYPE_DISK ||
6780 device->devtype == TYPE_ZBC) &&
6781 pqi_expose_device(device))
6782 output_len += scnprintf(buf + output_len,
6783 PAGE_SIZE - output_len,
6784 "BAY: %hhu ", bay);
6785
6786end_buffer:
6787 output_len += scnprintf(buf + output_len,
6788 PAGE_SIZE - output_len,
6789 "%s\n", active);
6790 }
6791
6792 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
694c5d5b 6793
cd128244
DC		 6794 return output_len;
6795}
6796
6c223761
KB		 6797static ssize_t pqi_sas_address_show(struct device *dev,
6798 struct device_attribute *attr, char *buffer)
6799{
6800 struct pqi_ctrl_info *ctrl_info;
6801 struct scsi_device *sdev;
6802 struct pqi_scsi_dev *device;
6803 unsigned long flags;
6804 u64 sas_address;
6805
6806 sdev = to_scsi_device(dev);
6807 ctrl_info = shost_to_hba(sdev->host);
6808
6809 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
6810
6811 device = sdev->hostdata;
8b664fef
KB		 6812 if (!device || !pqi_is_device_with_sas_address(device)) {
6813 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
6c223761
KB		 6814 return -ENODEV;
6815 }
694c5d5b 6816
6c223761
KB		 6817 sas_address = device->sas_address;
6818
6819 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
6820
a4256252 6821 return scnprintf(buffer, PAGE_SIZE, "0x%016llx\n", sas_address);
6c223761
KB		 6822}
6823
6824static ssize_t pqi_ssd_smart_path_enabled_show(struct device *dev,
6825 struct device_attribute *attr, char *buffer)
6826{
6827 struct pqi_ctrl_info *ctrl_info;
6828 struct scsi_device *sdev;
6829 struct pqi_scsi_dev *device;
6830 unsigned long flags;
6831
6832 sdev = to_scsi_device(dev);
6833 ctrl_info = shost_to_hba(sdev->host);
6834
6835 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
6836
6837 device = sdev->hostdata;
8b664fef
KB		 6838 if (!device) {
6839 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
6840 return -ENODEV;
6841 }
6842
588a63fe 6843 buffer[0] = device->raid_bypass_enabled ? '1' : '0';
6c223761
KB		 6844 buffer[1] = '\n';
6845 buffer[2] = '\0';
6846
6847 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
6848
6849 return 2;
6850}
6851
a9f93392
KB		 6852static ssize_t pqi_raid_level_show(struct device *dev,
6853 struct device_attribute *attr, char *buffer)
6854{
6855 struct pqi_ctrl_info *ctrl_info;
6856 struct scsi_device *sdev;
6857 struct pqi_scsi_dev *device;
6858 unsigned long flags;
6859 char *raid_level;
6860
6861 sdev = to_scsi_device(dev);
6862 ctrl_info = shost_to_hba(sdev->host);
6863
6864 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
6865
6866 device = sdev->hostdata;
8b664fef
KB		 6867 if (!device) {
6868 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
6869 return -ENODEV;
6870 }
a9f93392
KB		 6871
6872 if (pqi_is_logical_device(device))
6873 raid_level = pqi_raid_level_to_string(device->raid_level);
6874 else
6875 raid_level = "N/A";
6876
6877 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
6878
a4256252 6879 return scnprintf(buffer, PAGE_SIZE, "%s\n", raid_level);
a9f93392
KB		 6880}
6881
8b664fef
KB		 6882static ssize_t pqi_raid_bypass_cnt_show(struct device *dev,
6883 struct device_attribute *attr, char *buffer)
6884{
6885 struct pqi_ctrl_info *ctrl_info;
6886 struct scsi_device *sdev;
6887 struct pqi_scsi_dev *device;
6888 unsigned long flags;
6889 int raid_bypass_cnt;
6890
6891 sdev = to_scsi_device(dev);
6892 ctrl_info = shost_to_hba(sdev->host);
6893
6894 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
6895
6896 device = sdev->hostdata;
6897 if (!device) {
6898 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
6899 return -ENODEV;
6900 }
6901
6902 raid_bypass_cnt = atomic_read(&device->raid_bypass_cnt);
6903
6904 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
6905
a4256252 6906 return scnprintf(buffer, PAGE_SIZE, "0x%x\n", raid_bypass_cnt);
8b664fef
KB		 6907}
6908
cd128244
DC		 6909static DEVICE_ATTR(lunid, 0444, pqi_lunid_show, NULL);
6910static DEVICE_ATTR(unique_id, 0444, pqi_unique_id_show, NULL);
6911static DEVICE_ATTR(path_info, 0444, pqi_path_info_show, NULL);
cbe0c7b1 6912static DEVICE_ATTR(sas_address, 0444, pqi_sas_address_show, NULL);
8b664fef 6913static DEVICE_ATTR(ssd_smart_path_enabled, 0444, pqi_ssd_smart_path_enabled_show, NULL);
a9f93392 6914static DEVICE_ATTR(raid_level, 0444, pqi_raid_level_show, NULL);
8b664fef 6915static DEVICE_ATTR(raid_bypass_cnt, 0444, pqi_raid_bypass_cnt_show, NULL);
6c223761
KB		 6916
6917static struct device_attribute *pqi_sdev_attrs[] = {
cd128244
DC		 6918 &dev_attr_lunid,
6919 &dev_attr_unique_id,
6920 &dev_attr_path_info,
6c223761
KB		 6921 &dev_attr_sas_address,
6922 &dev_attr_ssd_smart_path_enabled,
a9f93392 6923 &dev_attr_raid_level,
8b664fef 6924 &dev_attr_raid_bypass_cnt,
6c223761
KB		 6925 NULL
6926};
6927
6928static struct scsi_host_template pqi_driver_template = {
6929 .module = THIS_MODULE,
6930 .name = DRIVER_NAME_SHORT,
6931 .proc_name = DRIVER_NAME_SHORT,
6932 .queuecommand = pqi_scsi_queue_command,
6933 .scan_start = pqi_scan_start,
6934 .scan_finished = pqi_scan_finished,
6935 .this_id = -1,
6c223761
KB		 6936 .eh_device_reset_handler = pqi_eh_device_reset_handler,
6937 .ioctl = pqi_ioctl,
6938 .slave_alloc = pqi_slave_alloc,
ce143793 6939 .slave_configure = pqi_slave_configure,
4d15ad38 6940 .slave_destroy = pqi_slave_destroy,
52198226 6941 .map_queues = pqi_map_queues,
6c223761
KB		 6942 .sdev_attrs = pqi_sdev_attrs,
6943 .shost_attrs = pqi_shost_attrs,
6944};
6945
6946static int pqi_register_scsi(struct pqi_ctrl_info *ctrl_info)
6947{
6948 int rc;
6949 struct Scsi_Host *shost;
6950
6951 shost = scsi_host_alloc(&pqi_driver_template, sizeof(ctrl_info));
6952 if (!shost) {
583891c9 6953 dev_err(&ctrl_info->pci_dev->dev, "scsi_host_alloc failed\n");
6c223761
KB		 6954 return -ENOMEM;
6955 }
6956
6957 shost->io_port = 0;
6958 shost->n_io_port = 0;
6959 shost->this_id = -1;
6960 shost->max_channel = PQI_MAX_BUS;
6961 shost->max_cmd_len = MAX_COMMAND_SIZE;
6962 shost->max_lun = ~0;
6963 shost->max_id = ~0;
6964 shost->max_sectors = ctrl_info->max_sectors;
6965 shost->can_queue = ctrl_info->scsi_ml_can_queue;
6966 shost->cmd_per_lun = shost->can_queue;
6967 shost->sg_tablesize = ctrl_info->sg_tablesize;
6968 shost->transportt = pqi_sas_transport_template;
52198226 6969 shost->irq = pci_irq_vector(ctrl_info->pci_dev, 0);
6c223761
KB		 6970 shost->unique_id = shost->irq;
6971 shost->nr_hw_queues = ctrl_info->num_queue_groups;
c6d3ee20 6972 shost->host_tagset = 1;
6c223761
KB		 6973 shost->hostdata[0] = (unsigned long)ctrl_info;
6974
6975 rc = scsi_add_host(shost, &ctrl_info->pci_dev->dev);
6976 if (rc) {
583891c9 6977 dev_err(&ctrl_info->pci_dev->dev, "scsi_add_host failed\n");
6c223761
KB		 6978 goto free_host;
6979 }
6980
6981 rc = pqi_add_sas_host(shost, ctrl_info);
6982 if (rc) {
583891c9 6983 dev_err(&ctrl_info->pci_dev->dev, "add SAS host failed\n");
6c223761
KB		 6984 goto remove_host;
6985 }
6986
6987 ctrl_info->scsi_host = shost;
6988
6989 return 0;
6990
6991remove_host:
6992 scsi_remove_host(shost);
6993free_host:
6994 scsi_host_put(shost);
6995
6996 return rc;
6997}
6998
6999static void pqi_unregister_scsi(struct pqi_ctrl_info *ctrl_info)
7000{
7001 struct Scsi_Host *shost;
7002
7003 pqi_delete_sas_host(ctrl_info);
7004
7005 shost = ctrl_info->scsi_host;
7006 if (!shost)
7007 return;
7008
7009 scsi_remove_host(shost);
7010 scsi_host_put(shost);
7011}
7012
336b6819
KB		 7013static int pqi_wait_for_pqi_reset_completion(struct pqi_ctrl_info *ctrl_info)
7014{
7015 int rc = 0;
7016 struct pqi_device_registers __iomem *pqi_registers;
7017 unsigned long timeout;
7018 unsigned int timeout_msecs;
7019 union pqi_reset_register reset_reg;
6c223761 7020
336b6819
KB		 7021 pqi_registers = ctrl_info->pqi_registers;
7022 timeout_msecs = readw(&pqi_registers->max_reset_timeout) * 100;
7023 timeout = msecs_to_jiffies(timeout_msecs) + jiffies;
7024
7025 while (1) {
7026 msleep(PQI_RESET_POLL_INTERVAL_MSECS);
7027 reset_reg.all_bits = readl(&pqi_registers->device_reset);
7028 if (reset_reg.bits.reset_action == PQI_RESET_ACTION_COMPLETED)
7029 break;
7030 pqi_check_ctrl_health(ctrl_info);
7031 if (pqi_ctrl_offline(ctrl_info)) {
7032 rc = -ENXIO;
7033 break;
7034 }
7035 if (time_after(jiffies, timeout)) {
7036 rc = -ETIMEDOUT;
7037 break;
7038 }
7039 }
7040
7041 return rc;
7042}
6c223761
KB		 7043
7044static int pqi_reset(struct pqi_ctrl_info *ctrl_info)
7045{
7046 int rc;
336b6819
KB		 7047 union pqi_reset_register reset_reg;
7048
7049 if (ctrl_info->pqi_reset_quiesce_supported) {
7050 rc = sis_pqi_reset_quiesce(ctrl_info);
7051 if (rc) {
7052 dev_err(&ctrl_info->pci_dev->dev,
583891c9 7053 "PQI reset failed during quiesce with error %d\n", rc);
336b6819
KB		 7054 return rc;
7055 }
7056 }
6c223761 7057
336b6819
KB		 7058 reset_reg.all_bits = 0;
7059 reset_reg.bits.reset_type = PQI_RESET_TYPE_HARD_RESET;
7060 reset_reg.bits.reset_action = PQI_RESET_ACTION_RESET;
6c223761 7061
336b6819 7062 writel(reset_reg.all_bits, &ctrl_info->pqi_registers->device_reset);
6c223761 7063
336b6819 7064 rc = pqi_wait_for_pqi_reset_completion(ctrl_info);
6c223761
KB		 7065 if (rc)
7066 dev_err(&ctrl_info->pci_dev->dev,
336b6819 7067 "PQI reset failed with error %d\n", rc);
6c223761
KB		 7068
7069 return rc;
7070}
7071
6d90615f
MB		 7072static int pqi_get_ctrl_serial_number(struct pqi_ctrl_info *ctrl_info)
7073{
7074 int rc;
7075 struct bmic_sense_subsystem_info *sense_info;
7076
7077 sense_info = kzalloc(sizeof(*sense_info), GFP_KERNEL);
7078 if (!sense_info)
7079 return -ENOMEM;
7080
7081 rc = pqi_sense_subsystem_info(ctrl_info, sense_info);
7082 if (rc)
7083 goto out;
7084
7085 memcpy(ctrl_info->serial_number, sense_info->ctrl_serial_number,
7086 sizeof(sense_info->ctrl_serial_number));
7087 ctrl_info->serial_number[sizeof(sense_info->ctrl_serial_number)] = '\0';
7088
7089out:
7090 kfree(sense_info);
7091
7092 return rc;
7093}
7094
7095static int pqi_get_ctrl_product_details(struct pqi_ctrl_info *ctrl_info)
6c223761
KB		 7096{
7097 int rc;
7098 struct bmic_identify_controller *identify;
7099
7100 identify = kmalloc(sizeof(*identify), GFP_KERNEL);
7101 if (!identify)
7102 return -ENOMEM;
7103
7104 rc = pqi_identify_controller(ctrl_info, identify);
7105 if (rc)
7106 goto out;
7107
598bef8d
KB		 7108 if (get_unaligned_le32(&identify->extra_controller_flags) &
7109 BMIC_IDENTIFY_EXTRA_FLAGS_LONG_FW_VERSION_SUPPORTED) {
7110 memcpy(ctrl_info->firmware_version,
7111 identify->firmware_version_long,
7112 sizeof(identify->firmware_version_long));
7113 } else {
7114 memcpy(ctrl_info->firmware_version,
7115 identify->firmware_version_short,
7116 sizeof(identify->firmware_version_short));
7117 ctrl_info->firmware_version
7118 [sizeof(identify->firmware_version_short)] = '\0';
7119 snprintf(ctrl_info->firmware_version +
7120 strlen(ctrl_info->firmware_version),
7121 sizeof(ctrl_info->firmware_version) -
7122 sizeof(identify->firmware_version_short),
7123 "-%u",
7124 get_unaligned_le16(&identify->firmware_build_number));
7125 }
6c223761 7126
6d90615f
MB		 7127 memcpy(ctrl_info->model, identify->product_id,
7128 sizeof(identify->product_id));
7129 ctrl_info->model[sizeof(identify->product_id)] = '\0';
7130
7131 memcpy(ctrl_info->vendor, identify->vendor_id,
7132 sizeof(identify->vendor_id));
7133 ctrl_info->vendor[sizeof(identify->vendor_id)] = '\0';
7134
6c223761
KB		 7135out:
7136 kfree(identify);
7137
7138 return rc;
7139}
7140
b212c251
KB		 7141struct pqi_config_table_section_info {
7142 struct pqi_ctrl_info *ctrl_info;
7143 void *section;
7144 u32 section_offset;
7145 void __iomem *section_iomem_addr;
7146};
7147
7148static inline bool pqi_is_firmware_feature_supported(
7149 struct pqi_config_table_firmware_features *firmware_features,
7150 unsigned int bit_position)
98f87667 7151{
b212c251 7152 unsigned int byte_index;
98f87667 7153
b212c251 7154 byte_index = bit_position / BITS_PER_BYTE;
98f87667 7155
b212c251
KB		 7156 if (byte_index >= le16_to_cpu(firmware_features->num_elements))
7157 return false;
98f87667 7158
b212c251
KB		 7159 return firmware_features->features_supported[byte_index] &
7160 (1 << (bit_position % BITS_PER_BYTE)) ? true : false;
7161}
7162
7163static inline bool pqi_is_firmware_feature_enabled(
7164 struct pqi_config_table_firmware_features *firmware_features,
7165 void __iomem *firmware_features_iomem_addr,
7166 unsigned int bit_position)
7167{
7168 unsigned int byte_index;
7169 u8 __iomem *features_enabled_iomem_addr;
7170
7171 byte_index = (bit_position / BITS_PER_BYTE) +
7172 (le16_to_cpu(firmware_features->num_elements) * 2);
7173
7174 features_enabled_iomem_addr = firmware_features_iomem_addr +
7175 offsetof(struct pqi_config_table_firmware_features,
7176 features_supported) + byte_index;
7177
7178 return *((__force u8 *)features_enabled_iomem_addr) &
7179 (1 << (bit_position % BITS_PER_BYTE)) ? true : false;
7180}
7181
7182static inline void pqi_request_firmware_feature(
7183 struct pqi_config_table_firmware_features *firmware_features,
7184 unsigned int bit_position)
7185{
7186 unsigned int byte_index;
7187
7188 byte_index = (bit_position / BITS_PER_BYTE) +
7189 le16_to_cpu(firmware_features->num_elements);
7190
7191 firmware_features->features_supported[byte_index] |=
7192 (1 << (bit_position % BITS_PER_BYTE));
7193}
7194
7195static int pqi_config_table_update(struct pqi_ctrl_info *ctrl_info,
7196 u16 first_section, u16 last_section)
7197{
7198 struct pqi_vendor_general_request request;
7199
7200 memset(&request, 0, sizeof(request));
7201
7202 request.header.iu_type = PQI_REQUEST_IU_VENDOR_GENERAL;
7203 put_unaligned_le16(sizeof(request) - PQI_REQUEST_HEADER_LENGTH,
7204 &request.header.iu_length);
7205 put_unaligned_le16(PQI_VENDOR_GENERAL_CONFIG_TABLE_UPDATE,
7206 &request.function_code);
7207 put_unaligned_le16(first_section,
7208 &request.data.config_table_update.first_section);
7209 put_unaligned_le16(last_section,
7210 &request.data.config_table_update.last_section);
7211
ae0c189d 7212 return pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0, NULL);
b212c251
KB		 7213}
7214
7215static int pqi_enable_firmware_features(struct pqi_ctrl_info *ctrl_info,
7216 struct pqi_config_table_firmware_features *firmware_features,
7217 void __iomem *firmware_features_iomem_addr)
7218{
7219 void *features_requested;
7220 void __iomem *features_requested_iomem_addr;
f6cc2a77 7221 void __iomem *host_max_known_feature_iomem_addr;
b212c251
KB		 7222
7223 features_requested = firmware_features->features_supported +
7224 le16_to_cpu(firmware_features->num_elements);
7225
7226 features_requested_iomem_addr = firmware_features_iomem_addr +
7227 (features_requested - (void *)firmware_features);
7228
7229 memcpy_toio(features_requested_iomem_addr, features_requested,
7230 le16_to_cpu(firmware_features->num_elements));
7231
f6cc2a77
KB		 7232 if (pqi_is_firmware_feature_supported(firmware_features,
7233 PQI_FIRMWARE_FEATURE_MAX_KNOWN_FEATURE)) {
7234 host_max_known_feature_iomem_addr =
7235 features_requested_iomem_addr +
7236 (le16_to_cpu(firmware_features->num_elements) * 2) +
7237 sizeof(__le16);
7238 writew(PQI_FIRMWARE_FEATURE_MAXIMUM,
7239 host_max_known_feature_iomem_addr);
7240 }
7241
b212c251
KB		 7242 return pqi_config_table_update(ctrl_info,
7243 PQI_CONFIG_TABLE_SECTION_FIRMWARE_FEATURES,
7244 PQI_CONFIG_TABLE_SECTION_FIRMWARE_FEATURES);
7245}
7246
7247struct pqi_firmware_feature {
7248 char *feature_name;
7249 unsigned int feature_bit;
7250 bool supported;
7251 bool enabled;
7252 void (*feature_status)(struct pqi_ctrl_info *ctrl_info,
7253 struct pqi_firmware_feature *firmware_feature);
7254};
7255
7256static void pqi_firmware_feature_status(struct pqi_ctrl_info *ctrl_info,
7257 struct pqi_firmware_feature *firmware_feature)
7258{
7259 if (!firmware_feature->supported) {
7260 dev_info(&ctrl_info->pci_dev->dev, "%s not supported by controller\n",
7261 firmware_feature->feature_name);
7262 return;
7263 }
7264
7265 if (firmware_feature->enabled) {
7266 dev_info(&ctrl_info->pci_dev->dev,
7267 "%s enabled\n", firmware_feature->feature_name);
7268 return;
7269 }
7270
7271 dev_err(&ctrl_info->pci_dev->dev, "failed to enable %s\n",
7272 firmware_feature->feature_name);
7273}
7274
21432010 7275static void pqi_ctrl_update_feature_flags(struct pqi_ctrl_info *ctrl_info,
7276 struct pqi_firmware_feature *firmware_feature)
7277{
7278 switch (firmware_feature->feature_bit) {
f6cc2a77
KB		 7279 case PQI_FIRMWARE_FEATURE_RAID_1_WRITE_BYPASS:
7280 ctrl_info->enable_r1_writes = firmware_feature->enabled;
7281 break;
7282 case PQI_FIRMWARE_FEATURE_RAID_5_WRITE_BYPASS:
7283 ctrl_info->enable_r5_writes = firmware_feature->enabled;
7284 break;
7285 case PQI_FIRMWARE_FEATURE_RAID_6_WRITE_BYPASS:
7286 ctrl_info->enable_r6_writes = firmware_feature->enabled;
7287 break;
21432010 7288 case PQI_FIRMWARE_FEATURE_SOFT_RESET_HANDSHAKE:
7289 ctrl_info->soft_reset_handshake_supported =
4ccc354b
KB		 7290 firmware_feature->enabled &&
7291 pqi_read_soft_reset_status(ctrl_info);
21432010 7292 break;
7293 case PQI_FIRMWARE_FEATURE_RAID_IU_TIMEOUT:
583891c9 7294 ctrl_info->raid_iu_timeout_supported = firmware_feature->enabled;
21432010 7295 break;
c2922f17 7296 case PQI_FIRMWARE_FEATURE_TMF_IU_TIMEOUT:
583891c9 7297 ctrl_info->tmf_iu_timeout_supported = firmware_feature->enabled;
c2922f17 7298 break;
7a84a821
KB		 7299 case PQI_FIRMWARE_FEATURE_UNIQUE_WWID_IN_REPORT_PHYS_LUN:
7300 ctrl_info->unique_wwid_in_report_phys_lun_supported =
c2922f17
MB		 7301 firmware_feature->enabled;
7302 break;
21432010 7303 }
7304
7305 pqi_firmware_feature_status(ctrl_info, firmware_feature);
7306}
7307
b212c251
KB		 7308static inline void pqi_firmware_feature_update(struct pqi_ctrl_info *ctrl_info,
7309 struct pqi_firmware_feature *firmware_feature)
7310{
7311 if (firmware_feature->feature_status)
7312 firmware_feature->feature_status(ctrl_info, firmware_feature);
7313}
7314
7315static DEFINE_MUTEX(pqi_firmware_features_mutex);
7316
7317static struct pqi_firmware_feature pqi_firmware_features[] = {
7318 {
7319 .feature_name = "Online Firmware Activation",
7320 .feature_bit = PQI_FIRMWARE_FEATURE_OFA,
7321 .feature_status = pqi_firmware_feature_status,
7322 },
7323 {
7324 .feature_name = "Serial Management Protocol",
7325 .feature_bit = PQI_FIRMWARE_FEATURE_SMP,
7326 .feature_status = pqi_firmware_feature_status,
7327 },
f6cc2a77
KB		 7328 {
7329 .feature_name = "Maximum Known Feature",
7330 .feature_bit = PQI_FIRMWARE_FEATURE_MAX_KNOWN_FEATURE,
7331 .feature_status = pqi_firmware_feature_status,
7332 },
7333 {
7334 .feature_name = "RAID 0 Read Bypass",
7335 .feature_bit = PQI_FIRMWARE_FEATURE_RAID_0_READ_BYPASS,
b212c251
KB		 7336 .feature_status = pqi_firmware_feature_status,
7337 },
7338 {
f6cc2a77
KB		 7339 .feature_name = "RAID 1 Read Bypass",
7340 .feature_bit = PQI_FIRMWARE_FEATURE_RAID_1_READ_BYPASS,
7341 .feature_status = pqi_firmware_feature_status,
7342 },
7343 {
7344 .feature_name = "RAID 5 Read Bypass",
7345 .feature_bit = PQI_FIRMWARE_FEATURE_RAID_5_READ_BYPASS,
b212c251
KB		 7346 .feature_status = pqi_firmware_feature_status,
7347 },
f6cc2a77
KB		 7348 {
7349 .feature_name = "RAID 6 Read Bypass",
7350 .feature_bit = PQI_FIRMWARE_FEATURE_RAID_6_READ_BYPASS,
7351 .feature_status = pqi_firmware_feature_status,
7352 },
7353 {
7354 .feature_name = "RAID 0 Write Bypass",
7355 .feature_bit = PQI_FIRMWARE_FEATURE_RAID_0_WRITE_BYPASS,
7356 .feature_status = pqi_firmware_feature_status,
7357 },
7358 {
7359 .feature_name = "RAID 1 Write Bypass",
7360 .feature_bit = PQI_FIRMWARE_FEATURE_RAID_1_WRITE_BYPASS,
7361 .feature_status = pqi_ctrl_update_feature_flags,
7362 },
7363 {
7364 .feature_name = "RAID 5 Write Bypass",
7365 .feature_bit = PQI_FIRMWARE_FEATURE_RAID_5_WRITE_BYPASS,
7366 .feature_status = pqi_ctrl_update_feature_flags,
7367 },
7368 {
7369 .feature_name = "RAID 6 Write Bypass",
7370 .feature_bit = PQI_FIRMWARE_FEATURE_RAID_6_WRITE_BYPASS,
7371 .feature_status = pqi_ctrl_update_feature_flags,
7372 },
4fd22c13
MR		 7373 {
7374 .feature_name = "New Soft Reset Handshake",
7375 .feature_bit = PQI_FIRMWARE_FEATURE_SOFT_RESET_HANDSHAKE,
21432010 7376 .feature_status = pqi_ctrl_update_feature_flags,
7377 },
7378 {
7379 .feature_name = "RAID IU Timeout",
7380 .feature_bit = PQI_FIRMWARE_FEATURE_RAID_IU_TIMEOUT,
7381 .feature_status = pqi_ctrl_update_feature_flags,
4fd22c13 7382 },
c2922f17
MB		 7383 {
7384 .feature_name = "TMF IU Timeout",
7385 .feature_bit = PQI_FIRMWARE_FEATURE_TMF_IU_TIMEOUT,
7386 .feature_status = pqi_ctrl_update_feature_flags,
7387 },
f6cc2a77
KB		 7388 {
7389 .feature_name = "RAID Bypass on encrypted logical volumes on NVMe",
7390 .feature_bit = PQI_FIRMWARE_FEATURE_RAID_BYPASS_ON_ENCRYPTED_NVME,
7391 .feature_status = pqi_firmware_feature_status,
7392 },
7a84a821
KB		 7393 {
7394 .feature_name = "Unique WWID in Report Physical LUN",
7395 .feature_bit = PQI_FIRMWARE_FEATURE_UNIQUE_WWID_IN_REPORT_PHYS_LUN,
7396 .feature_status = pqi_ctrl_update_feature_flags,
7397 },
b212c251
KB		 7398};
7399
7400static void pqi_process_firmware_features(
7401 struct pqi_config_table_section_info *section_info)
7402{
7403 int rc;
7404 struct pqi_ctrl_info *ctrl_info;
7405 struct pqi_config_table_firmware_features *firmware_features;
7406 void __iomem *firmware_features_iomem_addr;
7407 unsigned int i;
7408 unsigned int num_features_supported;
7409
7410 ctrl_info = section_info->ctrl_info;
7411 firmware_features = section_info->section;
7412 firmware_features_iomem_addr = section_info->section_iomem_addr;
7413
7414 for (i = 0, num_features_supported = 0;
7415 i < ARRAY_SIZE(pqi_firmware_features); i++) {
7416 if (pqi_is_firmware_feature_supported(firmware_features,
7417 pqi_firmware_features[i].feature_bit)) {
7418 pqi_firmware_features[i].supported = true;
7419 num_features_supported++;
7420 } else {
7421 pqi_firmware_feature_update(ctrl_info,
7422 &pqi_firmware_features[i]);
7423 }
7424 }
7425
7426 if (num_features_supported == 0)
7427 return;
7428
7429 for (i = 0; i < ARRAY_SIZE(pqi_firmware_features); i++) {
7430 if (!pqi_firmware_features[i].supported)
7431 continue;
7432 pqi_request_firmware_feature(firmware_features,
7433 pqi_firmware_features[i].feature_bit);
7434 }
7435
7436 rc = pqi_enable_firmware_features(ctrl_info, firmware_features,
7437 firmware_features_iomem_addr);
7438 if (rc) {
7439 dev_err(&ctrl_info->pci_dev->dev,
7440 "failed to enable firmware features in PQI configuration table\n");
7441 for (i = 0; i < ARRAY_SIZE(pqi_firmware_features); i++) {
7442 if (!pqi_firmware_features[i].supported)
7443 continue;
7444 pqi_firmware_feature_update(ctrl_info,
7445 &pqi_firmware_features[i]);
7446 }
7447 return;
7448 }
7449
7450 for (i = 0; i < ARRAY_SIZE(pqi_firmware_features); i++) {
7451 if (!pqi_firmware_features[i].supported)
7452 continue;
7453 if (pqi_is_firmware_feature_enabled(firmware_features,
7454 firmware_features_iomem_addr,
4fd22c13 7455 pqi_firmware_features[i].feature_bit)) {
583891c9 7456 pqi_firmware_features[i].enabled = true;
4fd22c13 7457 }
b212c251
KB		 7458 pqi_firmware_feature_update(ctrl_info,
7459 &pqi_firmware_features[i]);
7460 }
7461}
7462
7463static void pqi_init_firmware_features(void)
7464{
7465 unsigned int i;
7466
7467 for (i = 0; i < ARRAY_SIZE(pqi_firmware_features); i++) {
7468 pqi_firmware_features[i].supported = false;
7469 pqi_firmware_features[i].enabled = false;
7470 }
7471}
7472
7473static void pqi_process_firmware_features_section(
7474 struct pqi_config_table_section_info *section_info)
7475{
7476 mutex_lock(&pqi_firmware_features_mutex);
7477 pqi_init_firmware_features();
7478 pqi_process_firmware_features(section_info);
7479 mutex_unlock(&pqi_firmware_features_mutex);
7480}
7481
f6cc2a77
KB		 7482/*
7483 * Reset all controller settings that can be initialized during the processing
7484 * of the PQI Configuration Table.
7485 */
7486
4ccc354b
KB		 7487static void pqi_ctrl_reset_config(struct pqi_ctrl_info *ctrl_info)
7488{
7489 ctrl_info->heartbeat_counter = NULL;
7490 ctrl_info->soft_reset_status = NULL;
7491 ctrl_info->soft_reset_handshake_supported = false;
7492 ctrl_info->enable_r1_writes = false;
7493 ctrl_info->enable_r5_writes = false;
7494 ctrl_info->enable_r6_writes = false;
7495 ctrl_info->raid_iu_timeout_supported = false;
7496 ctrl_info->tmf_iu_timeout_supported = false;
7497 ctrl_info->unique_wwid_in_report_phys_lun_supported = false;
7498}
7499
98f87667
KB		 7500static int pqi_process_config_table(struct pqi_ctrl_info *ctrl_info)
7501{
7502 u32 table_length;
7503 u32 section_offset;
f6cc2a77 7504 bool firmware_feature_section_present;
98f87667
KB		 7505 void __iomem *table_iomem_addr;
7506 struct pqi_config_table *config_table;
7507 struct pqi_config_table_section_header *section;
b212c251 7508 struct pqi_config_table_section_info section_info;
f6cc2a77 7509 struct pqi_config_table_section_info feature_section_info;
98f87667
KB		 7510
7511 table_length = ctrl_info->config_table_length;
b212c251
KB		 7512 if (table_length == 0)
7513 return 0;
98f87667
KB		 7514
7515 config_table = kmalloc(table_length, GFP_KERNEL);
7516 if (!config_table) {
7517 dev_err(&ctrl_info->pci_dev->dev,
d87d5474 7518 "failed to allocate memory for PQI configuration table\n");
98f87667
KB		 7519 return -ENOMEM;
7520 }
7521
7522 /*
7523 * Copy the config table contents from I/O memory space into the
7524 * temporary buffer.
7525 */
583891c9 7526 table_iomem_addr = ctrl_info->iomem_base + ctrl_info->config_table_offset;
98f87667
KB		 7527 memcpy_fromio(config_table, table_iomem_addr, table_length);
7528
f6cc2a77 7529 firmware_feature_section_present = false;
b212c251 7530 section_info.ctrl_info = ctrl_info;
583891c9 7531 section_offset = get_unaligned_le32(&config_table->first_section_offset);
98f87667
KB		 7532
7533 while (section_offset) {
7534 section = (void *)config_table + section_offset;
7535
b212c251
KB		 7536 section_info.section = section;
7537 section_info.section_offset = section_offset;
583891c9 7538 section_info.section_iomem_addr = table_iomem_addr + section_offset;
b212c251 7539
98f87667 7540 switch (get_unaligned_le16(&section->section_id)) {
b212c251 7541 case PQI_CONFIG_TABLE_SECTION_FIRMWARE_FEATURES:
f6cc2a77
KB		 7542 firmware_feature_section_present = true;
7543 feature_section_info = section_info;
b212c251 7544 break;
98f87667 7545 case PQI_CONFIG_TABLE_SECTION_HEARTBEAT:
5a259e32
KB		 7546 if (pqi_disable_heartbeat)
7547 dev_warn(&ctrl_info->pci_dev->dev,
7548 "heartbeat disabled by module parameter\n");
7549 else
7550 ctrl_info->heartbeat_counter =
7551 table_iomem_addr +
7552 section_offset +
583891c9 7553 offsetof(struct pqi_config_table_heartbeat,
5a259e32 7554 heartbeat_counter);
98f87667 7555 break;
4fd22c13
MR		 7556 case PQI_CONFIG_TABLE_SECTION_SOFT_RESET:
7557 ctrl_info->soft_reset_status =
7558 table_iomem_addr +
7559 section_offset +
7560 offsetof(struct pqi_config_table_soft_reset,
583891c9 7561 soft_reset_status);
4fd22c13 7562 break;
98f87667
KB		 7563 }
7564
583891c9 7565 section_offset = get_unaligned_le16(&section->next_section_offset);
98f87667
KB		 7566 }
7567
f6cc2a77
KB		 7568 /*
7569 * We process the firmware feature section after all other sections
7570 * have been processed so that the feature bit callbacks can take
7571 * into account the settings configured by other sections.
7572 */
7573 if (firmware_feature_section_present)
7574 pqi_process_firmware_features_section(&feature_section_info);
7575
98f87667
KB		 7576 kfree(config_table);
7577
7578 return 0;
7579}
7580
162d7753
KB		 7581/* Switches the controller from PQI mode back into SIS mode. */
7582
7583static int pqi_revert_to_sis_mode(struct pqi_ctrl_info *ctrl_info)
7584{
7585 int rc;
7586
061ef06a 7587 pqi_change_irq_mode(ctrl_info, IRQ_MODE_NONE);
162d7753
KB		 7588 rc = pqi_reset(ctrl_info);
7589 if (rc)
7590 return rc;
4f078e24
KB		 7591 rc = sis_reenable_sis_mode(ctrl_info);
7592 if (rc) {
7593 dev_err(&ctrl_info->pci_dev->dev,
7594 "re-enabling SIS mode failed with error %d\n", rc);
7595 return rc;
7596 }
162d7753
KB		 7597 pqi_save_ctrl_mode(ctrl_info, SIS_MODE);
7598
7599 return 0;
7600}
7601
7602/*
7603 * If the controller isn't already in SIS mode, this function forces it into
7604 * SIS mode.
7605 */
7606
7607static int pqi_force_sis_mode(struct pqi_ctrl_info *ctrl_info)
ff6abb73
KB		 7608{
7609 if (!sis_is_firmware_running(ctrl_info))
7610 return -ENXIO;
7611
162d7753
KB		 7612 if (pqi_get_ctrl_mode(ctrl_info) == SIS_MODE)
7613 return 0;
7614
7615 if (sis_is_kernel_up(ctrl_info)) {
7616 pqi_save_ctrl_mode(ctrl_info, SIS_MODE);
7617 return 0;
ff6abb73
KB		 7618 }
7619
162d7753 7620 return pqi_revert_to_sis_mode(ctrl_info);
ff6abb73
KB		 7621}
7622
6c223761
KB		 7623static int pqi_ctrl_init(struct pqi_ctrl_info *ctrl_info)
7624{
7625 int rc;
2708a256 7626 u32 product_id;
6c223761 7627
0530736e
KB		 7628 if (reset_devices) {
7629 sis_soft_reset(ctrl_info);
3268b8a8 7630 msleep(PQI_POST_RESET_DELAY_SECS * PQI_HZ);
0530736e
KB		 7631 } else {
7632 rc = pqi_force_sis_mode(ctrl_info);
7633 if (rc)
7634 return rc;
7635 }
6c223761
KB		 7636
7637 /*
7638 * Wait until the controller is ready to start accepting SIS
7639 * commands.
7640 */
7641 rc = sis_wait_for_ctrl_ready(ctrl_info);
8845fdfa 7642 if (rc)
6c223761 7643 return rc;
6c223761
KB		 7644
7645 /*
7646 * Get the controller properties. This allows us to determine
7647 * whether or not it supports PQI mode.
7648 */
7649 rc = sis_get_ctrl_properties(ctrl_info);
7650 if (rc) {
7651 dev_err(&ctrl_info->pci_dev->dev,
7652 "error obtaining controller properties\n");
7653 return rc;
7654 }
7655
7656 rc = sis_get_pqi_capabilities(ctrl_info);
7657 if (rc) {
7658 dev_err(&ctrl_info->pci_dev->dev,
7659 "error obtaining controller capabilities\n");
7660 return rc;
7661 }
7662
2708a256
KB		 7663 product_id = sis_get_product_id(ctrl_info);
7664 ctrl_info->product_id = (u8)product_id;
7665 ctrl_info->product_revision = (u8)(product_id >> 8);
7666
d727a776
KB		 7667 if (reset_devices) {
7668 if (ctrl_info->max_outstanding_requests >
7669 PQI_MAX_OUTSTANDING_REQUESTS_KDUMP)
583891c9 7670 ctrl_info->max_outstanding_requests =
d727a776
KB		 7671 PQI_MAX_OUTSTANDING_REQUESTS_KDUMP;
7672 } else {
7673 if (ctrl_info->max_outstanding_requests >
7674 PQI_MAX_OUTSTANDING_REQUESTS)
583891c9 7675 ctrl_info->max_outstanding_requests =
d727a776
KB		 7676 PQI_MAX_OUTSTANDING_REQUESTS;
7677 }
6c223761
KB		 7678
7679 pqi_calculate_io_resources(ctrl_info);
7680
7681 rc = pqi_alloc_error_buffer(ctrl_info);
7682 if (rc) {
7683 dev_err(&ctrl_info->pci_dev->dev,
7684 "failed to allocate PQI error buffer\n");
7685 return rc;
7686 }
7687
7688 /*
7689 * If the function we are about to call succeeds, the
7690 * controller will transition from legacy SIS mode
7691 * into PQI mode.
7692 */
7693 rc = sis_init_base_struct_addr(ctrl_info);
7694 if (rc) {
7695 dev_err(&ctrl_info->pci_dev->dev,
7696 "error initializing PQI mode\n");
7697 return rc;
7698 }
7699
7700 /* Wait for the controller to complete the SIS -> PQI transition. */
7701 rc = pqi_wait_for_pqi_mode_ready(ctrl_info);
7702 if (rc) {
7703 dev_err(&ctrl_info->pci_dev->dev,
7704 "transition to PQI mode failed\n");
7705 return rc;
7706 }
7707
7708 /* From here on, we are running in PQI mode. */
7709 ctrl_info->pqi_mode_enabled = true;
ff6abb73 7710 pqi_save_ctrl_mode(ctrl_info, PQI_MODE);
6c223761
KB		 7711
7712 rc = pqi_alloc_admin_queues(ctrl_info);
7713 if (rc) {
7714 dev_err(&ctrl_info->pci_dev->dev,
d87d5474 7715 "failed to allocate admin queues\n");
6c223761
KB		 7716 return rc;
7717 }
7718
7719 rc = pqi_create_admin_queues(ctrl_info);
7720 if (rc) {
7721 dev_err(&ctrl_info->pci_dev->dev,
7722 "error creating admin queues\n");
7723 return rc;
7724 }
7725
7726 rc = pqi_report_device_capability(ctrl_info);
7727 if (rc) {
7728 dev_err(&ctrl_info->pci_dev->dev,
7729 "obtaining device capability failed\n");
7730 return rc;
7731 }
7732
7733 rc = pqi_validate_device_capability(ctrl_info);
7734 if (rc)
7735 return rc;
7736
7737 pqi_calculate_queue_resources(ctrl_info);
7738
7739 rc = pqi_enable_msix_interrupts(ctrl_info);
7740 if (rc)
7741 return rc;
7742
7743 if (ctrl_info->num_msix_vectors_enabled < ctrl_info->num_queue_groups) {
7744 ctrl_info->max_msix_vectors =
7745 ctrl_info->num_msix_vectors_enabled;
7746 pqi_calculate_queue_resources(ctrl_info);
7747 }
7748
7749 rc = pqi_alloc_io_resources(ctrl_info);
7750 if (rc)
7751 return rc;
7752
7753 rc = pqi_alloc_operational_queues(ctrl_info);
d87d5474
KB		 7754 if (rc) {
7755 dev_err(&ctrl_info->pci_dev->dev,
7756 "failed to allocate operational queues\n");
6c223761 7757 return rc;
d87d5474 7758 }
6c223761
KB		 7759
7760 pqi_init_operational_queues(ctrl_info);
7761
7762 rc = pqi_request_irqs(ctrl_info);
7763 if (rc)
7764 return rc;
7765
6c223761
KB		 7766 rc = pqi_create_queues(ctrl_info);
7767 if (rc)
7768 return rc;
7769
061ef06a
KB		 7770 pqi_change_irq_mode(ctrl_info, IRQ_MODE_MSIX);
7771
7772 ctrl_info->controller_online = true;
b212c251
KB		 7773
7774 rc = pqi_process_config_table(ctrl_info);
7775 if (rc)
7776 return rc;
7777
061ef06a 7778 pqi_start_heartbeat_timer(ctrl_info);
6c223761 7779
f6cc2a77
KB		 7780 if (ctrl_info->enable_r5_writes || ctrl_info->enable_r6_writes) {
7781 rc = pqi_get_advanced_raid_bypass_config(ctrl_info);
7782 if (rc) { /* Supported features not returned correctly. */
7783 dev_err(&ctrl_info->pci_dev->dev,
7784 "error obtaining advanced RAID bypass configuration\n");
7785 return rc;
7786 }
7787 ctrl_info->ciss_report_log_flags |=
7788 CISS_REPORT_LOG_FLAG_DRIVE_TYPE_MIX;
7789 }
7790
6a50d6ad 7791 rc = pqi_enable_events(ctrl_info);
6c223761
KB		 7792 if (rc) {
7793 dev_err(&ctrl_info->pci_dev->dev,
6a50d6ad 7794 "error enabling events\n");
6c223761
KB		 7795 return rc;
7796 }
7797
6c223761
KB		 7798 /* Register with the SCSI subsystem. */
7799 rc = pqi_register_scsi(ctrl_info);
7800 if (rc)
7801 return rc;
7802
6d90615f
MB		 7803 rc = pqi_get_ctrl_product_details(ctrl_info);
7804 if (rc) {
7805 dev_err(&ctrl_info->pci_dev->dev,
7806 "error obtaining product details\n");
7807 return rc;
7808 }
7809
7810 rc = pqi_get_ctrl_serial_number(ctrl_info);
6c223761
KB		 7811 if (rc) {
7812 dev_err(&ctrl_info->pci_dev->dev,
6d90615f 7813 "error obtaining ctrl serial number\n");
6c223761
KB		 7814 return rc;
7815 }
7816
171c2865
DC		 7817 rc = pqi_set_diag_rescan(ctrl_info);
7818 if (rc) {
7819 dev_err(&ctrl_info->pci_dev->dev,
7820 "error enabling multi-lun rescan\n");
7821 return rc;
7822 }
7823
6c223761
KB		 7824 rc = pqi_write_driver_version_to_host_wellness(ctrl_info);
7825 if (rc) {
7826 dev_err(&ctrl_info->pci_dev->dev,
7827 "error updating host wellness\n");
7828 return rc;
7829 }
7830
7831 pqi_schedule_update_time_worker(ctrl_info);
7832
7833 pqi_scan_scsi_devices(ctrl_info);
7834
7835 return 0;
7836}
7837
061ef06a
KB		 7838static void pqi_reinit_queues(struct pqi_ctrl_info *ctrl_info)
7839{
7840 unsigned int i;
7841 struct pqi_admin_queues *admin_queues;
7842 struct pqi_event_queue *event_queue;
7843
7844 admin_queues = &ctrl_info->admin_queues;
7845 admin_queues->iq_pi_copy = 0;
7846 admin_queues->oq_ci_copy = 0;
dac12fbc 7847 writel(0, admin_queues->oq_pi);
061ef06a
KB		 7848
7849 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
7850 ctrl_info->queue_groups[i].iq_pi_copy[RAID_PATH] = 0;
7851 ctrl_info->queue_groups[i].iq_pi_copy[AIO_PATH] = 0;
7852 ctrl_info->queue_groups[i].oq_ci_copy = 0;
7853
dac12fbc
KB		 7854 writel(0, ctrl_info->queue_groups[i].iq_ci[RAID_PATH]);
7855 writel(0, ctrl_info->queue_groups[i].iq_ci[AIO_PATH]);
7856 writel(0, ctrl_info->queue_groups[i].oq_pi);
061ef06a
KB		 7857 }
7858
7859 event_queue = &ctrl_info->event_queue;
dac12fbc 7860 writel(0, event_queue->oq_pi);
061ef06a
KB		 7861 event_queue->oq_ci_copy = 0;
7862}
7863
7864static int pqi_ctrl_init_resume(struct pqi_ctrl_info *ctrl_info)
7865{
7866 int rc;
7867
7868 rc = pqi_force_sis_mode(ctrl_info);
7869 if (rc)
7870 return rc;
7871
7872 /*
7873 * Wait until the controller is ready to start accepting SIS
7874 * commands.
7875 */
7876 rc = sis_wait_for_ctrl_ready_resume(ctrl_info);
7877 if (rc)
7878 return rc;
7879
4fd22c13
MR		 7880 /*
7881 * Get the controller properties. This allows us to determine
7882 * whether or not it supports PQI mode.
7883 */
7884 rc = sis_get_ctrl_properties(ctrl_info);
7885 if (rc) {
7886 dev_err(&ctrl_info->pci_dev->dev,
7887 "error obtaining controller properties\n");
7888 return rc;
7889 }
7890
7891 rc = sis_get_pqi_capabilities(ctrl_info);
7892 if (rc) {
7893 dev_err(&ctrl_info->pci_dev->dev,
7894 "error obtaining controller capabilities\n");
7895 return rc;
7896 }
7897
061ef06a
KB		 7898 /*
7899 * If the function we are about to call succeeds, the
7900 * controller will transition from legacy SIS mode
7901 * into PQI mode.
7902 */
7903 rc = sis_init_base_struct_addr(ctrl_info);
7904 if (rc) {
7905 dev_err(&ctrl_info->pci_dev->dev,
7906 "error initializing PQI mode\n");
7907 return rc;
7908 }
7909
7910 /* Wait for the controller to complete the SIS -> PQI transition. */
7911 rc = pqi_wait_for_pqi_mode_ready(ctrl_info);
7912 if (rc) {
7913 dev_err(&ctrl_info->pci_dev->dev,
7914 "transition to PQI mode failed\n");
7915 return rc;
7916 }
7917
7918 /* From here on, we are running in PQI mode. */
7919 ctrl_info->pqi_mode_enabled = true;
7920 pqi_save_ctrl_mode(ctrl_info, PQI_MODE);
7921
7922 pqi_reinit_queues(ctrl_info);
7923
7924 rc = pqi_create_admin_queues(ctrl_info);
7925 if (rc) {
7926 dev_err(&ctrl_info->pci_dev->dev,
7927 "error creating admin queues\n");
7928 return rc;
7929 }
7930
7931 rc = pqi_create_queues(ctrl_info);
7932 if (rc)
7933 return rc;
7934
7935 pqi_change_irq_mode(ctrl_info, IRQ_MODE_MSIX);
7936
7937 ctrl_info->controller_online = true;
061ef06a
KB		 7938 pqi_ctrl_unblock_requests(ctrl_info);
7939
4ccc354b
KB		 7940 pqi_ctrl_reset_config(ctrl_info);
7941
4fd22c13
MR		 7942 rc = pqi_process_config_table(ctrl_info);
7943 if (rc)
7944 return rc;
7945
7946 pqi_start_heartbeat_timer(ctrl_info);
7947
f6cc2a77
KB		 7948 if (ctrl_info->enable_r5_writes || ctrl_info->enable_r6_writes) {
7949 rc = pqi_get_advanced_raid_bypass_config(ctrl_info);
7950 if (rc) {
7951 dev_err(&ctrl_info->pci_dev->dev,
7952 "error obtaining advanced RAID bypass configuration\n");
7953 return rc;
7954 }
7955 ctrl_info->ciss_report_log_flags |=
7956 CISS_REPORT_LOG_FLAG_DRIVE_TYPE_MIX;
7957 }
7958
061ef06a
KB		 7959 rc = pqi_enable_events(ctrl_info);
7960 if (rc) {
7961 dev_err(&ctrl_info->pci_dev->dev,
d87d5474 7962 "error enabling events\n");
061ef06a
KB		 7963 return rc;
7964 }
7965
6d90615f 7966 rc = pqi_get_ctrl_product_details(ctrl_info);
4fd22c13
MR		 7967 if (rc) {
7968 dev_err(&ctrl_info->pci_dev->dev,
694c5d5b 7969 "error obtaining product details\n");
4fd22c13
MR		 7970 return rc;
7971 }
7972
171c2865
DC		 7973 rc = pqi_set_diag_rescan(ctrl_info);
7974 if (rc) {
7975 dev_err(&ctrl_info->pci_dev->dev,
7976 "error enabling multi-lun rescan\n");
7977 return rc;
7978 }
7979
061ef06a
KB		 7980 rc = pqi_write_driver_version_to_host_wellness(ctrl_info);
7981 if (rc) {
7982 dev_err(&ctrl_info->pci_dev->dev,
7983 "error updating host wellness\n");
7984 return rc;
7985 }
7986
2790cd4d
KB		 7987 if (pqi_ofa_in_progress(ctrl_info))
7988 pqi_ctrl_unblock_scan(ctrl_info);
061ef06a
KB		 7989
7990 pqi_scan_scsi_devices(ctrl_info);
7991
7992 return 0;
7993}
7994
583891c9 7995static inline int pqi_set_pcie_completion_timeout(struct pci_dev *pci_dev, u16 timeout)
a81ed5f3 7996{
d20df83b
BOS		 7997 int rc;
7998
7999 rc = pcie_capability_clear_and_set_word(pci_dev, PCI_EXP_DEVCTL2,
a81ed5f3 8000 PCI_EXP_DEVCTL2_COMP_TIMEOUT, timeout);
d20df83b
BOS		 8001
8002 return pcibios_err_to_errno(rc);
a81ed5f3
KB		 8003}
8004
6c223761
KB		 8005static int pqi_pci_init(struct pqi_ctrl_info *ctrl_info)
8006{
8007 int rc;
8008 u64 mask;
8009
8010 rc = pci_enable_device(ctrl_info->pci_dev);
8011 if (rc) {
8012 dev_err(&ctrl_info->pci_dev->dev,
8013 "failed to enable PCI device\n");
8014 return rc;
8015 }
8016
8017 if (sizeof(dma_addr_t) > 4)
8018 mask = DMA_BIT_MASK(64);
8019 else
8020 mask = DMA_BIT_MASK(32);
8021
1d94f06e 8022 rc = dma_set_mask_and_coherent(&ctrl_info->pci_dev->dev, mask);
6c223761
KB		 8023 if (rc) {
8024 dev_err(&ctrl_info->pci_dev->dev, "failed to set DMA mask\n");
8025 goto disable_device;
8026 }
8027
8028 rc = pci_request_regions(ctrl_info->pci_dev, DRIVER_NAME_SHORT);
8029 if (rc) {
8030 dev_err(&ctrl_info->pci_dev->dev,
8031 "failed to obtain PCI resources\n");
8032 goto disable_device;
8033 }
8034
4bdc0d67 8035 ctrl_info->iomem_base = ioremap(pci_resource_start(
6c223761
KB		 8036 ctrl_info->pci_dev, 0),
8037 sizeof(struct pqi_ctrl_registers));
8038 if (!ctrl_info->iomem_base) {
8039 dev_err(&ctrl_info->pci_dev->dev,
8040 "failed to map memory for controller registers\n");
8041 rc = -ENOMEM;
8042 goto release_regions;
8043 }
8044
a81ed5f3
KB		 8045#define PCI_EXP_COMP_TIMEOUT_65_TO_210_MS 0x6
8046
8047 /* Increase the PCIe completion timeout. */
8048 rc = pqi_set_pcie_completion_timeout(ctrl_info->pci_dev,
8049 PCI_EXP_COMP_TIMEOUT_65_TO_210_MS);
8050 if (rc) {
8051 dev_err(&ctrl_info->pci_dev->dev,
8052 "failed to set PCIe completion timeout\n");
8053 goto release_regions;
8054 }
8055
6c223761
KB		 8056 /* Enable bus mastering. */
8057 pci_set_master(ctrl_info->pci_dev);
8058
cbe0c7b1
KB		 8059 ctrl_info->registers = ctrl_info->iomem_base;
8060 ctrl_info->pqi_registers = &ctrl_info->registers->pqi_registers;
8061
6c223761
KB		 8062 pci_set_drvdata(ctrl_info->pci_dev, ctrl_info);
8063
8064 return 0;
8065
8066release_regions:
8067 pci_release_regions(ctrl_info->pci_dev);
8068disable_device:
8069 pci_disable_device(ctrl_info->pci_dev);
8070
8071 return rc;
8072}
8073
8074static void pqi_cleanup_pci_init(struct pqi_ctrl_info *ctrl_info)
8075{
8076 iounmap(ctrl_info->iomem_base);
8077 pci_release_regions(ctrl_info->pci_dev);
cbe0c7b1
KB		 8078 if (pci_is_enabled(ctrl_info->pci_dev))
8079 pci_disable_device(ctrl_info->pci_dev);
6c223761
KB		 8080 pci_set_drvdata(ctrl_info->pci_dev, NULL);
8081}
8082
8083static struct pqi_ctrl_info *pqi_alloc_ctrl_info(int numa_node)
8084{
8085 struct pqi_ctrl_info *ctrl_info;
8086
8087 ctrl_info = kzalloc_node(sizeof(struct pqi_ctrl_info),
8088 GFP_KERNEL, numa_node);
8089 if (!ctrl_info)
8090 return NULL;
8091
8092 mutex_init(&ctrl_info->scan_mutex);
7561a7e4 8093 mutex_init(&ctrl_info->lun_reset_mutex);
4fd22c13 8094 mutex_init(&ctrl_info->ofa_mutex);
6c223761
KB		 8095
8096 INIT_LIST_HEAD(&ctrl_info->scsi_device_list);
8097 spin_lock_init(&ctrl_info->scsi_device_list_lock);
8098
8099 INIT_WORK(&ctrl_info->event_work, pqi_event_worker);
8100 atomic_set(&ctrl_info->num_interrupts, 0);
8101
8102 INIT_DELAYED_WORK(&ctrl_info->rescan_work, pqi_rescan_worker);
8103 INIT_DELAYED_WORK(&ctrl_info->update_time_work, pqi_update_time_worker);
8104
74a0f573 8105 timer_setup(&ctrl_info->heartbeat_timer, pqi_heartbeat_timer_handler, 0);
5f310425 8106 INIT_WORK(&ctrl_info->ctrl_offline_work, pqi_ctrl_offline_worker);
98f87667 8107
2790cd4d
KB		 8108 INIT_WORK(&ctrl_info->ofa_memory_alloc_work, pqi_ofa_memory_alloc_worker);
8109 INIT_WORK(&ctrl_info->ofa_quiesce_work, pqi_ofa_quiesce_worker);
8110
6c223761
KB		 8111 sema_init(&ctrl_info->sync_request_sem,
8112 PQI_RESERVED_IO_SLOTS_SYNCHRONOUS_REQUESTS);
7561a7e4 8113 init_waitqueue_head(&ctrl_info->block_requests_wait);
6c223761
KB		 8114
8115 ctrl_info->ctrl_id = atomic_inc_return(&pqi_controller_count) - 1;
061ef06a 8116 ctrl_info->irq_mode = IRQ_MODE_NONE;
6c223761
KB		 8117 ctrl_info->max_msix_vectors = PQI_MAX_MSIX_VECTORS;
8118
f6cc2a77
KB		 8119 ctrl_info->ciss_report_log_flags = CISS_REPORT_LOG_FLAG_UNIQUE_LUN_ID;
8120 ctrl_info->max_transfer_encrypted_sas_sata =
8121 PQI_DEFAULT_MAX_TRANSFER_ENCRYPTED_SAS_SATA;
8122 ctrl_info->max_transfer_encrypted_nvme =
8123 PQI_DEFAULT_MAX_TRANSFER_ENCRYPTED_NVME;
8124 ctrl_info->max_write_raid_5_6 = PQI_DEFAULT_MAX_WRITE_RAID_5_6;
8125 ctrl_info->max_write_raid_1_10_2drive = ~0;
8126 ctrl_info->max_write_raid_1_10_3drive = ~0;
8127
6c223761
KB		 8128 return ctrl_info;
8129}
8130
8131static inline void pqi_free_ctrl_info(struct pqi_ctrl_info *ctrl_info)
8132{
8133 kfree(ctrl_info);
8134}
8135
8136static void pqi_free_interrupts(struct pqi_ctrl_info *ctrl_info)
8137{
98bf061b
KB		 8138 pqi_free_irqs(ctrl_info);
8139 pqi_disable_msix_interrupts(ctrl_info);
6c223761
KB		 8140}
8141
8142static void pqi_free_ctrl_resources(struct pqi_ctrl_info *ctrl_info)
8143{
8144 pqi_stop_heartbeat_timer(ctrl_info);
8145 pqi_free_interrupts(ctrl_info);
8146 if (ctrl_info->queue_memory_base)
8147 dma_free_coherent(&ctrl_info->pci_dev->dev,
8148 ctrl_info->queue_memory_length,
8149 ctrl_info->queue_memory_base,
8150 ctrl_info->queue_memory_base_dma_handle);
8151 if (ctrl_info->admin_queue_memory_base)
8152 dma_free_coherent(&ctrl_info->pci_dev->dev,
8153 ctrl_info->admin_queue_memory_length,
8154 ctrl_info->admin_queue_memory_base,
8155 ctrl_info->admin_queue_memory_base_dma_handle);
8156 pqi_free_all_io_requests(ctrl_info);
8157 if (ctrl_info->error_buffer)
8158 dma_free_coherent(&ctrl_info->pci_dev->dev,
8159 ctrl_info->error_buffer_length,
8160 ctrl_info->error_buffer,
8161 ctrl_info->error_buffer_dma_handle);
8162 if (ctrl_info->iomem_base)
8163 pqi_cleanup_pci_init(ctrl_info);
8164 pqi_free_ctrl_info(ctrl_info);
8165}
8166
8167static void pqi_remove_ctrl(struct pqi_ctrl_info *ctrl_info)
8168{
061ef06a
KB		 8169 pqi_cancel_rescan_worker(ctrl_info);
8170 pqi_cancel_update_time_worker(ctrl_info);
e57a1f9b 8171 pqi_unregister_scsi(ctrl_info);
162d7753
KB		 8172 if (ctrl_info->pqi_mode_enabled)
8173 pqi_revert_to_sis_mode(ctrl_info);
6c223761
KB		 8174 pqi_free_ctrl_resources(ctrl_info);
8175}
8176
4fd22c13
MR		 8177static void pqi_ofa_ctrl_quiesce(struct pqi_ctrl_info *ctrl_info)
8178{
2790cd4d
KB		 8179 pqi_ctrl_block_scan(ctrl_info);
8180 pqi_scsi_block_requests(ctrl_info);
8181 pqi_ctrl_block_device_reset(ctrl_info);
4fd22c13
MR		 8182 pqi_ctrl_block_requests(ctrl_info);
8183 pqi_ctrl_wait_until_quiesced(ctrl_info);
4fd22c13 8184 pqi_stop_heartbeat_timer(ctrl_info);
4fd22c13
MR		 8185}
8186
8187static void pqi_ofa_ctrl_unquiesce(struct pqi_ctrl_info *ctrl_info)
8188{
4fd22c13 8189 pqi_start_heartbeat_timer(ctrl_info);
2790cd4d
KB		 8190 pqi_ctrl_unblock_requests(ctrl_info);
8191 pqi_ctrl_unblock_device_reset(ctrl_info);
8192 pqi_scsi_unblock_requests(ctrl_info);
8193 pqi_ctrl_unblock_scan(ctrl_info);
4fd22c13
MR		 8194}
8195
2790cd4d 8196static int pqi_ofa_alloc_mem(struct pqi_ctrl_info *ctrl_info, u32 total_size, u32 chunk_size)
4fd22c13 8197{
4fd22c13 8198 int i;
2790cd4d 8199 u32 sg_count;
4fd22c13
MR		 8200 struct device *dev;
8201 struct pqi_ofa_memory *ofap;
2790cd4d
KB		 8202 struct pqi_sg_descriptor *mem_descriptor;
8203 dma_addr_t dma_handle;
4fd22c13
MR		 8204
8205 ofap = ctrl_info->pqi_ofa_mem_virt_addr;
8206
2790cd4d
KB		 8207 sg_count = DIV_ROUND_UP(total_size, chunk_size);
8208 if (sg_count == 0 || sg_count > PQI_OFA_MAX_SG_DESCRIPTORS)
4fd22c13
MR		 8209 goto out;
8210
2790cd4d 8211 ctrl_info->pqi_ofa_chunk_virt_addr = kmalloc_array(sg_count, sizeof(void *), GFP_KERNEL);
4fd22c13
MR		 8212 if (!ctrl_info->pqi_ofa_chunk_virt_addr)
8213 goto out;
8214
2790cd4d 8215 dev = &ctrl_info->pci_dev->dev;
4fd22c13 8216
2790cd4d 8217 for (i = 0; i < sg_count; i++) {
4fd22c13 8218 ctrl_info->pqi_ofa_chunk_virt_addr[i] =
2790cd4d 8219 dma_alloc_coherent(dev, chunk_size, &dma_handle, GFP_KERNEL);
4fd22c13 8220 if (!ctrl_info->pqi_ofa_chunk_virt_addr[i])
2790cd4d 8221 goto out_free_chunks;
4fd22c13 8222 mem_descriptor = &ofap->sg_descriptor[i];
583891c9
KB		 8223 put_unaligned_le64((u64)dma_handle, &mem_descriptor->address);
8224 put_unaligned_le32(chunk_size, &mem_descriptor->length);
4fd22c13
MR		 8225 }
8226
4fd22c13
MR		 8227 put_unaligned_le32(CISS_SG_LAST, &mem_descriptor->flags);
8228 put_unaligned_le16(sg_count, &ofap->num_memory_descriptors);
2790cd4d 8229 put_unaligned_le32(sg_count * chunk_size, &ofap->bytes_allocated);
4fd22c13
MR		 8230
8231 return 0;
8232
8233out_free_chunks:
8234 while (--i >= 0) {
8235 mem_descriptor = &ofap->sg_descriptor[i];
8236 dma_free_coherent(dev, chunk_size,
2790cd4d
KB		 8237 ctrl_info->pqi_ofa_chunk_virt_addr[i],
8238 get_unaligned_le64(&mem_descriptor->address));
4fd22c13
MR		 8239 }
8240 kfree(ctrl_info->pqi_ofa_chunk_virt_addr);
8241
8242out:
4fd22c13
MR		 8243 return -ENOMEM;
8244}
8245
8246static int pqi_ofa_alloc_host_buffer(struct pqi_ctrl_info *ctrl_info)
8247{
8248 u32 total_size;
2790cd4d 8249 u32 chunk_size;
4fd22c13 8250 u32 min_chunk_size;
4fd22c13 8251
2790cd4d
KB		 8252 if (ctrl_info->ofa_bytes_requested == 0)
8253 return 0;
4fd22c13 8254
2790cd4d
KB		 8255 total_size = PAGE_ALIGN(ctrl_info->ofa_bytes_requested);
8256 min_chunk_size = DIV_ROUND_UP(total_size, PQI_OFA_MAX_SG_DESCRIPTORS);
8257 min_chunk_size = PAGE_ALIGN(min_chunk_size);
4fd22c13 8258
2790cd4d
KB		 8259 for (chunk_size = total_size; chunk_size >= min_chunk_size;) {
8260 if (pqi_ofa_alloc_mem(ctrl_info, total_size, chunk_size) == 0)
4fd22c13 8261 return 0;
2790cd4d
KB		 8262 chunk_size /= 2;
8263 chunk_size = PAGE_ALIGN(chunk_size);
8264 }
4fd22c13
MR		 8265
8266 return -ENOMEM;
8267}
8268
2790cd4d 8269static void pqi_ofa_setup_host_buffer(struct pqi_ctrl_info *ctrl_info)
4fd22c13 8270{
4fd22c13 8271 struct device *dev;
2790cd4d 8272 struct pqi_ofa_memory *ofap;
4fd22c13
MR		 8273
8274 dev = &ctrl_info->pci_dev->dev;
4fd22c13 8275
2790cd4d
KB		 8276 ofap = dma_alloc_coherent(dev, sizeof(*ofap),
8277 &ctrl_info->pqi_ofa_mem_dma_handle, GFP_KERNEL);
8278 if (!ofap)
4fd22c13
MR		 8279 return;
8280
2790cd4d 8281 ctrl_info->pqi_ofa_mem_virt_addr = ofap;
4fd22c13
MR		 8282
8283 if (pqi_ofa_alloc_host_buffer(ctrl_info) < 0) {
2790cd4d
KB		 8284 dev_err(dev,
8285 "failed to allocate host buffer for Online Firmware Activation\n");
8286 dma_free_coherent(dev, sizeof(*ofap), ofap, ctrl_info->pqi_ofa_mem_dma_handle);
8287 ctrl_info->pqi_ofa_mem_virt_addr = NULL;
8288 return;
4fd22c13 8289 }
694c5d5b 8290
2790cd4d
KB		 8291 put_unaligned_le16(PQI_OFA_VERSION, &ofap->version);
8292 memcpy(&ofap->signature, PQI_OFA_SIGNATURE, sizeof(ofap->signature));
4fd22c13
MR		 8293}
8294
8295static void pqi_ofa_free_host_buffer(struct pqi_ctrl_info *ctrl_info)
8296{
2790cd4d
KB		 8297 unsigned int i;
8298 struct device *dev;
4fd22c13 8299 struct pqi_ofa_memory *ofap;
2790cd4d
KB		 8300 struct pqi_sg_descriptor *mem_descriptor;
8301 unsigned int num_memory_descriptors;
4fd22c13
MR		 8302
8303 ofap = ctrl_info->pqi_ofa_mem_virt_addr;
4fd22c13
MR		 8304 if (!ofap)
8305 return;
8306
2790cd4d
KB		 8307 dev = &ctrl_info->pci_dev->dev;
8308
8309 if (get_unaligned_le32(&ofap->bytes_allocated) == 0)
4fd22c13
MR		 8310 goto out;
8311
8312 mem_descriptor = ofap->sg_descriptor;
2790cd4d
KB		 8313 num_memory_descriptors =
8314 get_unaligned_le16(&ofap->num_memory_descriptors);
4fd22c13 8315
2790cd4d
KB		 8316 for (i = 0; i < num_memory_descriptors; i++) {
8317 dma_free_coherent(dev,
4fd22c13
MR		 8318 get_unaligned_le32(&mem_descriptor[i].length),
8319 ctrl_info->pqi_ofa_chunk_virt_addr[i],
8320 get_unaligned_le64(&mem_descriptor[i].address));
8321 }
8322 kfree(ctrl_info->pqi_ofa_chunk_virt_addr);
8323
8324out:
2790cd4d
KB		 8325 dma_free_coherent(dev, sizeof(*ofap), ofap,
8326 ctrl_info->pqi_ofa_mem_dma_handle);
4fd22c13
MR		 8327 ctrl_info->pqi_ofa_mem_virt_addr = NULL;
8328}
8329
8330static int pqi_ofa_host_memory_update(struct pqi_ctrl_info *ctrl_info)
8331{
2790cd4d 8332 u32 buffer_length;
4fd22c13 8333 struct pqi_vendor_general_request request;
4fd22c13
MR		 8334 struct pqi_ofa_memory *ofap;
8335
8336 memset(&request, 0, sizeof(request));
8337
4fd22c13
MR		 8338 request.header.iu_type = PQI_REQUEST_IU_VENDOR_GENERAL;
8339 put_unaligned_le16(sizeof(request) - PQI_REQUEST_HEADER_LENGTH,
8340 &request.header.iu_length);
8341 put_unaligned_le16(PQI_VENDOR_GENERAL_HOST_MEMORY_UPDATE,
8342 &request.function_code);
8343
2790cd4d
KB		 8344 ofap = ctrl_info->pqi_ofa_mem_virt_addr;
8345
4fd22c13 8346 if (ofap) {
2790cd4d 8347 buffer_length = offsetof(struct pqi_ofa_memory, sg_descriptor) +
4fd22c13
MR		 8348 get_unaligned_le16(&ofap->num_memory_descriptors) *
8349 sizeof(struct pqi_sg_descriptor);
8350
8351 put_unaligned_le64((u64)ctrl_info->pqi_ofa_mem_dma_handle,
8352 &request.data.ofa_memory_allocation.buffer_address);
2790cd4d 8353 put_unaligned_le32(buffer_length,
4fd22c13 8354 &request.data.ofa_memory_allocation.buffer_length);
4fd22c13
MR		 8355 }
8356
ae0c189d 8357 return pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0, NULL);
4fd22c13
MR		 8358}
8359
2790cd4d 8360static int pqi_ofa_ctrl_restart(struct pqi_ctrl_info *ctrl_info, unsigned int delay_secs)
4fd22c13 8361{
2790cd4d
KB		 8362 ssleep(delay_secs);
8363
4fd22c13
MR		 8364 return pqi_ctrl_init_resume(ctrl_info);
8365}
8366
3c50976f
KB		 8367static void pqi_perform_lockup_action(void)
8368{
8369 switch (pqi_lockup_action) {
8370 case PANIC:
8371 panic("FATAL: Smart Family Controller lockup detected");
8372 break;
8373 case REBOOT:
8374 emergency_restart();
8375 break;
8376 case NONE:
8377 default:
8378 break;
8379 }
8380}
8381
5f310425
KB		 8382static struct pqi_raid_error_info pqi_ctrl_offline_raid_error_info = {
8383 .data_out_result = PQI_DATA_IN_OUT_HARDWARE_ERROR,
8384 .status = SAM_STAT_CHECK_CONDITION,
8385};
8386
8387static void pqi_fail_all_outstanding_requests(struct pqi_ctrl_info *ctrl_info)
376fb880
KB		 8388{
8389 unsigned int i;
376fb880 8390 struct pqi_io_request *io_request;
376fb880
KB		 8391 struct scsi_cmnd *scmd;
8392
5f310425
KB		 8393 for (i = 0; i < ctrl_info->max_io_slots; i++) {
8394 io_request = &ctrl_info->io_request_pool[i];
8395 if (atomic_read(&io_request->refcount) == 0)
8396 continue;
376fb880 8397
5f310425
KB		 8398 scmd = io_request->scmd;
8399 if (scmd) {
8400 set_host_byte(scmd, DID_NO_CONNECT);
8401 } else {
8402 io_request->status = -ENXIO;
8403 io_request->error_info =
8404 &pqi_ctrl_offline_raid_error_info;
376fb880 8405 }
5f310425
KB		 8406
8407 io_request->io_complete_callback(io_request,
8408 io_request->context);
376fb880
KB		 8409 }
8410}
8411
5f310425 8412static void pqi_take_ctrl_offline_deferred(struct pqi_ctrl_info *ctrl_info)
376fb880 8413{
5f310425
KB		 8414 pqi_perform_lockup_action();
8415 pqi_stop_heartbeat_timer(ctrl_info);
8416 pqi_free_interrupts(ctrl_info);
8417 pqi_cancel_rescan_worker(ctrl_info);
8418 pqi_cancel_update_time_worker(ctrl_info);
8419 pqi_ctrl_wait_until_quiesced(ctrl_info);
8420 pqi_fail_all_outstanding_requests(ctrl_info);
5f310425
KB		 8421 pqi_ctrl_unblock_requests(ctrl_info);
8422}
8423
8424static void pqi_ctrl_offline_worker(struct work_struct *work)
8425{
8426 struct pqi_ctrl_info *ctrl_info;
8427
8428 ctrl_info = container_of(work, struct pqi_ctrl_info, ctrl_offline_work);
8429 pqi_take_ctrl_offline_deferred(ctrl_info);
376fb880
KB		 8430}
8431
8432static void pqi_take_ctrl_offline(struct pqi_ctrl_info *ctrl_info)
8433{
5f310425
KB		 8434 if (!ctrl_info->controller_online)
8435 return;
8436
376fb880 8437 ctrl_info->controller_online = false;
5f310425
KB		 8438 ctrl_info->pqi_mode_enabled = false;
8439 pqi_ctrl_block_requests(ctrl_info);
5a259e32
KB		 8440 if (!pqi_disable_ctrl_shutdown)
8441 sis_shutdown_ctrl(ctrl_info);
376fb880
KB		 8442 pci_disable_device(ctrl_info->pci_dev);
8443 dev_err(&ctrl_info->pci_dev->dev, "controller offline\n");
5f310425 8444 schedule_work(&ctrl_info->ctrl_offline_work);
376fb880
KB		 8445}
8446
d91d7820 8447static void pqi_print_ctrl_info(struct pci_dev *pci_dev,
6c223761
KB		 8448 const struct pci_device_id *id)
8449{
8450 char *ctrl_description;
8451
37b36847 8452 if (id->driver_data)
6c223761 8453 ctrl_description = (char *)id->driver_data;
37b36847
KB		 8454 else
8455 ctrl_description = "Microsemi Smart Family Controller";
6c223761 8456
d91d7820 8457 dev_info(&pci_dev->dev, "%s found\n", ctrl_description);
6c223761
KB		 8458}
8459
d91d7820
KB		 8460static int pqi_pci_probe(struct pci_dev *pci_dev,
8461 const struct pci_device_id *id)
6c223761
KB		 8462{
8463 int rc;
62dc51fb 8464 int node, cp_node;
6c223761
KB		 8465 struct pqi_ctrl_info *ctrl_info;
8466
d91d7820 8467 pqi_print_ctrl_info(pci_dev, id);
6c223761
KB		 8468
8469 if (pqi_disable_device_id_wildcards &&
8470 id->subvendor == PCI_ANY_ID &&
8471 id->subdevice == PCI_ANY_ID) {
d91d7820 8472 dev_warn(&pci_dev->dev,
6c223761
KB		 8473 "controller not probed because device ID wildcards are disabled\n");
8474 return -ENODEV;
8475 }
8476
8477 if (id->subvendor == PCI_ANY_ID || id->subdevice == PCI_ANY_ID)
d91d7820 8478 dev_warn(&pci_dev->dev,
6c223761
KB		 8479 "controller device ID matched using wildcards\n");
8480
d91d7820 8481 node = dev_to_node(&pci_dev->dev);
62dc51fb
SB		 8482 if (node == NUMA_NO_NODE) {
8483 cp_node = cpu_to_node(0);
8484 if (cp_node == NUMA_NO_NODE)
8485 cp_node = 0;
8486 set_dev_node(&pci_dev->dev, cp_node);
8487 }
6c223761
KB		 8488
8489 ctrl_info = pqi_alloc_ctrl_info(node);
8490 if (!ctrl_info) {
d91d7820 8491 dev_err(&pci_dev->dev,
6c223761
KB		 8492 "failed to allocate controller info block\n");
8493 return -ENOMEM;
8494 }
8495
d91d7820 8496 ctrl_info->pci_dev = pci_dev;
6c223761
KB		 8497
8498 rc = pqi_pci_init(ctrl_info);
8499 if (rc)
8500 goto error;
8501
8502 rc = pqi_ctrl_init(ctrl_info);
8503 if (rc)
8504 goto error;
8505
8506 return 0;
8507
8508error:
8509 pqi_remove_ctrl(ctrl_info);
8510
8511 return rc;
8512}
8513
d91d7820 8514static void pqi_pci_remove(struct pci_dev *pci_dev)
6c223761
KB		 8515{
8516 struct pqi_ctrl_info *ctrl_info;
8517
d91d7820 8518 ctrl_info = pci_get_drvdata(pci_dev);
6c223761
KB		 8519 if (!ctrl_info)
8520 return;
8521
8522 pqi_remove_ctrl(ctrl_info);
8523}
8524
0530736e
KB		 8525static void pqi_crash_if_pending_command(struct pqi_ctrl_info *ctrl_info)
8526{
8527 unsigned int i;
8528 struct pqi_io_request *io_request;
8529 struct scsi_cmnd *scmd;
8530
8531 for (i = 0; i < ctrl_info->max_io_slots; i++) {
8532 io_request = &ctrl_info->io_request_pool[i];
8533 if (atomic_read(&io_request->refcount) == 0)
8534 continue;
8535 scmd = io_request->scmd;
8536 WARN_ON(scmd != NULL); /* IO command from SML */
8537 WARN_ON(scmd == NULL); /* Non-IO cmd or driver initiated*/
8538 }
8539}
8540
d91d7820 8541static void pqi_shutdown(struct pci_dev *pci_dev)
6c223761
KB		 8542{
8543 int rc;
8544 struct pqi_ctrl_info *ctrl_info;
8545
d91d7820 8546 ctrl_info = pci_get_drvdata(pci_dev);
0530736e
KB		 8547 if (!ctrl_info) {
8548 dev_err(&pci_dev->dev,
8549 "cache could not be flushed\n");
8550 return;
8551 }
8552
0530736e 8553 pqi_wait_until_ofa_finished(ctrl_info);
0530736e 8554
9fa82023 8555 pqi_scsi_block_requests(ctrl_info);
0530736e 8556 pqi_ctrl_block_device_reset(ctrl_info);
9fa82023
KB		 8557 pqi_ctrl_block_requests(ctrl_info);
8558 pqi_ctrl_wait_until_quiesced(ctrl_info);
6c223761
KB		 8559
8560 /*
8561 * Write all data in the controller's battery-backed cache to
8562 * storage.
8563 */
58322fe0 8564 rc = pqi_flush_cache(ctrl_info, SHUTDOWN);
0530736e
KB		 8565 if (rc)
8566 dev_err(&pci_dev->dev,
8567 "unable to flush controller cache\n");
8568
0530736e
KB		 8569 pqi_crash_if_pending_command(ctrl_info);
8570 pqi_reset(ctrl_info);
6c223761
KB		 8571}
8572
3c50976f
KB		 8573static void pqi_process_lockup_action_param(void)
8574{
8575 unsigned int i;
8576
8577 if (!pqi_lockup_action_param)
8578 return;
8579
8580 for (i = 0; i < ARRAY_SIZE(pqi_lockup_actions); i++) {
8581 if (strcmp(pqi_lockup_action_param,
8582 pqi_lockup_actions[i].name) == 0) {
8583 pqi_lockup_action = pqi_lockup_actions[i].action;
8584 return;
8585 }
8586 }
8587
8588 pr_warn("%s: invalid lockup action setting \"%s\" - supported settings: none, reboot, panic\n",
8589 DRIVER_NAME_SHORT, pqi_lockup_action_param);
8590}
8591
8592static void pqi_process_module_params(void)
8593{
8594 pqi_process_lockup_action_param();
8595}
8596
5c146686 8597static __maybe_unused int pqi_suspend(struct pci_dev *pci_dev, pm_message_t state)
061ef06a
KB		 8598{
8599 struct pqi_ctrl_info *ctrl_info;
8600
8601 ctrl_info = pci_get_drvdata(pci_dev);
8602
4fd22c13 8603 pqi_wait_until_ofa_finished(ctrl_info);
9fa82023
KB		 8604
8605 pqi_ctrl_block_scan(ctrl_info);
8606 pqi_scsi_block_requests(ctrl_info);
8607 pqi_ctrl_block_device_reset(ctrl_info);
061ef06a
KB		 8608 pqi_ctrl_block_requests(ctrl_info);
8609 pqi_ctrl_wait_until_quiesced(ctrl_info);
9fa82023 8610 pqi_flush_cache(ctrl_info, SUSPEND);
061ef06a
KB		 8611 pqi_stop_heartbeat_timer(ctrl_info);
8612
9fa82023
KB		 8613 pqi_crash_if_pending_command(ctrl_info);
8614
061ef06a
KB		 8615 if (state.event == PM_EVENT_FREEZE)
8616 return 0;
8617
8618 pci_save_state(pci_dev);
8619 pci_set_power_state(pci_dev, pci_choose_state(pci_dev, state));
8620
8621 ctrl_info->controller_online = false;
8622 ctrl_info->pqi_mode_enabled = false;
8623
8624 return 0;
8625}
8626
5c146686 8627static __maybe_unused int pqi_resume(struct pci_dev *pci_dev)
061ef06a
KB		 8628{
8629 int rc;
8630 struct pqi_ctrl_info *ctrl_info;
8631
8632 ctrl_info = pci_get_drvdata(pci_dev);
8633
8634 if (pci_dev->current_state != PCI_D0) {
8635 ctrl_info->max_hw_queue_index = 0;
8636 pqi_free_interrupts(ctrl_info);
8637 pqi_change_irq_mode(ctrl_info, IRQ_MODE_INTX);
8638 rc = request_irq(pci_irq_vector(pci_dev, 0), pqi_irq_handler,
8639 IRQF_SHARED, DRIVER_NAME_SHORT,
8640 &ctrl_info->queue_groups[0]);
8641 if (rc) {
8642 dev_err(&ctrl_info->pci_dev->dev,
8643 "irq %u init failed with error %d\n",
8644 pci_dev->irq, rc);
8645 return rc;
8646 }
43e97ef4 8647 pqi_ctrl_unblock_device_reset(ctrl_info);
061ef06a 8648 pqi_ctrl_unblock_requests(ctrl_info);
9fa82023 8649 pqi_scsi_unblock_requests(ctrl_info);
43e97ef4 8650 pqi_ctrl_unblock_scan(ctrl_info);
061ef06a
KB		 8651 return 0;
8652 }
8653
8654 pci_set_power_state(pci_dev, PCI_D0);
8655 pci_restore_state(pci_dev);
8656
43e97ef4
KB		 8657 pqi_ctrl_unblock_device_reset(ctrl_info);
8658 pqi_ctrl_unblock_requests(ctrl_info);
8659 pqi_scsi_unblock_requests(ctrl_info);
8660 pqi_ctrl_unblock_scan(ctrl_info);
8661
061ef06a
KB		 8662 return pqi_ctrl_init_resume(ctrl_info);
8663}
8664
6c223761
KB		 8665/* Define the PCI IDs for the controllers that we support. */
8666static const struct pci_device_id pqi_pci_id_table[] = {
b0f9408b
KB		 8667 {
8668 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8669 0x105b, 0x1211)
8670 },
8671 {
8672 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8673 0x105b, 0x1321)
8674 },
7eddabff
KB		 8675 {
8676 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8677 0x152d, 0x8a22)
8678 },
8679 {
8680 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8681 0x152d, 0x8a23)
8682 },
8683 {
8684 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8685 0x152d, 0x8a24)
8686 },
8687 {
8688 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8689 0x152d, 0x8a36)
8690 },
8691 {
8692 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8693 0x152d, 0x8a37)
8694 },
75fbeacc
KB		 8695 {
8696 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8697 0x193d, 0x8460)
8698 },
0595a0b4
AK		 8699 {
8700 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8701 0x193d, 0x1104)
8702 },
8703 {
8704 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8705 0x193d, 0x1105)
8706 },
8707 {
8708 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8709 0x193d, 0x1106)
8710 },
8711 {
8712 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8713 0x193d, 0x1107)
8714 },
b0f9408b
KB		 8715 {
8716 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8717 0x193d, 0x8460)
8718 },
8719 {
8720 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8721 0x193d, 0x8461)
8722 },
84a77fef
MB		 8723 {
8724 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8725 0x193d, 0xc460)
8726 },
8727 {
8728 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8729 0x193d, 0xc461)
8730 },
b0f9408b
KB		 8731 {
8732 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8733 0x193d, 0xf460)
8734 },
8735 {
8736 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8737 0x193d, 0xf461)
8738 },
8739 {
8740 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8741 0x1bd4, 0x0045)
8742 },
8743 {
8744 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8745 0x1bd4, 0x0046)
8746 },
8747 {
8748 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8749 0x1bd4, 0x0047)
8750 },
8751 {
8752 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8753 0x1bd4, 0x0048)
8754 },
9f8d05fa
KB		 8755 {
8756 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8757 0x1bd4, 0x004a)
8758 },
8759 {
8760 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8761 0x1bd4, 0x004b)
8762 },
8763 {
8764 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8765 0x1bd4, 0x004c)
8766 },
63a7956a
GW		 8767 {
8768 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8769 0x1bd4, 0x004f)
8770 },
75fbeacc
KB		 8771 {
8772 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8773 0x1bd4, 0x0051)
8774 },
8775 {
8776 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8777 0x1bd4, 0x0052)
8778 },
8779 {
8780 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8781 0x1bd4, 0x0053)
8782 },
8783 {
8784 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8785 0x1bd4, 0x0054)
8786 },
c1b10475
AK		 8787 {
8788 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8789 0x19e5, 0xd227)
8790 },
8791 {
8792 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8793 0x19e5, 0xd228)
8794 },
8795 {
8796 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8797 0x19e5, 0xd229)
8798 },
8799 {
8800 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8801 0x19e5, 0xd22a)
8802 },
8803 {
8804 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8805 0x19e5, 0xd22b)
8806 },
8807 {
8808 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8809 0x19e5, 0xd22c)
8810 },
6c223761
KB		 8811 {
8812 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8813 PCI_VENDOR_ID_ADAPTEC2, 0x0110)
8814 },
8815 {
8816 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
55790064 8817 PCI_VENDOR_ID_ADAPTEC2, 0x0608)
6c223761
KB		 8818 },
8819 {
8820 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8821 PCI_VENDOR_ID_ADAPTEC2, 0x0800)
6c223761
KB		 8822 },
8823 {
8824 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8825 PCI_VENDOR_ID_ADAPTEC2, 0x0801)
6c223761
KB		 8826 },
8827 {
8828 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8829 PCI_VENDOR_ID_ADAPTEC2, 0x0802)
6c223761
KB		 8830 },
8831 {
8832 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8833 PCI_VENDOR_ID_ADAPTEC2, 0x0803)
6c223761
KB		 8834 },
8835 {
8836 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8837 PCI_VENDOR_ID_ADAPTEC2, 0x0804)
6c223761
KB		 8838 },
8839 {
8840 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8841 PCI_VENDOR_ID_ADAPTEC2, 0x0805)
6c223761
KB		 8842 },
8843 {
8844 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8845 PCI_VENDOR_ID_ADAPTEC2, 0x0806)
6c223761 8846 },
55790064
KB		 8847 {
8848 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8849 PCI_VENDOR_ID_ADAPTEC2, 0x0807)
8850 },
63a7956a
GW		 8851 {
8852 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8853 PCI_VENDOR_ID_ADAPTEC2, 0x0808)
8854 },
8855 {
8856 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8857 PCI_VENDOR_ID_ADAPTEC2, 0x0809)
8858 },
3af06083
MR		 8859 {
8860 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8861 PCI_VENDOR_ID_ADAPTEC2, 0x080a)
8862 },
6c223761
KB		 8863 {
8864 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8865 PCI_VENDOR_ID_ADAPTEC2, 0x0900)
6c223761
KB		 8866 },
8867 {
8868 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8869 PCI_VENDOR_ID_ADAPTEC2, 0x0901)
6c223761
KB		 8870 },
8871 {
8872 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8873 PCI_VENDOR_ID_ADAPTEC2, 0x0902)
6c223761
KB		 8874 },
8875 {
8876 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8877 PCI_VENDOR_ID_ADAPTEC2, 0x0903)
6c223761
KB		 8878 },
8879 {
8880 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8881 PCI_VENDOR_ID_ADAPTEC2, 0x0904)
6c223761
KB		 8882 },
8883 {
8884 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8885 PCI_VENDOR_ID_ADAPTEC2, 0x0905)
6c223761
KB		 8886 },
8887 {
8888 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8889 PCI_VENDOR_ID_ADAPTEC2, 0x0906)
6c223761
KB		 8890 },
8891 {
8892 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8893 PCI_VENDOR_ID_ADAPTEC2, 0x0907)
6c223761
KB		 8894 },
8895 {
8896 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8897 PCI_VENDOR_ID_ADAPTEC2, 0x0908)
6c223761 8898 },
55790064
KB		 8899 {
8900 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8901 PCI_VENDOR_ID_ADAPTEC2, 0x090a)
8902 },
6c223761
KB		 8903 {
8904 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8905 PCI_VENDOR_ID_ADAPTEC2, 0x1200)
6c223761
KB		 8906 },
8907 {
8908 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8909 PCI_VENDOR_ID_ADAPTEC2, 0x1201)
6c223761
KB		 8910 },
8911 {
8912 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8913 PCI_VENDOR_ID_ADAPTEC2, 0x1202)
6c223761
KB		 8914 },
8915 {
8916 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8917 PCI_VENDOR_ID_ADAPTEC2, 0x1280)
6c223761
KB		 8918 },
8919 {
8920 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8921 PCI_VENDOR_ID_ADAPTEC2, 0x1281)
6c223761 8922 },
b0f9408b
KB		 8923 {
8924 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8925 PCI_VENDOR_ID_ADAPTEC2, 0x1282)
8926 },
6c223761
KB		 8927 {
8928 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8929 PCI_VENDOR_ID_ADAPTEC2, 0x1300)
6c223761
KB		 8930 },
8931 {
8932 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff 8933 PCI_VENDOR_ID_ADAPTEC2, 0x1301)
6c223761 8934 },
bd809e8d
KB		 8935 {
8936 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8937 PCI_VENDOR_ID_ADAPTEC2, 0x1302)
8938 },
8939 {
8940 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8941 PCI_VENDOR_ID_ADAPTEC2, 0x1303)
8942 },
6c223761
KB		 8943 {
8944 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
7eddabff
KB		 8945 PCI_VENDOR_ID_ADAPTEC2, 0x1380)
8946 },
75fbeacc
KB		 8947 {
8948 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8949 PCI_VENDOR_ID_ADAPTEC2, 0x1400)
8950 },
8951 {
8952 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8953 PCI_VENDOR_ID_ADAPTEC2, 0x1402)
8954 },
8955 {
8956 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8957 PCI_VENDOR_ID_ADAPTEC2, 0x1410)
8958 },
8959 {
8960 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8961 PCI_VENDOR_ID_ADAPTEC2, 0x1411)
8962 },
8963 {
8964 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8965 PCI_VENDOR_ID_ADAPTEC2, 0x1412)
8966 },
8967 {
8968 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8969 PCI_VENDOR_ID_ADAPTEC2, 0x1420)
8970 },
8971 {
8972 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8973 PCI_VENDOR_ID_ADAPTEC2, 0x1430)
8974 },
8975 {
8976 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8977 PCI_VENDOR_ID_ADAPTEC2, 0x1440)
8978 },
8979 {
8980 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8981 PCI_VENDOR_ID_ADAPTEC2, 0x1441)
8982 },
8983 {
8984 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8985 PCI_VENDOR_ID_ADAPTEC2, 0x1450)
8986 },
8987 {
8988 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8989 PCI_VENDOR_ID_ADAPTEC2, 0x1452)
8990 },
8991 {
8992 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8993 PCI_VENDOR_ID_ADAPTEC2, 0x1460)
8994 },
8995 {
8996 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
8997 PCI_VENDOR_ID_ADAPTEC2, 0x1461)
8998 },
8999 {
9000 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9001 PCI_VENDOR_ID_ADAPTEC2, 0x1462)
9002 },
9003 {
9004 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9005 PCI_VENDOR_ID_ADAPTEC2, 0x1470)
9006 },
9007 {
9008 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9009 PCI_VENDOR_ID_ADAPTEC2, 0x1471)
9010 },
9011 {
9012 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9013 PCI_VENDOR_ID_ADAPTEC2, 0x1472)
9014 },
9015 {
9016 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9017 PCI_VENDOR_ID_ADAPTEC2, 0x1480)
9018 },
9019 {
9020 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9021 PCI_VENDOR_ID_ADAPTEC2, 0x1490)
9022 },
9023 {
9024 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9025 PCI_VENDOR_ID_ADAPTEC2, 0x1491)
9026 },
9027 {
9028 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9029 PCI_VENDOR_ID_ADAPTEC2, 0x14a0)
9030 },
9031 {
9032 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9033 PCI_VENDOR_ID_ADAPTEC2, 0x14a1)
9034 },
9035 {
9036 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9037 PCI_VENDOR_ID_ADAPTEC2, 0x14b0)
9038 },
9039 {
9040 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9041 PCI_VENDOR_ID_ADAPTEC2, 0x14b1)
9042 },
9043 {
9044 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9045 PCI_VENDOR_ID_ADAPTEC2, 0x14c0)
9046 },
9047 {
9048 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9049 PCI_VENDOR_ID_ADAPTEC2, 0x14c1)
9050 },
9051 {
9052 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9053 PCI_VENDOR_ID_ADAPTEC2, 0x14d0)
9054 },
9055 {
9056 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9057 PCI_VENDOR_ID_ADAPTEC2, 0x14e0)
9058 },
9059 {
9060 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9061 PCI_VENDOR_ID_ADAPTEC2, 0x14f0)
9062 },
9f8d05fa
KB		 9063 {
9064 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9065 PCI_VENDOR_ID_ADVANTECH, 0x8312)
9066 },
55790064
KB		 9067 {
9068 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9069 PCI_VENDOR_ID_DELL, 0x1fe0)
9070 },
7eddabff
KB		 9071 {
9072 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9073 PCI_VENDOR_ID_HP, 0x0600)
9074 },
9075 {
9076 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9077 PCI_VENDOR_ID_HP, 0x0601)
9078 },
9079 {
9080 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9081 PCI_VENDOR_ID_HP, 0x0602)
9082 },
9083 {
9084 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9085 PCI_VENDOR_ID_HP, 0x0603)
9086 },
9087 {
9088 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
55790064 9089 PCI_VENDOR_ID_HP, 0x0609)
7eddabff
KB		 9090 },
9091 {
9092 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9093 PCI_VENDOR_ID_HP, 0x0650)
9094 },
9095 {
9096 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9097 PCI_VENDOR_ID_HP, 0x0651)
9098 },
9099 {
9100 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9101 PCI_VENDOR_ID_HP, 0x0652)
9102 },
9103 {
9104 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9105 PCI_VENDOR_ID_HP, 0x0653)
9106 },
9107 {
9108 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9109 PCI_VENDOR_ID_HP, 0x0654)
9110 },
9111 {
9112 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9113 PCI_VENDOR_ID_HP, 0x0655)
9114 },
7eddabff
KB		 9115 {
9116 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9117 PCI_VENDOR_ID_HP, 0x0700)
9118 },
9119 {
9120 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9121 PCI_VENDOR_ID_HP, 0x0701)
6c223761
KB		 9122 },
9123 {
9124 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9125 PCI_VENDOR_ID_HP, 0x1001)
9126 },
75fbeacc
KB		 9127 {
9128 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9129 PCI_VENDOR_ID_HP, 0x1002)
9130 },
6c223761
KB		 9131 {
9132 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9133 PCI_VENDOR_ID_HP, 0x1100)
9134 },
9135 {
9136 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9137 PCI_VENDOR_ID_HP, 0x1101)
9138 },
75fbeacc
KB		 9139 {
9140 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9141 0x1590, 0x0294)
9142 },
9143 {
9144 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9145 0x1590, 0x02db)
9146 },
9147 {
9148 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9149 0x1590, 0x02dc)
9150 },
9151 {
9152 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9153 0x1590, 0x032e)
9154 },
8bdb3b9c
GW		 9155 {
9156 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9157 0x1d8d, 0x0800)
9158 },
9159 {
9160 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9161 0x1d8d, 0x0908)
9162 },
9163 {
9164 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9165 0x1d8d, 0x0806)
9166 },
9167 {
9168 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9169 0x1d8d, 0x0916)
9170 },
71ecc60d
GW		 9171 {
9172 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9173 PCI_VENDOR_ID_GIGABYTE, 0x1000)
9174 },
6c223761
KB		 9175 {
9176 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9177 PCI_ANY_ID, PCI_ANY_ID)
9178 },
9179 { 0 }
9180};
9181
9182MODULE_DEVICE_TABLE(pci, pqi_pci_id_table);
9183
9184static struct pci_driver pqi_pci_driver = {
9185 .name = DRIVER_NAME_SHORT,
9186 .id_table = pqi_pci_id_table,
9187 .probe = pqi_pci_probe,
9188 .remove = pqi_pci_remove,
9189 .shutdown = pqi_shutdown,
061ef06a
KB		 9190#if defined(CONFIG_PM)
9191 .suspend = pqi_suspend,
9192 .resume = pqi_resume,
9193#endif
6c223761
KB		 9194};
9195
9196static int __init pqi_init(void)
9197{
9198 int rc;
9199
9200 pr_info(DRIVER_NAME "\n");
9201
8b664fef 9202 pqi_sas_transport_template = sas_attach_transport(&pqi_sas_transport_functions);
6c223761
KB		 9203 if (!pqi_sas_transport_template)
9204 return -ENODEV;
9205
3c50976f
KB		 9206 pqi_process_module_params();
9207
6c223761
KB		 9208 rc = pci_register_driver(&pqi_pci_driver);
9209 if (rc)
9210 sas_release_transport(pqi_sas_transport_template);
9211
9212 return rc;
9213}
9214
9215static void __exit pqi_cleanup(void)
9216{
9217 pci_unregister_driver(&pqi_pci_driver);
9218 sas_release_transport(pqi_sas_transport_template);
9219}
9220
9221module_init(pqi_init);
9222module_exit(pqi_cleanup);
9223
9224static void __attribute__((unused)) verify_structures(void)
9225{
9226 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
9227 sis_host_to_ctrl_doorbell) != 0x20);
9228 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
9229 sis_interrupt_mask) != 0x34);
9230 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
9231 sis_ctrl_to_host_doorbell) != 0x9c);
9232 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
9233 sis_ctrl_to_host_doorbell_clear) != 0xa0);
ff6abb73
KB		 9234 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
9235 sis_driver_scratch) != 0xb0);
2708a256
KB		 9236 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
9237 sis_product_identifier) != 0xb4);
6c223761
KB		 9238 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
9239 sis_firmware_status) != 0xbc);
9240 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
9241 sis_mailbox) != 0x1000);
9242 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
9243 pqi_registers) != 0x4000);
9244
9245 BUILD_BUG_ON(offsetof(struct pqi_iu_header,
9246 iu_type) != 0x0);
9247 BUILD_BUG_ON(offsetof(struct pqi_iu_header,
9248 iu_length) != 0x2);
9249 BUILD_BUG_ON(offsetof(struct pqi_iu_header,
9250 response_queue_id) != 0x4);
9251 BUILD_BUG_ON(offsetof(struct pqi_iu_header,
ae0c189d 9252 driver_flags) != 0x6);
6c223761
KB		 9253 BUILD_BUG_ON(sizeof(struct pqi_iu_header) != 0x8);
9254
9255 BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
9256 status) != 0x0);
9257 BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
9258 service_response) != 0x1);
9259 BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
9260 data_present) != 0x2);
9261 BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
9262 reserved) != 0x3);
9263 BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
9264 residual_count) != 0x4);
9265 BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
9266 data_length) != 0x8);
9267 BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
9268 reserved1) != 0xa);
9269 BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
9270 data) != 0xc);
9271 BUILD_BUG_ON(sizeof(struct pqi_aio_error_info) != 0x10c);
9272
9273 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
9274 data_in_result) != 0x0);
9275 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
9276 data_out_result) != 0x1);
9277 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
9278 reserved) != 0x2);
9279 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
9280 status) != 0x5);
9281 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
9282 status_qualifier) != 0x6);
9283 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
9284 sense_data_length) != 0x8);
9285 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
9286 response_data_length) != 0xa);
9287 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
9288 data_in_transferred) != 0xc);
9289 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
9290 data_out_transferred) != 0x10);
9291 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
9292 data) != 0x14);
9293 BUILD_BUG_ON(sizeof(struct pqi_raid_error_info) != 0x114);
9294
9295 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
9296 signature) != 0x0);
9297 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
9298 function_and_status_code) != 0x8);
9299 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
9300 max_admin_iq_elements) != 0x10);
9301 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
9302 max_admin_oq_elements) != 0x11);
9303 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
9304 admin_iq_element_length) != 0x12);
9305 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
9306 admin_oq_element_length) != 0x13);
9307 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
9308 max_reset_timeout) != 0x14);
9309 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
9310 legacy_intx_status) != 0x18);
9311 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
9312 legacy_intx_mask_set) != 0x1c);
9313 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
9314 legacy_intx_mask_clear) != 0x20);
9315 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
9316 device_status) != 0x40);
9317 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
9318 admin_iq_pi_offset) != 0x48);
9319 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
9320 admin_oq_ci_offset) != 0x50);
9321 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
9322 admin_iq_element_array_addr) != 0x58);
9323 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
9324 admin_oq_element_array_addr) != 0x60);
9325 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
9326 admin_iq_ci_addr) != 0x68);
9327 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
9328 admin_oq_pi_addr) != 0x70);
9329 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
9330 admin_iq_num_elements) != 0x78);
9331 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
9332 admin_oq_num_elements) != 0x79);
9333 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
9334 admin_queue_int_msg_num) != 0x7a);
9335 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
9336 device_error) != 0x80);
9337 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
9338 error_details) != 0x88);
9339 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
9340 device_reset) != 0x90);
9341 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
9342 power_action) != 0x94);
9343 BUILD_BUG_ON(sizeof(struct pqi_device_registers) != 0x100);
9344
9345 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
9346 header.iu_type) != 0);
9347 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
9348 header.iu_length) != 2);
9349 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
ae0c189d 9350 header.driver_flags) != 6);
6c223761
KB		 9351 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
9352 request_id) != 8);
9353 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
9354 function_code) != 10);
9355 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
9356 data.report_device_capability.buffer_length) != 44);
9357 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
9358 data.report_device_capability.sg_descriptor) != 48);
9359 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
9360 data.create_operational_iq.queue_id) != 12);
9361 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
9362 data.create_operational_iq.element_array_addr) != 16);
9363 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
9364 data.create_operational_iq.ci_addr) != 24);
9365 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
9366 data.create_operational_iq.num_elements) != 32);
9367 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
9368 data.create_operational_iq.element_length) != 34);
9369 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
9370 data.create_operational_iq.queue_protocol) != 36);
9371 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
9372 data.create_operational_oq.queue_id) != 12);
9373 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
9374 data.create_operational_oq.element_array_addr) != 16);
9375 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
9376 data.create_operational_oq.pi_addr) != 24);
9377 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
9378 data.create_operational_oq.num_elements) != 32);
9379 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
9380 data.create_operational_oq.element_length) != 34);
9381 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
9382 data.create_operational_oq.queue_protocol) != 36);
9383 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
9384 data.create_operational_oq.int_msg_num) != 40);
9385 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
9386 data.create_operational_oq.coalescing_count) != 42);
9387 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
9388 data.create_operational_oq.min_coalescing_time) != 44);
9389 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
9390 data.create_operational_oq.max_coalescing_time) != 48);
9391 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
9392 data.delete_operational_queue.queue_id) != 12);
9393 BUILD_BUG_ON(sizeof(struct pqi_general_admin_request) != 64);
c593642c 9394 BUILD_BUG_ON(sizeof_field(struct pqi_general_admin_request,
6c223761 9395 data.create_operational_iq) != 64 - 11);
c593642c 9396 BUILD_BUG_ON(sizeof_field(struct pqi_general_admin_request,
6c223761 9397 data.create_operational_oq) != 64 - 11);
c593642c 9398 BUILD_BUG_ON(sizeof_field(struct pqi_general_admin_request,
6c223761
KB		 9399 data.delete_operational_queue) != 64 - 11);
9400
9401 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
9402 header.iu_type) != 0);
9403 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
9404 header.iu_length) != 2);
9405 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
ae0c189d 9406 header.driver_flags) != 6);
6c223761
KB		 9407 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
9408 request_id) != 8);
9409 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
9410 function_code) != 10);
9411 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
9412 status) != 11);
9413 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
9414 data.create_operational_iq.status_descriptor) != 12);
9415 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
9416 data.create_operational_iq.iq_pi_offset) != 16);
9417 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
9418 data.create_operational_oq.status_descriptor) != 12);
9419 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
9420 data.create_operational_oq.oq_ci_offset) != 16);
9421 BUILD_BUG_ON(sizeof(struct pqi_general_admin_response) != 64);
9422
9423 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
9424 header.iu_type) != 0);
9425 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
9426 header.iu_length) != 2);
9427 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
9428 header.response_queue_id) != 4);
9429 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
ae0c189d 9430 header.driver_flags) != 6);
6c223761
KB		 9431 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
9432 request_id) != 8);
9433 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
9434 nexus_id) != 10);
9435 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
9436 buffer_length) != 12);
9437 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
9438 lun_number) != 16);
9439 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
9440 protocol_specific) != 24);
9441 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
9442 error_index) != 27);
9443 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
9444 cdb) != 32);
21432010 9445 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
9446 timeout) != 60);
6c223761
KB		 9447 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
9448 sg_descriptors) != 64);
9449 BUILD_BUG_ON(sizeof(struct pqi_raid_path_request) !=
9450 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
9451
9452 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
9453 header.iu_type) != 0);
9454 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
9455 header.iu_length) != 2);
9456 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
9457 header.response_queue_id) != 4);
9458 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
ae0c189d 9459 header.driver_flags) != 6);
6c223761
KB		 9460 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
9461 request_id) != 8);
9462 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
9463 nexus_id) != 12);
9464 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
9465 buffer_length) != 16);
9466 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
9467 data_encryption_key_index) != 22);
9468 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
9469 encrypt_tweak_lower) != 24);
9470 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
9471 encrypt_tweak_upper) != 28);
9472 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
9473 cdb) != 32);
9474 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
9475 error_index) != 48);
9476 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
9477 num_sg_descriptors) != 50);
9478 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
9479 cdb_length) != 51);
9480 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
9481 lun_number) != 52);
9482 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
9483 sg_descriptors) != 64);
9484 BUILD_BUG_ON(sizeof(struct pqi_aio_path_request) !=
9485 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
9486
9487 BUILD_BUG_ON(offsetof(struct pqi_io_response,
9488 header.iu_type) != 0);
9489 BUILD_BUG_ON(offsetof(struct pqi_io_response,
9490 header.iu_length) != 2);
9491 BUILD_BUG_ON(offsetof(struct pqi_io_response,
9492 request_id) != 8);
9493 BUILD_BUG_ON(offsetof(struct pqi_io_response,
9494 error_index) != 10);
9495
9496 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
9497 header.iu_type) != 0);
9498 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
9499 header.iu_length) != 2);
9500 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
9501 header.response_queue_id) != 4);
9502 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
9503 request_id) != 8);
9504 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
9505 data.report_event_configuration.buffer_length) != 12);
9506 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
9507 data.report_event_configuration.sg_descriptors) != 16);
9508 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
9509 data.set_event_configuration.global_event_oq_id) != 10);
9510 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
9511 data.set_event_configuration.buffer_length) != 12);
9512 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
9513 data.set_event_configuration.sg_descriptors) != 16);
9514
9515 BUILD_BUG_ON(offsetof(struct pqi_iu_layer_descriptor,
9516 max_inbound_iu_length) != 6);
9517 BUILD_BUG_ON(offsetof(struct pqi_iu_layer_descriptor,
9518 max_outbound_iu_length) != 14);
9519 BUILD_BUG_ON(sizeof(struct pqi_iu_layer_descriptor) != 16);
9520
9521 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
9522 data_length) != 0);
9523 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
9524 iq_arbitration_priority_support_bitmask) != 8);
9525 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
9526 maximum_aw_a) != 9);
9527 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
9528 maximum_aw_b) != 10);
9529 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
9530 maximum_aw_c) != 11);
9531 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
9532 max_inbound_queues) != 16);
9533 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
9534 max_elements_per_iq) != 18);
9535 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
9536 max_iq_element_length) != 24);
9537 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
9538 min_iq_element_length) != 26);
9539 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
9540 max_outbound_queues) != 30);
9541 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
9542 max_elements_per_oq) != 32);
9543 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
9544 intr_coalescing_time_granularity) != 34);
9545 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
9546 max_oq_element_length) != 36);
9547 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
9548 min_oq_element_length) != 38);
9549 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
9550 iu_layer_descriptors) != 64);
9551 BUILD_BUG_ON(sizeof(struct pqi_device_capability) != 576);
9552
9553 BUILD_BUG_ON(offsetof(struct pqi_event_descriptor,
9554 event_type) != 0);
9555 BUILD_BUG_ON(offsetof(struct pqi_event_descriptor,
9556 oq_id) != 2);
9557 BUILD_BUG_ON(sizeof(struct pqi_event_descriptor) != 4);
9558
9559 BUILD_BUG_ON(offsetof(struct pqi_event_config,
9560 num_event_descriptors) != 2);
9561 BUILD_BUG_ON(offsetof(struct pqi_event_config,
9562 descriptors) != 4);
9563
061ef06a
KB		 9564 BUILD_BUG_ON(PQI_NUM_SUPPORTED_EVENTS !=
9565 ARRAY_SIZE(pqi_supported_event_types));
9566
6c223761
KB		 9567 BUILD_BUG_ON(offsetof(struct pqi_event_response,
9568 header.iu_type) != 0);
9569 BUILD_BUG_ON(offsetof(struct pqi_event_response,
9570 header.iu_length) != 2);
9571 BUILD_BUG_ON(offsetof(struct pqi_event_response,
9572 event_type) != 8);
9573 BUILD_BUG_ON(offsetof(struct pqi_event_response,
9574 event_id) != 10);
9575 BUILD_BUG_ON(offsetof(struct pqi_event_response,
9576 additional_event_id) != 12);
9577 BUILD_BUG_ON(offsetof(struct pqi_event_response,
9578 data) != 16);
9579 BUILD_BUG_ON(sizeof(struct pqi_event_response) != 32);
9580
9581 BUILD_BUG_ON(offsetof(struct pqi_event_acknowledge_request,
9582 header.iu_type) != 0);
9583 BUILD_BUG_ON(offsetof(struct pqi_event_acknowledge_request,
9584 header.iu_length) != 2);
9585 BUILD_BUG_ON(offsetof(struct pqi_event_acknowledge_request,
9586 event_type) != 8);
9587 BUILD_BUG_ON(offsetof(struct pqi_event_acknowledge_request,
9588 event_id) != 10);
9589 BUILD_BUG_ON(offsetof(struct pqi_event_acknowledge_request,
9590 additional_event_id) != 12);
9591 BUILD_BUG_ON(sizeof(struct pqi_event_acknowledge_request) != 16);
9592
9593 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
9594 header.iu_type) != 0);
9595 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
9596 header.iu_length) != 2);
9597 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
9598 request_id) != 8);
9599 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
9600 nexus_id) != 10);
9601 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
c2922f17
MB		 9602 timeout) != 14);
9603 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
6c223761
KB		 9604 lun_number) != 16);
9605 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
9606 protocol_specific) != 24);
9607 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
9608 outbound_queue_id_to_manage) != 26);
9609 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
9610 request_id_to_manage) != 28);
9611 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
9612 task_management_function) != 30);
9613 BUILD_BUG_ON(sizeof(struct pqi_task_management_request) != 32);
9614
9615 BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
9616 header.iu_type) != 0);
9617 BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
9618 header.iu_length) != 2);
9619 BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
9620 request_id) != 8);
9621 BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
9622 nexus_id) != 10);
9623 BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
9624 additional_response_info) != 12);
9625 BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
9626 response_code) != 15);
9627 BUILD_BUG_ON(sizeof(struct pqi_task_management_response) != 16);
9628
9629 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
9630 configured_logical_drive_count) != 0);
9631 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
9632 configuration_signature) != 1);
9633 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
598bef8d 9634 firmware_version_short) != 5);
6c223761
KB		 9635 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
9636 extended_logical_unit_count) != 154);
9637 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
9638 firmware_build_number) != 190);
598bef8d
KB		 9639 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
9640 vendor_id) != 200);
9641 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
9642 product_id) != 208);
9643 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
9644 extra_controller_flags) != 286);
6c223761
KB		 9645 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
9646 controller_mode) != 292);
598bef8d
KB		 9647 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
9648 spare_part_number) != 293);
9649 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
9650 firmware_version_long) != 325);
6c223761 9651
1be42f46
KB		 9652 BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
9653 phys_bay_in_box) != 115);
9654 BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
9655 device_type) != 120);
9656 BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
9657 redundant_path_present_map) != 1736);
9658 BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
9659 active_path_number) != 1738);
9660 BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
9661 alternate_paths_phys_connector) != 1739);
9662 BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
9663 alternate_paths_phys_box_on_port) != 1755);
9664 BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
9665 current_queue_depth_limit) != 1796);
9666 BUILD_BUG_ON(sizeof(struct bmic_identify_physical_device) != 2560);
9667
f6cc2a77
KB		 9668 BUILD_BUG_ON(sizeof(struct bmic_sense_feature_buffer_header) != 4);
9669 BUILD_BUG_ON(offsetof(struct bmic_sense_feature_buffer_header,
9670 page_code) != 0);
9671 BUILD_BUG_ON(offsetof(struct bmic_sense_feature_buffer_header,
9672 subpage_code) != 1);
9673 BUILD_BUG_ON(offsetof(struct bmic_sense_feature_buffer_header,
9674 buffer_length) != 2);
9675
9676 BUILD_BUG_ON(sizeof(struct bmic_sense_feature_page_header) != 4);
9677 BUILD_BUG_ON(offsetof(struct bmic_sense_feature_page_header,
9678 page_code) != 0);
9679 BUILD_BUG_ON(offsetof(struct bmic_sense_feature_page_header,
9680 subpage_code) != 1);
9681 BUILD_BUG_ON(offsetof(struct bmic_sense_feature_page_header,
9682 page_length) != 2);
9683
9684 BUILD_BUG_ON(sizeof(struct bmic_sense_feature_io_page_aio_subpage)
9685 != 18);
9686 BUILD_BUG_ON(offsetof(struct bmic_sense_feature_io_page_aio_subpage,
9687 header) != 0);
9688 BUILD_BUG_ON(offsetof(struct bmic_sense_feature_io_page_aio_subpage,
9689 firmware_read_support) != 4);
9690 BUILD_BUG_ON(offsetof(struct bmic_sense_feature_io_page_aio_subpage,
9691 driver_read_support) != 5);
9692 BUILD_BUG_ON(offsetof(struct bmic_sense_feature_io_page_aio_subpage,
9693 firmware_write_support) != 6);
9694 BUILD_BUG_ON(offsetof(struct bmic_sense_feature_io_page_aio_subpage,
9695 driver_write_support) != 7);
9696 BUILD_BUG_ON(offsetof(struct bmic_sense_feature_io_page_aio_subpage,
9697 max_transfer_encrypted_sas_sata) != 8);
9698 BUILD_BUG_ON(offsetof(struct bmic_sense_feature_io_page_aio_subpage,
9699 max_transfer_encrypted_nvme) != 10);
9700 BUILD_BUG_ON(offsetof(struct bmic_sense_feature_io_page_aio_subpage,
9701 max_write_raid_5_6) != 12);
9702 BUILD_BUG_ON(offsetof(struct bmic_sense_feature_io_page_aio_subpage,
9703 max_write_raid_1_10_2drive) != 14);
9704 BUILD_BUG_ON(offsetof(struct bmic_sense_feature_io_page_aio_subpage,
9705 max_write_raid_1_10_3drive) != 16);
9706
6c223761
KB		 9707 BUILD_BUG_ON(PQI_ADMIN_IQ_NUM_ELEMENTS > 255);
9708 BUILD_BUG_ON(PQI_ADMIN_OQ_NUM_ELEMENTS > 255);
9709 BUILD_BUG_ON(PQI_ADMIN_IQ_ELEMENT_LENGTH %
9710 PQI_QUEUE_ELEMENT_LENGTH_ALIGNMENT != 0);
9711 BUILD_BUG_ON(PQI_ADMIN_OQ_ELEMENT_LENGTH %
9712 PQI_QUEUE_ELEMENT_LENGTH_ALIGNMENT != 0);
9713 BUILD_BUG_ON(PQI_OPERATIONAL_IQ_ELEMENT_LENGTH > 1048560);
9714 BUILD_BUG_ON(PQI_OPERATIONAL_IQ_ELEMENT_LENGTH %
9715 PQI_QUEUE_ELEMENT_LENGTH_ALIGNMENT != 0);
9716 BUILD_BUG_ON(PQI_OPERATIONAL_OQ_ELEMENT_LENGTH > 1048560);
9717 BUILD_BUG_ON(PQI_OPERATIONAL_OQ_ELEMENT_LENGTH %
9718 PQI_QUEUE_ELEMENT_LENGTH_ALIGNMENT != 0);
9719
9720 BUILD_BUG_ON(PQI_RESERVED_IO_SLOTS >= PQI_MAX_OUTSTANDING_REQUESTS);
d727a776
KB		 9721 BUILD_BUG_ON(PQI_RESERVED_IO_SLOTS >=
9722 PQI_MAX_OUTSTANDING_REQUESTS_KDUMP);
6c223761 9723}
Linux 6.x block layer and io_uring tree(s)