projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'for-v3.15-rc/omap-fixes-a' of git://git.kernel.org/pub/scm/linux/kernel...
[linux-2.6-block.git] / drivers / block / skd_main.c
CommitLineData
e67f86b3
AB		 1/* Copyright 2012 STEC, Inc.
2 *
3 * This file is licensed under the terms of the 3-clause
4 * BSD License (http://opensource.org/licenses/BSD-3-Clause)
5 * or the GNU GPL-2.0 (http://www.gnu.org/licenses/gpl-2.0.html),
6 * at your option. Both licenses are also available in the LICENSE file
7 * distributed with this project. This file may not be copied, modified,
8 * or distributed except in accordance with those terms.
9 * Gordoni Waidhofer <gwaidhofer@stec-inc.com>
10 * Initial Driver Design!
11 * Thomas Swann <tswann@stec-inc.com>
12 * Interrupt handling.
13 * Ramprasad Chinthekindi <rchinthekindi@stec-inc.com>
14 * biomode implementation.
15 * Akhil Bhansali <abhansali@stec-inc.com>
16 * Added support for DISCARD / FLUSH and FUA.
17 */
18
19#include <linux/kernel.h>
20#include <linux/module.h>
21#include <linux/init.h>
22#include <linux/pci.h>
23#include <linux/slab.h>
24#include <linux/spinlock.h>
25#include <linux/blkdev.h>
26#include <linux/sched.h>
27#include <linux/interrupt.h>
28#include <linux/compiler.h>
29#include <linux/workqueue.h>
30#include <linux/bitops.h>
31#include <linux/delay.h>
32#include <linux/time.h>
33#include <linux/hdreg.h>
34#include <linux/dma-mapping.h>
35#include <linux/completion.h>
36#include <linux/scatterlist.h>
37#include <linux/version.h>
38#include <linux/err.h>
39#include <linux/scatterlist.h>
40#include <linux/aer.h>
41#include <linux/ctype.h>
42#include <linux/wait.h>
43#include <linux/uio.h>
44#include <scsi/scsi.h>
e67f86b3
AB		 45#include <scsi/sg.h>
46#include <linux/io.h>
47#include <linux/uaccess.h>
4ca90b53 48#include <asm/unaligned.h>
e67f86b3
AB		 49
50#include "skd_s1120.h"
51
52static int skd_dbg_level;
53static int skd_isr_comp_limit = 4;
54
55enum {
56 STEC_LINK_2_5GTS = 0,
57 STEC_LINK_5GTS = 1,
58 STEC_LINK_8GTS = 2,
59 STEC_LINK_UNKNOWN = 0xFF
60};
61
62enum {
63 SKD_FLUSH_INITIALIZER,
64 SKD_FLUSH_ZERO_SIZE_FIRST,
65 SKD_FLUSH_DATA_SECOND,
66};
67
e67f86b3
AB		 68#define SKD_ASSERT(expr) \
69 do { \
70 if (unlikely(!(expr))) { \
71 pr_err("Assertion failed! %s,%s,%s,line=%d\n", \
72 # expr, __FILE__, __func__, __LINE__); \
73 } \
74 } while (0)
75
e67f86b3
AB		 76#define DRV_NAME "skd"
77#define DRV_VERSION "2.2.1"
78#define DRV_BUILD_ID "0260"
79#define PFX DRV_NAME ": "
80#define DRV_BIN_VERSION 0x100
81#define DRV_VER_COMPL "2.2.1." DRV_BUILD_ID
82
83MODULE_AUTHOR("bug-reports: support@stec-inc.com");
84MODULE_LICENSE("Dual BSD/GPL");
85
38d4a1bb 86MODULE_DESCRIPTION("STEC s1120 PCIe SSD block driver (b" DRV_BUILD_ID ")");
e67f86b3
AB		 87MODULE_VERSION(DRV_VERSION "-" DRV_BUILD_ID);
88
89#define PCI_VENDOR_ID_STEC 0x1B39
90#define PCI_DEVICE_ID_S1120 0x0001
91
92#define SKD_FUA_NV (1 << 1)
93#define SKD_MINORS_PER_DEVICE 16
94
95#define SKD_MAX_QUEUE_DEPTH 200u
96
97#define SKD_PAUSE_TIMEOUT (5 * 1000)
98
99#define SKD_N_FITMSG_BYTES (512u)
100
101#define SKD_N_SPECIAL_CONTEXT 32u
102#define SKD_N_SPECIAL_FITMSG_BYTES (128u)
103
104/* SG elements are 32 bytes, so we can make this 4096 and still be under the
105 * 128KB limit. That allows 4096*4K = 16M xfer size
106 */
107#define SKD_N_SG_PER_REQ_DEFAULT 256u
108#define SKD_N_SG_PER_SPECIAL 256u
109
110#define SKD_N_COMPLETION_ENTRY 256u
111#define SKD_N_READ_CAP_BYTES (8u)
112
113#define SKD_N_INTERNAL_BYTES (512u)
114
115/* 5 bits of uniqifier, 0xF800 */
116#define SKD_ID_INCR (0x400)
117#define SKD_ID_TABLE_MASK (3u << 8u)
118#define SKD_ID_RW_REQUEST (0u << 8u)
119#define SKD_ID_INTERNAL (1u << 8u)
120#define SKD_ID_SPECIAL_REQUEST (2u << 8u)
121#define SKD_ID_FIT_MSG (3u << 8u)
122#define SKD_ID_SLOT_MASK 0x00FFu
123#define SKD_ID_SLOT_AND_TABLE_MASK 0x03FFu
124
125#define SKD_N_TIMEOUT_SLOT 4u
126#define SKD_TIMEOUT_SLOT_MASK 3u
127
128#define SKD_N_MAX_SECTORS 2048u
129
130#define SKD_MAX_RETRIES 2u
131
132#define SKD_TIMER_SECONDS(seconds) (seconds)
133#define SKD_TIMER_MINUTES(minutes) ((minutes) * (60))
134
135#define INQ_STD_NBYTES 36
136#define SKD_DISCARD_CDB_LENGTH 24
137
138enum skd_drvr_state {
139 SKD_DRVR_STATE_LOAD,
140 SKD_DRVR_STATE_IDLE,
141 SKD_DRVR_STATE_BUSY,
142 SKD_DRVR_STATE_STARTING,
143 SKD_DRVR_STATE_ONLINE,
144 SKD_DRVR_STATE_PAUSING,
145 SKD_DRVR_STATE_PAUSED,
146 SKD_DRVR_STATE_DRAINING_TIMEOUT,
147 SKD_DRVR_STATE_RESTARTING,
148 SKD_DRVR_STATE_RESUMING,
149 SKD_DRVR_STATE_STOPPING,
150 SKD_DRVR_STATE_FAULT,
151 SKD_DRVR_STATE_DISAPPEARED,
152 SKD_DRVR_STATE_PROTOCOL_MISMATCH,
153 SKD_DRVR_STATE_BUSY_ERASE,
154 SKD_DRVR_STATE_BUSY_SANITIZE,
155 SKD_DRVR_STATE_BUSY_IMMINENT,
156 SKD_DRVR_STATE_WAIT_BOOT,
157 SKD_DRVR_STATE_SYNCING,
158};
159
160#define SKD_WAIT_BOOT_TIMO SKD_TIMER_SECONDS(90u)
161#define SKD_STARTING_TIMO SKD_TIMER_SECONDS(8u)
162#define SKD_RESTARTING_TIMO SKD_TIMER_MINUTES(4u)
163#define SKD_DRAINING_TIMO SKD_TIMER_SECONDS(6u)
164#define SKD_BUSY_TIMO SKD_TIMER_MINUTES(20u)
165#define SKD_STARTED_BUSY_TIMO SKD_TIMER_SECONDS(60u)
166#define SKD_START_WAIT_SECONDS 90u
167
168enum skd_req_state {
169 SKD_REQ_STATE_IDLE,
170 SKD_REQ_STATE_SETUP,
171 SKD_REQ_STATE_BUSY,
172 SKD_REQ_STATE_COMPLETED,
173 SKD_REQ_STATE_TIMEOUT,
174 SKD_REQ_STATE_ABORTED,
175};
176
177enum skd_fit_msg_state {
178 SKD_MSG_STATE_IDLE,
179 SKD_MSG_STATE_BUSY,
180};
181
182enum skd_check_status_action {
183 SKD_CHECK_STATUS_REPORT_GOOD,
184 SKD_CHECK_STATUS_REPORT_SMART_ALERT,
185 SKD_CHECK_STATUS_REQUEUE_REQUEST,
186 SKD_CHECK_STATUS_REPORT_ERROR,
187 SKD_CHECK_STATUS_BUSY_IMMINENT,
188};
189
190struct skd_fitmsg_context {
191 enum skd_fit_msg_state state;
192
193 struct skd_fitmsg_context *next;
194
195 u32 id;
196 u16 outstanding;
197
198 u32 length;
199 u32 offset;
200
201 u8 *msg_buf;
202 dma_addr_t mb_dma_address;
203};
204
205struct skd_request_context {
206 enum skd_req_state state;
207
208 struct skd_request_context *next;
209
210 u16 id;
211 u32 fitmsg_id;
212
213 struct request *req;
e67f86b3
AB		 214 u8 flush_cmd;
215 u8 discard_page;
216
217 u32 timeout_stamp;
218 u8 sg_data_dir;
219 struct scatterlist *sg;
220 u32 n_sg;
221 u32 sg_byte_count;
222
223 struct fit_sg_descriptor *sksg_list;
224 dma_addr_t sksg_dma_address;
225
226 struct fit_completion_entry_v1 completion;
227
228 struct fit_comp_error_info err_info;
229
230};
231#define SKD_DATA_DIR_HOST_TO_CARD 1
232#define SKD_DATA_DIR_CARD_TO_HOST 2
233#define SKD_DATA_DIR_NONE 3 /* especially for DISCARD requests. */
234
235struct skd_special_context {
236 struct skd_request_context req;
237
238 u8 orphaned;
239
240 void *data_buf;
241 dma_addr_t db_dma_address;
242
243 u8 *msg_buf;
244 dma_addr_t mb_dma_address;
245};
246
247struct skd_sg_io {
248 fmode_t mode;
249 void __user *argp;
250
251 struct sg_io_hdr sg;
252
253 u8 cdb[16];
254
255 u32 dxfer_len;
256 u32 iovcnt;
257 struct sg_iovec *iov;
258 struct sg_iovec no_iov_iov;
259
260 struct skd_special_context *skspcl;
261};
262
263typedef enum skd_irq_type {
264 SKD_IRQ_LEGACY,
265 SKD_IRQ_MSI,
266 SKD_IRQ_MSIX
267} skd_irq_type_t;
268
269#define SKD_MAX_BARS 2
270
271struct skd_device {
272 volatile void __iomem *mem_map[SKD_MAX_BARS];
273 resource_size_t mem_phys[SKD_MAX_BARS];
274 u32 mem_size[SKD_MAX_BARS];
275
276 skd_irq_type_t irq_type;
277 u32 msix_count;
278 struct skd_msix_entry *msix_entries;
279
280 struct pci_dev *pdev;
281 int pcie_error_reporting_is_enabled;
282
283 spinlock_t lock;
284 struct gendisk *disk;
285 struct request_queue *queue;
286 struct device *class_dev;
287 int gendisk_on;
288 int sync_done;
289
290 atomic_t device_count;
291 u32 devno;
292 u32 major;
293 char name[32];
294 char isr_name[30];
295
296 enum skd_drvr_state state;
297 u32 drive_state;
298
299 u32 in_flight;
300 u32 cur_max_queue_depth;
301 u32 queue_low_water_mark;
302 u32 dev_max_queue_depth;
303
304 u32 num_fitmsg_context;
305 u32 num_req_context;
306
307 u32 timeout_slot[SKD_N_TIMEOUT_SLOT];
308 u32 timeout_stamp;
309 struct skd_fitmsg_context *skmsg_free_list;
310 struct skd_fitmsg_context *skmsg_table;
311
312 struct skd_request_context *skreq_free_list;
313 struct skd_request_context *skreq_table;
314
315 struct skd_special_context *skspcl_free_list;
316 struct skd_special_context *skspcl_table;
317
318 struct skd_special_context internal_skspcl;
319 u32 read_cap_blocksize;
320 u32 read_cap_last_lba;
321 int read_cap_is_valid;
322 int inquiry_is_valid;
323 u8 inq_serial_num[13]; /*12 chars plus null term */
324 u8 id_str[80]; /* holds a composite name (pci + sernum) */
325
326 u8 skcomp_cycle;
327 u32 skcomp_ix;
328 struct fit_completion_entry_v1 *skcomp_table;
329 struct fit_comp_error_info *skerr_table;
330 dma_addr_t cq_dma_address;
331
332 wait_queue_head_t waitq;
333
334 struct timer_list timer;
335 u32 timer_countdown;
336 u32 timer_substate;
337
338 int n_special;
339 int sgs_per_request;
340 u32 last_mtd;
341
342 u32 proto_ver;
343
344 int dbg_level;
345 u32 connect_time_stamp;
346 int connect_retries;
347#define SKD_MAX_CONNECT_RETRIES 16
348 u32 drive_jiffies;
349
350 u32 timo_slot;
351
e67f86b3 352
38d4a1bb 353 struct work_struct completion_worker;
e67f86b3
AB		 354};
355
356#define SKD_WRITEL(DEV, VAL, OFF) skd_reg_write32(DEV, VAL, OFF)
357#define SKD_READL(DEV, OFF) skd_reg_read32(DEV, OFF)
358#define SKD_WRITEQ(DEV, VAL, OFF) skd_reg_write64(DEV, VAL, OFF)
359
360static inline u32 skd_reg_read32(struct skd_device *skdev, u32 offset)
361{
362 u32 val;
363
364 if (likely(skdev->dbg_level < 2))
365 return readl(skdev->mem_map[1] + offset);
366 else {
367 barrier();
368 val = readl(skdev->mem_map[1] + offset);
369 barrier();
2e44b427 370 pr_debug("%s:%s:%d offset %x = %x\n",
371 skdev->name, __func__, __LINE__, offset, val);
e67f86b3
AB		 372 return val;
373 }
374
375}
376
377static inline void skd_reg_write32(struct skd_device *skdev, u32 val,
378 u32 offset)
379{
380 if (likely(skdev->dbg_level < 2)) {
381 writel(val, skdev->mem_map[1] + offset);
382 barrier();
e67f86b3
AB		 383 } else {
384 barrier();
385 writel(val, skdev->mem_map[1] + offset);
386 barrier();
2e44b427 387 pr_debug("%s:%s:%d offset %x = %x\n",
388 skdev->name, __func__, __LINE__, offset, val);
e67f86b3
AB		 389 }
390}
391
392static inline void skd_reg_write64(struct skd_device *skdev, u64 val,
393 u32 offset)
394{
395 if (likely(skdev->dbg_level < 2)) {
396 writeq(val, skdev->mem_map[1] + offset);
397 barrier();
e67f86b3
AB		 398 } else {
399 barrier();
400 writeq(val, skdev->mem_map[1] + offset);
401 barrier();
2e44b427 402 pr_debug("%s:%s:%d offset %x = %016llx\n",
403 skdev->name, __func__, __LINE__, offset, val);
e67f86b3
AB		 404 }
405}
406
407
408#define SKD_IRQ_DEFAULT SKD_IRQ_MSI
409static int skd_isr_type = SKD_IRQ_DEFAULT;
410
411module_param(skd_isr_type, int, 0444);
412MODULE_PARM_DESC(skd_isr_type, "Interrupt type capability."
413 " (0==legacy, 1==MSI, 2==MSI-X, default==1)");
414
415#define SKD_MAX_REQ_PER_MSG_DEFAULT 1
416static int skd_max_req_per_msg = SKD_MAX_REQ_PER_MSG_DEFAULT;
417
418module_param(skd_max_req_per_msg, int, 0444);
419MODULE_PARM_DESC(skd_max_req_per_msg,
420 "Maximum SCSI requests packed in a single message."
421 " (1-14, default==1)");
422
423#define SKD_MAX_QUEUE_DEPTH_DEFAULT 64
424#define SKD_MAX_QUEUE_DEPTH_DEFAULT_STR "64"
425static int skd_max_queue_depth = SKD_MAX_QUEUE_DEPTH_DEFAULT;
426
427module_param(skd_max_queue_depth, int, 0444);
428MODULE_PARM_DESC(skd_max_queue_depth,
429 "Maximum SCSI requests issued to s1120."
430 " (1-200, default==" SKD_MAX_QUEUE_DEPTH_DEFAULT_STR ")");
431
432static int skd_sgs_per_request = SKD_N_SG_PER_REQ_DEFAULT;
433module_param(skd_sgs_per_request, int, 0444);
434MODULE_PARM_DESC(skd_sgs_per_request,
435 "Maximum SG elements per block request."
436 " (1-4096, default==256)");
437
438static int skd_max_pass_thru = SKD_N_SPECIAL_CONTEXT;
439module_param(skd_max_pass_thru, int, 0444);
440MODULE_PARM_DESC(skd_max_pass_thru,
441 "Maximum SCSI pass-thru at a time." " (1-50, default==32)");
442
443module_param(skd_dbg_level, int, 0444);
444MODULE_PARM_DESC(skd_dbg_level, "s1120 debug level (0,1,2)");
445
446module_param(skd_isr_comp_limit, int, 0444);
447MODULE_PARM_DESC(skd_isr_comp_limit, "s1120 isr comp limit (0=none) default=4");
448
e67f86b3
AB		 449/* Major device number dynamically assigned. */
450static u32 skd_major;
451
e67f86b3
AB		 452static void skd_destruct(struct skd_device *skdev);
453static const struct block_device_operations skd_blockdev_ops;
454static void skd_send_fitmsg(struct skd_device *skdev,
455 struct skd_fitmsg_context *skmsg);
456static void skd_send_special_fitmsg(struct skd_device *skdev,
457 struct skd_special_context *skspcl);
458static void skd_request_fn(struct request_queue *rq);
459static void skd_end_request(struct skd_device *skdev,
460 struct skd_request_context *skreq, int error);
461static int skd_preop_sg_list(struct skd_device *skdev,
462 struct skd_request_context *skreq);
463static void skd_postop_sg_list(struct skd_device *skdev,
464 struct skd_request_context *skreq);
465
466static void skd_restart_device(struct skd_device *skdev);
467static int skd_quiesce_dev(struct skd_device *skdev);
468static int skd_unquiesce_dev(struct skd_device *skdev);
469static void skd_release_special(struct skd_device *skdev,
470 struct skd_special_context *skspcl);
471static void skd_disable_interrupts(struct skd_device *skdev);
472static void skd_isr_fwstate(struct skd_device *skdev);
473static void skd_recover_requests(struct skd_device *skdev, int requeue);
474static void skd_soft_reset(struct skd_device *skdev);
475
476static const char *skd_name(struct skd_device *skdev);
477const char *skd_drive_state_to_str(int state);
478const char *skd_skdev_state_to_str(enum skd_drvr_state state);
479static void skd_log_skdev(struct skd_device *skdev, const char *event);
480static void skd_log_skmsg(struct skd_device *skdev,
481 struct skd_fitmsg_context *skmsg, const char *event);
482static void skd_log_skreq(struct skd_device *skdev,
483 struct skd_request_context *skreq, const char *event);
484
e67f86b3
AB		 485/*
486 *****************************************************************************
487 * READ/WRITE REQUESTS
488 *****************************************************************************
489 */
fcd37eb3 490static void skd_fail_all_pending(struct skd_device *skdev)
e67f86b3
AB		 491{
492 struct request_queue *q = skdev->queue;
493 struct request *req;
494
495 for (;; ) {
496 req = blk_peek_request(q);
497 if (req == NULL)
498 break;
499 blk_start_request(req);
500 __blk_end_request_all(req, -EIO);
501 }
502}
503
e67f86b3
AB		 504static void
505skd_prep_rw_cdb(struct skd_scsi_request *scsi_req,
506 int data_dir, unsigned lba,
507 unsigned count)
508{
509 if (data_dir == READ)
510 scsi_req->cdb[0] = 0x28;
511 else
512 scsi_req->cdb[0] = 0x2a;
513
514 scsi_req->cdb[1] = 0;
515 scsi_req->cdb[2] = (lba & 0xff000000) >> 24;
516 scsi_req->cdb[3] = (lba & 0xff0000) >> 16;
517 scsi_req->cdb[4] = (lba & 0xff00) >> 8;
518 scsi_req->cdb[5] = (lba & 0xff);
519 scsi_req->cdb[6] = 0;
520 scsi_req->cdb[7] = (count & 0xff00) >> 8;
521 scsi_req->cdb[8] = count & 0xff;
522 scsi_req->cdb[9] = 0;
523}
524
525static void
526skd_prep_zerosize_flush_cdb(struct skd_scsi_request *scsi_req,
38d4a1bb 527 struct skd_request_context *skreq)
e67f86b3
AB		 528{
529 skreq->flush_cmd = 1;
530
531 scsi_req->cdb[0] = 0x35;
532 scsi_req->cdb[1] = 0;
533 scsi_req->cdb[2] = 0;
534 scsi_req->cdb[3] = 0;
535 scsi_req->cdb[4] = 0;
536 scsi_req->cdb[5] = 0;
537 scsi_req->cdb[6] = 0;
538 scsi_req->cdb[7] = 0;
539 scsi_req->cdb[8] = 0;
540 scsi_req->cdb[9] = 0;
541}
542
543static void
544skd_prep_discard_cdb(struct skd_scsi_request *scsi_req,
38d4a1bb
MS		 545 struct skd_request_context *skreq,
546 struct page *page,
547 u32 lba, u32 count)
e67f86b3
AB		 548{
549 char *buf;
550 unsigned long len;
551 struct request *req;
552
553 buf = page_address(page);
554 len = SKD_DISCARD_CDB_LENGTH;
555
556 scsi_req->cdb[0] = UNMAP;
557 scsi_req->cdb[8] = len;
558
559 put_unaligned_be16(6 + 16, &buf[0]);
560 put_unaligned_be16(16, &buf[2]);
561 put_unaligned_be64(lba, &buf[8]);
562 put_unaligned_be32(count, &buf[16]);
563
fcd37eb3
JA		 564 req = skreq->req;
565 blk_add_request_payload(req, page, len);
566 req->buffer = buf;
e67f86b3
AB		 567}
568
569static void skd_request_fn_not_online(struct request_queue *q);
570
571static void skd_request_fn(struct request_queue *q)
572{
573 struct skd_device *skdev = q->queuedata;
574 struct skd_fitmsg_context *skmsg = NULL;
575 struct fit_msg_hdr *fmh = NULL;
576 struct skd_request_context *skreq;
577 struct request *req = NULL;
e67f86b3
AB		 578 struct skd_scsi_request *scsi_req;
579 struct page *page;
580 unsigned long io_flags;
581 int error;
582 u32 lba;
583 u32 count;
584 int data_dir;
585 u32 be_lba;
586 u32 be_count;
587 u64 be_dmaa;
588 u64 cmdctxt;
589 u32 timo_slot;
590 void *cmd_ptr;
591 int flush, fua;
592
593 if (skdev->state != SKD_DRVR_STATE_ONLINE) {
594 skd_request_fn_not_online(q);
595 return;
596 }
597
6a5ec65b 598 if (blk_queue_stopped(skdev->queue)) {
e67f86b3
AB		 599 if (skdev->skmsg_free_list == NULL ||
600 skdev->skreq_free_list == NULL ||
601 skdev->in_flight >= skdev->queue_low_water_mark)
602 /* There is still some kind of shortage */
603 return;
604
6a5ec65b 605 queue_flag_clear(QUEUE_FLAG_STOPPED, skdev->queue);
e67f86b3
AB		 606 }
607
608 /*
609 * Stop conditions:
610 * - There are no more native requests
611 * - There are already the maximum number of requests in progress
612 * - There are no more skd_request_context entries
613 * - There are no more FIT msg buffers
614 */
615 for (;; ) {
616
617 flush = fua = 0;
618
fcd37eb3 619 req = blk_peek_request(q);
e67f86b3 620
fcd37eb3
JA		 621 /* Are there any native requests to start? */
622 if (req == NULL)
623 break;
e67f86b3 624
fcd37eb3
JA		 625 lba = (u32)blk_rq_pos(req);
626 count = blk_rq_sectors(req);
627 data_dir = rq_data_dir(req);
628 io_flags = req->cmd_flags;
e67f86b3 629
fcd37eb3
JA		 630 if (io_flags & REQ_FLUSH)
631 flush++;
e67f86b3 632
fcd37eb3
JA		 633 if (io_flags & REQ_FUA)
634 fua++;
e67f86b3 635
fcd37eb3
JA		 636 pr_debug("%s:%s:%d new req=%p lba=%u(0x%x) "
637 "count=%u(0x%x) dir=%d\n",
638 skdev->name, __func__, __LINE__,
639 req, lba, lba, count, count, data_dir);
e67f86b3 640
38d4a1bb 641 /* At this point we know there is a request */
e67f86b3
AB		 642
643 /* Are too many requets already in progress? */
644 if (skdev->in_flight >= skdev->cur_max_queue_depth) {
2e44b427 645 pr_debug("%s:%s:%d qdepth %d, limit %d\n",
646 skdev->name, __func__, __LINE__,
647 skdev->in_flight, skdev->cur_max_queue_depth);
e67f86b3
AB		 648 break;
649 }
650
651 /* Is a skd_request_context available? */
652 skreq = skdev->skreq_free_list;
653 if (skreq == NULL) {
2e44b427 654 pr_debug("%s:%s:%d Out of req=%p\n",
655 skdev->name, __func__, __LINE__, q);
e67f86b3
AB		 656 break;
657 }
658 SKD_ASSERT(skreq->state == SKD_REQ_STATE_IDLE);
659 SKD_ASSERT((skreq->id & SKD_ID_INCR) == 0);
660
661 /* Now we check to see if we can get a fit msg */
662 if (skmsg == NULL) {
663 if (skdev->skmsg_free_list == NULL) {
2e44b427 664 pr_debug("%s:%s:%d Out of msg\n",
665 skdev->name, __func__, __LINE__);
e67f86b3
AB		 666 break;
667 }
668 }
669
670 skreq->flush_cmd = 0;
671 skreq->n_sg = 0;
672 skreq->sg_byte_count = 0;
673 skreq->discard_page = 0;
674
675 /*
38d4a1bb 676 * OK to now dequeue request from q.
e67f86b3
AB		 677 *
678 * At this point we are comitted to either start or reject
679 * the native request. Note that skd_request_context is
680 * available but is still at the head of the free list.
681 */
fcd37eb3
JA		 682 blk_start_request(req);
683 skreq->req = req;
684 skreq->fitmsg_id = 0;
e67f86b3
AB		 685
686 /* Either a FIT msg is in progress or we have to start one. */
687 if (skmsg == NULL) {
688 /* Are there any FIT msg buffers available? */
689 skmsg = skdev->skmsg_free_list;
690 if (skmsg == NULL) {
2e44b427 691 pr_debug("%s:%s:%d Out of msg skdev=%p\n",
692 skdev->name, __func__, __LINE__,
693 skdev);
e67f86b3
AB		 694 break;
695 }
696 SKD_ASSERT(skmsg->state == SKD_MSG_STATE_IDLE);
697 SKD_ASSERT((skmsg->id & SKD_ID_INCR) == 0);
698
699 skdev->skmsg_free_list = skmsg->next;
700
701 skmsg->state = SKD_MSG_STATE_BUSY;
702 skmsg->id += SKD_ID_INCR;
703
704 /* Initialize the FIT msg header */
705 fmh = (struct fit_msg_hdr *)skmsg->msg_buf;
706 memset(fmh, 0, sizeof(*fmh));
707 fmh->protocol_id = FIT_PROTOCOL_ID_SOFIT;
708 skmsg->length = sizeof(*fmh);
709 }
710
711 skreq->fitmsg_id = skmsg->id;
712
713 /*
714 * Note that a FIT msg may have just been started
715 * but contains no SoFIT requests yet.
716 */
717
718 /*
719 * Transcode the request, checking as we go. The outcome of
720 * the transcoding is represented by the error variable.
721 */
722 cmd_ptr = &skmsg->msg_buf[skmsg->length];
723 memset(cmd_ptr, 0, 32);
724
725 be_lba = cpu_to_be32(lba);
726 be_count = cpu_to_be32(count);
727 be_dmaa = cpu_to_be64((u64)skreq->sksg_dma_address);
728 cmdctxt = skreq->id + SKD_ID_INCR;
729
730 scsi_req = cmd_ptr;
731 scsi_req->hdr.tag = cmdctxt;
732 scsi_req->hdr.sg_list_dma_address = be_dmaa;
733
734 if (data_dir == READ)
735 skreq->sg_data_dir = SKD_DATA_DIR_CARD_TO_HOST;
736 else
737 skreq->sg_data_dir = SKD_DATA_DIR_HOST_TO_CARD;
738
739 if (io_flags & REQ_DISCARD) {
740 page = alloc_page(GFP_ATOMIC | __GFP_ZERO);
741 if (!page) {
742 pr_err("request_fn:Page allocation failed.\n");
743 skd_end_request(skdev, skreq, -ENOMEM);
744 break;
745 }
746 skreq->discard_page = 1;
747 skd_prep_discard_cdb(scsi_req, skreq, page, lba, count);
748
749 } else if (flush == SKD_FLUSH_ZERO_SIZE_FIRST) {
750 skd_prep_zerosize_flush_cdb(scsi_req, skreq);
751 SKD_ASSERT(skreq->flush_cmd == 1);
752
753 } else {
754 skd_prep_rw_cdb(scsi_req, data_dir, lba, count);
755 }
756
757 if (fua)
758 scsi_req->cdb[1] |= SKD_FUA_NV;
759
fcd37eb3 760 if (!req->bio)
e67f86b3
AB		 761 goto skip_sg;
762
763 error = skd_preop_sg_list(skdev, skreq);
764
765 if (error != 0) {
766 /*
767 * Complete the native request with error.
768 * Note that the request context is still at the
769 * head of the free list, and that the SoFIT request
770 * was encoded into the FIT msg buffer but the FIT
771 * msg length has not been updated. In short, the
772 * only resource that has been allocated but might
773 * not be used is that the FIT msg could be empty.
774 */
2e44b427 775 pr_debug("%s:%s:%d error Out\n",
776 skdev->name, __func__, __LINE__);
e67f86b3
AB		 777 skd_end_request(skdev, skreq, error);
778 continue;
779 }
780
781skip_sg:
782 scsi_req->hdr.sg_list_len_bytes =
783 cpu_to_be32(skreq->sg_byte_count);
784
785 /* Complete resource allocations. */
786 skdev->skreq_free_list = skreq->next;
787 skreq->state = SKD_REQ_STATE_BUSY;
788 skreq->id += SKD_ID_INCR;
789
790 skmsg->length += sizeof(struct skd_scsi_request);
791 fmh->num_protocol_cmds_coalesced++;
792
793 /*
794 * Update the active request counts.
795 * Capture the timeout timestamp.
796 */
797 skreq->timeout_stamp = skdev->timeout_stamp;
798 timo_slot = skreq->timeout_stamp & SKD_TIMEOUT_SLOT_MASK;
799 skdev->timeout_slot[timo_slot]++;
800 skdev->in_flight++;
2e44b427 801 pr_debug("%s:%s:%d req=0x%x busy=%d\n",
802 skdev->name, __func__, __LINE__,
803 skreq->id, skdev->in_flight);
e67f86b3
AB		 804
805 /*
806 * If the FIT msg buffer is full send it.
807 */
808 if (skmsg->length >= SKD_N_FITMSG_BYTES ||
809 fmh->num_protocol_cmds_coalesced >= skd_max_req_per_msg) {
810 skd_send_fitmsg(skdev, skmsg);
811 skmsg = NULL;
812 fmh = NULL;
813 }
814 }
815
816 /*
817 * Is a FIT msg in progress? If it is empty put the buffer back
818 * on the free list. If it is non-empty send what we got.
819 * This minimizes latency when there are fewer requests than
820 * what fits in a FIT msg.
821 */
822 if (skmsg != NULL) {
823 /* Bigger than just a FIT msg header? */
824 if (skmsg->length > sizeof(struct fit_msg_hdr)) {
2e44b427 825 pr_debug("%s:%s:%d sending msg=%p, len %d\n",
826 skdev->name, __func__, __LINE__,
827 skmsg, skmsg->length);
e67f86b3
AB		 828 skd_send_fitmsg(skdev, skmsg);
829 } else {
830 /*
831 * The FIT msg is empty. It means we got started
832 * on the msg, but the requests were rejected.
833 */
834 skmsg->state = SKD_MSG_STATE_IDLE;
835 skmsg->id += SKD_ID_INCR;
836 skmsg->next = skdev->skmsg_free_list;
837 skdev->skmsg_free_list = skmsg;
838 }
839 skmsg = NULL;
840 fmh = NULL;
841 }
842
843 /*
844 * If req is non-NULL it means there is something to do but
845 * we are out of a resource.
846 */
fcd37eb3 847 if (req)
6a5ec65b 848 blk_stop_queue(skdev->queue);
e67f86b3
AB		 849}
850
38d4a1bb
MS		 851static void skd_end_request(struct skd_device *skdev,
852 struct skd_request_context *skreq, int error)
e67f86b3
AB		 853{
854 struct request *req = skreq->req;
855 unsigned int io_flags = req->cmd_flags;
856
857 if ((io_flags & REQ_DISCARD) &&
858 (skreq->discard_page == 1)) {
38d4a1bb 859 pr_debug("%s:%s:%d, free the page!",
2e44b427 860 skdev->name, __func__, __LINE__);
e67f86b3
AB		 861 free_page((unsigned long)req->buffer);
862 req->buffer = NULL;
863 }
864
865 if (unlikely(error)) {
866 struct request *req = skreq->req;
867 char *cmd = (rq_data_dir(req) == READ) ? "read" : "write";
868 u32 lba = (u32)blk_rq_pos(req);
869 u32 count = blk_rq_sectors(req);
870
871 pr_err("(%s): Error cmd=%s sect=%u count=%u id=0x%x\n",
872 skd_name(skdev), cmd, lba, count, skreq->id);
873 } else
2e44b427 874 pr_debug("%s:%s:%d id=0x%x error=%d\n",
875 skdev->name, __func__, __LINE__, skreq->id, error);
e67f86b3
AB		 876
877 __blk_end_request_all(skreq->req, error);
878}
879
fcd37eb3 880static int skd_preop_sg_list(struct skd_device *skdev,
38d4a1bb 881 struct skd_request_context *skreq)
e67f86b3
AB		 882{
883 struct request *req = skreq->req;
884 int writing = skreq->sg_data_dir == SKD_DATA_DIR_HOST_TO_CARD;
885 int pci_dir = writing ? PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE;
886 struct scatterlist *sg = &skreq->sg[0];
887 int n_sg;
888 int i;
889
890 skreq->sg_byte_count = 0;
891
892 /* SKD_ASSERT(skreq->sg_data_dir == SKD_DATA_DIR_HOST_TO_CARD ||
893 skreq->sg_data_dir == SKD_DATA_DIR_CARD_TO_HOST); */
894
895 n_sg = blk_rq_map_sg(skdev->queue, req, sg);
896 if (n_sg <= 0)
897 return -EINVAL;
898
899 /*
900 * Map scatterlist to PCI bus addresses.
901 * Note PCI might change the number of entries.
902 */
903 n_sg = pci_map_sg(skdev->pdev, sg, n_sg, pci_dir);
904 if (n_sg <= 0)
905 return -EINVAL;
906
907 SKD_ASSERT(n_sg <= skdev->sgs_per_request);
908
909 skreq->n_sg = n_sg;
910
911 for (i = 0; i < n_sg; i++) {
912 struct fit_sg_descriptor *sgd = &skreq->sksg_list[i];
913 u32 cnt = sg_dma_len(&sg[i]);
914 uint64_t dma_addr = sg_dma_address(&sg[i]);
915
916 sgd->control = FIT_SGD_CONTROL_NOT_LAST;
917 sgd->byte_count = cnt;
918 skreq->sg_byte_count += cnt;
919 sgd->host_side_addr = dma_addr;
920 sgd->dev_side_addr = 0;
921 }
922
923 skreq->sksg_list[n_sg - 1].next_desc_ptr = 0LL;
924 skreq->sksg_list[n_sg - 1].control = FIT_SGD_CONTROL_LAST;
925
926 if (unlikely(skdev->dbg_level > 1)) {
2e44b427 927 pr_debug("%s:%s:%d skreq=%x sksg_list=%p sksg_dma=%llx\n",
928 skdev->name, __func__, __LINE__,
929 skreq->id, skreq->sksg_list, skreq->sksg_dma_address);
e67f86b3
AB		 930 for (i = 0; i < n_sg; i++) {
931 struct fit_sg_descriptor *sgd = &skreq->sksg_list[i];
2e44b427 932 pr_debug("%s:%s:%d sg[%d] count=%u ctrl=0x%x "
933 "addr=0x%llx next=0x%llx\n",
934 skdev->name, __func__, __LINE__,
935 i, sgd->byte_count, sgd->control,
936 sgd->host_side_addr, sgd->next_desc_ptr);
e67f86b3
AB		 937 }
938 }
939
940 return 0;
941}
942
fcd37eb3 943static void skd_postop_sg_list(struct skd_device *skdev,
38d4a1bb 944 struct skd_request_context *skreq)
e67f86b3
AB		 945{
946 int writing = skreq->sg_data_dir == SKD_DATA_DIR_HOST_TO_CARD;
947 int pci_dir = writing ? PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE;
948
949 /*
950 * restore the next ptr for next IO request so we
951 * don't have to set it every time.
952 */
953 skreq->sksg_list[skreq->n_sg - 1].next_desc_ptr =
954 skreq->sksg_dma_address +
955 ((skreq->n_sg) * sizeof(struct fit_sg_descriptor));
956 pci_unmap_sg(skdev->pdev, &skreq->sg[0], skreq->n_sg, pci_dir);
957}
958
e67f86b3
AB		 959static void skd_request_fn_not_online(struct request_queue *q)
960{
961 struct skd_device *skdev = q->queuedata;
962 int error;
963
964 SKD_ASSERT(skdev->state != SKD_DRVR_STATE_ONLINE);
965
966 skd_log_skdev(skdev, "req_not_online");
967 switch (skdev->state) {
968 case SKD_DRVR_STATE_PAUSING:
969 case SKD_DRVR_STATE_PAUSED:
970 case SKD_DRVR_STATE_STARTING:
971 case SKD_DRVR_STATE_RESTARTING:
972 case SKD_DRVR_STATE_WAIT_BOOT:
973 /* In case of starting, we haven't started the queue,
974 * so we can't get here... but requests are
975 * possibly hanging out waiting for us because we
976 * reported the dev/skd0 already. They'll wait
977 * forever if connect doesn't complete.
978 * What to do??? delay dev/skd0 ??
979 */
980 case SKD_DRVR_STATE_BUSY:
981 case SKD_DRVR_STATE_BUSY_IMMINENT:
982 case SKD_DRVR_STATE_BUSY_ERASE:
983 case SKD_DRVR_STATE_DRAINING_TIMEOUT:
984 return;
985
986 case SKD_DRVR_STATE_BUSY_SANITIZE:
987 case SKD_DRVR_STATE_STOPPING:
988 case SKD_DRVR_STATE_SYNCING:
989 case SKD_DRVR_STATE_FAULT:
990 case SKD_DRVR_STATE_DISAPPEARED:
991 default:
992 error = -EIO;
993 break;
994 }
995
996 /* If we get here, terminate all pending block requeusts
997 * with EIO and any scsi pass thru with appropriate sense
998 */
999
1000 skd_fail_all_pending(skdev);
1001}
1002
1003/*
1004 *****************************************************************************
1005 * TIMER
1006 *****************************************************************************
1007 */
1008
1009static void skd_timer_tick_not_online(struct skd_device *skdev);
1010
1011static void skd_timer_tick(ulong arg)
1012{
1013 struct skd_device *skdev = (struct skd_device *)arg;
1014
1015 u32 timo_slot;
1016 u32 overdue_timestamp;
1017 unsigned long reqflags;
1018 u32 state;
1019
1020 if (skdev->state == SKD_DRVR_STATE_FAULT)
1021 /* The driver has declared fault, and we want it to
1022 * stay that way until driver is reloaded.
1023 */
1024 return;
1025
1026 spin_lock_irqsave(&skdev->lock, reqflags);
1027
1028 state = SKD_READL(skdev, FIT_STATUS);
1029 state &= FIT_SR_DRIVE_STATE_MASK;
1030 if (state != skdev->drive_state)
1031 skd_isr_fwstate(skdev);
1032
1033 if (skdev->state != SKD_DRVR_STATE_ONLINE) {
1034 skd_timer_tick_not_online(skdev);
1035 goto timer_func_out;
1036 }
1037 skdev->timeout_stamp++;
1038 timo_slot = skdev->timeout_stamp & SKD_TIMEOUT_SLOT_MASK;
1039
1040 /*
1041 * All requests that happened during the previous use of
1042 * this slot should be done by now. The previous use was
1043 * over 7 seconds ago.
1044 */
1045 if (skdev->timeout_slot[timo_slot] == 0)
1046 goto timer_func_out;
1047
1048 /* Something is overdue */
1049 overdue_timestamp = skdev->timeout_stamp - SKD_N_TIMEOUT_SLOT;
1050
2e44b427 1051 pr_debug("%s:%s:%d found %d timeouts, draining busy=%d\n",
1052 skdev->name, __func__, __LINE__,
1053 skdev->timeout_slot[timo_slot], skdev->in_flight);
e67f86b3
AB		 1054 pr_err("(%s): Overdue IOs (%d), busy %d\n",
1055 skd_name(skdev), skdev->timeout_slot[timo_slot],
1056 skdev->in_flight);
1057
1058 skdev->timer_countdown = SKD_DRAINING_TIMO;
1059 skdev->state = SKD_DRVR_STATE_DRAINING_TIMEOUT;
1060 skdev->timo_slot = timo_slot;
6a5ec65b 1061 blk_stop_queue(skdev->queue);
e67f86b3
AB		 1062
1063timer_func_out:
1064 mod_timer(&skdev->timer, (jiffies + HZ));
1065
1066 spin_unlock_irqrestore(&skdev->lock, reqflags);
1067}
1068
1069static void skd_timer_tick_not_online(struct skd_device *skdev)
1070{
1071 switch (skdev->state) {
1072 case SKD_DRVR_STATE_IDLE:
1073 case SKD_DRVR_STATE_LOAD:
1074 break;
1075 case SKD_DRVR_STATE_BUSY_SANITIZE:
2e44b427 1076 pr_debug("%s:%s:%d drive busy sanitize[%x], driver[%x]\n",
1077 skdev->name, __func__, __LINE__,
1078 skdev->drive_state, skdev->state);
e67f86b3
AB		 1079 /* If we've been in sanitize for 3 seconds, we figure we're not
1080 * going to get anymore completions, so recover requests now
1081 */
1082 if (skdev->timer_countdown > 0) {
1083 skdev->timer_countdown--;
1084 return;
1085 }
1086 skd_recover_requests(skdev, 0);
1087 break;
1088
1089 case SKD_DRVR_STATE_BUSY:
1090 case SKD_DRVR_STATE_BUSY_IMMINENT:
1091 case SKD_DRVR_STATE_BUSY_ERASE:
2e44b427 1092 pr_debug("%s:%s:%d busy[%x], countdown=%d\n",
1093 skdev->name, __func__, __LINE__,
1094 skdev->state, skdev->timer_countdown);
e67f86b3
AB		 1095 if (skdev->timer_countdown > 0) {
1096 skdev->timer_countdown--;
1097 return;
1098 }
2e44b427 1099 pr_debug("%s:%s:%d busy[%x], timedout=%d, restarting device.",
1100 skdev->name, __func__, __LINE__,
1101 skdev->state, skdev->timer_countdown);
e67f86b3
AB		 1102 skd_restart_device(skdev);
1103 break;
1104
1105 case SKD_DRVR_STATE_WAIT_BOOT:
1106 case SKD_DRVR_STATE_STARTING:
1107 if (skdev->timer_countdown > 0) {
1108 skdev->timer_countdown--;
1109 return;
1110 }
1111 /* For now, we fault the drive. Could attempt resets to
1112 * revcover at some point. */
1113 skdev->state = SKD_DRVR_STATE_FAULT;
1114
1115 pr_err("(%s): DriveFault Connect Timeout (%x)\n",
1116 skd_name(skdev), skdev->drive_state);
1117
1118 /*start the queue so we can respond with error to requests */
1119 /* wakeup anyone waiting for startup complete */
6a5ec65b 1120 blk_start_queue(skdev->queue);
e67f86b3
AB		 1121 skdev->gendisk_on = -1;
1122 wake_up_interruptible(&skdev->waitq);
1123 break;
1124
1125 case SKD_DRVR_STATE_ONLINE:
1126 /* shouldn't get here. */
1127 break;
1128
1129 case SKD_DRVR_STATE_PAUSING:
1130 case SKD_DRVR_STATE_PAUSED:
1131 break;
1132
1133 case SKD_DRVR_STATE_DRAINING_TIMEOUT:
2e44b427 1134 pr_debug("%s:%s:%d "
1135 "draining busy [%d] tick[%d] qdb[%d] tmls[%d]\n",
1136 skdev->name, __func__, __LINE__,
1137 skdev->timo_slot,
1138 skdev->timer_countdown,
1139 skdev->in_flight,
1140 skdev->timeout_slot[skdev->timo_slot]);
e67f86b3
AB		 1141 /* if the slot has cleared we can let the I/O continue */
1142 if (skdev->timeout_slot[skdev->timo_slot] == 0) {
2e44b427 1143 pr_debug("%s:%s:%d Slot drained, starting queue.\n",
1144 skdev->name, __func__, __LINE__);
e67f86b3 1145 skdev->state = SKD_DRVR_STATE_ONLINE;
6a5ec65b 1146 blk_start_queue(skdev->queue);
e67f86b3
AB		 1147 return;
1148 }
1149 if (skdev->timer_countdown > 0) {
1150 skdev->timer_countdown--;
1151 return;
1152 }
1153 skd_restart_device(skdev);
1154 break;
1155
1156 case SKD_DRVR_STATE_RESTARTING:
1157 if (skdev->timer_countdown > 0) {
1158 skdev->timer_countdown--;
1159 return;
1160 }
1161 /* For now, we fault the drive. Could attempt resets to
1162 * revcover at some point. */
1163 skdev->state = SKD_DRVR_STATE_FAULT;
1164 pr_err("(%s): DriveFault Reconnect Timeout (%x)\n",
1165 skd_name(skdev), skdev->drive_state);
1166
1167 /*
1168 * Recovering does two things:
1169 * 1. completes IO with error
1170 * 2. reclaims dma resources
1171 * When is it safe to recover requests?
1172 * - if the drive state is faulted
1173 * - if the state is still soft reset after out timeout
1174 * - if the drive registers are dead (state = FF)
1175 * If it is "unsafe", we still need to recover, so we will
1176 * disable pci bus mastering and disable our interrupts.
1177 */
1178
1179 if ((skdev->drive_state == FIT_SR_DRIVE_SOFT_RESET) ||
1180 (skdev->drive_state == FIT_SR_DRIVE_FAULT) ||
1181 (skdev->drive_state == FIT_SR_DRIVE_STATE_MASK))
1182 /* It never came out of soft reset. Try to
1183 * recover the requests and then let them
1184 * fail. This is to mitigate hung processes. */
1185 skd_recover_requests(skdev, 0);
1186 else {
1187 pr_err("(%s): Disable BusMaster (%x)\n",
1188 skd_name(skdev), skdev->drive_state);
1189 pci_disable_device(skdev->pdev);
1190 skd_disable_interrupts(skdev);
1191 skd_recover_requests(skdev, 0);
1192 }
1193
1194 /*start the queue so we can respond with error to requests */
1195 /* wakeup anyone waiting for startup complete */
6a5ec65b 1196 blk_start_queue(skdev->queue);
e67f86b3
AB		 1197 skdev->gendisk_on = -1;
1198 wake_up_interruptible(&skdev->waitq);
1199 break;
1200
1201 case SKD_DRVR_STATE_RESUMING:
1202 case SKD_DRVR_STATE_STOPPING:
1203 case SKD_DRVR_STATE_SYNCING:
1204 case SKD_DRVR_STATE_FAULT:
1205 case SKD_DRVR_STATE_DISAPPEARED:
1206 default:
1207 break;
1208 }
1209}
1210
1211static int skd_start_timer(struct skd_device *skdev)
1212{
1213 int rc;
1214
1215 init_timer(&skdev->timer);
1216 setup_timer(&skdev->timer, skd_timer_tick, (ulong)skdev);
1217
1218 rc = mod_timer(&skdev->timer, (jiffies + HZ));
1219 if (rc)
1220 pr_err("%s: failed to start timer %d\n",
1221 __func__, rc);
1222 return rc;
1223}
1224
1225static void skd_kill_timer(struct skd_device *skdev)
1226{
1227 del_timer_sync(&skdev->timer);
1228}
1229
1230/*
1231 *****************************************************************************
1232 * IOCTL
1233 *****************************************************************************
1234 */
1235static int skd_ioctl_sg_io(struct skd_device *skdev,
1236 fmode_t mode, void __user *argp);
1237static int skd_sg_io_get_and_check_args(struct skd_device *skdev,
1238 struct skd_sg_io *sksgio);
1239static int skd_sg_io_obtain_skspcl(struct skd_device *skdev,
1240 struct skd_sg_io *sksgio);
1241static int skd_sg_io_prep_buffering(struct skd_device *skdev,
1242 struct skd_sg_io *sksgio);
1243static int skd_sg_io_copy_buffer(struct skd_device *skdev,
1244 struct skd_sg_io *sksgio, int dxfer_dir);
1245static int skd_sg_io_send_fitmsg(struct skd_device *skdev,
1246 struct skd_sg_io *sksgio);
1247static int skd_sg_io_await(struct skd_device *skdev, struct skd_sg_io *sksgio);
1248static int skd_sg_io_release_skspcl(struct skd_device *skdev,
1249 struct skd_sg_io *sksgio);
1250static int skd_sg_io_put_status(struct skd_device *skdev,
1251 struct skd_sg_io *sksgio);
1252
1253static void skd_complete_special(struct skd_device *skdev,
1254 volatile struct fit_completion_entry_v1
1255 *skcomp,
1256 volatile struct fit_comp_error_info *skerr,
1257 struct skd_special_context *skspcl);
1258
1259static int skd_bdev_ioctl(struct block_device *bdev, fmode_t mode,
1260 uint cmd_in, ulong arg)
1261{
1262 int rc = 0;
1263 struct gendisk *disk = bdev->bd_disk;
1264 struct skd_device *skdev = disk->private_data;
1265 void __user *p = (void *)arg;
1266
2e44b427 1267 pr_debug("%s:%s:%d %s: CMD[%s] ioctl mode 0x%x, cmd 0x%x arg %0lx\n",
1268 skdev->name, __func__, __LINE__,
1269 disk->disk_name, current->comm, mode, cmd_in, arg);
e67f86b3
AB		 1270
1271 if (!capable(CAP_SYS_ADMIN))
1272 return -EPERM;
1273
1274 switch (cmd_in) {
1275 case SG_SET_TIMEOUT:
1276 case SG_GET_TIMEOUT:
1277 case SG_GET_VERSION_NUM:
1278 rc = scsi_cmd_ioctl(disk->queue, disk, mode, cmd_in, p);
1279 break;
1280 case SG_IO:
1281 rc = skd_ioctl_sg_io(skdev, mode, p);
1282 break;
1283
1284 default:
1285 rc = -ENOTTY;
1286 break;
1287 }
1288
2e44b427 1289 pr_debug("%s:%s:%d %s: completion rc %d\n",
1290 skdev->name, __func__, __LINE__, disk->disk_name, rc);
e67f86b3
AB		 1291 return rc;
1292}
1293
1294static int skd_ioctl_sg_io(struct skd_device *skdev, fmode_t mode,
1295 void __user *argp)
1296{
1297 int rc;
1298 struct skd_sg_io sksgio;
1299
1300 memset(&sksgio, 0, sizeof(sksgio));
1301 sksgio.mode = mode;
1302 sksgio.argp = argp;
1303 sksgio.iov = &sksgio.no_iov_iov;
1304
1305 switch (skdev->state) {
1306 case SKD_DRVR_STATE_ONLINE:
1307 case SKD_DRVR_STATE_BUSY_IMMINENT:
1308 break;
1309
1310 default:
2e44b427 1311 pr_debug("%s:%s:%d drive not online\n",
1312 skdev->name, __func__, __LINE__);
e67f86b3
AB		 1313 rc = -ENXIO;
1314 goto out;
1315 }
1316
f721bb0d
AB		 1317 rc = skd_sg_io_get_and_check_args(skdev, &sksgio);
1318 if (rc)
1319 goto out;
1320
1321 rc = skd_sg_io_obtain_skspcl(skdev, &sksgio);
1322 if (rc)
1323 goto out;
1324
1325 rc = skd_sg_io_prep_buffering(skdev, &sksgio);
1326 if (rc)
1327 goto out;
1328
1329 rc = skd_sg_io_copy_buffer(skdev, &sksgio, SG_DXFER_TO_DEV);
1330 if (rc)
e67f86b3
AB		 1331 goto out;
1332
f721bb0d
AB		 1333 rc = skd_sg_io_send_fitmsg(skdev, &sksgio);
1334 if (rc)
e67f86b3
AB		 1335 goto out;
1336
f721bb0d
AB		 1337 rc = skd_sg_io_await(skdev, &sksgio);
1338 if (rc)
1339 goto out;
1340
1341 rc = skd_sg_io_copy_buffer(skdev, &sksgio, SG_DXFER_FROM_DEV);
1342 if (rc)
1343 goto out;
1344
1345 rc = skd_sg_io_put_status(skdev, &sksgio);
1346 if (rc)
e67f86b3
AB		 1347 goto out;
1348
1349 rc = 0;
1350
1351out:
1352 skd_sg_io_release_skspcl(skdev, &sksgio);
1353
1354 if (sksgio.iov != NULL && sksgio.iov != &sksgio.no_iov_iov)
1355 kfree(sksgio.iov);
1356 return rc;
1357}
1358
1359static int skd_sg_io_get_and_check_args(struct skd_device *skdev,
1360 struct skd_sg_io *sksgio)
1361{
1362 struct sg_io_hdr *sgp = &sksgio->sg;
1363 int i, acc;
1364
1365 if (!access_ok(VERIFY_WRITE, sksgio->argp, sizeof(sg_io_hdr_t))) {
2e44b427 1366 pr_debug("%s:%s:%d access sg failed %p\n",
1367 skdev->name, __func__, __LINE__, sksgio->argp);
e67f86b3
AB		 1368 return -EFAULT;
1369 }
1370
1371 if (__copy_from_user(sgp, sksgio->argp, sizeof(sg_io_hdr_t))) {
2e44b427 1372 pr_debug("%s:%s:%d copy_from_user sg failed %p\n",
1373 skdev->name, __func__, __LINE__, sksgio->argp);
e67f86b3
AB		 1374 return -EFAULT;
1375 }
1376
1377 if (sgp->interface_id != SG_INTERFACE_ID_ORIG) {
2e44b427 1378 pr_debug("%s:%s:%d interface_id invalid 0x%x\n",
1379 skdev->name, __func__, __LINE__, sgp->interface_id);
e67f86b3
AB		 1380 return -EINVAL;
1381 }
1382
1383 if (sgp->cmd_len > sizeof(sksgio->cdb)) {
2e44b427 1384 pr_debug("%s:%s:%d cmd_len invalid %d\n",
1385 skdev->name, __func__, __LINE__, sgp->cmd_len);
e67f86b3
AB		 1386 return -EINVAL;
1387 }
1388
1389 if (sgp->iovec_count > 256) {
2e44b427 1390 pr_debug("%s:%s:%d iovec_count invalid %d\n",
1391 skdev->name, __func__, __LINE__, sgp->iovec_count);
e67f86b3
AB		 1392 return -EINVAL;
1393 }
1394
1395 if (sgp->dxfer_len > (PAGE_SIZE * SKD_N_SG_PER_SPECIAL)) {
2e44b427 1396 pr_debug("%s:%s:%d dxfer_len invalid %d\n",
1397 skdev->name, __func__, __LINE__, sgp->dxfer_len);
e67f86b3
AB		 1398 return -EINVAL;
1399 }
1400
1401 switch (sgp->dxfer_direction) {
1402 case SG_DXFER_NONE:
1403 acc = -1;
1404 break;
1405
1406 case SG_DXFER_TO_DEV:
1407 acc = VERIFY_READ;
1408 break;
1409
1410 case SG_DXFER_FROM_DEV:
1411 case SG_DXFER_TO_FROM_DEV:
1412 acc = VERIFY_WRITE;
1413 break;
1414
1415 default:
2e44b427 1416 pr_debug("%s:%s:%d dxfer_dir invalid %d\n",
1417 skdev->name, __func__, __LINE__, sgp->dxfer_direction);
e67f86b3
AB		 1418 return -EINVAL;
1419 }
1420
1421 if (copy_from_user(sksgio->cdb, sgp->cmdp, sgp->cmd_len)) {
2e44b427 1422 pr_debug("%s:%s:%d copy_from_user cmdp failed %p\n",
1423 skdev->name, __func__, __LINE__, sgp->cmdp);
e67f86b3
AB		 1424 return -EFAULT;
1425 }
1426
1427 if (sgp->mx_sb_len != 0) {
1428 if (!access_ok(VERIFY_WRITE, sgp->sbp, sgp->mx_sb_len)) {
2e44b427 1429 pr_debug("%s:%s:%d access sbp failed %p\n",
1430 skdev->name, __func__, __LINE__, sgp->sbp);
e67f86b3
AB		 1431 return -EFAULT;
1432 }
1433 }
1434
1435 if (sgp->iovec_count == 0) {
1436 sksgio->iov[0].iov_base = sgp->dxferp;
1437 sksgio->iov[0].iov_len = sgp->dxfer_len;
1438 sksgio->iovcnt = 1;
1439 sksgio->dxfer_len = sgp->dxfer_len;
1440 } else {
1441 struct sg_iovec *iov;
1442 uint nbytes = sizeof(*iov) * sgp->iovec_count;
1443 size_t iov_data_len;
1444
1445 iov = kmalloc(nbytes, GFP_KERNEL);
1446 if (iov == NULL) {
2e44b427 1447 pr_debug("%s:%s:%d alloc iovec failed %d\n",
1448 skdev->name, __func__, __LINE__,
1449 sgp->iovec_count);
e67f86b3
AB		 1450 return -ENOMEM;
1451 }
1452 sksgio->iov = iov;
1453 sksgio->iovcnt = sgp->iovec_count;
1454
1455 if (copy_from_user(iov, sgp->dxferp, nbytes)) {
2e44b427 1456 pr_debug("%s:%s:%d copy_from_user iovec failed %p\n",
1457 skdev->name, __func__, __LINE__, sgp->dxferp);
e67f86b3
AB		 1458 return -EFAULT;
1459 }
1460
1461 /*
1462 * Sum up the vecs, making sure they don't overflow
1463 */
1464 iov_data_len = 0;
1465 for (i = 0; i < sgp->iovec_count; i++) {
1466 if (iov_data_len + iov[i].iov_len < iov_data_len)
1467 return -EINVAL;
1468 iov_data_len += iov[i].iov_len;
1469 }
1470
1471 /* SG_IO howto says that the shorter of the two wins */
1472 if (sgp->dxfer_len < iov_data_len) {
1473 sksgio->iovcnt = iov_shorten((struct iovec *)iov,
1474 sgp->iovec_count,
1475 sgp->dxfer_len);
1476 sksgio->dxfer_len = sgp->dxfer_len;
1477 } else
1478 sksgio->dxfer_len = iov_data_len;
1479 }
1480
1481 if (sgp->dxfer_direction != SG_DXFER_NONE) {
1482 struct sg_iovec *iov = sksgio->iov;
1483 for (i = 0; i < sksgio->iovcnt; i++, iov++) {
1484 if (!access_ok(acc, iov->iov_base, iov->iov_len)) {
2e44b427 1485 pr_debug("%s:%s:%d access data failed %p/%d\n",
1486 skdev->name, __func__, __LINE__,
1487 iov->iov_base, (int)iov->iov_len);
e67f86b3
AB		 1488 return -EFAULT;
1489 }
1490 }
1491 }
1492
1493 return 0;
1494}
1495
1496static int skd_sg_io_obtain_skspcl(struct skd_device *skdev,
1497 struct skd_sg_io *sksgio)
1498{
1499 struct skd_special_context *skspcl = NULL;
1500 int rc;
1501
38d4a1bb 1502 for (;;) {
e67f86b3
AB		 1503 ulong flags;
1504
1505 spin_lock_irqsave(&skdev->lock, flags);
1506 skspcl = skdev->skspcl_free_list;
1507 if (skspcl != NULL) {
1508 skdev->skspcl_free_list =
1509 (struct skd_special_context *)skspcl->req.next;
1510 skspcl->req.id += SKD_ID_INCR;
1511 skspcl->req.state = SKD_REQ_STATE_SETUP;
1512 skspcl->orphaned = 0;
1513 skspcl->req.n_sg = 0;
1514 }
1515 spin_unlock_irqrestore(&skdev->lock, flags);
1516
1517 if (skspcl != NULL) {
1518 rc = 0;
1519 break;
1520 }
1521
2e44b427 1522 pr_debug("%s:%s:%d blocking\n",
1523 skdev->name, __func__, __LINE__);
e67f86b3
AB		 1524
1525 rc = wait_event_interruptible_timeout(
1526 skdev->waitq,
1527 (skdev->skspcl_free_list != NULL),
1528 msecs_to_jiffies(sksgio->sg.timeout));
1529
2e44b427 1530 pr_debug("%s:%s:%d unblocking, rc=%d\n",
1531 skdev->name, __func__, __LINE__, rc);
e67f86b3
AB		 1532
1533 if (rc <= 0) {
1534 if (rc == 0)
1535 rc = -ETIMEDOUT;
1536 else
1537 rc = -EINTR;
1538 break;
1539 }
1540 /*
1541 * If we get here rc > 0 meaning the timeout to
1542 * wait_event_interruptible_timeout() had time left, hence the
1543 * sought event -- non-empty free list -- happened.
1544 * Retry the allocation.
1545 */
1546 }
1547 sksgio->skspcl = skspcl;
1548
1549 return rc;
1550}
1551
1552static int skd_skreq_prep_buffering(struct skd_device *skdev,
1553 struct skd_request_context *skreq,
1554 u32 dxfer_len)
1555{
1556 u32 resid = dxfer_len;
1557
1558 /*
1559 * The DMA engine must have aligned addresses and byte counts.
1560 */
1561 resid += (-resid) & 3;
1562 skreq->sg_byte_count = resid;
1563
1564 skreq->n_sg = 0;
1565
1566 while (resid > 0) {
1567 u32 nbytes = PAGE_SIZE;
1568 u32 ix = skreq->n_sg;
1569 struct scatterlist *sg = &skreq->sg[ix];
1570 struct fit_sg_descriptor *sksg = &skreq->sksg_list[ix];
1571 struct page *page;
1572
1573 if (nbytes > resid)
1574 nbytes = resid;
1575
1576 page = alloc_page(GFP_KERNEL);
1577 if (page == NULL)
1578 return -ENOMEM;
1579
1580 sg_set_page(sg, page, nbytes, 0);
1581
1582 /* TODO: This should be going through a pci_???()
1583 * routine to do proper mapping. */
1584 sksg->control = FIT_SGD_CONTROL_NOT_LAST;
1585 sksg->byte_count = nbytes;
1586
1587 sksg->host_side_addr = sg_phys(sg);
1588
1589 sksg->dev_side_addr = 0;
1590 sksg->next_desc_ptr = skreq->sksg_dma_address +
1591 (ix + 1) * sizeof(*sksg);
1592
1593 skreq->n_sg++;
1594 resid -= nbytes;
1595 }
1596
1597 if (skreq->n_sg > 0) {
1598 u32 ix = skreq->n_sg - 1;
1599 struct fit_sg_descriptor *sksg = &skreq->sksg_list[ix];
1600
1601 sksg->control = FIT_SGD_CONTROL_LAST;
1602 sksg->next_desc_ptr = 0;
1603 }
1604
1605 if (unlikely(skdev->dbg_level > 1)) {
1606 u32 i;
1607
2e44b427 1608 pr_debug("%s:%s:%d skreq=%x sksg_list=%p sksg_dma=%llx\n",
1609 skdev->name, __func__, __LINE__,
1610 skreq->id, skreq->sksg_list, skreq->sksg_dma_address);
e67f86b3
AB		 1611 for (i = 0; i < skreq->n_sg; i++) {
1612 struct fit_sg_descriptor *sgd = &skreq->sksg_list[i];
1613
2e44b427 1614 pr_debug("%s:%s:%d sg[%d] count=%u ctrl=0x%x "
1615 "addr=0x%llx next=0x%llx\n",
1616 skdev->name, __func__, __LINE__,
1617 i, sgd->byte_count, sgd->control,
1618 sgd->host_side_addr, sgd->next_desc_ptr);
e67f86b3
AB		 1619 }
1620 }
1621
1622 return 0;
1623}
1624
1625static int skd_sg_io_prep_buffering(struct skd_device *skdev,
1626 struct skd_sg_io *sksgio)
1627{
1628 struct skd_special_context *skspcl = sksgio->skspcl;
1629 struct skd_request_context *skreq = &skspcl->req;
1630 u32 dxfer_len = sksgio->dxfer_len;
1631 int rc;
1632
1633 rc = skd_skreq_prep_buffering(skdev, skreq, dxfer_len);
1634 /*
1635 * Eventually, errors or not, skd_release_special() is called
1636 * to recover allocations including partial allocations.
1637 */
1638 return rc;
1639}
1640
1641static int skd_sg_io_copy_buffer(struct skd_device *skdev,
1642 struct skd_sg_io *sksgio, int dxfer_dir)
1643{
1644 struct skd_special_context *skspcl = sksgio->skspcl;
1645 u32 iov_ix = 0;
1646 struct sg_iovec curiov;
1647 u32 sksg_ix = 0;
1648 u8 *bufp = NULL;
1649 u32 buf_len = 0;
1650 u32 resid = sksgio->dxfer_len;
1651 int rc;
1652
1653 curiov.iov_len = 0;
1654 curiov.iov_base = NULL;
1655
1656 if (dxfer_dir != sksgio->sg.dxfer_direction) {
1657 if (dxfer_dir != SG_DXFER_TO_DEV ||
1658 sksgio->sg.dxfer_direction != SG_DXFER_TO_FROM_DEV)
1659 return 0;
1660 }
1661
1662 while (resid > 0) {
1663 u32 nbytes = PAGE_SIZE;
1664
1665 if (curiov.iov_len == 0) {
1666 curiov = sksgio->iov[iov_ix++];
1667 continue;
1668 }
1669
1670 if (buf_len == 0) {
1671 struct page *page;
1672 page = sg_page(&skspcl->req.sg[sksg_ix++]);
1673 bufp = page_address(page);
1674 buf_len = PAGE_SIZE;
1675 }
1676
1677 nbytes = min_t(u32, nbytes, resid);
1678 nbytes = min_t(u32, nbytes, curiov.iov_len);
1679 nbytes = min_t(u32, nbytes, buf_len);
1680
1681 if (dxfer_dir == SG_DXFER_TO_DEV)
1682 rc = __copy_from_user(bufp, curiov.iov_base, nbytes);
1683 else
1684 rc = __copy_to_user(curiov.iov_base, bufp, nbytes);
1685
1686 if (rc)
1687 return -EFAULT;
1688
1689 resid -= nbytes;
1690 curiov.iov_len -= nbytes;
1691 curiov.iov_base += nbytes;
1692 buf_len -= nbytes;
1693 }
1694
1695 return 0;
1696}
1697
1698static int skd_sg_io_send_fitmsg(struct skd_device *skdev,
1699 struct skd_sg_io *sksgio)
1700{
1701 struct skd_special_context *skspcl = sksgio->skspcl;
1702 struct fit_msg_hdr *fmh = (struct fit_msg_hdr *)skspcl->msg_buf;
1703 struct skd_scsi_request *scsi_req = (struct skd_scsi_request *)&fmh[1];
1704
1705 memset(skspcl->msg_buf, 0, SKD_N_SPECIAL_FITMSG_BYTES);
1706
1707 /* Initialize the FIT msg header */
1708 fmh->protocol_id = FIT_PROTOCOL_ID_SOFIT;
1709 fmh->num_protocol_cmds_coalesced = 1;
1710
1711 /* Initialize the SCSI request */
1712 if (sksgio->sg.dxfer_direction != SG_DXFER_NONE)
1713 scsi_req->hdr.sg_list_dma_address =
1714 cpu_to_be64(skspcl->req.sksg_dma_address);
1715 scsi_req->hdr.tag = skspcl->req.id;
1716 scsi_req->hdr.sg_list_len_bytes =
1717 cpu_to_be32(skspcl->req.sg_byte_count);
1718 memcpy(scsi_req->cdb, sksgio->cdb, sizeof(scsi_req->cdb));
1719
1720 skspcl->req.state = SKD_REQ_STATE_BUSY;
1721 skd_send_special_fitmsg(skdev, skspcl);
1722
1723 return 0;
1724}
1725
1726static int skd_sg_io_await(struct skd_device *skdev, struct skd_sg_io *sksgio)
1727{
1728 unsigned long flags;
1729 int rc;
1730
1731 rc = wait_event_interruptible_timeout(skdev->waitq,
1732 (sksgio->skspcl->req.state !=
1733 SKD_REQ_STATE_BUSY),
1734 msecs_to_jiffies(sksgio->sg.
1735 timeout));
1736
1737 spin_lock_irqsave(&skdev->lock, flags);
1738
1739 if (sksgio->skspcl->req.state == SKD_REQ_STATE_ABORTED) {
2e44b427 1740 pr_debug("%s:%s:%d skspcl %p aborted\n",
1741 skdev->name, __func__, __LINE__, sksgio->skspcl);
e67f86b3
AB		 1742
1743 /* Build check cond, sense and let command finish. */
1744 /* For a timeout, we must fabricate completion and sense
1745 * data to complete the command */
1746 sksgio->skspcl->req.completion.status =
1747 SAM_STAT_CHECK_CONDITION;
1748
1749 memset(&sksgio->skspcl->req.err_info, 0,
1750 sizeof(sksgio->skspcl->req.err_info));
1751 sksgio->skspcl->req.err_info.type = 0x70;
1752 sksgio->skspcl->req.err_info.key = ABORTED_COMMAND;
1753 sksgio->skspcl->req.err_info.code = 0x44;
1754 sksgio->skspcl->req.err_info.qual = 0;
1755 rc = 0;
1756 } else if (sksgio->skspcl->req.state != SKD_REQ_STATE_BUSY)
1757 /* No longer on the adapter. We finish. */
1758 rc = 0;
1759 else {
1760 /* Something's gone wrong. Still busy. Timeout or
1761 * user interrupted (control-C). Mark as an orphan
1762 * so it will be disposed when completed. */
1763 sksgio->skspcl->orphaned = 1;
1764 sksgio->skspcl = NULL;
1765 if (rc == 0) {
2e44b427 1766 pr_debug("%s:%s:%d timed out %p (%u ms)\n",
1767 skdev->name, __func__, __LINE__,
1768 sksgio, sksgio->sg.timeout);
e67f86b3
AB		 1769 rc = -ETIMEDOUT;
1770 } else {
2e44b427 1771 pr_debug("%s:%s:%d cntlc %p\n",
1772 skdev->name, __func__, __LINE__, sksgio);
e67f86b3
AB		 1773 rc = -EINTR;
1774 }
1775 }
1776
1777 spin_unlock_irqrestore(&skdev->lock, flags);
1778
1779 return rc;
1780}
1781
1782static int skd_sg_io_put_status(struct skd_device *skdev,
1783 struct skd_sg_io *sksgio)
1784{
1785 struct sg_io_hdr *sgp = &sksgio->sg;
1786 struct skd_special_context *skspcl = sksgio->skspcl;
1787 int resid = 0;
1788
1789 u32 nb = be32_to_cpu(skspcl->req.completion.num_returned_bytes);
1790
1791 sgp->status = skspcl->req.completion.status;
1792 resid = sksgio->dxfer_len - nb;
1793
1794 sgp->masked_status = sgp->status & STATUS_MASK;
1795 sgp->msg_status = 0;
1796 sgp->host_status = 0;
1797 sgp->driver_status = 0;
1798 sgp->resid = resid;
1799 if (sgp->masked_status || sgp->host_status || sgp->driver_status)
1800 sgp->info |= SG_INFO_CHECK;
1801
2e44b427 1802 pr_debug("%s:%s:%d status %x masked %x resid 0x%x\n",
1803 skdev->name, __func__, __LINE__,
1804 sgp->status, sgp->masked_status, sgp->resid);
e67f86b3
AB		 1805
1806 if (sgp->masked_status == SAM_STAT_CHECK_CONDITION) {
1807 if (sgp->mx_sb_len > 0) {
1808 struct fit_comp_error_info *ei = &skspcl->req.err_info;
1809 u32 nbytes = sizeof(*ei);
1810
1811 nbytes = min_t(u32, nbytes, sgp->mx_sb_len);
1812
1813 sgp->sb_len_wr = nbytes;
1814
1815 if (__copy_to_user(sgp->sbp, ei, nbytes)) {
2e44b427 1816 pr_debug("%s:%s:%d copy_to_user sense failed %p\n",
1817 skdev->name, __func__, __LINE__,
1818 sgp->sbp);
e67f86b3
AB		 1819 return -EFAULT;
1820 }
1821 }
1822 }
1823
1824 if (__copy_to_user(sksgio->argp, sgp, sizeof(sg_io_hdr_t))) {
2e44b427 1825 pr_debug("%s:%s:%d copy_to_user sg failed %p\n",
1826 skdev->name, __func__, __LINE__, sksgio->argp);
e67f86b3
AB		 1827 return -EFAULT;
1828 }
1829
1830 return 0;
1831}
1832
1833static int skd_sg_io_release_skspcl(struct skd_device *skdev,
1834 struct skd_sg_io *sksgio)
1835{
1836 struct skd_special_context *skspcl = sksgio->skspcl;
1837
1838 if (skspcl != NULL) {
1839 ulong flags;
1840
1841 sksgio->skspcl = NULL;
1842
1843 spin_lock_irqsave(&skdev->lock, flags);
1844 skd_release_special(skdev, skspcl);
1845 spin_unlock_irqrestore(&skdev->lock, flags);
1846 }
1847
1848 return 0;
1849}
1850
1851/*
1852 *****************************************************************************
1853 * INTERNAL REQUESTS -- generated by driver itself
1854 *****************************************************************************
1855 */
1856
1857static int skd_format_internal_skspcl(struct skd_device *skdev)
1858{
1859 struct skd_special_context *skspcl = &skdev->internal_skspcl;
1860 struct fit_sg_descriptor *sgd = &skspcl->req.sksg_list[0];
1861 struct fit_msg_hdr *fmh;
1862 uint64_t dma_address;
1863 struct skd_scsi_request *scsi;
1864
1865 fmh = (struct fit_msg_hdr *)&skspcl->msg_buf[0];
1866 fmh->protocol_id = FIT_PROTOCOL_ID_SOFIT;
1867 fmh->num_protocol_cmds_coalesced = 1;
1868
1869 scsi = (struct skd_scsi_request *)&skspcl->msg_buf[64];
1870 memset(scsi, 0, sizeof(*scsi));
1871 dma_address = skspcl->req.sksg_dma_address;
1872 scsi->hdr.sg_list_dma_address = cpu_to_be64(dma_address);
1873 sgd->control = FIT_SGD_CONTROL_LAST;
1874 sgd->byte_count = 0;
1875 sgd->host_side_addr = skspcl->db_dma_address;
1876 sgd->dev_side_addr = 0;
1877 sgd->next_desc_ptr = 0LL;
1878
1879 return 1;
1880}
1881
1882#define WR_BUF_SIZE SKD_N_INTERNAL_BYTES
1883
1884static void skd_send_internal_skspcl(struct skd_device *skdev,
1885 struct skd_special_context *skspcl,
1886 u8 opcode)
1887{
1888 struct fit_sg_descriptor *sgd = &skspcl->req.sksg_list[0];
1889 struct skd_scsi_request *scsi;
1890 unsigned char *buf = skspcl->data_buf;
1891 int i;
1892
1893 if (skspcl->req.state != SKD_REQ_STATE_IDLE)
1894 /*
1895 * A refresh is already in progress.
1896 * Just wait for it to finish.
1897 */
1898 return;
1899
1900 SKD_ASSERT((skspcl->req.id & SKD_ID_INCR) == 0);
1901 skspcl->req.state = SKD_REQ_STATE_BUSY;
1902 skspcl->req.id += SKD_ID_INCR;
1903
1904 scsi = (struct skd_scsi_request *)&skspcl->msg_buf[64];
1905 scsi->hdr.tag = skspcl->req.id;
1906
1907 memset(scsi->cdb, 0, sizeof(scsi->cdb));
1908
1909 switch (opcode) {
1910 case TEST_UNIT_READY:
1911 scsi->cdb[0] = TEST_UNIT_READY;
1912 sgd->byte_count = 0;
1913 scsi->hdr.sg_list_len_bytes = 0;
1914 break;
1915
1916 case READ_CAPACITY:
1917 scsi->cdb[0] = READ_CAPACITY;
1918 sgd->byte_count = SKD_N_READ_CAP_BYTES;
1919 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
1920 break;
1921
1922 case INQUIRY:
1923 scsi->cdb[0] = INQUIRY;
1924 scsi->cdb[1] = 0x01; /* evpd */
1925 scsi->cdb[2] = 0x80; /* serial number page */
1926 scsi->cdb[4] = 0x10;
1927 sgd->byte_count = 16;
1928 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
1929 break;
1930
1931 case SYNCHRONIZE_CACHE:
1932 scsi->cdb[0] = SYNCHRONIZE_CACHE;
1933 sgd->byte_count = 0;
1934 scsi->hdr.sg_list_len_bytes = 0;
1935 break;
1936
1937 case WRITE_BUFFER:
1938 scsi->cdb[0] = WRITE_BUFFER;
1939 scsi->cdb[1] = 0x02;
1940 scsi->cdb[7] = (WR_BUF_SIZE & 0xFF00) >> 8;
1941 scsi->cdb[8] = WR_BUF_SIZE & 0xFF;
1942 sgd->byte_count = WR_BUF_SIZE;
1943 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
1944 /* fill incrementing byte pattern */
1945 for (i = 0; i < sgd->byte_count; i++)
1946 buf[i] = i & 0xFF;
1947 break;
1948
1949 case READ_BUFFER:
1950 scsi->cdb[0] = READ_BUFFER;
1951 scsi->cdb[1] = 0x02;
1952 scsi->cdb[7] = (WR_BUF_SIZE & 0xFF00) >> 8;
1953 scsi->cdb[8] = WR_BUF_SIZE & 0xFF;
1954 sgd->byte_count = WR_BUF_SIZE;
1955 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
1956 memset(skspcl->data_buf, 0, sgd->byte_count);
1957 break;
1958
1959 default:
1960 SKD_ASSERT("Don't know what to send");
1961 return;
1962
1963 }
1964 skd_send_special_fitmsg(skdev, skspcl);
1965}
1966
1967static void skd_refresh_device_data(struct skd_device *skdev)
1968{
1969 struct skd_special_context *skspcl = &skdev->internal_skspcl;
1970
1971 skd_send_internal_skspcl(skdev, skspcl, TEST_UNIT_READY);
1972}
1973
1974static int skd_chk_read_buf(struct skd_device *skdev,
1975 struct skd_special_context *skspcl)
1976{
1977 unsigned char *buf = skspcl->data_buf;
1978 int i;
1979
1980 /* check for incrementing byte pattern */
1981 for (i = 0; i < WR_BUF_SIZE; i++)
1982 if (buf[i] != (i & 0xFF))
1983 return 1;
1984
1985 return 0;
1986}
1987
1988static void skd_log_check_status(struct skd_device *skdev, u8 status, u8 key,
1989 u8 code, u8 qual, u8 fruc)
1990{
1991 /* If the check condition is of special interest, log a message */
1992 if ((status == SAM_STAT_CHECK_CONDITION) && (key == 0x02)
1993 && (code == 0x04) && (qual == 0x06)) {
1994 pr_err("(%s): *** LOST_WRITE_DATA ERROR *** key/asc/"
1995 "ascq/fruc %02x/%02x/%02x/%02x\n",
1996 skd_name(skdev), key, code, qual, fruc);
1997 }
1998}
1999
2000static void skd_complete_internal(struct skd_device *skdev,
2001 volatile struct fit_completion_entry_v1
2002 *skcomp,
2003 volatile struct fit_comp_error_info *skerr,
2004 struct skd_special_context *skspcl)
2005{
2006 u8 *buf = skspcl->data_buf;
2007 u8 status;
2008 int i;
2009 struct skd_scsi_request *scsi =
2010 (struct skd_scsi_request *)&skspcl->msg_buf[64];
2011
2012 SKD_ASSERT(skspcl == &skdev->internal_skspcl);
2013
2e44b427 2014 pr_debug("%s:%s:%d complete internal %x\n",
2015 skdev->name, __func__, __LINE__, scsi->cdb[0]);
e67f86b3
AB		 2016
2017 skspcl->req.completion = *skcomp;
2018 skspcl->req.state = SKD_REQ_STATE_IDLE;
2019 skspcl->req.id += SKD_ID_INCR;
2020
2021 status = skspcl->req.completion.status;
2022
2023 skd_log_check_status(skdev, status, skerr->key, skerr->code,
2024 skerr->qual, skerr->fruc);
2025
2026 switch (scsi->cdb[0]) {
2027 case TEST_UNIT_READY:
2028 if (status == SAM_STAT_GOOD)
2029 skd_send_internal_skspcl(skdev, skspcl, WRITE_BUFFER);
2030 else if ((status == SAM_STAT_CHECK_CONDITION) &&
2031 (skerr->key == MEDIUM_ERROR))
2032 skd_send_internal_skspcl(skdev, skspcl, WRITE_BUFFER);
2033 else {
2034 if (skdev->state == SKD_DRVR_STATE_STOPPING) {
2e44b427 2035 pr_debug("%s:%s:%d TUR failed, don't send anymore state 0x%x\n",
2036 skdev->name, __func__, __LINE__,
2037 skdev->state);
e67f86b3
AB		 2038 return;
2039 }
2e44b427 2040 pr_debug("%s:%s:%d **** TUR failed, retry skerr\n",
2041 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2042 skd_send_internal_skspcl(skdev, skspcl, 0x00);
2043 }
2044 break;
2045
2046 case WRITE_BUFFER:
2047 if (status == SAM_STAT_GOOD)
2048 skd_send_internal_skspcl(skdev, skspcl, READ_BUFFER);
2049 else {
2050 if (skdev->state == SKD_DRVR_STATE_STOPPING) {
2e44b427 2051 pr_debug("%s:%s:%d write buffer failed, don't send anymore state 0x%x\n",
2052 skdev->name, __func__, __LINE__,
2053 skdev->state);
e67f86b3
AB		 2054 return;
2055 }
2e44b427 2056 pr_debug("%s:%s:%d **** write buffer failed, retry skerr\n",
2057 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2058 skd_send_internal_skspcl(skdev, skspcl, 0x00);
2059 }
2060 break;
2061
2062 case READ_BUFFER:
2063 if (status == SAM_STAT_GOOD) {
2064 if (skd_chk_read_buf(skdev, skspcl) == 0)
2065 skd_send_internal_skspcl(skdev, skspcl,
2066 READ_CAPACITY);
2067 else {
2068 pr_err(
2069 "(%s):*** W/R Buffer mismatch %d ***\n",
2070 skd_name(skdev), skdev->connect_retries);
2071 if (skdev->connect_retries <
2072 SKD_MAX_CONNECT_RETRIES) {
2073 skdev->connect_retries++;
2074 skd_soft_reset(skdev);
2075 } else {
2076 pr_err(
2077 "(%s): W/R Buffer Connect Error\n",
2078 skd_name(skdev));
2079 return;
2080 }
2081 }
2082
2083 } else {
2084 if (skdev->state == SKD_DRVR_STATE_STOPPING) {
2e44b427 2085 pr_debug("%s:%s:%d "
2086 "read buffer failed, don't send anymore state 0x%x\n",
2087 skdev->name, __func__, __LINE__,
2088 skdev->state);
e67f86b3
AB		 2089 return;
2090 }
2e44b427 2091 pr_debug("%s:%s:%d "
2092 "**** read buffer failed, retry skerr\n",
2093 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2094 skd_send_internal_skspcl(skdev, skspcl, 0x00);
2095 }
2096 break;
2097
2098 case READ_CAPACITY:
2099 skdev->read_cap_is_valid = 0;
2100 if (status == SAM_STAT_GOOD) {
2101 skdev->read_cap_last_lba =
2102 (buf[0] << 24) | (buf[1] << 16) |
2103 (buf[2] << 8) | buf[3];
2104 skdev->read_cap_blocksize =
2105 (buf[4] << 24) | (buf[5] << 16) |
2106 (buf[6] << 8) | buf[7];
2107
2e44b427 2108 pr_debug("%s:%s:%d last lba %d, bs %d\n",
2109 skdev->name, __func__, __LINE__,
2110 skdev->read_cap_last_lba,
2111 skdev->read_cap_blocksize);
e67f86b3
AB		 2112
2113 set_capacity(skdev->disk, skdev->read_cap_last_lba + 1);
2114
2115 skdev->read_cap_is_valid = 1;
2116
2117 skd_send_internal_skspcl(skdev, skspcl, INQUIRY);
2118 } else if ((status == SAM_STAT_CHECK_CONDITION) &&
2119 (skerr->key == MEDIUM_ERROR)) {
2120 skdev->read_cap_last_lba = ~0;
2121 set_capacity(skdev->disk, skdev->read_cap_last_lba + 1);
2e44b427 2122 pr_debug("%s:%s:%d "
2123 "**** MEDIUM ERROR caused READCAP to fail, ignore failure and continue to inquiry\n",
2124 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2125 skd_send_internal_skspcl(skdev, skspcl, INQUIRY);
2126 } else {
2e44b427 2127 pr_debug("%s:%s:%d **** READCAP failed, retry TUR\n",
2128 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2129 skd_send_internal_skspcl(skdev, skspcl,
2130 TEST_UNIT_READY);
2131 }
2132 break;
2133
2134 case INQUIRY:
2135 skdev->inquiry_is_valid = 0;
2136 if (status == SAM_STAT_GOOD) {
2137 skdev->inquiry_is_valid = 1;
2138
2139 for (i = 0; i < 12; i++)
2140 skdev->inq_serial_num[i] = buf[i + 4];
2141 skdev->inq_serial_num[12] = 0;
2142 }
2143
2144 if (skd_unquiesce_dev(skdev) < 0)
2e44b427 2145 pr_debug("%s:%s:%d **** failed, to ONLINE device\n",
2146 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2147 /* connection is complete */
2148 skdev->connect_retries = 0;
2149 break;
2150
2151 case SYNCHRONIZE_CACHE:
2152 if (status == SAM_STAT_GOOD)
2153 skdev->sync_done = 1;
2154 else
2155 skdev->sync_done = -1;
2156 wake_up_interruptible(&skdev->waitq);
2157 break;
2158
2159 default:
2160 SKD_ASSERT("we didn't send this");
2161 }
2162}
2163
2164/*
2165 *****************************************************************************
2166 * FIT MESSAGES
2167 *****************************************************************************
2168 */
2169
2170static void skd_send_fitmsg(struct skd_device *skdev,
2171 struct skd_fitmsg_context *skmsg)
2172{
2173 u64 qcmd;
2174 struct fit_msg_hdr *fmh;
2175
2e44b427 2176 pr_debug("%s:%s:%d dma address 0x%llx, busy=%d\n",
2177 skdev->name, __func__, __LINE__,
2178 skmsg->mb_dma_address, skdev->in_flight);
2179 pr_debug("%s:%s:%d msg_buf 0x%p, offset %x\n",
2180 skdev->name, __func__, __LINE__,
2181 skmsg->msg_buf, skmsg->offset);
e67f86b3
AB		 2182
2183 qcmd = skmsg->mb_dma_address;
2184 qcmd |= FIT_QCMD_QID_NORMAL;
2185
2186 fmh = (struct fit_msg_hdr *)skmsg->msg_buf;
2187 skmsg->outstanding = fmh->num_protocol_cmds_coalesced;
2188
2189 if (unlikely(skdev->dbg_level > 1)) {
2190 u8 *bp = (u8 *)skmsg->msg_buf;
2191 int i;
2192 for (i = 0; i < skmsg->length; i += 8) {
2e44b427 2193 pr_debug("%s:%s:%d msg[%2d] %02x %02x %02x %02x "
2194 "%02x %02x %02x %02x\n",
2195 skdev->name, __func__, __LINE__,
2196 i, bp[i + 0], bp[i + 1], bp[i + 2],
2197 bp[i + 3], bp[i + 4], bp[i + 5],
2198 bp[i + 6], bp[i + 7]);
e67f86b3
AB		 2199 if (i == 0)
2200 i = 64 - 8;
2201 }
2202 }
2203
2204 if (skmsg->length > 256)
2205 qcmd |= FIT_QCMD_MSGSIZE_512;
2206 else if (skmsg->length > 128)
2207 qcmd |= FIT_QCMD_MSGSIZE_256;
2208 else if (skmsg->length > 64)
2209 qcmd |= FIT_QCMD_MSGSIZE_128;
2210 else
2211 /*
2212 * This makes no sense because the FIT msg header is
2213 * 64 bytes. If the msg is only 64 bytes long it has
2214 * no payload.
2215 */
2216 qcmd |= FIT_QCMD_MSGSIZE_64;
2217
2218 SKD_WRITEQ(skdev, qcmd, FIT_Q_COMMAND);
2219
2220}
2221
2222static void skd_send_special_fitmsg(struct skd_device *skdev,
2223 struct skd_special_context *skspcl)
2224{
2225 u64 qcmd;
2226
2227 if (unlikely(skdev->dbg_level > 1)) {
2228 u8 *bp = (u8 *)skspcl->msg_buf;
2229 int i;
2230
2231 for (i = 0; i < SKD_N_SPECIAL_FITMSG_BYTES; i += 8) {
2e44b427 2232 pr_debug("%s:%s:%d spcl[%2d] %02x %02x %02x %02x "
2233 "%02x %02x %02x %02x\n",
2234 skdev->name, __func__, __LINE__, i,
2235 bp[i + 0], bp[i + 1], bp[i + 2], bp[i + 3],
2236 bp[i + 4], bp[i + 5], bp[i + 6], bp[i + 7]);
e67f86b3
AB		 2237 if (i == 0)
2238 i = 64 - 8;
2239 }
2240
2e44b427 2241 pr_debug("%s:%s:%d skspcl=%p id=%04x sksg_list=%p sksg_dma=%llx\n",
2242 skdev->name, __func__, __LINE__,
2243 skspcl, skspcl->req.id, skspcl->req.sksg_list,
2244 skspcl->req.sksg_dma_address);
e67f86b3
AB		 2245 for (i = 0; i < skspcl->req.n_sg; i++) {
2246 struct fit_sg_descriptor *sgd =
2247 &skspcl->req.sksg_list[i];
2248
2e44b427 2249 pr_debug("%s:%s:%d sg[%d] count=%u ctrl=0x%x "
2250 "addr=0x%llx next=0x%llx\n",
2251 skdev->name, __func__, __LINE__,
2252 i, sgd->byte_count, sgd->control,
2253 sgd->host_side_addr, sgd->next_desc_ptr);
e67f86b3
AB		 2254 }
2255 }
2256
2257 /*
2258 * Special FIT msgs are always 128 bytes: a 64-byte FIT hdr
2259 * and one 64-byte SSDI command.
2260 */
2261 qcmd = skspcl->mb_dma_address;
2262 qcmd |= FIT_QCMD_QID_NORMAL + FIT_QCMD_MSGSIZE_128;
2263
2264 SKD_WRITEQ(skdev, qcmd, FIT_Q_COMMAND);
2265}
2266
2267/*
2268 *****************************************************************************
2269 * COMPLETION QUEUE
2270 *****************************************************************************
2271 */
2272
2273static void skd_complete_other(struct skd_device *skdev,
2274 volatile struct fit_completion_entry_v1 *skcomp,
2275 volatile struct fit_comp_error_info *skerr);
2276
e67f86b3
AB		 2277struct sns_info {
2278 u8 type;
2279 u8 stat;
2280 u8 key;
2281 u8 asc;
2282 u8 ascq;
2283 u8 mask;
2284 enum skd_check_status_action action;
2285};
2286
2287static struct sns_info skd_chkstat_table[] = {
2288 /* Good */
2289 { 0x70, 0x02, RECOVERED_ERROR, 0, 0, 0x1c,
2290 SKD_CHECK_STATUS_REPORT_GOOD },
2291
2292 /* Smart alerts */
2293 { 0x70, 0x02, NO_SENSE, 0x0B, 0x00, 0x1E, /* warnings */
2294 SKD_CHECK_STATUS_REPORT_SMART_ALERT },
2295 { 0x70, 0x02, NO_SENSE, 0x5D, 0x00, 0x1E, /* thresholds */
2296 SKD_CHECK_STATUS_REPORT_SMART_ALERT },
2297 { 0x70, 0x02, RECOVERED_ERROR, 0x0B, 0x01, 0x1F, /* temperature over trigger */
2298 SKD_CHECK_STATUS_REPORT_SMART_ALERT },
2299
2300 /* Retry (with limits) */
2301 { 0x70, 0x02, 0x0B, 0, 0, 0x1C, /* This one is for DMA ERROR */
2302 SKD_CHECK_STATUS_REQUEUE_REQUEST },
2303 { 0x70, 0x02, 0x06, 0x0B, 0x00, 0x1E, /* warnings */
2304 SKD_CHECK_STATUS_REQUEUE_REQUEST },
2305 { 0x70, 0x02, 0x06, 0x5D, 0x00, 0x1E, /* thresholds */
2306 SKD_CHECK_STATUS_REQUEUE_REQUEST },
2307 { 0x70, 0x02, 0x06, 0x80, 0x30, 0x1F, /* backup power */
2308 SKD_CHECK_STATUS_REQUEUE_REQUEST },
2309
2310 /* Busy (or about to be) */
2311 { 0x70, 0x02, 0x06, 0x3f, 0x01, 0x1F, /* fw changed */
2312 SKD_CHECK_STATUS_BUSY_IMMINENT },
2313};
2314
2315/*
2316 * Look up status and sense data to decide how to handle the error
2317 * from the device.
2318 * mask says which fields must match e.g., mask=0x18 means check
2319 * type and stat, ignore key, asc, ascq.
2320 */
2321
38d4a1bb
MS		 2322static enum skd_check_status_action
2323skd_check_status(struct skd_device *skdev,
2324 u8 cmp_status, volatile struct fit_comp_error_info *skerr)
e67f86b3
AB		 2325{
2326 int i, n;
2327
2328 pr_err("(%s): key/asc/ascq/fruc %02x/%02x/%02x/%02x\n",
2329 skd_name(skdev), skerr->key, skerr->code, skerr->qual,
2330 skerr->fruc);
2331
2e44b427 2332 pr_debug("%s:%s:%d stat: t=%02x stat=%02x k=%02x c=%02x q=%02x fruc=%02x\n",
2333 skdev->name, __func__, __LINE__, skerr->type, cmp_status,
2334 skerr->key, skerr->code, skerr->qual, skerr->fruc);
e67f86b3
AB		 2335
2336 /* Does the info match an entry in the good category? */
2337 n = sizeof(skd_chkstat_table) / sizeof(skd_chkstat_table[0]);
2338 for (i = 0; i < n; i++) {
2339 struct sns_info *sns = &skd_chkstat_table[i];
2340
2341 if (sns->mask & 0x10)
2342 if (skerr->type != sns->type)
2343 continue;
2344
2345 if (sns->mask & 0x08)
2346 if (cmp_status != sns->stat)
2347 continue;
2348
2349 if (sns->mask & 0x04)
2350 if (skerr->key != sns->key)
2351 continue;
2352
2353 if (sns->mask & 0x02)
2354 if (skerr->code != sns->asc)
2355 continue;
2356
2357 if (sns->mask & 0x01)
2358 if (skerr->qual != sns->ascq)
2359 continue;
2360
2361 if (sns->action == SKD_CHECK_STATUS_REPORT_SMART_ALERT) {
2362 pr_err("(%s): SMART Alert: sense key/asc/ascq "
2363 "%02x/%02x/%02x\n",
2364 skd_name(skdev), skerr->key,
2365 skerr->code, skerr->qual);
2366 }
2367 return sns->action;
2368 }
2369
2370 /* No other match, so nonzero status means error,
2371 * zero status means good
2372 */
2373 if (cmp_status) {
2e44b427 2374 pr_debug("%s:%s:%d status check: error\n",
2375 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2376 return SKD_CHECK_STATUS_REPORT_ERROR;
2377 }
2378
2e44b427 2379 pr_debug("%s:%s:%d status check good default\n",
2380 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2381 return SKD_CHECK_STATUS_REPORT_GOOD;
2382}
2383
2384static void skd_resolve_req_exception(struct skd_device *skdev,
2385 struct skd_request_context *skreq)
2386{
2387 u8 cmp_status = skreq->completion.status;
2388
2389 switch (skd_check_status(skdev, cmp_status, &skreq->err_info)) {
2390 case SKD_CHECK_STATUS_REPORT_GOOD:
2391 case SKD_CHECK_STATUS_REPORT_SMART_ALERT:
2392 skd_end_request(skdev, skreq, 0);
2393 break;
2394
2395 case SKD_CHECK_STATUS_BUSY_IMMINENT:
2396 skd_log_skreq(skdev, skreq, "retry(busy)");
38d4a1bb 2397 blk_requeue_request(skdev->queue, skreq->req);
e67f86b3
AB		 2398 pr_info("(%s) drive BUSY imminent\n", skd_name(skdev));
2399 skdev->state = SKD_DRVR_STATE_BUSY_IMMINENT;
2400 skdev->timer_countdown = SKD_TIMER_MINUTES(20);
2401 skd_quiesce_dev(skdev);
2402 break;
2403
2404 case SKD_CHECK_STATUS_REQUEUE_REQUEST:
fcd37eb3
JA		 2405 if ((unsigned long) ++skreq->req->special < SKD_MAX_RETRIES) {
2406 skd_log_skreq(skdev, skreq, "retry");
38d4a1bb 2407 blk_requeue_request(skdev->queue, skreq->req);
fcd37eb3 2408 break;
e67f86b3
AB		 2409 }
2410 /* fall through to report error */
2411
2412 case SKD_CHECK_STATUS_REPORT_ERROR:
2413 default:
2414 skd_end_request(skdev, skreq, -EIO);
2415 break;
2416 }
2417}
2418
e67f86b3
AB		 2419/* assume spinlock is already held */
2420static void skd_release_skreq(struct skd_device *skdev,
2421 struct skd_request_context *skreq)
2422{
2423 u32 msg_slot;
2424 struct skd_fitmsg_context *skmsg;
2425
2426 u32 timo_slot;
2427
2428 /*
2429 * Reclaim the FIT msg buffer if this is
2430 * the first of the requests it carried to
2431 * be completed. The FIT msg buffer used to
2432 * send this request cannot be reused until
2433 * we are sure the s1120 card has copied
2434 * it to its memory. The FIT msg might have
2435 * contained several requests. As soon as
2436 * any of them are completed we know that
2437 * the entire FIT msg was transferred.
2438 * Only the first completed request will
2439 * match the FIT msg buffer id. The FIT
2440 * msg buffer id is immediately updated.
2441 * When subsequent requests complete the FIT
2442 * msg buffer id won't match, so we know
2443 * quite cheaply that it is already done.
2444 */
2445 msg_slot = skreq->fitmsg_id & SKD_ID_SLOT_MASK;
2446 SKD_ASSERT(msg_slot < skdev->num_fitmsg_context);
2447
2448 skmsg = &skdev->skmsg_table[msg_slot];
2449 if (skmsg->id == skreq->fitmsg_id) {
2450 SKD_ASSERT(skmsg->state == SKD_MSG_STATE_BUSY);
2451 SKD_ASSERT(skmsg->outstanding > 0);
2452 skmsg->outstanding--;
2453 if (skmsg->outstanding == 0) {
2454 skmsg->state = SKD_MSG_STATE_IDLE;
2455 skmsg->id += SKD_ID_INCR;
2456 skmsg->next = skdev->skmsg_free_list;
2457 skdev->skmsg_free_list = skmsg;
2458 }
2459 }
2460
2461 /*
2462 * Decrease the number of active requests.
2463 * Also decrements the count in the timeout slot.
2464 */
2465 SKD_ASSERT(skdev->in_flight > 0);
2466 skdev->in_flight -= 1;
2467
2468 timo_slot = skreq->timeout_stamp & SKD_TIMEOUT_SLOT_MASK;
2469 SKD_ASSERT(skdev->timeout_slot[timo_slot] > 0);
2470 skdev->timeout_slot[timo_slot] -= 1;
2471
2472 /*
2473 * Reset backpointer
2474 */
fcd37eb3 2475 skreq->req = NULL;
e67f86b3
AB		 2476
2477 /*
2478 * Reclaim the skd_request_context
2479 */
2480 skreq->state = SKD_REQ_STATE_IDLE;
2481 skreq->id += SKD_ID_INCR;
2482 skreq->next = skdev->skreq_free_list;
2483 skdev->skreq_free_list = skreq;
2484}
2485
2486#define DRIVER_INQ_EVPD_PAGE_CODE 0xDA
2487
2488static void skd_do_inq_page_00(struct skd_device *skdev,
2489 volatile struct fit_completion_entry_v1 *skcomp,
2490 volatile struct fit_comp_error_info *skerr,
2491 uint8_t *cdb, uint8_t *buf)
2492{
2493 uint16_t insert_pt, max_bytes, drive_pages, drive_bytes, new_size;
2494
2495 /* Caller requested "supported pages". The driver needs to insert
2496 * its page.
2497 */
2e44b427 2498 pr_debug("%s:%s:%d skd_do_driver_inquiry: modify supported pages.\n",
2499 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2500
2501 /* If the device rejected the request because the CDB was
2502 * improperly formed, then just leave.
2503 */
2504 if (skcomp->status == SAM_STAT_CHECK_CONDITION &&
2505 skerr->key == ILLEGAL_REQUEST && skerr->code == 0x24)
2506 return;
2507
2508 /* Get the amount of space the caller allocated */
2509 max_bytes = (cdb[3] << 8) | cdb[4];
2510
2511 /* Get the number of pages actually returned by the device */
2512 drive_pages = (buf[2] << 8) | buf[3];
2513 drive_bytes = drive_pages + 4;
2514 new_size = drive_pages + 1;
2515
2516 /* Supported pages must be in numerical order, so find where
2517 * the driver page needs to be inserted into the list of
2518 * pages returned by the device.
2519 */
2520 for (insert_pt = 4; insert_pt < drive_bytes; insert_pt++) {
2521 if (buf[insert_pt] == DRIVER_INQ_EVPD_PAGE_CODE)
2522 return; /* Device using this page code. abort */
2523 else if (buf[insert_pt] > DRIVER_INQ_EVPD_PAGE_CODE)
2524 break;
2525 }
2526
2527 if (insert_pt < max_bytes) {
2528 uint16_t u;
2529
2530 /* Shift everything up one byte to make room. */
2531 for (u = new_size + 3; u > insert_pt; u--)
2532 buf[u] = buf[u - 1];
2533 buf[insert_pt] = DRIVER_INQ_EVPD_PAGE_CODE;
2534
2535 /* SCSI byte order increment of num_returned_bytes by 1 */
2536 skcomp->num_returned_bytes =
2537 be32_to_cpu(skcomp->num_returned_bytes) + 1;
2538 skcomp->num_returned_bytes =
2539 be32_to_cpu(skcomp->num_returned_bytes);
2540 }
2541
2542 /* update page length field to reflect the driver's page too */
2543 buf[2] = (uint8_t)((new_size >> 8) & 0xFF);
2544 buf[3] = (uint8_t)((new_size >> 0) & 0xFF);
2545}
2546
2547static void skd_get_link_info(struct pci_dev *pdev, u8 *speed, u8 *width)
2548{
2549 int pcie_reg;
2550 u16 pci_bus_speed;
2551 u8 pci_lanes;
2552
2553 pcie_reg = pci_find_capability(pdev, PCI_CAP_ID_EXP);
2554 if (pcie_reg) {
2555 u16 linksta;
2556 pci_read_config_word(pdev, pcie_reg + PCI_EXP_LNKSTA, &linksta);
2557
2558 pci_bus_speed = linksta & 0xF;
2559 pci_lanes = (linksta & 0x3F0) >> 4;
2560 } else {
2561 *speed = STEC_LINK_UNKNOWN;
2562 *width = 0xFF;
2563 return;
2564 }
2565
2566 switch (pci_bus_speed) {
2567 case 1:
2568 *speed = STEC_LINK_2_5GTS;
2569 break;
2570 case 2:
2571 *speed = STEC_LINK_5GTS;
2572 break;
2573 case 3:
2574 *speed = STEC_LINK_8GTS;
2575 break;
2576 default:
2577 *speed = STEC_LINK_UNKNOWN;
2578 break;
2579 }
2580
2581 if (pci_lanes <= 0x20)
2582 *width = pci_lanes;
2583 else
2584 *width = 0xFF;
2585}
2586
2587static void skd_do_inq_page_da(struct skd_device *skdev,
2588 volatile struct fit_completion_entry_v1 *skcomp,
2589 volatile struct fit_comp_error_info *skerr,
2590 uint8_t *cdb, uint8_t *buf)
2591{
fec23f63 2592 struct pci_dev *pdev = skdev->pdev;
e67f86b3
AB		 2593 unsigned max_bytes;
2594 struct driver_inquiry_data inq;
2595 u16 val;
2596
2e44b427 2597 pr_debug("%s:%s:%d skd_do_driver_inquiry: return driver page\n",
2598 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2599
2600 memset(&inq, 0, sizeof(inq));
2601
2602 inq.page_code = DRIVER_INQ_EVPD_PAGE_CODE;
2603
fec23f63
BZ		 2604 skd_get_link_info(pdev, &inq.pcie_link_speed, &inq.pcie_link_lanes);
2605 inq.pcie_bus_number = cpu_to_be16(pdev->bus->number);
2606 inq.pcie_device_number = PCI_SLOT(pdev->devfn);
2607 inq.pcie_function_number = PCI_FUNC(pdev->devfn);
e67f86b3 2608
fec23f63
BZ		 2609 pci_read_config_word(pdev, PCI_VENDOR_ID, &val);
2610 inq.pcie_vendor_id = cpu_to_be16(val);
e67f86b3 2611
fec23f63
BZ		 2612 pci_read_config_word(pdev, PCI_DEVICE_ID, &val);
2613 inq.pcie_device_id = cpu_to_be16(val);
e67f86b3 2614
fec23f63
BZ		 2615 pci_read_config_word(pdev, PCI_SUBSYSTEM_VENDOR_ID, &val);
2616 inq.pcie_subsystem_vendor_id = cpu_to_be16(val);
e67f86b3 2617
fec23f63
BZ		 2618 pci_read_config_word(pdev, PCI_SUBSYSTEM_ID, &val);
2619 inq.pcie_subsystem_device_id = cpu_to_be16(val);
e67f86b3
AB		 2620
2621 /* Driver version, fixed lenth, padded with spaces on the right */
2622 inq.driver_version_length = sizeof(inq.driver_version);
2623 memset(&inq.driver_version, ' ', sizeof(inq.driver_version));
2624 memcpy(inq.driver_version, DRV_VER_COMPL,
2625 min(sizeof(inq.driver_version), strlen(DRV_VER_COMPL)));
2626
2627 inq.page_length = cpu_to_be16((sizeof(inq) - 4));
2628
2629 /* Clear the error set by the device */
2630 skcomp->status = SAM_STAT_GOOD;
2631 memset((void *)skerr, 0, sizeof(*skerr));
2632
2633 /* copy response into output buffer */
2634 max_bytes = (cdb[3] << 8) | cdb[4];
2635 memcpy(buf, &inq, min_t(unsigned, max_bytes, sizeof(inq)));
2636
2637 skcomp->num_returned_bytes =
2638 be32_to_cpu(min_t(uint16_t, max_bytes, sizeof(inq)));
2639}
2640
2641static void skd_do_driver_inq(struct skd_device *skdev,
2642 volatile struct fit_completion_entry_v1 *skcomp,
2643 volatile struct fit_comp_error_info *skerr,
2644 uint8_t *cdb, uint8_t *buf)
2645{
2646 if (!buf)
2647 return;
2648 else if (cdb[0] != INQUIRY)
2649 return; /* Not an INQUIRY */
2650 else if ((cdb[1] & 1) == 0)
2651 return; /* EVPD not set */
2652 else if (cdb[2] == 0)
2653 /* Need to add driver's page to supported pages list */
2654 skd_do_inq_page_00(skdev, skcomp, skerr, cdb, buf);
2655 else if (cdb[2] == DRIVER_INQ_EVPD_PAGE_CODE)
2656 /* Caller requested driver's page */
2657 skd_do_inq_page_da(skdev, skcomp, skerr, cdb, buf);
2658}
2659
2660static unsigned char *skd_sg_1st_page_ptr(struct scatterlist *sg)
2661{
2662 if (!sg)
2663 return NULL;
2664 if (!sg_page(sg))
2665 return NULL;
2666 return sg_virt(sg);
2667}
2668
2669static void skd_process_scsi_inq(struct skd_device *skdev,
2670 volatile struct fit_completion_entry_v1
2671 *skcomp,
2672 volatile struct fit_comp_error_info *skerr,
2673 struct skd_special_context *skspcl)
2674{
2675 uint8_t *buf;
2676 struct fit_msg_hdr *fmh = (struct fit_msg_hdr *)skspcl->msg_buf;
2677 struct skd_scsi_request *scsi_req = (struct skd_scsi_request *)&fmh[1];
2678
2679 dma_sync_sg_for_cpu(skdev->class_dev, skspcl->req.sg, skspcl->req.n_sg,
2680 skspcl->req.sg_data_dir);
2681 buf = skd_sg_1st_page_ptr(skspcl->req.sg);
2682
2683 if (buf)
2684 skd_do_driver_inq(skdev, skcomp, skerr, scsi_req->cdb, buf);
2685}
2686
2687
2688static int skd_isr_completion_posted(struct skd_device *skdev,
2689 int limit, int *enqueued)
2690{
2691 volatile struct fit_completion_entry_v1 *skcmp = NULL;
2692 volatile struct fit_comp_error_info *skerr;
2693 u16 req_id;
2694 u32 req_slot;
2695 struct skd_request_context *skreq;
2696 u16 cmp_cntxt = 0;
2697 u8 cmp_status = 0;
2698 u8 cmp_cycle = 0;
2699 u32 cmp_bytes = 0;
2700 int rc = 0;
2701 int processed = 0;
e67f86b3
AB		 2702
2703 for (;; ) {
2704 SKD_ASSERT(skdev->skcomp_ix < SKD_N_COMPLETION_ENTRY);
2705
2706 skcmp = &skdev->skcomp_table[skdev->skcomp_ix];
2707 cmp_cycle = skcmp->cycle;
2708 cmp_cntxt = skcmp->tag;
2709 cmp_status = skcmp->status;
2710 cmp_bytes = be32_to_cpu(skcmp->num_returned_bytes);
2711
2712 skerr = &skdev->skerr_table[skdev->skcomp_ix];
2713
2e44b427 2714 pr_debug("%s:%s:%d "
2715 "cycle=%d ix=%d got cycle=%d cmdctxt=0x%x stat=%d "
2716 "busy=%d rbytes=0x%x proto=%d\n",
2717 skdev->name, __func__, __LINE__, skdev->skcomp_cycle,
2718 skdev->skcomp_ix, cmp_cycle, cmp_cntxt, cmp_status,
2719 skdev->in_flight, cmp_bytes, skdev->proto_ver);
e67f86b3
AB		 2720
2721 if (cmp_cycle != skdev->skcomp_cycle) {
2e44b427 2722 pr_debug("%s:%s:%d end of completions\n",
2723 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2724 break;
2725 }
2726 /*
2727 * Update the completion queue head index and possibly
2728 * the completion cycle count. 8-bit wrap-around.
2729 */
2730 skdev->skcomp_ix++;
2731 if (skdev->skcomp_ix >= SKD_N_COMPLETION_ENTRY) {
2732 skdev->skcomp_ix = 0;
2733 skdev->skcomp_cycle++;
2734 }
2735
2736 /*
2737 * The command context is a unique 32-bit ID. The low order
2738 * bits help locate the request. The request is usually a
2739 * r/w request (see skd_start() above) or a special request.
2740 */
2741 req_id = cmp_cntxt;
2742 req_slot = req_id & SKD_ID_SLOT_AND_TABLE_MASK;
2743
2744 /* Is this other than a r/w request? */
2745 if (req_slot >= skdev->num_req_context) {
2746 /*
2747 * This is not a completion for a r/w request.
2748 */
2749 skd_complete_other(skdev, skcmp, skerr);
2750 continue;
2751 }
2752
2753 skreq = &skdev->skreq_table[req_slot];
2754
2755 /*
2756 * Make sure the request ID for the slot matches.
2757 */
2758 if (skreq->id != req_id) {
2e44b427 2759 pr_debug("%s:%s:%d mismatch comp_id=0x%x req_id=0x%x\n",
2760 skdev->name, __func__, __LINE__,
2761 req_id, skreq->id);
e67f86b3
AB		 2762 {
2763 u16 new_id = cmp_cntxt;
2764 pr_err("(%s): Completion mismatch "
2765 "comp_id=0x%04x skreq=0x%04x new=0x%04x\n",
2766 skd_name(skdev), req_id,
2767 skreq->id, new_id);
2768
2769 continue;
2770 }
2771 }
2772
2773 SKD_ASSERT(skreq->state == SKD_REQ_STATE_BUSY);
2774
2775 if (skreq->state == SKD_REQ_STATE_ABORTED) {
2e44b427 2776 pr_debug("%s:%s:%d reclaim req %p id=%04x\n",
2777 skdev->name, __func__, __LINE__,
2778 skreq, skreq->id);
e67f86b3
AB		 2779 /* a previously timed out command can
2780 * now be cleaned up */
2781 skd_release_skreq(skdev, skreq);
2782 continue;
2783 }
2784
2785 skreq->completion = *skcmp;
2786 if (unlikely(cmp_status == SAM_STAT_CHECK_CONDITION)) {
2787 skreq->err_info = *skerr;
2788 skd_log_check_status(skdev, cmp_status, skerr->key,
2789 skerr->code, skerr->qual,
2790 skerr->fruc);
2791 }
2792 /* Release DMA resources for the request. */
2793 if (skreq->n_sg > 0)
2794 skd_postop_sg_list(skdev, skreq);
2795
fcd37eb3 2796 if (!skreq->req) {
2e44b427 2797 pr_debug("%s:%s:%d NULL backptr skdreq %p, "
2798 "req=0x%x req_id=0x%x\n",
2799 skdev->name, __func__, __LINE__,
2800 skreq, skreq->id, req_id);
e67f86b3
AB		 2801 } else {
2802 /*
2803 * Capture the outcome and post it back to the
2804 * native request.
2805 */
fcd37eb3
JA		 2806 if (likely(cmp_status == SAM_STAT_GOOD))
2807 skd_end_request(skdev, skreq, 0);
2808 else
e67f86b3 2809 skd_resolve_req_exception(skdev, skreq);
e67f86b3
AB		 2810 }
2811
2812 /*
2813 * Release the skreq, its FIT msg (if one), timeout slot,
2814 * and queue depth.
2815 */
2816 skd_release_skreq(skdev, skreq);
2817
2818 /* skd_isr_comp_limit equal zero means no limit */
2819 if (limit) {
2820 if (++processed >= limit) {
2821 rc = 1;
2822 break;
2823 }
2824 }
2825 }
2826
2827 if ((skdev->state == SKD_DRVR_STATE_PAUSING)
2828 && (skdev->in_flight) == 0) {
2829 skdev->state = SKD_DRVR_STATE_PAUSED;
2830 wake_up_interruptible(&skdev->waitq);
2831 }
2832
2833 return rc;
2834}
2835
2836static void skd_complete_other(struct skd_device *skdev,
2837 volatile struct fit_completion_entry_v1 *skcomp,
2838 volatile struct fit_comp_error_info *skerr)
2839{
2840 u32 req_id = 0;
2841 u32 req_table;
2842 u32 req_slot;
2843 struct skd_special_context *skspcl;
2844
2845 req_id = skcomp->tag;
2846 req_table = req_id & SKD_ID_TABLE_MASK;
2847 req_slot = req_id & SKD_ID_SLOT_MASK;
2848
2e44b427 2849 pr_debug("%s:%s:%d table=0x%x id=0x%x slot=%d\n",
2850 skdev->name, __func__, __LINE__,
2851 req_table, req_id, req_slot);
e67f86b3
AB		 2852
2853 /*
2854 * Based on the request id, determine how to dispatch this completion.
2855 * This swich/case is finding the good cases and forwarding the
2856 * completion entry. Errors are reported below the switch.
2857 */
2858 switch (req_table) {
2859 case SKD_ID_RW_REQUEST:
2860 /*
2861 * The caller, skd_completion_posted_isr() above,
2862 * handles r/w requests. The only way we get here
2863 * is if the req_slot is out of bounds.
2864 */
2865 break;
2866
2867 case SKD_ID_SPECIAL_REQUEST:
2868 /*
2869 * Make sure the req_slot is in bounds and that the id
2870 * matches.
2871 */
2872 if (req_slot < skdev->n_special) {
2873 skspcl = &skdev->skspcl_table[req_slot];
2874 if (skspcl->req.id == req_id &&
2875 skspcl->req.state == SKD_REQ_STATE_BUSY) {
2876 skd_complete_special(skdev,
2877 skcomp, skerr, skspcl);
2878 return;
2879 }
2880 }
2881 break;
2882
2883 case SKD_ID_INTERNAL:
2884 if (req_slot == 0) {
2885 skspcl = &skdev->internal_skspcl;
2886 if (skspcl->req.id == req_id &&
2887 skspcl->req.state == SKD_REQ_STATE_BUSY) {
2888 skd_complete_internal(skdev,
2889 skcomp, skerr, skspcl);
2890 return;
2891 }
2892 }
2893 break;
2894
2895 case SKD_ID_FIT_MSG:
2896 /*
2897 * These id's should never appear in a completion record.
2898 */
2899 break;
2900
2901 default:
2902 /*
2903 * These id's should never appear anywhere;
2904 */
2905 break;
2906 }
2907
2908 /*
2909 * If we get here it is a bad or stale id.
2910 */
2911}
2912
2913static void skd_complete_special(struct skd_device *skdev,
2914 volatile struct fit_completion_entry_v1
2915 *skcomp,
2916 volatile struct fit_comp_error_info *skerr,
2917 struct skd_special_context *skspcl)
2918{
2e44b427 2919 pr_debug("%s:%s:%d completing special request %p\n",
2920 skdev->name, __func__, __LINE__, skspcl);
e67f86b3
AB		 2921 if (skspcl->orphaned) {
2922 /* Discard orphaned request */
2923 /* ?: Can this release directly or does it need
2924 * to use a worker? */
2e44b427 2925 pr_debug("%s:%s:%d release orphaned %p\n",
2926 skdev->name, __func__, __LINE__, skspcl);
e67f86b3
AB		 2927 skd_release_special(skdev, skspcl);
2928 return;
2929 }
2930
2931 skd_process_scsi_inq(skdev, skcomp, skerr, skspcl);
2932
2933 skspcl->req.state = SKD_REQ_STATE_COMPLETED;
2934 skspcl->req.completion = *skcomp;
2935 skspcl->req.err_info = *skerr;
2936
2937 skd_log_check_status(skdev, skspcl->req.completion.status, skerr->key,
2938 skerr->code, skerr->qual, skerr->fruc);
2939
2940 wake_up_interruptible(&skdev->waitq);
2941}
2942
2943/* assume spinlock is already held */
2944static void skd_release_special(struct skd_device *skdev,
2945 struct skd_special_context *skspcl)
2946{
2947 int i, was_depleted;
2948
2949 for (i = 0; i < skspcl->req.n_sg; i++) {
e67f86b3
AB		 2950 struct page *page = sg_page(&skspcl->req.sg[i]);
2951 __free_page(page);
2952 }
2953
2954 was_depleted = (skdev->skspcl_free_list == NULL);
2955
2956 skspcl->req.state = SKD_REQ_STATE_IDLE;
2957 skspcl->req.id += SKD_ID_INCR;
2958 skspcl->req.next =
2959 (struct skd_request_context *)skdev->skspcl_free_list;
2960 skdev->skspcl_free_list = (struct skd_special_context *)skspcl;
2961
2962 if (was_depleted) {
2e44b427 2963 pr_debug("%s:%s:%d skspcl was depleted\n",
2964 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2965 /* Free list was depleted. Their might be waiters. */
2966 wake_up_interruptible(&skdev->waitq);
2967 }
2968}
2969
2970static void skd_reset_skcomp(struct skd_device *skdev)
2971{
2972 u32 nbytes;
2973 struct fit_completion_entry_v1 *skcomp;
2974
2975 nbytes = sizeof(*skcomp) * SKD_N_COMPLETION_ENTRY;
2976 nbytes += sizeof(struct fit_comp_error_info) * SKD_N_COMPLETION_ENTRY;
2977
2978 memset(skdev->skcomp_table, 0, nbytes);
2979
2980 skdev->skcomp_ix = 0;
2981 skdev->skcomp_cycle = 1;
2982}
2983
2984/*
2985 *****************************************************************************
2986 * INTERRUPTS
2987 *****************************************************************************
2988 */
2989static void skd_completion_worker(struct work_struct *work)
2990{
2991 struct skd_device *skdev =
2992 container_of(work, struct skd_device, completion_worker);
2993 unsigned long flags;
2994 int flush_enqueued = 0;
2995
2996 spin_lock_irqsave(&skdev->lock, flags);
2997
2998 /*
2999 * pass in limit=0, which means no limit..
3000 * process everything in compq
3001 */
3002 skd_isr_completion_posted(skdev, 0, &flush_enqueued);
3003 skd_request_fn(skdev->queue);
3004
3005 spin_unlock_irqrestore(&skdev->lock, flags);
3006}
3007
3008static void skd_isr_msg_from_dev(struct skd_device *skdev);
3009
3010irqreturn_t
3011static skd_isr(int irq, void *ptr)
3012{
3013 struct skd_device *skdev;
3014 u32 intstat;
3015 u32 ack;
3016 int rc = 0;
3017 int deferred = 0;
3018 int flush_enqueued = 0;
3019
3020 skdev = (struct skd_device *)ptr;
3021 spin_lock(&skdev->lock);
3022
3023 for (;; ) {
3024 intstat = SKD_READL(skdev, FIT_INT_STATUS_HOST);
3025
3026 ack = FIT_INT_DEF_MASK;
3027 ack &= intstat;
3028
2e44b427 3029 pr_debug("%s:%s:%d intstat=0x%x ack=0x%x\n",
3030 skdev->name, __func__, __LINE__, intstat, ack);
e67f86b3
AB		 3031
3032 /* As long as there is an int pending on device, keep
3033 * running loop. When none, get out, but if we've never
3034 * done any processing, call completion handler?
3035 */
3036 if (ack == 0) {
3037 /* No interrupts on device, but run the completion
3038 * processor anyway?
3039 */
3040 if (rc == 0)
3041 if (likely (skdev->state
3042 == SKD_DRVR_STATE_ONLINE))
3043 deferred = 1;
3044 break;
3045 }
3046
3047 rc = IRQ_HANDLED;
3048
3049 SKD_WRITEL(skdev, ack, FIT_INT_STATUS_HOST);
3050
3051 if (likely((skdev->state != SKD_DRVR_STATE_LOAD) &&
3052 (skdev->state != SKD_DRVR_STATE_STOPPING))) {
3053 if (intstat & FIT_ISH_COMPLETION_POSTED) {
3054 /*
3055 * If we have already deferred completion
3056 * processing, don't bother running it again
3057 */
3058 if (deferred == 0)
3059 deferred =
3060 skd_isr_completion_posted(skdev,
3061 skd_isr_comp_limit, &flush_enqueued);
3062 }
3063
3064 if (intstat & FIT_ISH_FW_STATE_CHANGE) {
3065 skd_isr_fwstate(skdev);
3066 if (skdev->state == SKD_DRVR_STATE_FAULT ||
3067 skdev->state ==
3068 SKD_DRVR_STATE_DISAPPEARED) {
3069 spin_unlock(&skdev->lock);
3070 return rc;
3071 }
3072 }
3073
3074 if (intstat & FIT_ISH_MSG_FROM_DEV)
3075 skd_isr_msg_from_dev(skdev);
3076 }
3077 }
3078
3079 if (unlikely(flush_enqueued))
3080 skd_request_fn(skdev->queue);
3081
3082 if (deferred)
3083 schedule_work(&skdev->completion_worker);
3084 else if (!flush_enqueued)
3085 skd_request_fn(skdev->queue);
3086
3087 spin_unlock(&skdev->lock);
3088
3089 return rc;
3090}
3091
e67f86b3
AB		 3092static void skd_drive_fault(struct skd_device *skdev)
3093{
3094 skdev->state = SKD_DRVR_STATE_FAULT;
3095 pr_err("(%s): Drive FAULT\n", skd_name(skdev));
3096}
3097
3098static void skd_drive_disappeared(struct skd_device *skdev)
3099{
3100 skdev->state = SKD_DRVR_STATE_DISAPPEARED;
3101 pr_err("(%s): Drive DISAPPEARED\n", skd_name(skdev));
3102}
3103
3104static void skd_isr_fwstate(struct skd_device *skdev)
3105{
3106 u32 sense;
3107 u32 state;
3108 u32 mtd;
3109 int prev_driver_state = skdev->state;
3110
3111 sense = SKD_READL(skdev, FIT_STATUS);
3112 state = sense & FIT_SR_DRIVE_STATE_MASK;
3113
3114 pr_err("(%s): s1120 state %s(%d)=>%s(%d)\n",
3115 skd_name(skdev),
3116 skd_drive_state_to_str(skdev->drive_state), skdev->drive_state,
3117 skd_drive_state_to_str(state), state);
3118
3119 skdev->drive_state = state;
3120
3121 switch (skdev->drive_state) {
3122 case FIT_SR_DRIVE_INIT:
3123 if (skdev->state == SKD_DRVR_STATE_PROTOCOL_MISMATCH) {
3124 skd_disable_interrupts(skdev);
3125 break;
3126 }
3127 if (skdev->state == SKD_DRVR_STATE_RESTARTING)
3128 skd_recover_requests(skdev, 0);
3129 if (skdev->state == SKD_DRVR_STATE_WAIT_BOOT) {
3130 skdev->timer_countdown = SKD_STARTING_TIMO;
3131 skdev->state = SKD_DRVR_STATE_STARTING;
3132 skd_soft_reset(skdev);
3133 break;
3134 }
3135 mtd = FIT_MXD_CONS(FIT_MTD_FITFW_INIT, 0, 0);
3136 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3137 skdev->last_mtd = mtd;
3138 break;
3139
3140 case FIT_SR_DRIVE_ONLINE:
3141 skdev->cur_max_queue_depth = skd_max_queue_depth;
3142 if (skdev->cur_max_queue_depth > skdev->dev_max_queue_depth)
3143 skdev->cur_max_queue_depth = skdev->dev_max_queue_depth;
3144
3145 skdev->queue_low_water_mark =
3146 skdev->cur_max_queue_depth * 2 / 3 + 1;
3147 if (skdev->queue_low_water_mark < 1)
3148 skdev->queue_low_water_mark = 1;
3149 pr_info(
3150 "(%s): Queue depth limit=%d dev=%d lowat=%d\n",
3151 skd_name(skdev),
3152 skdev->cur_max_queue_depth,
3153 skdev->dev_max_queue_depth, skdev->queue_low_water_mark);
3154
3155 skd_refresh_device_data(skdev);
3156 break;
3157
3158 case FIT_SR_DRIVE_BUSY:
3159 skdev->state = SKD_DRVR_STATE_BUSY;
3160 skdev->timer_countdown = SKD_BUSY_TIMO;
3161 skd_quiesce_dev(skdev);
3162 break;
3163 case FIT_SR_DRIVE_BUSY_SANITIZE:
3164 /* set timer for 3 seconds, we'll abort any unfinished
3165 * commands after that expires
3166 */
3167 skdev->state = SKD_DRVR_STATE_BUSY_SANITIZE;
3168 skdev->timer_countdown = SKD_TIMER_SECONDS(3);
6a5ec65b 3169 blk_start_queue(skdev->queue);
e67f86b3
AB		 3170 break;
3171 case FIT_SR_DRIVE_BUSY_ERASE:
3172 skdev->state = SKD_DRVR_STATE_BUSY_ERASE;
3173 skdev->timer_countdown = SKD_BUSY_TIMO;
3174 break;
3175 case FIT_SR_DRIVE_OFFLINE:
3176 skdev->state = SKD_DRVR_STATE_IDLE;
3177 break;
3178 case FIT_SR_DRIVE_SOFT_RESET:
3179 switch (skdev->state) {
3180 case SKD_DRVR_STATE_STARTING:
3181 case SKD_DRVR_STATE_RESTARTING:
3182 /* Expected by a caller of skd_soft_reset() */
3183 break;
3184 default:
3185 skdev->state = SKD_DRVR_STATE_RESTARTING;
3186 break;
3187 }
3188 break;
3189 case FIT_SR_DRIVE_FW_BOOTING:
2e44b427 3190 pr_debug("%s:%s:%d ISR FIT_SR_DRIVE_FW_BOOTING %s\n",
3191 skdev->name, __func__, __LINE__, skdev->name);
e67f86b3
AB		 3192 skdev->state = SKD_DRVR_STATE_WAIT_BOOT;
3193 skdev->timer_countdown = SKD_WAIT_BOOT_TIMO;
3194 break;
3195
3196 case FIT_SR_DRIVE_DEGRADED:
3197 case FIT_SR_PCIE_LINK_DOWN:
3198 case FIT_SR_DRIVE_NEED_FW_DOWNLOAD:
3199 break;
3200
3201 case FIT_SR_DRIVE_FAULT:
3202 skd_drive_fault(skdev);
3203 skd_recover_requests(skdev, 0);
6a5ec65b 3204 blk_start_queue(skdev->queue);
e67f86b3
AB		 3205 break;
3206
3207 /* PCIe bus returned all Fs? */
3208 case 0xFF:
3209 pr_info("(%s): state=0x%x sense=0x%x\n",
3210 skd_name(skdev), state, sense);
3211 skd_drive_disappeared(skdev);
3212 skd_recover_requests(skdev, 0);
6a5ec65b 3213 blk_start_queue(skdev->queue);
e67f86b3
AB		 3214 break;
3215 default:
3216 /*
3217 * Uknown FW State. Wait for a state we recognize.
3218 */
3219 break;
3220 }
3221 pr_err("(%s): Driver state %s(%d)=>%s(%d)\n",
3222 skd_name(skdev),
3223 skd_skdev_state_to_str(prev_driver_state), prev_driver_state,
3224 skd_skdev_state_to_str(skdev->state), skdev->state);
3225}
3226
3227static void skd_recover_requests(struct skd_device *skdev, int requeue)
3228{
3229 int i;
3230
3231 for (i = 0; i < skdev->num_req_context; i++) {
3232 struct skd_request_context *skreq = &skdev->skreq_table[i];
3233
3234 if (skreq->state == SKD_REQ_STATE_BUSY) {
3235 skd_log_skreq(skdev, skreq, "recover");
3236
3237 SKD_ASSERT((skreq->id & SKD_ID_INCR) != 0);
fcd37eb3 3238 SKD_ASSERT(skreq->req != NULL);
e67f86b3
AB		 3239
3240 /* Release DMA resources for the request. */
3241 if (skreq->n_sg > 0)
3242 skd_postop_sg_list(skdev, skreq);
3243
fcd37eb3
JA		 3244 if (requeue &&
3245 (unsigned long) ++skreq->req->special <
3246 SKD_MAX_RETRIES)
38d4a1bb 3247 blk_requeue_request(skdev->queue, skreq->req);
fcd37eb3 3248 else
e67f86b3
AB		 3249 skd_end_request(skdev, skreq, -EIO);
3250
fcd37eb3 3251 skreq->req = NULL;
e67f86b3
AB		 3252
3253 skreq->state = SKD_REQ_STATE_IDLE;
3254 skreq->id += SKD_ID_INCR;
e67f86b3
AB		 3255 }
3256 if (i > 0)
3257 skreq[-1].next = skreq;
3258 skreq->next = NULL;
3259 }
3260 skdev->skreq_free_list = skdev->skreq_table;
3261
3262 for (i = 0; i < skdev->num_fitmsg_context; i++) {
3263 struct skd_fitmsg_context *skmsg = &skdev->skmsg_table[i];
3264
3265 if (skmsg->state == SKD_MSG_STATE_BUSY) {
3266 skd_log_skmsg(skdev, skmsg, "salvaged");
3267 SKD_ASSERT((skmsg->id & SKD_ID_INCR) != 0);
3268 skmsg->state = SKD_MSG_STATE_IDLE;
3269 skmsg->id += SKD_ID_INCR;
3270 }
3271 if (i > 0)
3272 skmsg[-1].next = skmsg;
3273 skmsg->next = NULL;
3274 }
3275 skdev->skmsg_free_list = skdev->skmsg_table;
3276
3277 for (i = 0; i < skdev->n_special; i++) {
3278 struct skd_special_context *skspcl = &skdev->skspcl_table[i];
3279
3280 /* If orphaned, reclaim it because it has already been reported
3281 * to the process as an error (it was just waiting for
3282 * a completion that didn't come, and now it will never come)
3283 * If busy, change to a state that will cause it to error
3284 * out in the wait routine and let it do the normal
3285 * reporting and reclaiming
3286 */
3287 if (skspcl->req.state == SKD_REQ_STATE_BUSY) {
3288 if (skspcl->orphaned) {
2e44b427 3289 pr_debug("%s:%s:%d orphaned %p\n",
3290 skdev->name, __func__, __LINE__,
3291 skspcl);
e67f86b3
AB		 3292 skd_release_special(skdev, skspcl);
3293 } else {
2e44b427 3294 pr_debug("%s:%s:%d not orphaned %p\n",
3295 skdev->name, __func__, __LINE__,
3296 skspcl);
e67f86b3
AB		 3297 skspcl->req.state = SKD_REQ_STATE_ABORTED;
3298 }
3299 }
3300 }
3301 skdev->skspcl_free_list = skdev->skspcl_table;
3302
3303 for (i = 0; i < SKD_N_TIMEOUT_SLOT; i++)
3304 skdev->timeout_slot[i] = 0;
3305
3306 skdev->in_flight = 0;
3307}
3308
3309static void skd_isr_msg_from_dev(struct skd_device *skdev)
3310{
3311 u32 mfd;
3312 u32 mtd;
3313 u32 data;
3314
3315 mfd = SKD_READL(skdev, FIT_MSG_FROM_DEVICE);
3316
2e44b427 3317 pr_debug("%s:%s:%d mfd=0x%x last_mtd=0x%x\n",
3318 skdev->name, __func__, __LINE__, mfd, skdev->last_mtd);
e67f86b3
AB		 3319
3320 /* ignore any mtd that is an ack for something we didn't send */
3321 if (FIT_MXD_TYPE(mfd) != FIT_MXD_TYPE(skdev->last_mtd))
3322 return;
3323
3324 switch (FIT_MXD_TYPE(mfd)) {
3325 case FIT_MTD_FITFW_INIT:
3326 skdev->proto_ver = FIT_PROTOCOL_MAJOR_VER(mfd);
3327
3328 if (skdev->proto_ver != FIT_PROTOCOL_VERSION_1) {
3329 pr_err("(%s): protocol mismatch\n",
3330 skdev->name);
3331 pr_err("(%s): got=%d support=%d\n",
3332 skdev->name, skdev->proto_ver,
3333 FIT_PROTOCOL_VERSION_1);
3334 pr_err("(%s): please upgrade driver\n",
3335 skdev->name);
3336 skdev->state = SKD_DRVR_STATE_PROTOCOL_MISMATCH;
3337 skd_soft_reset(skdev);
3338 break;
3339 }
3340 mtd = FIT_MXD_CONS(FIT_MTD_GET_CMDQ_DEPTH, 0, 0);
3341 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3342 skdev->last_mtd = mtd;
3343 break;
3344
3345 case FIT_MTD_GET_CMDQ_DEPTH:
3346 skdev->dev_max_queue_depth = FIT_MXD_DATA(mfd);
3347 mtd = FIT_MXD_CONS(FIT_MTD_SET_COMPQ_DEPTH, 0,
3348 SKD_N_COMPLETION_ENTRY);
3349 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3350 skdev->last_mtd = mtd;
3351 break;
3352
3353 case FIT_MTD_SET_COMPQ_DEPTH:
3354 SKD_WRITEQ(skdev, skdev->cq_dma_address, FIT_MSG_TO_DEVICE_ARG);
3355 mtd = FIT_MXD_CONS(FIT_MTD_SET_COMPQ_ADDR, 0, 0);
3356 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3357 skdev->last_mtd = mtd;
3358 break;
3359
3360 case FIT_MTD_SET_COMPQ_ADDR:
3361 skd_reset_skcomp(skdev);
3362 mtd = FIT_MXD_CONS(FIT_MTD_CMD_LOG_HOST_ID, 0, skdev->devno);
3363 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3364 skdev->last_mtd = mtd;
3365 break;
3366
3367 case FIT_MTD_CMD_LOG_HOST_ID:
3368 skdev->connect_time_stamp = get_seconds();
3369 data = skdev->connect_time_stamp & 0xFFFF;
3370 mtd = FIT_MXD_CONS(FIT_MTD_CMD_LOG_TIME_STAMP_LO, 0, data);
3371 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3372 skdev->last_mtd = mtd;
3373 break;
3374
3375 case FIT_MTD_CMD_LOG_TIME_STAMP_LO:
3376 skdev->drive_jiffies = FIT_MXD_DATA(mfd);
3377 data = (skdev->connect_time_stamp >> 16) & 0xFFFF;
3378 mtd = FIT_MXD_CONS(FIT_MTD_CMD_LOG_TIME_STAMP_HI, 0, data);
3379 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3380 skdev->last_mtd = mtd;
3381 break;
3382
3383 case FIT_MTD_CMD_LOG_TIME_STAMP_HI:
3384 skdev->drive_jiffies |= (FIT_MXD_DATA(mfd) << 16);
3385 mtd = FIT_MXD_CONS(FIT_MTD_ARM_QUEUE, 0, 0);
3386 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3387 skdev->last_mtd = mtd;
3388
3389 pr_err("(%s): Time sync driver=0x%x device=0x%x\n",
3390 skd_name(skdev),
3391 skdev->connect_time_stamp, skdev->drive_jiffies);
3392 break;
3393
3394 case FIT_MTD_ARM_QUEUE:
3395 skdev->last_mtd = 0;
3396 /*
3397 * State should be, or soon will be, FIT_SR_DRIVE_ONLINE.
3398 */
3399 break;
3400
3401 default:
3402 break;
3403 }
3404}
3405
3406static void skd_disable_interrupts(struct skd_device *skdev)
3407{
3408 u32 sense;
3409
3410 sense = SKD_READL(skdev, FIT_CONTROL);
3411 sense &= ~FIT_CR_ENABLE_INTERRUPTS;
3412 SKD_WRITEL(skdev, sense, FIT_CONTROL);
2e44b427 3413 pr_debug("%s:%s:%d sense 0x%x\n",
3414 skdev->name, __func__, __LINE__, sense);
e67f86b3
AB		 3415
3416 /* Note that the 1s is written. A 1-bit means
3417 * disable, a 0 means enable.
3418 */
3419 SKD_WRITEL(skdev, ~0, FIT_INT_MASK_HOST);
3420}
3421
3422static void skd_enable_interrupts(struct skd_device *skdev)
3423{
3424 u32 val;
3425
3426 /* unmask interrupts first */
3427 val = FIT_ISH_FW_STATE_CHANGE +
3428 FIT_ISH_COMPLETION_POSTED + FIT_ISH_MSG_FROM_DEV;
3429
3430 /* Note that the compliment of mask is written. A 1-bit means
3431 * disable, a 0 means enable. */
3432 SKD_WRITEL(skdev, ~val, FIT_INT_MASK_HOST);
2e44b427 3433 pr_debug("%s:%s:%d interrupt mask=0x%x\n",
3434 skdev->name, __func__, __LINE__, ~val);
e67f86b3
AB		 3435
3436 val = SKD_READL(skdev, FIT_CONTROL);
3437 val |= FIT_CR_ENABLE_INTERRUPTS;
2e44b427 3438 pr_debug("%s:%s:%d control=0x%x\n",
3439 skdev->name, __func__, __LINE__, val);
e67f86b3
AB		 3440 SKD_WRITEL(skdev, val, FIT_CONTROL);
3441}
3442
3443/*
3444 *****************************************************************************
3445 * START, STOP, RESTART, QUIESCE, UNQUIESCE
3446 *****************************************************************************
3447 */
3448
3449static void skd_soft_reset(struct skd_device *skdev)
3450{
3451 u32 val;
3452
3453 val = SKD_READL(skdev, FIT_CONTROL);
3454 val |= (FIT_CR_SOFT_RESET);
2e44b427 3455 pr_debug("%s:%s:%d control=0x%x\n",
3456 skdev->name, __func__, __LINE__, val);
e67f86b3
AB		 3457 SKD_WRITEL(skdev, val, FIT_CONTROL);
3458}
3459
3460static void skd_start_device(struct skd_device *skdev)
3461{
3462 unsigned long flags;
3463 u32 sense;
3464 u32 state;
3465
3466 spin_lock_irqsave(&skdev->lock, flags);
3467
3468 /* ack all ghost interrupts */
3469 SKD_WRITEL(skdev, FIT_INT_DEF_MASK, FIT_INT_STATUS_HOST);
3470
3471 sense = SKD_READL(skdev, FIT_STATUS);
3472
2e44b427 3473 pr_debug("%s:%s:%d initial status=0x%x\n",
3474 skdev->name, __func__, __LINE__, sense);
e67f86b3
AB		 3475
3476 state = sense & FIT_SR_DRIVE_STATE_MASK;
3477 skdev->drive_state = state;
3478 skdev->last_mtd = 0;
3479
3480 skdev->state = SKD_DRVR_STATE_STARTING;
3481 skdev->timer_countdown = SKD_STARTING_TIMO;
3482
3483 skd_enable_interrupts(skdev);
3484
3485 switch (skdev->drive_state) {
3486 case FIT_SR_DRIVE_OFFLINE:
3487 pr_err("(%s): Drive offline...\n", skd_name(skdev));
3488 break;
3489
3490 case FIT_SR_DRIVE_FW_BOOTING:
2e44b427 3491 pr_debug("%s:%s:%d FIT_SR_DRIVE_FW_BOOTING %s\n",
3492 skdev->name, __func__, __LINE__, skdev->name);
e67f86b3
AB		 3493 skdev->state = SKD_DRVR_STATE_WAIT_BOOT;
3494 skdev->timer_countdown = SKD_WAIT_BOOT_TIMO;
3495 break;
3496
3497 case FIT_SR_DRIVE_BUSY_SANITIZE:
3498 pr_info("(%s): Start: BUSY_SANITIZE\n",
3499 skd_name(skdev));
3500 skdev->state = SKD_DRVR_STATE_BUSY_SANITIZE;
3501 skdev->timer_countdown = SKD_STARTED_BUSY_TIMO;
3502 break;
3503
3504 case FIT_SR_DRIVE_BUSY_ERASE:
3505 pr_info("(%s): Start: BUSY_ERASE\n", skd_name(skdev));
3506 skdev->state = SKD_DRVR_STATE_BUSY_ERASE;
3507 skdev->timer_countdown = SKD_STARTED_BUSY_TIMO;
3508 break;
3509
3510 case FIT_SR_DRIVE_INIT:
3511 case FIT_SR_DRIVE_ONLINE:
3512 skd_soft_reset(skdev);
3513 break;
3514
3515 case FIT_SR_DRIVE_BUSY:
3516 pr_err("(%s): Drive Busy...\n", skd_name(skdev));
3517 skdev->state = SKD_DRVR_STATE_BUSY;
3518 skdev->timer_countdown = SKD_STARTED_BUSY_TIMO;
3519 break;
3520
3521 case FIT_SR_DRIVE_SOFT_RESET:
3522 pr_err("(%s) drive soft reset in prog\n",
3523 skd_name(skdev));
3524 break;
3525
3526 case FIT_SR_DRIVE_FAULT:
3527 /* Fault state is bad...soft reset won't do it...
3528 * Hard reset, maybe, but does it work on device?
3529 * For now, just fault so the system doesn't hang.
3530 */
3531 skd_drive_fault(skdev);
3532 /*start the queue so we can respond with error to requests */
2e44b427 3533 pr_debug("%s:%s:%d starting %s queue\n",
3534 skdev->name, __func__, __LINE__, skdev->name);
6a5ec65b 3535 blk_start_queue(skdev->queue);
e67f86b3
AB		 3536 skdev->gendisk_on = -1;
3537 wake_up_interruptible(&skdev->waitq);
3538 break;
3539
3540 case 0xFF:
3541 /* Most likely the device isn't there or isn't responding
3542 * to the BAR1 addresses. */
3543 skd_drive_disappeared(skdev);
3544 /*start the queue so we can respond with error to requests */
2e44b427 3545 pr_debug("%s:%s:%d starting %s queue to error-out reqs\n",
3546 skdev->name, __func__, __LINE__, skdev->name);
6a5ec65b 3547 blk_start_queue(skdev->queue);
e67f86b3
AB		 3548 skdev->gendisk_on = -1;
3549 wake_up_interruptible(&skdev->waitq);
3550 break;
3551
3552 default:
3553 pr_err("(%s) Start: unknown state %x\n",
3554 skd_name(skdev), skdev->drive_state);
3555 break;
3556 }
3557
3558 state = SKD_READL(skdev, FIT_CONTROL);
2e44b427 3559 pr_debug("%s:%s:%d FIT Control Status=0x%x\n",
3560 skdev->name, __func__, __LINE__, state);
e67f86b3
AB		 3561
3562 state = SKD_READL(skdev, FIT_INT_STATUS_HOST);
2e44b427 3563 pr_debug("%s:%s:%d Intr Status=0x%x\n",
3564 skdev->name, __func__, __LINE__, state);
e67f86b3
AB		 3565
3566 state = SKD_READL(skdev, FIT_INT_MASK_HOST);
2e44b427 3567 pr_debug("%s:%s:%d Intr Mask=0x%x\n",
3568 skdev->name, __func__, __LINE__, state);
e67f86b3
AB		 3569
3570 state = SKD_READL(skdev, FIT_MSG_FROM_DEVICE);
2e44b427 3571 pr_debug("%s:%s:%d Msg from Dev=0x%x\n",
3572 skdev->name, __func__, __LINE__, state);
e67f86b3
AB		 3573
3574 state = SKD_READL(skdev, FIT_HW_VERSION);
2e44b427 3575 pr_debug("%s:%s:%d HW version=0x%x\n",
3576 skdev->name, __func__, __LINE__, state);
e67f86b3
AB		 3577
3578 spin_unlock_irqrestore(&skdev->lock, flags);
3579}
3580
3581static void skd_stop_device(struct skd_device *skdev)
3582{
3583 unsigned long flags;
3584 struct skd_special_context *skspcl = &skdev->internal_skspcl;
3585 u32 dev_state;
3586 int i;
3587
3588 spin_lock_irqsave(&skdev->lock, flags);
3589
3590 if (skdev->state != SKD_DRVR_STATE_ONLINE) {
3591 pr_err("(%s): skd_stop_device not online no sync\n",
3592 skd_name(skdev));
3593 goto stop_out;
3594 }
3595
3596 if (skspcl->req.state != SKD_REQ_STATE_IDLE) {
3597 pr_err("(%s): skd_stop_device no special\n",
3598 skd_name(skdev));
3599 goto stop_out;
3600 }
3601
3602 skdev->state = SKD_DRVR_STATE_SYNCING;
3603 skdev->sync_done = 0;
3604
3605 skd_send_internal_skspcl(skdev, skspcl, SYNCHRONIZE_CACHE);
3606
3607 spin_unlock_irqrestore(&skdev->lock, flags);
3608
3609 wait_event_interruptible_timeout(skdev->waitq,
3610 (skdev->sync_done), (10 * HZ));
3611
3612 spin_lock_irqsave(&skdev->lock, flags);
3613
3614 switch (skdev->sync_done) {
3615 case 0:
3616 pr_err("(%s): skd_stop_device no sync\n",
3617 skd_name(skdev));
3618 break;
3619 case 1:
3620 pr_err("(%s): skd_stop_device sync done\n",
3621 skd_name(skdev));
3622 break;
3623 default:
3624 pr_err("(%s): skd_stop_device sync error\n",
3625 skd_name(skdev));
3626 }
3627
3628stop_out:
3629 skdev->state = SKD_DRVR_STATE_STOPPING;
3630 spin_unlock_irqrestore(&skdev->lock, flags);
3631
3632 skd_kill_timer(skdev);
3633
3634 spin_lock_irqsave(&skdev->lock, flags);
3635 skd_disable_interrupts(skdev);
3636
3637 /* ensure all ints on device are cleared */
3638 /* soft reset the device to unload with a clean slate */
3639 SKD_WRITEL(skdev, FIT_INT_DEF_MASK, FIT_INT_STATUS_HOST);
3640 SKD_WRITEL(skdev, FIT_CR_SOFT_RESET, FIT_CONTROL);
3641
3642 spin_unlock_irqrestore(&skdev->lock, flags);
3643
3644 /* poll every 100ms, 1 second timeout */
3645 for (i = 0; i < 10; i++) {
3646 dev_state =
3647 SKD_READL(skdev, FIT_STATUS) & FIT_SR_DRIVE_STATE_MASK;
3648 if (dev_state == FIT_SR_DRIVE_INIT)
3649 break;
3650 set_current_state(TASK_INTERRUPTIBLE);
3651 schedule_timeout(msecs_to_jiffies(100));
3652 }
3653
3654 if (dev_state != FIT_SR_DRIVE_INIT)
3655 pr_err("(%s): skd_stop_device state error 0x%02x\n",
3656 skd_name(skdev), dev_state);
3657}
3658
3659/* assume spinlock is held */
3660static void skd_restart_device(struct skd_device *skdev)
3661{
3662 u32 state;
3663
3664 /* ack all ghost interrupts */
3665 SKD_WRITEL(skdev, FIT_INT_DEF_MASK, FIT_INT_STATUS_HOST);
3666
3667 state = SKD_READL(skdev, FIT_STATUS);
3668
2e44b427 3669 pr_debug("%s:%s:%d drive status=0x%x\n",
3670 skdev->name, __func__, __LINE__, state);
e67f86b3
AB		 3671
3672 state &= FIT_SR_DRIVE_STATE_MASK;
3673 skdev->drive_state = state;
3674 skdev->last_mtd = 0;
3675
3676 skdev->state = SKD_DRVR_STATE_RESTARTING;
3677 skdev->timer_countdown = SKD_RESTARTING_TIMO;
3678
3679 skd_soft_reset(skdev);
3680}
3681
3682/* assume spinlock is held */
3683static int skd_quiesce_dev(struct skd_device *skdev)
3684{
3685 int rc = 0;
3686
3687 switch (skdev->state) {
3688 case SKD_DRVR_STATE_BUSY:
3689 case SKD_DRVR_STATE_BUSY_IMMINENT:
2e44b427 3690 pr_debug("%s:%s:%d stopping %s queue\n",
3691 skdev->name, __func__, __LINE__, skdev->name);
6a5ec65b 3692 blk_stop_queue(skdev->queue);
e67f86b3
AB		 3693 break;
3694 case SKD_DRVR_STATE_ONLINE:
3695 case SKD_DRVR_STATE_STOPPING:
3696 case SKD_DRVR_STATE_SYNCING:
3697 case SKD_DRVR_STATE_PAUSING:
3698 case SKD_DRVR_STATE_PAUSED:
3699 case SKD_DRVR_STATE_STARTING:
3700 case SKD_DRVR_STATE_RESTARTING:
3701 case SKD_DRVR_STATE_RESUMING:
3702 default:
3703 rc = -EINVAL;
2e44b427 3704 pr_debug("%s:%s:%d state [%d] not implemented\n",
3705 skdev->name, __func__, __LINE__, skdev->state);
e67f86b3
AB		 3706 }
3707 return rc;
3708}
3709
3710/* assume spinlock is held */
3711static int skd_unquiesce_dev(struct skd_device *skdev)
3712{
3713 int prev_driver_state = skdev->state;
3714
3715 skd_log_skdev(skdev, "unquiesce");
3716 if (skdev->state == SKD_DRVR_STATE_ONLINE) {
2e44b427 3717 pr_debug("%s:%s:%d **** device already ONLINE\n",
3718 skdev->name, __func__, __LINE__);
e67f86b3
AB		 3719 return 0;
3720 }
3721 if (skdev->drive_state != FIT_SR_DRIVE_ONLINE) {
3722 /*
3723 * If there has been an state change to other than
3724 * ONLINE, we will rely on controller state change
3725 * to come back online and restart the queue.
3726 * The BUSY state means that driver is ready to
3727 * continue normal processing but waiting for controller
3728 * to become available.
3729 */
3730 skdev->state = SKD_DRVR_STATE_BUSY;
2e44b427 3731 pr_debug("%s:%s:%d drive BUSY state\n",
3732 skdev->name, __func__, __LINE__);
e67f86b3
AB		 3733 return 0;
3734 }
3735
3736 /*
3737 * Drive has just come online, driver is either in startup,
3738 * paused performing a task, or bust waiting for hardware.
3739 */
3740 switch (skdev->state) {
3741 case SKD_DRVR_STATE_PAUSED:
3742 case SKD_DRVR_STATE_BUSY:
3743 case SKD_DRVR_STATE_BUSY_IMMINENT:
3744 case SKD_DRVR_STATE_BUSY_ERASE:
3745 case SKD_DRVR_STATE_STARTING:
3746 case SKD_DRVR_STATE_RESTARTING:
3747 case SKD_DRVR_STATE_FAULT:
3748 case SKD_DRVR_STATE_IDLE:
3749 case SKD_DRVR_STATE_LOAD:
3750 skdev->state = SKD_DRVR_STATE_ONLINE;
3751 pr_err("(%s): Driver state %s(%d)=>%s(%d)\n",
3752 skd_name(skdev),
3753 skd_skdev_state_to_str(prev_driver_state),
3754 prev_driver_state, skd_skdev_state_to_str(skdev->state),
3755 skdev->state);
2e44b427 3756 pr_debug("%s:%s:%d **** device ONLINE...starting block queue\n",
3757 skdev->name, __func__, __LINE__);
3758 pr_debug("%s:%s:%d starting %s queue\n",
3759 skdev->name, __func__, __LINE__, skdev->name);
e67f86b3 3760 pr_info("(%s): STEC s1120 ONLINE\n", skd_name(skdev));
6a5ec65b 3761 blk_start_queue(skdev->queue);
e67f86b3
AB		 3762 skdev->gendisk_on = 1;
3763 wake_up_interruptible(&skdev->waitq);
3764 break;
3765
3766 case SKD_DRVR_STATE_DISAPPEARED:
3767 default:
2e44b427 3768 pr_debug("%s:%s:%d **** driver state %d, not implemented \n",
3769 skdev->name, __func__, __LINE__,
3770 skdev->state);
e67f86b3
AB		 3771 return -EBUSY;
3772 }
3773 return 0;
3774}
3775
3776/*
3777 *****************************************************************************
3778 * PCIe MSI/MSI-X INTERRUPT HANDLERS
3779 *****************************************************************************
3780 */
3781
3782static irqreturn_t skd_reserved_isr(int irq, void *skd_host_data)
3783{
3784 struct skd_device *skdev = skd_host_data;
3785 unsigned long flags;
3786
3787 spin_lock_irqsave(&skdev->lock, flags);
2e44b427 3788 pr_debug("%s:%s:%d MSIX = 0x%x\n",
3789 skdev->name, __func__, __LINE__,
3790 SKD_READL(skdev, FIT_INT_STATUS_HOST));
e67f86b3
AB		 3791 pr_err("(%s): MSIX reserved irq %d = 0x%x\n", skd_name(skdev),
3792 irq, SKD_READL(skdev, FIT_INT_STATUS_HOST));
3793 SKD_WRITEL(skdev, FIT_INT_RESERVED_MASK, FIT_INT_STATUS_HOST);
3794 spin_unlock_irqrestore(&skdev->lock, flags);
3795 return IRQ_HANDLED;
3796}
3797
3798static irqreturn_t skd_statec_isr(int irq, void *skd_host_data)
3799{
3800 struct skd_device *skdev = skd_host_data;
3801 unsigned long flags;
3802
3803 spin_lock_irqsave(&skdev->lock, flags);
2e44b427 3804 pr_debug("%s:%s:%d MSIX = 0x%x\n",
3805 skdev->name, __func__, __LINE__,
3806 SKD_READL(skdev, FIT_INT_STATUS_HOST));
e67f86b3
AB		 3807 SKD_WRITEL(skdev, FIT_ISH_FW_STATE_CHANGE, FIT_INT_STATUS_HOST);
3808 skd_isr_fwstate(skdev);
3809 spin_unlock_irqrestore(&skdev->lock, flags);
3810 return IRQ_HANDLED;
3811}
3812
3813static irqreturn_t skd_comp_q(int irq, void *skd_host_data)
3814{
3815 struct skd_device *skdev = skd_host_data;
3816 unsigned long flags;
3817 int flush_enqueued = 0;
3818 int deferred;
3819
3820 spin_lock_irqsave(&skdev->lock, flags);
2e44b427 3821 pr_debug("%s:%s:%d MSIX = 0x%x\n",
3822 skdev->name, __func__, __LINE__,
3823 SKD_READL(skdev, FIT_INT_STATUS_HOST));
e67f86b3
AB		 3824 SKD_WRITEL(skdev, FIT_ISH_COMPLETION_POSTED, FIT_INT_STATUS_HOST);
3825 deferred = skd_isr_completion_posted(skdev, skd_isr_comp_limit,
3826 &flush_enqueued);
e67f86b3
AB		 3827 if (flush_enqueued)
3828 skd_request_fn(skdev->queue);
3829
3830 if (deferred)
3831 schedule_work(&skdev->completion_worker);
3832 else if (!flush_enqueued)
3833 skd_request_fn(skdev->queue);
3834
3835 spin_unlock_irqrestore(&skdev->lock, flags);
3836
3837 return IRQ_HANDLED;
3838}
3839
3840static irqreturn_t skd_msg_isr(int irq, void *skd_host_data)
3841{
3842 struct skd_device *skdev = skd_host_data;
3843 unsigned long flags;
3844
3845 spin_lock_irqsave(&skdev->lock, flags);
2e44b427 3846 pr_debug("%s:%s:%d MSIX = 0x%x\n",
3847 skdev->name, __func__, __LINE__,
3848 SKD_READL(skdev, FIT_INT_STATUS_HOST));
e67f86b3
AB		 3849 SKD_WRITEL(skdev, FIT_ISH_MSG_FROM_DEV, FIT_INT_STATUS_HOST);
3850 skd_isr_msg_from_dev(skdev);
3851 spin_unlock_irqrestore(&skdev->lock, flags);
3852 return IRQ_HANDLED;
3853}
3854
3855static irqreturn_t skd_qfull_isr(int irq, void *skd_host_data)
3856{
3857 struct skd_device *skdev = skd_host_data;
3858 unsigned long flags;
3859
3860 spin_lock_irqsave(&skdev->lock, flags);
2e44b427 3861 pr_debug("%s:%s:%d MSIX = 0x%x\n",
3862 skdev->name, __func__, __LINE__,
3863 SKD_READL(skdev, FIT_INT_STATUS_HOST));
e67f86b3
AB		 3864 SKD_WRITEL(skdev, FIT_INT_QUEUE_FULL, FIT_INT_STATUS_HOST);
3865 spin_unlock_irqrestore(&skdev->lock, flags);
3866 return IRQ_HANDLED;
3867}
3868
3869/*
3870 *****************************************************************************
3871 * PCIe MSI/MSI-X SETUP
3872 *****************************************************************************
3873 */
3874
3875struct skd_msix_entry {
3876 int have_irq;
3877 u32 vector;
3878 u32 entry;
3879 struct skd_device *rsp;
3880 char isr_name[30];
3881};
3882
3883struct skd_init_msix_entry {
3884 const char *name;
3885 irq_handler_t handler;
3886};
3887
3888#define SKD_MAX_MSIX_COUNT 13
3889#define SKD_MIN_MSIX_COUNT 7
3890#define SKD_BASE_MSIX_IRQ 4
3891
3892static struct skd_init_msix_entry msix_entries[SKD_MAX_MSIX_COUNT] = {
3893 { "(DMA 0)", skd_reserved_isr },
3894 { "(DMA 1)", skd_reserved_isr },
3895 { "(DMA 2)", skd_reserved_isr },
3896 { "(DMA 3)", skd_reserved_isr },
3897 { "(State Change)", skd_statec_isr },
3898 { "(COMPL_Q)", skd_comp_q },
3899 { "(MSG)", skd_msg_isr },
3900 { "(Reserved)", skd_reserved_isr },
3901 { "(Reserved)", skd_reserved_isr },
3902 { "(Queue Full 0)", skd_qfull_isr },
3903 { "(Queue Full 1)", skd_qfull_isr },
3904 { "(Queue Full 2)", skd_qfull_isr },
3905 { "(Queue Full 3)", skd_qfull_isr },
3906};
3907
3908static void skd_release_msix(struct skd_device *skdev)
3909{
3910 struct skd_msix_entry *qentry;
3911 int i;
3912
46817769
AG		 3913 if (skdev->msix_entries) {
3914 for (i = 0; i < skdev->msix_count; i++) {
3915 qentry = &skdev->msix_entries[i];
3916 skdev = qentry->rsp;
3917
3918 if (qentry->have_irq)
3919 devm_free_irq(&skdev->pdev->dev,
3920 qentry->vector, qentry->rsp);
3921 }
e67f86b3 3922
46817769 3923 kfree(skdev->msix_entries);
e67f86b3 3924 }
46817769
AG		 3925
3926 if (skdev->msix_count)
3927 pci_disable_msix(skdev->pdev);
3928
e67f86b3
AB		 3929 skdev->msix_count = 0;
3930 skdev->msix_entries = NULL;
3931}
3932
3933static int skd_acquire_msix(struct skd_device *skdev)
3934{
a9df8625 3935 int i, rc;
46817769
AG		 3936 struct pci_dev *pdev = skdev->pdev;
3937 struct msix_entry *entries;
e67f86b3
AB		 3938 struct skd_msix_entry *qentry;
3939
a9df8625
AG		 3940 entries = kzalloc(sizeof(struct msix_entry) * SKD_MAX_MSIX_COUNT,
3941 GFP_KERNEL);
e67f86b3
AB		 3942 if (!entries)
3943 return -ENOMEM;
3944
a9df8625 3945 for (i = 0; i < SKD_MAX_MSIX_COUNT; i++)
e67f86b3
AB		 3946 entries[i].entry = i;
3947
a9df8625
AG		 3948 rc = pci_enable_msix_range(pdev, entries,
3949 SKD_MIN_MSIX_COUNT, SKD_MAX_MSIX_COUNT);
3950 if (rc < 0) {
3951 pr_err("(%s): failed to enable MSI-X %d\n",
3952 skd_name(skdev), rc);
e67f86b3 3953 goto msix_out;
e67f86b3 3954 }
46817769 3955
a9df8625 3956 skdev->msix_count = rc;
e67f86b3
AB		 3957 skdev->msix_entries = kzalloc(sizeof(struct skd_msix_entry) *
3958 skdev->msix_count, GFP_KERNEL);
3959 if (!skdev->msix_entries) {
3960 rc = -ENOMEM;
e67f86b3
AB		 3961 pr_err("(%s): msix table allocation error\n",
3962 skd_name(skdev));
3963 goto msix_out;
3964 }
3965
e67f86b3 3966 for (i = 0; i < skdev->msix_count; i++) {
1bc5ce5d 3967 qentry = &skdev->msix_entries[i];
e67f86b3
AB		 3968 qentry->vector = entries[i].vector;
3969 qentry->entry = entries[i].entry;
3970 qentry->rsp = NULL;
3971 qentry->have_irq = 0;
2e44b427 3972 pr_debug("%s:%s:%d %s: <%s> msix (%d) vec %d, entry %x\n",
3973 skdev->name, __func__, __LINE__,
3974 pci_name(pdev), skdev->name,
3975 i, qentry->vector, qentry->entry);
e67f86b3
AB		 3976 }
3977
3978 /* Enable MSI-X vectors for the base queue */
c5e3035c 3979 for (i = 0; i < skdev->msix_count; i++) {
e67f86b3
AB		 3980 qentry = &skdev->msix_entries[i];
3981 snprintf(qentry->isr_name, sizeof(qentry->isr_name),
3982 "%s%d-msix %s", DRV_NAME, skdev->devno,
3983 msix_entries[i].name);
3984 rc = devm_request_irq(&skdev->pdev->dev, qentry->vector,
3985 msix_entries[i].handler, 0,
3986 qentry->isr_name, skdev);
3987 if (rc) {
3988 pr_err("(%s): Unable to register(%d) MSI-X "
3989 "handler %d: %s\n",
3990 skd_name(skdev), rc, i, qentry->isr_name);
3991 goto msix_out;
3992 } else {
3993 qentry->have_irq = 1;
3994 qentry->rsp = skdev;
3995 }
3996 }
2e44b427 3997 pr_debug("%s:%s:%d %s: <%s> msix %d irq(s) enabled\n",
3998 skdev->name, __func__, __LINE__,
3999 pci_name(pdev), skdev->name, skdev->msix_count);
e67f86b3
AB		 4000 return 0;
4001
4002msix_out:
4003 if (entries)
4004 kfree(entries);
4005 skd_release_msix(skdev);
4006 return rc;
4007}
4008
4009static int skd_acquire_irq(struct skd_device *skdev)
4010{
4011 int rc;
4012 struct pci_dev *pdev;
4013
4014 pdev = skdev->pdev;
4015 skdev->msix_count = 0;
4016
4017RETRY_IRQ_TYPE:
4018 switch (skdev->irq_type) {
4019 case SKD_IRQ_MSIX:
4020 rc = skd_acquire_msix(skdev);
4021 if (!rc)
4022 pr_info("(%s): MSI-X %d irqs enabled\n",
4023 skd_name(skdev), skdev->msix_count);
4024 else {
4025 pr_err(
4026 "(%s): failed to enable MSI-X, re-trying with MSI %d\n",
4027 skd_name(skdev), rc);
4028 skdev->irq_type = SKD_IRQ_MSI;
4029 goto RETRY_IRQ_TYPE;
4030 }
4031 break;
4032 case SKD_IRQ_MSI:
4033 snprintf(skdev->isr_name, sizeof(skdev->isr_name), "%s%d-msi",
4034 DRV_NAME, skdev->devno);
a9df8625
AG		 4035 rc = pci_enable_msi_range(pdev, 1, 1);
4036 if (rc > 0) {
e67f86b3
AB		 4037 rc = devm_request_irq(&pdev->dev, pdev->irq, skd_isr, 0,
4038 skdev->isr_name, skdev);
4039 if (rc) {
4040 pci_disable_msi(pdev);
4041 pr_err(
4042 "(%s): failed to allocate the MSI interrupt %d\n",
4043 skd_name(skdev), rc);
4044 goto RETRY_IRQ_LEGACY;
4045 }
4046 pr_info("(%s): MSI irq %d enabled\n",
4047 skd_name(skdev), pdev->irq);
4048 } else {
4049RETRY_IRQ_LEGACY:
4050 pr_err(
4051 "(%s): failed to enable MSI, re-trying with LEGACY %d\n",
4052 skd_name(skdev), rc);
4053 skdev->irq_type = SKD_IRQ_LEGACY;
4054 goto RETRY_IRQ_TYPE;
4055 }
4056 break;
4057 case SKD_IRQ_LEGACY:
4058 snprintf(skdev->isr_name, sizeof(skdev->isr_name),
4059 "%s%d-legacy", DRV_NAME, skdev->devno);
4060 rc = devm_request_irq(&pdev->dev, pdev->irq, skd_isr,
4061 IRQF_SHARED, skdev->isr_name, skdev);
4062 if (!rc)
4063 pr_info("(%s): LEGACY irq %d enabled\n",
4064 skd_name(skdev), pdev->irq);
4065 else
4066 pr_err("(%s): request LEGACY irq error %d\n",
4067 skd_name(skdev), rc);
4068 break;
4069 default:
4070 pr_info("(%s): irq_type %d invalid, re-set to %d\n",
4071 skd_name(skdev), skdev->irq_type, SKD_IRQ_DEFAULT);
4072 skdev->irq_type = SKD_IRQ_LEGACY;
4073 goto RETRY_IRQ_TYPE;
4074 }
4075 return rc;
4076}
4077
4078static void skd_release_irq(struct skd_device *skdev)
4079{
4080 switch (skdev->irq_type) {
4081 case SKD_IRQ_MSIX:
4082 skd_release_msix(skdev);
4083 break;
4084 case SKD_IRQ_MSI:
4085 devm_free_irq(&skdev->pdev->dev, skdev->pdev->irq, skdev);
4086 pci_disable_msi(skdev->pdev);
4087 break;
4088 case SKD_IRQ_LEGACY:
4089 devm_free_irq(&skdev->pdev->dev, skdev->pdev->irq, skdev);
4090 break;
4091 default:
4092 pr_err("(%s): wrong irq type %d!",
4093 skd_name(skdev), skdev->irq_type);
4094 break;
4095 }
4096}
4097
4098/*
4099 *****************************************************************************
4100 * CONSTRUCT
4101 *****************************************************************************
4102 */
4103
e67f86b3
AB		 4104static int skd_cons_skcomp(struct skd_device *skdev)
4105{
4106 int rc = 0;
4107 struct fit_completion_entry_v1 *skcomp;
4108 u32 nbytes;
4109
4110 nbytes = sizeof(*skcomp) * SKD_N_COMPLETION_ENTRY;
4111 nbytes += sizeof(struct fit_comp_error_info) * SKD_N_COMPLETION_ENTRY;
4112
2e44b427 4113 pr_debug("%s:%s:%d comp pci_alloc, total bytes %d entries %d\n",
4114 skdev->name, __func__, __LINE__,
4115 nbytes, SKD_N_COMPLETION_ENTRY);
e67f86b3
AB		 4116
4117 skcomp = pci_alloc_consistent(skdev->pdev, nbytes,
4118 &skdev->cq_dma_address);
4119
4120 if (skcomp == NULL) {
4121 rc = -ENOMEM;
4122 goto err_out;
4123 }
4124
4125 memset(skcomp, 0, nbytes);
4126
4127 skdev->skcomp_table = skcomp;
4128 skdev->skerr_table = (struct fit_comp_error_info *)((char *)skcomp +
4129 sizeof(*skcomp) *
4130 SKD_N_COMPLETION_ENTRY);
4131
4132err_out:
4133 return rc;
4134}
4135
4136static int skd_cons_skmsg(struct skd_device *skdev)
4137{
4138 int rc = 0;
4139 u32 i;
4140
2e44b427 4141 pr_debug("%s:%s:%d skmsg_table kzalloc, struct %lu, count %u total %lu\n",
4142 skdev->name, __func__, __LINE__,
4143 sizeof(struct skd_fitmsg_context),
4144 skdev->num_fitmsg_context,
4145 sizeof(struct skd_fitmsg_context) * skdev->num_fitmsg_context);
e67f86b3
AB		 4146
4147 skdev->skmsg_table = kzalloc(sizeof(struct skd_fitmsg_context)
4148 *skdev->num_fitmsg_context, GFP_KERNEL);
4149 if (skdev->skmsg_table == NULL) {
4150 rc = -ENOMEM;
4151 goto err_out;
4152 }
4153
4154 for (i = 0; i < skdev->num_fitmsg_context; i++) {
4155 struct skd_fitmsg_context *skmsg;
4156
4157 skmsg = &skdev->skmsg_table[i];
4158
4159 skmsg->id = i + SKD_ID_FIT_MSG;
4160
4161 skmsg->state = SKD_MSG_STATE_IDLE;
4162 skmsg->msg_buf = pci_alloc_consistent(skdev->pdev,
4163 SKD_N_FITMSG_BYTES + 64,
4164 &skmsg->mb_dma_address);
4165
4166 if (skmsg->msg_buf == NULL) {
4167 rc = -ENOMEM;
4168 goto err_out;
4169 }
4170
4171 skmsg->offset = (u32)((u64)skmsg->msg_buf &
4172 (~FIT_QCMD_BASE_ADDRESS_MASK));
4173 skmsg->msg_buf += ~FIT_QCMD_BASE_ADDRESS_MASK;
4174 skmsg->msg_buf = (u8 *)((u64)skmsg->msg_buf &
4175 FIT_QCMD_BASE_ADDRESS_MASK);
4176 skmsg->mb_dma_address += ~FIT_QCMD_BASE_ADDRESS_MASK;
4177 skmsg->mb_dma_address &= FIT_QCMD_BASE_ADDRESS_MASK;
4178 memset(skmsg->msg_buf, 0, SKD_N_FITMSG_BYTES);
4179
4180 skmsg->next = &skmsg[1];
4181 }
4182
4183 /* Free list is in order starting with the 0th entry. */
4184 skdev->skmsg_table[i - 1].next = NULL;
4185 skdev->skmsg_free_list = skdev->skmsg_table;
4186
4187err_out:
4188 return rc;
4189}
4190
542d7b00
BZ		 4191static struct fit_sg_descriptor *skd_cons_sg_list(struct skd_device *skdev,
4192 u32 n_sg,
4193 dma_addr_t *ret_dma_addr)
4194{
4195 struct fit_sg_descriptor *sg_list;
4196 u32 nbytes;
4197
4198 nbytes = sizeof(*sg_list) * n_sg;
4199
4200 sg_list = pci_alloc_consistent(skdev->pdev, nbytes, ret_dma_addr);
4201
4202 if (sg_list != NULL) {
4203 uint64_t dma_address = *ret_dma_addr;
4204 u32 i;
4205
4206 memset(sg_list, 0, nbytes);
4207
4208 for (i = 0; i < n_sg - 1; i++) {
4209 uint64_t ndp_off;
4210 ndp_off = (i + 1) * sizeof(struct fit_sg_descriptor);
4211
4212 sg_list[i].next_desc_ptr = dma_address + ndp_off;
4213 }
4214 sg_list[i].next_desc_ptr = 0LL;
4215 }
4216
4217 return sg_list;
4218}
4219
e67f86b3
AB		 4220static int skd_cons_skreq(struct skd_device *skdev)
4221{
4222 int rc = 0;
4223 u32 i;
4224
2e44b427 4225 pr_debug("%s:%s:%d skreq_table kzalloc, struct %lu, count %u total %lu\n",
4226 skdev->name, __func__, __LINE__,
4227 sizeof(struct skd_request_context),
4228 skdev->num_req_context,
4229 sizeof(struct skd_request_context) * skdev->num_req_context);
e67f86b3
AB		 4230
4231 skdev->skreq_table = kzalloc(sizeof(struct skd_request_context)
4232 * skdev->num_req_context, GFP_KERNEL);
4233 if (skdev->skreq_table == NULL) {
4234 rc = -ENOMEM;
4235 goto err_out;
4236 }
4237
2e44b427 4238 pr_debug("%s:%s:%d alloc sg_table sg_per_req %u scatlist %lu total %lu\n",
4239 skdev->name, __func__, __LINE__,
4240 skdev->sgs_per_request, sizeof(struct scatterlist),
4241 skdev->sgs_per_request * sizeof(struct scatterlist));
e67f86b3
AB		 4242
4243 for (i = 0; i < skdev->num_req_context; i++) {
4244 struct skd_request_context *skreq;
4245
4246 skreq = &skdev->skreq_table[i];
4247
4248 skreq->id = i + SKD_ID_RW_REQUEST;
4249 skreq->state = SKD_REQ_STATE_IDLE;
4250
4251 skreq->sg = kzalloc(sizeof(struct scatterlist) *
4252 skdev->sgs_per_request, GFP_KERNEL);
4253 if (skreq->sg == NULL) {
4254 rc = -ENOMEM;
4255 goto err_out;
4256 }
4257 sg_init_table(skreq->sg, skdev->sgs_per_request);
4258
4259 skreq->sksg_list = skd_cons_sg_list(skdev,
4260 skdev->sgs_per_request,
4261 &skreq->sksg_dma_address);
4262
4263 if (skreq->sksg_list == NULL) {
4264 rc = -ENOMEM;
4265 goto err_out;
4266 }
4267
4268 skreq->next = &skreq[1];
4269 }
4270
4271 /* Free list is in order starting with the 0th entry. */
4272 skdev->skreq_table[i - 1].next = NULL;
4273 skdev->skreq_free_list = skdev->skreq_table;
4274
4275err_out:
4276 return rc;
4277}
4278
4279static int skd_cons_skspcl(struct skd_device *skdev)
4280{
4281 int rc = 0;
4282 u32 i, nbytes;
4283
2e44b427 4284 pr_debug("%s:%s:%d skspcl_table kzalloc, struct %lu, count %u total %lu\n",
4285 skdev->name, __func__, __LINE__,
4286 sizeof(struct skd_special_context),
4287 skdev->n_special,
4288 sizeof(struct skd_special_context) * skdev->n_special);
e67f86b3
AB		 4289
4290 skdev->skspcl_table = kzalloc(sizeof(struct skd_special_context)
4291 * skdev->n_special, GFP_KERNEL);
4292 if (skdev->skspcl_table == NULL) {
4293 rc = -ENOMEM;
4294 goto err_out;
4295 }
4296
4297 for (i = 0; i < skdev->n_special; i++) {
4298 struct skd_special_context *skspcl;
4299
4300 skspcl = &skdev->skspcl_table[i];
4301
4302 skspcl->req.id = i + SKD_ID_SPECIAL_REQUEST;
4303 skspcl->req.state = SKD_REQ_STATE_IDLE;
4304
4305 skspcl->req.next = &skspcl[1].req;
4306
4307 nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
4308
4309 skspcl->msg_buf = pci_alloc_consistent(skdev->pdev, nbytes,
4310 &skspcl->mb_dma_address);
4311 if (skspcl->msg_buf == NULL) {
4312 rc = -ENOMEM;
4313 goto err_out;
4314 }
4315
4316 memset(skspcl->msg_buf, 0, nbytes);
4317
4318 skspcl->req.sg = kzalloc(sizeof(struct scatterlist) *
4319 SKD_N_SG_PER_SPECIAL, GFP_KERNEL);
4320 if (skspcl->req.sg == NULL) {
4321 rc = -ENOMEM;
4322 goto err_out;
4323 }
4324
4325 skspcl->req.sksg_list = skd_cons_sg_list(skdev,
4326 SKD_N_SG_PER_SPECIAL,
4327 &skspcl->req.
4328 sksg_dma_address);
4329 if (skspcl->req.sksg_list == NULL) {
4330 rc = -ENOMEM;
4331 goto err_out;
4332 }
4333 }
4334
4335 /* Free list is in order starting with the 0th entry. */
4336 skdev->skspcl_table[i - 1].req.next = NULL;
4337 skdev->skspcl_free_list = skdev->skspcl_table;
4338
4339 return rc;
4340
4341err_out:
4342 return rc;
4343}
4344
4345static int skd_cons_sksb(struct skd_device *skdev)
4346{
4347 int rc = 0;
4348 struct skd_special_context *skspcl;
4349 u32 nbytes;
4350
4351 skspcl = &skdev->internal_skspcl;
4352
4353 skspcl->req.id = 0 + SKD_ID_INTERNAL;
4354 skspcl->req.state = SKD_REQ_STATE_IDLE;
4355
4356 nbytes = SKD_N_INTERNAL_BYTES;
4357
4358 skspcl->data_buf = pci_alloc_consistent(skdev->pdev, nbytes,
4359 &skspcl->db_dma_address);
4360 if (skspcl->data_buf == NULL) {
4361 rc = -ENOMEM;
4362 goto err_out;
4363 }
4364
4365 memset(skspcl->data_buf, 0, nbytes);
4366
4367 nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
4368 skspcl->msg_buf = pci_alloc_consistent(skdev->pdev, nbytes,
4369 &skspcl->mb_dma_address);
4370 if (skspcl->msg_buf == NULL) {
4371 rc = -ENOMEM;
4372 goto err_out;
4373 }
4374
4375 memset(skspcl->msg_buf, 0, nbytes);
4376
4377 skspcl->req.sksg_list = skd_cons_sg_list(skdev, 1,
4378 &skspcl->req.sksg_dma_address);
4379 if (skspcl->req.sksg_list == NULL) {
4380 rc = -ENOMEM;
4381 goto err_out;
4382 }
4383
4384 if (!skd_format_internal_skspcl(skdev)) {
4385 rc = -EINVAL;
4386 goto err_out;
4387 }
4388
4389err_out:
4390 return rc;
4391}
4392
e67f86b3
AB		 4393static int skd_cons_disk(struct skd_device *skdev)
4394{
4395 int rc = 0;
4396 struct gendisk *disk;
4397 struct request_queue *q;
4398 unsigned long flags;
4399
4400 disk = alloc_disk(SKD_MINORS_PER_DEVICE);
4401 if (!disk) {
4402 rc = -ENOMEM;
4403 goto err_out;
4404 }
4405
4406 skdev->disk = disk;
4407 sprintf(disk->disk_name, DRV_NAME "%u", skdev->devno);
4408
4409 disk->major = skdev->major;
4410 disk->first_minor = skdev->devno * SKD_MINORS_PER_DEVICE;
4411 disk->fops = &skd_blockdev_ops;
4412 disk->private_data = skdev;
4413
fcd37eb3 4414 q = blk_init_queue(skd_request_fn, &skdev->lock);
e67f86b3
AB		 4415 if (!q) {
4416 rc = -ENOMEM;
4417 goto err_out;
4418 }
4419
4420 skdev->queue = q;
4421 disk->queue = q;
4422 q->queuedata = skdev;
4423
e67f86b3
AB		 4424 blk_queue_flush(q, REQ_FLUSH | REQ_FUA);
4425 blk_queue_max_segments(q, skdev->sgs_per_request);
4426 blk_queue_max_hw_sectors(q, SKD_N_MAX_SECTORS);
4427
4428 /* set sysfs ptimal_io_size to 8K */
4429 blk_queue_io_opt(q, 8192);
4430
4431 /* DISCARD Flag initialization. */
4432 q->limits.discard_granularity = 8192;
4433 q->limits.discard_alignment = 0;
4434 q->limits.max_discard_sectors = UINT_MAX >> 9;
4435 q->limits.discard_zeroes_data = 1;
4436 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q);
4437 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, q);
4438
4439 spin_lock_irqsave(&skdev->lock, flags);
2e44b427 4440 pr_debug("%s:%s:%d stopping %s queue\n",
4441 skdev->name, __func__, __LINE__, skdev->name);
6a5ec65b 4442 blk_stop_queue(skdev->queue);
e67f86b3
AB		 4443 spin_unlock_irqrestore(&skdev->lock, flags);
4444
4445err_out:
4446 return rc;
4447}
4448
542d7b00
BZ		 4449#define SKD_N_DEV_TABLE 16u
4450static u32 skd_next_devno;
e67f86b3 4451
542d7b00 4452static struct skd_device *skd_construct(struct pci_dev *pdev)
e67f86b3 4453{
542d7b00
BZ		 4454 struct skd_device *skdev;
4455 int blk_major = skd_major;
4456 int rc;
e67f86b3 4457
542d7b00 4458 skdev = kzalloc(sizeof(*skdev), GFP_KERNEL);
e67f86b3 4459
542d7b00
BZ		 4460 if (!skdev) {
4461 pr_err(PFX "(%s): memory alloc failure\n",
4462 pci_name(pdev));
4463 return NULL;
4464 }
e67f86b3 4465
542d7b00
BZ		 4466 skdev->state = SKD_DRVR_STATE_LOAD;
4467 skdev->pdev = pdev;
4468 skdev->devno = skd_next_devno++;
4469 skdev->major = blk_major;
4470 skdev->irq_type = skd_isr_type;
4471 sprintf(skdev->name, DRV_NAME "%d", skdev->devno);
4472 skdev->dev_max_queue_depth = 0;
e67f86b3 4473
542d7b00
BZ		 4474 skdev->num_req_context = skd_max_queue_depth;
4475 skdev->num_fitmsg_context = skd_max_queue_depth;
4476 skdev->n_special = skd_max_pass_thru;
4477 skdev->cur_max_queue_depth = 1;
4478 skdev->queue_low_water_mark = 1;
4479 skdev->proto_ver = 99;
4480 skdev->sgs_per_request = skd_sgs_per_request;
4481 skdev->dbg_level = skd_dbg_level;
e67f86b3 4482
542d7b00 4483 atomic_set(&skdev->device_count, 0);
e67f86b3 4484
542d7b00
BZ		 4485 spin_lock_init(&skdev->lock);
4486
4487 INIT_WORK(&skdev->completion_worker, skd_completion_worker);
e67f86b3 4488
2e44b427 4489 pr_debug("%s:%s:%d skcomp\n", skdev->name, __func__, __LINE__);
542d7b00
BZ		 4490 rc = skd_cons_skcomp(skdev);
4491 if (rc < 0)
4492 goto err_out;
e67f86b3 4493
542d7b00
BZ		 4494 pr_debug("%s:%s:%d skmsg\n", skdev->name, __func__, __LINE__);
4495 rc = skd_cons_skmsg(skdev);
4496 if (rc < 0)
4497 goto err_out;
4498
4499 pr_debug("%s:%s:%d skreq\n", skdev->name, __func__, __LINE__);
4500 rc = skd_cons_skreq(skdev);
4501 if (rc < 0)
4502 goto err_out;
4503
4504 pr_debug("%s:%s:%d skspcl\n", skdev->name, __func__, __LINE__);
4505 rc = skd_cons_skspcl(skdev);
4506 if (rc < 0)
4507 goto err_out;
4508
4509 pr_debug("%s:%s:%d sksb\n", skdev->name, __func__, __LINE__);
4510 rc = skd_cons_sksb(skdev);
4511 if (rc < 0)
4512 goto err_out;
4513
4514 pr_debug("%s:%s:%d disk\n", skdev->name, __func__, __LINE__);
4515 rc = skd_cons_disk(skdev);
4516 if (rc < 0)
4517 goto err_out;
4518
4519 pr_debug("%s:%s:%d VICTORY\n", skdev->name, __func__, __LINE__);
4520 return skdev;
4521
4522err_out:
4523 pr_debug("%s:%s:%d construct failed\n",
4524 skdev->name, __func__, __LINE__);
4525 skd_destruct(skdev);
4526 return NULL;
e67f86b3
AB		 4527}
4528
542d7b00
BZ		 4529/*
4530 *****************************************************************************
4531 * DESTRUCT (FREE)
4532 *****************************************************************************
4533 */
4534
e67f86b3
AB		 4535static void skd_free_skcomp(struct skd_device *skdev)
4536{
4537 if (skdev->skcomp_table != NULL) {
4538 u32 nbytes;
4539
4540 nbytes = sizeof(skdev->skcomp_table[0]) *
4541 SKD_N_COMPLETION_ENTRY;
4542 pci_free_consistent(skdev->pdev, nbytes,
4543 skdev->skcomp_table, skdev->cq_dma_address);
4544 }
4545
4546 skdev->skcomp_table = NULL;
4547 skdev->cq_dma_address = 0;
4548}
4549
4550static void skd_free_skmsg(struct skd_device *skdev)
4551{
4552 u32 i;
4553
4554 if (skdev->skmsg_table == NULL)
4555 return;
4556
4557 for (i = 0; i < skdev->num_fitmsg_context; i++) {
4558 struct skd_fitmsg_context *skmsg;
4559
4560 skmsg = &skdev->skmsg_table[i];
4561
4562 if (skmsg->msg_buf != NULL) {
4563 skmsg->msg_buf += skmsg->offset;
4564 skmsg->mb_dma_address += skmsg->offset;
4565 pci_free_consistent(skdev->pdev, SKD_N_FITMSG_BYTES,
4566 skmsg->msg_buf,
4567 skmsg->mb_dma_address);
4568 }
4569 skmsg->msg_buf = NULL;
4570 skmsg->mb_dma_address = 0;
4571 }
4572
4573 kfree(skdev->skmsg_table);
4574 skdev->skmsg_table = NULL;
4575}
4576
542d7b00
BZ		 4577static void skd_free_sg_list(struct skd_device *skdev,
4578 struct fit_sg_descriptor *sg_list,
4579 u32 n_sg, dma_addr_t dma_addr)
4580{
4581 if (sg_list != NULL) {
4582 u32 nbytes;
4583
4584 nbytes = sizeof(*sg_list) * n_sg;
4585
4586 pci_free_consistent(skdev->pdev, nbytes, sg_list, dma_addr);
4587 }
4588}
4589
e67f86b3
AB		 4590static void skd_free_skreq(struct skd_device *skdev)
4591{
4592 u32 i;
4593
4594 if (skdev->skreq_table == NULL)
4595 return;
4596
4597 for (i = 0; i < skdev->num_req_context; i++) {
4598 struct skd_request_context *skreq;
4599
4600 skreq = &skdev->skreq_table[i];
4601
4602 skd_free_sg_list(skdev, skreq->sksg_list,
4603 skdev->sgs_per_request,
4604 skreq->sksg_dma_address);
4605
4606 skreq->sksg_list = NULL;
4607 skreq->sksg_dma_address = 0;
4608
4609 kfree(skreq->sg);
4610 }
4611
4612 kfree(skdev->skreq_table);
4613 skdev->skreq_table = NULL;
4614}
4615
4616static void skd_free_skspcl(struct skd_device *skdev)
4617{
4618 u32 i;
4619 u32 nbytes;
4620
4621 if (skdev->skspcl_table == NULL)
4622 return;
4623
4624 for (i = 0; i < skdev->n_special; i++) {
4625 struct skd_special_context *skspcl;
4626
4627 skspcl = &skdev->skspcl_table[i];
4628
4629 if (skspcl->msg_buf != NULL) {
4630 nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
4631 pci_free_consistent(skdev->pdev, nbytes,
4632 skspcl->msg_buf,
4633 skspcl->mb_dma_address);
4634 }
4635
4636 skspcl->msg_buf = NULL;
4637 skspcl->mb_dma_address = 0;
4638
4639 skd_free_sg_list(skdev, skspcl->req.sksg_list,
4640 SKD_N_SG_PER_SPECIAL,
4641 skspcl->req.sksg_dma_address);
4642
4643 skspcl->req.sksg_list = NULL;
4644 skspcl->req.sksg_dma_address = 0;
4645
4646 kfree(skspcl->req.sg);
4647 }
4648
4649 kfree(skdev->skspcl_table);
4650 skdev->skspcl_table = NULL;
4651}
4652
4653static void skd_free_sksb(struct skd_device *skdev)
4654{
4655 struct skd_special_context *skspcl;
4656 u32 nbytes;
4657
4658 skspcl = &skdev->internal_skspcl;
4659
4660 if (skspcl->data_buf != NULL) {
4661 nbytes = SKD_N_INTERNAL_BYTES;
4662
4663 pci_free_consistent(skdev->pdev, nbytes,
4664 skspcl->data_buf, skspcl->db_dma_address);
4665 }
4666
4667 skspcl->data_buf = NULL;
4668 skspcl->db_dma_address = 0;
4669
4670 if (skspcl->msg_buf != NULL) {
4671 nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
4672 pci_free_consistent(skdev->pdev, nbytes,
4673 skspcl->msg_buf, skspcl->mb_dma_address);
4674 }
4675
4676 skspcl->msg_buf = NULL;
4677 skspcl->mb_dma_address = 0;
4678
4679 skd_free_sg_list(skdev, skspcl->req.sksg_list, 1,
4680 skspcl->req.sksg_dma_address);
4681
4682 skspcl->req.sksg_list = NULL;
4683 skspcl->req.sksg_dma_address = 0;
4684}
4685
e67f86b3
AB		 4686static void skd_free_disk(struct skd_device *skdev)
4687{
4688 struct gendisk *disk = skdev->disk;
4689
4690 if (disk != NULL) {
4691 struct request_queue *q = disk->queue;
4692
4693 if (disk->flags & GENHD_FL_UP)
4694 del_gendisk(disk);
4695 if (q)
4696 blk_cleanup_queue(q);
4697 put_disk(disk);
4698 }
4699 skdev->disk = NULL;
4700}
4701
542d7b00
BZ		 4702static void skd_destruct(struct skd_device *skdev)
4703{
4704 if (skdev == NULL)
4705 return;
4706
4707
4708 pr_debug("%s:%s:%d disk\n", skdev->name, __func__, __LINE__);
4709 skd_free_disk(skdev);
4710
4711 pr_debug("%s:%s:%d sksb\n", skdev->name, __func__, __LINE__);
4712 skd_free_sksb(skdev);
4713
4714 pr_debug("%s:%s:%d skspcl\n", skdev->name, __func__, __LINE__);
4715 skd_free_skspcl(skdev);
4716
4717 pr_debug("%s:%s:%d skreq\n", skdev->name, __func__, __LINE__);
4718 skd_free_skreq(skdev);
4719
4720 pr_debug("%s:%s:%d skmsg\n", skdev->name, __func__, __LINE__);
4721 skd_free_skmsg(skdev);
e67f86b3 4722
542d7b00
BZ		 4723 pr_debug("%s:%s:%d skcomp\n", skdev->name, __func__, __LINE__);
4724 skd_free_skcomp(skdev);
4725
4726 pr_debug("%s:%s:%d skdev\n", skdev->name, __func__, __LINE__);
4727 kfree(skdev);
4728}
e67f86b3
AB		 4729
4730/*
4731 *****************************************************************************
4732 * BLOCK DEVICE (BDEV) GLUE
4733 *****************************************************************************
4734 */
4735
4736static int skd_bdev_getgeo(struct block_device *bdev, struct hd_geometry *geo)
4737{
4738 struct skd_device *skdev;
4739 u64 capacity;
4740
4741 skdev = bdev->bd_disk->private_data;
4742
2e44b427 4743 pr_debug("%s:%s:%d %s: CMD[%s] getgeo device\n",
4744 skdev->name, __func__, __LINE__,
4745 bdev->bd_disk->disk_name, current->comm);
e67f86b3
AB		 4746
4747 if (skdev->read_cap_is_valid) {
4748 capacity = get_capacity(skdev->disk);
4749 geo->heads = 64;
4750 geo->sectors = 255;
4751 geo->cylinders = (capacity) / (255 * 64);
4752
4753 return 0;
4754 }
4755 return -EIO;
4756}
4757
4758static int skd_bdev_attach(struct skd_device *skdev)
4759{
2e44b427 4760 pr_debug("%s:%s:%d add_disk\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4761 add_disk(skdev->disk);
4762 return 0;
4763}
4764
4765static const struct block_device_operations skd_blockdev_ops = {
4766 .owner = THIS_MODULE,
4767 .ioctl = skd_bdev_ioctl,
4768 .getgeo = skd_bdev_getgeo,
4769};
4770
4771
4772/*
4773 *****************************************************************************
4774 * PCIe DRIVER GLUE
4775 *****************************************************************************
4776 */
4777
4778static DEFINE_PCI_DEVICE_TABLE(skd_pci_tbl) = {
4779 { PCI_VENDOR_ID_STEC, PCI_DEVICE_ID_S1120,
4780 PCI_ANY_ID, PCI_ANY_ID, 0, 0, },
4781 { 0 } /* terminate list */
4782};
4783
4784MODULE_DEVICE_TABLE(pci, skd_pci_tbl);
4785
4786static char *skd_pci_info(struct skd_device *skdev, char *str)
4787{
4788 int pcie_reg;
4789
4790 strcpy(str, "PCIe (");
4791 pcie_reg = pci_find_capability(skdev->pdev, PCI_CAP_ID_EXP);
4792
4793 if (pcie_reg) {
4794
4795 char lwstr[6];
4796 uint16_t pcie_lstat, lspeed, lwidth;
4797
4798 pcie_reg += 0x12;
4799 pci_read_config_word(skdev->pdev, pcie_reg, &pcie_lstat);
4800 lspeed = pcie_lstat & (0xF);
4801 lwidth = (pcie_lstat & 0x3F0) >> 4;
4802
4803 if (lspeed == 1)
4804 strcat(str, "2.5GT/s ");
4805 else if (lspeed == 2)
4806 strcat(str, "5.0GT/s ");
4807 else
4808 strcat(str, "<unknown> ");
4809 snprintf(lwstr, sizeof(lwstr), "%dX)", lwidth);
4810 strcat(str, lwstr);
4811 }
4812 return str;
4813}
4814
4815static int skd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
4816{
4817 int i;
4818 int rc = 0;
4819 char pci_str[32];
4820 struct skd_device *skdev;
4821
4822 pr_info("STEC s1120 Driver(%s) version %s-b%s\n",
4823 DRV_NAME, DRV_VERSION, DRV_BUILD_ID);
4824 pr_info("(skd?:??:[%s]): vendor=%04X device=%04x\n",
4825 pci_name(pdev), pdev->vendor, pdev->device);
4826
4827 rc = pci_enable_device(pdev);
4828 if (rc)
4829 return rc;
4830 rc = pci_request_regions(pdev, DRV_NAME);
4831 if (rc)
4832 goto err_out;
4833 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
4834 if (!rc) {
4835 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64))) {
4836
4837 pr_err("(%s): consistent DMA mask error %d\n",
4838 pci_name(pdev), rc);
4839 }
4840 } else {
4841 (rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)));
4842 if (rc) {
4843
4844 pr_err("(%s): DMA mask error %d\n",
4845 pci_name(pdev), rc);
4846 goto err_out_regions;
4847 }
4848 }
4849
b8df6647
BZ		 4850 if (!skd_major) {
4851 rc = register_blkdev(0, DRV_NAME);
4852 if (rc < 0)
4853 goto err_out_regions;
4854 BUG_ON(!rc);
4855 skd_major = rc;
4856 }
4857
e67f86b3 4858 skdev = skd_construct(pdev);
1762b57f
WY		 4859 if (skdev == NULL) {
4860 rc = -ENOMEM;
e67f86b3 4861 goto err_out_regions;
1762b57f 4862 }
e67f86b3
AB		 4863
4864 skd_pci_info(skdev, pci_str);
4865 pr_info("(%s): %s 64bit\n", skd_name(skdev), pci_str);
4866
4867 pci_set_master(pdev);
4868 rc = pci_enable_pcie_error_reporting(pdev);
4869 if (rc) {
4870 pr_err(
4871 "(%s): bad enable of PCIe error reporting rc=%d\n",
4872 skd_name(skdev), rc);
4873 skdev->pcie_error_reporting_is_enabled = 0;
4874 } else
4875 skdev->pcie_error_reporting_is_enabled = 1;
4876
4877
4878 pci_set_drvdata(pdev, skdev);
ebedd16d 4879
e67f86b3
AB		 4880 skdev->disk->driverfs_dev = &pdev->dev;
4881
4882 for (i = 0; i < SKD_MAX_BARS; i++) {
4883 skdev->mem_phys[i] = pci_resource_start(pdev, i);
4884 skdev->mem_size[i] = (u32)pci_resource_len(pdev, i);
4885 skdev->mem_map[i] = ioremap(skdev->mem_phys[i],
4886 skdev->mem_size[i]);
4887 if (!skdev->mem_map[i]) {
4888 pr_err("(%s): Unable to map adapter memory!\n",
4889 skd_name(skdev));
4890 rc = -ENODEV;
4891 goto err_out_iounmap;
4892 }
2e44b427 4893 pr_debug("%s:%s:%d mem_map=%p, phyd=%016llx, size=%d\n",
4894 skdev->name, __func__, __LINE__,
4895 skdev->mem_map[i],
4896 (uint64_t)skdev->mem_phys[i], skdev->mem_size[i]);
e67f86b3
AB		 4897 }
4898
4899 rc = skd_acquire_irq(skdev);
4900 if (rc) {
4901 pr_err("(%s): interrupt resource error %d\n",
4902 skd_name(skdev), rc);
4903 goto err_out_iounmap;
4904 }
4905
4906 rc = skd_start_timer(skdev);
4907 if (rc)
4908 goto err_out_timer;
4909
4910 init_waitqueue_head(&skdev->waitq);
4911
4912 skd_start_device(skdev);
4913
4914 rc = wait_event_interruptible_timeout(skdev->waitq,
4915 (skdev->gendisk_on),
4916 (SKD_START_WAIT_SECONDS * HZ));
4917 if (skdev->gendisk_on > 0) {
4918 /* device came on-line after reset */
4919 skd_bdev_attach(skdev);
4920 rc = 0;
4921 } else {
4922 /* we timed out, something is wrong with the device,
4923 don't add the disk structure */
4924 pr_err(
4925 "(%s): error: waiting for s1120 timed out %d!\n",
4926 skd_name(skdev), rc);
4927 /* in case of no error; we timeout with ENXIO */
4928 if (!rc)
4929 rc = -ENXIO;
4930 goto err_out_timer;
4931 }
4932
4933
4934#ifdef SKD_VMK_POLL_HANDLER
4935 if (skdev->irq_type == SKD_IRQ_MSIX) {
4936 /* MSIX completion handler is being used for coredump */
4937 vmklnx_scsi_register_poll_handler(skdev->scsi_host,
4938 skdev->msix_entries[5].vector,
4939 skd_comp_q, skdev);
4940 } else {
4941 vmklnx_scsi_register_poll_handler(skdev->scsi_host,
4942 skdev->pdev->irq, skd_isr,
4943 skdev);
4944 }
4945#endif /* SKD_VMK_POLL_HANDLER */
4946
4947 return rc;
4948
4949err_out_timer:
4950 skd_stop_device(skdev);
4951 skd_release_irq(skdev);
4952
4953err_out_iounmap:
4954 for (i = 0; i < SKD_MAX_BARS; i++)
4955 if (skdev->mem_map[i])
4956 iounmap(skdev->mem_map[i]);
4957
4958 if (skdev->pcie_error_reporting_is_enabled)
4959 pci_disable_pcie_error_reporting(pdev);
4960
4961 skd_destruct(skdev);
4962
4963err_out_regions:
4964 pci_release_regions(pdev);
4965
4966err_out:
4967 pci_disable_device(pdev);
4968 pci_set_drvdata(pdev, NULL);
4969 return rc;
4970}
4971
4972static void skd_pci_remove(struct pci_dev *pdev)
4973{
4974 int i;
4975 struct skd_device *skdev;
4976
4977 skdev = pci_get_drvdata(pdev);
4978 if (!skdev) {
4979 pr_err("%s: no device data for PCI\n", pci_name(pdev));
4980 return;
4981 }
4982 skd_stop_device(skdev);
4983 skd_release_irq(skdev);
4984
4985 for (i = 0; i < SKD_MAX_BARS; i++)
4986 if (skdev->mem_map[i])
4987 iounmap((u32 *)skdev->mem_map[i]);
4988
4989 if (skdev->pcie_error_reporting_is_enabled)
4990 pci_disable_pcie_error_reporting(pdev);
4991
4992 skd_destruct(skdev);
4993
4994 pci_release_regions(pdev);
4995 pci_disable_device(pdev);
4996 pci_set_drvdata(pdev, NULL);
4997
4998 return;
4999}
5000
5001static int skd_pci_suspend(struct pci_dev *pdev, pm_message_t state)
5002{
5003 int i;
5004 struct skd_device *skdev;
5005
5006 skdev = pci_get_drvdata(pdev);
5007 if (!skdev) {
5008 pr_err("%s: no device data for PCI\n", pci_name(pdev));
5009 return -EIO;
5010 }
5011
5012 skd_stop_device(skdev);
5013
5014 skd_release_irq(skdev);
5015
5016 for (i = 0; i < SKD_MAX_BARS; i++)
5017 if (skdev->mem_map[i])
5018 iounmap((u32 *)skdev->mem_map[i]);
5019
5020 if (skdev->pcie_error_reporting_is_enabled)
5021 pci_disable_pcie_error_reporting(pdev);
5022
5023 pci_release_regions(pdev);
5024 pci_save_state(pdev);
5025 pci_disable_device(pdev);
5026 pci_set_power_state(pdev, pci_choose_state(pdev, state));
5027 return 0;
5028}
5029
5030static int skd_pci_resume(struct pci_dev *pdev)
5031{
5032 int i;
5033 int rc = 0;
5034 struct skd_device *skdev;
5035
5036 skdev = pci_get_drvdata(pdev);
5037 if (!skdev) {
5038 pr_err("%s: no device data for PCI\n", pci_name(pdev));
5039 return -1;
5040 }
5041
5042 pci_set_power_state(pdev, PCI_D0);
5043 pci_enable_wake(pdev, PCI_D0, 0);
5044 pci_restore_state(pdev);
5045
5046 rc = pci_enable_device(pdev);
5047 if (rc)
5048 return rc;
5049 rc = pci_request_regions(pdev, DRV_NAME);
5050 if (rc)
5051 goto err_out;
5052 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
5053 if (!rc) {
5054 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64))) {
5055
5056 pr_err("(%s): consistent DMA mask error %d\n",
5057 pci_name(pdev), rc);
5058 }
5059 } else {
5060 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
5061 if (rc) {
5062
5063 pr_err("(%s): DMA mask error %d\n",
5064 pci_name(pdev), rc);
5065 goto err_out_regions;
5066 }
5067 }
5068
5069 pci_set_master(pdev);
5070 rc = pci_enable_pcie_error_reporting(pdev);
5071 if (rc) {
5072 pr_err("(%s): bad enable of PCIe error reporting rc=%d\n",
5073 skdev->name, rc);
5074 skdev->pcie_error_reporting_is_enabled = 0;
5075 } else
5076 skdev->pcie_error_reporting_is_enabled = 1;
5077
5078 for (i = 0; i < SKD_MAX_BARS; i++) {
5079
5080 skdev->mem_phys[i] = pci_resource_start(pdev, i);
5081 skdev->mem_size[i] = (u32)pci_resource_len(pdev, i);
5082 skdev->mem_map[i] = ioremap(skdev->mem_phys[i],
5083 skdev->mem_size[i]);
5084 if (!skdev->mem_map[i]) {
5085 pr_err("(%s): Unable to map adapter memory!\n",
5086 skd_name(skdev));
5087 rc = -ENODEV;
5088 goto err_out_iounmap;
5089 }
2e44b427 5090 pr_debug("%s:%s:%d mem_map=%p, phyd=%016llx, size=%d\n",
5091 skdev->name, __func__, __LINE__,
5092 skdev->mem_map[i],
5093 (uint64_t)skdev->mem_phys[i], skdev->mem_size[i]);
e67f86b3
AB		 5094 }
5095 rc = skd_acquire_irq(skdev);
5096 if (rc) {
5097
5098 pr_err("(%s): interrupt resource error %d\n",
5099 pci_name(pdev), rc);
5100 goto err_out_iounmap;
5101 }
5102
5103 rc = skd_start_timer(skdev);
5104 if (rc)
5105 goto err_out_timer;
5106
5107 init_waitqueue_head(&skdev->waitq);
5108
5109 skd_start_device(skdev);
5110
5111 return rc;
5112
5113err_out_timer:
5114 skd_stop_device(skdev);
5115 skd_release_irq(skdev);
5116
5117err_out_iounmap:
5118 for (i = 0; i < SKD_MAX_BARS; i++)
5119 if (skdev->mem_map[i])
5120 iounmap(skdev->mem_map[i]);
5121
5122 if (skdev->pcie_error_reporting_is_enabled)
5123 pci_disable_pcie_error_reporting(pdev);
5124
5125err_out_regions:
5126 pci_release_regions(pdev);
5127
5128err_out:
5129 pci_disable_device(pdev);
5130 return rc;
5131}
5132
5133static void skd_pci_shutdown(struct pci_dev *pdev)
5134{
5135 struct skd_device *skdev;
5136
5137 pr_err("skd_pci_shutdown called\n");
5138
5139 skdev = pci_get_drvdata(pdev);
5140 if (!skdev) {
5141 pr_err("%s: no device data for PCI\n", pci_name(pdev));
5142 return;
5143 }
5144
5145 pr_err("%s: calling stop\n", skd_name(skdev));
5146 skd_stop_device(skdev);
5147}
5148
5149static struct pci_driver skd_driver = {
5150 .name = DRV_NAME,
5151 .id_table = skd_pci_tbl,
5152 .probe = skd_pci_probe,
5153 .remove = skd_pci_remove,
5154 .suspend = skd_pci_suspend,
5155 .resume = skd_pci_resume,
5156 .shutdown = skd_pci_shutdown,
5157};
5158
5159/*
5160 *****************************************************************************
5161 * LOGGING SUPPORT
5162 *****************************************************************************
5163 */
5164
5165static const char *skd_name(struct skd_device *skdev)
5166{
5167 memset(skdev->id_str, 0, sizeof(skdev->id_str));
5168
5169 if (skdev->inquiry_is_valid)
5170 snprintf(skdev->id_str, sizeof(skdev->id_str), "%s:%s:[%s]",
5171 skdev->name, skdev->inq_serial_num,
5172 pci_name(skdev->pdev));
5173 else
5174 snprintf(skdev->id_str, sizeof(skdev->id_str), "%s:??:[%s]",
5175 skdev->name, pci_name(skdev->pdev));
5176
5177 return skdev->id_str;
5178}
5179
5180const char *skd_drive_state_to_str(int state)
5181{
5182 switch (state) {
5183 case FIT_SR_DRIVE_OFFLINE:
5184 return "OFFLINE";
5185 case FIT_SR_DRIVE_INIT:
5186 return "INIT";
5187 case FIT_SR_DRIVE_ONLINE:
5188 return "ONLINE";
5189 case FIT_SR_DRIVE_BUSY:
5190 return "BUSY";
5191 case FIT_SR_DRIVE_FAULT:
5192 return "FAULT";
5193 case FIT_SR_DRIVE_DEGRADED:
5194 return "DEGRADED";
5195 case FIT_SR_PCIE_LINK_DOWN:
5196 return "INK_DOWN";
5197 case FIT_SR_DRIVE_SOFT_RESET:
5198 return "SOFT_RESET";
5199 case FIT_SR_DRIVE_NEED_FW_DOWNLOAD:
5200 return "NEED_FW";
5201 case FIT_SR_DRIVE_INIT_FAULT:
5202 return "INIT_FAULT";
5203 case FIT_SR_DRIVE_BUSY_SANITIZE:
5204 return "BUSY_SANITIZE";
5205 case FIT_SR_DRIVE_BUSY_ERASE:
5206 return "BUSY_ERASE";
5207 case FIT_SR_DRIVE_FW_BOOTING:
5208 return "FW_BOOTING";
5209 default:
5210 return "???";
5211 }
5212}
5213
5214const char *skd_skdev_state_to_str(enum skd_drvr_state state)
5215{
5216 switch (state) {
5217 case SKD_DRVR_STATE_LOAD:
5218 return "LOAD";
5219 case SKD_DRVR_STATE_IDLE:
5220 return "IDLE";
5221 case SKD_DRVR_STATE_BUSY:
5222 return "BUSY";
5223 case SKD_DRVR_STATE_STARTING:
5224 return "STARTING";
5225 case SKD_DRVR_STATE_ONLINE:
5226 return "ONLINE";
5227 case SKD_DRVR_STATE_PAUSING:
5228 return "PAUSING";
5229 case SKD_DRVR_STATE_PAUSED:
5230 return "PAUSED";
5231 case SKD_DRVR_STATE_DRAINING_TIMEOUT:
5232 return "DRAINING_TIMEOUT";
5233 case SKD_DRVR_STATE_RESTARTING:
5234 return "RESTARTING";
5235 case SKD_DRVR_STATE_RESUMING:
5236 return "RESUMING";
5237 case SKD_DRVR_STATE_STOPPING:
5238 return "STOPPING";
5239 case SKD_DRVR_STATE_SYNCING:
5240 return "SYNCING";
5241 case SKD_DRVR_STATE_FAULT:
5242 return "FAULT";
5243 case SKD_DRVR_STATE_DISAPPEARED:
5244 return "DISAPPEARED";
5245 case SKD_DRVR_STATE_BUSY_ERASE:
5246 return "BUSY_ERASE";
5247 case SKD_DRVR_STATE_BUSY_SANITIZE:
5248 return "BUSY_SANITIZE";
5249 case SKD_DRVR_STATE_BUSY_IMMINENT:
5250 return "BUSY_IMMINENT";
5251 case SKD_DRVR_STATE_WAIT_BOOT:
5252 return "WAIT_BOOT";
5253
5254 default:
5255 return "???";
5256 }
5257}
5258
a26ba7fa 5259static const char *skd_skmsg_state_to_str(enum skd_fit_msg_state state)
e67f86b3
AB		 5260{
5261 switch (state) {
5262 case SKD_MSG_STATE_IDLE:
5263 return "IDLE";
5264 case SKD_MSG_STATE_BUSY:
5265 return "BUSY";
5266 default:
5267 return "???";
5268 }
5269}
5270
a26ba7fa 5271static const char *skd_skreq_state_to_str(enum skd_req_state state)
e67f86b3
AB		 5272{
5273 switch (state) {
5274 case SKD_REQ_STATE_IDLE:
5275 return "IDLE";
5276 case SKD_REQ_STATE_SETUP:
5277 return "SETUP";
5278 case SKD_REQ_STATE_BUSY:
5279 return "BUSY";
5280 case SKD_REQ_STATE_COMPLETED:
5281 return "COMPLETED";
5282 case SKD_REQ_STATE_TIMEOUT:
5283 return "TIMEOUT";
5284 case SKD_REQ_STATE_ABORTED:
5285 return "ABORTED";
5286 default:
5287 return "???";
5288 }
5289}
5290
5291static void skd_log_skdev(struct skd_device *skdev, const char *event)
5292{
2e44b427 5293 pr_debug("%s:%s:%d (%s) skdev=%p event='%s'\n",
5294 skdev->name, __func__, __LINE__, skdev->name, skdev, event);
5295 pr_debug("%s:%s:%d drive_state=%s(%d) driver_state=%s(%d)\n",
5296 skdev->name, __func__, __LINE__,
5297 skd_drive_state_to_str(skdev->drive_state), skdev->drive_state,
5298 skd_skdev_state_to_str(skdev->state), skdev->state);
5299 pr_debug("%s:%s:%d busy=%d limit=%d dev=%d lowat=%d\n",
5300 skdev->name, __func__, __LINE__,
5301 skdev->in_flight, skdev->cur_max_queue_depth,
5302 skdev->dev_max_queue_depth, skdev->queue_low_water_mark);
5303 pr_debug("%s:%s:%d timestamp=0x%x cycle=%d cycle_ix=%d\n",
5304 skdev->name, __func__, __LINE__,
5305 skdev->timeout_stamp, skdev->skcomp_cycle, skdev->skcomp_ix);
e67f86b3
AB		 5306}
5307
5308static void skd_log_skmsg(struct skd_device *skdev,
5309 struct skd_fitmsg_context *skmsg, const char *event)
5310{
2e44b427 5311 pr_debug("%s:%s:%d (%s) skmsg=%p event='%s'\n",
5312 skdev->name, __func__, __LINE__, skdev->name, skmsg, event);
5313 pr_debug("%s:%s:%d state=%s(%d) id=0x%04x length=%d\n",
5314 skdev->name, __func__, __LINE__,
5315 skd_skmsg_state_to_str(skmsg->state), skmsg->state,
5316 skmsg->id, skmsg->length);
e67f86b3
AB		 5317}
5318
5319static void skd_log_skreq(struct skd_device *skdev,
5320 struct skd_request_context *skreq, const char *event)
5321{
2e44b427 5322 pr_debug("%s:%s:%d (%s) skreq=%p event='%s'\n",
5323 skdev->name, __func__, __LINE__, skdev->name, skreq, event);
5324 pr_debug("%s:%s:%d state=%s(%d) id=0x%04x fitmsg=0x%04x\n",
5325 skdev->name, __func__, __LINE__,
5326 skd_skreq_state_to_str(skreq->state), skreq->state,
5327 skreq->id, skreq->fitmsg_id);
5328 pr_debug("%s:%s:%d timo=0x%x sg_dir=%d n_sg=%d\n",
5329 skdev->name, __func__, __LINE__,
5330 skreq->timeout_stamp, skreq->sg_data_dir, skreq->n_sg);
e67f86b3 5331
fcd37eb3
JA		 5332 if (skreq->req != NULL) {
5333 struct request *req = skreq->req;
5334 u32 lba = (u32)blk_rq_pos(req);
5335 u32 count = blk_rq_sectors(req);
e67f86b3 5336
fcd37eb3
JA		 5337 pr_debug("%s:%s:%d "
5338 "req=%p lba=%u(0x%x) count=%u(0x%x) dir=%d\n",
5339 skdev->name, __func__, __LINE__,
5340 req, lba, lba, count, count,
5341 (int)rq_data_dir(req));
5342 } else
5343 pr_debug("%s:%s:%d req=NULL\n",
5344 skdev->name, __func__, __LINE__);
e67f86b3
AB		 5345}
5346
5347/*
5348 *****************************************************************************
5349 * MODULE GLUE
5350 *****************************************************************************
5351 */
5352
5353static int __init skd_init(void)
5354{
e67f86b3
AB		 5355 pr_info(PFX " v%s-b%s loaded\n", DRV_VERSION, DRV_BUILD_ID);
5356
5357 switch (skd_isr_type) {
5358 case SKD_IRQ_LEGACY:
5359 case SKD_IRQ_MSI:
5360 case SKD_IRQ_MSIX:
5361 break;
5362 default:
fbed149a 5363 pr_err(PFX "skd_isr_type %d invalid, re-set to %d\n",
e67f86b3
AB		 5364 skd_isr_type, SKD_IRQ_DEFAULT);
5365 skd_isr_type = SKD_IRQ_DEFAULT;
5366 }
5367
fbed149a
BZ		 5368 if (skd_max_queue_depth < 1 ||
5369 skd_max_queue_depth > SKD_MAX_QUEUE_DEPTH) {
5370 pr_err(PFX "skd_max_queue_depth %d invalid, re-set to %d\n",
e67f86b3
AB		 5371 skd_max_queue_depth, SKD_MAX_QUEUE_DEPTH_DEFAULT);
5372 skd_max_queue_depth = SKD_MAX_QUEUE_DEPTH_DEFAULT;
5373 }
5374
5375 if (skd_max_req_per_msg < 1 || skd_max_req_per_msg > 14) {
fbed149a 5376 pr_err(PFX "skd_max_req_per_msg %d invalid, re-set to %d\n",
e67f86b3
AB		 5377 skd_max_req_per_msg, SKD_MAX_REQ_PER_MSG_DEFAULT);
5378 skd_max_req_per_msg = SKD_MAX_REQ_PER_MSG_DEFAULT;
5379 }
5380
5381 if (skd_sgs_per_request < 1 || skd_sgs_per_request > 4096) {
fbed149a 5382 pr_err(PFX "skd_sg_per_request %d invalid, re-set to %d\n",
e67f86b3
AB		 5383 skd_sgs_per_request, SKD_N_SG_PER_REQ_DEFAULT);
5384 skd_sgs_per_request = SKD_N_SG_PER_REQ_DEFAULT;
5385 }
5386
5387 if (skd_dbg_level < 0 || skd_dbg_level > 2) {
fbed149a 5388 pr_err(PFX "skd_dbg_level %d invalid, re-set to %d\n",
e67f86b3
AB		 5389 skd_dbg_level, 0);
5390 skd_dbg_level = 0;
5391 }
5392
5393 if (skd_isr_comp_limit < 0) {
fbed149a 5394 pr_err(PFX "skd_isr_comp_limit %d invalid, set to %d\n",
e67f86b3
AB		 5395 skd_isr_comp_limit, 0);
5396 skd_isr_comp_limit = 0;
5397 }
5398
5399 if (skd_max_pass_thru < 1 || skd_max_pass_thru > 50) {
fbed149a 5400 pr_err(PFX "skd_max_pass_thru %d invalid, re-set to %d\n",
e67f86b3
AB		 5401 skd_max_pass_thru, SKD_N_SPECIAL_CONTEXT);
5402 skd_max_pass_thru = SKD_N_SPECIAL_CONTEXT;
5403 }
5404
b8df6647 5405 return pci_register_driver(&skd_driver);
e67f86b3
AB		 5406}
5407
5408static void __exit skd_exit(void)
5409{
5410 pr_info(PFX " v%s-b%s unloading\n", DRV_VERSION, DRV_BUILD_ID);
5411
e67f86b3 5412 pci_unregister_driver(&skd_driver);
b8df6647
BZ		 5413
5414 if (skd_major)
5415 unregister_blkdev(skd_major, DRV_NAME);
e67f86b3
AB		 5416}
5417
e67f86b3
AB		 5418module_init(skd_init);
5419module_exit(skd_exit);
Linux 6.x block layer and io_uring tree(s)