projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
skd: cleanup the skd_*() function block wrapping
[linux-2.6-block.git] / drivers / block / skd_main.c
CommitLineData
e67f86b3
AB		 1/* Copyright 2012 STEC, Inc.
2 *
3 * This file is licensed under the terms of the 3-clause
4 * BSD License (http://opensource.org/licenses/BSD-3-Clause)
5 * or the GNU GPL-2.0 (http://www.gnu.org/licenses/gpl-2.0.html),
6 * at your option. Both licenses are also available in the LICENSE file
7 * distributed with this project. This file may not be copied, modified,
8 * or distributed except in accordance with those terms.
9 * Gordoni Waidhofer <gwaidhofer@stec-inc.com>
10 * Initial Driver Design!
11 * Thomas Swann <tswann@stec-inc.com>
12 * Interrupt handling.
13 * Ramprasad Chinthekindi <rchinthekindi@stec-inc.com>
14 * biomode implementation.
15 * Akhil Bhansali <abhansali@stec-inc.com>
16 * Added support for DISCARD / FLUSH and FUA.
17 */
18
19#include <linux/kernel.h>
20#include <linux/module.h>
21#include <linux/init.h>
22#include <linux/pci.h>
23#include <linux/slab.h>
24#include <linux/spinlock.h>
25#include <linux/blkdev.h>
26#include <linux/sched.h>
27#include <linux/interrupt.h>
28#include <linux/compiler.h>
29#include <linux/workqueue.h>
30#include <linux/bitops.h>
31#include <linux/delay.h>
32#include <linux/time.h>
33#include <linux/hdreg.h>
34#include <linux/dma-mapping.h>
35#include <linux/completion.h>
36#include <linux/scatterlist.h>
37#include <linux/version.h>
38#include <linux/err.h>
39#include <linux/scatterlist.h>
40#include <linux/aer.h>
41#include <linux/ctype.h>
42#include <linux/wait.h>
43#include <linux/uio.h>
44#include <scsi/scsi.h>
45#include <scsi/scsi_host.h>
46#include <scsi/scsi_tcq.h>
47#include <scsi/scsi_cmnd.h>
48#include <scsi/sg.h>
49#include <linux/io.h>
50#include <linux/uaccess.h>
51#include <asm-generic/unaligned.h>
52
53#include "skd_s1120.h"
54
55static int skd_dbg_level;
56static int skd_isr_comp_limit = 4;
57
58enum {
59 STEC_LINK_2_5GTS = 0,
60 STEC_LINK_5GTS = 1,
61 STEC_LINK_8GTS = 2,
62 STEC_LINK_UNKNOWN = 0xFF
63};
64
65enum {
66 SKD_FLUSH_INITIALIZER,
67 SKD_FLUSH_ZERO_SIZE_FIRST,
68 SKD_FLUSH_DATA_SECOND,
69};
70
e67f86b3
AB		 71#define SKD_ASSERT(expr) \
72 do { \
73 if (unlikely(!(expr))) { \
74 pr_err("Assertion failed! %s,%s,%s,line=%d\n", \
75 # expr, __FILE__, __func__, __LINE__); \
76 } \
77 } while (0)
78
e67f86b3
AB		 79#define DRV_NAME "skd"
80#define DRV_VERSION "2.2.1"
81#define DRV_BUILD_ID "0260"
82#define PFX DRV_NAME ": "
83#define DRV_BIN_VERSION 0x100
84#define DRV_VER_COMPL "2.2.1." DRV_BUILD_ID
85
86MODULE_AUTHOR("bug-reports: support@stec-inc.com");
87MODULE_LICENSE("Dual BSD/GPL");
88
89MODULE_DESCRIPTION("STEC s1120 PCIe SSD block/BIO driver (b" DRV_BUILD_ID ")");
90MODULE_VERSION(DRV_VERSION "-" DRV_BUILD_ID);
91
92#define PCI_VENDOR_ID_STEC 0x1B39
93#define PCI_DEVICE_ID_S1120 0x0001
94
95#define SKD_FUA_NV (1 << 1)
96#define SKD_MINORS_PER_DEVICE 16
97
98#define SKD_MAX_QUEUE_DEPTH 200u
99
100#define SKD_PAUSE_TIMEOUT (5 * 1000)
101
102#define SKD_N_FITMSG_BYTES (512u)
103
104#define SKD_N_SPECIAL_CONTEXT 32u
105#define SKD_N_SPECIAL_FITMSG_BYTES (128u)
106
107/* SG elements are 32 bytes, so we can make this 4096 and still be under the
108 * 128KB limit. That allows 4096*4K = 16M xfer size
109 */
110#define SKD_N_SG_PER_REQ_DEFAULT 256u
111#define SKD_N_SG_PER_SPECIAL 256u
112
113#define SKD_N_COMPLETION_ENTRY 256u
114#define SKD_N_READ_CAP_BYTES (8u)
115
116#define SKD_N_INTERNAL_BYTES (512u)
117
118/* 5 bits of uniqifier, 0xF800 */
119#define SKD_ID_INCR (0x400)
120#define SKD_ID_TABLE_MASK (3u << 8u)
121#define SKD_ID_RW_REQUEST (0u << 8u)
122#define SKD_ID_INTERNAL (1u << 8u)
123#define SKD_ID_SPECIAL_REQUEST (2u << 8u)
124#define SKD_ID_FIT_MSG (3u << 8u)
125#define SKD_ID_SLOT_MASK 0x00FFu
126#define SKD_ID_SLOT_AND_TABLE_MASK 0x03FFu
127
128#define SKD_N_TIMEOUT_SLOT 4u
129#define SKD_TIMEOUT_SLOT_MASK 3u
130
131#define SKD_N_MAX_SECTORS 2048u
132
133#define SKD_MAX_RETRIES 2u
134
135#define SKD_TIMER_SECONDS(seconds) (seconds)
136#define SKD_TIMER_MINUTES(minutes) ((minutes) * (60))
137
138#define INQ_STD_NBYTES 36
139#define SKD_DISCARD_CDB_LENGTH 24
140
141enum skd_drvr_state {
142 SKD_DRVR_STATE_LOAD,
143 SKD_DRVR_STATE_IDLE,
144 SKD_DRVR_STATE_BUSY,
145 SKD_DRVR_STATE_STARTING,
146 SKD_DRVR_STATE_ONLINE,
147 SKD_DRVR_STATE_PAUSING,
148 SKD_DRVR_STATE_PAUSED,
149 SKD_DRVR_STATE_DRAINING_TIMEOUT,
150 SKD_DRVR_STATE_RESTARTING,
151 SKD_DRVR_STATE_RESUMING,
152 SKD_DRVR_STATE_STOPPING,
153 SKD_DRVR_STATE_FAULT,
154 SKD_DRVR_STATE_DISAPPEARED,
155 SKD_DRVR_STATE_PROTOCOL_MISMATCH,
156 SKD_DRVR_STATE_BUSY_ERASE,
157 SKD_DRVR_STATE_BUSY_SANITIZE,
158 SKD_DRVR_STATE_BUSY_IMMINENT,
159 SKD_DRVR_STATE_WAIT_BOOT,
160 SKD_DRVR_STATE_SYNCING,
161};
162
163#define SKD_WAIT_BOOT_TIMO SKD_TIMER_SECONDS(90u)
164#define SKD_STARTING_TIMO SKD_TIMER_SECONDS(8u)
165#define SKD_RESTARTING_TIMO SKD_TIMER_MINUTES(4u)
166#define SKD_DRAINING_TIMO SKD_TIMER_SECONDS(6u)
167#define SKD_BUSY_TIMO SKD_TIMER_MINUTES(20u)
168#define SKD_STARTED_BUSY_TIMO SKD_TIMER_SECONDS(60u)
169#define SKD_START_WAIT_SECONDS 90u
170
171enum skd_req_state {
172 SKD_REQ_STATE_IDLE,
173 SKD_REQ_STATE_SETUP,
174 SKD_REQ_STATE_BUSY,
175 SKD_REQ_STATE_COMPLETED,
176 SKD_REQ_STATE_TIMEOUT,
177 SKD_REQ_STATE_ABORTED,
178};
179
180enum skd_fit_msg_state {
181 SKD_MSG_STATE_IDLE,
182 SKD_MSG_STATE_BUSY,
183};
184
185enum skd_check_status_action {
186 SKD_CHECK_STATUS_REPORT_GOOD,
187 SKD_CHECK_STATUS_REPORT_SMART_ALERT,
188 SKD_CHECK_STATUS_REQUEUE_REQUEST,
189 SKD_CHECK_STATUS_REPORT_ERROR,
190 SKD_CHECK_STATUS_BUSY_IMMINENT,
191};
192
193struct skd_fitmsg_context {
194 enum skd_fit_msg_state state;
195
196 struct skd_fitmsg_context *next;
197
198 u32 id;
199 u16 outstanding;
200
201 u32 length;
202 u32 offset;
203
204 u8 *msg_buf;
205 dma_addr_t mb_dma_address;
206};
207
208struct skd_request_context {
209 enum skd_req_state state;
210
211 struct skd_request_context *next;
212
213 u16 id;
214 u32 fitmsg_id;
215
216 struct request *req;
e67f86b3
AB		 217 u8 flush_cmd;
218 u8 discard_page;
219
220 u32 timeout_stamp;
221 u8 sg_data_dir;
222 struct scatterlist *sg;
223 u32 n_sg;
224 u32 sg_byte_count;
225
226 struct fit_sg_descriptor *sksg_list;
227 dma_addr_t sksg_dma_address;
228
229 struct fit_completion_entry_v1 completion;
230
231 struct fit_comp_error_info err_info;
232
233};
234#define SKD_DATA_DIR_HOST_TO_CARD 1
235#define SKD_DATA_DIR_CARD_TO_HOST 2
236#define SKD_DATA_DIR_NONE 3 /* especially for DISCARD requests. */
237
238struct skd_special_context {
239 struct skd_request_context req;
240
241 u8 orphaned;
242
243 void *data_buf;
244 dma_addr_t db_dma_address;
245
246 u8 *msg_buf;
247 dma_addr_t mb_dma_address;
248};
249
250struct skd_sg_io {
251 fmode_t mode;
252 void __user *argp;
253
254 struct sg_io_hdr sg;
255
256 u8 cdb[16];
257
258 u32 dxfer_len;
259 u32 iovcnt;
260 struct sg_iovec *iov;
261 struct sg_iovec no_iov_iov;
262
263 struct skd_special_context *skspcl;
264};
265
266typedef enum skd_irq_type {
267 SKD_IRQ_LEGACY,
268 SKD_IRQ_MSI,
269 SKD_IRQ_MSIX
270} skd_irq_type_t;
271
272#define SKD_MAX_BARS 2
273
274struct skd_device {
275 volatile void __iomem *mem_map[SKD_MAX_BARS];
276 resource_size_t mem_phys[SKD_MAX_BARS];
277 u32 mem_size[SKD_MAX_BARS];
278
279 skd_irq_type_t irq_type;
280 u32 msix_count;
281 struct skd_msix_entry *msix_entries;
282
283 struct pci_dev *pdev;
284 int pcie_error_reporting_is_enabled;
285
286 spinlock_t lock;
287 struct gendisk *disk;
288 struct request_queue *queue;
289 struct device *class_dev;
290 int gendisk_on;
291 int sync_done;
292
293 atomic_t device_count;
294 u32 devno;
295 u32 major;
296 char name[32];
297 char isr_name[30];
298
299 enum skd_drvr_state state;
300 u32 drive_state;
301
302 u32 in_flight;
303 u32 cur_max_queue_depth;
304 u32 queue_low_water_mark;
305 u32 dev_max_queue_depth;
306
307 u32 num_fitmsg_context;
308 u32 num_req_context;
309
310 u32 timeout_slot[SKD_N_TIMEOUT_SLOT];
311 u32 timeout_stamp;
312 struct skd_fitmsg_context *skmsg_free_list;
313 struct skd_fitmsg_context *skmsg_table;
314
315 struct skd_request_context *skreq_free_list;
316 struct skd_request_context *skreq_table;
317
318 struct skd_special_context *skspcl_free_list;
319 struct skd_special_context *skspcl_table;
320
321 struct skd_special_context internal_skspcl;
322 u32 read_cap_blocksize;
323 u32 read_cap_last_lba;
324 int read_cap_is_valid;
325 int inquiry_is_valid;
326 u8 inq_serial_num[13]; /*12 chars plus null term */
327 u8 id_str[80]; /* holds a composite name (pci + sernum) */
328
329 u8 skcomp_cycle;
330 u32 skcomp_ix;
331 struct fit_completion_entry_v1 *skcomp_table;
332 struct fit_comp_error_info *skerr_table;
333 dma_addr_t cq_dma_address;
334
335 wait_queue_head_t waitq;
336
337 struct timer_list timer;
338 u32 timer_countdown;
339 u32 timer_substate;
340
341 int n_special;
342 int sgs_per_request;
343 u32 last_mtd;
344
345 u32 proto_ver;
346
347 int dbg_level;
348 u32 connect_time_stamp;
349 int connect_retries;
350#define SKD_MAX_CONNECT_RETRIES 16
351 u32 drive_jiffies;
352
353 u32 timo_slot;
354
e67f86b3 355 struct work_struct completion_worker;
e67f86b3
AB		 356};
357
358#define SKD_FLUSH_JOB "skd-flush-jobs"
359struct kmem_cache *skd_flush_slab;
360
361/*
362 * These commands hold "nonzero size FLUSH bios",
363 * which are enqueud in skdev->flush_list during
364 * completion of "zero size FLUSH commands".
365 * It will be active in biomode.
366 */
367struct skd_flush_cmd {
368 void *cmd;
369 struct list_head flist;
370};
371
372#define SKD_WRITEL(DEV, VAL, OFF) skd_reg_write32(DEV, VAL, OFF)
373#define SKD_READL(DEV, OFF) skd_reg_read32(DEV, OFF)
374#define SKD_WRITEQ(DEV, VAL, OFF) skd_reg_write64(DEV, VAL, OFF)
375
376static inline u32 skd_reg_read32(struct skd_device *skdev, u32 offset)
377{
378 u32 val;
379
380 if (likely(skdev->dbg_level < 2))
381 return readl(skdev->mem_map[1] + offset);
382 else {
383 barrier();
384 val = readl(skdev->mem_map[1] + offset);
385 barrier();
2e44b427 386 pr_debug("%s:%s:%d offset %x = %x\n",
387 skdev->name, __func__, __LINE__, offset, val);
e67f86b3
AB		 388 return val;
389 }
390
391}
392
393static inline void skd_reg_write32(struct skd_device *skdev, u32 val,
394 u32 offset)
395{
396 if (likely(skdev->dbg_level < 2)) {
397 writel(val, skdev->mem_map[1] + offset);
398 barrier();
e67f86b3
AB		 399 } else {
400 barrier();
401 writel(val, skdev->mem_map[1] + offset);
402 barrier();
2e44b427 403 pr_debug("%s:%s:%d offset %x = %x\n",
404 skdev->name, __func__, __LINE__, offset, val);
e67f86b3
AB		 405 }
406}
407
408static inline void skd_reg_write64(struct skd_device *skdev, u64 val,
409 u32 offset)
410{
411 if (likely(skdev->dbg_level < 2)) {
412 writeq(val, skdev->mem_map[1] + offset);
413 barrier();
e67f86b3
AB		 414 } else {
415 barrier();
416 writeq(val, skdev->mem_map[1] + offset);
417 barrier();
2e44b427 418 pr_debug("%s:%s:%d offset %x = %016llx\n",
419 skdev->name, __func__, __LINE__, offset, val);
e67f86b3
AB		 420 }
421}
422
423
424#define SKD_IRQ_DEFAULT SKD_IRQ_MSI
425static int skd_isr_type = SKD_IRQ_DEFAULT;
426
427module_param(skd_isr_type, int, 0444);
428MODULE_PARM_DESC(skd_isr_type, "Interrupt type capability."
429 " (0==legacy, 1==MSI, 2==MSI-X, default==1)");
430
431#define SKD_MAX_REQ_PER_MSG_DEFAULT 1
432static int skd_max_req_per_msg = SKD_MAX_REQ_PER_MSG_DEFAULT;
433
434module_param(skd_max_req_per_msg, int, 0444);
435MODULE_PARM_DESC(skd_max_req_per_msg,
436 "Maximum SCSI requests packed in a single message."
437 " (1-14, default==1)");
438
439#define SKD_MAX_QUEUE_DEPTH_DEFAULT 64
440#define SKD_MAX_QUEUE_DEPTH_DEFAULT_STR "64"
441static int skd_max_queue_depth = SKD_MAX_QUEUE_DEPTH_DEFAULT;
442
443module_param(skd_max_queue_depth, int, 0444);
444MODULE_PARM_DESC(skd_max_queue_depth,
445 "Maximum SCSI requests issued to s1120."
446 " (1-200, default==" SKD_MAX_QUEUE_DEPTH_DEFAULT_STR ")");
447
448static int skd_sgs_per_request = SKD_N_SG_PER_REQ_DEFAULT;
449module_param(skd_sgs_per_request, int, 0444);
450MODULE_PARM_DESC(skd_sgs_per_request,
451 "Maximum SG elements per block request."
452 " (1-4096, default==256)");
453
454static int skd_max_pass_thru = SKD_N_SPECIAL_CONTEXT;
455module_param(skd_max_pass_thru, int, 0444);
456MODULE_PARM_DESC(skd_max_pass_thru,
457 "Maximum SCSI pass-thru at a time." " (1-50, default==32)");
458
459module_param(skd_dbg_level, int, 0444);
460MODULE_PARM_DESC(skd_dbg_level, "s1120 debug level (0,1,2)");
461
462module_param(skd_isr_comp_limit, int, 0444);
463MODULE_PARM_DESC(skd_isr_comp_limit, "s1120 isr comp limit (0=none) default=4");
464
e67f86b3
AB		 465/* Major device number dynamically assigned. */
466static u32 skd_major;
467
468static struct skd_device *skd_construct(struct pci_dev *pdev);
469static void skd_destruct(struct skd_device *skdev);
470static const struct block_device_operations skd_blockdev_ops;
471static void skd_send_fitmsg(struct skd_device *skdev,
472 struct skd_fitmsg_context *skmsg);
473static void skd_send_special_fitmsg(struct skd_device *skdev,
474 struct skd_special_context *skspcl);
475static void skd_request_fn(struct request_queue *rq);
476static void skd_end_request(struct skd_device *skdev,
477 struct skd_request_context *skreq, int error);
478static int skd_preop_sg_list(struct skd_device *skdev,
479 struct skd_request_context *skreq);
480static void skd_postop_sg_list(struct skd_device *skdev,
481 struct skd_request_context *skreq);
482
483static void skd_restart_device(struct skd_device *skdev);
484static int skd_quiesce_dev(struct skd_device *skdev);
485static int skd_unquiesce_dev(struct skd_device *skdev);
486static void skd_release_special(struct skd_device *skdev,
487 struct skd_special_context *skspcl);
488static void skd_disable_interrupts(struct skd_device *skdev);
489static void skd_isr_fwstate(struct skd_device *skdev);
490static void skd_recover_requests(struct skd_device *skdev, int requeue);
491static void skd_soft_reset(struct skd_device *skdev);
492
493static const char *skd_name(struct skd_device *skdev);
494const char *skd_drive_state_to_str(int state);
495const char *skd_skdev_state_to_str(enum skd_drvr_state state);
496static void skd_log_skdev(struct skd_device *skdev, const char *event);
497static void skd_log_skmsg(struct skd_device *skdev,
498 struct skd_fitmsg_context *skmsg, const char *event);
499static void skd_log_skreq(struct skd_device *skdev,
500 struct skd_request_context *skreq, const char *event);
501
e67f86b3
AB		 502/*
503 *****************************************************************************
504 * READ/WRITE REQUESTS
505 *****************************************************************************
506 */
fcd37eb3 507static void skd_fail_all_pending(struct skd_device *skdev)
e67f86b3
AB		 508{
509 struct request_queue *q = skdev->queue;
510 struct request *req;
511
512 for (;; ) {
513 req = blk_peek_request(q);
514 if (req == NULL)
515 break;
516 blk_start_request(req);
517 __blk_end_request_all(req, -EIO);
518 }
519}
520
e67f86b3
AB		 521static void
522skd_prep_rw_cdb(struct skd_scsi_request *scsi_req,
523 int data_dir, unsigned lba,
524 unsigned count)
525{
526 if (data_dir == READ)
527 scsi_req->cdb[0] = 0x28;
528 else
529 scsi_req->cdb[0] = 0x2a;
530
531 scsi_req->cdb[1] = 0;
532 scsi_req->cdb[2] = (lba & 0xff000000) >> 24;
533 scsi_req->cdb[3] = (lba & 0xff0000) >> 16;
534 scsi_req->cdb[4] = (lba & 0xff00) >> 8;
535 scsi_req->cdb[5] = (lba & 0xff);
536 scsi_req->cdb[6] = 0;
537 scsi_req->cdb[7] = (count & 0xff00) >> 8;
538 scsi_req->cdb[8] = count & 0xff;
539 scsi_req->cdb[9] = 0;
540}
541
542static void
543skd_prep_zerosize_flush_cdb(struct skd_scsi_request *scsi_req,
544 struct skd_request_context *skreq)
545{
546 skreq->flush_cmd = 1;
547
548 scsi_req->cdb[0] = 0x35;
549 scsi_req->cdb[1] = 0;
550 scsi_req->cdb[2] = 0;
551 scsi_req->cdb[3] = 0;
552 scsi_req->cdb[4] = 0;
553 scsi_req->cdb[5] = 0;
554 scsi_req->cdb[6] = 0;
555 scsi_req->cdb[7] = 0;
556 scsi_req->cdb[8] = 0;
557 scsi_req->cdb[9] = 0;
558}
559
560static void
561skd_prep_discard_cdb(struct skd_scsi_request *scsi_req,
562 struct skd_request_context *skreq,
563 struct page *page,
564 u32 lba, u32 count)
565{
566 char *buf;
567 unsigned long len;
568 struct request *req;
569
570 buf = page_address(page);
571 len = SKD_DISCARD_CDB_LENGTH;
572
573 scsi_req->cdb[0] = UNMAP;
574 scsi_req->cdb[8] = len;
575
576 put_unaligned_be16(6 + 16, &buf[0]);
577 put_unaligned_be16(16, &buf[2]);
578 put_unaligned_be64(lba, &buf[8]);
579 put_unaligned_be32(count, &buf[16]);
580
fcd37eb3
JA		 581 req = skreq->req;
582 blk_add_request_payload(req, page, len);
583 req->buffer = buf;
e67f86b3
AB		 584}
585
586static void skd_request_fn_not_online(struct request_queue *q);
587
588static void skd_request_fn(struct request_queue *q)
589{
590 struct skd_device *skdev = q->queuedata;
591 struct skd_fitmsg_context *skmsg = NULL;
592 struct fit_msg_hdr *fmh = NULL;
593 struct skd_request_context *skreq;
594 struct request *req = NULL;
e67f86b3
AB		 595 struct skd_scsi_request *scsi_req;
596 struct page *page;
597 unsigned long io_flags;
598 int error;
599 u32 lba;
600 u32 count;
601 int data_dir;
602 u32 be_lba;
603 u32 be_count;
604 u64 be_dmaa;
605 u64 cmdctxt;
606 u32 timo_slot;
607 void *cmd_ptr;
608 int flush, fua;
609
610 if (skdev->state != SKD_DRVR_STATE_ONLINE) {
611 skd_request_fn_not_online(q);
612 return;
613 }
614
6a5ec65b 615 if (blk_queue_stopped(skdev->queue)) {
e67f86b3
AB		 616 if (skdev->skmsg_free_list == NULL ||
617 skdev->skreq_free_list == NULL ||
618 skdev->in_flight >= skdev->queue_low_water_mark)
619 /* There is still some kind of shortage */
620 return;
621
6a5ec65b 622 queue_flag_clear(QUEUE_FLAG_STOPPED, skdev->queue);
e67f86b3
AB		 623 }
624
625 /*
626 * Stop conditions:
627 * - There are no more native requests
628 * - There are already the maximum number of requests in progress
629 * - There are no more skd_request_context entries
630 * - There are no more FIT msg buffers
631 */
632 for (;; ) {
633
634 flush = fua = 0;
635
fcd37eb3 636 req = blk_peek_request(q);
e67f86b3 637
fcd37eb3
JA		 638 /* Are there any native requests to start? */
639 if (req == NULL)
640 break;
e67f86b3 641
fcd37eb3
JA		 642 lba = (u32)blk_rq_pos(req);
643 count = blk_rq_sectors(req);
644 data_dir = rq_data_dir(req);
645 io_flags = req->cmd_flags;
e67f86b3 646
fcd37eb3
JA		 647 if (io_flags & REQ_FLUSH)
648 flush++;
e67f86b3 649
fcd37eb3
JA		 650 if (io_flags & REQ_FUA)
651 fua++;
e67f86b3 652
fcd37eb3
JA		 653 pr_debug("%s:%s:%d new req=%p lba=%u(0x%x) "
654 "count=%u(0x%x) dir=%d\n",
655 skdev->name, __func__, __LINE__,
656 req, lba, lba, count, count, data_dir);
e67f86b3
AB		 657
658 /* At this point we know there is a request
659 * (from our bio q or req q depending on the way
660 * the driver is built do checks for resources.
661 */
662
663 /* Are too many requets already in progress? */
664 if (skdev->in_flight >= skdev->cur_max_queue_depth) {
2e44b427 665 pr_debug("%s:%s:%d qdepth %d, limit %d\n",
666 skdev->name, __func__, __LINE__,
667 skdev->in_flight, skdev->cur_max_queue_depth);
e67f86b3
AB		 668 break;
669 }
670
671 /* Is a skd_request_context available? */
672 skreq = skdev->skreq_free_list;
673 if (skreq == NULL) {
2e44b427 674 pr_debug("%s:%s:%d Out of req=%p\n",
675 skdev->name, __func__, __LINE__, q);
e67f86b3
AB		 676 break;
677 }
678 SKD_ASSERT(skreq->state == SKD_REQ_STATE_IDLE);
679 SKD_ASSERT((skreq->id & SKD_ID_INCR) == 0);
680
681 /* Now we check to see if we can get a fit msg */
682 if (skmsg == NULL) {
683 if (skdev->skmsg_free_list == NULL) {
2e44b427 684 pr_debug("%s:%s:%d Out of msg\n",
685 skdev->name, __func__, __LINE__);
e67f86b3
AB		 686 break;
687 }
688 }
689
690 skreq->flush_cmd = 0;
691 skreq->n_sg = 0;
692 skreq->sg_byte_count = 0;
693 skreq->discard_page = 0;
694
695 /*
696 * OK to now dequeue request from either bio or q.
697 *
698 * At this point we are comitted to either start or reject
699 * the native request. Note that skd_request_context is
700 * available but is still at the head of the free list.
701 */
fcd37eb3
JA		 702 blk_start_request(req);
703 skreq->req = req;
704 skreq->fitmsg_id = 0;
e67f86b3
AB		 705
706 /* Either a FIT msg is in progress or we have to start one. */
707 if (skmsg == NULL) {
708 /* Are there any FIT msg buffers available? */
709 skmsg = skdev->skmsg_free_list;
710 if (skmsg == NULL) {
2e44b427 711 pr_debug("%s:%s:%d Out of msg skdev=%p\n",
712 skdev->name, __func__, __LINE__,
713 skdev);
e67f86b3
AB		 714 break;
715 }
716 SKD_ASSERT(skmsg->state == SKD_MSG_STATE_IDLE);
717 SKD_ASSERT((skmsg->id & SKD_ID_INCR) == 0);
718
719 skdev->skmsg_free_list = skmsg->next;
720
721 skmsg->state = SKD_MSG_STATE_BUSY;
722 skmsg->id += SKD_ID_INCR;
723
724 /* Initialize the FIT msg header */
725 fmh = (struct fit_msg_hdr *)skmsg->msg_buf;
726 memset(fmh, 0, sizeof(*fmh));
727 fmh->protocol_id = FIT_PROTOCOL_ID_SOFIT;
728 skmsg->length = sizeof(*fmh);
729 }
730
731 skreq->fitmsg_id = skmsg->id;
732
733 /*
734 * Note that a FIT msg may have just been started
735 * but contains no SoFIT requests yet.
736 */
737
738 /*
739 * Transcode the request, checking as we go. The outcome of
740 * the transcoding is represented by the error variable.
741 */
742 cmd_ptr = &skmsg->msg_buf[skmsg->length];
743 memset(cmd_ptr, 0, 32);
744
745 be_lba = cpu_to_be32(lba);
746 be_count = cpu_to_be32(count);
747 be_dmaa = cpu_to_be64((u64)skreq->sksg_dma_address);
748 cmdctxt = skreq->id + SKD_ID_INCR;
749
750 scsi_req = cmd_ptr;
751 scsi_req->hdr.tag = cmdctxt;
752 scsi_req->hdr.sg_list_dma_address = be_dmaa;
753
754 if (data_dir == READ)
755 skreq->sg_data_dir = SKD_DATA_DIR_CARD_TO_HOST;
756 else
757 skreq->sg_data_dir = SKD_DATA_DIR_HOST_TO_CARD;
758
759 if (io_flags & REQ_DISCARD) {
760 page = alloc_page(GFP_ATOMIC | __GFP_ZERO);
761 if (!page) {
762 pr_err("request_fn:Page allocation failed.\n");
763 skd_end_request(skdev, skreq, -ENOMEM);
764 break;
765 }
766 skreq->discard_page = 1;
767 skd_prep_discard_cdb(scsi_req, skreq, page, lba, count);
768
769 } else if (flush == SKD_FLUSH_ZERO_SIZE_FIRST) {
770 skd_prep_zerosize_flush_cdb(scsi_req, skreq);
771 SKD_ASSERT(skreq->flush_cmd == 1);
772
773 } else {
774 skd_prep_rw_cdb(scsi_req, data_dir, lba, count);
775 }
776
777 if (fua)
778 scsi_req->cdb[1] |= SKD_FUA_NV;
779
fcd37eb3 780 if (!req->bio)
e67f86b3
AB		 781 goto skip_sg;
782
783 error = skd_preop_sg_list(skdev, skreq);
784
785 if (error != 0) {
786 /*
787 * Complete the native request with error.
788 * Note that the request context is still at the
789 * head of the free list, and that the SoFIT request
790 * was encoded into the FIT msg buffer but the FIT
791 * msg length has not been updated. In short, the
792 * only resource that has been allocated but might
793 * not be used is that the FIT msg could be empty.
794 */
2e44b427 795 pr_debug("%s:%s:%d error Out\n",
796 skdev->name, __func__, __LINE__);
e67f86b3
AB		 797 skd_end_request(skdev, skreq, error);
798 continue;
799 }
800
801skip_sg:
802 scsi_req->hdr.sg_list_len_bytes =
803 cpu_to_be32(skreq->sg_byte_count);
804
805 /* Complete resource allocations. */
806 skdev->skreq_free_list = skreq->next;
807 skreq->state = SKD_REQ_STATE_BUSY;
808 skreq->id += SKD_ID_INCR;
809
810 skmsg->length += sizeof(struct skd_scsi_request);
811 fmh->num_protocol_cmds_coalesced++;
812
813 /*
814 * Update the active request counts.
815 * Capture the timeout timestamp.
816 */
817 skreq->timeout_stamp = skdev->timeout_stamp;
818 timo_slot = skreq->timeout_stamp & SKD_TIMEOUT_SLOT_MASK;
819 skdev->timeout_slot[timo_slot]++;
820 skdev->in_flight++;
2e44b427 821 pr_debug("%s:%s:%d req=0x%x busy=%d\n",
822 skdev->name, __func__, __LINE__,
823 skreq->id, skdev->in_flight);
e67f86b3
AB		 824
825 /*
826 * If the FIT msg buffer is full send it.
827 */
828 if (skmsg->length >= SKD_N_FITMSG_BYTES ||
829 fmh->num_protocol_cmds_coalesced >= skd_max_req_per_msg) {
830 skd_send_fitmsg(skdev, skmsg);
831 skmsg = NULL;
832 fmh = NULL;
833 }
834 }
835
836 /*
837 * Is a FIT msg in progress? If it is empty put the buffer back
838 * on the free list. If it is non-empty send what we got.
839 * This minimizes latency when there are fewer requests than
840 * what fits in a FIT msg.
841 */
842 if (skmsg != NULL) {
843 /* Bigger than just a FIT msg header? */
844 if (skmsg->length > sizeof(struct fit_msg_hdr)) {
2e44b427 845 pr_debug("%s:%s:%d sending msg=%p, len %d\n",
846 skdev->name, __func__, __LINE__,
847 skmsg, skmsg->length);
e67f86b3
AB		 848 skd_send_fitmsg(skdev, skmsg);
849 } else {
850 /*
851 * The FIT msg is empty. It means we got started
852 * on the msg, but the requests were rejected.
853 */
854 skmsg->state = SKD_MSG_STATE_IDLE;
855 skmsg->id += SKD_ID_INCR;
856 skmsg->next = skdev->skmsg_free_list;
857 skdev->skmsg_free_list = skmsg;
858 }
859 skmsg = NULL;
860 fmh = NULL;
861 }
862
863 /*
864 * If req is non-NULL it means there is something to do but
865 * we are out of a resource.
866 */
fcd37eb3 867 if (req)
6a5ec65b 868 blk_stop_queue(skdev->queue);
e67f86b3
AB		 869}
870
871static void skd_end_request_blk(struct skd_device *skdev,
872 struct skd_request_context *skreq, int error)
873{
874 struct request *req = skreq->req;
875 unsigned int io_flags = req->cmd_flags;
876
877 if ((io_flags & REQ_DISCARD) &&
878 (skreq->discard_page == 1)) {
2e44b427 879 pr_debug("%s:%s:%d skd_end_request_blk, free the page!",
880 skdev->name, __func__, __LINE__);
e67f86b3
AB		 881 free_page((unsigned long)req->buffer);
882 req->buffer = NULL;
883 }
884
885 if (unlikely(error)) {
886 struct request *req = skreq->req;
887 char *cmd = (rq_data_dir(req) == READ) ? "read" : "write";
888 u32 lba = (u32)blk_rq_pos(req);
889 u32 count = blk_rq_sectors(req);
890
891 pr_err("(%s): Error cmd=%s sect=%u count=%u id=0x%x\n",
892 skd_name(skdev), cmd, lba, count, skreq->id);
893 } else
2e44b427 894 pr_debug("%s:%s:%d id=0x%x error=%d\n",
895 skdev->name, __func__, __LINE__, skreq->id, error);
e67f86b3
AB		 896
897 __blk_end_request_all(skreq->req, error);
898}
899
fcd37eb3 900static int skd_preop_sg_list(struct skd_device *skdev,
e67f86b3
AB		 901 struct skd_request_context *skreq)
902{
903 struct request *req = skreq->req;
904 int writing = skreq->sg_data_dir == SKD_DATA_DIR_HOST_TO_CARD;
905 int pci_dir = writing ? PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE;
906 struct scatterlist *sg = &skreq->sg[0];
907 int n_sg;
908 int i;
909
910 skreq->sg_byte_count = 0;
911
912 /* SKD_ASSERT(skreq->sg_data_dir == SKD_DATA_DIR_HOST_TO_CARD ||
913 skreq->sg_data_dir == SKD_DATA_DIR_CARD_TO_HOST); */
914
915 n_sg = blk_rq_map_sg(skdev->queue, req, sg);
916 if (n_sg <= 0)
917 return -EINVAL;
918
919 /*
920 * Map scatterlist to PCI bus addresses.
921 * Note PCI might change the number of entries.
922 */
923 n_sg = pci_map_sg(skdev->pdev, sg, n_sg, pci_dir);
924 if (n_sg <= 0)
925 return -EINVAL;
926
927 SKD_ASSERT(n_sg <= skdev->sgs_per_request);
928
929 skreq->n_sg = n_sg;
930
931 for (i = 0; i < n_sg; i++) {
932 struct fit_sg_descriptor *sgd = &skreq->sksg_list[i];
933 u32 cnt = sg_dma_len(&sg[i]);
934 uint64_t dma_addr = sg_dma_address(&sg[i]);
935
936 sgd->control = FIT_SGD_CONTROL_NOT_LAST;
937 sgd->byte_count = cnt;
938 skreq->sg_byte_count += cnt;
939 sgd->host_side_addr = dma_addr;
940 sgd->dev_side_addr = 0;
941 }
942
943 skreq->sksg_list[n_sg - 1].next_desc_ptr = 0LL;
944 skreq->sksg_list[n_sg - 1].control = FIT_SGD_CONTROL_LAST;
945
946 if (unlikely(skdev->dbg_level > 1)) {
2e44b427 947 pr_debug("%s:%s:%d skreq=%x sksg_list=%p sksg_dma=%llx\n",
948 skdev->name, __func__, __LINE__,
949 skreq->id, skreq->sksg_list, skreq->sksg_dma_address);
e67f86b3
AB		 950 for (i = 0; i < n_sg; i++) {
951 struct fit_sg_descriptor *sgd = &skreq->sksg_list[i];
2e44b427 952 pr_debug("%s:%s:%d sg[%d] count=%u ctrl=0x%x "
953 "addr=0x%llx next=0x%llx\n",
954 skdev->name, __func__, __LINE__,
955 i, sgd->byte_count, sgd->control,
956 sgd->host_side_addr, sgd->next_desc_ptr);
e67f86b3
AB		 957 }
958 }
959
960 return 0;
961}
962
fcd37eb3 963static void skd_postop_sg_list(struct skd_device *skdev,
e67f86b3
AB		 964 struct skd_request_context *skreq)
965{
966 int writing = skreq->sg_data_dir == SKD_DATA_DIR_HOST_TO_CARD;
967 int pci_dir = writing ? PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE;
968
969 /*
970 * restore the next ptr for next IO request so we
971 * don't have to set it every time.
972 */
973 skreq->sksg_list[skreq->n_sg - 1].next_desc_ptr =
974 skreq->sksg_dma_address +
975 ((skreq->n_sg) * sizeof(struct fit_sg_descriptor));
976 pci_unmap_sg(skdev->pdev, &skreq->sg[0], skreq->n_sg, pci_dir);
977}
978
e67f86b3
AB		 979static void skd_end_request(struct skd_device *skdev,
980 struct skd_request_context *skreq, int error)
981{
fcd37eb3 982 skd_end_request_blk(skdev, skreq, error);
e67f86b3
AB		 983}
984
985static void skd_request_fn_not_online(struct request_queue *q)
986{
987 struct skd_device *skdev = q->queuedata;
988 int error;
989
990 SKD_ASSERT(skdev->state != SKD_DRVR_STATE_ONLINE);
991
992 skd_log_skdev(skdev, "req_not_online");
993 switch (skdev->state) {
994 case SKD_DRVR_STATE_PAUSING:
995 case SKD_DRVR_STATE_PAUSED:
996 case SKD_DRVR_STATE_STARTING:
997 case SKD_DRVR_STATE_RESTARTING:
998 case SKD_DRVR_STATE_WAIT_BOOT:
999 /* In case of starting, we haven't started the queue,
1000 * so we can't get here... but requests are
1001 * possibly hanging out waiting for us because we
1002 * reported the dev/skd0 already. They'll wait
1003 * forever if connect doesn't complete.
1004 * What to do??? delay dev/skd0 ??
1005 */
1006 case SKD_DRVR_STATE_BUSY:
1007 case SKD_DRVR_STATE_BUSY_IMMINENT:
1008 case SKD_DRVR_STATE_BUSY_ERASE:
1009 case SKD_DRVR_STATE_DRAINING_TIMEOUT:
1010 return;
1011
1012 case SKD_DRVR_STATE_BUSY_SANITIZE:
1013 case SKD_DRVR_STATE_STOPPING:
1014 case SKD_DRVR_STATE_SYNCING:
1015 case SKD_DRVR_STATE_FAULT:
1016 case SKD_DRVR_STATE_DISAPPEARED:
1017 default:
1018 error = -EIO;
1019 break;
1020 }
1021
1022 /* If we get here, terminate all pending block requeusts
1023 * with EIO and any scsi pass thru with appropriate sense
1024 */
1025
1026 skd_fail_all_pending(skdev);
1027}
1028
1029/*
1030 *****************************************************************************
1031 * TIMER
1032 *****************************************************************************
1033 */
1034
1035static void skd_timer_tick_not_online(struct skd_device *skdev);
1036
1037static void skd_timer_tick(ulong arg)
1038{
1039 struct skd_device *skdev = (struct skd_device *)arg;
1040
1041 u32 timo_slot;
1042 u32 overdue_timestamp;
1043 unsigned long reqflags;
1044 u32 state;
1045
1046 if (skdev->state == SKD_DRVR_STATE_FAULT)
1047 /* The driver has declared fault, and we want it to
1048 * stay that way until driver is reloaded.
1049 */
1050 return;
1051
1052 spin_lock_irqsave(&skdev->lock, reqflags);
1053
1054 state = SKD_READL(skdev, FIT_STATUS);
1055 state &= FIT_SR_DRIVE_STATE_MASK;
1056 if (state != skdev->drive_state)
1057 skd_isr_fwstate(skdev);
1058
1059 if (skdev->state != SKD_DRVR_STATE_ONLINE) {
1060 skd_timer_tick_not_online(skdev);
1061 goto timer_func_out;
1062 }
1063 skdev->timeout_stamp++;
1064 timo_slot = skdev->timeout_stamp & SKD_TIMEOUT_SLOT_MASK;
1065
1066 /*
1067 * All requests that happened during the previous use of
1068 * this slot should be done by now. The previous use was
1069 * over 7 seconds ago.
1070 */
1071 if (skdev->timeout_slot[timo_slot] == 0)
1072 goto timer_func_out;
1073
1074 /* Something is overdue */
1075 overdue_timestamp = skdev->timeout_stamp - SKD_N_TIMEOUT_SLOT;
1076
2e44b427 1077 pr_debug("%s:%s:%d found %d timeouts, draining busy=%d\n",
1078 skdev->name, __func__, __LINE__,
1079 skdev->timeout_slot[timo_slot], skdev->in_flight);
e67f86b3
AB		 1080 pr_err("(%s): Overdue IOs (%d), busy %d\n",
1081 skd_name(skdev), skdev->timeout_slot[timo_slot],
1082 skdev->in_flight);
1083
1084 skdev->timer_countdown = SKD_DRAINING_TIMO;
1085 skdev->state = SKD_DRVR_STATE_DRAINING_TIMEOUT;
1086 skdev->timo_slot = timo_slot;
6a5ec65b 1087 blk_stop_queue(skdev->queue);
e67f86b3
AB		 1088
1089timer_func_out:
1090 mod_timer(&skdev->timer, (jiffies + HZ));
1091
1092 spin_unlock_irqrestore(&skdev->lock, reqflags);
1093}
1094
1095static void skd_timer_tick_not_online(struct skd_device *skdev)
1096{
1097 switch (skdev->state) {
1098 case SKD_DRVR_STATE_IDLE:
1099 case SKD_DRVR_STATE_LOAD:
1100 break;
1101 case SKD_DRVR_STATE_BUSY_SANITIZE:
2e44b427 1102 pr_debug("%s:%s:%d drive busy sanitize[%x], driver[%x]\n",
1103 skdev->name, __func__, __LINE__,
1104 skdev->drive_state, skdev->state);
e67f86b3
AB		 1105 /* If we've been in sanitize for 3 seconds, we figure we're not
1106 * going to get anymore completions, so recover requests now
1107 */
1108 if (skdev->timer_countdown > 0) {
1109 skdev->timer_countdown--;
1110 return;
1111 }
1112 skd_recover_requests(skdev, 0);
1113 break;
1114
1115 case SKD_DRVR_STATE_BUSY:
1116 case SKD_DRVR_STATE_BUSY_IMMINENT:
1117 case SKD_DRVR_STATE_BUSY_ERASE:
2e44b427 1118 pr_debug("%s:%s:%d busy[%x], countdown=%d\n",
1119 skdev->name, __func__, __LINE__,
1120 skdev->state, skdev->timer_countdown);
e67f86b3
AB		 1121 if (skdev->timer_countdown > 0) {
1122 skdev->timer_countdown--;
1123 return;
1124 }
2e44b427 1125 pr_debug("%s:%s:%d busy[%x], timedout=%d, restarting device.",
1126 skdev->name, __func__, __LINE__,
1127 skdev->state, skdev->timer_countdown);
e67f86b3
AB		 1128 skd_restart_device(skdev);
1129 break;
1130
1131 case SKD_DRVR_STATE_WAIT_BOOT:
1132 case SKD_DRVR_STATE_STARTING:
1133 if (skdev->timer_countdown > 0) {
1134 skdev->timer_countdown--;
1135 return;
1136 }
1137 /* For now, we fault the drive. Could attempt resets to
1138 * revcover at some point. */
1139 skdev->state = SKD_DRVR_STATE_FAULT;
1140
1141 pr_err("(%s): DriveFault Connect Timeout (%x)\n",
1142 skd_name(skdev), skdev->drive_state);
1143
1144 /*start the queue so we can respond with error to requests */
1145 /* wakeup anyone waiting for startup complete */
6a5ec65b 1146 blk_start_queue(skdev->queue);
e67f86b3
AB		 1147 skdev->gendisk_on = -1;
1148 wake_up_interruptible(&skdev->waitq);
1149 break;
1150
1151 case SKD_DRVR_STATE_ONLINE:
1152 /* shouldn't get here. */
1153 break;
1154
1155 case SKD_DRVR_STATE_PAUSING:
1156 case SKD_DRVR_STATE_PAUSED:
1157 break;
1158
1159 case SKD_DRVR_STATE_DRAINING_TIMEOUT:
2e44b427 1160 pr_debug("%s:%s:%d "
1161 "draining busy [%d] tick[%d] qdb[%d] tmls[%d]\n",
1162 skdev->name, __func__, __LINE__,
1163 skdev->timo_slot,
1164 skdev->timer_countdown,
1165 skdev->in_flight,
1166 skdev->timeout_slot[skdev->timo_slot]);
e67f86b3
AB		 1167 /* if the slot has cleared we can let the I/O continue */
1168 if (skdev->timeout_slot[skdev->timo_slot] == 0) {
2e44b427 1169 pr_debug("%s:%s:%d Slot drained, starting queue.\n",
1170 skdev->name, __func__, __LINE__);
e67f86b3 1171 skdev->state = SKD_DRVR_STATE_ONLINE;
6a5ec65b 1172 blk_start_queue(skdev->queue);
e67f86b3
AB		 1173 return;
1174 }
1175 if (skdev->timer_countdown > 0) {
1176 skdev->timer_countdown--;
1177 return;
1178 }
1179 skd_restart_device(skdev);
1180 break;
1181
1182 case SKD_DRVR_STATE_RESTARTING:
1183 if (skdev->timer_countdown > 0) {
1184 skdev->timer_countdown--;
1185 return;
1186 }
1187 /* For now, we fault the drive. Could attempt resets to
1188 * revcover at some point. */
1189 skdev->state = SKD_DRVR_STATE_FAULT;
1190 pr_err("(%s): DriveFault Reconnect Timeout (%x)\n",
1191 skd_name(skdev), skdev->drive_state);
1192
1193 /*
1194 * Recovering does two things:
1195 * 1. completes IO with error
1196 * 2. reclaims dma resources
1197 * When is it safe to recover requests?
1198 * - if the drive state is faulted
1199 * - if the state is still soft reset after out timeout
1200 * - if the drive registers are dead (state = FF)
1201 * If it is "unsafe", we still need to recover, so we will
1202 * disable pci bus mastering and disable our interrupts.
1203 */
1204
1205 if ((skdev->drive_state == FIT_SR_DRIVE_SOFT_RESET) ||
1206 (skdev->drive_state == FIT_SR_DRIVE_FAULT) ||
1207 (skdev->drive_state == FIT_SR_DRIVE_STATE_MASK))
1208 /* It never came out of soft reset. Try to
1209 * recover the requests and then let them
1210 * fail. This is to mitigate hung processes. */
1211 skd_recover_requests(skdev, 0);
1212 else {
1213 pr_err("(%s): Disable BusMaster (%x)\n",
1214 skd_name(skdev), skdev->drive_state);
1215 pci_disable_device(skdev->pdev);
1216 skd_disable_interrupts(skdev);
1217 skd_recover_requests(skdev, 0);
1218 }
1219
1220 /*start the queue so we can respond with error to requests */
1221 /* wakeup anyone waiting for startup complete */
6a5ec65b 1222 blk_start_queue(skdev->queue);
e67f86b3
AB		 1223 skdev->gendisk_on = -1;
1224 wake_up_interruptible(&skdev->waitq);
1225 break;
1226
1227 case SKD_DRVR_STATE_RESUMING:
1228 case SKD_DRVR_STATE_STOPPING:
1229 case SKD_DRVR_STATE_SYNCING:
1230 case SKD_DRVR_STATE_FAULT:
1231 case SKD_DRVR_STATE_DISAPPEARED:
1232 default:
1233 break;
1234 }
1235}
1236
1237static int skd_start_timer(struct skd_device *skdev)
1238{
1239 int rc;
1240
1241 init_timer(&skdev->timer);
1242 setup_timer(&skdev->timer, skd_timer_tick, (ulong)skdev);
1243
1244 rc = mod_timer(&skdev->timer, (jiffies + HZ));
1245 if (rc)
1246 pr_err("%s: failed to start timer %d\n",
1247 __func__, rc);
1248 return rc;
1249}
1250
1251static void skd_kill_timer(struct skd_device *skdev)
1252{
1253 del_timer_sync(&skdev->timer);
1254}
1255
1256/*
1257 *****************************************************************************
1258 * IOCTL
1259 *****************************************************************************
1260 */
1261static int skd_ioctl_sg_io(struct skd_device *skdev,
1262 fmode_t mode, void __user *argp);
1263static int skd_sg_io_get_and_check_args(struct skd_device *skdev,
1264 struct skd_sg_io *sksgio);
1265static int skd_sg_io_obtain_skspcl(struct skd_device *skdev,
1266 struct skd_sg_io *sksgio);
1267static int skd_sg_io_prep_buffering(struct skd_device *skdev,
1268 struct skd_sg_io *sksgio);
1269static int skd_sg_io_copy_buffer(struct skd_device *skdev,
1270 struct skd_sg_io *sksgio, int dxfer_dir);
1271static int skd_sg_io_send_fitmsg(struct skd_device *skdev,
1272 struct skd_sg_io *sksgio);
1273static int skd_sg_io_await(struct skd_device *skdev, struct skd_sg_io *sksgio);
1274static int skd_sg_io_release_skspcl(struct skd_device *skdev,
1275 struct skd_sg_io *sksgio);
1276static int skd_sg_io_put_status(struct skd_device *skdev,
1277 struct skd_sg_io *sksgio);
1278
1279static void skd_complete_special(struct skd_device *skdev,
1280 volatile struct fit_completion_entry_v1
1281 *skcomp,
1282 volatile struct fit_comp_error_info *skerr,
1283 struct skd_special_context *skspcl);
1284
1285static int skd_bdev_ioctl(struct block_device *bdev, fmode_t mode,
1286 uint cmd_in, ulong arg)
1287{
1288 int rc = 0;
1289 struct gendisk *disk = bdev->bd_disk;
1290 struct skd_device *skdev = disk->private_data;
1291 void __user *p = (void *)arg;
1292
2e44b427 1293 pr_debug("%s:%s:%d %s: CMD[%s] ioctl mode 0x%x, cmd 0x%x arg %0lx\n",
1294 skdev->name, __func__, __LINE__,
1295 disk->disk_name, current->comm, mode, cmd_in, arg);
e67f86b3
AB		 1296
1297 if (!capable(CAP_SYS_ADMIN))
1298 return -EPERM;
1299
1300 switch (cmd_in) {
1301 case SG_SET_TIMEOUT:
1302 case SG_GET_TIMEOUT:
1303 case SG_GET_VERSION_NUM:
1304 rc = scsi_cmd_ioctl(disk->queue, disk, mode, cmd_in, p);
1305 break;
1306 case SG_IO:
1307 rc = skd_ioctl_sg_io(skdev, mode, p);
1308 break;
1309
1310 default:
1311 rc = -ENOTTY;
1312 break;
1313 }
1314
2e44b427 1315 pr_debug("%s:%s:%d %s: completion rc %d\n",
1316 skdev->name, __func__, __LINE__, disk->disk_name, rc);
e67f86b3
AB		 1317 return rc;
1318}
1319
1320static int skd_ioctl_sg_io(struct skd_device *skdev, fmode_t mode,
1321 void __user *argp)
1322{
1323 int rc;
1324 struct skd_sg_io sksgio;
1325
1326 memset(&sksgio, 0, sizeof(sksgio));
1327 sksgio.mode = mode;
1328 sksgio.argp = argp;
1329 sksgio.iov = &sksgio.no_iov_iov;
1330
1331 switch (skdev->state) {
1332 case SKD_DRVR_STATE_ONLINE:
1333 case SKD_DRVR_STATE_BUSY_IMMINENT:
1334 break;
1335
1336 default:
2e44b427 1337 pr_debug("%s:%s:%d drive not online\n",
1338 skdev->name, __func__, __LINE__);
e67f86b3
AB		 1339 rc = -ENXIO;
1340 goto out;
1341 }
1342
f721bb0d
AB		 1343 rc = skd_sg_io_get_and_check_args(skdev, &sksgio);
1344 if (rc)
1345 goto out;
1346
1347 rc = skd_sg_io_obtain_skspcl(skdev, &sksgio);
1348 if (rc)
1349 goto out;
1350
1351 rc = skd_sg_io_prep_buffering(skdev, &sksgio);
1352 if (rc)
1353 goto out;
1354
1355 rc = skd_sg_io_copy_buffer(skdev, &sksgio, SG_DXFER_TO_DEV);
1356 if (rc)
e67f86b3
AB		 1357 goto out;
1358
f721bb0d
AB		 1359 rc = skd_sg_io_send_fitmsg(skdev, &sksgio);
1360 if (rc)
e67f86b3
AB		 1361 goto out;
1362
f721bb0d
AB		 1363 rc = skd_sg_io_await(skdev, &sksgio);
1364 if (rc)
1365 goto out;
1366
1367 rc = skd_sg_io_copy_buffer(skdev, &sksgio, SG_DXFER_FROM_DEV);
1368 if (rc)
1369 goto out;
1370
1371 rc = skd_sg_io_put_status(skdev, &sksgio);
1372 if (rc)
e67f86b3
AB		 1373 goto out;
1374
1375 rc = 0;
1376
1377out:
1378 skd_sg_io_release_skspcl(skdev, &sksgio);
1379
1380 if (sksgio.iov != NULL && sksgio.iov != &sksgio.no_iov_iov)
1381 kfree(sksgio.iov);
1382 return rc;
1383}
1384
1385static int skd_sg_io_get_and_check_args(struct skd_device *skdev,
1386 struct skd_sg_io *sksgio)
1387{
1388 struct sg_io_hdr *sgp = &sksgio->sg;
1389 int i, acc;
1390
1391 if (!access_ok(VERIFY_WRITE, sksgio->argp, sizeof(sg_io_hdr_t))) {
2e44b427 1392 pr_debug("%s:%s:%d access sg failed %p\n",
1393 skdev->name, __func__, __LINE__, sksgio->argp);
e67f86b3
AB		 1394 return -EFAULT;
1395 }
1396
1397 if (__copy_from_user(sgp, sksgio->argp, sizeof(sg_io_hdr_t))) {
2e44b427 1398 pr_debug("%s:%s:%d copy_from_user sg failed %p\n",
1399 skdev->name, __func__, __LINE__, sksgio->argp);
e67f86b3
AB		 1400 return -EFAULT;
1401 }
1402
1403 if (sgp->interface_id != SG_INTERFACE_ID_ORIG) {
2e44b427 1404 pr_debug("%s:%s:%d interface_id invalid 0x%x\n",
1405 skdev->name, __func__, __LINE__, sgp->interface_id);
e67f86b3
AB		 1406 return -EINVAL;
1407 }
1408
1409 if (sgp->cmd_len > sizeof(sksgio->cdb)) {
2e44b427 1410 pr_debug("%s:%s:%d cmd_len invalid %d\n",
1411 skdev->name, __func__, __LINE__, sgp->cmd_len);
e67f86b3
AB		 1412 return -EINVAL;
1413 }
1414
1415 if (sgp->iovec_count > 256) {
2e44b427 1416 pr_debug("%s:%s:%d iovec_count invalid %d\n",
1417 skdev->name, __func__, __LINE__, sgp->iovec_count);
e67f86b3
AB		 1418 return -EINVAL;
1419 }
1420
1421 if (sgp->dxfer_len > (PAGE_SIZE * SKD_N_SG_PER_SPECIAL)) {
2e44b427 1422 pr_debug("%s:%s:%d dxfer_len invalid %d\n",
1423 skdev->name, __func__, __LINE__, sgp->dxfer_len);
e67f86b3
AB		 1424 return -EINVAL;
1425 }
1426
1427 switch (sgp->dxfer_direction) {
1428 case SG_DXFER_NONE:
1429 acc = -1;
1430 break;
1431
1432 case SG_DXFER_TO_DEV:
1433 acc = VERIFY_READ;
1434 break;
1435
1436 case SG_DXFER_FROM_DEV:
1437 case SG_DXFER_TO_FROM_DEV:
1438 acc = VERIFY_WRITE;
1439 break;
1440
1441 default:
2e44b427 1442 pr_debug("%s:%s:%d dxfer_dir invalid %d\n",
1443 skdev->name, __func__, __LINE__, sgp->dxfer_direction);
e67f86b3
AB		 1444 return -EINVAL;
1445 }
1446
1447 if (copy_from_user(sksgio->cdb, sgp->cmdp, sgp->cmd_len)) {
2e44b427 1448 pr_debug("%s:%s:%d copy_from_user cmdp failed %p\n",
1449 skdev->name, __func__, __LINE__, sgp->cmdp);
e67f86b3
AB		 1450 return -EFAULT;
1451 }
1452
1453 if (sgp->mx_sb_len != 0) {
1454 if (!access_ok(VERIFY_WRITE, sgp->sbp, sgp->mx_sb_len)) {
2e44b427 1455 pr_debug("%s:%s:%d access sbp failed %p\n",
1456 skdev->name, __func__, __LINE__, sgp->sbp);
e67f86b3
AB		 1457 return -EFAULT;
1458 }
1459 }
1460
1461 if (sgp->iovec_count == 0) {
1462 sksgio->iov[0].iov_base = sgp->dxferp;
1463 sksgio->iov[0].iov_len = sgp->dxfer_len;
1464 sksgio->iovcnt = 1;
1465 sksgio->dxfer_len = sgp->dxfer_len;
1466 } else {
1467 struct sg_iovec *iov;
1468 uint nbytes = sizeof(*iov) * sgp->iovec_count;
1469 size_t iov_data_len;
1470
1471 iov = kmalloc(nbytes, GFP_KERNEL);
1472 if (iov == NULL) {
2e44b427 1473 pr_debug("%s:%s:%d alloc iovec failed %d\n",
1474 skdev->name, __func__, __LINE__,
1475 sgp->iovec_count);
e67f86b3
AB		 1476 return -ENOMEM;
1477 }
1478 sksgio->iov = iov;
1479 sksgio->iovcnt = sgp->iovec_count;
1480
1481 if (copy_from_user(iov, sgp->dxferp, nbytes)) {
2e44b427 1482 pr_debug("%s:%s:%d copy_from_user iovec failed %p\n",
1483 skdev->name, __func__, __LINE__, sgp->dxferp);
e67f86b3
AB		 1484 return -EFAULT;
1485 }
1486
1487 /*
1488 * Sum up the vecs, making sure they don't overflow
1489 */
1490 iov_data_len = 0;
1491 for (i = 0; i < sgp->iovec_count; i++) {
1492 if (iov_data_len + iov[i].iov_len < iov_data_len)
1493 return -EINVAL;
1494 iov_data_len += iov[i].iov_len;
1495 }
1496
1497 /* SG_IO howto says that the shorter of the two wins */
1498 if (sgp->dxfer_len < iov_data_len) {
1499 sksgio->iovcnt = iov_shorten((struct iovec *)iov,
1500 sgp->iovec_count,
1501 sgp->dxfer_len);
1502 sksgio->dxfer_len = sgp->dxfer_len;
1503 } else
1504 sksgio->dxfer_len = iov_data_len;
1505 }
1506
1507 if (sgp->dxfer_direction != SG_DXFER_NONE) {
1508 struct sg_iovec *iov = sksgio->iov;
1509 for (i = 0; i < sksgio->iovcnt; i++, iov++) {
1510 if (!access_ok(acc, iov->iov_base, iov->iov_len)) {
2e44b427 1511 pr_debug("%s:%s:%d access data failed %p/%d\n",
1512 skdev->name, __func__, __LINE__,
1513 iov->iov_base, (int)iov->iov_len);
e67f86b3
AB		 1514 return -EFAULT;
1515 }
1516 }
1517 }
1518
1519 return 0;
1520}
1521
1522static int skd_sg_io_obtain_skspcl(struct skd_device *skdev,
1523 struct skd_sg_io *sksgio)
1524{
1525 struct skd_special_context *skspcl = NULL;
1526 int rc;
1527
1528 for (;; ) {
1529 ulong flags;
1530
1531 spin_lock_irqsave(&skdev->lock, flags);
1532 skspcl = skdev->skspcl_free_list;
1533 if (skspcl != NULL) {
1534 skdev->skspcl_free_list =
1535 (struct skd_special_context *)skspcl->req.next;
1536 skspcl->req.id += SKD_ID_INCR;
1537 skspcl->req.state = SKD_REQ_STATE_SETUP;
1538 skspcl->orphaned = 0;
1539 skspcl->req.n_sg = 0;
1540 }
1541 spin_unlock_irqrestore(&skdev->lock, flags);
1542
1543 if (skspcl != NULL) {
1544 rc = 0;
1545 break;
1546 }
1547
2e44b427 1548 pr_debug("%s:%s:%d blocking\n",
1549 skdev->name, __func__, __LINE__);
e67f86b3
AB		 1550
1551 rc = wait_event_interruptible_timeout(
1552 skdev->waitq,
1553 (skdev->skspcl_free_list != NULL),
1554 msecs_to_jiffies(sksgio->sg.timeout));
1555
2e44b427 1556 pr_debug("%s:%s:%d unblocking, rc=%d\n",
1557 skdev->name, __func__, __LINE__, rc);
e67f86b3
AB		 1558
1559 if (rc <= 0) {
1560 if (rc == 0)
1561 rc = -ETIMEDOUT;
1562 else
1563 rc = -EINTR;
1564 break;
1565 }
1566 /*
1567 * If we get here rc > 0 meaning the timeout to
1568 * wait_event_interruptible_timeout() had time left, hence the
1569 * sought event -- non-empty free list -- happened.
1570 * Retry the allocation.
1571 */
1572 }
1573 sksgio->skspcl = skspcl;
1574
1575 return rc;
1576}
1577
1578static int skd_skreq_prep_buffering(struct skd_device *skdev,
1579 struct skd_request_context *skreq,
1580 u32 dxfer_len)
1581{
1582 u32 resid = dxfer_len;
1583
1584 /*
1585 * The DMA engine must have aligned addresses and byte counts.
1586 */
1587 resid += (-resid) & 3;
1588 skreq->sg_byte_count = resid;
1589
1590 skreq->n_sg = 0;
1591
1592 while (resid > 0) {
1593 u32 nbytes = PAGE_SIZE;
1594 u32 ix = skreq->n_sg;
1595 struct scatterlist *sg = &skreq->sg[ix];
1596 struct fit_sg_descriptor *sksg = &skreq->sksg_list[ix];
1597 struct page *page;
1598
1599 if (nbytes > resid)
1600 nbytes = resid;
1601
1602 page = alloc_page(GFP_KERNEL);
1603 if (page == NULL)
1604 return -ENOMEM;
1605
1606 sg_set_page(sg, page, nbytes, 0);
1607
1608 /* TODO: This should be going through a pci_???()
1609 * routine to do proper mapping. */
1610 sksg->control = FIT_SGD_CONTROL_NOT_LAST;
1611 sksg->byte_count = nbytes;
1612
1613 sksg->host_side_addr = sg_phys(sg);
1614
1615 sksg->dev_side_addr = 0;
1616 sksg->next_desc_ptr = skreq->sksg_dma_address +
1617 (ix + 1) * sizeof(*sksg);
1618
1619 skreq->n_sg++;
1620 resid -= nbytes;
1621 }
1622
1623 if (skreq->n_sg > 0) {
1624 u32 ix = skreq->n_sg - 1;
1625 struct fit_sg_descriptor *sksg = &skreq->sksg_list[ix];
1626
1627 sksg->control = FIT_SGD_CONTROL_LAST;
1628 sksg->next_desc_ptr = 0;
1629 }
1630
1631 if (unlikely(skdev->dbg_level > 1)) {
1632 u32 i;
1633
2e44b427 1634 pr_debug("%s:%s:%d skreq=%x sksg_list=%p sksg_dma=%llx\n",
1635 skdev->name, __func__, __LINE__,
1636 skreq->id, skreq->sksg_list, skreq->sksg_dma_address);
e67f86b3
AB		 1637 for (i = 0; i < skreq->n_sg; i++) {
1638 struct fit_sg_descriptor *sgd = &skreq->sksg_list[i];
1639
2e44b427 1640 pr_debug("%s:%s:%d sg[%d] count=%u ctrl=0x%x "
1641 "addr=0x%llx next=0x%llx\n",
1642 skdev->name, __func__, __LINE__,
1643 i, sgd->byte_count, sgd->control,
1644 sgd->host_side_addr, sgd->next_desc_ptr);
e67f86b3
AB		 1645 }
1646 }
1647
1648 return 0;
1649}
1650
1651static int skd_sg_io_prep_buffering(struct skd_device *skdev,
1652 struct skd_sg_io *sksgio)
1653{
1654 struct skd_special_context *skspcl = sksgio->skspcl;
1655 struct skd_request_context *skreq = &skspcl->req;
1656 u32 dxfer_len = sksgio->dxfer_len;
1657 int rc;
1658
1659 rc = skd_skreq_prep_buffering(skdev, skreq, dxfer_len);
1660 /*
1661 * Eventually, errors or not, skd_release_special() is called
1662 * to recover allocations including partial allocations.
1663 */
1664 return rc;
1665}
1666
1667static int skd_sg_io_copy_buffer(struct skd_device *skdev,
1668 struct skd_sg_io *sksgio, int dxfer_dir)
1669{
1670 struct skd_special_context *skspcl = sksgio->skspcl;
1671 u32 iov_ix = 0;
1672 struct sg_iovec curiov;
1673 u32 sksg_ix = 0;
1674 u8 *bufp = NULL;
1675 u32 buf_len = 0;
1676 u32 resid = sksgio->dxfer_len;
1677 int rc;
1678
1679 curiov.iov_len = 0;
1680 curiov.iov_base = NULL;
1681
1682 if (dxfer_dir != sksgio->sg.dxfer_direction) {
1683 if (dxfer_dir != SG_DXFER_TO_DEV ||
1684 sksgio->sg.dxfer_direction != SG_DXFER_TO_FROM_DEV)
1685 return 0;
1686 }
1687
1688 while (resid > 0) {
1689 u32 nbytes = PAGE_SIZE;
1690
1691 if (curiov.iov_len == 0) {
1692 curiov = sksgio->iov[iov_ix++];
1693 continue;
1694 }
1695
1696 if (buf_len == 0) {
1697 struct page *page;
1698 page = sg_page(&skspcl->req.sg[sksg_ix++]);
1699 bufp = page_address(page);
1700 buf_len = PAGE_SIZE;
1701 }
1702
1703 nbytes = min_t(u32, nbytes, resid);
1704 nbytes = min_t(u32, nbytes, curiov.iov_len);
1705 nbytes = min_t(u32, nbytes, buf_len);
1706
1707 if (dxfer_dir == SG_DXFER_TO_DEV)
1708 rc = __copy_from_user(bufp, curiov.iov_base, nbytes);
1709 else
1710 rc = __copy_to_user(curiov.iov_base, bufp, nbytes);
1711
1712 if (rc)
1713 return -EFAULT;
1714
1715 resid -= nbytes;
1716 curiov.iov_len -= nbytes;
1717 curiov.iov_base += nbytes;
1718 buf_len -= nbytes;
1719 }
1720
1721 return 0;
1722}
1723
1724static int skd_sg_io_send_fitmsg(struct skd_device *skdev,
1725 struct skd_sg_io *sksgio)
1726{
1727 struct skd_special_context *skspcl = sksgio->skspcl;
1728 struct fit_msg_hdr *fmh = (struct fit_msg_hdr *)skspcl->msg_buf;
1729 struct skd_scsi_request *scsi_req = (struct skd_scsi_request *)&fmh[1];
1730
1731 memset(skspcl->msg_buf, 0, SKD_N_SPECIAL_FITMSG_BYTES);
1732
1733 /* Initialize the FIT msg header */
1734 fmh->protocol_id = FIT_PROTOCOL_ID_SOFIT;
1735 fmh->num_protocol_cmds_coalesced = 1;
1736
1737 /* Initialize the SCSI request */
1738 if (sksgio->sg.dxfer_direction != SG_DXFER_NONE)
1739 scsi_req->hdr.sg_list_dma_address =
1740 cpu_to_be64(skspcl->req.sksg_dma_address);
1741 scsi_req->hdr.tag = skspcl->req.id;
1742 scsi_req->hdr.sg_list_len_bytes =
1743 cpu_to_be32(skspcl->req.sg_byte_count);
1744 memcpy(scsi_req->cdb, sksgio->cdb, sizeof(scsi_req->cdb));
1745
1746 skspcl->req.state = SKD_REQ_STATE_BUSY;
1747 skd_send_special_fitmsg(skdev, skspcl);
1748
1749 return 0;
1750}
1751
1752static int skd_sg_io_await(struct skd_device *skdev, struct skd_sg_io *sksgio)
1753{
1754 unsigned long flags;
1755 int rc;
1756
1757 rc = wait_event_interruptible_timeout(skdev->waitq,
1758 (sksgio->skspcl->req.state !=
1759 SKD_REQ_STATE_BUSY),
1760 msecs_to_jiffies(sksgio->sg.
1761 timeout));
1762
1763 spin_lock_irqsave(&skdev->lock, flags);
1764
1765 if (sksgio->skspcl->req.state == SKD_REQ_STATE_ABORTED) {
2e44b427 1766 pr_debug("%s:%s:%d skspcl %p aborted\n",
1767 skdev->name, __func__, __LINE__, sksgio->skspcl);
e67f86b3
AB		 1768
1769 /* Build check cond, sense and let command finish. */
1770 /* For a timeout, we must fabricate completion and sense
1771 * data to complete the command */
1772 sksgio->skspcl->req.completion.status =
1773 SAM_STAT_CHECK_CONDITION;
1774
1775 memset(&sksgio->skspcl->req.err_info, 0,
1776 sizeof(sksgio->skspcl->req.err_info));
1777 sksgio->skspcl->req.err_info.type = 0x70;
1778 sksgio->skspcl->req.err_info.key = ABORTED_COMMAND;
1779 sksgio->skspcl->req.err_info.code = 0x44;
1780 sksgio->skspcl->req.err_info.qual = 0;
1781 rc = 0;
1782 } else if (sksgio->skspcl->req.state != SKD_REQ_STATE_BUSY)
1783 /* No longer on the adapter. We finish. */
1784 rc = 0;
1785 else {
1786 /* Something's gone wrong. Still busy. Timeout or
1787 * user interrupted (control-C). Mark as an orphan
1788 * so it will be disposed when completed. */
1789 sksgio->skspcl->orphaned = 1;
1790 sksgio->skspcl = NULL;
1791 if (rc == 0) {
2e44b427 1792 pr_debug("%s:%s:%d timed out %p (%u ms)\n",
1793 skdev->name, __func__, __LINE__,
1794 sksgio, sksgio->sg.timeout);
e67f86b3
AB		 1795 rc = -ETIMEDOUT;
1796 } else {
2e44b427 1797 pr_debug("%s:%s:%d cntlc %p\n",
1798 skdev->name, __func__, __LINE__, sksgio);
e67f86b3
AB		 1799 rc = -EINTR;
1800 }
1801 }
1802
1803 spin_unlock_irqrestore(&skdev->lock, flags);
1804
1805 return rc;
1806}
1807
1808static int skd_sg_io_put_status(struct skd_device *skdev,
1809 struct skd_sg_io *sksgio)
1810{
1811 struct sg_io_hdr *sgp = &sksgio->sg;
1812 struct skd_special_context *skspcl = sksgio->skspcl;
1813 int resid = 0;
1814
1815 u32 nb = be32_to_cpu(skspcl->req.completion.num_returned_bytes);
1816
1817 sgp->status = skspcl->req.completion.status;
1818 resid = sksgio->dxfer_len - nb;
1819
1820 sgp->masked_status = sgp->status & STATUS_MASK;
1821 sgp->msg_status = 0;
1822 sgp->host_status = 0;
1823 sgp->driver_status = 0;
1824 sgp->resid = resid;
1825 if (sgp->masked_status || sgp->host_status || sgp->driver_status)
1826 sgp->info |= SG_INFO_CHECK;
1827
2e44b427 1828 pr_debug("%s:%s:%d status %x masked %x resid 0x%x\n",
1829 skdev->name, __func__, __LINE__,
1830 sgp->status, sgp->masked_status, sgp->resid);
e67f86b3
AB		 1831
1832 if (sgp->masked_status == SAM_STAT_CHECK_CONDITION) {
1833 if (sgp->mx_sb_len > 0) {
1834 struct fit_comp_error_info *ei = &skspcl->req.err_info;
1835 u32 nbytes = sizeof(*ei);
1836
1837 nbytes = min_t(u32, nbytes, sgp->mx_sb_len);
1838
1839 sgp->sb_len_wr = nbytes;
1840
1841 if (__copy_to_user(sgp->sbp, ei, nbytes)) {
2e44b427 1842 pr_debug("%s:%s:%d copy_to_user sense failed %p\n",
1843 skdev->name, __func__, __LINE__,
1844 sgp->sbp);
e67f86b3
AB		 1845 return -EFAULT;
1846 }
1847 }
1848 }
1849
1850 if (__copy_to_user(sksgio->argp, sgp, sizeof(sg_io_hdr_t))) {
2e44b427 1851 pr_debug("%s:%s:%d copy_to_user sg failed %p\n",
1852 skdev->name, __func__, __LINE__, sksgio->argp);
e67f86b3
AB		 1853 return -EFAULT;
1854 }
1855
1856 return 0;
1857}
1858
1859static int skd_sg_io_release_skspcl(struct skd_device *skdev,
1860 struct skd_sg_io *sksgio)
1861{
1862 struct skd_special_context *skspcl = sksgio->skspcl;
1863
1864 if (skspcl != NULL) {
1865 ulong flags;
1866
1867 sksgio->skspcl = NULL;
1868
1869 spin_lock_irqsave(&skdev->lock, flags);
1870 skd_release_special(skdev, skspcl);
1871 spin_unlock_irqrestore(&skdev->lock, flags);
1872 }
1873
1874 return 0;
1875}
1876
1877/*
1878 *****************************************************************************
1879 * INTERNAL REQUESTS -- generated by driver itself
1880 *****************************************************************************
1881 */
1882
1883static int skd_format_internal_skspcl(struct skd_device *skdev)
1884{
1885 struct skd_special_context *skspcl = &skdev->internal_skspcl;
1886 struct fit_sg_descriptor *sgd = &skspcl->req.sksg_list[0];
1887 struct fit_msg_hdr *fmh;
1888 uint64_t dma_address;
1889 struct skd_scsi_request *scsi;
1890
1891 fmh = (struct fit_msg_hdr *)&skspcl->msg_buf[0];
1892 fmh->protocol_id = FIT_PROTOCOL_ID_SOFIT;
1893 fmh->num_protocol_cmds_coalesced = 1;
1894
1895 scsi = (struct skd_scsi_request *)&skspcl->msg_buf[64];
1896 memset(scsi, 0, sizeof(*scsi));
1897 dma_address = skspcl->req.sksg_dma_address;
1898 scsi->hdr.sg_list_dma_address = cpu_to_be64(dma_address);
1899 sgd->control = FIT_SGD_CONTROL_LAST;
1900 sgd->byte_count = 0;
1901 sgd->host_side_addr = skspcl->db_dma_address;
1902 sgd->dev_side_addr = 0;
1903 sgd->next_desc_ptr = 0LL;
1904
1905 return 1;
1906}
1907
1908#define WR_BUF_SIZE SKD_N_INTERNAL_BYTES
1909
1910static void skd_send_internal_skspcl(struct skd_device *skdev,
1911 struct skd_special_context *skspcl,
1912 u8 opcode)
1913{
1914 struct fit_sg_descriptor *sgd = &skspcl->req.sksg_list[0];
1915 struct skd_scsi_request *scsi;
1916 unsigned char *buf = skspcl->data_buf;
1917 int i;
1918
1919 if (skspcl->req.state != SKD_REQ_STATE_IDLE)
1920 /*
1921 * A refresh is already in progress.
1922 * Just wait for it to finish.
1923 */
1924 return;
1925
1926 SKD_ASSERT((skspcl->req.id & SKD_ID_INCR) == 0);
1927 skspcl->req.state = SKD_REQ_STATE_BUSY;
1928 skspcl->req.id += SKD_ID_INCR;
1929
1930 scsi = (struct skd_scsi_request *)&skspcl->msg_buf[64];
1931 scsi->hdr.tag = skspcl->req.id;
1932
1933 memset(scsi->cdb, 0, sizeof(scsi->cdb));
1934
1935 switch (opcode) {
1936 case TEST_UNIT_READY:
1937 scsi->cdb[0] = TEST_UNIT_READY;
1938 sgd->byte_count = 0;
1939 scsi->hdr.sg_list_len_bytes = 0;
1940 break;
1941
1942 case READ_CAPACITY:
1943 scsi->cdb[0] = READ_CAPACITY;
1944 sgd->byte_count = SKD_N_READ_CAP_BYTES;
1945 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
1946 break;
1947
1948 case INQUIRY:
1949 scsi->cdb[0] = INQUIRY;
1950 scsi->cdb[1] = 0x01; /* evpd */
1951 scsi->cdb[2] = 0x80; /* serial number page */
1952 scsi->cdb[4] = 0x10;
1953 sgd->byte_count = 16;
1954 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
1955 break;
1956
1957 case SYNCHRONIZE_CACHE:
1958 scsi->cdb[0] = SYNCHRONIZE_CACHE;
1959 sgd->byte_count = 0;
1960 scsi->hdr.sg_list_len_bytes = 0;
1961 break;
1962
1963 case WRITE_BUFFER:
1964 scsi->cdb[0] = WRITE_BUFFER;
1965 scsi->cdb[1] = 0x02;
1966 scsi->cdb[7] = (WR_BUF_SIZE & 0xFF00) >> 8;
1967 scsi->cdb[8] = WR_BUF_SIZE & 0xFF;
1968 sgd->byte_count = WR_BUF_SIZE;
1969 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
1970 /* fill incrementing byte pattern */
1971 for (i = 0; i < sgd->byte_count; i++)
1972 buf[i] = i & 0xFF;
1973 break;
1974
1975 case READ_BUFFER:
1976 scsi->cdb[0] = READ_BUFFER;
1977 scsi->cdb[1] = 0x02;
1978 scsi->cdb[7] = (WR_BUF_SIZE & 0xFF00) >> 8;
1979 scsi->cdb[8] = WR_BUF_SIZE & 0xFF;
1980 sgd->byte_count = WR_BUF_SIZE;
1981 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
1982 memset(skspcl->data_buf, 0, sgd->byte_count);
1983 break;
1984
1985 default:
1986 SKD_ASSERT("Don't know what to send");
1987 return;
1988
1989 }
1990 skd_send_special_fitmsg(skdev, skspcl);
1991}
1992
1993static void skd_refresh_device_data(struct skd_device *skdev)
1994{
1995 struct skd_special_context *skspcl = &skdev->internal_skspcl;
1996
1997 skd_send_internal_skspcl(skdev, skspcl, TEST_UNIT_READY);
1998}
1999
2000static int skd_chk_read_buf(struct skd_device *skdev,
2001 struct skd_special_context *skspcl)
2002{
2003 unsigned char *buf = skspcl->data_buf;
2004 int i;
2005
2006 /* check for incrementing byte pattern */
2007 for (i = 0; i < WR_BUF_SIZE; i++)
2008 if (buf[i] != (i & 0xFF))
2009 return 1;
2010
2011 return 0;
2012}
2013
2014static void skd_log_check_status(struct skd_device *skdev, u8 status, u8 key,
2015 u8 code, u8 qual, u8 fruc)
2016{
2017 /* If the check condition is of special interest, log a message */
2018 if ((status == SAM_STAT_CHECK_CONDITION) && (key == 0x02)
2019 && (code == 0x04) && (qual == 0x06)) {
2020 pr_err("(%s): *** LOST_WRITE_DATA ERROR *** key/asc/"
2021 "ascq/fruc %02x/%02x/%02x/%02x\n",
2022 skd_name(skdev), key, code, qual, fruc);
2023 }
2024}
2025
2026static void skd_complete_internal(struct skd_device *skdev,
2027 volatile struct fit_completion_entry_v1
2028 *skcomp,
2029 volatile struct fit_comp_error_info *skerr,
2030 struct skd_special_context *skspcl)
2031{
2032 u8 *buf = skspcl->data_buf;
2033 u8 status;
2034 int i;
2035 struct skd_scsi_request *scsi =
2036 (struct skd_scsi_request *)&skspcl->msg_buf[64];
2037
2038 SKD_ASSERT(skspcl == &skdev->internal_skspcl);
2039
2e44b427 2040 pr_debug("%s:%s:%d complete internal %x\n",
2041 skdev->name, __func__, __LINE__, scsi->cdb[0]);
e67f86b3
AB		 2042
2043 skspcl->req.completion = *skcomp;
2044 skspcl->req.state = SKD_REQ_STATE_IDLE;
2045 skspcl->req.id += SKD_ID_INCR;
2046
2047 status = skspcl->req.completion.status;
2048
2049 skd_log_check_status(skdev, status, skerr->key, skerr->code,
2050 skerr->qual, skerr->fruc);
2051
2052 switch (scsi->cdb[0]) {
2053 case TEST_UNIT_READY:
2054 if (status == SAM_STAT_GOOD)
2055 skd_send_internal_skspcl(skdev, skspcl, WRITE_BUFFER);
2056 else if ((status == SAM_STAT_CHECK_CONDITION) &&
2057 (skerr->key == MEDIUM_ERROR))
2058 skd_send_internal_skspcl(skdev, skspcl, WRITE_BUFFER);
2059 else {
2060 if (skdev->state == SKD_DRVR_STATE_STOPPING) {
2e44b427 2061 pr_debug("%s:%s:%d TUR failed, don't send anymore state 0x%x\n",
2062 skdev->name, __func__, __LINE__,
2063 skdev->state);
e67f86b3
AB		 2064 return;
2065 }
2e44b427 2066 pr_debug("%s:%s:%d **** TUR failed, retry skerr\n",
2067 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2068 skd_send_internal_skspcl(skdev, skspcl, 0x00);
2069 }
2070 break;
2071
2072 case WRITE_BUFFER:
2073 if (status == SAM_STAT_GOOD)
2074 skd_send_internal_skspcl(skdev, skspcl, READ_BUFFER);
2075 else {
2076 if (skdev->state == SKD_DRVR_STATE_STOPPING) {
2e44b427 2077 pr_debug("%s:%s:%d write buffer failed, don't send anymore state 0x%x\n",
2078 skdev->name, __func__, __LINE__,
2079 skdev->state);
e67f86b3
AB		 2080 return;
2081 }
2e44b427 2082 pr_debug("%s:%s:%d **** write buffer failed, retry skerr\n",
2083 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2084 skd_send_internal_skspcl(skdev, skspcl, 0x00);
2085 }
2086 break;
2087
2088 case READ_BUFFER:
2089 if (status == SAM_STAT_GOOD) {
2090 if (skd_chk_read_buf(skdev, skspcl) == 0)
2091 skd_send_internal_skspcl(skdev, skspcl,
2092 READ_CAPACITY);
2093 else {
2094 pr_err(
2095 "(%s):*** W/R Buffer mismatch %d ***\n",
2096 skd_name(skdev), skdev->connect_retries);
2097 if (skdev->connect_retries <
2098 SKD_MAX_CONNECT_RETRIES) {
2099 skdev->connect_retries++;
2100 skd_soft_reset(skdev);
2101 } else {
2102 pr_err(
2103 "(%s): W/R Buffer Connect Error\n",
2104 skd_name(skdev));
2105 return;
2106 }
2107 }
2108
2109 } else {
2110 if (skdev->state == SKD_DRVR_STATE_STOPPING) {
2e44b427 2111 pr_debug("%s:%s:%d "
2112 "read buffer failed, don't send anymore state 0x%x\n",
2113 skdev->name, __func__, __LINE__,
2114 skdev->state);
e67f86b3
AB		 2115 return;
2116 }
2e44b427 2117 pr_debug("%s:%s:%d "
2118 "**** read buffer failed, retry skerr\n",
2119 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2120 skd_send_internal_skspcl(skdev, skspcl, 0x00);
2121 }
2122 break;
2123
2124 case READ_CAPACITY:
2125 skdev->read_cap_is_valid = 0;
2126 if (status == SAM_STAT_GOOD) {
2127 skdev->read_cap_last_lba =
2128 (buf[0] << 24) | (buf[1] << 16) |
2129 (buf[2] << 8) | buf[3];
2130 skdev->read_cap_blocksize =
2131 (buf[4] << 24) | (buf[5] << 16) |
2132 (buf[6] << 8) | buf[7];
2133
2e44b427 2134 pr_debug("%s:%s:%d last lba %d, bs %d\n",
2135 skdev->name, __func__, __LINE__,
2136 skdev->read_cap_last_lba,
2137 skdev->read_cap_blocksize);
e67f86b3
AB		 2138
2139 set_capacity(skdev->disk, skdev->read_cap_last_lba + 1);
2140
2141 skdev->read_cap_is_valid = 1;
2142
2143 skd_send_internal_skspcl(skdev, skspcl, INQUIRY);
2144 } else if ((status == SAM_STAT_CHECK_CONDITION) &&
2145 (skerr->key == MEDIUM_ERROR)) {
2146 skdev->read_cap_last_lba = ~0;
2147 set_capacity(skdev->disk, skdev->read_cap_last_lba + 1);
2e44b427 2148 pr_debug("%s:%s:%d "
2149 "**** MEDIUM ERROR caused READCAP to fail, ignore failure and continue to inquiry\n",
2150 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2151 skd_send_internal_skspcl(skdev, skspcl, INQUIRY);
2152 } else {
2e44b427 2153 pr_debug("%s:%s:%d **** READCAP failed, retry TUR\n",
2154 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2155 skd_send_internal_skspcl(skdev, skspcl,
2156 TEST_UNIT_READY);
2157 }
2158 break;
2159
2160 case INQUIRY:
2161 skdev->inquiry_is_valid = 0;
2162 if (status == SAM_STAT_GOOD) {
2163 skdev->inquiry_is_valid = 1;
2164
2165 for (i = 0; i < 12; i++)
2166 skdev->inq_serial_num[i] = buf[i + 4];
2167 skdev->inq_serial_num[12] = 0;
2168 }
2169
2170 if (skd_unquiesce_dev(skdev) < 0)
2e44b427 2171 pr_debug("%s:%s:%d **** failed, to ONLINE device\n",
2172 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2173 /* connection is complete */
2174 skdev->connect_retries = 0;
2175 break;
2176
2177 case SYNCHRONIZE_CACHE:
2178 if (status == SAM_STAT_GOOD)
2179 skdev->sync_done = 1;
2180 else
2181 skdev->sync_done = -1;
2182 wake_up_interruptible(&skdev->waitq);
2183 break;
2184
2185 default:
2186 SKD_ASSERT("we didn't send this");
2187 }
2188}
2189
2190/*
2191 *****************************************************************************
2192 * FIT MESSAGES
2193 *****************************************************************************
2194 */
2195
2196static void skd_send_fitmsg(struct skd_device *skdev,
2197 struct skd_fitmsg_context *skmsg)
2198{
2199 u64 qcmd;
2200 struct fit_msg_hdr *fmh;
2201
2e44b427 2202 pr_debug("%s:%s:%d dma address 0x%llx, busy=%d\n",
2203 skdev->name, __func__, __LINE__,
2204 skmsg->mb_dma_address, skdev->in_flight);
2205 pr_debug("%s:%s:%d msg_buf 0x%p, offset %x\n",
2206 skdev->name, __func__, __LINE__,
2207 skmsg->msg_buf, skmsg->offset);
e67f86b3
AB		 2208
2209 qcmd = skmsg->mb_dma_address;
2210 qcmd |= FIT_QCMD_QID_NORMAL;
2211
2212 fmh = (struct fit_msg_hdr *)skmsg->msg_buf;
2213 skmsg->outstanding = fmh->num_protocol_cmds_coalesced;
2214
2215 if (unlikely(skdev->dbg_level > 1)) {
2216 u8 *bp = (u8 *)skmsg->msg_buf;
2217 int i;
2218 for (i = 0; i < skmsg->length; i += 8) {
2e44b427 2219 pr_debug("%s:%s:%d msg[%2d] %02x %02x %02x %02x "
2220 "%02x %02x %02x %02x\n",
2221 skdev->name, __func__, __LINE__,
2222 i, bp[i + 0], bp[i + 1], bp[i + 2],
2223 bp[i + 3], bp[i + 4], bp[i + 5],
2224 bp[i + 6], bp[i + 7]);
e67f86b3
AB		 2225 if (i == 0)
2226 i = 64 - 8;
2227 }
2228 }
2229
2230 if (skmsg->length > 256)
2231 qcmd |= FIT_QCMD_MSGSIZE_512;
2232 else if (skmsg->length > 128)
2233 qcmd |= FIT_QCMD_MSGSIZE_256;
2234 else if (skmsg->length > 64)
2235 qcmd |= FIT_QCMD_MSGSIZE_128;
2236 else
2237 /*
2238 * This makes no sense because the FIT msg header is
2239 * 64 bytes. If the msg is only 64 bytes long it has
2240 * no payload.
2241 */
2242 qcmd |= FIT_QCMD_MSGSIZE_64;
2243
2244 SKD_WRITEQ(skdev, qcmd, FIT_Q_COMMAND);
2245
2246}
2247
2248static void skd_send_special_fitmsg(struct skd_device *skdev,
2249 struct skd_special_context *skspcl)
2250{
2251 u64 qcmd;
2252
2253 if (unlikely(skdev->dbg_level > 1)) {
2254 u8 *bp = (u8 *)skspcl->msg_buf;
2255 int i;
2256
2257 for (i = 0; i < SKD_N_SPECIAL_FITMSG_BYTES; i += 8) {
2e44b427 2258 pr_debug("%s:%s:%d spcl[%2d] %02x %02x %02x %02x "
2259 "%02x %02x %02x %02x\n",
2260 skdev->name, __func__, __LINE__, i,
2261 bp[i + 0], bp[i + 1], bp[i + 2], bp[i + 3],
2262 bp[i + 4], bp[i + 5], bp[i + 6], bp[i + 7]);
e67f86b3
AB		 2263 if (i == 0)
2264 i = 64 - 8;
2265 }
2266
2e44b427 2267 pr_debug("%s:%s:%d skspcl=%p id=%04x sksg_list=%p sksg_dma=%llx\n",
2268 skdev->name, __func__, __LINE__,
2269 skspcl, skspcl->req.id, skspcl->req.sksg_list,
2270 skspcl->req.sksg_dma_address);
e67f86b3
AB		 2271 for (i = 0; i < skspcl->req.n_sg; i++) {
2272 struct fit_sg_descriptor *sgd =
2273 &skspcl->req.sksg_list[i];
2274
2e44b427 2275 pr_debug("%s:%s:%d sg[%d] count=%u ctrl=0x%x "
2276 "addr=0x%llx next=0x%llx\n",
2277 skdev->name, __func__, __LINE__,
2278 i, sgd->byte_count, sgd->control,
2279 sgd->host_side_addr, sgd->next_desc_ptr);
e67f86b3
AB		 2280 }
2281 }
2282
2283 /*
2284 * Special FIT msgs are always 128 bytes: a 64-byte FIT hdr
2285 * and one 64-byte SSDI command.
2286 */
2287 qcmd = skspcl->mb_dma_address;
2288 qcmd |= FIT_QCMD_QID_NORMAL + FIT_QCMD_MSGSIZE_128;
2289
2290 SKD_WRITEQ(skdev, qcmd, FIT_Q_COMMAND);
2291}
2292
2293/*
2294 *****************************************************************************
2295 * COMPLETION QUEUE
2296 *****************************************************************************
2297 */
2298
2299static void skd_complete_other(struct skd_device *skdev,
2300 volatile struct fit_completion_entry_v1 *skcomp,
2301 volatile struct fit_comp_error_info *skerr);
2302
2303
2304static void skd_requeue_request(struct skd_device *skdev,
2305 struct skd_request_context *skreq);
2306
2307struct sns_info {
2308 u8 type;
2309 u8 stat;
2310 u8 key;
2311 u8 asc;
2312 u8 ascq;
2313 u8 mask;
2314 enum skd_check_status_action action;
2315};
2316
2317static struct sns_info skd_chkstat_table[] = {
2318 /* Good */
2319 { 0x70, 0x02, RECOVERED_ERROR, 0, 0, 0x1c,
2320 SKD_CHECK_STATUS_REPORT_GOOD },
2321
2322 /* Smart alerts */
2323 { 0x70, 0x02, NO_SENSE, 0x0B, 0x00, 0x1E, /* warnings */
2324 SKD_CHECK_STATUS_REPORT_SMART_ALERT },
2325 { 0x70, 0x02, NO_SENSE, 0x5D, 0x00, 0x1E, /* thresholds */
2326 SKD_CHECK_STATUS_REPORT_SMART_ALERT },
2327 { 0x70, 0x02, RECOVERED_ERROR, 0x0B, 0x01, 0x1F, /* temperature over trigger */
2328 SKD_CHECK_STATUS_REPORT_SMART_ALERT },
2329
2330 /* Retry (with limits) */
2331 { 0x70, 0x02, 0x0B, 0, 0, 0x1C, /* This one is for DMA ERROR */
2332 SKD_CHECK_STATUS_REQUEUE_REQUEST },
2333 { 0x70, 0x02, 0x06, 0x0B, 0x00, 0x1E, /* warnings */
2334 SKD_CHECK_STATUS_REQUEUE_REQUEST },
2335 { 0x70, 0x02, 0x06, 0x5D, 0x00, 0x1E, /* thresholds */
2336 SKD_CHECK_STATUS_REQUEUE_REQUEST },
2337 { 0x70, 0x02, 0x06, 0x80, 0x30, 0x1F, /* backup power */
2338 SKD_CHECK_STATUS_REQUEUE_REQUEST },
2339
2340 /* Busy (or about to be) */
2341 { 0x70, 0x02, 0x06, 0x3f, 0x01, 0x1F, /* fw changed */
2342 SKD_CHECK_STATUS_BUSY_IMMINENT },
2343};
2344
2345/*
2346 * Look up status and sense data to decide how to handle the error
2347 * from the device.
2348 * mask says which fields must match e.g., mask=0x18 means check
2349 * type and stat, ignore key, asc, ascq.
2350 */
2351
2352static enum skd_check_status_action skd_check_status(struct skd_device *skdev,
2353 u8 cmp_status,
2354 volatile struct fit_comp_error_info *skerr)
2355{
2356 int i, n;
2357
2358 pr_err("(%s): key/asc/ascq/fruc %02x/%02x/%02x/%02x\n",
2359 skd_name(skdev), skerr->key, skerr->code, skerr->qual,
2360 skerr->fruc);
2361
2e44b427 2362 pr_debug("%s:%s:%d stat: t=%02x stat=%02x k=%02x c=%02x q=%02x fruc=%02x\n",
2363 skdev->name, __func__, __LINE__, skerr->type, cmp_status,
2364 skerr->key, skerr->code, skerr->qual, skerr->fruc);
e67f86b3
AB		 2365
2366 /* Does the info match an entry in the good category? */
2367 n = sizeof(skd_chkstat_table) / sizeof(skd_chkstat_table[0]);
2368 for (i = 0; i < n; i++) {
2369 struct sns_info *sns = &skd_chkstat_table[i];
2370
2371 if (sns->mask & 0x10)
2372 if (skerr->type != sns->type)
2373 continue;
2374
2375 if (sns->mask & 0x08)
2376 if (cmp_status != sns->stat)
2377 continue;
2378
2379 if (sns->mask & 0x04)
2380 if (skerr->key != sns->key)
2381 continue;
2382
2383 if (sns->mask & 0x02)
2384 if (skerr->code != sns->asc)
2385 continue;
2386
2387 if (sns->mask & 0x01)
2388 if (skerr->qual != sns->ascq)
2389 continue;
2390
2391 if (sns->action == SKD_CHECK_STATUS_REPORT_SMART_ALERT) {
2392 pr_err("(%s): SMART Alert: sense key/asc/ascq "
2393 "%02x/%02x/%02x\n",
2394 skd_name(skdev), skerr->key,
2395 skerr->code, skerr->qual);
2396 }
2397 return sns->action;
2398 }
2399
2400 /* No other match, so nonzero status means error,
2401 * zero status means good
2402 */
2403 if (cmp_status) {
2e44b427 2404 pr_debug("%s:%s:%d status check: error\n",
2405 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2406 return SKD_CHECK_STATUS_REPORT_ERROR;
2407 }
2408
2e44b427 2409 pr_debug("%s:%s:%d status check good default\n",
2410 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2411 return SKD_CHECK_STATUS_REPORT_GOOD;
2412}
2413
2414static void skd_resolve_req_exception(struct skd_device *skdev,
2415 struct skd_request_context *skreq)
2416{
2417 u8 cmp_status = skreq->completion.status;
2418
2419 switch (skd_check_status(skdev, cmp_status, &skreq->err_info)) {
2420 case SKD_CHECK_STATUS_REPORT_GOOD:
2421 case SKD_CHECK_STATUS_REPORT_SMART_ALERT:
2422 skd_end_request(skdev, skreq, 0);
2423 break;
2424
2425 case SKD_CHECK_STATUS_BUSY_IMMINENT:
2426 skd_log_skreq(skdev, skreq, "retry(busy)");
2427 skd_requeue_request(skdev, skreq);
2428 pr_info("(%s) drive BUSY imminent\n", skd_name(skdev));
2429 skdev->state = SKD_DRVR_STATE_BUSY_IMMINENT;
2430 skdev->timer_countdown = SKD_TIMER_MINUTES(20);
2431 skd_quiesce_dev(skdev);
2432 break;
2433
2434 case SKD_CHECK_STATUS_REQUEUE_REQUEST:
fcd37eb3
JA		 2435 if ((unsigned long) ++skreq->req->special < SKD_MAX_RETRIES) {
2436 skd_log_skreq(skdev, skreq, "retry");
2437 skd_requeue_request(skdev, skreq);
2438 break;
e67f86b3
AB		 2439 }
2440 /* fall through to report error */
2441
2442 case SKD_CHECK_STATUS_REPORT_ERROR:
2443 default:
2444 skd_end_request(skdev, skreq, -EIO);
2445 break;
2446 }
2447}
2448
2449static void skd_requeue_request(struct skd_device *skdev,
2450 struct skd_request_context *skreq)
2451{
fcd37eb3 2452 blk_requeue_request(skdev->queue, skreq->req);
e67f86b3
AB		 2453}
2454
2455
2456
2457/* assume spinlock is already held */
2458static void skd_release_skreq(struct skd_device *skdev,
2459 struct skd_request_context *skreq)
2460{
2461 u32 msg_slot;
2462 struct skd_fitmsg_context *skmsg;
2463
2464 u32 timo_slot;
2465
2466 /*
2467 * Reclaim the FIT msg buffer if this is
2468 * the first of the requests it carried to
2469 * be completed. The FIT msg buffer used to
2470 * send this request cannot be reused until
2471 * we are sure the s1120 card has copied
2472 * it to its memory. The FIT msg might have
2473 * contained several requests. As soon as
2474 * any of them are completed we know that
2475 * the entire FIT msg was transferred.
2476 * Only the first completed request will
2477 * match the FIT msg buffer id. The FIT
2478 * msg buffer id is immediately updated.
2479 * When subsequent requests complete the FIT
2480 * msg buffer id won't match, so we know
2481 * quite cheaply that it is already done.
2482 */
2483 msg_slot = skreq->fitmsg_id & SKD_ID_SLOT_MASK;
2484 SKD_ASSERT(msg_slot < skdev->num_fitmsg_context);
2485
2486 skmsg = &skdev->skmsg_table[msg_slot];
2487 if (skmsg->id == skreq->fitmsg_id) {
2488 SKD_ASSERT(skmsg->state == SKD_MSG_STATE_BUSY);
2489 SKD_ASSERT(skmsg->outstanding > 0);
2490 skmsg->outstanding--;
2491 if (skmsg->outstanding == 0) {
2492 skmsg->state = SKD_MSG_STATE_IDLE;
2493 skmsg->id += SKD_ID_INCR;
2494 skmsg->next = skdev->skmsg_free_list;
2495 skdev->skmsg_free_list = skmsg;
2496 }
2497 }
2498
2499 /*
2500 * Decrease the number of active requests.
2501 * Also decrements the count in the timeout slot.
2502 */
2503 SKD_ASSERT(skdev->in_flight > 0);
2504 skdev->in_flight -= 1;
2505
2506 timo_slot = skreq->timeout_stamp & SKD_TIMEOUT_SLOT_MASK;
2507 SKD_ASSERT(skdev->timeout_slot[timo_slot] > 0);
2508 skdev->timeout_slot[timo_slot] -= 1;
2509
2510 /*
2511 * Reset backpointer
2512 */
fcd37eb3 2513 skreq->req = NULL;
e67f86b3
AB		 2514
2515 /*
2516 * Reclaim the skd_request_context
2517 */
2518 skreq->state = SKD_REQ_STATE_IDLE;
2519 skreq->id += SKD_ID_INCR;
2520 skreq->next = skdev->skreq_free_list;
2521 skdev->skreq_free_list = skreq;
2522}
2523
2524#define DRIVER_INQ_EVPD_PAGE_CODE 0xDA
2525
2526static void skd_do_inq_page_00(struct skd_device *skdev,
2527 volatile struct fit_completion_entry_v1 *skcomp,
2528 volatile struct fit_comp_error_info *skerr,
2529 uint8_t *cdb, uint8_t *buf)
2530{
2531 uint16_t insert_pt, max_bytes, drive_pages, drive_bytes, new_size;
2532
2533 /* Caller requested "supported pages". The driver needs to insert
2534 * its page.
2535 */
2e44b427 2536 pr_debug("%s:%s:%d skd_do_driver_inquiry: modify supported pages.\n",
2537 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2538
2539 /* If the device rejected the request because the CDB was
2540 * improperly formed, then just leave.
2541 */
2542 if (skcomp->status == SAM_STAT_CHECK_CONDITION &&
2543 skerr->key == ILLEGAL_REQUEST && skerr->code == 0x24)
2544 return;
2545
2546 /* Get the amount of space the caller allocated */
2547 max_bytes = (cdb[3] << 8) | cdb[4];
2548
2549 /* Get the number of pages actually returned by the device */
2550 drive_pages = (buf[2] << 8) | buf[3];
2551 drive_bytes = drive_pages + 4;
2552 new_size = drive_pages + 1;
2553
2554 /* Supported pages must be in numerical order, so find where
2555 * the driver page needs to be inserted into the list of
2556 * pages returned by the device.
2557 */
2558 for (insert_pt = 4; insert_pt < drive_bytes; insert_pt++) {
2559 if (buf[insert_pt] == DRIVER_INQ_EVPD_PAGE_CODE)
2560 return; /* Device using this page code. abort */
2561 else if (buf[insert_pt] > DRIVER_INQ_EVPD_PAGE_CODE)
2562 break;
2563 }
2564
2565 if (insert_pt < max_bytes) {
2566 uint16_t u;
2567
2568 /* Shift everything up one byte to make room. */
2569 for (u = new_size + 3; u > insert_pt; u--)
2570 buf[u] = buf[u - 1];
2571 buf[insert_pt] = DRIVER_INQ_EVPD_PAGE_CODE;
2572
2573 /* SCSI byte order increment of num_returned_bytes by 1 */
2574 skcomp->num_returned_bytes =
2575 be32_to_cpu(skcomp->num_returned_bytes) + 1;
2576 skcomp->num_returned_bytes =
2577 be32_to_cpu(skcomp->num_returned_bytes);
2578 }
2579
2580 /* update page length field to reflect the driver's page too */
2581 buf[2] = (uint8_t)((new_size >> 8) & 0xFF);
2582 buf[3] = (uint8_t)((new_size >> 0) & 0xFF);
2583}
2584
2585static void skd_get_link_info(struct pci_dev *pdev, u8 *speed, u8 *width)
2586{
2587 int pcie_reg;
2588 u16 pci_bus_speed;
2589 u8 pci_lanes;
2590
2591 pcie_reg = pci_find_capability(pdev, PCI_CAP_ID_EXP);
2592 if (pcie_reg) {
2593 u16 linksta;
2594 pci_read_config_word(pdev, pcie_reg + PCI_EXP_LNKSTA, &linksta);
2595
2596 pci_bus_speed = linksta & 0xF;
2597 pci_lanes = (linksta & 0x3F0) >> 4;
2598 } else {
2599 *speed = STEC_LINK_UNKNOWN;
2600 *width = 0xFF;
2601 return;
2602 }
2603
2604 switch (pci_bus_speed) {
2605 case 1:
2606 *speed = STEC_LINK_2_5GTS;
2607 break;
2608 case 2:
2609 *speed = STEC_LINK_5GTS;
2610 break;
2611 case 3:
2612 *speed = STEC_LINK_8GTS;
2613 break;
2614 default:
2615 *speed = STEC_LINK_UNKNOWN;
2616 break;
2617 }
2618
2619 if (pci_lanes <= 0x20)
2620 *width = pci_lanes;
2621 else
2622 *width = 0xFF;
2623}
2624
2625static void skd_do_inq_page_da(struct skd_device *skdev,
2626 volatile struct fit_completion_entry_v1 *skcomp,
2627 volatile struct fit_comp_error_info *skerr,
2628 uint8_t *cdb, uint8_t *buf)
2629{
2630 unsigned max_bytes;
2631 struct driver_inquiry_data inq;
2632 u16 val;
2633
2e44b427 2634 pr_debug("%s:%s:%d skd_do_driver_inquiry: return driver page\n",
2635 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2636
2637 memset(&inq, 0, sizeof(inq));
2638
2639 inq.page_code = DRIVER_INQ_EVPD_PAGE_CODE;
2640
2641 if (skdev->pdev && skdev->pdev->bus) {
2642 skd_get_link_info(skdev->pdev,
2643 &inq.pcie_link_speed, &inq.pcie_link_lanes);
2644 inq.pcie_bus_number = cpu_to_be16(skdev->pdev->bus->number);
2645 inq.pcie_device_number = PCI_SLOT(skdev->pdev->devfn);
2646 inq.pcie_function_number = PCI_FUNC(skdev->pdev->devfn);
2647
2648 pci_read_config_word(skdev->pdev, PCI_VENDOR_ID, &val);
2649 inq.pcie_vendor_id = cpu_to_be16(val);
2650
2651 pci_read_config_word(skdev->pdev, PCI_DEVICE_ID, &val);
2652 inq.pcie_device_id = cpu_to_be16(val);
2653
2654 pci_read_config_word(skdev->pdev, PCI_SUBSYSTEM_VENDOR_ID,
2655 &val);
2656 inq.pcie_subsystem_vendor_id = cpu_to_be16(val);
2657
2658 pci_read_config_word(skdev->pdev, PCI_SUBSYSTEM_ID, &val);
2659 inq.pcie_subsystem_device_id = cpu_to_be16(val);
2660 } else {
2661 inq.pcie_bus_number = 0xFFFF;
2662 inq.pcie_device_number = 0xFF;
2663 inq.pcie_function_number = 0xFF;
2664 inq.pcie_link_speed = 0xFF;
2665 inq.pcie_link_lanes = 0xFF;
2666 inq.pcie_vendor_id = 0xFFFF;
2667 inq.pcie_device_id = 0xFFFF;
2668 inq.pcie_subsystem_vendor_id = 0xFFFF;
2669 inq.pcie_subsystem_device_id = 0xFFFF;
2670 }
2671
2672 /* Driver version, fixed lenth, padded with spaces on the right */
2673 inq.driver_version_length = sizeof(inq.driver_version);
2674 memset(&inq.driver_version, ' ', sizeof(inq.driver_version));
2675 memcpy(inq.driver_version, DRV_VER_COMPL,
2676 min(sizeof(inq.driver_version), strlen(DRV_VER_COMPL)));
2677
2678 inq.page_length = cpu_to_be16((sizeof(inq) - 4));
2679
2680 /* Clear the error set by the device */
2681 skcomp->status = SAM_STAT_GOOD;
2682 memset((void *)skerr, 0, sizeof(*skerr));
2683
2684 /* copy response into output buffer */
2685 max_bytes = (cdb[3] << 8) | cdb[4];
2686 memcpy(buf, &inq, min_t(unsigned, max_bytes, sizeof(inq)));
2687
2688 skcomp->num_returned_bytes =
2689 be32_to_cpu(min_t(uint16_t, max_bytes, sizeof(inq)));
2690}
2691
2692static void skd_do_driver_inq(struct skd_device *skdev,
2693 volatile struct fit_completion_entry_v1 *skcomp,
2694 volatile struct fit_comp_error_info *skerr,
2695 uint8_t *cdb, uint8_t *buf)
2696{
2697 if (!buf)
2698 return;
2699 else if (cdb[0] != INQUIRY)
2700 return; /* Not an INQUIRY */
2701 else if ((cdb[1] & 1) == 0)
2702 return; /* EVPD not set */
2703 else if (cdb[2] == 0)
2704 /* Need to add driver's page to supported pages list */
2705 skd_do_inq_page_00(skdev, skcomp, skerr, cdb, buf);
2706 else if (cdb[2] == DRIVER_INQ_EVPD_PAGE_CODE)
2707 /* Caller requested driver's page */
2708 skd_do_inq_page_da(skdev, skcomp, skerr, cdb, buf);
2709}
2710
2711static unsigned char *skd_sg_1st_page_ptr(struct scatterlist *sg)
2712{
2713 if (!sg)
2714 return NULL;
2715 if (!sg_page(sg))
2716 return NULL;
2717 return sg_virt(sg);
2718}
2719
2720static void skd_process_scsi_inq(struct skd_device *skdev,
2721 volatile struct fit_completion_entry_v1
2722 *skcomp,
2723 volatile struct fit_comp_error_info *skerr,
2724 struct skd_special_context *skspcl)
2725{
2726 uint8_t *buf;
2727 struct fit_msg_hdr *fmh = (struct fit_msg_hdr *)skspcl->msg_buf;
2728 struct skd_scsi_request *scsi_req = (struct skd_scsi_request *)&fmh[1];
2729
2730 dma_sync_sg_for_cpu(skdev->class_dev, skspcl->req.sg, skspcl->req.n_sg,
2731 skspcl->req.sg_data_dir);
2732 buf = skd_sg_1st_page_ptr(skspcl->req.sg);
2733
2734 if (buf)
2735 skd_do_driver_inq(skdev, skcomp, skerr, scsi_req->cdb, buf);
2736}
2737
2738
2739static int skd_isr_completion_posted(struct skd_device *skdev,
2740 int limit, int *enqueued)
2741{
2742 volatile struct fit_completion_entry_v1 *skcmp = NULL;
2743 volatile struct fit_comp_error_info *skerr;
2744 u16 req_id;
2745 u32 req_slot;
2746 struct skd_request_context *skreq;
2747 u16 cmp_cntxt = 0;
2748 u8 cmp_status = 0;
2749 u8 cmp_cycle = 0;
2750 u32 cmp_bytes = 0;
2751 int rc = 0;
2752 int processed = 0;
e67f86b3
AB		 2753
2754 for (;; ) {
2755 SKD_ASSERT(skdev->skcomp_ix < SKD_N_COMPLETION_ENTRY);
2756
2757 skcmp = &skdev->skcomp_table[skdev->skcomp_ix];
2758 cmp_cycle = skcmp->cycle;
2759 cmp_cntxt = skcmp->tag;
2760 cmp_status = skcmp->status;
2761 cmp_bytes = be32_to_cpu(skcmp->num_returned_bytes);
2762
2763 skerr = &skdev->skerr_table[skdev->skcomp_ix];
2764
2e44b427 2765 pr_debug("%s:%s:%d "
2766 "cycle=%d ix=%d got cycle=%d cmdctxt=0x%x stat=%d "
2767 "busy=%d rbytes=0x%x proto=%d\n",
2768 skdev->name, __func__, __LINE__, skdev->skcomp_cycle,
2769 skdev->skcomp_ix, cmp_cycle, cmp_cntxt, cmp_status,
2770 skdev->in_flight, cmp_bytes, skdev->proto_ver);
e67f86b3
AB		 2771
2772 if (cmp_cycle != skdev->skcomp_cycle) {
2e44b427 2773 pr_debug("%s:%s:%d end of completions\n",
2774 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2775 break;
2776 }
2777 /*
2778 * Update the completion queue head index and possibly
2779 * the completion cycle count. 8-bit wrap-around.
2780 */
2781 skdev->skcomp_ix++;
2782 if (skdev->skcomp_ix >= SKD_N_COMPLETION_ENTRY) {
2783 skdev->skcomp_ix = 0;
2784 skdev->skcomp_cycle++;
2785 }
2786
2787 /*
2788 * The command context is a unique 32-bit ID. The low order
2789 * bits help locate the request. The request is usually a
2790 * r/w request (see skd_start() above) or a special request.
2791 */
2792 req_id = cmp_cntxt;
2793 req_slot = req_id & SKD_ID_SLOT_AND_TABLE_MASK;
2794
2795 /* Is this other than a r/w request? */
2796 if (req_slot >= skdev->num_req_context) {
2797 /*
2798 * This is not a completion for a r/w request.
2799 */
2800 skd_complete_other(skdev, skcmp, skerr);
2801 continue;
2802 }
2803
2804 skreq = &skdev->skreq_table[req_slot];
2805
2806 /*
2807 * Make sure the request ID for the slot matches.
2808 */
2809 if (skreq->id != req_id) {
2e44b427 2810 pr_debug("%s:%s:%d mismatch comp_id=0x%x req_id=0x%x\n",
2811 skdev->name, __func__, __LINE__,
2812 req_id, skreq->id);
e67f86b3
AB		 2813 {
2814 u16 new_id = cmp_cntxt;
2815 pr_err("(%s): Completion mismatch "
2816 "comp_id=0x%04x skreq=0x%04x new=0x%04x\n",
2817 skd_name(skdev), req_id,
2818 skreq->id, new_id);
2819
2820 continue;
2821 }
2822 }
2823
2824 SKD_ASSERT(skreq->state == SKD_REQ_STATE_BUSY);
2825
2826 if (skreq->state == SKD_REQ_STATE_ABORTED) {
2e44b427 2827 pr_debug("%s:%s:%d reclaim req %p id=%04x\n",
2828 skdev->name, __func__, __LINE__,
2829 skreq, skreq->id);
e67f86b3
AB		 2830 /* a previously timed out command can
2831 * now be cleaned up */
2832 skd_release_skreq(skdev, skreq);
2833 continue;
2834 }
2835
2836 skreq->completion = *skcmp;
2837 if (unlikely(cmp_status == SAM_STAT_CHECK_CONDITION)) {
2838 skreq->err_info = *skerr;
2839 skd_log_check_status(skdev, cmp_status, skerr->key,
2840 skerr->code, skerr->qual,
2841 skerr->fruc);
2842 }
2843 /* Release DMA resources for the request. */
2844 if (skreq->n_sg > 0)
2845 skd_postop_sg_list(skdev, skreq);
2846
fcd37eb3 2847 if (!skreq->req) {
2e44b427 2848 pr_debug("%s:%s:%d NULL backptr skdreq %p, "
2849 "req=0x%x req_id=0x%x\n",
2850 skdev->name, __func__, __LINE__,
2851 skreq, skreq->id, req_id);
e67f86b3
AB		 2852 } else {
2853 /*
2854 * Capture the outcome and post it back to the
2855 * native request.
2856 */
fcd37eb3
JA		 2857 if (likely(cmp_status == SAM_STAT_GOOD))
2858 skd_end_request(skdev, skreq, 0);
2859 else
e67f86b3 2860 skd_resolve_req_exception(skdev, skreq);
e67f86b3
AB		 2861 }
2862
2863 /*
2864 * Release the skreq, its FIT msg (if one), timeout slot,
2865 * and queue depth.
2866 */
2867 skd_release_skreq(skdev, skreq);
2868
2869 /* skd_isr_comp_limit equal zero means no limit */
2870 if (limit) {
2871 if (++processed >= limit) {
2872 rc = 1;
2873 break;
2874 }
2875 }
2876 }
2877
2878 if ((skdev->state == SKD_DRVR_STATE_PAUSING)
2879 && (skdev->in_flight) == 0) {
2880 skdev->state = SKD_DRVR_STATE_PAUSED;
2881 wake_up_interruptible(&skdev->waitq);
2882 }
2883
2884 return rc;
2885}
2886
2887static void skd_complete_other(struct skd_device *skdev,
2888 volatile struct fit_completion_entry_v1 *skcomp,
2889 volatile struct fit_comp_error_info *skerr)
2890{
2891 u32 req_id = 0;
2892 u32 req_table;
2893 u32 req_slot;
2894 struct skd_special_context *skspcl;
2895
2896 req_id = skcomp->tag;
2897 req_table = req_id & SKD_ID_TABLE_MASK;
2898 req_slot = req_id & SKD_ID_SLOT_MASK;
2899
2e44b427 2900 pr_debug("%s:%s:%d table=0x%x id=0x%x slot=%d\n",
2901 skdev->name, __func__, __LINE__,
2902 req_table, req_id, req_slot);
e67f86b3
AB		 2903
2904 /*
2905 * Based on the request id, determine how to dispatch this completion.
2906 * This swich/case is finding the good cases and forwarding the
2907 * completion entry. Errors are reported below the switch.
2908 */
2909 switch (req_table) {
2910 case SKD_ID_RW_REQUEST:
2911 /*
2912 * The caller, skd_completion_posted_isr() above,
2913 * handles r/w requests. The only way we get here
2914 * is if the req_slot is out of bounds.
2915 */
2916 break;
2917
2918 case SKD_ID_SPECIAL_REQUEST:
2919 /*
2920 * Make sure the req_slot is in bounds and that the id
2921 * matches.
2922 */
2923 if (req_slot < skdev->n_special) {
2924 skspcl = &skdev->skspcl_table[req_slot];
2925 if (skspcl->req.id == req_id &&
2926 skspcl->req.state == SKD_REQ_STATE_BUSY) {
2927 skd_complete_special(skdev,
2928 skcomp, skerr, skspcl);
2929 return;
2930 }
2931 }
2932 break;
2933
2934 case SKD_ID_INTERNAL:
2935 if (req_slot == 0) {
2936 skspcl = &skdev->internal_skspcl;
2937 if (skspcl->req.id == req_id &&
2938 skspcl->req.state == SKD_REQ_STATE_BUSY) {
2939 skd_complete_internal(skdev,
2940 skcomp, skerr, skspcl);
2941 return;
2942 }
2943 }
2944 break;
2945
2946 case SKD_ID_FIT_MSG:
2947 /*
2948 * These id's should never appear in a completion record.
2949 */
2950 break;
2951
2952 default:
2953 /*
2954 * These id's should never appear anywhere;
2955 */
2956 break;
2957 }
2958
2959 /*
2960 * If we get here it is a bad or stale id.
2961 */
2962}
2963
2964static void skd_complete_special(struct skd_device *skdev,
2965 volatile struct fit_completion_entry_v1
2966 *skcomp,
2967 volatile struct fit_comp_error_info *skerr,
2968 struct skd_special_context *skspcl)
2969{
2e44b427 2970 pr_debug("%s:%s:%d completing special request %p\n",
2971 skdev->name, __func__, __LINE__, skspcl);
e67f86b3
AB		 2972 if (skspcl->orphaned) {
2973 /* Discard orphaned request */
2974 /* ?: Can this release directly or does it need
2975 * to use a worker? */
2e44b427 2976 pr_debug("%s:%s:%d release orphaned %p\n",
2977 skdev->name, __func__, __LINE__, skspcl);
e67f86b3
AB		 2978 skd_release_special(skdev, skspcl);
2979 return;
2980 }
2981
2982 skd_process_scsi_inq(skdev, skcomp, skerr, skspcl);
2983
2984 skspcl->req.state = SKD_REQ_STATE_COMPLETED;
2985 skspcl->req.completion = *skcomp;
2986 skspcl->req.err_info = *skerr;
2987
2988 skd_log_check_status(skdev, skspcl->req.completion.status, skerr->key,
2989 skerr->code, skerr->qual, skerr->fruc);
2990
2991 wake_up_interruptible(&skdev->waitq);
2992}
2993
2994/* assume spinlock is already held */
2995static void skd_release_special(struct skd_device *skdev,
2996 struct skd_special_context *skspcl)
2997{
2998 int i, was_depleted;
2999
3000 for (i = 0; i < skspcl->req.n_sg; i++) {
3001
3002 struct page *page = sg_page(&skspcl->req.sg[i]);
3003 __free_page(page);
3004 }
3005
3006 was_depleted = (skdev->skspcl_free_list == NULL);
3007
3008 skspcl->req.state = SKD_REQ_STATE_IDLE;
3009 skspcl->req.id += SKD_ID_INCR;
3010 skspcl->req.next =
3011 (struct skd_request_context *)skdev->skspcl_free_list;
3012 skdev->skspcl_free_list = (struct skd_special_context *)skspcl;
3013
3014 if (was_depleted) {
2e44b427 3015 pr_debug("%s:%s:%d skspcl was depleted\n",
3016 skdev->name, __func__, __LINE__);
e67f86b3
AB		 3017 /* Free list was depleted. Their might be waiters. */
3018 wake_up_interruptible(&skdev->waitq);
3019 }
3020}
3021
3022static void skd_reset_skcomp(struct skd_device *skdev)
3023{
3024 u32 nbytes;
3025 struct fit_completion_entry_v1 *skcomp;
3026
3027 nbytes = sizeof(*skcomp) * SKD_N_COMPLETION_ENTRY;
3028 nbytes += sizeof(struct fit_comp_error_info) * SKD_N_COMPLETION_ENTRY;
3029
3030 memset(skdev->skcomp_table, 0, nbytes);
3031
3032 skdev->skcomp_ix = 0;
3033 skdev->skcomp_cycle = 1;
3034}
3035
3036/*
3037 *****************************************************************************
3038 * INTERRUPTS
3039 *****************************************************************************
3040 */
3041static void skd_completion_worker(struct work_struct *work)
3042{
3043 struct skd_device *skdev =
3044 container_of(work, struct skd_device, completion_worker);
3045 unsigned long flags;
3046 int flush_enqueued = 0;
3047
3048 spin_lock_irqsave(&skdev->lock, flags);
3049
3050 /*
3051 * pass in limit=0, which means no limit..
3052 * process everything in compq
3053 */
3054 skd_isr_completion_posted(skdev, 0, &flush_enqueued);
3055 skd_request_fn(skdev->queue);
3056
3057 spin_unlock_irqrestore(&skdev->lock, flags);
3058}
3059
3060static void skd_isr_msg_from_dev(struct skd_device *skdev);
3061
3062irqreturn_t
3063static skd_isr(int irq, void *ptr)
3064{
3065 struct skd_device *skdev;
3066 u32 intstat;
3067 u32 ack;
3068 int rc = 0;
3069 int deferred = 0;
3070 int flush_enqueued = 0;
3071
3072 skdev = (struct skd_device *)ptr;
3073 spin_lock(&skdev->lock);
3074
3075 for (;; ) {
3076 intstat = SKD_READL(skdev, FIT_INT_STATUS_HOST);
3077
3078 ack = FIT_INT_DEF_MASK;
3079 ack &= intstat;
3080
2e44b427 3081 pr_debug("%s:%s:%d intstat=0x%x ack=0x%x\n",
3082 skdev->name, __func__, __LINE__, intstat, ack);
e67f86b3
AB		 3083
3084 /* As long as there is an int pending on device, keep
3085 * running loop. When none, get out, but if we've never
3086 * done any processing, call completion handler?
3087 */
3088 if (ack == 0) {
3089 /* No interrupts on device, but run the completion
3090 * processor anyway?
3091 */
3092 if (rc == 0)
3093 if (likely (skdev->state
3094 == SKD_DRVR_STATE_ONLINE))
3095 deferred = 1;
3096 break;
3097 }
3098
3099 rc = IRQ_HANDLED;
3100
3101 SKD_WRITEL(skdev, ack, FIT_INT_STATUS_HOST);
3102
3103 if (likely((skdev->state != SKD_DRVR_STATE_LOAD) &&
3104 (skdev->state != SKD_DRVR_STATE_STOPPING))) {
3105 if (intstat & FIT_ISH_COMPLETION_POSTED) {
3106 /*
3107 * If we have already deferred completion
3108 * processing, don't bother running it again
3109 */
3110 if (deferred == 0)
3111 deferred =
3112 skd_isr_completion_posted(skdev,
3113 skd_isr_comp_limit, &flush_enqueued);
3114 }
3115
3116 if (intstat & FIT_ISH_FW_STATE_CHANGE) {
3117 skd_isr_fwstate(skdev);
3118 if (skdev->state == SKD_DRVR_STATE_FAULT ||
3119 skdev->state ==
3120 SKD_DRVR_STATE_DISAPPEARED) {
3121 spin_unlock(&skdev->lock);
3122 return rc;
3123 }
3124 }
3125
3126 if (intstat & FIT_ISH_MSG_FROM_DEV)
3127 skd_isr_msg_from_dev(skdev);
3128 }
3129 }
3130
3131 if (unlikely(flush_enqueued))
3132 skd_request_fn(skdev->queue);
3133
3134 if (deferred)
3135 schedule_work(&skdev->completion_worker);
3136 else if (!flush_enqueued)
3137 skd_request_fn(skdev->queue);
3138
3139 spin_unlock(&skdev->lock);
3140
3141 return rc;
3142}
3143
3144
3145static void skd_drive_fault(struct skd_device *skdev)
3146{
3147 skdev->state = SKD_DRVR_STATE_FAULT;
3148 pr_err("(%s): Drive FAULT\n", skd_name(skdev));
3149}
3150
3151static void skd_drive_disappeared(struct skd_device *skdev)
3152{
3153 skdev->state = SKD_DRVR_STATE_DISAPPEARED;
3154 pr_err("(%s): Drive DISAPPEARED\n", skd_name(skdev));
3155}
3156
3157static void skd_isr_fwstate(struct skd_device *skdev)
3158{
3159 u32 sense;
3160 u32 state;
3161 u32 mtd;
3162 int prev_driver_state = skdev->state;
3163
3164 sense = SKD_READL(skdev, FIT_STATUS);
3165 state = sense & FIT_SR_DRIVE_STATE_MASK;
3166
3167 pr_err("(%s): s1120 state %s(%d)=>%s(%d)\n",
3168 skd_name(skdev),
3169 skd_drive_state_to_str(skdev->drive_state), skdev->drive_state,
3170 skd_drive_state_to_str(state), state);
3171
3172 skdev->drive_state = state;
3173
3174 switch (skdev->drive_state) {
3175 case FIT_SR_DRIVE_INIT:
3176 if (skdev->state == SKD_DRVR_STATE_PROTOCOL_MISMATCH) {
3177 skd_disable_interrupts(skdev);
3178 break;
3179 }
3180 if (skdev->state == SKD_DRVR_STATE_RESTARTING)
3181 skd_recover_requests(skdev, 0);
3182 if (skdev->state == SKD_DRVR_STATE_WAIT_BOOT) {
3183 skdev->timer_countdown = SKD_STARTING_TIMO;
3184 skdev->state = SKD_DRVR_STATE_STARTING;
3185 skd_soft_reset(skdev);
3186 break;
3187 }
3188 mtd = FIT_MXD_CONS(FIT_MTD_FITFW_INIT, 0, 0);
3189 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3190 skdev->last_mtd = mtd;
3191 break;
3192
3193 case FIT_SR_DRIVE_ONLINE:
3194 skdev->cur_max_queue_depth = skd_max_queue_depth;
3195 if (skdev->cur_max_queue_depth > skdev->dev_max_queue_depth)
3196 skdev->cur_max_queue_depth = skdev->dev_max_queue_depth;
3197
3198 skdev->queue_low_water_mark =
3199 skdev->cur_max_queue_depth * 2 / 3 + 1;
3200 if (skdev->queue_low_water_mark < 1)
3201 skdev->queue_low_water_mark = 1;
3202 pr_info(
3203 "(%s): Queue depth limit=%d dev=%d lowat=%d\n",
3204 skd_name(skdev),
3205 skdev->cur_max_queue_depth,
3206 skdev->dev_max_queue_depth, skdev->queue_low_water_mark);
3207
3208 skd_refresh_device_data(skdev);
3209 break;
3210
3211 case FIT_SR_DRIVE_BUSY:
3212 skdev->state = SKD_DRVR_STATE_BUSY;
3213 skdev->timer_countdown = SKD_BUSY_TIMO;
3214 skd_quiesce_dev(skdev);
3215 break;
3216 case FIT_SR_DRIVE_BUSY_SANITIZE:
3217 /* set timer for 3 seconds, we'll abort any unfinished
3218 * commands after that expires
3219 */
3220 skdev->state = SKD_DRVR_STATE_BUSY_SANITIZE;
3221 skdev->timer_countdown = SKD_TIMER_SECONDS(3);
6a5ec65b 3222 blk_start_queue(skdev->queue);
e67f86b3
AB		 3223 break;
3224 case FIT_SR_DRIVE_BUSY_ERASE:
3225 skdev->state = SKD_DRVR_STATE_BUSY_ERASE;
3226 skdev->timer_countdown = SKD_BUSY_TIMO;
3227 break;
3228 case FIT_SR_DRIVE_OFFLINE:
3229 skdev->state = SKD_DRVR_STATE_IDLE;
3230 break;
3231 case FIT_SR_DRIVE_SOFT_RESET:
3232 switch (skdev->state) {
3233 case SKD_DRVR_STATE_STARTING:
3234 case SKD_DRVR_STATE_RESTARTING:
3235 /* Expected by a caller of skd_soft_reset() */
3236 break;
3237 default:
3238 skdev->state = SKD_DRVR_STATE_RESTARTING;
3239 break;
3240 }
3241 break;
3242 case FIT_SR_DRIVE_FW_BOOTING:
2e44b427 3243 pr_debug("%s:%s:%d ISR FIT_SR_DRIVE_FW_BOOTING %s\n",
3244 skdev->name, __func__, __LINE__, skdev->name);
e67f86b3
AB		 3245 skdev->state = SKD_DRVR_STATE_WAIT_BOOT;
3246 skdev->timer_countdown = SKD_WAIT_BOOT_TIMO;
3247 break;
3248
3249 case FIT_SR_DRIVE_DEGRADED:
3250 case FIT_SR_PCIE_LINK_DOWN:
3251 case FIT_SR_DRIVE_NEED_FW_DOWNLOAD:
3252 break;
3253
3254 case FIT_SR_DRIVE_FAULT:
3255 skd_drive_fault(skdev);
3256 skd_recover_requests(skdev, 0);
6a5ec65b 3257 blk_start_queue(skdev->queue);
e67f86b3
AB		 3258 break;
3259
3260 /* PCIe bus returned all Fs? */
3261 case 0xFF:
3262 pr_info("(%s): state=0x%x sense=0x%x\n",
3263 skd_name(skdev), state, sense);
3264 skd_drive_disappeared(skdev);
3265 skd_recover_requests(skdev, 0);
6a5ec65b 3266 blk_start_queue(skdev->queue);
e67f86b3
AB		 3267 break;
3268 default:
3269 /*
3270 * Uknown FW State. Wait for a state we recognize.
3271 */
3272 break;
3273 }
3274 pr_err("(%s): Driver state %s(%d)=>%s(%d)\n",
3275 skd_name(skdev),
3276 skd_skdev_state_to_str(prev_driver_state), prev_driver_state,
3277 skd_skdev_state_to_str(skdev->state), skdev->state);
3278}
3279
3280static void skd_recover_requests(struct skd_device *skdev, int requeue)
3281{
3282 int i;
3283
3284 for (i = 0; i < skdev->num_req_context; i++) {
3285 struct skd_request_context *skreq = &skdev->skreq_table[i];
3286
3287 if (skreq->state == SKD_REQ_STATE_BUSY) {
3288 skd_log_skreq(skdev, skreq, "recover");
3289
3290 SKD_ASSERT((skreq->id & SKD_ID_INCR) != 0);
fcd37eb3 3291 SKD_ASSERT(skreq->req != NULL);
e67f86b3
AB		 3292
3293 /* Release DMA resources for the request. */
3294 if (skreq->n_sg > 0)
3295 skd_postop_sg_list(skdev, skreq);
3296
fcd37eb3
JA		 3297 if (requeue &&
3298 (unsigned long) ++skreq->req->special <
3299 SKD_MAX_RETRIES)
3300 skd_requeue_request(skdev, skreq);
3301 else
e67f86b3
AB		 3302 skd_end_request(skdev, skreq, -EIO);
3303
fcd37eb3 3304 skreq->req = NULL;
e67f86b3
AB		 3305
3306 skreq->state = SKD_REQ_STATE_IDLE;
3307 skreq->id += SKD_ID_INCR;
3308
3309
3310 }
3311 if (i > 0)
3312 skreq[-1].next = skreq;
3313 skreq->next = NULL;
3314 }
3315 skdev->skreq_free_list = skdev->skreq_table;
3316
3317 for (i = 0; i < skdev->num_fitmsg_context; i++) {
3318 struct skd_fitmsg_context *skmsg = &skdev->skmsg_table[i];
3319
3320 if (skmsg->state == SKD_MSG_STATE_BUSY) {
3321 skd_log_skmsg(skdev, skmsg, "salvaged");
3322 SKD_ASSERT((skmsg->id & SKD_ID_INCR) != 0);
3323 skmsg->state = SKD_MSG_STATE_IDLE;
3324 skmsg->id += SKD_ID_INCR;
3325 }
3326 if (i > 0)
3327 skmsg[-1].next = skmsg;
3328 skmsg->next = NULL;
3329 }
3330 skdev->skmsg_free_list = skdev->skmsg_table;
3331
3332 for (i = 0; i < skdev->n_special; i++) {
3333 struct skd_special_context *skspcl = &skdev->skspcl_table[i];
3334
3335 /* If orphaned, reclaim it because it has already been reported
3336 * to the process as an error (it was just waiting for
3337 * a completion that didn't come, and now it will never come)
3338 * If busy, change to a state that will cause it to error
3339 * out in the wait routine and let it do the normal
3340 * reporting and reclaiming
3341 */
3342 if (skspcl->req.state == SKD_REQ_STATE_BUSY) {
3343 if (skspcl->orphaned) {
2e44b427 3344 pr_debug("%s:%s:%d orphaned %p\n",
3345 skdev->name, __func__, __LINE__,
3346 skspcl);
e67f86b3
AB		 3347 skd_release_special(skdev, skspcl);
3348 } else {
2e44b427 3349 pr_debug("%s:%s:%d not orphaned %p\n",
3350 skdev->name, __func__, __LINE__,
3351 skspcl);
e67f86b3
AB		 3352 skspcl->req.state = SKD_REQ_STATE_ABORTED;
3353 }
3354 }
3355 }
3356 skdev->skspcl_free_list = skdev->skspcl_table;
3357
3358 for (i = 0; i < SKD_N_TIMEOUT_SLOT; i++)
3359 skdev->timeout_slot[i] = 0;
3360
3361 skdev->in_flight = 0;
3362}
3363
3364static void skd_isr_msg_from_dev(struct skd_device *skdev)
3365{
3366 u32 mfd;
3367 u32 mtd;
3368 u32 data;
3369
3370 mfd = SKD_READL(skdev, FIT_MSG_FROM_DEVICE);
3371
2e44b427 3372 pr_debug("%s:%s:%d mfd=0x%x last_mtd=0x%x\n",
3373 skdev->name, __func__, __LINE__, mfd, skdev->last_mtd);
e67f86b3
AB		 3374
3375 /* ignore any mtd that is an ack for something we didn't send */
3376 if (FIT_MXD_TYPE(mfd) != FIT_MXD_TYPE(skdev->last_mtd))
3377 return;
3378
3379 switch (FIT_MXD_TYPE(mfd)) {
3380 case FIT_MTD_FITFW_INIT:
3381 skdev->proto_ver = FIT_PROTOCOL_MAJOR_VER(mfd);
3382
3383 if (skdev->proto_ver != FIT_PROTOCOL_VERSION_1) {
3384 pr_err("(%s): protocol mismatch\n",
3385 skdev->name);
3386 pr_err("(%s): got=%d support=%d\n",
3387 skdev->name, skdev->proto_ver,
3388 FIT_PROTOCOL_VERSION_1);
3389 pr_err("(%s): please upgrade driver\n",
3390 skdev->name);
3391 skdev->state = SKD_DRVR_STATE_PROTOCOL_MISMATCH;
3392 skd_soft_reset(skdev);
3393 break;
3394 }
3395 mtd = FIT_MXD_CONS(FIT_MTD_GET_CMDQ_DEPTH, 0, 0);
3396 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3397 skdev->last_mtd = mtd;
3398 break;
3399
3400 case FIT_MTD_GET_CMDQ_DEPTH:
3401 skdev->dev_max_queue_depth = FIT_MXD_DATA(mfd);
3402 mtd = FIT_MXD_CONS(FIT_MTD_SET_COMPQ_DEPTH, 0,
3403 SKD_N_COMPLETION_ENTRY);
3404 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3405 skdev->last_mtd = mtd;
3406 break;
3407
3408 case FIT_MTD_SET_COMPQ_DEPTH:
3409 SKD_WRITEQ(skdev, skdev->cq_dma_address, FIT_MSG_TO_DEVICE_ARG);
3410 mtd = FIT_MXD_CONS(FIT_MTD_SET_COMPQ_ADDR, 0, 0);
3411 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3412 skdev->last_mtd = mtd;
3413 break;
3414
3415 case FIT_MTD_SET_COMPQ_ADDR:
3416 skd_reset_skcomp(skdev);
3417 mtd = FIT_MXD_CONS(FIT_MTD_CMD_LOG_HOST_ID, 0, skdev->devno);
3418 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3419 skdev->last_mtd = mtd;
3420 break;
3421
3422 case FIT_MTD_CMD_LOG_HOST_ID:
3423 skdev->connect_time_stamp = get_seconds();
3424 data = skdev->connect_time_stamp & 0xFFFF;
3425 mtd = FIT_MXD_CONS(FIT_MTD_CMD_LOG_TIME_STAMP_LO, 0, data);
3426 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3427 skdev->last_mtd = mtd;
3428 break;
3429
3430 case FIT_MTD_CMD_LOG_TIME_STAMP_LO:
3431 skdev->drive_jiffies = FIT_MXD_DATA(mfd);
3432 data = (skdev->connect_time_stamp >> 16) & 0xFFFF;
3433 mtd = FIT_MXD_CONS(FIT_MTD_CMD_LOG_TIME_STAMP_HI, 0, data);
3434 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3435 skdev->last_mtd = mtd;
3436 break;
3437
3438 case FIT_MTD_CMD_LOG_TIME_STAMP_HI:
3439 skdev->drive_jiffies |= (FIT_MXD_DATA(mfd) << 16);
3440 mtd = FIT_MXD_CONS(FIT_MTD_ARM_QUEUE, 0, 0);
3441 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3442 skdev->last_mtd = mtd;
3443
3444 pr_err("(%s): Time sync driver=0x%x device=0x%x\n",
3445 skd_name(skdev),
3446 skdev->connect_time_stamp, skdev->drive_jiffies);
3447 break;
3448
3449 case FIT_MTD_ARM_QUEUE:
3450 skdev->last_mtd = 0;
3451 /*
3452 * State should be, or soon will be, FIT_SR_DRIVE_ONLINE.
3453 */
3454 break;
3455
3456 default:
3457 break;
3458 }
3459}
3460
3461static void skd_disable_interrupts(struct skd_device *skdev)
3462{
3463 u32 sense;
3464
3465 sense = SKD_READL(skdev, FIT_CONTROL);
3466 sense &= ~FIT_CR_ENABLE_INTERRUPTS;
3467 SKD_WRITEL(skdev, sense, FIT_CONTROL);
2e44b427 3468 pr_debug("%s:%s:%d sense 0x%x\n",
3469 skdev->name, __func__, __LINE__, sense);
e67f86b3
AB		 3470
3471 /* Note that the 1s is written. A 1-bit means
3472 * disable, a 0 means enable.
3473 */
3474 SKD_WRITEL(skdev, ~0, FIT_INT_MASK_HOST);
3475}
3476
3477static void skd_enable_interrupts(struct skd_device *skdev)
3478{
3479 u32 val;
3480
3481 /* unmask interrupts first */
3482 val = FIT_ISH_FW_STATE_CHANGE +
3483 FIT_ISH_COMPLETION_POSTED + FIT_ISH_MSG_FROM_DEV;
3484
3485 /* Note that the compliment of mask is written. A 1-bit means
3486 * disable, a 0 means enable. */
3487 SKD_WRITEL(skdev, ~val, FIT_INT_MASK_HOST);
2e44b427 3488 pr_debug("%s:%s:%d interrupt mask=0x%x\n",
3489 skdev->name, __func__, __LINE__, ~val);
e67f86b3
AB		 3490
3491 val = SKD_READL(skdev, FIT_CONTROL);
3492 val |= FIT_CR_ENABLE_INTERRUPTS;
2e44b427 3493 pr_debug("%s:%s:%d control=0x%x\n",
3494 skdev->name, __func__, __LINE__, val);
e67f86b3
AB		 3495 SKD_WRITEL(skdev, val, FIT_CONTROL);
3496}
3497
3498/*
3499 *****************************************************************************
3500 * START, STOP, RESTART, QUIESCE, UNQUIESCE
3501 *****************************************************************************
3502 */
3503
3504static void skd_soft_reset(struct skd_device *skdev)
3505{
3506 u32 val;
3507
3508 val = SKD_READL(skdev, FIT_CONTROL);
3509 val |= (FIT_CR_SOFT_RESET);
2e44b427 3510 pr_debug("%s:%s:%d control=0x%x\n",
3511 skdev->name, __func__, __LINE__, val);
e67f86b3
AB		 3512 SKD_WRITEL(skdev, val, FIT_CONTROL);
3513}
3514
3515static void skd_start_device(struct skd_device *skdev)
3516{
3517 unsigned long flags;
3518 u32 sense;
3519 u32 state;
3520
3521 spin_lock_irqsave(&skdev->lock, flags);
3522
3523 /* ack all ghost interrupts */
3524 SKD_WRITEL(skdev, FIT_INT_DEF_MASK, FIT_INT_STATUS_HOST);
3525
3526 sense = SKD_READL(skdev, FIT_STATUS);
3527
2e44b427 3528 pr_debug("%s:%s:%d initial status=0x%x\n",
3529 skdev->name, __func__, __LINE__, sense);
e67f86b3
AB		 3530
3531 state = sense & FIT_SR_DRIVE_STATE_MASK;
3532 skdev->drive_state = state;
3533 skdev->last_mtd = 0;
3534
3535 skdev->state = SKD_DRVR_STATE_STARTING;
3536 skdev->timer_countdown = SKD_STARTING_TIMO;
3537
3538 skd_enable_interrupts(skdev);
3539
3540 switch (skdev->drive_state) {
3541 case FIT_SR_DRIVE_OFFLINE:
3542 pr_err("(%s): Drive offline...\n", skd_name(skdev));
3543 break;
3544
3545 case FIT_SR_DRIVE_FW_BOOTING:
2e44b427 3546 pr_debug("%s:%s:%d FIT_SR_DRIVE_FW_BOOTING %s\n",
3547 skdev->name, __func__, __LINE__, skdev->name);
e67f86b3
AB		 3548 skdev->state = SKD_DRVR_STATE_WAIT_BOOT;
3549 skdev->timer_countdown = SKD_WAIT_BOOT_TIMO;
3550 break;
3551
3552 case FIT_SR_DRIVE_BUSY_SANITIZE:
3553 pr_info("(%s): Start: BUSY_SANITIZE\n",
3554 skd_name(skdev));
3555 skdev->state = SKD_DRVR_STATE_BUSY_SANITIZE;
3556 skdev->timer_countdown = SKD_STARTED_BUSY_TIMO;
3557 break;
3558
3559 case FIT_SR_DRIVE_BUSY_ERASE:
3560 pr_info("(%s): Start: BUSY_ERASE\n", skd_name(skdev));
3561 skdev->state = SKD_DRVR_STATE_BUSY_ERASE;
3562 skdev->timer_countdown = SKD_STARTED_BUSY_TIMO;
3563 break;
3564
3565 case FIT_SR_DRIVE_INIT:
3566 case FIT_SR_DRIVE_ONLINE:
3567 skd_soft_reset(skdev);
3568 break;
3569
3570 case FIT_SR_DRIVE_BUSY:
3571 pr_err("(%s): Drive Busy...\n", skd_name(skdev));
3572 skdev->state = SKD_DRVR_STATE_BUSY;
3573 skdev->timer_countdown = SKD_STARTED_BUSY_TIMO;
3574 break;
3575
3576 case FIT_SR_DRIVE_SOFT_RESET:
3577 pr_err("(%s) drive soft reset in prog\n",
3578 skd_name(skdev));
3579 break;
3580
3581 case FIT_SR_DRIVE_FAULT:
3582 /* Fault state is bad...soft reset won't do it...
3583 * Hard reset, maybe, but does it work on device?
3584 * For now, just fault so the system doesn't hang.
3585 */
3586 skd_drive_fault(skdev);
3587 /*start the queue so we can respond with error to requests */
2e44b427 3588 pr_debug("%s:%s:%d starting %s queue\n",
3589 skdev->name, __func__, __LINE__, skdev->name);
6a5ec65b 3590 blk_start_queue(skdev->queue);
e67f86b3
AB		 3591 skdev->gendisk_on = -1;
3592 wake_up_interruptible(&skdev->waitq);
3593 break;
3594
3595 case 0xFF:
3596 /* Most likely the device isn't there or isn't responding
3597 * to the BAR1 addresses. */
3598 skd_drive_disappeared(skdev);
3599 /*start the queue so we can respond with error to requests */
2e44b427 3600 pr_debug("%s:%s:%d starting %s queue to error-out reqs\n",
3601 skdev->name, __func__, __LINE__, skdev->name);
6a5ec65b 3602 blk_start_queue(skdev->queue);
e67f86b3
AB		 3603 skdev->gendisk_on = -1;
3604 wake_up_interruptible(&skdev->waitq);
3605 break;
3606
3607 default:
3608 pr_err("(%s) Start: unknown state %x\n",
3609 skd_name(skdev), skdev->drive_state);
3610 break;
3611 }
3612
3613 state = SKD_READL(skdev, FIT_CONTROL);
2e44b427 3614 pr_debug("%s:%s:%d FIT Control Status=0x%x\n",
3615 skdev->name, __func__, __LINE__, state);
e67f86b3
AB		 3616
3617 state = SKD_READL(skdev, FIT_INT_STATUS_HOST);
2e44b427 3618 pr_debug("%s:%s:%d Intr Status=0x%x\n",
3619 skdev->name, __func__, __LINE__, state);
e67f86b3
AB		 3620
3621 state = SKD_READL(skdev, FIT_INT_MASK_HOST);
2e44b427 3622 pr_debug("%s:%s:%d Intr Mask=0x%x\n",
3623 skdev->name, __func__, __LINE__, state);
e67f86b3
AB		 3624
3625 state = SKD_READL(skdev, FIT_MSG_FROM_DEVICE);
2e44b427 3626 pr_debug("%s:%s:%d Msg from Dev=0x%x\n",
3627 skdev->name, __func__, __LINE__, state);
e67f86b3
AB		 3628
3629 state = SKD_READL(skdev, FIT_HW_VERSION);
2e44b427 3630 pr_debug("%s:%s:%d HW version=0x%x\n",
3631 skdev->name, __func__, __LINE__, state);
e67f86b3
AB		 3632
3633 spin_unlock_irqrestore(&skdev->lock, flags);
3634}
3635
3636static void skd_stop_device(struct skd_device *skdev)
3637{
3638 unsigned long flags;
3639 struct skd_special_context *skspcl = &skdev->internal_skspcl;
3640 u32 dev_state;
3641 int i;
3642
3643 spin_lock_irqsave(&skdev->lock, flags);
3644
3645 if (skdev->state != SKD_DRVR_STATE_ONLINE) {
3646 pr_err("(%s): skd_stop_device not online no sync\n",
3647 skd_name(skdev));
3648 goto stop_out;
3649 }
3650
3651 if (skspcl->req.state != SKD_REQ_STATE_IDLE) {
3652 pr_err("(%s): skd_stop_device no special\n",
3653 skd_name(skdev));
3654 goto stop_out;
3655 }
3656
3657 skdev->state = SKD_DRVR_STATE_SYNCING;
3658 skdev->sync_done = 0;
3659
3660 skd_send_internal_skspcl(skdev, skspcl, SYNCHRONIZE_CACHE);
3661
3662 spin_unlock_irqrestore(&skdev->lock, flags);
3663
3664 wait_event_interruptible_timeout(skdev->waitq,
3665 (skdev->sync_done), (10 * HZ));
3666
3667 spin_lock_irqsave(&skdev->lock, flags);
3668
3669 switch (skdev->sync_done) {
3670 case 0:
3671 pr_err("(%s): skd_stop_device no sync\n",
3672 skd_name(skdev));
3673 break;
3674 case 1:
3675 pr_err("(%s): skd_stop_device sync done\n",
3676 skd_name(skdev));
3677 break;
3678 default:
3679 pr_err("(%s): skd_stop_device sync error\n",
3680 skd_name(skdev));
3681 }
3682
3683stop_out:
3684 skdev->state = SKD_DRVR_STATE_STOPPING;
3685 spin_unlock_irqrestore(&skdev->lock, flags);
3686
3687 skd_kill_timer(skdev);
3688
3689 spin_lock_irqsave(&skdev->lock, flags);
3690 skd_disable_interrupts(skdev);
3691
3692 /* ensure all ints on device are cleared */
3693 /* soft reset the device to unload with a clean slate */
3694 SKD_WRITEL(skdev, FIT_INT_DEF_MASK, FIT_INT_STATUS_HOST);
3695 SKD_WRITEL(skdev, FIT_CR_SOFT_RESET, FIT_CONTROL);
3696
3697 spin_unlock_irqrestore(&skdev->lock, flags);
3698
3699 /* poll every 100ms, 1 second timeout */
3700 for (i = 0; i < 10; i++) {
3701 dev_state =
3702 SKD_READL(skdev, FIT_STATUS) & FIT_SR_DRIVE_STATE_MASK;
3703 if (dev_state == FIT_SR_DRIVE_INIT)
3704 break;
3705 set_current_state(TASK_INTERRUPTIBLE);
3706 schedule_timeout(msecs_to_jiffies(100));
3707 }
3708
3709 if (dev_state != FIT_SR_DRIVE_INIT)
3710 pr_err("(%s): skd_stop_device state error 0x%02x\n",
3711 skd_name(skdev), dev_state);
3712}
3713
3714/* assume spinlock is held */
3715static void skd_restart_device(struct skd_device *skdev)
3716{
3717 u32 state;
3718
3719 /* ack all ghost interrupts */
3720 SKD_WRITEL(skdev, FIT_INT_DEF_MASK, FIT_INT_STATUS_HOST);
3721
3722 state = SKD_READL(skdev, FIT_STATUS);
3723
2e44b427 3724 pr_debug("%s:%s:%d drive status=0x%x\n",
3725 skdev->name, __func__, __LINE__, state);
e67f86b3
AB		 3726
3727 state &= FIT_SR_DRIVE_STATE_MASK;
3728 skdev->drive_state = state;
3729 skdev->last_mtd = 0;
3730
3731 skdev->state = SKD_DRVR_STATE_RESTARTING;
3732 skdev->timer_countdown = SKD_RESTARTING_TIMO;
3733
3734 skd_soft_reset(skdev);
3735}
3736
3737/* assume spinlock is held */
3738static int skd_quiesce_dev(struct skd_device *skdev)
3739{
3740 int rc = 0;
3741
3742 switch (skdev->state) {
3743 case SKD_DRVR_STATE_BUSY:
3744 case SKD_DRVR_STATE_BUSY_IMMINENT:
2e44b427 3745 pr_debug("%s:%s:%d stopping %s queue\n",
3746 skdev->name, __func__, __LINE__, skdev->name);
6a5ec65b 3747 blk_stop_queue(skdev->queue);
e67f86b3
AB		 3748 break;
3749 case SKD_DRVR_STATE_ONLINE:
3750 case SKD_DRVR_STATE_STOPPING:
3751 case SKD_DRVR_STATE_SYNCING:
3752 case SKD_DRVR_STATE_PAUSING:
3753 case SKD_DRVR_STATE_PAUSED:
3754 case SKD_DRVR_STATE_STARTING:
3755 case SKD_DRVR_STATE_RESTARTING:
3756 case SKD_DRVR_STATE_RESUMING:
3757 default:
3758 rc = -EINVAL;
2e44b427 3759 pr_debug("%s:%s:%d state [%d] not implemented\n",
3760 skdev->name, __func__, __LINE__, skdev->state);
e67f86b3
AB		 3761 }
3762 return rc;
3763}
3764
3765/* assume spinlock is held */
3766static int skd_unquiesce_dev(struct skd_device *skdev)
3767{
3768 int prev_driver_state = skdev->state;
3769
3770 skd_log_skdev(skdev, "unquiesce");
3771 if (skdev->state == SKD_DRVR_STATE_ONLINE) {
2e44b427 3772 pr_debug("%s:%s:%d **** device already ONLINE\n",
3773 skdev->name, __func__, __LINE__);
e67f86b3
AB		 3774 return 0;
3775 }
3776 if (skdev->drive_state != FIT_SR_DRIVE_ONLINE) {
3777 /*
3778 * If there has been an state change to other than
3779 * ONLINE, we will rely on controller state change
3780 * to come back online and restart the queue.
3781 * The BUSY state means that driver is ready to
3782 * continue normal processing but waiting for controller
3783 * to become available.
3784 */
3785 skdev->state = SKD_DRVR_STATE_BUSY;
2e44b427 3786 pr_debug("%s:%s:%d drive BUSY state\n",
3787 skdev->name, __func__, __LINE__);
e67f86b3
AB		 3788 return 0;
3789 }
3790
3791 /*
3792 * Drive has just come online, driver is either in startup,
3793 * paused performing a task, or bust waiting for hardware.
3794 */
3795 switch (skdev->state) {
3796 case SKD_DRVR_STATE_PAUSED:
3797 case SKD_DRVR_STATE_BUSY:
3798 case SKD_DRVR_STATE_BUSY_IMMINENT:
3799 case SKD_DRVR_STATE_BUSY_ERASE:
3800 case SKD_DRVR_STATE_STARTING:
3801 case SKD_DRVR_STATE_RESTARTING:
3802 case SKD_DRVR_STATE_FAULT:
3803 case SKD_DRVR_STATE_IDLE:
3804 case SKD_DRVR_STATE_LOAD:
3805 skdev->state = SKD_DRVR_STATE_ONLINE;
3806 pr_err("(%s): Driver state %s(%d)=>%s(%d)\n",
3807 skd_name(skdev),
3808 skd_skdev_state_to_str(prev_driver_state),
3809 prev_driver_state, skd_skdev_state_to_str(skdev->state),
3810 skdev->state);
2e44b427 3811 pr_debug("%s:%s:%d **** device ONLINE...starting block queue\n",
3812 skdev->name, __func__, __LINE__);
3813 pr_debug("%s:%s:%d starting %s queue\n",
3814 skdev->name, __func__, __LINE__, skdev->name);
e67f86b3 3815 pr_info("(%s): STEC s1120 ONLINE\n", skd_name(skdev));
6a5ec65b 3816 blk_start_queue(skdev->queue);
e67f86b3
AB		 3817 skdev->gendisk_on = 1;
3818 wake_up_interruptible(&skdev->waitq);
3819 break;
3820
3821 case SKD_DRVR_STATE_DISAPPEARED:
3822 default:
2e44b427 3823 pr_debug("%s:%s:%d **** driver state %d, not implemented \n",
3824 skdev->name, __func__, __LINE__,
3825 skdev->state);
e67f86b3
AB		 3826 return -EBUSY;
3827 }
3828 return 0;
3829}
3830
3831/*
3832 *****************************************************************************
3833 * PCIe MSI/MSI-X INTERRUPT HANDLERS
3834 *****************************************************************************
3835 */
3836
3837static irqreturn_t skd_reserved_isr(int irq, void *skd_host_data)
3838{
3839 struct skd_device *skdev = skd_host_data;
3840 unsigned long flags;
3841
3842 spin_lock_irqsave(&skdev->lock, flags);
2e44b427 3843 pr_debug("%s:%s:%d MSIX = 0x%x\n",
3844 skdev->name, __func__, __LINE__,
3845 SKD_READL(skdev, FIT_INT_STATUS_HOST));
e67f86b3
AB		 3846 pr_err("(%s): MSIX reserved irq %d = 0x%x\n", skd_name(skdev),
3847 irq, SKD_READL(skdev, FIT_INT_STATUS_HOST));
3848 SKD_WRITEL(skdev, FIT_INT_RESERVED_MASK, FIT_INT_STATUS_HOST);
3849 spin_unlock_irqrestore(&skdev->lock, flags);
3850 return IRQ_HANDLED;
3851}
3852
3853static irqreturn_t skd_statec_isr(int irq, void *skd_host_data)
3854{
3855 struct skd_device *skdev = skd_host_data;
3856 unsigned long flags;
3857
3858 spin_lock_irqsave(&skdev->lock, flags);
2e44b427 3859 pr_debug("%s:%s:%d MSIX = 0x%x\n",
3860 skdev->name, __func__, __LINE__,
3861 SKD_READL(skdev, FIT_INT_STATUS_HOST));
e67f86b3
AB		 3862 SKD_WRITEL(skdev, FIT_ISH_FW_STATE_CHANGE, FIT_INT_STATUS_HOST);
3863 skd_isr_fwstate(skdev);
3864 spin_unlock_irqrestore(&skdev->lock, flags);
3865 return IRQ_HANDLED;
3866}
3867
3868static irqreturn_t skd_comp_q(int irq, void *skd_host_data)
3869{
3870 struct skd_device *skdev = skd_host_data;
3871 unsigned long flags;
3872 int flush_enqueued = 0;
3873 int deferred;
3874
3875 spin_lock_irqsave(&skdev->lock, flags);
2e44b427 3876 pr_debug("%s:%s:%d MSIX = 0x%x\n",
3877 skdev->name, __func__, __LINE__,
3878 SKD_READL(skdev, FIT_INT_STATUS_HOST));
e67f86b3
AB		 3879 SKD_WRITEL(skdev, FIT_ISH_COMPLETION_POSTED, FIT_INT_STATUS_HOST);
3880 deferred = skd_isr_completion_posted(skdev, skd_isr_comp_limit,
3881 &flush_enqueued);
3882
3883 if (flush_enqueued)
3884 skd_request_fn(skdev->queue);
3885
3886 if (deferred)
3887 schedule_work(&skdev->completion_worker);
3888 else if (!flush_enqueued)
3889 skd_request_fn(skdev->queue);
3890
3891 spin_unlock_irqrestore(&skdev->lock, flags);
3892
3893 return IRQ_HANDLED;
3894}
3895
3896static irqreturn_t skd_msg_isr(int irq, void *skd_host_data)
3897{
3898 struct skd_device *skdev = skd_host_data;
3899 unsigned long flags;
3900
3901 spin_lock_irqsave(&skdev->lock, flags);
2e44b427 3902 pr_debug("%s:%s:%d MSIX = 0x%x\n",
3903 skdev->name, __func__, __LINE__,
3904 SKD_READL(skdev, FIT_INT_STATUS_HOST));
e67f86b3
AB		 3905 SKD_WRITEL(skdev, FIT_ISH_MSG_FROM_DEV, FIT_INT_STATUS_HOST);
3906 skd_isr_msg_from_dev(skdev);
3907 spin_unlock_irqrestore(&skdev->lock, flags);
3908 return IRQ_HANDLED;
3909}
3910
3911static irqreturn_t skd_qfull_isr(int irq, void *skd_host_data)
3912{
3913 struct skd_device *skdev = skd_host_data;
3914 unsigned long flags;
3915
3916 spin_lock_irqsave(&skdev->lock, flags);
2e44b427 3917 pr_debug("%s:%s:%d MSIX = 0x%x\n",
3918 skdev->name, __func__, __LINE__,
3919 SKD_READL(skdev, FIT_INT_STATUS_HOST));
e67f86b3
AB		 3920 SKD_WRITEL(skdev, FIT_INT_QUEUE_FULL, FIT_INT_STATUS_HOST);
3921 spin_unlock_irqrestore(&skdev->lock, flags);
3922 return IRQ_HANDLED;
3923}
3924
3925/*
3926 *****************************************************************************
3927 * PCIe MSI/MSI-X SETUP
3928 *****************************************************************************
3929 */
3930
3931struct skd_msix_entry {
3932 int have_irq;
3933 u32 vector;
3934 u32 entry;
3935 struct skd_device *rsp;
3936 char isr_name[30];
3937};
3938
3939struct skd_init_msix_entry {
3940 const char *name;
3941 irq_handler_t handler;
3942};
3943
3944#define SKD_MAX_MSIX_COUNT 13
3945#define SKD_MIN_MSIX_COUNT 7
3946#define SKD_BASE_MSIX_IRQ 4
3947
3948static struct skd_init_msix_entry msix_entries[SKD_MAX_MSIX_COUNT] = {
3949 { "(DMA 0)", skd_reserved_isr },
3950 { "(DMA 1)", skd_reserved_isr },
3951 { "(DMA 2)", skd_reserved_isr },
3952 { "(DMA 3)", skd_reserved_isr },
3953 { "(State Change)", skd_statec_isr },
3954 { "(COMPL_Q)", skd_comp_q },
3955 { "(MSG)", skd_msg_isr },
3956 { "(Reserved)", skd_reserved_isr },
3957 { "(Reserved)", skd_reserved_isr },
3958 { "(Queue Full 0)", skd_qfull_isr },
3959 { "(Queue Full 1)", skd_qfull_isr },
3960 { "(Queue Full 2)", skd_qfull_isr },
3961 { "(Queue Full 3)", skd_qfull_isr },
3962};
3963
3964static void skd_release_msix(struct skd_device *skdev)
3965{
3966 struct skd_msix_entry *qentry;
3967 int i;
3968
3969 if (skdev->msix_entries == NULL)
3970 return;
3971 for (i = 0; i < skdev->msix_count; i++) {
3972 qentry = &skdev->msix_entries[i];
3973 skdev = qentry->rsp;
3974
3975 if (qentry->have_irq)
3976 devm_free_irq(&skdev->pdev->dev,
3977 qentry->vector, qentry->rsp);
3978 }
3979 pci_disable_msix(skdev->pdev);
3980 kfree(skdev->msix_entries);
3981 skdev->msix_count = 0;
3982 skdev->msix_entries = NULL;
3983}
3984
3985static int skd_acquire_msix(struct skd_device *skdev)
3986{
3987 int i, rc;
3988 struct pci_dev *pdev;
3989 struct msix_entry *entries = NULL;
3990 struct skd_msix_entry *qentry;
3991
3992 pdev = skdev->pdev;
3993 skdev->msix_count = SKD_MAX_MSIX_COUNT;
3994 entries = kzalloc(sizeof(struct msix_entry) * SKD_MAX_MSIX_COUNT,
3995 GFP_KERNEL);
3996 if (!entries)
3997 return -ENOMEM;
3998
3999 for (i = 0; i < SKD_MAX_MSIX_COUNT; i++)
4000 entries[i].entry = i;
4001
4002 rc = pci_enable_msix(pdev, entries, SKD_MAX_MSIX_COUNT);
4003 if (rc < 0)
4004 goto msix_out;
4005 if (rc) {
4006 if (rc < SKD_MIN_MSIX_COUNT) {
4007 pr_err("(%s): failed to enable MSI-X %d\n",
4008 skd_name(skdev), rc);
4009 goto msix_out;
4010 }
2e44b427 4011 pr_debug("%s:%s:%d %s: <%s> allocated %d MSI-X vectors\n",
4012 skdev->name, __func__, __LINE__,
4013 pci_name(pdev), skdev->name, rc);
e67f86b3
AB		 4014
4015 skdev->msix_count = rc;
4016 rc = pci_enable_msix(pdev, entries, skdev->msix_count);
4017 if (rc) {
4018 pr_err("(%s): failed to enable MSI-X "
4019 "support (%d) %d\n",
4020 skd_name(skdev), skdev->msix_count, rc);
4021 goto msix_out;
4022 }
4023 }
4024 skdev->msix_entries = kzalloc(sizeof(struct skd_msix_entry) *
4025 skdev->msix_count, GFP_KERNEL);
4026 if (!skdev->msix_entries) {
4027 rc = -ENOMEM;
4028 skdev->msix_count = 0;
4029 pr_err("(%s): msix table allocation error\n",
4030 skd_name(skdev));
4031 goto msix_out;
4032 }
4033
4034 qentry = skdev->msix_entries;
4035 for (i = 0; i < skdev->msix_count; i++) {
4036 qentry->vector = entries[i].vector;
4037 qentry->entry = entries[i].entry;
4038 qentry->rsp = NULL;
4039 qentry->have_irq = 0;
2e44b427 4040 pr_debug("%s:%s:%d %s: <%s> msix (%d) vec %d, entry %x\n",
4041 skdev->name, __func__, __LINE__,
4042 pci_name(pdev), skdev->name,
4043 i, qentry->vector, qentry->entry);
e67f86b3
AB		 4044 qentry++;
4045 }
4046
4047 /* Enable MSI-X vectors for the base queue */
4048 for (i = 0; i < SKD_MAX_MSIX_COUNT; i++) {
4049 qentry = &skdev->msix_entries[i];
4050 snprintf(qentry->isr_name, sizeof(qentry->isr_name),
4051 "%s%d-msix %s", DRV_NAME, skdev->devno,
4052 msix_entries[i].name);
4053 rc = devm_request_irq(&skdev->pdev->dev, qentry->vector,
4054 msix_entries[i].handler, 0,
4055 qentry->isr_name, skdev);
4056 if (rc) {
4057 pr_err("(%s): Unable to register(%d) MSI-X "
4058 "handler %d: %s\n",
4059 skd_name(skdev), rc, i, qentry->isr_name);
4060 goto msix_out;
4061 } else {
4062 qentry->have_irq = 1;
4063 qentry->rsp = skdev;
4064 }
4065 }
2e44b427 4066 pr_debug("%s:%s:%d %s: <%s> msix %d irq(s) enabled\n",
4067 skdev->name, __func__, __LINE__,
4068 pci_name(pdev), skdev->name, skdev->msix_count);
e67f86b3
AB		 4069 return 0;
4070
4071msix_out:
4072 if (entries)
4073 kfree(entries);
4074 skd_release_msix(skdev);
4075 return rc;
4076}
4077
4078static int skd_acquire_irq(struct skd_device *skdev)
4079{
4080 int rc;
4081 struct pci_dev *pdev;
4082
4083 pdev = skdev->pdev;
4084 skdev->msix_count = 0;
4085
4086RETRY_IRQ_TYPE:
4087 switch (skdev->irq_type) {
4088 case SKD_IRQ_MSIX:
4089 rc = skd_acquire_msix(skdev);
4090 if (!rc)
4091 pr_info("(%s): MSI-X %d irqs enabled\n",
4092 skd_name(skdev), skdev->msix_count);
4093 else {
4094 pr_err(
4095 "(%s): failed to enable MSI-X, re-trying with MSI %d\n",
4096 skd_name(skdev), rc);
4097 skdev->irq_type = SKD_IRQ_MSI;
4098 goto RETRY_IRQ_TYPE;
4099 }
4100 break;
4101 case SKD_IRQ_MSI:
4102 snprintf(skdev->isr_name, sizeof(skdev->isr_name), "%s%d-msi",
4103 DRV_NAME, skdev->devno);
4104 rc = pci_enable_msi(pdev);
4105 if (!rc) {
4106 rc = devm_request_irq(&pdev->dev, pdev->irq, skd_isr, 0,
4107 skdev->isr_name, skdev);
4108 if (rc) {
4109 pci_disable_msi(pdev);
4110 pr_err(
4111 "(%s): failed to allocate the MSI interrupt %d\n",
4112 skd_name(skdev), rc);
4113 goto RETRY_IRQ_LEGACY;
4114 }
4115 pr_info("(%s): MSI irq %d enabled\n",
4116 skd_name(skdev), pdev->irq);
4117 } else {
4118RETRY_IRQ_LEGACY:
4119 pr_err(
4120 "(%s): failed to enable MSI, re-trying with LEGACY %d\n",
4121 skd_name(skdev), rc);
4122 skdev->irq_type = SKD_IRQ_LEGACY;
4123 goto RETRY_IRQ_TYPE;
4124 }
4125 break;
4126 case SKD_IRQ_LEGACY:
4127 snprintf(skdev->isr_name, sizeof(skdev->isr_name),
4128 "%s%d-legacy", DRV_NAME, skdev->devno);
4129 rc = devm_request_irq(&pdev->dev, pdev->irq, skd_isr,
4130 IRQF_SHARED, skdev->isr_name, skdev);
4131 if (!rc)
4132 pr_info("(%s): LEGACY irq %d enabled\n",
4133 skd_name(skdev), pdev->irq);
4134 else
4135 pr_err("(%s): request LEGACY irq error %d\n",
4136 skd_name(skdev), rc);
4137 break;
4138 default:
4139 pr_info("(%s): irq_type %d invalid, re-set to %d\n",
4140 skd_name(skdev), skdev->irq_type, SKD_IRQ_DEFAULT);
4141 skdev->irq_type = SKD_IRQ_LEGACY;
4142 goto RETRY_IRQ_TYPE;
4143 }
4144 return rc;
4145}
4146
4147static void skd_release_irq(struct skd_device *skdev)
4148{
4149 switch (skdev->irq_type) {
4150 case SKD_IRQ_MSIX:
4151 skd_release_msix(skdev);
4152 break;
4153 case SKD_IRQ_MSI:
4154 devm_free_irq(&skdev->pdev->dev, skdev->pdev->irq, skdev);
4155 pci_disable_msi(skdev->pdev);
4156 break;
4157 case SKD_IRQ_LEGACY:
4158 devm_free_irq(&skdev->pdev->dev, skdev->pdev->irq, skdev);
4159 break;
4160 default:
4161 pr_err("(%s): wrong irq type %d!",
4162 skd_name(skdev), skdev->irq_type);
4163 break;
4164 }
4165}
4166
4167/*
4168 *****************************************************************************
4169 * CONSTRUCT
4170 *****************************************************************************
4171 */
4172
4173static int skd_cons_skcomp(struct skd_device *skdev);
4174static int skd_cons_skmsg(struct skd_device *skdev);
4175static int skd_cons_skreq(struct skd_device *skdev);
4176static int skd_cons_skspcl(struct skd_device *skdev);
4177static int skd_cons_sksb(struct skd_device *skdev);
4178static struct fit_sg_descriptor *skd_cons_sg_list(struct skd_device *skdev,
4179 u32 n_sg,
4180 dma_addr_t *ret_dma_addr);
4181static int skd_cons_disk(struct skd_device *skdev);
4182
4183#define SKD_N_DEV_TABLE 16u
4184static u32 skd_next_devno;
4185
4186static struct skd_device *skd_construct(struct pci_dev *pdev)
4187{
4188 struct skd_device *skdev;
4189 int blk_major = skd_major;
4190 int rc;
4191
4192 skdev = kzalloc(sizeof(*skdev), GFP_KERNEL);
4193
4194 if (!skdev) {
4195 pr_err(PFX "(%s): memory alloc failure\n",
4196 pci_name(pdev));
4197 return NULL;
4198 }
4199
4200 skdev->state = SKD_DRVR_STATE_LOAD;
4201 skdev->pdev = pdev;
4202 skdev->devno = skd_next_devno++;
4203 skdev->major = blk_major;
4204 skdev->irq_type = skd_isr_type;
4205 sprintf(skdev->name, DRV_NAME "%d", skdev->devno);
4206 skdev->dev_max_queue_depth = 0;
4207
4208 skdev->num_req_context = skd_max_queue_depth;
4209 skdev->num_fitmsg_context = skd_max_queue_depth;
4210 skdev->n_special = skd_max_pass_thru;
4211 skdev->cur_max_queue_depth = 1;
4212 skdev->queue_low_water_mark = 1;
4213 skdev->proto_ver = 99;
4214 skdev->sgs_per_request = skd_sgs_per_request;
4215 skdev->dbg_level = skd_dbg_level;
4216
e67f86b3
AB		 4217 atomic_set(&skdev->device_count, 0);
4218
4219 spin_lock_init(&skdev->lock);
4220
4221 INIT_WORK(&skdev->completion_worker, skd_completion_worker);
e67f86b3 4222
2e44b427 4223 pr_debug("%s:%s:%d skcomp\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4224 rc = skd_cons_skcomp(skdev);
4225 if (rc < 0)
4226 goto err_out;
4227
2e44b427 4228 pr_debug("%s:%s:%d skmsg\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4229 rc = skd_cons_skmsg(skdev);
4230 if (rc < 0)
4231 goto err_out;
4232
2e44b427 4233 pr_debug("%s:%s:%d skreq\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4234 rc = skd_cons_skreq(skdev);
4235 if (rc < 0)
4236 goto err_out;
4237
2e44b427 4238 pr_debug("%s:%s:%d skspcl\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4239 rc = skd_cons_skspcl(skdev);
4240 if (rc < 0)
4241 goto err_out;
4242
2e44b427 4243 pr_debug("%s:%s:%d sksb\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4244 rc = skd_cons_sksb(skdev);
4245 if (rc < 0)
4246 goto err_out;
4247
2e44b427 4248 pr_debug("%s:%s:%d disk\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4249 rc = skd_cons_disk(skdev);
4250 if (rc < 0)
4251 goto err_out;
4252
2e44b427 4253 pr_debug("%s:%s:%d VICTORY\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4254 return skdev;
4255
4256err_out:
2e44b427 4257 pr_debug("%s:%s:%d construct failed\n",
4258 skdev->name, __func__, __LINE__);
e67f86b3
AB		 4259 skd_destruct(skdev);
4260 return NULL;
4261}
4262
4263static int skd_cons_skcomp(struct skd_device *skdev)
4264{
4265 int rc = 0;
4266 struct fit_completion_entry_v1 *skcomp;
4267 u32 nbytes;
4268
4269 nbytes = sizeof(*skcomp) * SKD_N_COMPLETION_ENTRY;
4270 nbytes += sizeof(struct fit_comp_error_info) * SKD_N_COMPLETION_ENTRY;
4271
2e44b427 4272 pr_debug("%s:%s:%d comp pci_alloc, total bytes %d entries %d\n",
4273 skdev->name, __func__, __LINE__,
4274 nbytes, SKD_N_COMPLETION_ENTRY);
e67f86b3
AB		 4275
4276 skcomp = pci_alloc_consistent(skdev->pdev, nbytes,
4277 &skdev->cq_dma_address);
4278
4279 if (skcomp == NULL) {
4280 rc = -ENOMEM;
4281 goto err_out;
4282 }
4283
4284 memset(skcomp, 0, nbytes);
4285
4286 skdev->skcomp_table = skcomp;
4287 skdev->skerr_table = (struct fit_comp_error_info *)((char *)skcomp +
4288 sizeof(*skcomp) *
4289 SKD_N_COMPLETION_ENTRY);
4290
4291err_out:
4292 return rc;
4293}
4294
4295static int skd_cons_skmsg(struct skd_device *skdev)
4296{
4297 int rc = 0;
4298 u32 i;
4299
2e44b427 4300 pr_debug("%s:%s:%d skmsg_table kzalloc, struct %lu, count %u total %lu\n",
4301 skdev->name, __func__, __LINE__,
4302 sizeof(struct skd_fitmsg_context),
4303 skdev->num_fitmsg_context,
4304 sizeof(struct skd_fitmsg_context) * skdev->num_fitmsg_context);
e67f86b3
AB		 4305
4306 skdev->skmsg_table = kzalloc(sizeof(struct skd_fitmsg_context)
4307 *skdev->num_fitmsg_context, GFP_KERNEL);
4308 if (skdev->skmsg_table == NULL) {
4309 rc = -ENOMEM;
4310 goto err_out;
4311 }
4312
4313 for (i = 0; i < skdev->num_fitmsg_context; i++) {
4314 struct skd_fitmsg_context *skmsg;
4315
4316 skmsg = &skdev->skmsg_table[i];
4317
4318 skmsg->id = i + SKD_ID_FIT_MSG;
4319
4320 skmsg->state = SKD_MSG_STATE_IDLE;
4321 skmsg->msg_buf = pci_alloc_consistent(skdev->pdev,
4322 SKD_N_FITMSG_BYTES + 64,
4323 &skmsg->mb_dma_address);
4324
4325 if (skmsg->msg_buf == NULL) {
4326 rc = -ENOMEM;
4327 goto err_out;
4328 }
4329
4330 skmsg->offset = (u32)((u64)skmsg->msg_buf &
4331 (~FIT_QCMD_BASE_ADDRESS_MASK));
4332 skmsg->msg_buf += ~FIT_QCMD_BASE_ADDRESS_MASK;
4333 skmsg->msg_buf = (u8 *)((u64)skmsg->msg_buf &
4334 FIT_QCMD_BASE_ADDRESS_MASK);
4335 skmsg->mb_dma_address += ~FIT_QCMD_BASE_ADDRESS_MASK;
4336 skmsg->mb_dma_address &= FIT_QCMD_BASE_ADDRESS_MASK;
4337 memset(skmsg->msg_buf, 0, SKD_N_FITMSG_BYTES);
4338
4339 skmsg->next = &skmsg[1];
4340 }
4341
4342 /* Free list is in order starting with the 0th entry. */
4343 skdev->skmsg_table[i - 1].next = NULL;
4344 skdev->skmsg_free_list = skdev->skmsg_table;
4345
4346err_out:
4347 return rc;
4348}
4349
4350static int skd_cons_skreq(struct skd_device *skdev)
4351{
4352 int rc = 0;
4353 u32 i;
4354
2e44b427 4355 pr_debug("%s:%s:%d skreq_table kzalloc, struct %lu, count %u total %lu\n",
4356 skdev->name, __func__, __LINE__,
4357 sizeof(struct skd_request_context),
4358 skdev->num_req_context,
4359 sizeof(struct skd_request_context) * skdev->num_req_context);
e67f86b3
AB		 4360
4361 skdev->skreq_table = kzalloc(sizeof(struct skd_request_context)
4362 * skdev->num_req_context, GFP_KERNEL);
4363 if (skdev->skreq_table == NULL) {
4364 rc = -ENOMEM;
4365 goto err_out;
4366 }
4367
2e44b427 4368 pr_debug("%s:%s:%d alloc sg_table sg_per_req %u scatlist %lu total %lu\n",
4369 skdev->name, __func__, __LINE__,
4370 skdev->sgs_per_request, sizeof(struct scatterlist),
4371 skdev->sgs_per_request * sizeof(struct scatterlist));
e67f86b3
AB		 4372
4373 for (i = 0; i < skdev->num_req_context; i++) {
4374 struct skd_request_context *skreq;
4375
4376 skreq = &skdev->skreq_table[i];
4377
4378 skreq->id = i + SKD_ID_RW_REQUEST;
4379 skreq->state = SKD_REQ_STATE_IDLE;
4380
4381 skreq->sg = kzalloc(sizeof(struct scatterlist) *
4382 skdev->sgs_per_request, GFP_KERNEL);
4383 if (skreq->sg == NULL) {
4384 rc = -ENOMEM;
4385 goto err_out;
4386 }
4387 sg_init_table(skreq->sg, skdev->sgs_per_request);
4388
4389 skreq->sksg_list = skd_cons_sg_list(skdev,
4390 skdev->sgs_per_request,
4391 &skreq->sksg_dma_address);
4392
4393 if (skreq->sksg_list == NULL) {
4394 rc = -ENOMEM;
4395 goto err_out;
4396 }
4397
4398 skreq->next = &skreq[1];
4399 }
4400
4401 /* Free list is in order starting with the 0th entry. */
4402 skdev->skreq_table[i - 1].next = NULL;
4403 skdev->skreq_free_list = skdev->skreq_table;
4404
4405err_out:
4406 return rc;
4407}
4408
4409static int skd_cons_skspcl(struct skd_device *skdev)
4410{
4411 int rc = 0;
4412 u32 i, nbytes;
4413
2e44b427 4414 pr_debug("%s:%s:%d skspcl_table kzalloc, struct %lu, count %u total %lu\n",
4415 skdev->name, __func__, __LINE__,
4416 sizeof(struct skd_special_context),
4417 skdev->n_special,
4418 sizeof(struct skd_special_context) * skdev->n_special);
e67f86b3
AB		 4419
4420 skdev->skspcl_table = kzalloc(sizeof(struct skd_special_context)
4421 * skdev->n_special, GFP_KERNEL);
4422 if (skdev->skspcl_table == NULL) {
4423 rc = -ENOMEM;
4424 goto err_out;
4425 }
4426
4427 for (i = 0; i < skdev->n_special; i++) {
4428 struct skd_special_context *skspcl;
4429
4430 skspcl = &skdev->skspcl_table[i];
4431
4432 skspcl->req.id = i + SKD_ID_SPECIAL_REQUEST;
4433 skspcl->req.state = SKD_REQ_STATE_IDLE;
4434
4435 skspcl->req.next = &skspcl[1].req;
4436
4437 nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
4438
4439 skspcl->msg_buf = pci_alloc_consistent(skdev->pdev, nbytes,
4440 &skspcl->mb_dma_address);
4441 if (skspcl->msg_buf == NULL) {
4442 rc = -ENOMEM;
4443 goto err_out;
4444 }
4445
4446 memset(skspcl->msg_buf, 0, nbytes);
4447
4448 skspcl->req.sg = kzalloc(sizeof(struct scatterlist) *
4449 SKD_N_SG_PER_SPECIAL, GFP_KERNEL);
4450 if (skspcl->req.sg == NULL) {
4451 rc = -ENOMEM;
4452 goto err_out;
4453 }
4454
4455 skspcl->req.sksg_list = skd_cons_sg_list(skdev,
4456 SKD_N_SG_PER_SPECIAL,
4457 &skspcl->req.
4458 sksg_dma_address);
4459 if (skspcl->req.sksg_list == NULL) {
4460 rc = -ENOMEM;
4461 goto err_out;
4462 }
4463 }
4464
4465 /* Free list is in order starting with the 0th entry. */
4466 skdev->skspcl_table[i - 1].req.next = NULL;
4467 skdev->skspcl_free_list = skdev->skspcl_table;
4468
4469 return rc;
4470
4471err_out:
4472 return rc;
4473}
4474
4475static int skd_cons_sksb(struct skd_device *skdev)
4476{
4477 int rc = 0;
4478 struct skd_special_context *skspcl;
4479 u32 nbytes;
4480
4481 skspcl = &skdev->internal_skspcl;
4482
4483 skspcl->req.id = 0 + SKD_ID_INTERNAL;
4484 skspcl->req.state = SKD_REQ_STATE_IDLE;
4485
4486 nbytes = SKD_N_INTERNAL_BYTES;
4487
4488 skspcl->data_buf = pci_alloc_consistent(skdev->pdev, nbytes,
4489 &skspcl->db_dma_address);
4490 if (skspcl->data_buf == NULL) {
4491 rc = -ENOMEM;
4492 goto err_out;
4493 }
4494
4495 memset(skspcl->data_buf, 0, nbytes);
4496
4497 nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
4498 skspcl->msg_buf = pci_alloc_consistent(skdev->pdev, nbytes,
4499 &skspcl->mb_dma_address);
4500 if (skspcl->msg_buf == NULL) {
4501 rc = -ENOMEM;
4502 goto err_out;
4503 }
4504
4505 memset(skspcl->msg_buf, 0, nbytes);
4506
4507 skspcl->req.sksg_list = skd_cons_sg_list(skdev, 1,
4508 &skspcl->req.sksg_dma_address);
4509 if (skspcl->req.sksg_list == NULL) {
4510 rc = -ENOMEM;
4511 goto err_out;
4512 }
4513
4514 if (!skd_format_internal_skspcl(skdev)) {
4515 rc = -EINVAL;
4516 goto err_out;
4517 }
4518
4519err_out:
4520 return rc;
4521}
4522
4523static struct fit_sg_descriptor *skd_cons_sg_list(struct skd_device *skdev,
4524 u32 n_sg,
4525 dma_addr_t *ret_dma_addr)
4526{
4527 struct fit_sg_descriptor *sg_list;
4528 u32 nbytes;
4529
4530 nbytes = sizeof(*sg_list) * n_sg;
4531
4532 sg_list = pci_alloc_consistent(skdev->pdev, nbytes, ret_dma_addr);
4533
4534 if (sg_list != NULL) {
4535 uint64_t dma_address = *ret_dma_addr;
4536 u32 i;
4537
4538 memset(sg_list, 0, nbytes);
4539
4540 for (i = 0; i < n_sg - 1; i++) {
4541 uint64_t ndp_off;
4542 ndp_off = (i + 1) * sizeof(struct fit_sg_descriptor);
4543
4544 sg_list[i].next_desc_ptr = dma_address + ndp_off;
4545 }
4546 sg_list[i].next_desc_ptr = 0LL;
4547 }
4548
4549 return sg_list;
4550}
4551
4552static int skd_cons_disk(struct skd_device *skdev)
4553{
4554 int rc = 0;
4555 struct gendisk *disk;
4556 struct request_queue *q;
4557 unsigned long flags;
4558
4559 disk = alloc_disk(SKD_MINORS_PER_DEVICE);
4560 if (!disk) {
4561 rc = -ENOMEM;
4562 goto err_out;
4563 }
4564
4565 skdev->disk = disk;
4566 sprintf(disk->disk_name, DRV_NAME "%u", skdev->devno);
4567
4568 disk->major = skdev->major;
4569 disk->first_minor = skdev->devno * SKD_MINORS_PER_DEVICE;
4570 disk->fops = &skd_blockdev_ops;
4571 disk->private_data = skdev;
4572
fcd37eb3 4573 q = blk_init_queue(skd_request_fn, &skdev->lock);
e67f86b3
AB		 4574 if (!q) {
4575 rc = -ENOMEM;
4576 goto err_out;
4577 }
4578
4579 skdev->queue = q;
4580 disk->queue = q;
4581 q->queuedata = skdev;
4582
e67f86b3
AB		 4583 blk_queue_flush(q, REQ_FLUSH | REQ_FUA);
4584 blk_queue_max_segments(q, skdev->sgs_per_request);
4585 blk_queue_max_hw_sectors(q, SKD_N_MAX_SECTORS);
4586
4587 /* set sysfs ptimal_io_size to 8K */
4588 blk_queue_io_opt(q, 8192);
4589
4590 /* DISCARD Flag initialization. */
4591 q->limits.discard_granularity = 8192;
4592 q->limits.discard_alignment = 0;
4593 q->limits.max_discard_sectors = UINT_MAX >> 9;
4594 q->limits.discard_zeroes_data = 1;
4595 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q);
4596 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, q);
4597
4598 spin_lock_irqsave(&skdev->lock, flags);
2e44b427 4599 pr_debug("%s:%s:%d stopping %s queue\n",
4600 skdev->name, __func__, __LINE__, skdev->name);
6a5ec65b 4601 blk_stop_queue(skdev->queue);
e67f86b3
AB		 4602 spin_unlock_irqrestore(&skdev->lock, flags);
4603
4604err_out:
4605 return rc;
4606}
4607
4608/*
4609 *****************************************************************************
4610 * DESTRUCT (FREE)
4611 *****************************************************************************
4612 */
4613
4614static void skd_free_skcomp(struct skd_device *skdev);
4615static void skd_free_skmsg(struct skd_device *skdev);
4616static void skd_free_skreq(struct skd_device *skdev);
4617static void skd_free_skspcl(struct skd_device *skdev);
4618static void skd_free_sksb(struct skd_device *skdev);
4619static void skd_free_sg_list(struct skd_device *skdev,
4620 struct fit_sg_descriptor *sg_list,
4621 u32 n_sg, dma_addr_t dma_addr);
4622static void skd_free_disk(struct skd_device *skdev);
4623
4624static void skd_destruct(struct skd_device *skdev)
4625{
4626 if (skdev == NULL)
4627 return;
4628
4629
2e44b427 4630 pr_debug("%s:%s:%d disk\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4631 skd_free_disk(skdev);
4632
2e44b427 4633 pr_debug("%s:%s:%d sksb\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4634 skd_free_sksb(skdev);
4635
2e44b427 4636 pr_debug("%s:%s:%d skspcl\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4637 skd_free_skspcl(skdev);
4638
2e44b427 4639 pr_debug("%s:%s:%d skreq\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4640 skd_free_skreq(skdev);
4641
2e44b427 4642 pr_debug("%s:%s:%d skmsg\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4643 skd_free_skmsg(skdev);
4644
2e44b427 4645 pr_debug("%s:%s:%d skcomp\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4646 skd_free_skcomp(skdev);
4647
2e44b427 4648 pr_debug("%s:%s:%d skdev\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4649 kfree(skdev);
4650}
4651
4652static void skd_free_skcomp(struct skd_device *skdev)
4653{
4654 if (skdev->skcomp_table != NULL) {
4655 u32 nbytes;
4656
4657 nbytes = sizeof(skdev->skcomp_table[0]) *
4658 SKD_N_COMPLETION_ENTRY;
4659 pci_free_consistent(skdev->pdev, nbytes,
4660 skdev->skcomp_table, skdev->cq_dma_address);
4661 }
4662
4663 skdev->skcomp_table = NULL;
4664 skdev->cq_dma_address = 0;
4665}
4666
4667static void skd_free_skmsg(struct skd_device *skdev)
4668{
4669 u32 i;
4670
4671 if (skdev->skmsg_table == NULL)
4672 return;
4673
4674 for (i = 0; i < skdev->num_fitmsg_context; i++) {
4675 struct skd_fitmsg_context *skmsg;
4676
4677 skmsg = &skdev->skmsg_table[i];
4678
4679 if (skmsg->msg_buf != NULL) {
4680 skmsg->msg_buf += skmsg->offset;
4681 skmsg->mb_dma_address += skmsg->offset;
4682 pci_free_consistent(skdev->pdev, SKD_N_FITMSG_BYTES,
4683 skmsg->msg_buf,
4684 skmsg->mb_dma_address);
4685 }
4686 skmsg->msg_buf = NULL;
4687 skmsg->mb_dma_address = 0;
4688 }
4689
4690 kfree(skdev->skmsg_table);
4691 skdev->skmsg_table = NULL;
4692}
4693
4694static void skd_free_skreq(struct skd_device *skdev)
4695{
4696 u32 i;
4697
4698 if (skdev->skreq_table == NULL)
4699 return;
4700
4701 for (i = 0; i < skdev->num_req_context; i++) {
4702 struct skd_request_context *skreq;
4703
4704 skreq = &skdev->skreq_table[i];
4705
4706 skd_free_sg_list(skdev, skreq->sksg_list,
4707 skdev->sgs_per_request,
4708 skreq->sksg_dma_address);
4709
4710 skreq->sksg_list = NULL;
4711 skreq->sksg_dma_address = 0;
4712
4713 kfree(skreq->sg);
4714 }
4715
4716 kfree(skdev->skreq_table);
4717 skdev->skreq_table = NULL;
4718}
4719
4720static void skd_free_skspcl(struct skd_device *skdev)
4721{
4722 u32 i;
4723 u32 nbytes;
4724
4725 if (skdev->skspcl_table == NULL)
4726 return;
4727
4728 for (i = 0; i < skdev->n_special; i++) {
4729 struct skd_special_context *skspcl;
4730
4731 skspcl = &skdev->skspcl_table[i];
4732
4733 if (skspcl->msg_buf != NULL) {
4734 nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
4735 pci_free_consistent(skdev->pdev, nbytes,
4736 skspcl->msg_buf,
4737 skspcl->mb_dma_address);
4738 }
4739
4740 skspcl->msg_buf = NULL;
4741 skspcl->mb_dma_address = 0;
4742
4743 skd_free_sg_list(skdev, skspcl->req.sksg_list,
4744 SKD_N_SG_PER_SPECIAL,
4745 skspcl->req.sksg_dma_address);
4746
4747 skspcl->req.sksg_list = NULL;
4748 skspcl->req.sksg_dma_address = 0;
4749
4750 kfree(skspcl->req.sg);
4751 }
4752
4753 kfree(skdev->skspcl_table);
4754 skdev->skspcl_table = NULL;
4755}
4756
4757static void skd_free_sksb(struct skd_device *skdev)
4758{
4759 struct skd_special_context *skspcl;
4760 u32 nbytes;
4761
4762 skspcl = &skdev->internal_skspcl;
4763
4764 if (skspcl->data_buf != NULL) {
4765 nbytes = SKD_N_INTERNAL_BYTES;
4766
4767 pci_free_consistent(skdev->pdev, nbytes,
4768 skspcl->data_buf, skspcl->db_dma_address);
4769 }
4770
4771 skspcl->data_buf = NULL;
4772 skspcl->db_dma_address = 0;
4773
4774 if (skspcl->msg_buf != NULL) {
4775 nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
4776 pci_free_consistent(skdev->pdev, nbytes,
4777 skspcl->msg_buf, skspcl->mb_dma_address);
4778 }
4779
4780 skspcl->msg_buf = NULL;
4781 skspcl->mb_dma_address = 0;
4782
4783 skd_free_sg_list(skdev, skspcl->req.sksg_list, 1,
4784 skspcl->req.sksg_dma_address);
4785
4786 skspcl->req.sksg_list = NULL;
4787 skspcl->req.sksg_dma_address = 0;
4788}
4789
4790static void skd_free_sg_list(struct skd_device *skdev,
4791 struct fit_sg_descriptor *sg_list,
4792 u32 n_sg, dma_addr_t dma_addr)
4793{
4794 if (sg_list != NULL) {
4795 u32 nbytes;
4796
4797 nbytes = sizeof(*sg_list) * n_sg;
4798
4799 pci_free_consistent(skdev->pdev, nbytes, sg_list, dma_addr);
4800 }
4801}
4802
4803static void skd_free_disk(struct skd_device *skdev)
4804{
4805 struct gendisk *disk = skdev->disk;
4806
4807 if (disk != NULL) {
4808 struct request_queue *q = disk->queue;
4809
4810 if (disk->flags & GENHD_FL_UP)
4811 del_gendisk(disk);
4812 if (q)
4813 blk_cleanup_queue(q);
4814 put_disk(disk);
4815 }
4816 skdev->disk = NULL;
4817}
4818
4819
4820
4821/*
4822 *****************************************************************************
4823 * BLOCK DEVICE (BDEV) GLUE
4824 *****************************************************************************
4825 */
4826
4827static int skd_bdev_getgeo(struct block_device *bdev, struct hd_geometry *geo)
4828{
4829 struct skd_device *skdev;
4830 u64 capacity;
4831
4832 skdev = bdev->bd_disk->private_data;
4833
2e44b427 4834 pr_debug("%s:%s:%d %s: CMD[%s] getgeo device\n",
4835 skdev->name, __func__, __LINE__,
4836 bdev->bd_disk->disk_name, current->comm);
e67f86b3
AB		 4837
4838 if (skdev->read_cap_is_valid) {
4839 capacity = get_capacity(skdev->disk);
4840 geo->heads = 64;
4841 geo->sectors = 255;
4842 geo->cylinders = (capacity) / (255 * 64);
4843
4844 return 0;
4845 }
4846 return -EIO;
4847}
4848
4849static int skd_bdev_attach(struct skd_device *skdev)
4850{
2e44b427 4851 pr_debug("%s:%s:%d add_disk\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4852 add_disk(skdev->disk);
4853 return 0;
4854}
4855
4856static const struct block_device_operations skd_blockdev_ops = {
4857 .owner = THIS_MODULE,
4858 .ioctl = skd_bdev_ioctl,
4859 .getgeo = skd_bdev_getgeo,
4860};
4861
4862
4863/*
4864 *****************************************************************************
4865 * PCIe DRIVER GLUE
4866 *****************************************************************************
4867 */
4868
4869static DEFINE_PCI_DEVICE_TABLE(skd_pci_tbl) = {
4870 { PCI_VENDOR_ID_STEC, PCI_DEVICE_ID_S1120,
4871 PCI_ANY_ID, PCI_ANY_ID, 0, 0, },
4872 { 0 } /* terminate list */
4873};
4874
4875MODULE_DEVICE_TABLE(pci, skd_pci_tbl);
4876
4877static char *skd_pci_info(struct skd_device *skdev, char *str)
4878{
4879 int pcie_reg;
4880
4881 strcpy(str, "PCIe (");
4882 pcie_reg = pci_find_capability(skdev->pdev, PCI_CAP_ID_EXP);
4883
4884 if (pcie_reg) {
4885
4886 char lwstr[6];
4887 uint16_t pcie_lstat, lspeed, lwidth;
4888
4889 pcie_reg += 0x12;
4890 pci_read_config_word(skdev->pdev, pcie_reg, &pcie_lstat);
4891 lspeed = pcie_lstat & (0xF);
4892 lwidth = (pcie_lstat & 0x3F0) >> 4;
4893
4894 if (lspeed == 1)
4895 strcat(str, "2.5GT/s ");
4896 else if (lspeed == 2)
4897 strcat(str, "5.0GT/s ");
4898 else
4899 strcat(str, "<unknown> ");
4900 snprintf(lwstr, sizeof(lwstr), "%dX)", lwidth);
4901 strcat(str, lwstr);
4902 }
4903 return str;
4904}
4905
4906static int skd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
4907{
4908 int i;
4909 int rc = 0;
4910 char pci_str[32];
4911 struct skd_device *skdev;
4912
4913 pr_info("STEC s1120 Driver(%s) version %s-b%s\n",
4914 DRV_NAME, DRV_VERSION, DRV_BUILD_ID);
4915 pr_info("(skd?:??:[%s]): vendor=%04X device=%04x\n",
4916 pci_name(pdev), pdev->vendor, pdev->device);
4917
4918 rc = pci_enable_device(pdev);
4919 if (rc)
4920 return rc;
4921 rc = pci_request_regions(pdev, DRV_NAME);
4922 if (rc)
4923 goto err_out;
4924 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
4925 if (!rc) {
4926 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64))) {
4927
4928 pr_err("(%s): consistent DMA mask error %d\n",
4929 pci_name(pdev), rc);
4930 }
4931 } else {
4932 (rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)));
4933 if (rc) {
4934
4935 pr_err("(%s): DMA mask error %d\n",
4936 pci_name(pdev), rc);
4937 goto err_out_regions;
4938 }
4939 }
4940
4941 skdev = skd_construct(pdev);
1762b57f
WY		 4942 if (skdev == NULL) {
4943 rc = -ENOMEM;
e67f86b3 4944 goto err_out_regions;
1762b57f 4945 }
e67f86b3
AB		 4946
4947 skd_pci_info(skdev, pci_str);
4948 pr_info("(%s): %s 64bit\n", skd_name(skdev), pci_str);
4949
4950 pci_set_master(pdev);
4951 rc = pci_enable_pcie_error_reporting(pdev);
4952 if (rc) {
4953 pr_err(
4954 "(%s): bad enable of PCIe error reporting rc=%d\n",
4955 skd_name(skdev), rc);
4956 skdev->pcie_error_reporting_is_enabled = 0;
4957 } else
4958 skdev->pcie_error_reporting_is_enabled = 1;
4959
4960
4961 pci_set_drvdata(pdev, skdev);
4962 skdev->pdev = pdev;
4963 skdev->disk->driverfs_dev = &pdev->dev;
4964
4965 for (i = 0; i < SKD_MAX_BARS; i++) {
4966 skdev->mem_phys[i] = pci_resource_start(pdev, i);
4967 skdev->mem_size[i] = (u32)pci_resource_len(pdev, i);
4968 skdev->mem_map[i] = ioremap(skdev->mem_phys[i],
4969 skdev->mem_size[i]);
4970 if (!skdev->mem_map[i]) {
4971 pr_err("(%s): Unable to map adapter memory!\n",
4972 skd_name(skdev));
4973 rc = -ENODEV;
4974 goto err_out_iounmap;
4975 }
2e44b427 4976 pr_debug("%s:%s:%d mem_map=%p, phyd=%016llx, size=%d\n",
4977 skdev->name, __func__, __LINE__,
4978 skdev->mem_map[i],
4979 (uint64_t)skdev->mem_phys[i], skdev->mem_size[i]);
e67f86b3
AB		 4980 }
4981
4982 rc = skd_acquire_irq(skdev);
4983 if (rc) {
4984 pr_err("(%s): interrupt resource error %d\n",
4985 skd_name(skdev), rc);
4986 goto err_out_iounmap;
4987 }
4988
4989 rc = skd_start_timer(skdev);
4990 if (rc)
4991 goto err_out_timer;
4992
4993 init_waitqueue_head(&skdev->waitq);
4994
4995 skd_start_device(skdev);
4996
4997 rc = wait_event_interruptible_timeout(skdev->waitq,
4998 (skdev->gendisk_on),
4999 (SKD_START_WAIT_SECONDS * HZ));
5000 if (skdev->gendisk_on > 0) {
5001 /* device came on-line after reset */
5002 skd_bdev_attach(skdev);
5003 rc = 0;
5004 } else {
5005 /* we timed out, something is wrong with the device,
5006 don't add the disk structure */
5007 pr_err(
5008 "(%s): error: waiting for s1120 timed out %d!\n",
5009 skd_name(skdev), rc);
5010 /* in case of no error; we timeout with ENXIO */
5011 if (!rc)
5012 rc = -ENXIO;
5013 goto err_out_timer;
5014 }
5015
5016
5017#ifdef SKD_VMK_POLL_HANDLER
5018 if (skdev->irq_type == SKD_IRQ_MSIX) {
5019 /* MSIX completion handler is being used for coredump */
5020 vmklnx_scsi_register_poll_handler(skdev->scsi_host,
5021 skdev->msix_entries[5].vector,
5022 skd_comp_q, skdev);
5023 } else {
5024 vmklnx_scsi_register_poll_handler(skdev->scsi_host,
5025 skdev->pdev->irq, skd_isr,
5026 skdev);
5027 }
5028#endif /* SKD_VMK_POLL_HANDLER */
5029
5030 return rc;
5031
5032err_out_timer:
5033 skd_stop_device(skdev);
5034 skd_release_irq(skdev);
5035
5036err_out_iounmap:
5037 for (i = 0; i < SKD_MAX_BARS; i++)
5038 if (skdev->mem_map[i])
5039 iounmap(skdev->mem_map[i]);
5040
5041 if (skdev->pcie_error_reporting_is_enabled)
5042 pci_disable_pcie_error_reporting(pdev);
5043
5044 skd_destruct(skdev);
5045
5046err_out_regions:
5047 pci_release_regions(pdev);
5048
5049err_out:
5050 pci_disable_device(pdev);
5051 pci_set_drvdata(pdev, NULL);
5052 return rc;
5053}
5054
5055static void skd_pci_remove(struct pci_dev *pdev)
5056{
5057 int i;
5058 struct skd_device *skdev;
5059
5060 skdev = pci_get_drvdata(pdev);
5061 if (!skdev) {
5062 pr_err("%s: no device data for PCI\n", pci_name(pdev));
5063 return;
5064 }
5065 skd_stop_device(skdev);
5066 skd_release_irq(skdev);
5067
5068 for (i = 0; i < SKD_MAX_BARS; i++)
5069 if (skdev->mem_map[i])
5070 iounmap((u32 *)skdev->mem_map[i]);
5071
5072 if (skdev->pcie_error_reporting_is_enabled)
5073 pci_disable_pcie_error_reporting(pdev);
5074
5075 skd_destruct(skdev);
5076
5077 pci_release_regions(pdev);
5078 pci_disable_device(pdev);
5079 pci_set_drvdata(pdev, NULL);
5080
5081 return;
5082}
5083
5084static int skd_pci_suspend(struct pci_dev *pdev, pm_message_t state)
5085{
5086 int i;
5087 struct skd_device *skdev;
5088
5089 skdev = pci_get_drvdata(pdev);
5090 if (!skdev) {
5091 pr_err("%s: no device data for PCI\n", pci_name(pdev));
5092 return -EIO;
5093 }
5094
5095 skd_stop_device(skdev);
5096
5097 skd_release_irq(skdev);
5098
5099 for (i = 0; i < SKD_MAX_BARS; i++)
5100 if (skdev->mem_map[i])
5101 iounmap((u32 *)skdev->mem_map[i]);
5102
5103 if (skdev->pcie_error_reporting_is_enabled)
5104 pci_disable_pcie_error_reporting(pdev);
5105
5106 pci_release_regions(pdev);
5107 pci_save_state(pdev);
5108 pci_disable_device(pdev);
5109 pci_set_power_state(pdev, pci_choose_state(pdev, state));
5110 return 0;
5111}
5112
5113static int skd_pci_resume(struct pci_dev *pdev)
5114{
5115 int i;
5116 int rc = 0;
5117 struct skd_device *skdev;
5118
5119 skdev = pci_get_drvdata(pdev);
5120 if (!skdev) {
5121 pr_err("%s: no device data for PCI\n", pci_name(pdev));
5122 return -1;
5123 }
5124
5125 pci_set_power_state(pdev, PCI_D0);
5126 pci_enable_wake(pdev, PCI_D0, 0);
5127 pci_restore_state(pdev);
5128
5129 rc = pci_enable_device(pdev);
5130 if (rc)
5131 return rc;
5132 rc = pci_request_regions(pdev, DRV_NAME);
5133 if (rc)
5134 goto err_out;
5135 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
5136 if (!rc) {
5137 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64))) {
5138
5139 pr_err("(%s): consistent DMA mask error %d\n",
5140 pci_name(pdev), rc);
5141 }
5142 } else {
5143 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
5144 if (rc) {
5145
5146 pr_err("(%s): DMA mask error %d\n",
5147 pci_name(pdev), rc);
5148 goto err_out_regions;
5149 }
5150 }
5151
5152 pci_set_master(pdev);
5153 rc = pci_enable_pcie_error_reporting(pdev);
5154 if (rc) {
5155 pr_err("(%s): bad enable of PCIe error reporting rc=%d\n",
5156 skdev->name, rc);
5157 skdev->pcie_error_reporting_is_enabled = 0;
5158 } else
5159 skdev->pcie_error_reporting_is_enabled = 1;
5160
5161 for (i = 0; i < SKD_MAX_BARS; i++) {
5162
5163 skdev->mem_phys[i] = pci_resource_start(pdev, i);
5164 skdev->mem_size[i] = (u32)pci_resource_len(pdev, i);
5165 skdev->mem_map[i] = ioremap(skdev->mem_phys[i],
5166 skdev->mem_size[i]);
5167 if (!skdev->mem_map[i]) {
5168 pr_err("(%s): Unable to map adapter memory!\n",
5169 skd_name(skdev));
5170 rc = -ENODEV;
5171 goto err_out_iounmap;
5172 }
2e44b427 5173 pr_debug("%s:%s:%d mem_map=%p, phyd=%016llx, size=%d\n",
5174 skdev->name, __func__, __LINE__,
5175 skdev->mem_map[i],
5176 (uint64_t)skdev->mem_phys[i], skdev->mem_size[i]);
e67f86b3
AB		 5177 }
5178 rc = skd_acquire_irq(skdev);
5179 if (rc) {
5180
5181 pr_err("(%s): interrupt resource error %d\n",
5182 pci_name(pdev), rc);
5183 goto err_out_iounmap;
5184 }
5185
5186 rc = skd_start_timer(skdev);
5187 if (rc)
5188 goto err_out_timer;
5189
5190 init_waitqueue_head(&skdev->waitq);
5191
5192 skd_start_device(skdev);
5193
5194 return rc;
5195
5196err_out_timer:
5197 skd_stop_device(skdev);
5198 skd_release_irq(skdev);
5199
5200err_out_iounmap:
5201 for (i = 0; i < SKD_MAX_BARS; i++)
5202 if (skdev->mem_map[i])
5203 iounmap(skdev->mem_map[i]);
5204
5205 if (skdev->pcie_error_reporting_is_enabled)
5206 pci_disable_pcie_error_reporting(pdev);
5207
5208err_out_regions:
5209 pci_release_regions(pdev);
5210
5211err_out:
5212 pci_disable_device(pdev);
5213 return rc;
5214}
5215
5216static void skd_pci_shutdown(struct pci_dev *pdev)
5217{
5218 struct skd_device *skdev;
5219
5220 pr_err("skd_pci_shutdown called\n");
5221
5222 skdev = pci_get_drvdata(pdev);
5223 if (!skdev) {
5224 pr_err("%s: no device data for PCI\n", pci_name(pdev));
5225 return;
5226 }
5227
5228 pr_err("%s: calling stop\n", skd_name(skdev));
5229 skd_stop_device(skdev);
5230}
5231
5232static struct pci_driver skd_driver = {
5233 .name = DRV_NAME,
5234 .id_table = skd_pci_tbl,
5235 .probe = skd_pci_probe,
5236 .remove = skd_pci_remove,
5237 .suspend = skd_pci_suspend,
5238 .resume = skd_pci_resume,
5239 .shutdown = skd_pci_shutdown,
5240};
5241
5242/*
5243 *****************************************************************************
5244 * LOGGING SUPPORT
5245 *****************************************************************************
5246 */
5247
5248static const char *skd_name(struct skd_device *skdev)
5249{
5250 memset(skdev->id_str, 0, sizeof(skdev->id_str));
5251
5252 if (skdev->inquiry_is_valid)
5253 snprintf(skdev->id_str, sizeof(skdev->id_str), "%s:%s:[%s]",
5254 skdev->name, skdev->inq_serial_num,
5255 pci_name(skdev->pdev));
5256 else
5257 snprintf(skdev->id_str, sizeof(skdev->id_str), "%s:??:[%s]",
5258 skdev->name, pci_name(skdev->pdev));
5259
5260 return skdev->id_str;
5261}
5262
5263const char *skd_drive_state_to_str(int state)
5264{
5265 switch (state) {
5266 case FIT_SR_DRIVE_OFFLINE:
5267 return "OFFLINE";
5268 case FIT_SR_DRIVE_INIT:
5269 return "INIT";
5270 case FIT_SR_DRIVE_ONLINE:
5271 return "ONLINE";
5272 case FIT_SR_DRIVE_BUSY:
5273 return "BUSY";
5274 case FIT_SR_DRIVE_FAULT:
5275 return "FAULT";
5276 case FIT_SR_DRIVE_DEGRADED:
5277 return "DEGRADED";
5278 case FIT_SR_PCIE_LINK_DOWN:
5279 return "INK_DOWN";
5280 case FIT_SR_DRIVE_SOFT_RESET:
5281 return "SOFT_RESET";
5282 case FIT_SR_DRIVE_NEED_FW_DOWNLOAD:
5283 return "NEED_FW";
5284 case FIT_SR_DRIVE_INIT_FAULT:
5285 return "INIT_FAULT";
5286 case FIT_SR_DRIVE_BUSY_SANITIZE:
5287 return "BUSY_SANITIZE";
5288 case FIT_SR_DRIVE_BUSY_ERASE:
5289 return "BUSY_ERASE";
5290 case FIT_SR_DRIVE_FW_BOOTING:
5291 return "FW_BOOTING";
5292 default:
5293 return "???";
5294 }
5295}
5296
5297const char *skd_skdev_state_to_str(enum skd_drvr_state state)
5298{
5299 switch (state) {
5300 case SKD_DRVR_STATE_LOAD:
5301 return "LOAD";
5302 case SKD_DRVR_STATE_IDLE:
5303 return "IDLE";
5304 case SKD_DRVR_STATE_BUSY:
5305 return "BUSY";
5306 case SKD_DRVR_STATE_STARTING:
5307 return "STARTING";
5308 case SKD_DRVR_STATE_ONLINE:
5309 return "ONLINE";
5310 case SKD_DRVR_STATE_PAUSING:
5311 return "PAUSING";
5312 case SKD_DRVR_STATE_PAUSED:
5313 return "PAUSED";
5314 case SKD_DRVR_STATE_DRAINING_TIMEOUT:
5315 return "DRAINING_TIMEOUT";
5316 case SKD_DRVR_STATE_RESTARTING:
5317 return "RESTARTING";
5318 case SKD_DRVR_STATE_RESUMING:
5319 return "RESUMING";
5320 case SKD_DRVR_STATE_STOPPING:
5321 return "STOPPING";
5322 case SKD_DRVR_STATE_SYNCING:
5323 return "SYNCING";
5324 case SKD_DRVR_STATE_FAULT:
5325 return "FAULT";
5326 case SKD_DRVR_STATE_DISAPPEARED:
5327 return "DISAPPEARED";
5328 case SKD_DRVR_STATE_BUSY_ERASE:
5329 return "BUSY_ERASE";
5330 case SKD_DRVR_STATE_BUSY_SANITIZE:
5331 return "BUSY_SANITIZE";
5332 case SKD_DRVR_STATE_BUSY_IMMINENT:
5333 return "BUSY_IMMINENT";
5334 case SKD_DRVR_STATE_WAIT_BOOT:
5335 return "WAIT_BOOT";
5336
5337 default:
5338 return "???";
5339 }
5340}
5341
5342const char *skd_skmsg_state_to_str(enum skd_fit_msg_state state)
5343{
5344 switch (state) {
5345 case SKD_MSG_STATE_IDLE:
5346 return "IDLE";
5347 case SKD_MSG_STATE_BUSY:
5348 return "BUSY";
5349 default:
5350 return "???";
5351 }
5352}
5353
5354const char *skd_skreq_state_to_str(enum skd_req_state state)
5355{
5356 switch (state) {
5357 case SKD_REQ_STATE_IDLE:
5358 return "IDLE";
5359 case SKD_REQ_STATE_SETUP:
5360 return "SETUP";
5361 case SKD_REQ_STATE_BUSY:
5362 return "BUSY";
5363 case SKD_REQ_STATE_COMPLETED:
5364 return "COMPLETED";
5365 case SKD_REQ_STATE_TIMEOUT:
5366 return "TIMEOUT";
5367 case SKD_REQ_STATE_ABORTED:
5368 return "ABORTED";
5369 default:
5370 return "???";
5371 }
5372}
5373
5374static void skd_log_skdev(struct skd_device *skdev, const char *event)
5375{
2e44b427 5376 pr_debug("%s:%s:%d (%s) skdev=%p event='%s'\n",
5377 skdev->name, __func__, __LINE__, skdev->name, skdev, event);
5378 pr_debug("%s:%s:%d drive_state=%s(%d) driver_state=%s(%d)\n",
5379 skdev->name, __func__, __LINE__,
5380 skd_drive_state_to_str(skdev->drive_state), skdev->drive_state,
5381 skd_skdev_state_to_str(skdev->state), skdev->state);
5382 pr_debug("%s:%s:%d busy=%d limit=%d dev=%d lowat=%d\n",
5383 skdev->name, __func__, __LINE__,
5384 skdev->in_flight, skdev->cur_max_queue_depth,
5385 skdev->dev_max_queue_depth, skdev->queue_low_water_mark);
5386 pr_debug("%s:%s:%d timestamp=0x%x cycle=%d cycle_ix=%d\n",
5387 skdev->name, __func__, __LINE__,
5388 skdev->timeout_stamp, skdev->skcomp_cycle, skdev->skcomp_ix);
e67f86b3
AB		 5389}
5390
5391static void skd_log_skmsg(struct skd_device *skdev,
5392 struct skd_fitmsg_context *skmsg, const char *event)
5393{
2e44b427 5394 pr_debug("%s:%s:%d (%s) skmsg=%p event='%s'\n",
5395 skdev->name, __func__, __LINE__, skdev->name, skmsg, event);
5396 pr_debug("%s:%s:%d state=%s(%d) id=0x%04x length=%d\n",
5397 skdev->name, __func__, __LINE__,
5398 skd_skmsg_state_to_str(skmsg->state), skmsg->state,
5399 skmsg->id, skmsg->length);
e67f86b3
AB		 5400}
5401
5402static void skd_log_skreq(struct skd_device *skdev,
5403 struct skd_request_context *skreq, const char *event)
5404{
2e44b427 5405 pr_debug("%s:%s:%d (%s) skreq=%p event='%s'\n",
5406 skdev->name, __func__, __LINE__, skdev->name, skreq, event);
5407 pr_debug("%s:%s:%d state=%s(%d) id=0x%04x fitmsg=0x%04x\n",
5408 skdev->name, __func__, __LINE__,
5409 skd_skreq_state_to_str(skreq->state), skreq->state,
5410 skreq->id, skreq->fitmsg_id);
5411 pr_debug("%s:%s:%d timo=0x%x sg_dir=%d n_sg=%d\n",
5412 skdev->name, __func__, __LINE__,
5413 skreq->timeout_stamp, skreq->sg_data_dir, skreq->n_sg);
e67f86b3 5414
fcd37eb3
JA		 5415 if (skreq->req != NULL) {
5416 struct request *req = skreq->req;
5417 u32 lba = (u32)blk_rq_pos(req);
5418 u32 count = blk_rq_sectors(req);
e67f86b3 5419
fcd37eb3
JA		 5420 pr_debug("%s:%s:%d "
5421 "req=%p lba=%u(0x%x) count=%u(0x%x) dir=%d\n",
5422 skdev->name, __func__, __LINE__,
5423 req, lba, lba, count, count,
5424 (int)rq_data_dir(req));
5425 } else
5426 pr_debug("%s:%s:%d req=NULL\n",
5427 skdev->name, __func__, __LINE__);
e67f86b3
AB		 5428}
5429
5430/*
5431 *****************************************************************************
5432 * MODULE GLUE
5433 *****************************************************************************
5434 */
5435
5436static int __init skd_init(void)
5437{
5438 int rc = 0;
5439
5440 pr_info(PFX " v%s-b%s loaded\n", DRV_VERSION, DRV_BUILD_ID);
5441
5442 switch (skd_isr_type) {
5443 case SKD_IRQ_LEGACY:
5444 case SKD_IRQ_MSI:
5445 case SKD_IRQ_MSIX:
5446 break;
5447 default:
5448 pr_info("skd_isr_type %d invalid, re-set to %d\n",
5449 skd_isr_type, SKD_IRQ_DEFAULT);
5450 skd_isr_type = SKD_IRQ_DEFAULT;
5451 }
5452
5453 skd_flush_slab = kmem_cache_create(SKD_FLUSH_JOB,
5454 sizeof(struct skd_flush_cmd),
5455 0, 0, NULL);
5456
5457 if (!skd_flush_slab) {
5458 pr_err("failed to allocated flush slab.\n");
5459 return -ENOMEM;
5460 }
5461
5462 if (skd_max_queue_depth < 1
5463 || skd_max_queue_depth > SKD_MAX_QUEUE_DEPTH) {
5464 pr_info(
5465 "skd_max_queue_depth %d invalid, re-set to %d\n",
5466 skd_max_queue_depth, SKD_MAX_QUEUE_DEPTH_DEFAULT);
5467 skd_max_queue_depth = SKD_MAX_QUEUE_DEPTH_DEFAULT;
5468 }
5469
5470 if (skd_max_req_per_msg < 1 || skd_max_req_per_msg > 14) {
5471 pr_info(
5472 "skd_max_req_per_msg %d invalid, re-set to %d\n",
5473 skd_max_req_per_msg, SKD_MAX_REQ_PER_MSG_DEFAULT);
5474 skd_max_req_per_msg = SKD_MAX_REQ_PER_MSG_DEFAULT;
5475 }
5476
5477 if (skd_sgs_per_request < 1 || skd_sgs_per_request > 4096) {
5478 pr_info(
5479 "skd_sg_per_request %d invalid, re-set to %d\n",
5480 skd_sgs_per_request, SKD_N_SG_PER_REQ_DEFAULT);
5481 skd_sgs_per_request = SKD_N_SG_PER_REQ_DEFAULT;
5482 }
5483
5484 if (skd_dbg_level < 0 || skd_dbg_level > 2) {
5485 pr_info("skd_dbg_level %d invalid, re-set to %d\n",
5486 skd_dbg_level, 0);
5487 skd_dbg_level = 0;
5488 }
5489
5490 if (skd_isr_comp_limit < 0) {
5491 pr_info("skd_isr_comp_limit %d invalid, set to %d\n",
5492 skd_isr_comp_limit, 0);
5493 skd_isr_comp_limit = 0;
5494 }
5495
5496 if (skd_max_pass_thru < 1 || skd_max_pass_thru > 50) {
5497 pr_info("skd_max_pass_thru %d invalid, re-set to %d\n",
5498 skd_max_pass_thru, SKD_N_SPECIAL_CONTEXT);
5499 skd_max_pass_thru = SKD_N_SPECIAL_CONTEXT;
5500 }
5501
5502 /* Obtain major device number. */
5503 rc = register_blkdev(0, DRV_NAME);
5504 if (rc < 0)
5505 return rc;
5506
5507 skd_major = rc;
5508
5509 return pci_register_driver(&skd_driver);
5510
5511}
5512
5513static void __exit skd_exit(void)
5514{
5515 pr_info(PFX " v%s-b%s unloading\n", DRV_VERSION, DRV_BUILD_ID);
5516
5517 unregister_blkdev(skd_major, DRV_NAME);
5518 pci_unregister_driver(&skd_driver);
5519
5520 kmem_cache_destroy(skd_flush_slab);
5521}
5522
e67f86b3
AB		 5523module_init(skd_init);
5524module_exit(skd_exit);
Linux 6.x block layer and io_uring tree(s)