projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
treewide: Replace GPLv2 boilerplate/reference with SPDX - rule 156
[linux-block.git] / drivers / block / rsxx / core.c
CommitLineData
1a59d1b8 1// SPDX-License-Identifier: GPL-2.0-or-later
8722ff8c 2/*
3* Filename: core.c
4*
8722ff8c 5* Authors: Joshua Morris <josh.h.morris@us.ibm.com>
6* Philip Kelleher <pjk1939@linux.vnet.ibm.com>
7*
8* (C) Copyright 2013 IBM Corporation
8722ff8c 9*/
10
11#include <linux/kernel.h>
12#include <linux/init.h>
13#include <linux/interrupt.h>
14#include <linux/module.h>
15#include <linux/pci.h>
16#include <linux/reboot.h>
17#include <linux/slab.h>
18#include <linux/bitops.h>
c95246c3 19#include <linux/delay.h>
36f988e9
PK		 20#include <linux/debugfs.h>
21#include <linux/seq_file.h>
8722ff8c 22
23#include <linux/genhd.h>
24#include <linux/idr.h>
25
26#include "rsxx_priv.h"
27#include "rsxx_cfg.h"
28
29#define NO_LEGACY 0
fb065cd9 30#define SYNC_START_TIMEOUT (10 * 60) /* 10 minutes */
8722ff8c 31
f730e3dc 32MODULE_DESCRIPTION("IBM Flash Adapter 900GB Full Height Device Driver");
9bb3c446 33MODULE_AUTHOR("Joshua Morris/Philip Kelleher, IBM");
8722ff8c 34MODULE_LICENSE("GPL");
35MODULE_VERSION(DRIVER_VERSION);
36
37static unsigned int force_legacy = NO_LEGACY;
38module_param(force_legacy, uint, 0444);
39MODULE_PARM_DESC(force_legacy, "Force the use of legacy type PCI interrupts");
40
fb065cd9
PK		 41static unsigned int sync_start = 1;
42module_param(sync_start, uint, 0444);
43MODULE_PARM_DESC(sync_start, "On by Default: Driver load will not complete "
44 "until the card startup has completed.");
45
8722ff8c 46static DEFINE_IDA(rsxx_disk_ida);
8722ff8c 47
36f988e9
PK		 48/* --------------------Debugfs Setup ------------------- */
49
36f988e9
PK		 50static int rsxx_attr_pci_regs_show(struct seq_file *m, void *p)
51{
52 struct rsxx_cardinfo *card = m->private;
53
54 seq_printf(m, "HWID 0x%08x\n",
55 ioread32(card->regmap + HWID));
56 seq_printf(m, "SCRATCH 0x%08x\n",
57 ioread32(card->regmap + SCRATCH));
58 seq_printf(m, "IER 0x%08x\n",
59 ioread32(card->regmap + IER));
60 seq_printf(m, "IPR 0x%08x\n",
61 ioread32(card->regmap + IPR));
62 seq_printf(m, "CREG_CMD 0x%08x\n",
63 ioread32(card->regmap + CREG_CMD));
64 seq_printf(m, "CREG_ADD 0x%08x\n",
65 ioread32(card->regmap + CREG_ADD));
66 seq_printf(m, "CREG_CNT 0x%08x\n",
67 ioread32(card->regmap + CREG_CNT));
68 seq_printf(m, "CREG_STAT 0x%08x\n",
69 ioread32(card->regmap + CREG_STAT));
70 seq_printf(m, "CREG_DATA0 0x%08x\n",
71 ioread32(card->regmap + CREG_DATA0));
72 seq_printf(m, "CREG_DATA1 0x%08x\n",
73 ioread32(card->regmap + CREG_DATA1));
74 seq_printf(m, "CREG_DATA2 0x%08x\n",
75 ioread32(card->regmap + CREG_DATA2));
76 seq_printf(m, "CREG_DATA3 0x%08x\n",
77 ioread32(card->regmap + CREG_DATA3));
78 seq_printf(m, "CREG_DATA4 0x%08x\n",
79 ioread32(card->regmap + CREG_DATA4));
80 seq_printf(m, "CREG_DATA5 0x%08x\n",
81 ioread32(card->regmap + CREG_DATA5));
82 seq_printf(m, "CREG_DATA6 0x%08x\n",
83 ioread32(card->regmap + CREG_DATA6));
84 seq_printf(m, "CREG_DATA7 0x%08x\n",
85 ioread32(card->regmap + CREG_DATA7));
86 seq_printf(m, "INTR_COAL 0x%08x\n",
87 ioread32(card->regmap + INTR_COAL));
88 seq_printf(m, "HW_ERROR 0x%08x\n",
89 ioread32(card->regmap + HW_ERROR));
90 seq_printf(m, "DEBUG0 0x%08x\n",
91 ioread32(card->regmap + PCI_DEBUG0));
92 seq_printf(m, "DEBUG1 0x%08x\n",
93 ioread32(card->regmap + PCI_DEBUG1));
94 seq_printf(m, "DEBUG2 0x%08x\n",
95 ioread32(card->regmap + PCI_DEBUG2));
96 seq_printf(m, "DEBUG3 0x%08x\n",
97 ioread32(card->regmap + PCI_DEBUG3));
98 seq_printf(m, "DEBUG4 0x%08x\n",
99 ioread32(card->regmap + PCI_DEBUG4));
100 seq_printf(m, "DEBUG5 0x%08x\n",
101 ioread32(card->regmap + PCI_DEBUG5));
102 seq_printf(m, "DEBUG6 0x%08x\n",
103 ioread32(card->regmap + PCI_DEBUG6));
104 seq_printf(m, "DEBUG7 0x%08x\n",
105 ioread32(card->regmap + PCI_DEBUG7));
106 seq_printf(m, "RECONFIG 0x%08x\n",
107 ioread32(card->regmap + PCI_RECONFIG));
108
109 return 0;
110}
111
112static int rsxx_attr_stats_show(struct seq_file *m, void *p)
113{
114 struct rsxx_cardinfo *card = m->private;
115 int i;
116
117 for (i = 0; i < card->n_targets; i++) {
118 seq_printf(m, "Ctrl %d CRC Errors = %d\n",
119 i, card->ctrl[i].stats.crc_errors);
120 seq_printf(m, "Ctrl %d Hard Errors = %d\n",
121 i, card->ctrl[i].stats.hard_errors);
122 seq_printf(m, "Ctrl %d Soft Errors = %d\n",
123 i, card->ctrl[i].stats.soft_errors);
124 seq_printf(m, "Ctrl %d Writes Issued = %d\n",
125 i, card->ctrl[i].stats.writes_issued);
126 seq_printf(m, "Ctrl %d Writes Failed = %d\n",
127 i, card->ctrl[i].stats.writes_failed);
128 seq_printf(m, "Ctrl %d Reads Issued = %d\n",
129 i, card->ctrl[i].stats.reads_issued);
130 seq_printf(m, "Ctrl %d Reads Failed = %d\n",
131 i, card->ctrl[i].stats.reads_failed);
132 seq_printf(m, "Ctrl %d Reads Retried = %d\n",
133 i, card->ctrl[i].stats.reads_retried);
134 seq_printf(m, "Ctrl %d Discards Issued = %d\n",
135 i, card->ctrl[i].stats.discards_issued);
136 seq_printf(m, "Ctrl %d Discards Failed = %d\n",
137 i, card->ctrl[i].stats.discards_failed);
138 seq_printf(m, "Ctrl %d DMA SW Errors = %d\n",
139 i, card->ctrl[i].stats.dma_sw_err);
140 seq_printf(m, "Ctrl %d DMA HW Faults = %d\n",
141 i, card->ctrl[i].stats.dma_hw_fault);
142 seq_printf(m, "Ctrl %d DMAs Cancelled = %d\n",
143 i, card->ctrl[i].stats.dma_cancelled);
144 seq_printf(m, "Ctrl %d SW Queue Depth = %d\n",
145 i, card->ctrl[i].stats.sw_q_depth);
146 seq_printf(m, "Ctrl %d HW Queue Depth = %d\n",
147 i, atomic_read(&card->ctrl[i].stats.hw_q_depth));
148 }
149
150 return 0;
151}
152
153static int rsxx_attr_stats_open(struct inode *inode, struct file *file)
154{
155 return single_open(file, rsxx_attr_stats_show, inode->i_private);
156}
157
158static int rsxx_attr_pci_regs_open(struct inode *inode, struct file *file)
159{
160 return single_open(file, rsxx_attr_pci_regs_show, inode->i_private);
161}
162
163static ssize_t rsxx_cram_read(struct file *fp, char __user *ubuf,
164 size_t cnt, loff_t *ppos)
165{
8e3fb059 166 struct rsxx_cardinfo *card = file_inode(fp)->i_private;
36f988e9 167 char *buf;
8e3fb059 168 ssize_t st;
36f988e9 169
8e3fb059 170 buf = kzalloc(cnt, GFP_KERNEL);
36f988e9
PK		 171 if (!buf)
172 return -ENOMEM;
173
8e3fb059
AV		 174 st = rsxx_creg_read(card, CREG_ADD_CRAM + (u32)*ppos, cnt, buf, 1);
175 if (!st)
176 st = copy_to_user(ubuf, buf, cnt);
177 kfree(buf);
36f988e9
PK		 178 if (st)
179 return st;
8e3fb059 180 *ppos += cnt;
36f988e9
PK		 181 return cnt;
182}
183
184static ssize_t rsxx_cram_write(struct file *fp, const char __user *ubuf,
185 size_t cnt, loff_t *ppos)
186{
8e3fb059 187 struct rsxx_cardinfo *card = file_inode(fp)->i_private;
36f988e9 188 char *buf;
8e3fb059 189 ssize_t st;
36f988e9 190
820351f0
AV		 191 buf = memdup_user(ubuf, cnt);
192 if (IS_ERR(buf))
193 return PTR_ERR(buf);
36f988e9 194
820351f0 195 st = rsxx_creg_write(card, CREG_ADD_CRAM + (u32)*ppos, cnt, buf, 1);
8e3fb059 196 kfree(buf);
36f988e9
PK		 197 if (st)
198 return st;
8e3fb059 199 *ppos += cnt;
36f988e9
PK		 200 return cnt;
201}
202
36f988e9
PK		 203static const struct file_operations debugfs_cram_fops = {
204 .owner = THIS_MODULE,
36f988e9
PK		 205 .read = rsxx_cram_read,
206 .write = rsxx_cram_write,
36f988e9
PK		 207};
208
209static const struct file_operations debugfs_stats_fops = {
210 .owner = THIS_MODULE,
211 .open = rsxx_attr_stats_open,
212 .read = seq_read,
213 .llseek = seq_lseek,
214 .release = single_release,
215};
216
217static const struct file_operations debugfs_pci_regs_fops = {
218 .owner = THIS_MODULE,
219 .open = rsxx_attr_pci_regs_open,
220 .read = seq_read,
221 .llseek = seq_lseek,
222 .release = single_release,
223};
224
225static void rsxx_debugfs_dev_new(struct rsxx_cardinfo *card)
226{
227 struct dentry *debugfs_stats;
228 struct dentry *debugfs_pci_regs;
229 struct dentry *debugfs_cram;
230
231 card->debugfs_dir = debugfs_create_dir(card->gendisk->disk_name, NULL);
232 if (IS_ERR_OR_NULL(card->debugfs_dir))
233 goto failed_debugfs_dir;
234
5657a819 235 debugfs_stats = debugfs_create_file("stats", 0444,
36f988e9
PK		 236 card->debugfs_dir, card,
237 &debugfs_stats_fops);
238 if (IS_ERR_OR_NULL(debugfs_stats))
239 goto failed_debugfs_stats;
240
5657a819 241 debugfs_pci_regs = debugfs_create_file("pci_regs", 0444,
36f988e9
PK		 242 card->debugfs_dir, card,
243 &debugfs_pci_regs_fops);
244 if (IS_ERR_OR_NULL(debugfs_pci_regs))
245 goto failed_debugfs_pci_regs;
246
5657a819 247 debugfs_cram = debugfs_create_file("cram", 0644,
36f988e9
PK		 248 card->debugfs_dir, card,
249 &debugfs_cram_fops);
250 if (IS_ERR_OR_NULL(debugfs_cram))
251 goto failed_debugfs_cram;
252
253 return;
254failed_debugfs_cram:
255 debugfs_remove(debugfs_pci_regs);
256failed_debugfs_pci_regs:
257 debugfs_remove(debugfs_stats);
258failed_debugfs_stats:
259 debugfs_remove(card->debugfs_dir);
260failed_debugfs_dir:
261 card->debugfs_dir = NULL;
262}
263
8722ff8c 264/*----------------- Interrupt Control & Handling -------------------*/
c95246c3
PK		 265
266static void rsxx_mask_interrupts(struct rsxx_cardinfo *card)
267{
268 card->isr_mask = 0;
269 card->ier_mask = 0;
270}
271
8722ff8c 272static void __enable_intr(unsigned int *mask, unsigned int intr)
273{
274 *mask |= intr;
275}
276
277static void __disable_intr(unsigned int *mask, unsigned int intr)
278{
279 *mask &= ~intr;
280}
281
282/*
283 * NOTE: Disabling the IER will disable the hardware interrupt.
284 * Disabling the ISR will disable the software handling of the ISR bit.
285 *
286 * Enable/Disable interrupt functions assume the card->irq_lock
287 * is held by the caller.
288 */
289void rsxx_enable_ier(struct rsxx_cardinfo *card, unsigned int intr)
290{
c95246c3
PK		 291 if (unlikely(card->halt) ||
292 unlikely(card->eeh_state))
8722ff8c 293 return;
294
295 __enable_intr(&card->ier_mask, intr);
296 iowrite32(card->ier_mask, card->regmap + IER);
297}
298
299void rsxx_disable_ier(struct rsxx_cardinfo *card, unsigned int intr)
300{
c95246c3
PK		 301 if (unlikely(card->eeh_state))
302 return;
303
8722ff8c 304 __disable_intr(&card->ier_mask, intr);
305 iowrite32(card->ier_mask, card->regmap + IER);
306}
307
308void rsxx_enable_ier_and_isr(struct rsxx_cardinfo *card,
309 unsigned int intr)
310{
c95246c3
PK		 311 if (unlikely(card->halt) ||
312 unlikely(card->eeh_state))
8722ff8c 313 return;
314
315 __enable_intr(&card->isr_mask, intr);
316 __enable_intr(&card->ier_mask, intr);
317 iowrite32(card->ier_mask, card->regmap + IER);
318}
319void rsxx_disable_ier_and_isr(struct rsxx_cardinfo *card,
320 unsigned int intr)
321{
c95246c3
PK		 322 if (unlikely(card->eeh_state))
323 return;
324
8722ff8c 325 __disable_intr(&card->isr_mask, intr);
326 __disable_intr(&card->ier_mask, intr);
327 iowrite32(card->ier_mask, card->regmap + IER);
328}
329
c206c709 330static irqreturn_t rsxx_isr(int irq, void *pdata)
8722ff8c 331{
c206c709 332 struct rsxx_cardinfo *card = pdata;
8722ff8c 333 unsigned int isr;
334 int handled = 0;
335 int reread_isr;
336 int i;
337
338 spin_lock(&card->irq_lock);
339
340 do {
341 reread_isr = 0;
342
c95246c3
PK		 343 if (unlikely(card->eeh_state))
344 break;
345
8722ff8c 346 isr = ioread32(card->regmap + ISR);
347 if (isr == 0xffffffff) {
348 /*
349 * A few systems seem to have an intermittent issue
350 * where PCI reads return all Fs, but retrying the read
351 * a little later will return as expected.
352 */
353 dev_info(CARD_TO_DEV(card),
354 "ISR = 0xFFFFFFFF, retrying later\n");
355 break;
356 }
357
358 isr &= card->isr_mask;
359 if (!isr)
360 break;
361
362 for (i = 0; i < card->n_targets; i++) {
363 if (isr & CR_INTR_DMA(i)) {
364 if (card->ier_mask & CR_INTR_DMA(i)) {
365 rsxx_disable_ier(card, CR_INTR_DMA(i));
366 reread_isr = 1;
367 }
368 queue_work(card->ctrl[i].done_wq,
369 &card->ctrl[i].dma_done_work);
370 handled++;
371 }
372 }
373
374 if (isr & CR_INTR_CREG) {
a3299ab1
PK		 375 queue_work(card->creg_ctrl.creg_wq,
376 &card->creg_ctrl.done_work);
8722ff8c 377 handled++;
378 }
379
380 if (isr & CR_INTR_EVENT) {
a3299ab1 381 queue_work(card->event_wq, &card->event_work);
8722ff8c 382 rsxx_disable_ier_and_isr(card, CR_INTR_EVENT);
383 handled++;
384 }
385 } while (reread_isr);
386
387 spin_unlock(&card->irq_lock);
388
389 return handled ? IRQ_HANDLED : IRQ_NONE;
390}
391
392/*----------------- Card Event Handler -------------------*/
f3791203 393static const char * const rsxx_card_state_to_str(unsigned int state)
c206c709 394{
f3791203 395 static const char * const state_strings[] = {
c206c709
PK		 396 "Unknown", "Shutdown", "Starting", "Formatting",
397 "Uninitialized", "Good", "Shutting Down",
398 "Fault", "Read Only Fault", "dStroying"
399 };
400
401 return state_strings[ffs(state)];
402}
403
8722ff8c 404static void card_state_change(struct rsxx_cardinfo *card,
405 unsigned int new_state)
406{
407 int st;
408
409 dev_info(CARD_TO_DEV(card),
410 "card state change detected.(%s -> %s)\n",
411 rsxx_card_state_to_str(card->state),
412 rsxx_card_state_to_str(new_state));
413
414 card->state = new_state;
415
416 /* Don't attach DMA interfaces if the card has an invalid config */
417 if (!card->config_valid)
418 return;
419
420 switch (new_state) {
421 case CARD_STATE_RD_ONLY_FAULT:
422 dev_crit(CARD_TO_DEV(card),
423 "Hardware has entered read-only mode!\n");
424 /*
425 * Fall through so the DMA devices can be attached and
426 * the user can attempt to pull off their data.
427 */
e16fb3a8 428 /* fall through */
8722ff8c 429 case CARD_STATE_GOOD:
430 st = rsxx_get_card_size8(card, &card->size8);
431 if (st)
432 dev_err(CARD_TO_DEV(card),
433 "Failed attaching DMA devices\n");
434
435 if (card->config_valid)
436 set_capacity(card->gendisk, card->size8 >> 9);
437 break;
438
439 case CARD_STATE_FAULT:
440 dev_crit(CARD_TO_DEV(card),
441 "Hardware Fault reported!\n");
442 /* Fall through. */
443
444 /* Everything else, detach DMA interface if it's attached. */
445 case CARD_STATE_SHUTDOWN:
446 case CARD_STATE_STARTING:
447 case CARD_STATE_FORMATTING:
448 case CARD_STATE_UNINITIALIZED:
449 case CARD_STATE_SHUTTING_DOWN:
450 /*
451 * dStroy is a term coined by marketing to represent the low level
452 * secure erase.
453 */
454 case CARD_STATE_DSTROYING:
455 set_capacity(card->gendisk, 0);
456 break;
457 }
458}
459
460static void card_event_handler(struct work_struct *work)
461{
462 struct rsxx_cardinfo *card;
463 unsigned int state;
464 unsigned long flags;
465 int st;
466
467 card = container_of(work, struct rsxx_cardinfo, event_work);
468
469 if (unlikely(card->halt))
470 return;
471
472 /*
473 * Enable the interrupt now to avoid any weird race conditions where a
474 * state change might occur while rsxx_get_card_state() is
475 * processing a returned creg cmd.
476 */
477 spin_lock_irqsave(&card->irq_lock, flags);
478 rsxx_enable_ier_and_isr(card, CR_INTR_EVENT);
479 spin_unlock_irqrestore(&card->irq_lock, flags);
480
481 st = rsxx_get_card_state(card, &state);
482 if (st) {
483 dev_info(CARD_TO_DEV(card),
484 "Failed reading state after event.\n");
485 return;
486 }
487
488 if (card->state != state)
489 card_state_change(card, state);
490
491 if (card->creg_ctrl.creg_stats.stat & CREG_STAT_LOG_PENDING)
492 rsxx_read_hw_log(card);
493}
494
8722ff8c 495/*----------------- Card Operations -------------------*/
496static int card_shutdown(struct rsxx_cardinfo *card)
497{
498 unsigned int state;
499 signed long start;
500 const int timeout = msecs_to_jiffies(120000);
501 int st;
502
503 /* We can't issue a shutdown if the card is in a transition state */
504 start = jiffies;
505 do {
506 st = rsxx_get_card_state(card, &state);
507 if (st)
508 return st;
509 } while (state == CARD_STATE_STARTING &&
510 (jiffies - start < timeout));
511
512 if (state == CARD_STATE_STARTING)
513 return -ETIMEDOUT;
514
515 /* Only issue a shutdown if we need to */
516 if ((state != CARD_STATE_SHUTTING_DOWN) &&
517 (state != CARD_STATE_SHUTDOWN)) {
518 st = rsxx_issue_card_cmd(card, CARD_CMD_SHUTDOWN);
519 if (st)
520 return st;
521 }
522
523 start = jiffies;
524 do {
525 st = rsxx_get_card_state(card, &state);
526 if (st)
527 return st;
528 } while (state != CARD_STATE_SHUTDOWN &&
529 (jiffies - start < timeout));
530
531 if (state != CARD_STATE_SHUTDOWN)
532 return -ETIMEDOUT;
533
534 return 0;
535}
536
4dcaf472 537static int rsxx_eeh_frozen(struct pci_dev *dev)
c95246c3
PK		 538{
539 struct rsxx_cardinfo *card = pci_get_drvdata(dev);
540 int i;
4dcaf472 541 int st;
c95246c3 542
f730e3dc 543 dev_warn(&dev->dev, "IBM Flash Adapter PCI: preparing for slot reset.\n");
c95246c3
PK		 544
545 card->eeh_state = 1;
546 rsxx_mask_interrupts(card);
547
548 /*
549 * We need to guarantee that the write for eeh_state and masking
550 * interrupts does not become reordered. This will prevent a possible
551 * race condition with the EEH code.
552 */
553 wmb();
554
555 pci_disable_device(dev);
556
4dcaf472
PK		 557 st = rsxx_eeh_save_issued_dmas(card);
558 if (st)
559 return st;
c95246c3
PK		 560
561 rsxx_eeh_save_issued_creg(card);
562
563 for (i = 0; i < card->n_targets; i++) {
564 if (card->ctrl[i].status.buf)
565 pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8,
566 card->ctrl[i].status.buf,
567 card->ctrl[i].status.dma_addr);
568 if (card->ctrl[i].cmd.buf)
569 pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8,
570 card->ctrl[i].cmd.buf,
571 card->ctrl[i].cmd.dma_addr);
572 }
4dcaf472
PK		 573
574 return 0;
c95246c3
PK		 575}
576
577static void rsxx_eeh_failure(struct pci_dev *dev)
578{
579 struct rsxx_cardinfo *card = pci_get_drvdata(dev);
580 int i;
0ab4743e 581 int cnt = 0;
c95246c3 582
f730e3dc 583 dev_err(&dev->dev, "IBM Flash Adapter PCI: disabling failed card.\n");
c95246c3
PK		 584
585 card->eeh_state = 1;
0ab4743e 586 card->halt = 1;
c95246c3 587
0ab4743e
PK		 588 for (i = 0; i < card->n_targets; i++) {
589 spin_lock_bh(&card->ctrl[i].queue_lock);
590 cnt = rsxx_cleanup_dma_queue(&card->ctrl[i],
e5feab22
PK		 591 &card->ctrl[i].queue,
592 COMPLETE_DMA);
0ab4743e
PK		 593 spin_unlock_bh(&card->ctrl[i].queue_lock);
594
595 cnt += rsxx_dma_cancel(&card->ctrl[i]);
c95246c3 596
0ab4743e
PK		 597 if (cnt)
598 dev_info(CARD_TO_DEV(card),
599 "Freed %d queued DMAs on channel %d\n",
600 cnt, card->ctrl[i].id);
601 }
c95246c3
PK		 602}
603
604static int rsxx_eeh_fifo_flush_poll(struct rsxx_cardinfo *card)
605{
606 unsigned int status;
607 int iter = 0;
608
609 /* We need to wait for the hardware to reset */
610 while (iter++ < 10) {
611 status = ioread32(card->regmap + PCI_RECONFIG);
612
613 if (status & RSXX_FLUSH_BUSY) {
614 ssleep(1);
615 continue;
616 }
617
618 if (status & RSXX_FLUSH_TIMEOUT)
619 dev_warn(CARD_TO_DEV(card), "HW: flash controller timeout\n");
620 return 0;
621 }
622
623 /* Hardware failed resetting itself. */
624 return -1;
625}
626
627static pci_ers_result_t rsxx_error_detected(struct pci_dev *dev,
628 enum pci_channel_state error)
629{
4dcaf472
PK		 630 int st;
631
c95246c3
PK		 632 if (dev->revision < RSXX_EEH_SUPPORT)
633 return PCI_ERS_RESULT_NONE;
634
635 if (error == pci_channel_io_perm_failure) {
636 rsxx_eeh_failure(dev);
637 return PCI_ERS_RESULT_DISCONNECT;
638 }
639
4dcaf472
PK		 640 st = rsxx_eeh_frozen(dev);
641 if (st) {
642 dev_err(&dev->dev, "Slot reset setup failed\n");
643 rsxx_eeh_failure(dev);
644 return PCI_ERS_RESULT_DISCONNECT;
645 }
646
c95246c3
PK		 647 return PCI_ERS_RESULT_NEED_RESET;
648}
649
650static pci_ers_result_t rsxx_slot_reset(struct pci_dev *dev)
651{
652 struct rsxx_cardinfo *card = pci_get_drvdata(dev);
653 unsigned long flags;
654 int i;
655 int st;
656
657 dev_warn(&dev->dev,
f730e3dc 658 "IBM Flash Adapter PCI: recovering from slot reset.\n");
c95246c3
PK		 659
660 st = pci_enable_device(dev);
661 if (st)
662 goto failed_hw_setup;
663
664 pci_set_master(dev);
665
666 st = rsxx_eeh_fifo_flush_poll(card);
667 if (st)
668 goto failed_hw_setup;
669
670 rsxx_dma_queue_reset(card);
671
672 for (i = 0; i < card->n_targets; i++) {
673 st = rsxx_hw_buffers_init(dev, &card->ctrl[i]);
674 if (st)
675 goto failed_hw_buffers_init;
676 }
677
678 if (card->config_valid)
679 rsxx_dma_configure(card);
680
681 /* Clears the ISR register from spurious interrupts */
682 st = ioread32(card->regmap + ISR);
683
684 card->eeh_state = 0;
685
c95246c3
PK		 686 spin_lock_irqsave(&card->irq_lock, flags);
687 if (card->n_targets & RSXX_MAX_TARGETS)
688 rsxx_enable_ier_and_isr(card, CR_INTR_ALL_G);
689 else
690 rsxx_enable_ier_and_isr(card, CR_INTR_ALL_C);
691 spin_unlock_irqrestore(&card->irq_lock, flags);
692
693 rsxx_kick_creg_queue(card);
694
695 for (i = 0; i < card->n_targets; i++) {
696 spin_lock(&card->ctrl[i].queue_lock);
697 if (list_empty(&card->ctrl[i].queue)) {
698 spin_unlock(&card->ctrl[i].queue_lock);
699 continue;
700 }
701 spin_unlock(&card->ctrl[i].queue_lock);
702
703 queue_work(card->ctrl[i].issue_wq,
704 &card->ctrl[i].issue_dma_work);
705 }
706
f730e3dc 707 dev_info(&dev->dev, "IBM Flash Adapter PCI: recovery complete.\n");
c95246c3
PK		 708
709 return PCI_ERS_RESULT_RECOVERED;
710
711failed_hw_buffers_init:
c95246c3
PK		 712 for (i = 0; i < card->n_targets; i++) {
713 if (card->ctrl[i].status.buf)
714 pci_free_consistent(card->dev,
715 STATUS_BUFFER_SIZE8,
716 card->ctrl[i].status.buf,
717 card->ctrl[i].status.dma_addr);
718 if (card->ctrl[i].cmd.buf)
719 pci_free_consistent(card->dev,
720 COMMAND_BUFFER_SIZE8,
721 card->ctrl[i].cmd.buf,
722 card->ctrl[i].cmd.dma_addr);
723 }
724failed_hw_setup:
725 rsxx_eeh_failure(dev);
726 return PCI_ERS_RESULT_DISCONNECT;
727
728}
729
8722ff8c 730/*----------------- Driver Initialization & Setup -------------------*/
731/* Returns: 0 if the driver is compatible with the device
732 -1 if the driver is NOT compatible with the device */
733static int rsxx_compatibility_check(struct rsxx_cardinfo *card)
734{
735 unsigned char pci_rev;
736
737 pci_read_config_byte(card->dev, PCI_REVISION_ID, &pci_rev);
738
739 if (pci_rev > RS70_PCI_REV_SUPPORTED)
740 return -1;
741 return 0;
742}
743
82bed4d5 744static int rsxx_pci_probe(struct pci_dev *dev,
8722ff8c 745 const struct pci_device_id *id)
746{
747 struct rsxx_cardinfo *card;
8722ff8c 748 int st;
fb065cd9 749 unsigned int sync_timeout;
8722ff8c 750
751 dev_info(&dev->dev, "PCI-Flash SSD discovered\n");
752
753 card = kzalloc(sizeof(*card), GFP_KERNEL);
754 if (!card)
755 return -ENOMEM;
756
757 card->dev = dev;
758 pci_set_drvdata(dev, card);
759
37ae133c
MW		 760 st = ida_alloc(&rsxx_disk_ida, GFP_KERNEL);
761 if (st < 0)
8722ff8c 762 goto failed_ida_get;
37ae133c 763 card->disk_id = st;
8722ff8c 764
765 st = pci_enable_device(dev);
766 if (st)
767 goto failed_enable;
768
769 pci_set_master(dev);
b0da3498 770 dma_set_max_seg_size(&dev->dev, RSXX_HW_BLK_SIZE);
8722ff8c 771
77a12e51 772 st = dma_set_mask(&dev->dev, DMA_BIT_MASK(64));
8722ff8c 773 if (st) {
774 dev_err(CARD_TO_DEV(card),
775 "No usable DMA configuration,aborting\n");
776 goto failed_dma_mask;
777 }
778
779 st = pci_request_regions(dev, DRIVER_NAME);
780 if (st) {
781 dev_err(CARD_TO_DEV(card),
782 "Failed to request memory region\n");
783 goto failed_request_regions;
784 }
785
786 if (pci_resource_len(dev, 0) == 0) {
787 dev_err(CARD_TO_DEV(card), "BAR0 has length 0!\n");
788 st = -ENOMEM;
789 goto failed_iomap;
790 }
791
792 card->regmap = pci_iomap(dev, 0, 0);
793 if (!card->regmap) {
794 dev_err(CARD_TO_DEV(card), "Failed to map BAR0\n");
795 st = -ENOMEM;
796 goto failed_iomap;
797 }
798
799 spin_lock_init(&card->irq_lock);
800 card->halt = 0;
c95246c3 801 card->eeh_state = 0;
8722ff8c 802
c206c709 803 spin_lock_irq(&card->irq_lock);
8722ff8c 804 rsxx_disable_ier_and_isr(card, CR_INTR_ALL);
c206c709 805 spin_unlock_irq(&card->irq_lock);
8722ff8c 806
807 if (!force_legacy) {
808 st = pci_enable_msi(dev);
809 if (st)
810 dev_warn(CARD_TO_DEV(card),
811 "Failed to enable MSI\n");
812 }
813
baf37812 814 st = request_irq(dev->irq, rsxx_isr, IRQF_SHARED,
8722ff8c 815 DRIVER_NAME, card);
816 if (st) {
817 dev_err(CARD_TO_DEV(card),
818 "Failed requesting IRQ%d\n", dev->irq);
819 goto failed_irq;
820 }
821
822 /************* Setup Processor Command Interface *************/
a3299ab1
PK		 823 st = rsxx_creg_setup(card);
824 if (st) {
825 dev_err(CARD_TO_DEV(card), "Failed to setup creg interface.\n");
826 goto failed_creg_setup;
827 }
8722ff8c 828
c206c709 829 spin_lock_irq(&card->irq_lock);
8722ff8c 830 rsxx_enable_ier_and_isr(card, CR_INTR_CREG);
c206c709 831 spin_unlock_irq(&card->irq_lock);
8722ff8c 832
833 st = rsxx_compatibility_check(card);
834 if (st) {
835 dev_warn(CARD_TO_DEV(card),
836 "Incompatible driver detected. Please update the driver.\n");
837 st = -EINVAL;
838 goto failed_compatiblity_check;
839 }
840
841 /************* Load Card Config *************/
842 st = rsxx_load_config(card);
843 if (st)
844 dev_err(CARD_TO_DEV(card),
845 "Failed loading card config\n");
846
847 /************* Setup DMA Engine *************/
848 st = rsxx_get_num_targets(card, &card->n_targets);
849 if (st)
850 dev_info(CARD_TO_DEV(card),
851 "Failed reading the number of DMA targets\n");
852
6396bb22
KC		 853 card->ctrl = kcalloc(card->n_targets, sizeof(*card->ctrl),
854 GFP_KERNEL);
8722ff8c 855 if (!card->ctrl) {
856 st = -ENOMEM;
857 goto failed_dma_setup;
858 }
859
860 st = rsxx_dma_setup(card);
861 if (st) {
862 dev_info(CARD_TO_DEV(card),
863 "Failed to setup DMA engine\n");
864 goto failed_dma_setup;
865 }
866
867 /************* Setup Card Event Handler *************/
a3299ab1
PK		 868 card->event_wq = create_singlethread_workqueue(DRIVER_NAME"_event");
869 if (!card->event_wq) {
870 dev_err(CARD_TO_DEV(card), "Failed card event setup.\n");
871 goto failed_event_handler;
872 }
873
8722ff8c 874 INIT_WORK(&card->event_work, card_event_handler);
875
876 st = rsxx_setup_dev(card);
877 if (st)
878 goto failed_create_dev;
879
880 rsxx_get_card_state(card, &card->state);
881
882 dev_info(CARD_TO_DEV(card),
883 "card state: %s\n",
884 rsxx_card_state_to_str(card->state));
885
886 /*
887 * Now that the DMA Engine and devices have been setup,
888 * we can enable the event interrupt(it kicks off actions in
889 * those layers so we couldn't enable it right away.)
890 */
c206c709 891 spin_lock_irq(&card->irq_lock);
8722ff8c 892 rsxx_enable_ier_and_isr(card, CR_INTR_EVENT);
c206c709 893 spin_unlock_irq(&card->irq_lock);
8722ff8c 894
895 if (card->state == CARD_STATE_SHUTDOWN) {
896 st = rsxx_issue_card_cmd(card, CARD_CMD_STARTUP);
897 if (st)
898 dev_crit(CARD_TO_DEV(card),
899 "Failed issuing card startup\n");
fb065cd9
PK		 900 if (sync_start) {
901 sync_timeout = SYNC_START_TIMEOUT;
902
903 dev_info(CARD_TO_DEV(card),
904 "Waiting for card to startup\n");
905
906 do {
907 ssleep(1);
908 sync_timeout--;
909
910 rsxx_get_card_state(card, &card->state);
911 } while (sync_timeout &&
912 (card->state == CARD_STATE_STARTING));
913
914 if (card->state == CARD_STATE_STARTING) {
915 dev_warn(CARD_TO_DEV(card),
916 "Card startup timed out\n");
917 card->size8 = 0;
918 } else {
919 dev_info(CARD_TO_DEV(card),
920 "card state: %s\n",
921 rsxx_card_state_to_str(card->state));
922 st = rsxx_get_card_size8(card, &card->size8);
923 if (st)
924 card->size8 = 0;
925 }
926 }
8722ff8c 927 } else if (card->state == CARD_STATE_GOOD ||
928 card->state == CARD_STATE_RD_ONLY_FAULT) {
929 st = rsxx_get_card_size8(card, &card->size8);
930 if (st)
931 card->size8 = 0;
932 }
933
934 rsxx_attach_dev(card);
935
36f988e9
PK		 936 /************* Setup Debugfs *************/
937 rsxx_debugfs_dev_new(card);
938
8722ff8c 939 return 0;
940
941failed_create_dev:
a3299ab1
PK		 942 destroy_workqueue(card->event_wq);
943 card->event_wq = NULL;
944failed_event_handler:
8722ff8c 945 rsxx_dma_destroy(card);
946failed_dma_setup:
947failed_compatiblity_check:
a3299ab1
PK		 948 destroy_workqueue(card->creg_ctrl.creg_wq);
949 card->creg_ctrl.creg_wq = NULL;
950failed_creg_setup:
c206c709 951 spin_lock_irq(&card->irq_lock);
8722ff8c 952 rsxx_disable_ier_and_isr(card, CR_INTR_ALL);
c206c709 953 spin_unlock_irq(&card->irq_lock);
8722ff8c 954 free_irq(dev->irq, card);
955 if (!force_legacy)
956 pci_disable_msi(dev);
957failed_irq:
958 pci_iounmap(dev, card->regmap);
959failed_iomap:
960 pci_release_regions(dev);
961failed_request_regions:
962failed_dma_mask:
963 pci_disable_device(dev);
964failed_enable:
37ae133c 965 ida_free(&rsxx_disk_ida, card->disk_id);
8722ff8c 966failed_ida_get:
967 kfree(card);
968
969 return st;
970}
971
82bed4d5 972static void rsxx_pci_remove(struct pci_dev *dev)
8722ff8c 973{
974 struct rsxx_cardinfo *card = pci_get_drvdata(dev);
975 unsigned long flags;
976 int st;
977 int i;
978
979 if (!card)
980 return;
981
982 dev_info(CARD_TO_DEV(card),
983 "Removing PCI-Flash SSD.\n");
984
985 rsxx_detach_dev(card);
986
987 for (i = 0; i < card->n_targets; i++) {
988 spin_lock_irqsave(&card->irq_lock, flags);
989 rsxx_disable_ier_and_isr(card, CR_INTR_DMA(i));
990 spin_unlock_irqrestore(&card->irq_lock, flags);
991 }
992
993 st = card_shutdown(card);
994 if (st)
995 dev_crit(CARD_TO_DEV(card), "Shutdown failed!\n");
996
997 /* Sync outstanding event handlers. */
998 spin_lock_irqsave(&card->irq_lock, flags);
999 rsxx_disable_ier_and_isr(card, CR_INTR_EVENT);
1000 spin_unlock_irqrestore(&card->irq_lock, flags);
1001
8722ff8c 1002 cancel_work_sync(&card->event_work);
1003
1004 rsxx_destroy_dev(card);
1005 rsxx_dma_destroy(card);
1006
1007 spin_lock_irqsave(&card->irq_lock, flags);
1008 rsxx_disable_ier_and_isr(card, CR_INTR_ALL);
1009 spin_unlock_irqrestore(&card->irq_lock, flags);
1ebfd109
PK		 1010
1011 /* Prevent work_structs from re-queuing themselves. */
1012 card->halt = 1;
1013
36f988e9
PK		 1014 debugfs_remove_recursive(card->debugfs_dir);
1015
8722ff8c 1016 free_irq(dev->irq, card);
1017
1018 if (!force_legacy)
1019 pci_disable_msi(dev);
1020
1021 rsxx_creg_destroy(card);
1022
1023 pci_iounmap(dev, card->regmap);
1024
1025 pci_disable_device(dev);
1026 pci_release_regions(dev);
1027
37ae133c 1028 ida_free(&rsxx_disk_ida, card->disk_id);
8722ff8c 1029 kfree(card);
1030}
1031
1032static int rsxx_pci_suspend(struct pci_dev *dev, pm_message_t state)
1033{
1034 /* We don't support suspend at this time. */
1035 return -ENOSYS;
1036}
1037
1038static void rsxx_pci_shutdown(struct pci_dev *dev)
1039{
1040 struct rsxx_cardinfo *card = pci_get_drvdata(dev);
1041 unsigned long flags;
1042 int i;
1043
1044 if (!card)
1045 return;
1046
1047 dev_info(CARD_TO_DEV(card), "Shutting down PCI-Flash SSD.\n");
1048
1049 rsxx_detach_dev(card);
1050
1051 for (i = 0; i < card->n_targets; i++) {
1052 spin_lock_irqsave(&card->irq_lock, flags);
1053 rsxx_disable_ier_and_isr(card, CR_INTR_DMA(i));
1054 spin_unlock_irqrestore(&card->irq_lock, flags);
1055 }
1056
1057 card_shutdown(card);
1058}
1059
c95246c3
PK		 1060static const struct pci_error_handlers rsxx_err_handler = {
1061 .error_detected = rsxx_error_detected,
1062 .slot_reset = rsxx_slot_reset,
1063};
1064
9baa3c34 1065static const struct pci_device_id rsxx_pci_ids[] = {
9bb3c446
PK		 1066 {PCI_DEVICE(PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_FS70_FLASH)},
1067 {PCI_DEVICE(PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_FS80_FLASH)},
8722ff8c 1068 {0,},
1069};
1070
1071MODULE_DEVICE_TABLE(pci, rsxx_pci_ids);
1072
1073static struct pci_driver rsxx_pci_driver = {
1074 .name = DRIVER_NAME,
1075 .id_table = rsxx_pci_ids,
1076 .probe = rsxx_pci_probe,
82bed4d5 1077 .remove = rsxx_pci_remove,
8722ff8c 1078 .suspend = rsxx_pci_suspend,
1079 .shutdown = rsxx_pci_shutdown,
c95246c3 1080 .err_handler = &rsxx_err_handler,
8722ff8c 1081};
1082
1083static int __init rsxx_core_init(void)
1084{
1085 int st;
1086
1087 st = rsxx_dev_init();
1088 if (st)
1089 return st;
1090
1091 st = rsxx_dma_init();
1092 if (st)
1093 goto dma_init_failed;
1094
1095 st = rsxx_creg_init();
1096 if (st)
1097 goto creg_init_failed;
1098
1099 return pci_register_driver(&rsxx_pci_driver);
1100
1101creg_init_failed:
1102 rsxx_dma_cleanup();
1103dma_init_failed:
1104 rsxx_dev_cleanup();
1105
1106 return st;
1107}
1108
1109static void __exit rsxx_core_cleanup(void)
1110{
1111 pci_unregister_driver(&rsxx_pci_driver);
1112 rsxx_creg_cleanup();
1113 rsxx_dma_cleanup();
1114 rsxx_dev_cleanup();
1115}
1116
1117module_init(rsxx_core_init);
1118module_exit(rsxx_core_cleanup);
Linux 6.x block layer and io_uring tree(s)