projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
block: Mark expected switch fall-throughs
[linux-2.6-block.git] / drivers / block / rsxx / core.c
CommitLineData
8722ff8c 1/*
2* Filename: core.c
3*
4*
5* Authors: Joshua Morris <josh.h.morris@us.ibm.com>
6* Philip Kelleher <pjk1939@linux.vnet.ibm.com>
7*
8* (C) Copyright 2013 IBM Corporation
9*
10* This program is free software; you can redistribute it and/or
11* modify it under the terms of the GNU General Public License as
12* published by the Free Software Foundation; either version 2 of the
13* License, or (at your option) any later version.
14*
15* This program is distributed in the hope that it will be useful, but
16* WITHOUT ANY WARRANTY; without even the implied warranty of
17* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18* General Public License for more details.
19*
20* You should have received a copy of the GNU General Public License
21* along with this program; if not, write to the Free Software Foundation,
22* Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
23*/
24
25#include <linux/kernel.h>
26#include <linux/init.h>
27#include <linux/interrupt.h>
28#include <linux/module.h>
29#include <linux/pci.h>
30#include <linux/reboot.h>
31#include <linux/slab.h>
32#include <linux/bitops.h>
c95246c3 33#include <linux/delay.h>
36f988e9
PK		 34#include <linux/debugfs.h>
35#include <linux/seq_file.h>
8722ff8c 36
37#include <linux/genhd.h>
38#include <linux/idr.h>
39
40#include "rsxx_priv.h"
41#include "rsxx_cfg.h"
42
43#define NO_LEGACY 0
fb065cd9 44#define SYNC_START_TIMEOUT (10 * 60) /* 10 minutes */
8722ff8c 45
f730e3dc 46MODULE_DESCRIPTION("IBM Flash Adapter 900GB Full Height Device Driver");
9bb3c446 47MODULE_AUTHOR("Joshua Morris/Philip Kelleher, IBM");
8722ff8c 48MODULE_LICENSE("GPL");
49MODULE_VERSION(DRIVER_VERSION);
50
51static unsigned int force_legacy = NO_LEGACY;
52module_param(force_legacy, uint, 0444);
53MODULE_PARM_DESC(force_legacy, "Force the use of legacy type PCI interrupts");
54
fb065cd9
PK		 55static unsigned int sync_start = 1;
56module_param(sync_start, uint, 0444);
57MODULE_PARM_DESC(sync_start, "On by Default: Driver load will not complete "
58 "until the card startup has completed.");
59
8722ff8c 60static DEFINE_IDA(rsxx_disk_ida);
8722ff8c 61
36f988e9
PK		 62/* --------------------Debugfs Setup ------------------- */
63
36f988e9
PK		 64static int rsxx_attr_pci_regs_show(struct seq_file *m, void *p)
65{
66 struct rsxx_cardinfo *card = m->private;
67
68 seq_printf(m, "HWID 0x%08x\n",
69 ioread32(card->regmap + HWID));
70 seq_printf(m, "SCRATCH 0x%08x\n",
71 ioread32(card->regmap + SCRATCH));
72 seq_printf(m, "IER 0x%08x\n",
73 ioread32(card->regmap + IER));
74 seq_printf(m, "IPR 0x%08x\n",
75 ioread32(card->regmap + IPR));
76 seq_printf(m, "CREG_CMD 0x%08x\n",
77 ioread32(card->regmap + CREG_CMD));
78 seq_printf(m, "CREG_ADD 0x%08x\n",
79 ioread32(card->regmap + CREG_ADD));
80 seq_printf(m, "CREG_CNT 0x%08x\n",
81 ioread32(card->regmap + CREG_CNT));
82 seq_printf(m, "CREG_STAT 0x%08x\n",
83 ioread32(card->regmap + CREG_STAT));
84 seq_printf(m, "CREG_DATA0 0x%08x\n",
85 ioread32(card->regmap + CREG_DATA0));
86 seq_printf(m, "CREG_DATA1 0x%08x\n",
87 ioread32(card->regmap + CREG_DATA1));
88 seq_printf(m, "CREG_DATA2 0x%08x\n",
89 ioread32(card->regmap + CREG_DATA2));
90 seq_printf(m, "CREG_DATA3 0x%08x\n",
91 ioread32(card->regmap + CREG_DATA3));
92 seq_printf(m, "CREG_DATA4 0x%08x\n",
93 ioread32(card->regmap + CREG_DATA4));
94 seq_printf(m, "CREG_DATA5 0x%08x\n",
95 ioread32(card->regmap + CREG_DATA5));
96 seq_printf(m, "CREG_DATA6 0x%08x\n",
97 ioread32(card->regmap + CREG_DATA6));
98 seq_printf(m, "CREG_DATA7 0x%08x\n",
99 ioread32(card->regmap + CREG_DATA7));
100 seq_printf(m, "INTR_COAL 0x%08x\n",
101 ioread32(card->regmap + INTR_COAL));
102 seq_printf(m, "HW_ERROR 0x%08x\n",
103 ioread32(card->regmap + HW_ERROR));
104 seq_printf(m, "DEBUG0 0x%08x\n",
105 ioread32(card->regmap + PCI_DEBUG0));
106 seq_printf(m, "DEBUG1 0x%08x\n",
107 ioread32(card->regmap + PCI_DEBUG1));
108 seq_printf(m, "DEBUG2 0x%08x\n",
109 ioread32(card->regmap + PCI_DEBUG2));
110 seq_printf(m, "DEBUG3 0x%08x\n",
111 ioread32(card->regmap + PCI_DEBUG3));
112 seq_printf(m, "DEBUG4 0x%08x\n",
113 ioread32(card->regmap + PCI_DEBUG4));
114 seq_printf(m, "DEBUG5 0x%08x\n",
115 ioread32(card->regmap + PCI_DEBUG5));
116 seq_printf(m, "DEBUG6 0x%08x\n",
117 ioread32(card->regmap + PCI_DEBUG6));
118 seq_printf(m, "DEBUG7 0x%08x\n",
119 ioread32(card->regmap + PCI_DEBUG7));
120 seq_printf(m, "RECONFIG 0x%08x\n",
121 ioread32(card->regmap + PCI_RECONFIG));
122
123 return 0;
124}
125
126static int rsxx_attr_stats_show(struct seq_file *m, void *p)
127{
128 struct rsxx_cardinfo *card = m->private;
129 int i;
130
131 for (i = 0; i < card->n_targets; i++) {
132 seq_printf(m, "Ctrl %d CRC Errors = %d\n",
133 i, card->ctrl[i].stats.crc_errors);
134 seq_printf(m, "Ctrl %d Hard Errors = %d\n",
135 i, card->ctrl[i].stats.hard_errors);
136 seq_printf(m, "Ctrl %d Soft Errors = %d\n",
137 i, card->ctrl[i].stats.soft_errors);
138 seq_printf(m, "Ctrl %d Writes Issued = %d\n",
139 i, card->ctrl[i].stats.writes_issued);
140 seq_printf(m, "Ctrl %d Writes Failed = %d\n",
141 i, card->ctrl[i].stats.writes_failed);
142 seq_printf(m, "Ctrl %d Reads Issued = %d\n",
143 i, card->ctrl[i].stats.reads_issued);
144 seq_printf(m, "Ctrl %d Reads Failed = %d\n",
145 i, card->ctrl[i].stats.reads_failed);
146 seq_printf(m, "Ctrl %d Reads Retried = %d\n",
147 i, card->ctrl[i].stats.reads_retried);
148 seq_printf(m, "Ctrl %d Discards Issued = %d\n",
149 i, card->ctrl[i].stats.discards_issued);
150 seq_printf(m, "Ctrl %d Discards Failed = %d\n",
151 i, card->ctrl[i].stats.discards_failed);
152 seq_printf(m, "Ctrl %d DMA SW Errors = %d\n",
153 i, card->ctrl[i].stats.dma_sw_err);
154 seq_printf(m, "Ctrl %d DMA HW Faults = %d\n",
155 i, card->ctrl[i].stats.dma_hw_fault);
156 seq_printf(m, "Ctrl %d DMAs Cancelled = %d\n",
157 i, card->ctrl[i].stats.dma_cancelled);
158 seq_printf(m, "Ctrl %d SW Queue Depth = %d\n",
159 i, card->ctrl[i].stats.sw_q_depth);
160 seq_printf(m, "Ctrl %d HW Queue Depth = %d\n",
161 i, atomic_read(&card->ctrl[i].stats.hw_q_depth));
162 }
163
164 return 0;
165}
166
167static int rsxx_attr_stats_open(struct inode *inode, struct file *file)
168{
169 return single_open(file, rsxx_attr_stats_show, inode->i_private);
170}
171
172static int rsxx_attr_pci_regs_open(struct inode *inode, struct file *file)
173{
174 return single_open(file, rsxx_attr_pci_regs_show, inode->i_private);
175}
176
177static ssize_t rsxx_cram_read(struct file *fp, char __user *ubuf,
178 size_t cnt, loff_t *ppos)
179{
8e3fb059 180 struct rsxx_cardinfo *card = file_inode(fp)->i_private;
36f988e9 181 char *buf;
8e3fb059 182 ssize_t st;
36f988e9 183
8e3fb059 184 buf = kzalloc(cnt, GFP_KERNEL);
36f988e9
PK		 185 if (!buf)
186 return -ENOMEM;
187
8e3fb059
AV		 188 st = rsxx_creg_read(card, CREG_ADD_CRAM + (u32)*ppos, cnt, buf, 1);
189 if (!st)
190 st = copy_to_user(ubuf, buf, cnt);
191 kfree(buf);
36f988e9
PK		 192 if (st)
193 return st;
8e3fb059 194 *ppos += cnt;
36f988e9
PK		 195 return cnt;
196}
197
198static ssize_t rsxx_cram_write(struct file *fp, const char __user *ubuf,
199 size_t cnt, loff_t *ppos)
200{
8e3fb059 201 struct rsxx_cardinfo *card = file_inode(fp)->i_private;
36f988e9 202 char *buf;
8e3fb059 203 ssize_t st;
36f988e9 204
820351f0
AV		 205 buf = memdup_user(ubuf, cnt);
206 if (IS_ERR(buf))
207 return PTR_ERR(buf);
36f988e9 208
820351f0 209 st = rsxx_creg_write(card, CREG_ADD_CRAM + (u32)*ppos, cnt, buf, 1);
8e3fb059 210 kfree(buf);
36f988e9
PK		 211 if (st)
212 return st;
8e3fb059 213 *ppos += cnt;
36f988e9
PK		 214 return cnt;
215}
216
36f988e9
PK		 217static const struct file_operations debugfs_cram_fops = {
218 .owner = THIS_MODULE,
36f988e9
PK		 219 .read = rsxx_cram_read,
220 .write = rsxx_cram_write,
36f988e9
PK		 221};
222
223static const struct file_operations debugfs_stats_fops = {
224 .owner = THIS_MODULE,
225 .open = rsxx_attr_stats_open,
226 .read = seq_read,
227 .llseek = seq_lseek,
228 .release = single_release,
229};
230
231static const struct file_operations debugfs_pci_regs_fops = {
232 .owner = THIS_MODULE,
233 .open = rsxx_attr_pci_regs_open,
234 .read = seq_read,
235 .llseek = seq_lseek,
236 .release = single_release,
237};
238
239static void rsxx_debugfs_dev_new(struct rsxx_cardinfo *card)
240{
241 struct dentry *debugfs_stats;
242 struct dentry *debugfs_pci_regs;
243 struct dentry *debugfs_cram;
244
245 card->debugfs_dir = debugfs_create_dir(card->gendisk->disk_name, NULL);
246 if (IS_ERR_OR_NULL(card->debugfs_dir))
247 goto failed_debugfs_dir;
248
5657a819 249 debugfs_stats = debugfs_create_file("stats", 0444,
36f988e9
PK		 250 card->debugfs_dir, card,
251 &debugfs_stats_fops);
252 if (IS_ERR_OR_NULL(debugfs_stats))
253 goto failed_debugfs_stats;
254
5657a819 255 debugfs_pci_regs = debugfs_create_file("pci_regs", 0444,
36f988e9
PK		 256 card->debugfs_dir, card,
257 &debugfs_pci_regs_fops);
258 if (IS_ERR_OR_NULL(debugfs_pci_regs))
259 goto failed_debugfs_pci_regs;
260
5657a819 261 debugfs_cram = debugfs_create_file("cram", 0644,
36f988e9
PK		 262 card->debugfs_dir, card,
263 &debugfs_cram_fops);
264 if (IS_ERR_OR_NULL(debugfs_cram))
265 goto failed_debugfs_cram;
266
267 return;
268failed_debugfs_cram:
269 debugfs_remove(debugfs_pci_regs);
270failed_debugfs_pci_regs:
271 debugfs_remove(debugfs_stats);
272failed_debugfs_stats:
273 debugfs_remove(card->debugfs_dir);
274failed_debugfs_dir:
275 card->debugfs_dir = NULL;
276}
277
8722ff8c 278/*----------------- Interrupt Control & Handling -------------------*/
c95246c3
PK		 279
280static void rsxx_mask_interrupts(struct rsxx_cardinfo *card)
281{
282 card->isr_mask = 0;
283 card->ier_mask = 0;
284}
285
8722ff8c 286static void __enable_intr(unsigned int *mask, unsigned int intr)
287{
288 *mask |= intr;
289}
290
291static void __disable_intr(unsigned int *mask, unsigned int intr)
292{
293 *mask &= ~intr;
294}
295
296/*
297 * NOTE: Disabling the IER will disable the hardware interrupt.
298 * Disabling the ISR will disable the software handling of the ISR bit.
299 *
300 * Enable/Disable interrupt functions assume the card->irq_lock
301 * is held by the caller.
302 */
303void rsxx_enable_ier(struct rsxx_cardinfo *card, unsigned int intr)
304{
c95246c3
PK		 305 if (unlikely(card->halt) ||
306 unlikely(card->eeh_state))
8722ff8c 307 return;
308
309 __enable_intr(&card->ier_mask, intr);
310 iowrite32(card->ier_mask, card->regmap + IER);
311}
312
313void rsxx_disable_ier(struct rsxx_cardinfo *card, unsigned int intr)
314{
c95246c3
PK		 315 if (unlikely(card->eeh_state))
316 return;
317
8722ff8c 318 __disable_intr(&card->ier_mask, intr);
319 iowrite32(card->ier_mask, card->regmap + IER);
320}
321
322void rsxx_enable_ier_and_isr(struct rsxx_cardinfo *card,
323 unsigned int intr)
324{
c95246c3
PK		 325 if (unlikely(card->halt) ||
326 unlikely(card->eeh_state))
8722ff8c 327 return;
328
329 __enable_intr(&card->isr_mask, intr);
330 __enable_intr(&card->ier_mask, intr);
331 iowrite32(card->ier_mask, card->regmap + IER);
332}
333void rsxx_disable_ier_and_isr(struct rsxx_cardinfo *card,
334 unsigned int intr)
335{
c95246c3
PK		 336 if (unlikely(card->eeh_state))
337 return;
338
8722ff8c 339 __disable_intr(&card->isr_mask, intr);
340 __disable_intr(&card->ier_mask, intr);
341 iowrite32(card->ier_mask, card->regmap + IER);
342}
343
c206c709 344static irqreturn_t rsxx_isr(int irq, void *pdata)
8722ff8c 345{
c206c709 346 struct rsxx_cardinfo *card = pdata;
8722ff8c 347 unsigned int isr;
348 int handled = 0;
349 int reread_isr;
350 int i;
351
352 spin_lock(&card->irq_lock);
353
354 do {
355 reread_isr = 0;
356
c95246c3
PK		 357 if (unlikely(card->eeh_state))
358 break;
359
8722ff8c 360 isr = ioread32(card->regmap + ISR);
361 if (isr == 0xffffffff) {
362 /*
363 * A few systems seem to have an intermittent issue
364 * where PCI reads return all Fs, but retrying the read
365 * a little later will return as expected.
366 */
367 dev_info(CARD_TO_DEV(card),
368 "ISR = 0xFFFFFFFF, retrying later\n");
369 break;
370 }
371
372 isr &= card->isr_mask;
373 if (!isr)
374 break;
375
376 for (i = 0; i < card->n_targets; i++) {
377 if (isr & CR_INTR_DMA(i)) {
378 if (card->ier_mask & CR_INTR_DMA(i)) {
379 rsxx_disable_ier(card, CR_INTR_DMA(i));
380 reread_isr = 1;
381 }
382 queue_work(card->ctrl[i].done_wq,
383 &card->ctrl[i].dma_done_work);
384 handled++;
385 }
386 }
387
388 if (isr & CR_INTR_CREG) {
a3299ab1
PK		 389 queue_work(card->creg_ctrl.creg_wq,
390 &card->creg_ctrl.done_work);
8722ff8c 391 handled++;
392 }
393
394 if (isr & CR_INTR_EVENT) {
a3299ab1 395 queue_work(card->event_wq, &card->event_work);
8722ff8c 396 rsxx_disable_ier_and_isr(card, CR_INTR_EVENT);
397 handled++;
398 }
399 } while (reread_isr);
400
401 spin_unlock(&card->irq_lock);
402
403 return handled ? IRQ_HANDLED : IRQ_NONE;
404}
405
406/*----------------- Card Event Handler -------------------*/
f3791203 407static const char * const rsxx_card_state_to_str(unsigned int state)
c206c709 408{
f3791203 409 static const char * const state_strings[] = {
c206c709
PK		 410 "Unknown", "Shutdown", "Starting", "Formatting",
411 "Uninitialized", "Good", "Shutting Down",
412 "Fault", "Read Only Fault", "dStroying"
413 };
414
415 return state_strings[ffs(state)];
416}
417
8722ff8c 418static void card_state_change(struct rsxx_cardinfo *card,
419 unsigned int new_state)
420{
421 int st;
422
423 dev_info(CARD_TO_DEV(card),
424 "card state change detected.(%s -> %s)\n",
425 rsxx_card_state_to_str(card->state),
426 rsxx_card_state_to_str(new_state));
427
428 card->state = new_state;
429
430 /* Don't attach DMA interfaces if the card has an invalid config */
431 if (!card->config_valid)
432 return;
433
434 switch (new_state) {
435 case CARD_STATE_RD_ONLY_FAULT:
436 dev_crit(CARD_TO_DEV(card),
437 "Hardware has entered read-only mode!\n");
438 /*
439 * Fall through so the DMA devices can be attached and
440 * the user can attempt to pull off their data.
441 */
e16fb3a8 442 /* fall through */
8722ff8c 443 case CARD_STATE_GOOD:
444 st = rsxx_get_card_size8(card, &card->size8);
445 if (st)
446 dev_err(CARD_TO_DEV(card),
447 "Failed attaching DMA devices\n");
448
449 if (card->config_valid)
450 set_capacity(card->gendisk, card->size8 >> 9);
451 break;
452
453 case CARD_STATE_FAULT:
454 dev_crit(CARD_TO_DEV(card),
455 "Hardware Fault reported!\n");
456 /* Fall through. */
457
458 /* Everything else, detach DMA interface if it's attached. */
459 case CARD_STATE_SHUTDOWN:
460 case CARD_STATE_STARTING:
461 case CARD_STATE_FORMATTING:
462 case CARD_STATE_UNINITIALIZED:
463 case CARD_STATE_SHUTTING_DOWN:
464 /*
465 * dStroy is a term coined by marketing to represent the low level
466 * secure erase.
467 */
468 case CARD_STATE_DSTROYING:
469 set_capacity(card->gendisk, 0);
470 break;
471 }
472}
473
474static void card_event_handler(struct work_struct *work)
475{
476 struct rsxx_cardinfo *card;
477 unsigned int state;
478 unsigned long flags;
479 int st;
480
481 card = container_of(work, struct rsxx_cardinfo, event_work);
482
483 if (unlikely(card->halt))
484 return;
485
486 /*
487 * Enable the interrupt now to avoid any weird race conditions where a
488 * state change might occur while rsxx_get_card_state() is
489 * processing a returned creg cmd.
490 */
491 spin_lock_irqsave(&card->irq_lock, flags);
492 rsxx_enable_ier_and_isr(card, CR_INTR_EVENT);
493 spin_unlock_irqrestore(&card->irq_lock, flags);
494
495 st = rsxx_get_card_state(card, &state);
496 if (st) {
497 dev_info(CARD_TO_DEV(card),
498 "Failed reading state after event.\n");
499 return;
500 }
501
502 if (card->state != state)
503 card_state_change(card, state);
504
505 if (card->creg_ctrl.creg_stats.stat & CREG_STAT_LOG_PENDING)
506 rsxx_read_hw_log(card);
507}
508
8722ff8c 509/*----------------- Card Operations -------------------*/
510static int card_shutdown(struct rsxx_cardinfo *card)
511{
512 unsigned int state;
513 signed long start;
514 const int timeout = msecs_to_jiffies(120000);
515 int st;
516
517 /* We can't issue a shutdown if the card is in a transition state */
518 start = jiffies;
519 do {
520 st = rsxx_get_card_state(card, &state);
521 if (st)
522 return st;
523 } while (state == CARD_STATE_STARTING &&
524 (jiffies - start < timeout));
525
526 if (state == CARD_STATE_STARTING)
527 return -ETIMEDOUT;
528
529 /* Only issue a shutdown if we need to */
530 if ((state != CARD_STATE_SHUTTING_DOWN) &&
531 (state != CARD_STATE_SHUTDOWN)) {
532 st = rsxx_issue_card_cmd(card, CARD_CMD_SHUTDOWN);
533 if (st)
534 return st;
535 }
536
537 start = jiffies;
538 do {
539 st = rsxx_get_card_state(card, &state);
540 if (st)
541 return st;
542 } while (state != CARD_STATE_SHUTDOWN &&
543 (jiffies - start < timeout));
544
545 if (state != CARD_STATE_SHUTDOWN)
546 return -ETIMEDOUT;
547
548 return 0;
549}
550
4dcaf472 551static int rsxx_eeh_frozen(struct pci_dev *dev)
c95246c3
PK		 552{
553 struct rsxx_cardinfo *card = pci_get_drvdata(dev);
554 int i;
4dcaf472 555 int st;
c95246c3 556
f730e3dc 557 dev_warn(&dev->dev, "IBM Flash Adapter PCI: preparing for slot reset.\n");
c95246c3
PK		 558
559 card->eeh_state = 1;
560 rsxx_mask_interrupts(card);
561
562 /*
563 * We need to guarantee that the write for eeh_state and masking
564 * interrupts does not become reordered. This will prevent a possible
565 * race condition with the EEH code.
566 */
567 wmb();
568
569 pci_disable_device(dev);
570
4dcaf472
PK		 571 st = rsxx_eeh_save_issued_dmas(card);
572 if (st)
573 return st;
c95246c3
PK		 574
575 rsxx_eeh_save_issued_creg(card);
576
577 for (i = 0; i < card->n_targets; i++) {
578 if (card->ctrl[i].status.buf)
579 pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8,
580 card->ctrl[i].status.buf,
581 card->ctrl[i].status.dma_addr);
582 if (card->ctrl[i].cmd.buf)
583 pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8,
584 card->ctrl[i].cmd.buf,
585 card->ctrl[i].cmd.dma_addr);
586 }
4dcaf472
PK		 587
588 return 0;
c95246c3
PK		 589}
590
591static void rsxx_eeh_failure(struct pci_dev *dev)
592{
593 struct rsxx_cardinfo *card = pci_get_drvdata(dev);
594 int i;
0ab4743e 595 int cnt = 0;
c95246c3 596
f730e3dc 597 dev_err(&dev->dev, "IBM Flash Adapter PCI: disabling failed card.\n");
c95246c3
PK		 598
599 card->eeh_state = 1;
0ab4743e 600 card->halt = 1;
c95246c3 601
0ab4743e
PK		 602 for (i = 0; i < card->n_targets; i++) {
603 spin_lock_bh(&card->ctrl[i].queue_lock);
604 cnt = rsxx_cleanup_dma_queue(&card->ctrl[i],
e5feab22
PK		 605 &card->ctrl[i].queue,
606 COMPLETE_DMA);
0ab4743e
PK		 607 spin_unlock_bh(&card->ctrl[i].queue_lock);
608
609 cnt += rsxx_dma_cancel(&card->ctrl[i]);
c95246c3 610
0ab4743e
PK		 611 if (cnt)
612 dev_info(CARD_TO_DEV(card),
613 "Freed %d queued DMAs on channel %d\n",
614 cnt, card->ctrl[i].id);
615 }
c95246c3
PK		 616}
617
618static int rsxx_eeh_fifo_flush_poll(struct rsxx_cardinfo *card)
619{
620 unsigned int status;
621 int iter = 0;
622
623 /* We need to wait for the hardware to reset */
624 while (iter++ < 10) {
625 status = ioread32(card->regmap + PCI_RECONFIG);
626
627 if (status & RSXX_FLUSH_BUSY) {
628 ssleep(1);
629 continue;
630 }
631
632 if (status & RSXX_FLUSH_TIMEOUT)
633 dev_warn(CARD_TO_DEV(card), "HW: flash controller timeout\n");
634 return 0;
635 }
636
637 /* Hardware failed resetting itself. */
638 return -1;
639}
640
641static pci_ers_result_t rsxx_error_detected(struct pci_dev *dev,
642 enum pci_channel_state error)
643{
4dcaf472
PK		 644 int st;
645
c95246c3
PK		 646 if (dev->revision < RSXX_EEH_SUPPORT)
647 return PCI_ERS_RESULT_NONE;
648
649 if (error == pci_channel_io_perm_failure) {
650 rsxx_eeh_failure(dev);
651 return PCI_ERS_RESULT_DISCONNECT;
652 }
653
4dcaf472
PK		 654 st = rsxx_eeh_frozen(dev);
655 if (st) {
656 dev_err(&dev->dev, "Slot reset setup failed\n");
657 rsxx_eeh_failure(dev);
658 return PCI_ERS_RESULT_DISCONNECT;
659 }
660
c95246c3
PK		 661 return PCI_ERS_RESULT_NEED_RESET;
662}
663
664static pci_ers_result_t rsxx_slot_reset(struct pci_dev *dev)
665{
666 struct rsxx_cardinfo *card = pci_get_drvdata(dev);
667 unsigned long flags;
668 int i;
669 int st;
670
671 dev_warn(&dev->dev,
f730e3dc 672 "IBM Flash Adapter PCI: recovering from slot reset.\n");
c95246c3
PK		 673
674 st = pci_enable_device(dev);
675 if (st)
676 goto failed_hw_setup;
677
678 pci_set_master(dev);
679
680 st = rsxx_eeh_fifo_flush_poll(card);
681 if (st)
682 goto failed_hw_setup;
683
684 rsxx_dma_queue_reset(card);
685
686 for (i = 0; i < card->n_targets; i++) {
687 st = rsxx_hw_buffers_init(dev, &card->ctrl[i]);
688 if (st)
689 goto failed_hw_buffers_init;
690 }
691
692 if (card->config_valid)
693 rsxx_dma_configure(card);
694
695 /* Clears the ISR register from spurious interrupts */
696 st = ioread32(card->regmap + ISR);
697
698 card->eeh_state = 0;
699
c95246c3
PK		 700 spin_lock_irqsave(&card->irq_lock, flags);
701 if (card->n_targets & RSXX_MAX_TARGETS)
702 rsxx_enable_ier_and_isr(card, CR_INTR_ALL_G);
703 else
704 rsxx_enable_ier_and_isr(card, CR_INTR_ALL_C);
705 spin_unlock_irqrestore(&card->irq_lock, flags);
706
707 rsxx_kick_creg_queue(card);
708
709 for (i = 0; i < card->n_targets; i++) {
710 spin_lock(&card->ctrl[i].queue_lock);
711 if (list_empty(&card->ctrl[i].queue)) {
712 spin_unlock(&card->ctrl[i].queue_lock);
713 continue;
714 }
715 spin_unlock(&card->ctrl[i].queue_lock);
716
717 queue_work(card->ctrl[i].issue_wq,
718 &card->ctrl[i].issue_dma_work);
719 }
720
f730e3dc 721 dev_info(&dev->dev, "IBM Flash Adapter PCI: recovery complete.\n");
c95246c3
PK		 722
723 return PCI_ERS_RESULT_RECOVERED;
724
725failed_hw_buffers_init:
c95246c3
PK		 726 for (i = 0; i < card->n_targets; i++) {
727 if (card->ctrl[i].status.buf)
728 pci_free_consistent(card->dev,
729 STATUS_BUFFER_SIZE8,
730 card->ctrl[i].status.buf,
731 card->ctrl[i].status.dma_addr);
732 if (card->ctrl[i].cmd.buf)
733 pci_free_consistent(card->dev,
734 COMMAND_BUFFER_SIZE8,
735 card->ctrl[i].cmd.buf,
736 card->ctrl[i].cmd.dma_addr);
737 }
738failed_hw_setup:
739 rsxx_eeh_failure(dev);
740 return PCI_ERS_RESULT_DISCONNECT;
741
742}
743
8722ff8c 744/*----------------- Driver Initialization & Setup -------------------*/
745/* Returns: 0 if the driver is compatible with the device
746 -1 if the driver is NOT compatible with the device */
747static int rsxx_compatibility_check(struct rsxx_cardinfo *card)
748{
749 unsigned char pci_rev;
750
751 pci_read_config_byte(card->dev, PCI_REVISION_ID, &pci_rev);
752
753 if (pci_rev > RS70_PCI_REV_SUPPORTED)
754 return -1;
755 return 0;
756}
757
82bed4d5 758static int rsxx_pci_probe(struct pci_dev *dev,
8722ff8c 759 const struct pci_device_id *id)
760{
761 struct rsxx_cardinfo *card;
8722ff8c 762 int st;
fb065cd9 763 unsigned int sync_timeout;
8722ff8c 764
765 dev_info(&dev->dev, "PCI-Flash SSD discovered\n");
766
767 card = kzalloc(sizeof(*card), GFP_KERNEL);
768 if (!card)
769 return -ENOMEM;
770
771 card->dev = dev;
772 pci_set_drvdata(dev, card);
773
37ae133c
MW		 774 st = ida_alloc(&rsxx_disk_ida, GFP_KERNEL);
775 if (st < 0)
8722ff8c 776 goto failed_ida_get;
37ae133c 777 card->disk_id = st;
8722ff8c 778
779 st = pci_enable_device(dev);
780 if (st)
781 goto failed_enable;
782
783 pci_set_master(dev);
b0da3498 784 dma_set_max_seg_size(&dev->dev, RSXX_HW_BLK_SIZE);
8722ff8c 785
77a12e51 786 st = dma_set_mask(&dev->dev, DMA_BIT_MASK(64));
8722ff8c 787 if (st) {
788 dev_err(CARD_TO_DEV(card),
789 "No usable DMA configuration,aborting\n");
790 goto failed_dma_mask;
791 }
792
793 st = pci_request_regions(dev, DRIVER_NAME);
794 if (st) {
795 dev_err(CARD_TO_DEV(card),
796 "Failed to request memory region\n");
797 goto failed_request_regions;
798 }
799
800 if (pci_resource_len(dev, 0) == 0) {
801 dev_err(CARD_TO_DEV(card), "BAR0 has length 0!\n");
802 st = -ENOMEM;
803 goto failed_iomap;
804 }
805
806 card->regmap = pci_iomap(dev, 0, 0);
807 if (!card->regmap) {
808 dev_err(CARD_TO_DEV(card), "Failed to map BAR0\n");
809 st = -ENOMEM;
810 goto failed_iomap;
811 }
812
813 spin_lock_init(&card->irq_lock);
814 card->halt = 0;
c95246c3 815 card->eeh_state = 0;
8722ff8c 816
c206c709 817 spin_lock_irq(&card->irq_lock);
8722ff8c 818 rsxx_disable_ier_and_isr(card, CR_INTR_ALL);
c206c709 819 spin_unlock_irq(&card->irq_lock);
8722ff8c 820
821 if (!force_legacy) {
822 st = pci_enable_msi(dev);
823 if (st)
824 dev_warn(CARD_TO_DEV(card),
825 "Failed to enable MSI\n");
826 }
827
baf37812 828 st = request_irq(dev->irq, rsxx_isr, IRQF_SHARED,
8722ff8c 829 DRIVER_NAME, card);
830 if (st) {
831 dev_err(CARD_TO_DEV(card),
832 "Failed requesting IRQ%d\n", dev->irq);
833 goto failed_irq;
834 }
835
836 /************* Setup Processor Command Interface *************/
a3299ab1
PK		 837 st = rsxx_creg_setup(card);
838 if (st) {
839 dev_err(CARD_TO_DEV(card), "Failed to setup creg interface.\n");
840 goto failed_creg_setup;
841 }
8722ff8c 842
c206c709 843 spin_lock_irq(&card->irq_lock);
8722ff8c 844 rsxx_enable_ier_and_isr(card, CR_INTR_CREG);
c206c709 845 spin_unlock_irq(&card->irq_lock);
8722ff8c 846
847 st = rsxx_compatibility_check(card);
848 if (st) {
849 dev_warn(CARD_TO_DEV(card),
850 "Incompatible driver detected. Please update the driver.\n");
851 st = -EINVAL;
852 goto failed_compatiblity_check;
853 }
854
855 /************* Load Card Config *************/
856 st = rsxx_load_config(card);
857 if (st)
858 dev_err(CARD_TO_DEV(card),
859 "Failed loading card config\n");
860
861 /************* Setup DMA Engine *************/
862 st = rsxx_get_num_targets(card, &card->n_targets);
863 if (st)
864 dev_info(CARD_TO_DEV(card),
865 "Failed reading the number of DMA targets\n");
866
6396bb22
KC		 867 card->ctrl = kcalloc(card->n_targets, sizeof(*card->ctrl),
868 GFP_KERNEL);
8722ff8c 869 if (!card->ctrl) {
870 st = -ENOMEM;
871 goto failed_dma_setup;
872 }
873
874 st = rsxx_dma_setup(card);
875 if (st) {
876 dev_info(CARD_TO_DEV(card),
877 "Failed to setup DMA engine\n");
878 goto failed_dma_setup;
879 }
880
881 /************* Setup Card Event Handler *************/
a3299ab1
PK		 882 card->event_wq = create_singlethread_workqueue(DRIVER_NAME"_event");
883 if (!card->event_wq) {
884 dev_err(CARD_TO_DEV(card), "Failed card event setup.\n");
885 goto failed_event_handler;
886 }
887
8722ff8c 888 INIT_WORK(&card->event_work, card_event_handler);
889
890 st = rsxx_setup_dev(card);
891 if (st)
892 goto failed_create_dev;
893
894 rsxx_get_card_state(card, &card->state);
895
896 dev_info(CARD_TO_DEV(card),
897 "card state: %s\n",
898 rsxx_card_state_to_str(card->state));
899
900 /*
901 * Now that the DMA Engine and devices have been setup,
902 * we can enable the event interrupt(it kicks off actions in
903 * those layers so we couldn't enable it right away.)
904 */
c206c709 905 spin_lock_irq(&card->irq_lock);
8722ff8c 906 rsxx_enable_ier_and_isr(card, CR_INTR_EVENT);
c206c709 907 spin_unlock_irq(&card->irq_lock);
8722ff8c 908
909 if (card->state == CARD_STATE_SHUTDOWN) {
910 st = rsxx_issue_card_cmd(card, CARD_CMD_STARTUP);
911 if (st)
912 dev_crit(CARD_TO_DEV(card),
913 "Failed issuing card startup\n");
fb065cd9
PK		 914 if (sync_start) {
915 sync_timeout = SYNC_START_TIMEOUT;
916
917 dev_info(CARD_TO_DEV(card),
918 "Waiting for card to startup\n");
919
920 do {
921 ssleep(1);
922 sync_timeout--;
923
924 rsxx_get_card_state(card, &card->state);
925 } while (sync_timeout &&
926 (card->state == CARD_STATE_STARTING));
927
928 if (card->state == CARD_STATE_STARTING) {
929 dev_warn(CARD_TO_DEV(card),
930 "Card startup timed out\n");
931 card->size8 = 0;
932 } else {
933 dev_info(CARD_TO_DEV(card),
934 "card state: %s\n",
935 rsxx_card_state_to_str(card->state));
936 st = rsxx_get_card_size8(card, &card->size8);
937 if (st)
938 card->size8 = 0;
939 }
940 }
8722ff8c 941 } else if (card->state == CARD_STATE_GOOD ||
942 card->state == CARD_STATE_RD_ONLY_FAULT) {
943 st = rsxx_get_card_size8(card, &card->size8);
944 if (st)
945 card->size8 = 0;
946 }
947
948 rsxx_attach_dev(card);
949
36f988e9
PK		 950 /************* Setup Debugfs *************/
951 rsxx_debugfs_dev_new(card);
952
8722ff8c 953 return 0;
954
955failed_create_dev:
a3299ab1
PK		 956 destroy_workqueue(card->event_wq);
957 card->event_wq = NULL;
958failed_event_handler:
8722ff8c 959 rsxx_dma_destroy(card);
960failed_dma_setup:
961failed_compatiblity_check:
a3299ab1
PK		 962 destroy_workqueue(card->creg_ctrl.creg_wq);
963 card->creg_ctrl.creg_wq = NULL;
964failed_creg_setup:
c206c709 965 spin_lock_irq(&card->irq_lock);
8722ff8c 966 rsxx_disable_ier_and_isr(card, CR_INTR_ALL);
c206c709 967 spin_unlock_irq(&card->irq_lock);
8722ff8c 968 free_irq(dev->irq, card);
969 if (!force_legacy)
970 pci_disable_msi(dev);
971failed_irq:
972 pci_iounmap(dev, card->regmap);
973failed_iomap:
974 pci_release_regions(dev);
975failed_request_regions:
976failed_dma_mask:
977 pci_disable_device(dev);
978failed_enable:
37ae133c 979 ida_free(&rsxx_disk_ida, card->disk_id);
8722ff8c 980failed_ida_get:
981 kfree(card);
982
983 return st;
984}
985
82bed4d5 986static void rsxx_pci_remove(struct pci_dev *dev)
8722ff8c 987{
988 struct rsxx_cardinfo *card = pci_get_drvdata(dev);
989 unsigned long flags;
990 int st;
991 int i;
992
993 if (!card)
994 return;
995
996 dev_info(CARD_TO_DEV(card),
997 "Removing PCI-Flash SSD.\n");
998
999 rsxx_detach_dev(card);
1000
1001 for (i = 0; i < card->n_targets; i++) {
1002 spin_lock_irqsave(&card->irq_lock, flags);
1003 rsxx_disable_ier_and_isr(card, CR_INTR_DMA(i));
1004 spin_unlock_irqrestore(&card->irq_lock, flags);
1005 }
1006
1007 st = card_shutdown(card);
1008 if (st)
1009 dev_crit(CARD_TO_DEV(card), "Shutdown failed!\n");
1010
1011 /* Sync outstanding event handlers. */
1012 spin_lock_irqsave(&card->irq_lock, flags);
1013 rsxx_disable_ier_and_isr(card, CR_INTR_EVENT);
1014 spin_unlock_irqrestore(&card->irq_lock, flags);
1015
8722ff8c 1016 cancel_work_sync(&card->event_work);
1017
1018 rsxx_destroy_dev(card);
1019 rsxx_dma_destroy(card);
1020
1021 spin_lock_irqsave(&card->irq_lock, flags);
1022 rsxx_disable_ier_and_isr(card, CR_INTR_ALL);
1023 spin_unlock_irqrestore(&card->irq_lock, flags);
1ebfd109
PK		 1024
1025 /* Prevent work_structs from re-queuing themselves. */
1026 card->halt = 1;
1027
36f988e9
PK		 1028 debugfs_remove_recursive(card->debugfs_dir);
1029
8722ff8c 1030 free_irq(dev->irq, card);
1031
1032 if (!force_legacy)
1033 pci_disable_msi(dev);
1034
1035 rsxx_creg_destroy(card);
1036
1037 pci_iounmap(dev, card->regmap);
1038
1039 pci_disable_device(dev);
1040 pci_release_regions(dev);
1041
37ae133c 1042 ida_free(&rsxx_disk_ida, card->disk_id);
8722ff8c 1043 kfree(card);
1044}
1045
1046static int rsxx_pci_suspend(struct pci_dev *dev, pm_message_t state)
1047{
1048 /* We don't support suspend at this time. */
1049 return -ENOSYS;
1050}
1051
1052static void rsxx_pci_shutdown(struct pci_dev *dev)
1053{
1054 struct rsxx_cardinfo *card = pci_get_drvdata(dev);
1055 unsigned long flags;
1056 int i;
1057
1058 if (!card)
1059 return;
1060
1061 dev_info(CARD_TO_DEV(card), "Shutting down PCI-Flash SSD.\n");
1062
1063 rsxx_detach_dev(card);
1064
1065 for (i = 0; i < card->n_targets; i++) {
1066 spin_lock_irqsave(&card->irq_lock, flags);
1067 rsxx_disable_ier_and_isr(card, CR_INTR_DMA(i));
1068 spin_unlock_irqrestore(&card->irq_lock, flags);
1069 }
1070
1071 card_shutdown(card);
1072}
1073
c95246c3
PK		 1074static const struct pci_error_handlers rsxx_err_handler = {
1075 .error_detected = rsxx_error_detected,
1076 .slot_reset = rsxx_slot_reset,
1077};
1078
9baa3c34 1079static const struct pci_device_id rsxx_pci_ids[] = {
9bb3c446
PK		 1080 {PCI_DEVICE(PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_FS70_FLASH)},
1081 {PCI_DEVICE(PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_FS80_FLASH)},
8722ff8c 1082 {0,},
1083};
1084
1085MODULE_DEVICE_TABLE(pci, rsxx_pci_ids);
1086
1087static struct pci_driver rsxx_pci_driver = {
1088 .name = DRIVER_NAME,
1089 .id_table = rsxx_pci_ids,
1090 .probe = rsxx_pci_probe,
82bed4d5 1091 .remove = rsxx_pci_remove,
8722ff8c 1092 .suspend = rsxx_pci_suspend,
1093 .shutdown = rsxx_pci_shutdown,
c95246c3 1094 .err_handler = &rsxx_err_handler,
8722ff8c 1095};
1096
1097static int __init rsxx_core_init(void)
1098{
1099 int st;
1100
1101 st = rsxx_dev_init();
1102 if (st)
1103 return st;
1104
1105 st = rsxx_dma_init();
1106 if (st)
1107 goto dma_init_failed;
1108
1109 st = rsxx_creg_init();
1110 if (st)
1111 goto creg_init_failed;
1112
1113 return pci_register_driver(&rsxx_pci_driver);
1114
1115creg_init_failed:
1116 rsxx_dma_cleanup();
1117dma_init_failed:
1118 rsxx_dev_cleanup();
1119
1120 return st;
1121}
1122
1123static void __exit rsxx_core_cleanup(void)
1124{
1125 pci_unregister_driver(&rsxx_pci_driver);
1126 rsxx_creg_cleanup();
1127 rsxx_dma_cleanup();
1128 rsxx_dev_cleanup();
1129}
1130
1131module_init(rsxx_core_init);
1132module_exit(rsxx_core_cleanup);
Linux 6.x block layer and io_uring tree(s)