projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'for-linus-20181026' of git://git.kernel.dk/linux-block
[linux-2.6-block.git] / drivers / ata / sata_nv.c
CommitLineData
1da177e4
LT		 1/*
2 * sata_nv.c - NVIDIA nForce SATA
3 *
4 * Copyright 2004 NVIDIA Corp. All rights reserved.
5 * Copyright 2004 Andrew Chew
6 *
aa7e16d6
JG		 7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2, or (at your option)
11 * any later version.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with this program; see the file COPYING. If not, write to
20 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
1da177e4 21 *
af36d7f0
JG		 22 *
23 * libata documentation is available via 'make {ps|pdf}docs',
19285f3c 24 * as Documentation/driver-api/libata.rst
af36d7f0
JG		 25 *
26 * No hardware documentation available outside of NVIDIA.
27 * This driver programs the NVIDIA SATA controller in a similar
28 * fashion as with other PCI IDE BMDMA controllers, with a few
29 * NV-specific details such as register offsets, SATA phy location,
30 * hotplug info, etc.
31 *
fbbb262d
RH		 32 * CK804/MCP04 controllers support an alternate programming interface
33 * similar to the ADMA specification (with some modifications).
34 * This allows the use of NCQ. Non-DMA-mapped ATA commands are still
35 * sent through the legacy interface.
36 *
1da177e4
LT		 37 */
38
1da177e4
LT		 39#include <linux/kernel.h>
40#include <linux/module.h>
5a0e3ad6 41#include <linux/gfp.h>
1da177e4 42#include <linux/pci.h>
1da177e4
LT		 43#include <linux/blkdev.h>
44#include <linux/delay.h>
45#include <linux/interrupt.h>
a9524a76 46#include <linux/device.h>
1da177e4 47#include <scsi/scsi_host.h>
fbbb262d 48#include <scsi/scsi_device.h>
1da177e4
LT		 49#include <linux/libata.h>
50
51#define DRV_NAME "sata_nv"
2a3103ce 52#define DRV_VERSION "3.5"
fbbb262d
RH		 53
54#define NV_ADMA_DMA_BOUNDARY 0xffffffffUL
1da177e4 55
10ad05df 56enum {
0d5ff566
TH		 57 NV_MMIO_BAR = 5,
58
10ad05df 59 NV_PORTS = 2,
14bdef98
EIB		 60 NV_PIO_MASK = ATA_PIO4,
61 NV_MWDMA_MASK = ATA_MWDMA2,
62 NV_UDMA_MASK = ATA_UDMA6,
10ad05df
JG		 63 NV_PORT0_SCR_REG_OFFSET = 0x00,
64 NV_PORT1_SCR_REG_OFFSET = 0x40,
1da177e4 65
27e4b274 66 /* INT_STATUS/ENABLE */
10ad05df 67 NV_INT_STATUS = 0x10,
10ad05df 68 NV_INT_ENABLE = 0x11,
27e4b274 69 NV_INT_STATUS_CK804 = 0x440,
10ad05df 70 NV_INT_ENABLE_CK804 = 0x441,
1da177e4 71
27e4b274
TH		 72 /* INT_STATUS/ENABLE bits */
73 NV_INT_DEV = 0x01,
74 NV_INT_PM = 0x02,
75 NV_INT_ADDED = 0x04,
76 NV_INT_REMOVED = 0x08,
77
78 NV_INT_PORT_SHIFT = 4, /* each port occupies 4 bits */
79
39f87582 80 NV_INT_ALL = 0x0f,
5a44efff
TH		 81 NV_INT_MASK = NV_INT_DEV |
82 NV_INT_ADDED | NV_INT_REMOVED,
39f87582 83
27e4b274 84 /* INT_CONFIG */
10ad05df
JG		 85 NV_INT_CONFIG = 0x12,
86 NV_INT_CONFIG_METHD = 0x01, // 0 = INT, 1 = SMI
1da177e4 87
10ad05df
JG		 88 // For PCI config register 20
89 NV_MCP_SATA_CFG_20 = 0x50,
90 NV_MCP_SATA_CFG_20_SATA_SPACE_EN = 0x04,
fbbb262d
RH		 91 NV_MCP_SATA_CFG_20_PORT0_EN = (1 << 17),
92 NV_MCP_SATA_CFG_20_PORT1_EN = (1 << 16),
93 NV_MCP_SATA_CFG_20_PORT0_PWB_EN = (1 << 14),
94 NV_MCP_SATA_CFG_20_PORT1_PWB_EN = (1 << 12),
95
96 NV_ADMA_MAX_CPBS = 32,
97 NV_ADMA_CPB_SZ = 128,
98 NV_ADMA_APRD_SZ = 16,
99 NV_ADMA_SGTBL_LEN = (1024 - NV_ADMA_CPB_SZ) /
100 NV_ADMA_APRD_SZ,
101 NV_ADMA_SGTBL_TOTAL_LEN = NV_ADMA_SGTBL_LEN + 5,
102 NV_ADMA_SGTBL_SZ = NV_ADMA_SGTBL_LEN * NV_ADMA_APRD_SZ,
103 NV_ADMA_PORT_PRIV_DMA_SZ = NV_ADMA_MAX_CPBS *
104 (NV_ADMA_CPB_SZ + NV_ADMA_SGTBL_SZ),
105
106 /* BAR5 offset to ADMA general registers */
107 NV_ADMA_GEN = 0x400,
108 NV_ADMA_GEN_CTL = 0x00,
109 NV_ADMA_NOTIFIER_CLEAR = 0x30,
110
111 /* BAR5 offset to ADMA ports */
112 NV_ADMA_PORT = 0x480,
113
114 /* size of ADMA port register space */
115 NV_ADMA_PORT_SIZE = 0x100,
116
117 /* ADMA port registers */
118 NV_ADMA_CTL = 0x40,
119 NV_ADMA_CPB_COUNT = 0x42,
120 NV_ADMA_NEXT_CPB_IDX = 0x43,
121 NV_ADMA_STAT = 0x44,
122 NV_ADMA_CPB_BASE_LOW = 0x48,
123 NV_ADMA_CPB_BASE_HIGH = 0x4C,
124 NV_ADMA_APPEND = 0x50,
125 NV_ADMA_NOTIFIER = 0x68,
126 NV_ADMA_NOTIFIER_ERROR = 0x6C,
127
128 /* NV_ADMA_CTL register bits */
129 NV_ADMA_CTL_HOTPLUG_IEN = (1 << 0),
130 NV_ADMA_CTL_CHANNEL_RESET = (1 << 5),
131 NV_ADMA_CTL_GO = (1 << 7),
132 NV_ADMA_CTL_AIEN = (1 << 8),
133 NV_ADMA_CTL_READ_NON_COHERENT = (1 << 11),
134 NV_ADMA_CTL_WRITE_NON_COHERENT = (1 << 12),
135
136 /* CPB response flag bits */
137 NV_CPB_RESP_DONE = (1 << 0),
138 NV_CPB_RESP_ATA_ERR = (1 << 3),
139 NV_CPB_RESP_CMD_ERR = (1 << 4),
140 NV_CPB_RESP_CPB_ERR = (1 << 7),
141
142 /* CPB control flag bits */
143 NV_CPB_CTL_CPB_VALID = (1 << 0),
144 NV_CPB_CTL_QUEUE = (1 << 1),
145 NV_CPB_CTL_APRD_VALID = (1 << 2),
146 NV_CPB_CTL_IEN = (1 << 3),
147 NV_CPB_CTL_FPDMA = (1 << 4),
148
149 /* APRD flags */
150 NV_APRD_WRITE = (1 << 1),
151 NV_APRD_END = (1 << 2),
152 NV_APRD_CONT = (1 << 3),
153
154 /* NV_ADMA_STAT flags */
155 NV_ADMA_STAT_TIMEOUT = (1 << 0),
156 NV_ADMA_STAT_HOTUNPLUG = (1 << 1),
157 NV_ADMA_STAT_HOTPLUG = (1 << 2),
158 NV_ADMA_STAT_CPBERR = (1 << 4),
159 NV_ADMA_STAT_SERROR = (1 << 5),
160 NV_ADMA_STAT_CMD_COMPLETE = (1 << 6),
161 NV_ADMA_STAT_IDLE = (1 << 8),
162 NV_ADMA_STAT_LEGACY = (1 << 9),
163 NV_ADMA_STAT_STOPPED = (1 << 10),
164 NV_ADMA_STAT_DONE = (1 << 12),
165 NV_ADMA_STAT_ERR = NV_ADMA_STAT_CPBERR |
2dcb407e 166 NV_ADMA_STAT_TIMEOUT,
fbbb262d
RH		 167
168 /* port flags */
169 NV_ADMA_PORT_REGISTER_MODE = (1 << 0),
2dec7555 170 NV_ADMA_ATAPI_SETUP_COMPLETE = (1 << 1),
fbbb262d 171
f140f0f1
KL		 172 /* MCP55 reg offset */
173 NV_CTL_MCP55 = 0x400,
174 NV_INT_STATUS_MCP55 = 0x440,
175 NV_INT_ENABLE_MCP55 = 0x444,
176 NV_NCQ_REG_MCP55 = 0x448,
177
178 /* MCP55 */
179 NV_INT_ALL_MCP55 = 0xffff,
180 NV_INT_PORT_SHIFT_MCP55 = 16, /* each port occupies 16 bits */
181 NV_INT_MASK_MCP55 = NV_INT_ALL_MCP55 & 0xfffd,
182
183 /* SWNCQ ENABLE BITS*/
184 NV_CTL_PRI_SWNCQ = 0x02,
185 NV_CTL_SEC_SWNCQ = 0x04,
186
187 /* SW NCQ status bits*/
188 NV_SWNCQ_IRQ_DEV = (1 << 0),
189 NV_SWNCQ_IRQ_PM = (1 << 1),
190 NV_SWNCQ_IRQ_ADDED = (1 << 2),
191 NV_SWNCQ_IRQ_REMOVED = (1 << 3),
192
193 NV_SWNCQ_IRQ_BACKOUT = (1 << 4),
194 NV_SWNCQ_IRQ_SDBFIS = (1 << 5),
195 NV_SWNCQ_IRQ_DHREGFIS = (1 << 6),
196 NV_SWNCQ_IRQ_DMASETUP = (1 << 7),
197
198 NV_SWNCQ_IRQ_HOTPLUG = NV_SWNCQ_IRQ_ADDED |
199 NV_SWNCQ_IRQ_REMOVED,
200
fbbb262d
RH		 201};
202
203/* ADMA Physical Region Descriptor - one SG segment */
204struct nv_adma_prd {
205 __le64 addr;
206 __le32 len;
207 u8 flags;
208 u8 packet_len;
209 __le16 reserved;
210};
211
212enum nv_adma_regbits {
213 CMDEND = (1 << 15), /* end of command list */
214 WNB = (1 << 14), /* wait-not-BSY */
215 IGN = (1 << 13), /* ignore this entry */
216 CS1n = (1 << (4 + 8)), /* std. PATA signals follow... */
217 DA2 = (1 << (2 + 8)),
218 DA1 = (1 << (1 + 8)),
219 DA0 = (1 << (0 + 8)),
220};
221
222/* ADMA Command Parameter Block
223 The first 5 SG segments are stored inside the Command Parameter Block itself.
224 If there are more than 5 segments the remainder are stored in a separate
225 memory area indicated by next_aprd. */
226struct nv_adma_cpb {
227 u8 resp_flags; /* 0 */
228 u8 reserved1; /* 1 */
229 u8 ctl_flags; /* 2 */
230 /* len is length of taskfile in 64 bit words */
2dcb407e 231 u8 len; /* 3 */
fbbb262d
RH		 232 u8 tag; /* 4 */
233 u8 next_cpb_idx; /* 5 */
234 __le16 reserved2; /* 6-7 */
235 __le16 tf[12]; /* 8-31 */
236 struct nv_adma_prd aprd[5]; /* 32-111 */
237 __le64 next_aprd; /* 112-119 */
238 __le64 reserved3; /* 120-127 */
10ad05df 239};
1da177e4 240
fbbb262d
RH		 241
242struct nv_adma_port_priv {
243 struct nv_adma_cpb *cpb;
244 dma_addr_t cpb_dma;
245 struct nv_adma_prd *aprd;
246 dma_addr_t aprd_dma;
2dcb407e
JG		 247 void __iomem *ctl_block;
248 void __iomem *gen_block;
249 void __iomem *notifier_clear_block;
8959d300 250 u64 adma_dma_mask;
fbbb262d 251 u8 flags;
5e5c74a5 252 int last_issue_ncq;
fbbb262d
RH		 253};
254
cdf56bcf
RH		 255struct nv_host_priv {
256 unsigned long type;
257};
258
f140f0f1
KL		 259struct defer_queue {
260 u32 defer_bits;
261 unsigned int head;
262 unsigned int tail;
263 unsigned int tag[ATA_MAX_QUEUE];
264};
265
266enum ncq_saw_flag_list {
267 ncq_saw_d2h = (1U << 0),
268 ncq_saw_dmas = (1U << 1),
269 ncq_saw_sdb = (1U << 2),
270 ncq_saw_backout = (1U << 3),
271};
272
273struct nv_swncq_port_priv {
f60d7011 274 struct ata_bmdma_prd *prd; /* our SG list */
f140f0f1
KL		 275 dma_addr_t prd_dma; /* and its DMA mapping */
276 void __iomem *sactive_block;
277 void __iomem *irq_block;
278 void __iomem *tag_block;
279 u32 qc_active;
280
281 unsigned int last_issue_tag;
282
283 /* fifo circular queue to store deferral command */
284 struct defer_queue defer_queue;
285
286 /* for NCQ interrupt analysis */
287 u32 dhfis_bits;
288 u32 dmafis_bits;
289 u32 sdbfis_bits;
290
291 unsigned int ncq_flags;
292};
293
294
5796d1c4 295#define NV_ADMA_CHECK_INTR(GCTL, PORT) ((GCTL) & (1 << (19 + (12 * (PORT)))))
fbbb262d 296
2dcb407e 297static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
58eb8cd5 298#ifdef CONFIG_PM_SLEEP
cdf56bcf 299static int nv_pci_device_resume(struct pci_dev *pdev);
438ac6d5 300#endif
cca3974e 301static void nv_ck804_host_stop(struct ata_host *host);
7d12e780
DH		 302static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance);
303static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance);
304static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance);
82ef04fb
TH		 305static int nv_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val);
306static int nv_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val);
1da177e4 307
7f4774b3
TH		 308static int nv_hardreset(struct ata_link *link, unsigned int *class,
309 unsigned long deadline);
39f87582
TH		 310static void nv_nf2_freeze(struct ata_port *ap);
311static void nv_nf2_thaw(struct ata_port *ap);
312static void nv_ck804_freeze(struct ata_port *ap);
313static void nv_ck804_thaw(struct ata_port *ap);
fbbb262d 314static int nv_adma_slave_config(struct scsi_device *sdev);
2dec7555 315static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc);
fbbb262d
RH		 316static void nv_adma_qc_prep(struct ata_queued_cmd *qc);
317static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc);
318static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance);
319static void nv_adma_irq_clear(struct ata_port *ap);
320static int nv_adma_port_start(struct ata_port *ap);
321static void nv_adma_port_stop(struct ata_port *ap);
438ac6d5 322#ifdef CONFIG_PM
cdf56bcf
RH		 323static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg);
324static int nv_adma_port_resume(struct ata_port *ap);
438ac6d5 325#endif
53014e25
RH		 326static void nv_adma_freeze(struct ata_port *ap);
327static void nv_adma_thaw(struct ata_port *ap);
fbbb262d
RH		 328static void nv_adma_error_handler(struct ata_port *ap);
329static void nv_adma_host_stop(struct ata_host *host);
f5ecac2d 330static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc);
f2fb344b 331static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf);
39f87582 332
f140f0f1
KL		 333static void nv_mcp55_thaw(struct ata_port *ap);
334static void nv_mcp55_freeze(struct ata_port *ap);
335static void nv_swncq_error_handler(struct ata_port *ap);
336static int nv_swncq_slave_config(struct scsi_device *sdev);
337static int nv_swncq_port_start(struct ata_port *ap);
338static void nv_swncq_qc_prep(struct ata_queued_cmd *qc);
339static void nv_swncq_fill_sg(struct ata_queued_cmd *qc);
340static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc);
341static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis);
342static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance);
343#ifdef CONFIG_PM
344static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg);
345static int nv_swncq_port_resume(struct ata_port *ap);
346#endif
347
1da177e4
LT		 348enum nv_host_type
349{
350 GENERIC,
351 NFORCE2,
27e4b274 352 NFORCE3 = NFORCE2, /* NF2 == NF3 as far as sata_nv is concerned */
fbbb262d 353 CK804,
f140f0f1 354 ADMA,
2d775708 355 MCP5x,
f140f0f1 356 SWNCQ,
1da177e4
LT		 357};
358
3b7d697d 359static const struct pci_device_id nv_pci_tbl[] = {
54bb3a94
JG		 360 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE2S_SATA), NFORCE2 },
361 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA), NFORCE3 },
362 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA2), NFORCE3 },
363 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA), CK804 },
364 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA2), CK804 },
365 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA), CK804 },
366 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA2), CK804 },
2d775708
TH		 367 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA), MCP5x },
368 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2), MCP5x },
369 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA), MCP5x },
370 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2), MCP5x },
e2e031eb
KL		 371 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA), GENERIC },
372 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA2), GENERIC },
373 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA3), GENERIC },
2d2744fc
JG		 374
375 { } /* terminate list */
1da177e4
LT		 376};
377
1da177e4
LT		 378static struct pci_driver nv_pci_driver = {
379 .name = DRV_NAME,
380 .id_table = nv_pci_tbl,
381 .probe = nv_init_one,
58eb8cd5 382#ifdef CONFIG_PM_SLEEP
cdf56bcf
RH		 383 .suspend = ata_pci_device_suspend,
384 .resume = nv_pci_device_resume,
438ac6d5 385#endif
1daf9ce7 386 .remove = ata_pci_remove_one,
1da177e4
LT		 387};
388
193515d5 389static struct scsi_host_template nv_sht = {
68d1d07b 390 ATA_BMDMA_SHT(DRV_NAME),
1da177e4
LT		 391};
392
fbbb262d 393static struct scsi_host_template nv_adma_sht = {
68d1d07b 394 ATA_NCQ_SHT(DRV_NAME),
fbbb262d 395 .can_queue = NV_ADMA_MAX_CPBS,
fbbb262d 396 .sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN,
fbbb262d
RH		 397 .dma_boundary = NV_ADMA_DMA_BOUNDARY,
398 .slave_configure = nv_adma_slave_config,
fbbb262d
RH		 399};
400
f140f0f1 401static struct scsi_host_template nv_swncq_sht = {
68d1d07b 402 ATA_NCQ_SHT(DRV_NAME),
ba80c3a5 403 .can_queue = ATA_MAX_QUEUE - 1,
f140f0f1 404 .sg_tablesize = LIBATA_MAX_PRD,
f140f0f1
KL		 405 .dma_boundary = ATA_DMA_BOUNDARY,
406 .slave_configure = nv_swncq_slave_config,
f140f0f1
KL		 407};
408
7f4774b3
TH		 409/*
410 * NV SATA controllers have various different problems with hardreset
411 * protocol depending on the specific controller and device.
412 *
413 * GENERIC:
414 *
415 * bko11195 reports that link doesn't come online after hardreset on
416 * generic nv's and there have been several other similar reports on
417 * linux-ide.
418 *
419 * bko12351#c23 reports that warmplug on MCP61 doesn't work with
420 * softreset.
421 *
422 * NF2/3:
423 *
424 * bko3352 reports nf2/3 controllers can't determine device signature
425 * reliably after hardreset. The following thread reports detection
426 * failure on cold boot with the standard debouncing timing.
427 *
428 * http://thread.gmane.org/gmane.linux.ide/34098
429 *
430 * bko12176 reports that hardreset fails to bring up the link during
431 * boot on nf2.
432 *
433 * CK804:
434 *
435 * For initial probing after boot and hot plugging, hardreset mostly
436 * works fine on CK804 but curiously, reprobing on the initial port
437 * by rescanning or rmmod/insmod fails to acquire the initial D2H Reg
438 * FIS in somewhat undeterministic way.
439 *
440 * SWNCQ:
441 *
442 * bko12351 reports that when SWNCQ is enabled, for hotplug to work,
443 * hardreset should be used and hardreset can't report proper
444 * signature, which suggests that mcp5x is closer to nf2 as long as
445 * reset quirkiness is concerned.
446 *
447 * bko12703 reports that boot probing fails for intel SSD with
448 * hardreset. Link fails to come online. Softreset works fine.
449 *
450 * The failures are varied but the following patterns seem true for
451 * all flavors.
452 *
453 * - Softreset during boot always works.
454 *
455 * - Hardreset during boot sometimes fails to bring up the link on
456 * certain comibnations and device signature acquisition is
457 * unreliable.
458 *
459 * - Hardreset is often necessary after hotplug.
460 *
461 * So, preferring softreset for boot probing and error handling (as
462 * hardreset might bring down the link) but using hardreset for
463 * post-boot probing should work around the above issues in most
464 * cases. Define nv_hardreset() which only kicks in for post-boot
465 * probing and use it for all variants.
466 */
467static struct ata_port_operations nv_generic_ops = {
029cfd6b 468 .inherits = &ata_bmdma_port_ops,
c96f1732 469 .lost_interrupt = ATA_OP_NULL,
1da177e4
LT		 470 .scr_read = nv_scr_read,
471 .scr_write = nv_scr_write,
7f4774b3 472 .hardreset = nv_hardreset,
1da177e4
LT		 473};
474
029cfd6b 475static struct ata_port_operations nv_nf2_ops = {
7dac745b 476 .inherits = &nv_generic_ops,
39f87582
TH		 477 .freeze = nv_nf2_freeze,
478 .thaw = nv_nf2_thaw,
ada364e8
TH		 479};
480
029cfd6b 481static struct ata_port_operations nv_ck804_ops = {
7f4774b3 482 .inherits = &nv_generic_ops,
39f87582
TH		 483 .freeze = nv_ck804_freeze,
484 .thaw = nv_ck804_thaw,
ada364e8
TH		 485 .host_stop = nv_ck804_host_stop,
486};
487
029cfd6b 488static struct ata_port_operations nv_adma_ops = {
3c324283 489 .inherits = &nv_ck804_ops,
029cfd6b 490
2dec7555 491 .check_atapi_dma = nv_adma_check_atapi_dma,
5682ed33 492 .sff_tf_read = nv_adma_tf_read,
31cc23b3 493 .qc_defer = ata_std_qc_defer,
fbbb262d
RH		 494 .qc_prep = nv_adma_qc_prep,
495 .qc_issue = nv_adma_qc_issue,
5682ed33 496 .sff_irq_clear = nv_adma_irq_clear,
029cfd6b 497
53014e25
RH		 498 .freeze = nv_adma_freeze,
499 .thaw = nv_adma_thaw,
fbbb262d 500 .error_handler = nv_adma_error_handler,
f5ecac2d 501 .post_internal_cmd = nv_adma_post_internal_cmd,
029cfd6b 502
fbbb262d
RH		 503 .port_start = nv_adma_port_start,
504 .port_stop = nv_adma_port_stop,
438ac6d5 505#ifdef CONFIG_PM
cdf56bcf
RH		 506 .port_suspend = nv_adma_port_suspend,
507 .port_resume = nv_adma_port_resume,
438ac6d5 508#endif
fbbb262d
RH		 509 .host_stop = nv_adma_host_stop,
510};
511
029cfd6b 512static struct ata_port_operations nv_swncq_ops = {
7f4774b3 513 .inherits = &nv_generic_ops,
029cfd6b 514
f140f0f1
KL		 515 .qc_defer = ata_std_qc_defer,
516 .qc_prep = nv_swncq_qc_prep,
517 .qc_issue = nv_swncq_qc_issue,
029cfd6b 518
f140f0f1
KL		 519 .freeze = nv_mcp55_freeze,
520 .thaw = nv_mcp55_thaw,
521 .error_handler = nv_swncq_error_handler,
029cfd6b 522
f140f0f1
KL		 523#ifdef CONFIG_PM
524 .port_suspend = nv_swncq_port_suspend,
525 .port_resume = nv_swncq_port_resume,
526#endif
527 .port_start = nv_swncq_port_start,
528};
529
95947193
TH		 530struct nv_pi_priv {
531 irq_handler_t irq_handler;
532 struct scsi_host_template *sht;
533};
534
535#define NV_PI_PRIV(_irq_handler, _sht) \
536 &(struct nv_pi_priv){ .irq_handler = _irq_handler, .sht = _sht }
537
1626aeb8 538static const struct ata_port_info nv_port_info[] = {
ada364e8
TH		 539 /* generic */
540 {
9cbe056f 541 .flags = ATA_FLAG_SATA,
ada364e8
TH		 542 .pio_mask = NV_PIO_MASK,
543 .mwdma_mask = NV_MWDMA_MASK,
544 .udma_mask = NV_UDMA_MASK,
545 .port_ops = &nv_generic_ops,
95947193 546 .private_data = NV_PI_PRIV(nv_generic_interrupt, &nv_sht),
ada364e8
TH		 547 },
548 /* nforce2/3 */
549 {
9cbe056f 550 .flags = ATA_FLAG_SATA,
ada364e8
TH		 551 .pio_mask = NV_PIO_MASK,
552 .mwdma_mask = NV_MWDMA_MASK,
553 .udma_mask = NV_UDMA_MASK,
554 .port_ops = &nv_nf2_ops,
95947193 555 .private_data = NV_PI_PRIV(nv_nf2_interrupt, &nv_sht),
ada364e8
TH		 556 },
557 /* ck804 */
558 {
9cbe056f 559 .flags = ATA_FLAG_SATA,
ada364e8
TH		 560 .pio_mask = NV_PIO_MASK,
561 .mwdma_mask = NV_MWDMA_MASK,
562 .udma_mask = NV_UDMA_MASK,
563 .port_ops = &nv_ck804_ops,
95947193 564 .private_data = NV_PI_PRIV(nv_ck804_interrupt, &nv_sht),
ada364e8 565 },
fbbb262d
RH		 566 /* ADMA */
567 {
9cbe056f 568 .flags = ATA_FLAG_SATA | ATA_FLAG_NCQ,
fbbb262d
RH		 569 .pio_mask = NV_PIO_MASK,
570 .mwdma_mask = NV_MWDMA_MASK,
571 .udma_mask = NV_UDMA_MASK,
572 .port_ops = &nv_adma_ops,
95947193 573 .private_data = NV_PI_PRIV(nv_adma_interrupt, &nv_adma_sht),
fbbb262d 574 },
2d775708
TH		 575 /* MCP5x */
576 {
9cbe056f 577 .flags = ATA_FLAG_SATA,
2d775708
TH		 578 .pio_mask = NV_PIO_MASK,
579 .mwdma_mask = NV_MWDMA_MASK,
580 .udma_mask = NV_UDMA_MASK,
7f4774b3 581 .port_ops = &nv_generic_ops,
2d775708
TH		 582 .private_data = NV_PI_PRIV(nv_generic_interrupt, &nv_sht),
583 },
f140f0f1
KL		 584 /* SWNCQ */
585 {
9cbe056f 586 .flags = ATA_FLAG_SATA | ATA_FLAG_NCQ,
f140f0f1
KL		 587 .pio_mask = NV_PIO_MASK,
588 .mwdma_mask = NV_MWDMA_MASK,
589 .udma_mask = NV_UDMA_MASK,
590 .port_ops = &nv_swncq_ops,
95947193 591 .private_data = NV_PI_PRIV(nv_swncq_interrupt, &nv_swncq_sht),
f140f0f1 592 },
1da177e4
LT		 593};
594
595MODULE_AUTHOR("NVIDIA");
596MODULE_DESCRIPTION("low-level driver for NVIDIA nForce SATA controller");
597MODULE_LICENSE("GPL");
598MODULE_DEVICE_TABLE(pci, nv_pci_tbl);
599MODULE_VERSION(DRV_VERSION);
600
90ab5ee9 601static bool adma_enabled;
c13aff32 602static bool swncq_enabled = true;
90ab5ee9 603static bool msi_enabled;
fbbb262d 604
2dec7555
RH		 605static void nv_adma_register_mode(struct ata_port *ap)
606{
2dec7555 607 struct nv_adma_port_priv *pp = ap->private_data;
cdf56bcf 608 void __iomem *mmio = pp->ctl_block;
a2cfe81a
RH		 609 u16 tmp, status;
610 int count = 0;
2dec7555
RH		 611
612 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
613 return;
614
a2cfe81a 615 status = readw(mmio + NV_ADMA_STAT);
2dcb407e 616 while (!(status & NV_ADMA_STAT_IDLE) && count < 20) {
a2cfe81a
RH		 617 ndelay(50);
618 status = readw(mmio + NV_ADMA_STAT);
619 count++;
620 }
2dcb407e 621 if (count == 20)
a9a79dfe
JP		 622 ata_port_warn(ap, "timeout waiting for ADMA IDLE, stat=0x%hx\n",
623 status);
a2cfe81a 624
2dec7555
RH		 625 tmp = readw(mmio + NV_ADMA_CTL);
626 writew(tmp & ~NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
627
a2cfe81a
RH		 628 count = 0;
629 status = readw(mmio + NV_ADMA_STAT);
2dcb407e 630 while (!(status & NV_ADMA_STAT_LEGACY) && count < 20) {
a2cfe81a
RH		 631 ndelay(50);
632 status = readw(mmio + NV_ADMA_STAT);
633 count++;
634 }
2dcb407e 635 if (count == 20)
a9a79dfe
JP		 636 ata_port_warn(ap,
637 "timeout waiting for ADMA LEGACY, stat=0x%hx\n",
638 status);
a2cfe81a 639
2dec7555
RH		 640 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
641}
642
643static void nv_adma_mode(struct ata_port *ap)
644{
2dec7555 645 struct nv_adma_port_priv *pp = ap->private_data;
cdf56bcf 646 void __iomem *mmio = pp->ctl_block;
a2cfe81a
RH		 647 u16 tmp, status;
648 int count = 0;
2dec7555
RH		 649
650 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE))
651 return;
f20b16ff 652
2dec7555
RH		 653 WARN_ON(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
654
655 tmp = readw(mmio + NV_ADMA_CTL);
656 writew(tmp | NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
657
a2cfe81a 658 status = readw(mmio + NV_ADMA_STAT);
2dcb407e 659 while (((status & NV_ADMA_STAT_LEGACY) ||
a2cfe81a
RH		 660 !(status & NV_ADMA_STAT_IDLE)) && count < 20) {
661 ndelay(50);
662 status = readw(mmio + NV_ADMA_STAT);
663 count++;
664 }
2dcb407e 665 if (count == 20)
a9a79dfe 666 ata_port_warn(ap,
a2cfe81a
RH		 667 "timeout waiting for ADMA LEGACY clear and IDLE, stat=0x%hx\n",
668 status);
669
2dec7555
RH		 670 pp->flags &= ~NV_ADMA_PORT_REGISTER_MODE;
671}
672
fbbb262d
RH		 673static int nv_adma_slave_config(struct scsi_device *sdev)
674{
675 struct ata_port *ap = ata_shost_to_port(sdev->host);
2dec7555 676 struct nv_adma_port_priv *pp = ap->private_data;
8959d300 677 struct nv_adma_port_priv *port0, *port1;
2dec7555 678 struct pci_dev *pdev = to_pci_dev(ap->host->dev);
8959d300 679 unsigned long segment_boundary, flags;
fbbb262d
RH		 680 unsigned short sg_tablesize;
681 int rc;
2dec7555
RH		 682 int adma_enable;
683 u32 current_reg, new_reg, config_mask;
fbbb262d
RH		 684
685 rc = ata_scsi_slave_config(sdev);
686
687 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
688 /* Not a proper libata device, ignore */
689 return rc;
690
8959d300
RH		 691 spin_lock_irqsave(ap->lock, flags);
692
9af5c9c9 693 if (ap->link.device[sdev->id].class == ATA_DEV_ATAPI) {
fbbb262d
RH		 694 /*
695 * NVIDIA reports that ADMA mode does not support ATAPI commands.
696 * Therefore ATAPI commands are sent through the legacy interface.
697 * However, the legacy interface only supports 32-bit DMA.
698 * Restrict DMA parameters as required by the legacy interface
699 * when an ATAPI device is connected.
700 */
fbbb262d
RH		 701 segment_boundary = ATA_DMA_BOUNDARY;
702 /* Subtract 1 since an extra entry may be needed for padding, see
703 libata-scsi.c */
704 sg_tablesize = LIBATA_MAX_PRD - 1;
f20b16ff 705
2dec7555
RH		 706 /* Since the legacy DMA engine is in use, we need to disable ADMA
707 on the port. */
708 adma_enable = 0;
709 nv_adma_register_mode(ap);
2dcb407e 710 } else {
fbbb262d
RH		 711 segment_boundary = NV_ADMA_DMA_BOUNDARY;
712 sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN;
2dec7555 713 adma_enable = 1;
fbbb262d 714 }
f20b16ff 715
2dec7555
RH		 716 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &current_reg);
717
2dcb407e 718 if (ap->port_no == 1)
2dec7555
RH		 719 config_mask = NV_MCP_SATA_CFG_20_PORT1_EN |
720 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
721 else
722 config_mask = NV_MCP_SATA_CFG_20_PORT0_EN |
723 NV_MCP_SATA_CFG_20_PORT0_PWB_EN;
f20b16ff 724
2dcb407e 725 if (adma_enable) {
2dec7555
RH		 726 new_reg = current_reg | config_mask;
727 pp->flags &= ~NV_ADMA_ATAPI_SETUP_COMPLETE;
2dcb407e 728 } else {
2dec7555
RH		 729 new_reg = current_reg & ~config_mask;
730 pp->flags |= NV_ADMA_ATAPI_SETUP_COMPLETE;
731 }
f20b16ff 732
2dcb407e 733 if (current_reg != new_reg)
2dec7555 734 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, new_reg);
f20b16ff 735
8959d300
RH		 736 port0 = ap->host->ports[0]->private_data;
737 port1 = ap->host->ports[1]->private_data;
8959d300
RH		 738 if ((port0->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
739 (port1->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)) {
258c9fde
CH		 740 /*
741 * We have to set the DMA mask to 32-bit if either port is in
742 * ATAPI mode, since they are on the same PCI device which is
743 * used for DMA mapping. If either SCSI device is not allocated
744 * yet, it's OK since that port will discover its correct
745 * setting when it does get allocated.
746 */
747 rc = dma_set_mask(&pdev->dev, ATA_DMA_MASK);
8959d300 748 } else {
258c9fde 749 rc = dma_set_mask(&pdev->dev, pp->adma_dma_mask);
8959d300
RH		 750 }
751
fbbb262d 752 blk_queue_segment_boundary(sdev->request_queue, segment_boundary);
8a78362c 753 blk_queue_max_segments(sdev->request_queue, sg_tablesize);
a9a79dfe
JP		 754 ata_port_info(ap,
755 "DMA mask 0x%llX, segment boundary 0x%lX, hw segs %hu\n",
756 (unsigned long long)*ap->host->dev->dma_mask,
757 segment_boundary, sg_tablesize);
8959d300
RH		 758
759 spin_unlock_irqrestore(ap->lock, flags);
760
fbbb262d
RH		 761 return rc;
762}
763
2dec7555
RH		 764static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc)
765{
766 struct nv_adma_port_priv *pp = qc->ap->private_data;
767 return !(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
768}
769
f2fb344b
RH		 770static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf)
771{
3f3debdb
RH		 772 /* Other than when internal or pass-through commands are executed,
773 the only time this function will be called in ADMA mode will be
774 if a command fails. In the failure case we don't care about going
775 into register mode with ADMA commands pending, as the commands will
776 all shortly be aborted anyway. We assume that NCQ commands are not
777 issued via passthrough, which is the only way that switching into
778 ADMA mode could abort outstanding commands. */
f2fb344b
RH		 779 nv_adma_register_mode(ap);
780
9363c382 781 ata_sff_tf_read(ap, tf);
f2fb344b
RH		 782}
783
2dec7555 784static unsigned int nv_adma_tf_to_cpb(struct ata_taskfile *tf, __le16 *cpb)
fbbb262d
RH		 785{
786 unsigned int idx = 0;
787
2dcb407e 788 if (tf->flags & ATA_TFLAG_ISADDR) {
ac3d6b86
RH		 789 if (tf->flags & ATA_TFLAG_LBA48) {
790 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->hob_feature | WNB);
791 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->hob_nsect);
792 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->hob_lbal);
793 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->hob_lbam);
794 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->hob_lbah);
795 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature);
796 } else
797 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature | WNB);
a84471fe 798
ac3d6b86
RH		 799 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->nsect);
800 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->lbal);
801 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->lbam);
802 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->lbah);
fbbb262d 803 }
a84471fe 804
2dcb407e 805 if (tf->flags & ATA_TFLAG_DEVICE)
ac3d6b86 806 cpb[idx++] = cpu_to_le16((ATA_REG_DEVICE << 8) | tf->device);
fbbb262d
RH		 807
808 cpb[idx++] = cpu_to_le16((ATA_REG_CMD << 8) | tf->command | CMDEND);
a84471fe 809
2dcb407e 810 while (idx < 12)
ac3d6b86 811 cpb[idx++] = cpu_to_le16(IGN);
fbbb262d
RH		 812
813 return idx;
814}
815
5bd28a4b 816static int nv_adma_check_cpb(struct ata_port *ap, int cpb_num, int force_err)
fbbb262d
RH		 817{
818 struct nv_adma_port_priv *pp = ap->private_data;
2dec7555 819 u8 flags = pp->cpb[cpb_num].resp_flags;
fbbb262d
RH		 820
821 VPRINTK("CPB %d, flags=0x%x\n", cpb_num, flags);
822
5bd28a4b
RH		 823 if (unlikely((force_err ||
824 flags & (NV_CPB_RESP_ATA_ERR |
825 NV_CPB_RESP_CMD_ERR |
826 NV_CPB_RESP_CPB_ERR)))) {
9af5c9c9 827 struct ata_eh_info *ehi = &ap->link.eh_info;
5bd28a4b
RH		 828 int freeze = 0;
829
830 ata_ehi_clear_desc(ehi);
2dcb407e 831 __ata_ehi_push_desc(ehi, "CPB resp_flags 0x%x: ", flags);
5bd28a4b 832 if (flags & NV_CPB_RESP_ATA_ERR) {
b64bbc39 833 ata_ehi_push_desc(ehi, "ATA error");
5bd28a4b
RH		 834 ehi->err_mask |= AC_ERR_DEV;
835 } else if (flags & NV_CPB_RESP_CMD_ERR) {
b64bbc39 836 ata_ehi_push_desc(ehi, "CMD error");
5bd28a4b
RH		 837 ehi->err_mask |= AC_ERR_DEV;
838 } else if (flags & NV_CPB_RESP_CPB_ERR) {
b64bbc39 839 ata_ehi_push_desc(ehi, "CPB error");
5bd28a4b
RH		 840 ehi->err_mask |= AC_ERR_SYSTEM;
841 freeze = 1;
842 } else {
843 /* notifier error, but no error in CPB flags? */
b64bbc39 844 ata_ehi_push_desc(ehi, "unknown");
5bd28a4b
RH		 845 ehi->err_mask |= AC_ERR_OTHER;
846 freeze = 1;
847 }
848 /* Kill all commands. EH will determine what actually failed. */
849 if (freeze)
850 ata_port_freeze(ap);
851 else
852 ata_port_abort(ap);
1aadf5c3 853 return -1;
fbbb262d 854 }
5bd28a4b 855
1aadf5c3
TH		 856 if (likely(flags & NV_CPB_RESP_DONE))
857 return 1;
5bd28a4b 858 return 0;
fbbb262d
RH		 859}
860
2dec7555
RH		 861static int nv_host_intr(struct ata_port *ap, u8 irq_stat)
862{
9af5c9c9 863 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, ap->link.active_tag);
2dec7555
RH		 864
865 /* freeze if hotplugged */
866 if (unlikely(irq_stat & (NV_INT_ADDED | NV_INT_REMOVED))) {
867 ata_port_freeze(ap);
868 return 1;
869 }
870
871 /* bail out if not our interrupt */
872 if (!(irq_stat & NV_INT_DEV))
873 return 0;
874
875 /* DEV interrupt w/ no active qc? */
876 if (unlikely(!qc || (qc->tf.flags & ATA_TFLAG_POLLING))) {
9363c382 877 ata_sff_check_status(ap);
2dec7555
RH		 878 return 1;
879 }
880
881 /* handle interrupt */
c3b28894 882 return ata_bmdma_port_intr(ap, qc);
2dec7555
RH		 883}
884
fbbb262d
RH		 885static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance)
886{
887 struct ata_host *host = dev_instance;
888 int i, handled = 0;
2dec7555 889 u32 notifier_clears[2];
fbbb262d
RH		 890
891 spin_lock(&host->lock);
892
893 for (i = 0; i < host->n_ports; i++) {
894 struct ata_port *ap = host->ports[i];
3e4ec344
TH		 895 struct nv_adma_port_priv *pp = ap->private_data;
896 void __iomem *mmio = pp->ctl_block;
897 u16 status;
898 u32 gen_ctl;
899 u32 notifier, notifier_error;
900
2dec7555 901 notifier_clears[i] = 0;
fbbb262d 902
3e4ec344
TH		 903 /* if ADMA is disabled, use standard ata interrupt handler */
904 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
905 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
906 >> (NV_INT_PORT_SHIFT * i);
907 handled += nv_host_intr(ap, irq_stat);
908 continue;
909 }
fbbb262d 910
3e4ec344
TH		 911 /* if in ATA register mode, check for standard interrupts */
912 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE) {
913 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
914 >> (NV_INT_PORT_SHIFT * i);
915 if (ata_tag_valid(ap->link.active_tag))
916 /** NV_INT_DEV indication seems unreliable
917 at times at least in ADMA mode. Force it
918 on always when a command is active, to
919 prevent losing interrupts. */
920 irq_stat |= NV_INT_DEV;
921 handled += nv_host_intr(ap, irq_stat);
922 }
923
924 notifier = readl(mmio + NV_ADMA_NOTIFIER);
925 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
926 notifier_clears[i] = notifier | notifier_error;
927
928 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
929
930 if (!NV_ADMA_CHECK_INTR(gen_ctl, ap->port_no) && !notifier &&
931 !notifier_error)
932 /* Nothing to do */
933 continue;
934
935 status = readw(mmio + NV_ADMA_STAT);
936
937 /*
938 * Clear status. Ensure the controller sees the
939 * clearing before we start looking at any of the CPB
940 * statuses, so that any CPB completions after this
941 * point in the handler will raise another interrupt.
942 */
943 writew(status, mmio + NV_ADMA_STAT);
944 readw(mmio + NV_ADMA_STAT); /* flush posted write */
945 rmb();
fbbb262d 946
3e4ec344
TH		 947 handled++; /* irq handled if we got here */
948
949 /* freeze if hotplugged or controller error */
950 if (unlikely(status & (NV_ADMA_STAT_HOTPLUG |
951 NV_ADMA_STAT_HOTUNPLUG |
952 NV_ADMA_STAT_TIMEOUT |
953 NV_ADMA_STAT_SERROR))) {
954 struct ata_eh_info *ehi = &ap->link.eh_info;
955
956 ata_ehi_clear_desc(ehi);
957 __ata_ehi_push_desc(ehi, "ADMA status 0x%08x: ", status);
958 if (status & NV_ADMA_STAT_TIMEOUT) {
959 ehi->err_mask |= AC_ERR_SYSTEM;
960 ata_ehi_push_desc(ehi, "timeout");
961 } else if (status & NV_ADMA_STAT_HOTPLUG) {
962 ata_ehi_hotplugged(ehi);
963 ata_ehi_push_desc(ehi, "hotplug");
964 } else if (status & NV_ADMA_STAT_HOTUNPLUG) {
965 ata_ehi_hotplugged(ehi);
966 ata_ehi_push_desc(ehi, "hot unplug");
967 } else if (status & NV_ADMA_STAT_SERROR) {
968 /* let EH analyze SError and figure out cause */
969 ata_ehi_push_desc(ehi, "SError");
970 } else
971 ata_ehi_push_desc(ehi, "unknown");
972 ata_port_freeze(ap);
973 continue;
974 }
975
976 if (status & (NV_ADMA_STAT_DONE |
977 NV_ADMA_STAT_CPBERR |
978 NV_ADMA_STAT_CMD_COMPLETE)) {
979 u32 check_commands = notifier_clears[i];
1aadf5c3 980 u32 done_mask = 0;
752e386c 981 int pos, rc;
3e4ec344
TH		 982
983 if (status & NV_ADMA_STAT_CPBERR) {
984 /* check all active commands */
985 if (ata_tag_valid(ap->link.active_tag))
986 check_commands = 1 <<
987 ap->link.active_tag;
988 else
989 check_commands = ap->link.sactive;
fbbb262d
RH		 990 }
991
3e4ec344 992 /* check CPBs for completed commands */
752e386c 993 while ((pos = ffs(check_commands))) {
3e4ec344 994 pos--;
752e386c 995 rc = nv_adma_check_cpb(ap, pos,
5796d1c4 996 notifier_error & (1 << pos));
1aadf5c3
TH		 997 if (rc > 0)
998 done_mask |= 1 << pos;
999 else if (unlikely(rc < 0))
752e386c 1000 check_commands = 0;
3e4ec344 1001 check_commands &= ~(1 << pos);
fbbb262d 1002 }
1aadf5c3 1003 ata_qc_complete_multiple(ap, ap->qc_active ^ done_mask);
fbbb262d
RH		 1004 }
1005 }
f20b16ff 1006
b447916e 1007 if (notifier_clears[0] || notifier_clears[1]) {
2dec7555
RH		 1008 /* Note: Both notifier clear registers must be written
1009 if either is set, even if one is zero, according to NVIDIA. */
cdf56bcf
RH		 1010 struct nv_adma_port_priv *pp = host->ports[0]->private_data;
1011 writel(notifier_clears[0], pp->notifier_clear_block);
1012 pp = host->ports[1]->private_data;
1013 writel(notifier_clears[1], pp->notifier_clear_block);
2dec7555 1014 }
fbbb262d
RH		 1015
1016 spin_unlock(&host->lock);
1017
1018 return IRQ_RETVAL(handled);
1019}
1020
53014e25
RH		 1021static void nv_adma_freeze(struct ata_port *ap)
1022{
1023 struct nv_adma_port_priv *pp = ap->private_data;
1024 void __iomem *mmio = pp->ctl_block;
1025 u16 tmp;
1026
1027 nv_ck804_freeze(ap);
1028
1029 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1030 return;
1031
1032 /* clear any outstanding CK804 notifications */
2dcb407e 1033 writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
53014e25
RH		 1034 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1035
1036 /* Disable interrupt */
1037 tmp = readw(mmio + NV_ADMA_CTL);
2dcb407e 1038 writew(tmp & ~(NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
53014e25 1039 mmio + NV_ADMA_CTL);
5796d1c4 1040 readw(mmio + NV_ADMA_CTL); /* flush posted write */
53014e25
RH		 1041}
1042
1043static void nv_adma_thaw(struct ata_port *ap)
1044{
1045 struct nv_adma_port_priv *pp = ap->private_data;
1046 void __iomem *mmio = pp->ctl_block;
1047 u16 tmp;
1048
1049 nv_ck804_thaw(ap);
1050
1051 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1052 return;
1053
1054 /* Enable interrupt */
1055 tmp = readw(mmio + NV_ADMA_CTL);
2dcb407e 1056 writew(tmp | (NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
53014e25 1057 mmio + NV_ADMA_CTL);
5796d1c4 1058 readw(mmio + NV_ADMA_CTL); /* flush posted write */
53014e25
RH		 1059}
1060
fbbb262d
RH		 1061static void nv_adma_irq_clear(struct ata_port *ap)
1062{
cdf56bcf
RH		 1063 struct nv_adma_port_priv *pp = ap->private_data;
1064 void __iomem *mmio = pp->ctl_block;
53014e25 1065 u32 notifier_clears[2];
fbbb262d 1066
53014e25 1067 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
37f65b8b 1068 ata_bmdma_irq_clear(ap);
53014e25
RH		 1069 return;
1070 }
1071
1072 /* clear any outstanding CK804 notifications */
2dcb407e 1073 writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
53014e25 1074 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
fbbb262d 1075
53014e25
RH		 1076 /* clear ADMA status */
1077 writew(0xffff, mmio + NV_ADMA_STAT);
a617c09f 1078
53014e25
RH		 1079 /* clear notifiers - note both ports need to be written with
1080 something even though we are only clearing on one */
1081 if (ap->port_no == 0) {
1082 notifier_clears[0] = 0xFFFFFFFF;
1083 notifier_clears[1] = 0;
1084 } else {
1085 notifier_clears[0] = 0;
1086 notifier_clears[1] = 0xFFFFFFFF;
1087 }
1088 pp = ap->host->ports[0]->private_data;
1089 writel(notifier_clears[0], pp->notifier_clear_block);
1090 pp = ap->host->ports[1]->private_data;
1091 writel(notifier_clears[1], pp->notifier_clear_block);
fbbb262d
RH		 1092}
1093
f5ecac2d 1094static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc)
fbbb262d 1095{
f5ecac2d 1096 struct nv_adma_port_priv *pp = qc->ap->private_data;
fbbb262d 1097
b447916e 1098 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
fe06e5f9 1099 ata_bmdma_post_internal_cmd(qc);
fbbb262d
RH		 1100}
1101
1102static int nv_adma_port_start(struct ata_port *ap)
1103{
1104 struct device *dev = ap->host->dev;
1105 struct nv_adma_port_priv *pp;
1106 int rc;
1107 void *mem;
1108 dma_addr_t mem_dma;
cdf56bcf 1109 void __iomem *mmio;
8959d300 1110 struct pci_dev *pdev = to_pci_dev(dev);
fbbb262d
RH		 1111 u16 tmp;
1112
1113 VPRINTK("ENTER\n");
1114
258c9fde
CH		 1115 /*
1116 * Ensure DMA mask is set to 32-bit before allocating legacy PRD and
1117 * pad buffers.
1118 */
1119 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
8959d300
RH		 1120 if (rc)
1121 return rc;
1122
c7087652
TH		 1123 /* we might fallback to bmdma, allocate bmdma resources */
1124 rc = ata_bmdma_port_start(ap);
fbbb262d
RH		 1125 if (rc)
1126 return rc;
1127
24dc5f33
TH		 1128 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1129 if (!pp)
1130 return -ENOMEM;
fbbb262d 1131
0d5ff566 1132 mmio = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_PORT +
cdf56bcf
RH		 1133 ap->port_no * NV_ADMA_PORT_SIZE;
1134 pp->ctl_block = mmio;
0d5ff566 1135 pp->gen_block = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_GEN;
cdf56bcf
RH		 1136 pp->notifier_clear_block = pp->gen_block +
1137 NV_ADMA_NOTIFIER_CLEAR + (4 * ap->port_no);
1138
258c9fde
CH		 1139 /*
1140 * Now that the legacy PRD and padding buffer are allocated we can
1141 * try to raise the DMA mask to allocate the CPB/APRD table.
1142 */
1143 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1144 if (rc) {
1145 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1146 if (rc)
1147 return rc;
1148 }
8959d300
RH		 1149 pp->adma_dma_mask = *dev->dma_mask;
1150
24dc5f33
TH		 1151 mem = dmam_alloc_coherent(dev, NV_ADMA_PORT_PRIV_DMA_SZ,
1152 &mem_dma, GFP_KERNEL);
1153 if (!mem)
1154 return -ENOMEM;
fbbb262d
RH		 1155 memset(mem, 0, NV_ADMA_PORT_PRIV_DMA_SZ);
1156
1157 /*
1158 * First item in chunk of DMA memory:
1159 * 128-byte command parameter block (CPB)
1160 * one for each command tag
1161 */
1162 pp->cpb = mem;
1163 pp->cpb_dma = mem_dma;
1164
1165 writel(mem_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
5796d1c4 1166 writel((mem_dma >> 16) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
fbbb262d
RH		 1167
1168 mem += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1169 mem_dma += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1170
1171 /*
1172 * Second item: block of ADMA_SGTBL_LEN s/g entries
1173 */
1174 pp->aprd = mem;
1175 pp->aprd_dma = mem_dma;
1176
1177 ap->private_data = pp;
1178
1179 /* clear any outstanding interrupt conditions */
1180 writew(0xffff, mmio + NV_ADMA_STAT);
1181
1182 /* initialize port variables */
1183 pp->flags = NV_ADMA_PORT_REGISTER_MODE;
1184
1185 /* clear CPB fetch count */
1186 writew(0, mmio + NV_ADMA_CPB_COUNT);
1187
cdf56bcf 1188 /* clear GO for register mode, enable interrupt */
fbbb262d 1189 tmp = readw(mmio + NV_ADMA_CTL);
5796d1c4
JG		 1190 writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1191 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
fbbb262d
RH		 1192
1193 tmp = readw(mmio + NV_ADMA_CTL);
1194 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
5796d1c4 1195 readw(mmio + NV_ADMA_CTL); /* flush posted write */
fbbb262d
RH		 1196 udelay(1);
1197 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
5796d1c4 1198 readw(mmio + NV_ADMA_CTL); /* flush posted write */
fbbb262d
RH		 1199
1200 return 0;
fbbb262d
RH		 1201}
1202
1203static void nv_adma_port_stop(struct ata_port *ap)
1204{
fbbb262d 1205 struct nv_adma_port_priv *pp = ap->private_data;
cdf56bcf 1206 void __iomem *mmio = pp->ctl_block;
fbbb262d
RH		 1207
1208 VPRINTK("ENTER\n");
fbbb262d 1209 writew(0, mmio + NV_ADMA_CTL);
fbbb262d
RH		 1210}
1211
438ac6d5 1212#ifdef CONFIG_PM
cdf56bcf
RH		 1213static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg)
1214{
1215 struct nv_adma_port_priv *pp = ap->private_data;
1216 void __iomem *mmio = pp->ctl_block;
1217
1218 /* Go to register mode - clears GO */
1219 nv_adma_register_mode(ap);
1220
1221 /* clear CPB fetch count */
1222 writew(0, mmio + NV_ADMA_CPB_COUNT);
1223
1224 /* disable interrupt, shut down port */
1225 writew(0, mmio + NV_ADMA_CTL);
1226
1227 return 0;
1228}
1229
1230static int nv_adma_port_resume(struct ata_port *ap)
1231{
1232 struct nv_adma_port_priv *pp = ap->private_data;
1233 void __iomem *mmio = pp->ctl_block;
1234 u16 tmp;
1235
1236 /* set CPB block location */
1237 writel(pp->cpb_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
5796d1c4 1238 writel((pp->cpb_dma >> 16) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
cdf56bcf
RH		 1239
1240 /* clear any outstanding interrupt conditions */
1241 writew(0xffff, mmio + NV_ADMA_STAT);
1242
1243 /* initialize port variables */
1244 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
1245
1246 /* clear CPB fetch count */
1247 writew(0, mmio + NV_ADMA_CPB_COUNT);
1248
1249 /* clear GO for register mode, enable interrupt */
1250 tmp = readw(mmio + NV_ADMA_CTL);
5796d1c4
JG		 1251 writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1252 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
cdf56bcf
RH		 1253
1254 tmp = readw(mmio + NV_ADMA_CTL);
1255 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
5796d1c4 1256 readw(mmio + NV_ADMA_CTL); /* flush posted write */
cdf56bcf
RH		 1257 udelay(1);
1258 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
5796d1c4 1259 readw(mmio + NV_ADMA_CTL); /* flush posted write */
cdf56bcf
RH		 1260
1261 return 0;
1262}
438ac6d5 1263#endif
fbbb262d 1264
9a829ccf 1265static void nv_adma_setup_port(struct ata_port *ap)
fbbb262d 1266{
9a829ccf
TH		 1267 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1268 struct ata_ioports *ioport = &ap->ioaddr;
fbbb262d
RH		 1269
1270 VPRINTK("ENTER\n");
1271
9a829ccf 1272 mmio += NV_ADMA_PORT + ap->port_no * NV_ADMA_PORT_SIZE;
fbbb262d 1273
0d5ff566
TH		 1274 ioport->cmd_addr = mmio;
1275 ioport->data_addr = mmio + (ATA_REG_DATA * 4);
fbbb262d 1276 ioport->error_addr =
0d5ff566
TH		 1277 ioport->feature_addr = mmio + (ATA_REG_ERR * 4);
1278 ioport->nsect_addr = mmio + (ATA_REG_NSECT * 4);
1279 ioport->lbal_addr = mmio + (ATA_REG_LBAL * 4);
1280 ioport->lbam_addr = mmio + (ATA_REG_LBAM * 4);
1281 ioport->lbah_addr = mmio + (ATA_REG_LBAH * 4);
1282 ioport->device_addr = mmio + (ATA_REG_DEVICE * 4);
fbbb262d 1283 ioport->status_addr =
0d5ff566 1284 ioport->command_addr = mmio + (ATA_REG_STATUS * 4);
fbbb262d 1285 ioport->altstatus_addr =
0d5ff566 1286 ioport->ctl_addr = mmio + 0x20;
fbbb262d
RH		 1287}
1288
9a829ccf 1289static int nv_adma_host_init(struct ata_host *host)
fbbb262d 1290{
9a829ccf 1291 struct pci_dev *pdev = to_pci_dev(host->dev);
fbbb262d
RH		 1292 unsigned int i;
1293 u32 tmp32;
1294
1295 VPRINTK("ENTER\n");
1296
1297 /* enable ADMA on the ports */
1298 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
1299 tmp32 |= NV_MCP_SATA_CFG_20_PORT0_EN |
1300 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
1301 NV_MCP_SATA_CFG_20_PORT1_EN |
1302 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
1303
1304 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
1305
9a829ccf
TH		 1306 for (i = 0; i < host->n_ports; i++)
1307 nv_adma_setup_port(host->ports[i]);
fbbb262d 1308
fbbb262d
RH		 1309 return 0;
1310}
1311
1312static void nv_adma_fill_aprd(struct ata_queued_cmd *qc,
1313 struct scatterlist *sg,
1314 int idx,
1315 struct nv_adma_prd *aprd)
1316{
41949ed5 1317 u8 flags = 0;
fbbb262d
RH		 1318 if (qc->tf.flags & ATA_TFLAG_WRITE)
1319 flags |= NV_APRD_WRITE;
1320 if (idx == qc->n_elem - 1)
1321 flags |= NV_APRD_END;
1322 else if (idx != 4)
1323 flags |= NV_APRD_CONT;
1324
1325 aprd->addr = cpu_to_le64(((u64)sg_dma_address(sg)));
1326 aprd->len = cpu_to_le32(((u32)sg_dma_len(sg))); /* len in bytes */
2dec7555 1327 aprd->flags = flags;
41949ed5 1328 aprd->packet_len = 0;
fbbb262d
RH		 1329}
1330
1331static void nv_adma_fill_sg(struct ata_queued_cmd *qc, struct nv_adma_cpb *cpb)
1332{
1333 struct nv_adma_port_priv *pp = qc->ap->private_data;
fbbb262d
RH		 1334 struct nv_adma_prd *aprd;
1335 struct scatterlist *sg;
ff2aeb1e 1336 unsigned int si;
fbbb262d
RH		 1337
1338 VPRINTK("ENTER\n");
1339
ff2aeb1e
TH		 1340 for_each_sg(qc->sg, sg, qc->n_elem, si) {
1341 aprd = (si < 5) ? &cpb->aprd[si] :
4e5b6260 1342 &pp->aprd[NV_ADMA_SGTBL_LEN * qc->hw_tag + (si-5)];
ff2aeb1e 1343 nv_adma_fill_aprd(qc, sg, si, aprd);
fbbb262d 1344 }
ff2aeb1e 1345 if (si > 5)
4e5b6260 1346 cpb->next_aprd = cpu_to_le64(((u64)(pp->aprd_dma + NV_ADMA_SGTBL_SZ * qc->hw_tag)));
41949ed5
RH		 1347 else
1348 cpb->next_aprd = cpu_to_le64(0);
fbbb262d
RH		 1349}
1350
382a6652
RH		 1351static int nv_adma_use_reg_mode(struct ata_queued_cmd *qc)
1352{
1353 struct nv_adma_port_priv *pp = qc->ap->private_data;
1354
1355 /* ADMA engine can only be used for non-ATAPI DMA commands,
3f3debdb 1356 or interrupt-driven no-data commands. */
b447916e 1357 if ((pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
3f3debdb 1358 (qc->tf.flags & ATA_TFLAG_POLLING))
382a6652
RH		 1359 return 1;
1360
b447916e 1361 if ((qc->flags & ATA_QCFLAG_DMAMAP) ||
382a6652
RH		 1362 (qc->tf.protocol == ATA_PROT_NODATA))
1363 return 0;
1364
1365 return 1;
1366}
1367
fbbb262d
RH		 1368static void nv_adma_qc_prep(struct ata_queued_cmd *qc)
1369{
1370 struct nv_adma_port_priv *pp = qc->ap->private_data;
4e5b6260 1371 struct nv_adma_cpb *cpb = &pp->cpb[qc->hw_tag];
fbbb262d 1372 u8 ctl_flags = NV_CPB_CTL_CPB_VALID |
fbbb262d
RH		 1373 NV_CPB_CTL_IEN;
1374
382a6652 1375 if (nv_adma_use_reg_mode(qc)) {
3f3debdb
RH		 1376 BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&
1377 (qc->flags & ATA_QCFLAG_DMAMAP));
2dec7555 1378 nv_adma_register_mode(qc->ap);
f47451c4 1379 ata_bmdma_qc_prep(qc);
fbbb262d
RH		 1380 return;
1381 }
1382
41949ed5
RH		 1383 cpb->resp_flags = NV_CPB_RESP_DONE;
1384 wmb();
1385 cpb->ctl_flags = 0;
1386 wmb();
fbbb262d
RH		 1387
1388 cpb->len = 3;
4e5b6260 1389 cpb->tag = qc->hw_tag;
fbbb262d
RH		 1390 cpb->next_cpb_idx = 0;
1391
1392 /* turn on NCQ flags for NCQ commands */
1393 if (qc->tf.protocol == ATA_PROT_NCQ)
1394 ctl_flags |= NV_CPB_CTL_QUEUE | NV_CPB_CTL_FPDMA;
1395
cdf56bcf
RH		 1396 VPRINTK("qc->flags = 0x%lx\n", qc->flags);
1397
fbbb262d
RH		 1398 nv_adma_tf_to_cpb(&qc->tf, cpb->tf);
1399
b447916e 1400 if (qc->flags & ATA_QCFLAG_DMAMAP) {
382a6652
RH		 1401 nv_adma_fill_sg(qc, cpb);
1402 ctl_flags |= NV_CPB_CTL_APRD_VALID;
1403 } else
1404 memset(&cpb->aprd[0], 0, sizeof(struct nv_adma_prd) * 5);
fbbb262d 1405
5796d1c4
JG		 1406 /* Be paranoid and don't let the device see NV_CPB_CTL_CPB_VALID
1407 until we are finished filling in all of the contents */
fbbb262d
RH		 1408 wmb();
1409 cpb->ctl_flags = ctl_flags;
41949ed5
RH		 1410 wmb();
1411 cpb->resp_flags = 0;
fbbb262d
RH		 1412}
1413
1414static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc)
1415{
2dec7555 1416 struct nv_adma_port_priv *pp = qc->ap->private_data;
cdf56bcf 1417 void __iomem *mmio = pp->ctl_block;
5e5c74a5 1418 int curr_ncq = (qc->tf.protocol == ATA_PROT_NCQ);
fbbb262d
RH		 1419
1420 VPRINTK("ENTER\n");
1421
3f3debdb
RH		 1422 /* We can't handle result taskfile with NCQ commands, since
1423 retrieving the taskfile switches us out of ADMA mode and would abort
1424 existing commands. */
1425 if (unlikely(qc->tf.protocol == ATA_PROT_NCQ &&
1426 (qc->flags & ATA_QCFLAG_RESULT_TF))) {
a9a79dfe 1427 ata_dev_err(qc->dev, "NCQ w/ RESULT_TF not allowed\n");
3f3debdb
RH		 1428 return AC_ERR_SYSTEM;
1429 }
1430
382a6652 1431 if (nv_adma_use_reg_mode(qc)) {
fbbb262d 1432 /* use ATA register mode */
382a6652 1433 VPRINTK("using ATA register mode: 0x%lx\n", qc->flags);
3f3debdb
RH		 1434 BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&
1435 (qc->flags & ATA_QCFLAG_DMAMAP));
fbbb262d 1436 nv_adma_register_mode(qc->ap);
360ff783 1437 return ata_bmdma_qc_issue(qc);
fbbb262d
RH		 1438 } else
1439 nv_adma_mode(qc->ap);
1440
1441 /* write append register, command tag in lower 8 bits
1442 and (number of cpbs to append -1) in top 8 bits */
1443 wmb();
5e5c74a5 1444
b447916e 1445 if (curr_ncq != pp->last_issue_ncq) {
5796d1c4
JG		 1446 /* Seems to need some delay before switching between NCQ and
1447 non-NCQ commands, else we get command timeouts and such. */
5e5c74a5
RH		 1448 udelay(20);
1449 pp->last_issue_ncq = curr_ncq;
1450 }
1451
4e5b6260 1452 writew(qc->hw_tag, mmio + NV_ADMA_APPEND);
fbbb262d 1453
4e5b6260 1454 DPRINTK("Issued tag %u\n", qc->hw_tag);
fbbb262d
RH		 1455
1456 return 0;
1457}
1458
7d12e780 1459static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance)
1da177e4 1460{
cca3974e 1461 struct ata_host *host = dev_instance;
1da177e4
LT		 1462 unsigned int i;
1463 unsigned int handled = 0;
1464 unsigned long flags;
1465
cca3974e 1466 spin_lock_irqsave(&host->lock, flags);
1da177e4 1467
cca3974e 1468 for (i = 0; i < host->n_ports; i++) {
3e4ec344
TH		 1469 struct ata_port *ap = host->ports[i];
1470 struct ata_queued_cmd *qc;
1da177e4 1471
3e4ec344
TH		 1472 qc = ata_qc_from_tag(ap, ap->link.active_tag);
1473 if (qc && (!(qc->tf.flags & ATA_TFLAG_POLLING))) {
c3b28894 1474 handled += ata_bmdma_port_intr(ap, qc);
3e4ec344
TH		 1475 } else {
1476 /*
1477 * No request pending? Clear interrupt status
1478 * anyway, in case there's one pending.
1479 */
1480 ap->ops->sff_check_status(ap);
1481 }
1da177e4
LT		 1482 }
1483
cca3974e 1484 spin_unlock_irqrestore(&host->lock, flags);
1da177e4
LT		 1485
1486 return IRQ_RETVAL(handled);
1487}
1488
cca3974e 1489static irqreturn_t nv_do_interrupt(struct ata_host *host, u8 irq_stat)
ada364e8
TH		 1490{
1491 int i, handled = 0;
1492
cca3974e 1493 for (i = 0; i < host->n_ports; i++) {
3e4ec344 1494 handled += nv_host_intr(host->ports[i], irq_stat);
ada364e8
TH		 1495 irq_stat >>= NV_INT_PORT_SHIFT;
1496 }
1497
1498 return IRQ_RETVAL(handled);
1499}
1500
7d12e780 1501static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance)
ada364e8 1502{
cca3974e 1503 struct ata_host *host = dev_instance;
ada364e8
TH		 1504 u8 irq_stat;
1505 irqreturn_t ret;
1506
cca3974e 1507 spin_lock(&host->lock);
0d5ff566 1508 irq_stat = ioread8(host->ports[0]->ioaddr.scr_addr + NV_INT_STATUS);
cca3974e
JG		 1509 ret = nv_do_interrupt(host, irq_stat);
1510 spin_unlock(&host->lock);
ada364e8
TH		 1511
1512 return ret;
1513}
1514
7d12e780 1515static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance)
ada364e8 1516{
cca3974e 1517 struct ata_host *host = dev_instance;
ada364e8
TH		 1518 u8 irq_stat;
1519 irqreturn_t ret;
1520
cca3974e 1521 spin_lock(&host->lock);
0d5ff566 1522 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
cca3974e
JG		 1523 ret = nv_do_interrupt(host, irq_stat);
1524 spin_unlock(&host->lock);
ada364e8
TH		 1525
1526 return ret;
1527}
1528
82ef04fb 1529static int nv_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val)
1da177e4 1530{
1da177e4 1531 if (sc_reg > SCR_CONTROL)
da3dbb17 1532 return -EINVAL;
1da177e4 1533
82ef04fb 1534 *val = ioread32(link->ap->ioaddr.scr_addr + (sc_reg * 4));
da3dbb17 1535 return 0;
1da177e4
LT		 1536}
1537
82ef04fb 1538static int nv_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val)
1da177e4 1539{
1da177e4 1540 if (sc_reg > SCR_CONTROL)
da3dbb17 1541 return -EINVAL;
1da177e4 1542
82ef04fb 1543 iowrite32(val, link->ap->ioaddr.scr_addr + (sc_reg * 4));
da3dbb17 1544 return 0;
1da177e4
LT		 1545}
1546
7f4774b3
TH		 1547static int nv_hardreset(struct ata_link *link, unsigned int *class,
1548 unsigned long deadline)
e8caa3c7 1549{
7f4774b3 1550 struct ata_eh_context *ehc = &link->eh_context;
e8caa3c7 1551
7f4774b3
TH		 1552 /* Do hardreset iff it's post-boot probing, please read the
1553 * comment above port ops for details.
1554 */
1555 if (!(link->ap->pflags & ATA_PFLAG_LOADING) &&
1556 !ata_dev_enabled(link->device))
1557 sata_link_hardreset(link, sata_deb_timing_hotplug, deadline,
1558 NULL, NULL);
6489e326
TH		 1559 else {
1560 const unsigned long *timing = sata_ehc_deb_timing(ehc);
1561 int rc;
1562
1563 if (!(ehc->i.flags & ATA_EHI_QUIET))
a9a79dfe
JP		 1564 ata_link_info(link,
1565 "nv: skipping hardreset on occupied port\n");
6489e326
TH		 1566
1567 /* make sure the link is online */
1568 rc = sata_link_resume(link, timing, deadline);
1569 /* whine about phy resume failure but proceed */
1570 if (rc && rc != -EOPNOTSUPP)
a9a79dfe
JP		 1571 ata_link_warn(link, "failed to resume link (errno=%d)\n",
1572 rc);
6489e326 1573 }
7f4774b3
TH		 1574
1575 /* device signature acquisition is unreliable */
1576 return -EAGAIN;
e8caa3c7
TH		 1577}
1578
39f87582
TH		 1579static void nv_nf2_freeze(struct ata_port *ap)
1580{
0d5ff566 1581 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
39f87582
TH		 1582 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1583 u8 mask;
1584
0d5ff566 1585 mask = ioread8(scr_addr + NV_INT_ENABLE);
39f87582 1586 mask &= ~(NV_INT_ALL << shift);
0d5ff566 1587 iowrite8(mask, scr_addr + NV_INT_ENABLE);
39f87582
TH		 1588}
1589
1590static void nv_nf2_thaw(struct ata_port *ap)
1591{
0d5ff566 1592 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
39f87582
TH		 1593 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1594 u8 mask;
1595
0d5ff566 1596 iowrite8(NV_INT_ALL << shift, scr_addr + NV_INT_STATUS);
39f87582 1597
0d5ff566 1598 mask = ioread8(scr_addr + NV_INT_ENABLE);
39f87582 1599 mask |= (NV_INT_MASK << shift);
0d5ff566 1600 iowrite8(mask, scr_addr + NV_INT_ENABLE);
39f87582
TH		 1601}
1602
1603static void nv_ck804_freeze(struct ata_port *ap)
1604{
0d5ff566 1605 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
39f87582
TH		 1606 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1607 u8 mask;
1608
1609 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1610 mask &= ~(NV_INT_ALL << shift);
1611 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1612}
1613
1614static void nv_ck804_thaw(struct ata_port *ap)
1615{
0d5ff566 1616 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
39f87582
TH		 1617 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1618 u8 mask;
1619
1620 writeb(NV_INT_ALL << shift, mmio_base + NV_INT_STATUS_CK804);
1621
1622 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1623 mask |= (NV_INT_MASK << shift);
1624 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1625}
1626
f140f0f1
KL		 1627static void nv_mcp55_freeze(struct ata_port *ap)
1628{
1629 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1630 int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1631 u32 mask;
1632
1633 writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1634
1635 mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1636 mask &= ~(NV_INT_ALL_MCP55 << shift);
1637 writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
f140f0f1
KL		 1638}
1639
1640static void nv_mcp55_thaw(struct ata_port *ap)
1641{
1642 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1643 int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1644 u32 mask;
1645
1646 writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1647
1648 mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1649 mask |= (NV_INT_MASK_MCP55 << shift);
1650 writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
f140f0f1
KL		 1651}
1652
fbbb262d
RH		 1653static void nv_adma_error_handler(struct ata_port *ap)
1654{
1655 struct nv_adma_port_priv *pp = ap->private_data;
b447916e 1656 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) {
cdf56bcf 1657 void __iomem *mmio = pp->ctl_block;
fbbb262d
RH		 1658 int i;
1659 u16 tmp;
a84471fe 1660
b447916e 1661 if (ata_tag_valid(ap->link.active_tag) || ap->link.sactive) {
2cb27853
RH		 1662 u32 notifier = readl(mmio + NV_ADMA_NOTIFIER);
1663 u32 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
1664 u32 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
1665 u32 status = readw(mmio + NV_ADMA_STAT);
08af7414
RH		 1666 u8 cpb_count = readb(mmio + NV_ADMA_CPB_COUNT);
1667 u8 next_cpb_idx = readb(mmio + NV_ADMA_NEXT_CPB_IDX);
2cb27853 1668
a9a79dfe 1669 ata_port_err(ap,
5796d1c4 1670 "EH in ADMA mode, notifier 0x%X "
08af7414
RH		 1671 "notifier_error 0x%X gen_ctl 0x%X status 0x%X "
1672 "next cpb count 0x%X next cpb idx 0x%x\n",
1673 notifier, notifier_error, gen_ctl, status,
1674 cpb_count, next_cpb_idx);
2cb27853 1675
b447916e 1676 for (i = 0; i < NV_ADMA_MAX_CPBS; i++) {
2cb27853 1677 struct nv_adma_cpb *cpb = &pp->cpb[i];
b447916e 1678 if ((ata_tag_valid(ap->link.active_tag) && i == ap->link.active_tag) ||
5796d1c4 1679 ap->link.sactive & (1 << i))
a9a79dfe 1680 ata_port_err(ap,
2cb27853
RH		 1681 "CPB %d: ctl_flags 0x%x, resp_flags 0x%x\n",
1682 i, cpb->ctl_flags, cpb->resp_flags);
1683 }
1684 }
fbbb262d 1685
fbbb262d
RH		 1686 /* Push us back into port register mode for error handling. */
1687 nv_adma_register_mode(ap);
1688
5796d1c4
JG		 1689 /* Mark all of the CPBs as invalid to prevent them from
1690 being executed */
b447916e 1691 for (i = 0; i < NV_ADMA_MAX_CPBS; i++)
fbbb262d
RH		 1692 pp->cpb[i].ctl_flags &= ~NV_CPB_CTL_CPB_VALID;
1693
1694 /* clear CPB fetch count */
1695 writew(0, mmio + NV_ADMA_CPB_COUNT);
1696
1697 /* Reset channel */
1698 tmp = readw(mmio + NV_ADMA_CTL);
1699 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
b447916e 1700 readw(mmio + NV_ADMA_CTL); /* flush posted write */
fbbb262d
RH		 1701 udelay(1);
1702 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
b447916e 1703 readw(mmio + NV_ADMA_CTL); /* flush posted write */
fbbb262d
RH		 1704 }
1705
fe06e5f9 1706 ata_bmdma_error_handler(ap);
fbbb262d
RH		 1707}
1708
f140f0f1
KL		 1709static void nv_swncq_qc_to_dq(struct ata_port *ap, struct ata_queued_cmd *qc)
1710{
1711 struct nv_swncq_port_priv *pp = ap->private_data;
1712 struct defer_queue *dq = &pp->defer_queue;
1713
1714 /* queue is full */
1715 WARN_ON(dq->tail - dq->head == ATA_MAX_QUEUE);
4e5b6260
JA		 1716 dq->defer_bits |= (1 << qc->hw_tag);
1717 dq->tag[dq->tail++ & (ATA_MAX_QUEUE - 1)] = qc->hw_tag;
f140f0f1
KL		 1718}
1719
1720static struct ata_queued_cmd *nv_swncq_qc_from_dq(struct ata_port *ap)
1721{
1722 struct nv_swncq_port_priv *pp = ap->private_data;
1723 struct defer_queue *dq = &pp->defer_queue;
1724 unsigned int tag;
1725
1726 if (dq->head == dq->tail) /* null queue */
1727 return NULL;
1728
1729 tag = dq->tag[dq->head & (ATA_MAX_QUEUE - 1)];
1730 dq->tag[dq->head++ & (ATA_MAX_QUEUE - 1)] = ATA_TAG_POISON;
1731 WARN_ON(!(dq->defer_bits & (1 << tag)));
1732 dq->defer_bits &= ~(1 << tag);
1733
1734 return ata_qc_from_tag(ap, tag);
1735}
1736
1737static void nv_swncq_fis_reinit(struct ata_port *ap)
1738{
1739 struct nv_swncq_port_priv *pp = ap->private_data;
1740
1741 pp->dhfis_bits = 0;
1742 pp->dmafis_bits = 0;
1743 pp->sdbfis_bits = 0;
1744 pp->ncq_flags = 0;
1745}
1746
1747static void nv_swncq_pp_reinit(struct ata_port *ap)
1748{
1749 struct nv_swncq_port_priv *pp = ap->private_data;
1750 struct defer_queue *dq = &pp->defer_queue;
1751
1752 dq->head = 0;
1753 dq->tail = 0;
1754 dq->defer_bits = 0;
1755 pp->qc_active = 0;
1756 pp->last_issue_tag = ATA_TAG_POISON;
1757 nv_swncq_fis_reinit(ap);
1758}
1759
1760static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis)
1761{
1762 struct nv_swncq_port_priv *pp = ap->private_data;
1763
1764 writew(fis, pp->irq_block);
1765}
1766
1767static void __ata_bmdma_stop(struct ata_port *ap)
1768{
1769 struct ata_queued_cmd qc;
1770
1771 qc.ap = ap;
1772 ata_bmdma_stop(&qc);
1773}
1774
1775static void nv_swncq_ncq_stop(struct ata_port *ap)
1776{
1777 struct nv_swncq_port_priv *pp = ap->private_data;
1778 unsigned int i;
1779 u32 sactive;
1780 u32 done_mask;
1781
e3ed8939 1782 ata_port_err(ap, "EH in SWNCQ mode,QC:qc_active 0x%llX sactive 0x%X\n",
a9a79dfe
JP		 1783 ap->qc_active, ap->link.sactive);
1784 ata_port_err(ap,
f140f0f1
KL		 1785 "SWNCQ:qc_active 0x%X defer_bits 0x%X last_issue_tag 0x%x\n "
1786 "dhfis 0x%X dmafis 0x%X sdbfis 0x%X\n",
1787 pp->qc_active, pp->defer_queue.defer_bits, pp->last_issue_tag,
1788 pp->dhfis_bits, pp->dmafis_bits, pp->sdbfis_bits);
1789
a9a79dfe
JP		 1790 ata_port_err(ap, "ATA_REG 0x%X ERR_REG 0x%X\n",
1791 ap->ops->sff_check_status(ap),
1792 ioread8(ap->ioaddr.error_addr));
f140f0f1
KL		 1793
1794 sactive = readl(pp->sactive_block);
1795 done_mask = pp->qc_active ^ sactive;
1796
a9a79dfe 1797 ata_port_err(ap, "tag : dhfis dmafis sdbfis sactive\n");
f140f0f1
KL		 1798 for (i = 0; i < ATA_MAX_QUEUE; i++) {
1799 u8 err = 0;
1800 if (pp->qc_active & (1 << i))
1801 err = 0;
1802 else if (done_mask & (1 << i))
1803 err = 1;
1804 else
1805 continue;
1806
a9a79dfe
JP		 1807 ata_port_err(ap,
1808 "tag 0x%x: %01x %01x %01x %01x %s\n", i,
1809 (pp->dhfis_bits >> i) & 0x1,
1810 (pp->dmafis_bits >> i) & 0x1,
1811 (pp->sdbfis_bits >> i) & 0x1,
1812 (sactive >> i) & 0x1,
1813 (err ? "error! tag doesn't exit" : " "));
f140f0f1
KL		 1814 }
1815
1816 nv_swncq_pp_reinit(ap);
5682ed33 1817 ap->ops->sff_irq_clear(ap);
f140f0f1
KL		 1818 __ata_bmdma_stop(ap);
1819 nv_swncq_irq_clear(ap, 0xffff);
1820}
1821
1822static void nv_swncq_error_handler(struct ata_port *ap)
1823{
1824 struct ata_eh_context *ehc = &ap->link.eh_context;
1825
1826 if (ap->link.sactive) {
1827 nv_swncq_ncq_stop(ap);
cf480626 1828 ehc->i.action |= ATA_EH_RESET;
f140f0f1
KL		 1829 }
1830
fe06e5f9 1831 ata_bmdma_error_handler(ap);
f140f0f1
KL		 1832}
1833
1834#ifdef CONFIG_PM
1835static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg)
1836{
1837 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1838 u32 tmp;
1839
1840 /* clear irq */
1841 writel(~0, mmio + NV_INT_STATUS_MCP55);
1842
1843 /* disable irq */
1844 writel(0, mmio + NV_INT_ENABLE_MCP55);
1845
1846 /* disable swncq */
1847 tmp = readl(mmio + NV_CTL_MCP55);
1848 tmp &= ~(NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ);
1849 writel(tmp, mmio + NV_CTL_MCP55);
1850
1851 return 0;
1852}
1853
1854static int nv_swncq_port_resume(struct ata_port *ap)
1855{
1856 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1857 u32 tmp;
1858
1859 /* clear irq */
1860 writel(~0, mmio + NV_INT_STATUS_MCP55);
1861
1862 /* enable irq */
1863 writel(0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1864
1865 /* enable swncq */
1866 tmp = readl(mmio + NV_CTL_MCP55);
1867 writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1868
1869 return 0;
1870}
1871#endif
1872
1873static void nv_swncq_host_init(struct ata_host *host)
1874{
1875 u32 tmp;
1876 void __iomem *mmio = host->iomap[NV_MMIO_BAR];
1877 struct pci_dev *pdev = to_pci_dev(host->dev);
1878 u8 regval;
1879
1880 /* disable ECO 398 */
1881 pci_read_config_byte(pdev, 0x7f, &regval);
1882 regval &= ~(1 << 7);
1883 pci_write_config_byte(pdev, 0x7f, regval);
1884
1885 /* enable swncq */
1886 tmp = readl(mmio + NV_CTL_MCP55);
1887 VPRINTK("HOST_CTL:0x%X\n", tmp);
1888 writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1889
1890 /* enable irq intr */
1891 tmp = readl(mmio + NV_INT_ENABLE_MCP55);
1892 VPRINTK("HOST_ENABLE:0x%X\n", tmp);
1893 writel(tmp | 0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1894
1895 /* clear port irq */
1896 writel(~0x0, mmio + NV_INT_STATUS_MCP55);
1897}
1898
1899static int nv_swncq_slave_config(struct scsi_device *sdev)
1900{
1901 struct ata_port *ap = ata_shost_to_port(sdev->host);
1902 struct pci_dev *pdev = to_pci_dev(ap->host->dev);
1903 struct ata_device *dev;
1904 int rc;
1905 u8 rev;
1906 u8 check_maxtor = 0;
1907 unsigned char model_num[ATA_ID_PROD_LEN + 1];
1908
1909 rc = ata_scsi_slave_config(sdev);
1910 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
1911 /* Not a proper libata device, ignore */
1912 return rc;
1913
1914 dev = &ap->link.device[sdev->id];
1915 if (!(ap->flags & ATA_FLAG_NCQ) || dev->class == ATA_DEV_ATAPI)
1916 return rc;
1917
1918 /* if MCP51 and Maxtor, then disable ncq */
1919 if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA ||
1920 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2)
1921 check_maxtor = 1;
1922
1923 /* if MCP55 and rev <= a2 and Maxtor, then disable ncq */
1924 if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA ||
1925 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2) {
1926 pci_read_config_byte(pdev, 0x8, &rev);
1927 if (rev <= 0xa2)
1928 check_maxtor = 1;
1929 }
1930
1931 if (!check_maxtor)
1932 return rc;
1933
1934 ata_id_c_string(dev->id, model_num, ATA_ID_PROD, sizeof(model_num));
1935
1936 if (strncmp(model_num, "Maxtor", 6) == 0) {
db5ed4df 1937 ata_scsi_change_queue_depth(sdev, 1);
a9a79dfe
JP		 1938 ata_dev_notice(dev, "Disabling SWNCQ mode (depth %x)\n",
1939 sdev->queue_depth);
f140f0f1
KL		 1940 }
1941
1942 return rc;
1943}
1944
1945static int nv_swncq_port_start(struct ata_port *ap)
1946{
1947 struct device *dev = ap->host->dev;
1948 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1949 struct nv_swncq_port_priv *pp;
1950 int rc;
1951
c7087652
TH		 1952 /* we might fallback to bmdma, allocate bmdma resources */
1953 rc = ata_bmdma_port_start(ap);
f140f0f1
KL		 1954 if (rc)
1955 return rc;
1956
1957 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1958 if (!pp)
1959 return -ENOMEM;
1960
1961 pp->prd = dmam_alloc_coherent(dev, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE,
1962 &pp->prd_dma, GFP_KERNEL);
1963 if (!pp->prd)
1964 return -ENOMEM;
1965 memset(pp->prd, 0, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE);
1966
1967 ap->private_data = pp;
1968 pp->sactive_block = ap->ioaddr.scr_addr + 4 * SCR_ACTIVE;
1969 pp->irq_block = mmio + NV_INT_STATUS_MCP55 + ap->port_no * 2;
1970 pp->tag_block = mmio + NV_NCQ_REG_MCP55 + ap->port_no * 2;
1971
1972 return 0;
1973}
1974
1975static void nv_swncq_qc_prep(struct ata_queued_cmd *qc)
1976{
1977 if (qc->tf.protocol != ATA_PROT_NCQ) {
f47451c4 1978 ata_bmdma_qc_prep(qc);
f140f0f1
KL		 1979 return;
1980 }
1981
1982 if (!(qc->flags & ATA_QCFLAG_DMAMAP))
1983 return;
1984
1985 nv_swncq_fill_sg(qc);
1986}
1987
1988static void nv_swncq_fill_sg(struct ata_queued_cmd *qc)
1989{
1990 struct ata_port *ap = qc->ap;
1991 struct scatterlist *sg;
f140f0f1 1992 struct nv_swncq_port_priv *pp = ap->private_data;
f60d7011 1993 struct ata_bmdma_prd *prd;
ff2aeb1e 1994 unsigned int si, idx;
f140f0f1 1995
4e5b6260 1996 prd = pp->prd + ATA_MAX_PRD * qc->hw_tag;
f140f0f1
KL		 1997
1998 idx = 0;
ff2aeb1e 1999 for_each_sg(qc->sg, sg, qc->n_elem, si) {
f140f0f1
KL		 2000 u32 addr, offset;
2001 u32 sg_len, len;
2002
2003 addr = (u32)sg_dma_address(sg);
2004 sg_len = sg_dma_len(sg);
2005
2006 while (sg_len) {
2007 offset = addr & 0xffff;
2008 len = sg_len;
2009 if ((offset + sg_len) > 0x10000)
2010 len = 0x10000 - offset;
2011
2012 prd[idx].addr = cpu_to_le32(addr);
2013 prd[idx].flags_len = cpu_to_le32(len & 0xffff);
2014
2015 idx++;
2016 sg_len -= len;
2017 addr += len;
2018 }
2019 }
2020
ff2aeb1e 2021 prd[idx - 1].flags_len |= cpu_to_le32(ATA_PRD_EOT);
f140f0f1
KL		 2022}
2023
2024static unsigned int nv_swncq_issue_atacmd(struct ata_port *ap,
2025 struct ata_queued_cmd *qc)
2026{
2027 struct nv_swncq_port_priv *pp = ap->private_data;
2028
2029 if (qc == NULL)
2030 return 0;
2031
2032 DPRINTK("Enter\n");
2033
4e5b6260
JA		 2034 writel((1 << qc->hw_tag), pp->sactive_block);
2035 pp->last_issue_tag = qc->hw_tag;
2036 pp->dhfis_bits &= ~(1 << qc->hw_tag);
2037 pp->dmafis_bits &= ~(1 << qc->hw_tag);
2038 pp->qc_active |= (0x1 << qc->hw_tag);
f140f0f1 2039
5682ed33
TH		 2040 ap->ops->sff_tf_load(ap, &qc->tf); /* load tf registers */
2041 ap->ops->sff_exec_command(ap, &qc->tf);
f140f0f1 2042
4e5b6260 2043 DPRINTK("Issued tag %u\n", qc->hw_tag);
f140f0f1
KL		 2044
2045 return 0;
2046}
2047
2048static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc)
2049{
2050 struct ata_port *ap = qc->ap;
2051 struct nv_swncq_port_priv *pp = ap->private_data;
2052
2053 if (qc->tf.protocol != ATA_PROT_NCQ)
360ff783 2054 return ata_bmdma_qc_issue(qc);
f140f0f1
KL		 2055
2056 DPRINTK("Enter\n");
2057
2058 if (!pp->qc_active)
2059 nv_swncq_issue_atacmd(ap, qc);
2060 else
2061 nv_swncq_qc_to_dq(ap, qc); /* add qc to defer queue */
2062
2063 return 0;
2064}
2065
2066static void nv_swncq_hotplug(struct ata_port *ap, u32 fis)
2067{
2068 u32 serror;
2069 struct ata_eh_info *ehi = &ap->link.eh_info;
2070
2071 ata_ehi_clear_desc(ehi);
2072
2073 /* AHCI needs SError cleared; otherwise, it might lock up */
2074 sata_scr_read(&ap->link, SCR_ERROR, &serror);
2075 sata_scr_write(&ap->link, SCR_ERROR, serror);
2076
2077 /* analyze @irq_stat */
2078 if (fis & NV_SWNCQ_IRQ_ADDED)
2079 ata_ehi_push_desc(ehi, "hot plug");
2080 else if (fis & NV_SWNCQ_IRQ_REMOVED)
2081 ata_ehi_push_desc(ehi, "hot unplug");
2082
2083 ata_ehi_hotplugged(ehi);
2084
2085 /* okay, let's hand over to EH */
2086 ehi->serror |= serror;
2087
2088 ata_port_freeze(ap);
2089}
2090
2091static int nv_swncq_sdbfis(struct ata_port *ap)
2092{
2093 struct ata_queued_cmd *qc;
2094 struct nv_swncq_port_priv *pp = ap->private_data;
2095 struct ata_eh_info *ehi = &ap->link.eh_info;
2096 u32 sactive;
f140f0f1 2097 u32 done_mask;
f140f0f1
KL		 2098 u8 host_stat;
2099 u8 lack_dhfis = 0;
2100
2101 host_stat = ap->ops->bmdma_status(ap);
2102 if (unlikely(host_stat & ATA_DMA_ERR)) {
25985edc 2103 /* error when transferring data to/from memory */
f140f0f1
KL		 2104 ata_ehi_clear_desc(ehi);
2105 ata_ehi_push_desc(ehi, "BMDMA stat 0x%x", host_stat);
2106 ehi->err_mask |= AC_ERR_HOST_BUS;
cf480626 2107 ehi->action |= ATA_EH_RESET;
f140f0f1
KL		 2108 return -EINVAL;
2109 }
2110
5682ed33 2111 ap->ops->sff_irq_clear(ap);
f140f0f1
KL		 2112 __ata_bmdma_stop(ap);
2113
2114 sactive = readl(pp->sactive_block);
2115 done_mask = pp->qc_active ^ sactive;
2116
1aadf5c3
TH		 2117 pp->qc_active &= ~done_mask;
2118 pp->dhfis_bits &= ~done_mask;
2119 pp->dmafis_bits &= ~done_mask;
2120 pp->sdbfis_bits |= done_mask;
2121 ata_qc_complete_multiple(ap, ap->qc_active ^ done_mask);
f140f0f1
KL		 2122
2123 if (!ap->qc_active) {
2124 DPRINTK("over\n");
2125 nv_swncq_pp_reinit(ap);
752e386c 2126 return 0;
f140f0f1
KL		 2127 }
2128
2129 if (pp->qc_active & pp->dhfis_bits)
752e386c 2130 return 0;
f140f0f1
KL		 2131
2132 if ((pp->ncq_flags & ncq_saw_backout) ||
2133 (pp->qc_active ^ pp->dhfis_bits))
752e386c 2134 /* if the controller can't get a device to host register FIS,
f140f0f1
KL		 2135 * The driver needs to reissue the new command.
2136 */
2137 lack_dhfis = 1;
2138
2139 DPRINTK("id 0x%x QC: qc_active 0x%x,"
2140 "SWNCQ:qc_active 0x%X defer_bits %X "
2141 "dhfis 0x%X dmafis 0x%X last_issue_tag %x\n",
2142 ap->print_id, ap->qc_active, pp->qc_active,
2143 pp->defer_queue.defer_bits, pp->dhfis_bits,
2144 pp->dmafis_bits, pp->last_issue_tag);
2145
2146 nv_swncq_fis_reinit(ap);
2147
2148 if (lack_dhfis) {
2149 qc = ata_qc_from_tag(ap, pp->last_issue_tag);
2150 nv_swncq_issue_atacmd(ap, qc);
752e386c 2151 return 0;
f140f0f1
KL		 2152 }
2153
2154 if (pp->defer_queue.defer_bits) {
2155 /* send deferral queue command */
2156 qc = nv_swncq_qc_from_dq(ap);
2157 WARN_ON(qc == NULL);
2158 nv_swncq_issue_atacmd(ap, qc);
2159 }
2160
752e386c 2161 return 0;
f140f0f1
KL		 2162}
2163
2164static inline u32 nv_swncq_tag(struct ata_port *ap)
2165{
2166 struct nv_swncq_port_priv *pp = ap->private_data;
2167 u32 tag;
2168
2169 tag = readb(pp->tag_block) >> 2;
2170 return (tag & 0x1f);
2171}
2172
752e386c 2173static void nv_swncq_dmafis(struct ata_port *ap)
f140f0f1
KL		 2174{
2175 struct ata_queued_cmd *qc;
2176 unsigned int rw;
2177 u8 dmactl;
2178 u32 tag;
2179 struct nv_swncq_port_priv *pp = ap->private_data;
2180
2181 __ata_bmdma_stop(ap);
2182 tag = nv_swncq_tag(ap);
2183
2184 DPRINTK("dma setup tag 0x%x\n", tag);
2185 qc = ata_qc_from_tag(ap, tag);
2186
2187 if (unlikely(!qc))
752e386c 2188 return;
f140f0f1
KL		 2189
2190 rw = qc->tf.flags & ATA_TFLAG_WRITE;
2191
2192 /* load PRD table addr. */
4e5b6260 2193 iowrite32(pp->prd_dma + ATA_PRD_TBL_SZ * qc->hw_tag,
f140f0f1
KL		 2194 ap->ioaddr.bmdma_addr + ATA_DMA_TABLE_OFS);
2195
2196 /* specify data direction, triple-check start bit is clear */
2197 dmactl = ioread8(ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
2198 dmactl &= ~ATA_DMA_WR;
2199 if (!rw)
2200 dmactl |= ATA_DMA_WR;
2201
2202 iowrite8(dmactl | ATA_DMA_START, ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
f140f0f1
KL		 2203}
2204
2205static void nv_swncq_host_interrupt(struct ata_port *ap, u16 fis)
2206{
2207 struct nv_swncq_port_priv *pp = ap->private_data;
2208 struct ata_queued_cmd *qc;
2209 struct ata_eh_info *ehi = &ap->link.eh_info;
2210 u32 serror;
2211 u8 ata_stat;
f140f0f1 2212
5682ed33 2213 ata_stat = ap->ops->sff_check_status(ap);
f140f0f1
KL		 2214 nv_swncq_irq_clear(ap, fis);
2215 if (!fis)
2216 return;
2217
2218 if (ap->pflags & ATA_PFLAG_FROZEN)
2219 return;
2220
2221 if (fis & NV_SWNCQ_IRQ_HOTPLUG) {
2222 nv_swncq_hotplug(ap, fis);
2223 return;
2224 }
2225
2226 if (!pp->qc_active)
2227 return;
2228
82ef04fb 2229 if (ap->ops->scr_read(&ap->link, SCR_ERROR, &serror))
f140f0f1 2230 return;
82ef04fb 2231 ap->ops->scr_write(&ap->link, SCR_ERROR, serror);
f140f0f1
KL		 2232
2233 if (ata_stat & ATA_ERR) {
2234 ata_ehi_clear_desc(ehi);
2235 ata_ehi_push_desc(ehi, "Ata error. fis:0x%X", fis);
2236 ehi->err_mask |= AC_ERR_DEV;
2237 ehi->serror |= serror;
cf480626 2238 ehi->action |= ATA_EH_RESET;
f140f0f1
KL		 2239 ata_port_freeze(ap);
2240 return;
2241 }
2242
2243 if (fis & NV_SWNCQ_IRQ_BACKOUT) {
2244 /* If the IRQ is backout, driver must issue
2245 * the new command again some time later.
2246 */
2247 pp->ncq_flags |= ncq_saw_backout;
2248 }
2249
2250 if (fis & NV_SWNCQ_IRQ_SDBFIS) {
2251 pp->ncq_flags |= ncq_saw_sdb;
2252 DPRINTK("id 0x%x SWNCQ: qc_active 0x%X "
2253 "dhfis 0x%X dmafis 0x%X sactive 0x%X\n",
2254 ap->print_id, pp->qc_active, pp->dhfis_bits,
2255 pp->dmafis_bits, readl(pp->sactive_block));
752e386c 2256 if (nv_swncq_sdbfis(ap) < 0)
f140f0f1
KL		 2257 goto irq_error;
2258 }
2259
2260 if (fis & NV_SWNCQ_IRQ_DHREGFIS) {
2261 /* The interrupt indicates the new command
2262 * was transmitted correctly to the drive.
2263 */
2264 pp->dhfis_bits |= (0x1 << pp->last_issue_tag);
2265 pp->ncq_flags |= ncq_saw_d2h;
2266 if (pp->ncq_flags & (ncq_saw_sdb | ncq_saw_backout)) {
2267 ata_ehi_push_desc(ehi, "illegal fis transaction");
2268 ehi->err_mask |= AC_ERR_HSM;
cf480626 2269 ehi->action |= ATA_EH_RESET;
f140f0f1
KL		 2270 goto irq_error;
2271 }
2272
2273 if (!(fis & NV_SWNCQ_IRQ_DMASETUP) &&
2274 !(pp->ncq_flags & ncq_saw_dmas)) {
5682ed33 2275 ata_stat = ap->ops->sff_check_status(ap);
f140f0f1
KL		 2276 if (ata_stat & ATA_BUSY)
2277 goto irq_exit;
2278
2279 if (pp->defer_queue.defer_bits) {
2280 DPRINTK("send next command\n");
2281 qc = nv_swncq_qc_from_dq(ap);
2282 nv_swncq_issue_atacmd(ap, qc);
2283 }
2284 }
2285 }
2286
2287 if (fis & NV_SWNCQ_IRQ_DMASETUP) {
2288 /* program the dma controller with appropriate PRD buffers
2289 * and start the DMA transfer for requested command.
2290 */
2291 pp->dmafis_bits |= (0x1 << nv_swncq_tag(ap));
2292 pp->ncq_flags |= ncq_saw_dmas;
752e386c 2293 nv_swncq_dmafis(ap);
f140f0f1
KL		 2294 }
2295
2296irq_exit:
2297 return;
2298irq_error:
2299 ata_ehi_push_desc(ehi, "fis:0x%x", fis);
2300 ata_port_freeze(ap);
2301 return;
2302}
2303
2304static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance)
2305{
2306 struct ata_host *host = dev_instance;
2307 unsigned int i;
2308 unsigned int handled = 0;
2309 unsigned long flags;
2310 u32 irq_stat;
2311
2312 spin_lock_irqsave(&host->lock, flags);
2313
2314 irq_stat = readl(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_MCP55);
2315
2316 for (i = 0; i < host->n_ports; i++) {
2317 struct ata_port *ap = host->ports[i];
2318
3e4ec344
TH		 2319 if (ap->link.sactive) {
2320 nv_swncq_host_interrupt(ap, (u16)irq_stat);
2321 handled = 1;
2322 } else {
2323 if (irq_stat) /* reserve Hotplug */
2324 nv_swncq_irq_clear(ap, 0xfff0);
f140f0f1 2325
3e4ec344 2326 handled += nv_host_intr(ap, (u8)irq_stat);
f140f0f1
KL		 2327 }
2328 irq_stat >>= NV_INT_PORT_SHIFT_MCP55;
2329 }
2330
2331 spin_unlock_irqrestore(&host->lock, flags);
2332
2333 return IRQ_RETVAL(handled);
2334}
2335
5796d1c4 2336static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
1da177e4 2337{
1626aeb8 2338 const struct ata_port_info *ppi[] = { NULL, NULL };
95947193 2339 struct nv_pi_priv *ipriv;
9a829ccf 2340 struct ata_host *host;
cdf56bcf 2341 struct nv_host_priv *hpriv;
1da177e4
LT		 2342 int rc;
2343 u32 bar;
0d5ff566 2344 void __iomem *base;
fbbb262d 2345 unsigned long type = ent->driver_data;
1da177e4
LT		 2346
2347 // Make sure this is a SATA controller by counting the number of bars
2348 // (NVIDIA SATA controllers will always have six bars). Otherwise,
2349 // it's an IDE controller and we ignore it.
5796d1c4 2350 for (bar = 0; bar < 6; bar++)
1da177e4
LT		 2351 if (pci_resource_start(pdev, bar) == 0)
2352 return -ENODEV;
2353
06296a1e 2354 ata_print_version_once(&pdev->dev, DRV_VERSION);
1da177e4 2355
24dc5f33 2356 rc = pcim_enable_device(pdev);
1da177e4 2357 if (rc)
24dc5f33 2358 return rc;
1da177e4 2359
9a829ccf 2360 /* determine type and allocate host */
f140f0f1 2361 if (type == CK804 && adma_enabled) {
a44fec1f 2362 dev_notice(&pdev->dev, "Using ADMA mode\n");
fbbb262d 2363 type = ADMA;
2d775708 2364 } else if (type == MCP5x && swncq_enabled) {
a44fec1f 2365 dev_notice(&pdev->dev, "Using SWNCQ mode\n");
2d775708 2366 type = SWNCQ;
360737a9
JG		 2367 }
2368
1626aeb8 2369 ppi[0] = &nv_port_info[type];
95947193 2370 ipriv = ppi[0]->private_data;
1c5afdf7 2371 rc = ata_pci_bmdma_prepare_host(pdev, ppi, &host);
9a829ccf
TH		 2372 if (rc)
2373 return rc;
1da177e4 2374
24dc5f33 2375 hpriv = devm_kzalloc(&pdev->dev, sizeof(*hpriv), GFP_KERNEL);
cdf56bcf 2376 if (!hpriv)
24dc5f33 2377 return -ENOMEM;
9a829ccf
TH		 2378 hpriv->type = type;
2379 host->private_data = hpriv;
cdf56bcf 2380
9a829ccf
TH		 2381 /* request and iomap NV_MMIO_BAR */
2382 rc = pcim_iomap_regions(pdev, 1 << NV_MMIO_BAR, DRV_NAME);
2383 if (rc)
2384 return rc;
1da177e4 2385
9a829ccf
TH		 2386 /* configure SCR access */
2387 base = host->iomap[NV_MMIO_BAR];
2388 host->ports[0]->ioaddr.scr_addr = base + NV_PORT0_SCR_REG_OFFSET;
2389 host->ports[1]->ioaddr.scr_addr = base + NV_PORT1_SCR_REG_OFFSET;
1da177e4 2390
ada364e8 2391 /* enable SATA space for CK804 */
fbbb262d 2392 if (type >= CK804) {
ada364e8
TH		 2393 u8 regval;
2394
2395 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2396 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2397 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2398 }
2399
9a829ccf 2400 /* init ADMA */
fbbb262d 2401 if (type == ADMA) {
9a829ccf 2402 rc = nv_adma_host_init(host);
fbbb262d 2403 if (rc)
24dc5f33 2404 return rc;
360737a9 2405 } else if (type == SWNCQ)
f140f0f1 2406 nv_swncq_host_init(host);
fbbb262d 2407
51c89499 2408 if (msi_enabled) {
a44fec1f 2409 dev_notice(&pdev->dev, "Using MSI\n");
51c89499
TV		 2410 pci_enable_msi(pdev);
2411 }
2412
9a829ccf 2413 pci_set_master(pdev);
95cc2c70 2414 return ata_pci_sff_activate_host(host, ipriv->irq_handler, ipriv->sht);
1da177e4
LT		 2415}
2416
58eb8cd5 2417#ifdef CONFIG_PM_SLEEP
cdf56bcf
RH		 2418static int nv_pci_device_resume(struct pci_dev *pdev)
2419{
0a86e1c8 2420 struct ata_host *host = pci_get_drvdata(pdev);
cdf56bcf 2421 struct nv_host_priv *hpriv = host->private_data;
ce053fa8 2422 int rc;
cdf56bcf 2423
ce053fa8 2424 rc = ata_pci_device_do_resume(pdev);
b447916e 2425 if (rc)
ce053fa8 2426 return rc;
cdf56bcf
RH		 2427
2428 if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
b447916e 2429 if (hpriv->type >= CK804) {
cdf56bcf
RH		 2430 u8 regval;
2431
2432 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2433 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2434 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2435 }
b447916e 2436 if (hpriv->type == ADMA) {
cdf56bcf
RH		 2437 u32 tmp32;
2438 struct nv_adma_port_priv *pp;
2439 /* enable/disable ADMA on the ports appropriately */
2440 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2441
2442 pp = host->ports[0]->private_data;
b447916e 2443 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
cdf56bcf 2444 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
5796d1c4 2445 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
cdf56bcf
RH		 2446 else
2447 tmp32 |= (NV_MCP_SATA_CFG_20_PORT0_EN |
5796d1c4 2448 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
cdf56bcf 2449 pp = host->ports[1]->private_data;
b447916e 2450 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
cdf56bcf 2451 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT1_EN |
5796d1c4 2452 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
cdf56bcf
RH		 2453 else
2454 tmp32 |= (NV_MCP_SATA_CFG_20_PORT1_EN |
5796d1c4 2455 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
cdf56bcf
RH		 2456
2457 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2458 }
2459 }
2460
2461 ata_host_resume(host);
2462
2463 return 0;
2464}
438ac6d5 2465#endif
cdf56bcf 2466
cca3974e 2467static void nv_ck804_host_stop(struct ata_host *host)
ada364e8 2468{
cca3974e 2469 struct pci_dev *pdev = to_pci_dev(host->dev);
ada364e8
TH		 2470 u8 regval;
2471
2472 /* disable SATA space for CK804 */
2473 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2474 regval &= ~NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2475 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
ada364e8
TH		 2476}
2477
fbbb262d
RH		 2478static void nv_adma_host_stop(struct ata_host *host)
2479{
2480 struct pci_dev *pdev = to_pci_dev(host->dev);
fbbb262d
RH		 2481 u32 tmp32;
2482
fbbb262d
RH		 2483 /* disable ADMA on the ports */
2484 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2485 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
2486 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
2487 NV_MCP_SATA_CFG_20_PORT1_EN |
2488 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2489
2490 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2491
2492 nv_ck804_host_stop(host);
2493}
2494
2fc75da0 2495module_pci_driver(nv_pci_driver);
1da177e4 2496
fbbb262d 2497module_param_named(adma, adma_enabled, bool, 0444);
55f784c8 2498MODULE_PARM_DESC(adma, "Enable use of ADMA (Default: false)");
f140f0f1 2499module_param_named(swncq, swncq_enabled, bool, 0444);
d21279f4 2500MODULE_PARM_DESC(swncq, "Enable use of SWNCQ (Default: true)");
51c89499
TV		 2501module_param_named(msi, msi_enabled, bool, 0444);
2502MODULE_PARM_DESC(msi, "Enable use of MSI (Default: false)");
Linux 6.x block layer and io_uring tree(s)