projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'printk-for-6.6' of git://git.kernel.org/pub/scm/linux/kernel/git/printk...
[linux-block.git] / drivers / ata / sata_nv.c
CommitLineData
c82ee6d3 1// SPDX-License-Identifier: GPL-2.0-or-later
1da177e4
LT		 2/*
3 * sata_nv.c - NVIDIA nForce SATA
4 *
5 * Copyright 2004 NVIDIA Corp. All rights reserved.
6 * Copyright 2004 Andrew Chew
7 *
af36d7f0 8 * libata documentation is available via 'make {ps|pdf}docs',
19285f3c 9 * as Documentation/driver-api/libata.rst
af36d7f0
JG		 10 *
11 * No hardware documentation available outside of NVIDIA.
12 * This driver programs the NVIDIA SATA controller in a similar
13 * fashion as with other PCI IDE BMDMA controllers, with a few
14 * NV-specific details such as register offsets, SATA phy location,
15 * hotplug info, etc.
16 *
fbbb262d
RH		 17 * CK804/MCP04 controllers support an alternate programming interface
18 * similar to the ADMA specification (with some modifications).
19 * This allows the use of NCQ. Non-DMA-mapped ATA commands are still
20 * sent through the legacy interface.
1da177e4
LT		 21 */
22
1da177e4
LT		 23#include <linux/kernel.h>
24#include <linux/module.h>
5a0e3ad6 25#include <linux/gfp.h>
1da177e4 26#include <linux/pci.h>
1da177e4
LT		 27#include <linux/blkdev.h>
28#include <linux/delay.h>
29#include <linux/interrupt.h>
a9524a76 30#include <linux/device.h>
1da177e4 31#include <scsi/scsi_host.h>
fbbb262d 32#include <scsi/scsi_device.h>
1da177e4 33#include <linux/libata.h>
c206a389 34#include <trace/events/libata.h>
1da177e4
LT		 35
36#define DRV_NAME "sata_nv"
2a3103ce 37#define DRV_VERSION "3.5"
fbbb262d
RH		 38
39#define NV_ADMA_DMA_BOUNDARY 0xffffffffUL
1da177e4 40
10ad05df 41enum {
0d5ff566
TH		 42 NV_MMIO_BAR = 5,
43
10ad05df 44 NV_PORTS = 2,
14bdef98
EIB		 45 NV_PIO_MASK = ATA_PIO4,
46 NV_MWDMA_MASK = ATA_MWDMA2,
47 NV_UDMA_MASK = ATA_UDMA6,
10ad05df
JG		 48 NV_PORT0_SCR_REG_OFFSET = 0x00,
49 NV_PORT1_SCR_REG_OFFSET = 0x40,
1da177e4 50
27e4b274 51 /* INT_STATUS/ENABLE */
10ad05df 52 NV_INT_STATUS = 0x10,
10ad05df 53 NV_INT_ENABLE = 0x11,
27e4b274 54 NV_INT_STATUS_CK804 = 0x440,
10ad05df 55 NV_INT_ENABLE_CK804 = 0x441,
1da177e4 56
27e4b274
TH		 57 /* INT_STATUS/ENABLE bits */
58 NV_INT_DEV = 0x01,
59 NV_INT_PM = 0x02,
60 NV_INT_ADDED = 0x04,
61 NV_INT_REMOVED = 0x08,
62
63 NV_INT_PORT_SHIFT = 4, /* each port occupies 4 bits */
64
39f87582 65 NV_INT_ALL = 0x0f,
5a44efff
TH		 66 NV_INT_MASK = NV_INT_DEV |
67 NV_INT_ADDED | NV_INT_REMOVED,
39f87582 68
27e4b274 69 /* INT_CONFIG */
10ad05df
JG		 70 NV_INT_CONFIG = 0x12,
71 NV_INT_CONFIG_METHD = 0x01, // 0 = INT, 1 = SMI
1da177e4 72
10ad05df
JG		 73 // For PCI config register 20
74 NV_MCP_SATA_CFG_20 = 0x50,
75 NV_MCP_SATA_CFG_20_SATA_SPACE_EN = 0x04,
fbbb262d
RH		 76 NV_MCP_SATA_CFG_20_PORT0_EN = (1 << 17),
77 NV_MCP_SATA_CFG_20_PORT1_EN = (1 << 16),
78 NV_MCP_SATA_CFG_20_PORT0_PWB_EN = (1 << 14),
79 NV_MCP_SATA_CFG_20_PORT1_PWB_EN = (1 << 12),
80
81 NV_ADMA_MAX_CPBS = 32,
82 NV_ADMA_CPB_SZ = 128,
83 NV_ADMA_APRD_SZ = 16,
84 NV_ADMA_SGTBL_LEN = (1024 - NV_ADMA_CPB_SZ) /
85 NV_ADMA_APRD_SZ,
86 NV_ADMA_SGTBL_TOTAL_LEN = NV_ADMA_SGTBL_LEN + 5,
87 NV_ADMA_SGTBL_SZ = NV_ADMA_SGTBL_LEN * NV_ADMA_APRD_SZ,
88 NV_ADMA_PORT_PRIV_DMA_SZ = NV_ADMA_MAX_CPBS *
89 (NV_ADMA_CPB_SZ + NV_ADMA_SGTBL_SZ),
90
91 /* BAR5 offset to ADMA general registers */
92 NV_ADMA_GEN = 0x400,
93 NV_ADMA_GEN_CTL = 0x00,
94 NV_ADMA_NOTIFIER_CLEAR = 0x30,
95
96 /* BAR5 offset to ADMA ports */
97 NV_ADMA_PORT = 0x480,
98
99 /* size of ADMA port register space */
100 NV_ADMA_PORT_SIZE = 0x100,
101
102 /* ADMA port registers */
103 NV_ADMA_CTL = 0x40,
104 NV_ADMA_CPB_COUNT = 0x42,
105 NV_ADMA_NEXT_CPB_IDX = 0x43,
106 NV_ADMA_STAT = 0x44,
107 NV_ADMA_CPB_BASE_LOW = 0x48,
108 NV_ADMA_CPB_BASE_HIGH = 0x4C,
109 NV_ADMA_APPEND = 0x50,
110 NV_ADMA_NOTIFIER = 0x68,
111 NV_ADMA_NOTIFIER_ERROR = 0x6C,
112
113 /* NV_ADMA_CTL register bits */
114 NV_ADMA_CTL_HOTPLUG_IEN = (1 << 0),
115 NV_ADMA_CTL_CHANNEL_RESET = (1 << 5),
116 NV_ADMA_CTL_GO = (1 << 7),
117 NV_ADMA_CTL_AIEN = (1 << 8),
118 NV_ADMA_CTL_READ_NON_COHERENT = (1 << 11),
119 NV_ADMA_CTL_WRITE_NON_COHERENT = (1 << 12),
120
121 /* CPB response flag bits */
122 NV_CPB_RESP_DONE = (1 << 0),
123 NV_CPB_RESP_ATA_ERR = (1 << 3),
124 NV_CPB_RESP_CMD_ERR = (1 << 4),
125 NV_CPB_RESP_CPB_ERR = (1 << 7),
126
127 /* CPB control flag bits */
128 NV_CPB_CTL_CPB_VALID = (1 << 0),
129 NV_CPB_CTL_QUEUE = (1 << 1),
130 NV_CPB_CTL_APRD_VALID = (1 << 2),
131 NV_CPB_CTL_IEN = (1 << 3),
132 NV_CPB_CTL_FPDMA = (1 << 4),
133
134 /* APRD flags */
135 NV_APRD_WRITE = (1 << 1),
136 NV_APRD_END = (1 << 2),
137 NV_APRD_CONT = (1 << 3),
138
139 /* NV_ADMA_STAT flags */
140 NV_ADMA_STAT_TIMEOUT = (1 << 0),
141 NV_ADMA_STAT_HOTUNPLUG = (1 << 1),
142 NV_ADMA_STAT_HOTPLUG = (1 << 2),
143 NV_ADMA_STAT_CPBERR = (1 << 4),
144 NV_ADMA_STAT_SERROR = (1 << 5),
145 NV_ADMA_STAT_CMD_COMPLETE = (1 << 6),
146 NV_ADMA_STAT_IDLE = (1 << 8),
147 NV_ADMA_STAT_LEGACY = (1 << 9),
148 NV_ADMA_STAT_STOPPED = (1 << 10),
149 NV_ADMA_STAT_DONE = (1 << 12),
150 NV_ADMA_STAT_ERR = NV_ADMA_STAT_CPBERR |
2dcb407e 151 NV_ADMA_STAT_TIMEOUT,
fbbb262d
RH		 152
153 /* port flags */
154 NV_ADMA_PORT_REGISTER_MODE = (1 << 0),
2dec7555 155 NV_ADMA_ATAPI_SETUP_COMPLETE = (1 << 1),
fbbb262d 156
f140f0f1
KL		 157 /* MCP55 reg offset */
158 NV_CTL_MCP55 = 0x400,
159 NV_INT_STATUS_MCP55 = 0x440,
160 NV_INT_ENABLE_MCP55 = 0x444,
161 NV_NCQ_REG_MCP55 = 0x448,
162
163 /* MCP55 */
164 NV_INT_ALL_MCP55 = 0xffff,
165 NV_INT_PORT_SHIFT_MCP55 = 16, /* each port occupies 16 bits */
166 NV_INT_MASK_MCP55 = NV_INT_ALL_MCP55 & 0xfffd,
167
168 /* SWNCQ ENABLE BITS*/
169 NV_CTL_PRI_SWNCQ = 0x02,
170 NV_CTL_SEC_SWNCQ = 0x04,
171
172 /* SW NCQ status bits*/
173 NV_SWNCQ_IRQ_DEV = (1 << 0),
174 NV_SWNCQ_IRQ_PM = (1 << 1),
175 NV_SWNCQ_IRQ_ADDED = (1 << 2),
176 NV_SWNCQ_IRQ_REMOVED = (1 << 3),
177
178 NV_SWNCQ_IRQ_BACKOUT = (1 << 4),
179 NV_SWNCQ_IRQ_SDBFIS = (1 << 5),
180 NV_SWNCQ_IRQ_DHREGFIS = (1 << 6),
181 NV_SWNCQ_IRQ_DMASETUP = (1 << 7),
182
183 NV_SWNCQ_IRQ_HOTPLUG = NV_SWNCQ_IRQ_ADDED |
184 NV_SWNCQ_IRQ_REMOVED,
185
fbbb262d
RH		 186};
187
188/* ADMA Physical Region Descriptor - one SG segment */
189struct nv_adma_prd {
190 __le64 addr;
191 __le32 len;
192 u8 flags;
193 u8 packet_len;
194 __le16 reserved;
195};
196
197enum nv_adma_regbits {
198 CMDEND = (1 << 15), /* end of command list */
199 WNB = (1 << 14), /* wait-not-BSY */
200 IGN = (1 << 13), /* ignore this entry */
201 CS1n = (1 << (4 + 8)), /* std. PATA signals follow... */
202 DA2 = (1 << (2 + 8)),
203 DA1 = (1 << (1 + 8)),
204 DA0 = (1 << (0 + 8)),
205};
206
207/* ADMA Command Parameter Block
208 The first 5 SG segments are stored inside the Command Parameter Block itself.
209 If there are more than 5 segments the remainder are stored in a separate
210 memory area indicated by next_aprd. */
211struct nv_adma_cpb {
212 u8 resp_flags; /* 0 */
213 u8 reserved1; /* 1 */
214 u8 ctl_flags; /* 2 */
215 /* len is length of taskfile in 64 bit words */
2dcb407e 216 u8 len; /* 3 */
fbbb262d
RH		 217 u8 tag; /* 4 */
218 u8 next_cpb_idx; /* 5 */
219 __le16 reserved2; /* 6-7 */
220 __le16 tf[12]; /* 8-31 */
221 struct nv_adma_prd aprd[5]; /* 32-111 */
222 __le64 next_aprd; /* 112-119 */
223 __le64 reserved3; /* 120-127 */
10ad05df 224};
1da177e4 225
fbbb262d
RH		 226
227struct nv_adma_port_priv {
228 struct nv_adma_cpb *cpb;
229 dma_addr_t cpb_dma;
230 struct nv_adma_prd *aprd;
231 dma_addr_t aprd_dma;
2dcb407e
JG		 232 void __iomem *ctl_block;
233 void __iomem *gen_block;
234 void __iomem *notifier_clear_block;
8959d300 235 u64 adma_dma_mask;
fbbb262d 236 u8 flags;
5e5c74a5 237 int last_issue_ncq;
fbbb262d
RH		 238};
239
cdf56bcf
RH		 240struct nv_host_priv {
241 unsigned long type;
242};
243
f140f0f1
KL		 244struct defer_queue {
245 u32 defer_bits;
246 unsigned int head;
247 unsigned int tail;
248 unsigned int tag[ATA_MAX_QUEUE];
249};
250
251enum ncq_saw_flag_list {
252 ncq_saw_d2h = (1U << 0),
253 ncq_saw_dmas = (1U << 1),
254 ncq_saw_sdb = (1U << 2),
255 ncq_saw_backout = (1U << 3),
256};
257
258struct nv_swncq_port_priv {
f60d7011 259 struct ata_bmdma_prd *prd; /* our SG list */
f140f0f1
KL		 260 dma_addr_t prd_dma; /* and its DMA mapping */
261 void __iomem *sactive_block;
262 void __iomem *irq_block;
263 void __iomem *tag_block;
264 u32 qc_active;
265
266 unsigned int last_issue_tag;
267
268 /* fifo circular queue to store deferral command */
269 struct defer_queue defer_queue;
270
271 /* for NCQ interrupt analysis */
272 u32 dhfis_bits;
273 u32 dmafis_bits;
274 u32 sdbfis_bits;
275
276 unsigned int ncq_flags;
277};
278
279
5796d1c4 280#define NV_ADMA_CHECK_INTR(GCTL, PORT) ((GCTL) & (1 << (19 + (12 * (PORT)))))
fbbb262d 281
2dcb407e 282static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
58eb8cd5 283#ifdef CONFIG_PM_SLEEP
cdf56bcf 284static int nv_pci_device_resume(struct pci_dev *pdev);
438ac6d5 285#endif
cca3974e 286static void nv_ck804_host_stop(struct ata_host *host);
7d12e780
DH		 287static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance);
288static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance);
289static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance);
82ef04fb
TH		 290static int nv_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val);
291static int nv_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val);
1da177e4 292
7f4774b3
TH		 293static int nv_hardreset(struct ata_link *link, unsigned int *class,
294 unsigned long deadline);
39f87582
TH		 295static void nv_nf2_freeze(struct ata_port *ap);
296static void nv_nf2_thaw(struct ata_port *ap);
297static void nv_ck804_freeze(struct ata_port *ap);
298static void nv_ck804_thaw(struct ata_port *ap);
fbbb262d 299static int nv_adma_slave_config(struct scsi_device *sdev);
2dec7555 300static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc);
95364f36 301static enum ata_completion_errors nv_adma_qc_prep(struct ata_queued_cmd *qc);
fbbb262d
RH		 302static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc);
303static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance);
304static void nv_adma_irq_clear(struct ata_port *ap);
305static int nv_adma_port_start(struct ata_port *ap);
306static void nv_adma_port_stop(struct ata_port *ap);
438ac6d5 307#ifdef CONFIG_PM
cdf56bcf
RH		 308static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg);
309static int nv_adma_port_resume(struct ata_port *ap);
438ac6d5 310#endif
53014e25
RH		 311static void nv_adma_freeze(struct ata_port *ap);
312static void nv_adma_thaw(struct ata_port *ap);
fbbb262d
RH		 313static void nv_adma_error_handler(struct ata_port *ap);
314static void nv_adma_host_stop(struct ata_host *host);
f5ecac2d 315static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc);
f2fb344b 316static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf);
39f87582 317
f140f0f1
KL		 318static void nv_mcp55_thaw(struct ata_port *ap);
319static void nv_mcp55_freeze(struct ata_port *ap);
320static void nv_swncq_error_handler(struct ata_port *ap);
321static int nv_swncq_slave_config(struct scsi_device *sdev);
322static int nv_swncq_port_start(struct ata_port *ap);
95364f36 323static enum ata_completion_errors nv_swncq_qc_prep(struct ata_queued_cmd *qc);
f140f0f1
KL		 324static void nv_swncq_fill_sg(struct ata_queued_cmd *qc);
325static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc);
326static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis);
327static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance);
328#ifdef CONFIG_PM
329static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg);
330static int nv_swncq_port_resume(struct ata_port *ap);
331#endif
332
1da177e4
LT		 333enum nv_host_type
334{
335 GENERIC,
336 NFORCE2,
27e4b274 337 NFORCE3 = NFORCE2, /* NF2 == NF3 as far as sata_nv is concerned */
fbbb262d 338 CK804,
f140f0f1 339 ADMA,
2d775708 340 MCP5x,
f140f0f1 341 SWNCQ,
1da177e4
LT		 342};
343
3b7d697d 344static const struct pci_device_id nv_pci_tbl[] = {
54bb3a94
JG		 345 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE2S_SATA), NFORCE2 },
346 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA), NFORCE3 },
347 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA2), NFORCE3 },
348 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA), CK804 },
349 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA2), CK804 },
350 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA), CK804 },
351 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA2), CK804 },
2d775708
TH		 352 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA), MCP5x },
353 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2), MCP5x },
354 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA), MCP5x },
355 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2), MCP5x },
e2e031eb
KL		 356 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA), GENERIC },
357 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA2), GENERIC },
358 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA3), GENERIC },
2d2744fc
JG		 359
360 { } /* terminate list */
1da177e4
LT		 361};
362
1da177e4
LT		 363static struct pci_driver nv_pci_driver = {
364 .name = DRV_NAME,
365 .id_table = nv_pci_tbl,
366 .probe = nv_init_one,
58eb8cd5 367#ifdef CONFIG_PM_SLEEP
cdf56bcf
RH		 368 .suspend = ata_pci_device_suspend,
369 .resume = nv_pci_device_resume,
438ac6d5 370#endif
1daf9ce7 371 .remove = ata_pci_remove_one,
1da177e4
LT		 372};
373
25df73d9 374static const struct scsi_host_template nv_sht = {
68d1d07b 375 ATA_BMDMA_SHT(DRV_NAME),
1da177e4
LT		 376};
377
25df73d9 378static const struct scsi_host_template nv_adma_sht = {
7d43b828 379 __ATA_BASE_SHT(DRV_NAME),
fbbb262d 380 .can_queue = NV_ADMA_MAX_CPBS,
fbbb262d 381 .sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN,
fbbb262d
RH		 382 .dma_boundary = NV_ADMA_DMA_BOUNDARY,
383 .slave_configure = nv_adma_slave_config,
c3f69c7f 384 .sdev_groups = ata_ncq_sdev_groups,
7d43b828
LJ		 385 .change_queue_depth = ata_scsi_change_queue_depth,
386 .tag_alloc_policy = BLK_TAG_ALLOC_RR,
fbbb262d
RH		 387};
388
25df73d9 389static const struct scsi_host_template nv_swncq_sht = {
7d43b828 390 __ATA_BASE_SHT(DRV_NAME),
ba80c3a5 391 .can_queue = ATA_MAX_QUEUE - 1,
f140f0f1 392 .sg_tablesize = LIBATA_MAX_PRD,
f140f0f1
KL		 393 .dma_boundary = ATA_DMA_BOUNDARY,
394 .slave_configure = nv_swncq_slave_config,
c3f69c7f 395 .sdev_groups = ata_ncq_sdev_groups,
7d43b828
LJ		 396 .change_queue_depth = ata_scsi_change_queue_depth,
397 .tag_alloc_policy = BLK_TAG_ALLOC_RR,
f140f0f1
KL		 398};
399
7f4774b3
TH		 400/*
401 * NV SATA controllers have various different problems with hardreset
402 * protocol depending on the specific controller and device.
403 *
404 * GENERIC:
405 *
406 * bko11195 reports that link doesn't come online after hardreset on
407 * generic nv's and there have been several other similar reports on
408 * linux-ide.
409 *
410 * bko12351#c23 reports that warmplug on MCP61 doesn't work with
411 * softreset.
412 *
413 * NF2/3:
414 *
415 * bko3352 reports nf2/3 controllers can't determine device signature
416 * reliably after hardreset. The following thread reports detection
417 * failure on cold boot with the standard debouncing timing.
418 *
419 * http://thread.gmane.org/gmane.linux.ide/34098
420 *
421 * bko12176 reports that hardreset fails to bring up the link during
422 * boot on nf2.
423 *
424 * CK804:
425 *
426 * For initial probing after boot and hot plugging, hardreset mostly
427 * works fine on CK804 but curiously, reprobing on the initial port
428 * by rescanning or rmmod/insmod fails to acquire the initial D2H Reg
429 * FIS in somewhat undeterministic way.
430 *
431 * SWNCQ:
432 *
433 * bko12351 reports that when SWNCQ is enabled, for hotplug to work,
434 * hardreset should be used and hardreset can't report proper
435 * signature, which suggests that mcp5x is closer to nf2 as long as
436 * reset quirkiness is concerned.
437 *
438 * bko12703 reports that boot probing fails for intel SSD with
439 * hardreset. Link fails to come online. Softreset works fine.
440 *
441 * The failures are varied but the following patterns seem true for
442 * all flavors.
443 *
444 * - Softreset during boot always works.
445 *
446 * - Hardreset during boot sometimes fails to bring up the link on
447 * certain comibnations and device signature acquisition is
448 * unreliable.
449 *
450 * - Hardreset is often necessary after hotplug.
451 *
452 * So, preferring softreset for boot probing and error handling (as
453 * hardreset might bring down the link) but using hardreset for
454 * post-boot probing should work around the above issues in most
455 * cases. Define nv_hardreset() which only kicks in for post-boot
456 * probing and use it for all variants.
457 */
458static struct ata_port_operations nv_generic_ops = {
029cfd6b 459 .inherits = &ata_bmdma_port_ops,
c96f1732 460 .lost_interrupt = ATA_OP_NULL,
1da177e4
LT		 461 .scr_read = nv_scr_read,
462 .scr_write = nv_scr_write,
7f4774b3 463 .hardreset = nv_hardreset,
1da177e4
LT		 464};
465
029cfd6b 466static struct ata_port_operations nv_nf2_ops = {
7dac745b 467 .inherits = &nv_generic_ops,
39f87582
TH		 468 .freeze = nv_nf2_freeze,
469 .thaw = nv_nf2_thaw,
ada364e8
TH		 470};
471
029cfd6b 472static struct ata_port_operations nv_ck804_ops = {
7f4774b3 473 .inherits = &nv_generic_ops,
39f87582
TH		 474 .freeze = nv_ck804_freeze,
475 .thaw = nv_ck804_thaw,
ada364e8
TH		 476 .host_stop = nv_ck804_host_stop,
477};
478
029cfd6b 479static struct ata_port_operations nv_adma_ops = {
3c324283 480 .inherits = &nv_ck804_ops,
029cfd6b 481
2dec7555 482 .check_atapi_dma = nv_adma_check_atapi_dma,
5682ed33 483 .sff_tf_read = nv_adma_tf_read,
31cc23b3 484 .qc_defer = ata_std_qc_defer,
fbbb262d
RH		 485 .qc_prep = nv_adma_qc_prep,
486 .qc_issue = nv_adma_qc_issue,
5682ed33 487 .sff_irq_clear = nv_adma_irq_clear,
029cfd6b 488
53014e25
RH		 489 .freeze = nv_adma_freeze,
490 .thaw = nv_adma_thaw,
fbbb262d 491 .error_handler = nv_adma_error_handler,
f5ecac2d 492 .post_internal_cmd = nv_adma_post_internal_cmd,
029cfd6b 493
fbbb262d
RH		 494 .port_start = nv_adma_port_start,
495 .port_stop = nv_adma_port_stop,
438ac6d5 496#ifdef CONFIG_PM
cdf56bcf
RH		 497 .port_suspend = nv_adma_port_suspend,
498 .port_resume = nv_adma_port_resume,
438ac6d5 499#endif
fbbb262d
RH		 500 .host_stop = nv_adma_host_stop,
501};
502
029cfd6b 503static struct ata_port_operations nv_swncq_ops = {
7f4774b3 504 .inherits = &nv_generic_ops,
029cfd6b 505
f140f0f1
KL		 506 .qc_defer = ata_std_qc_defer,
507 .qc_prep = nv_swncq_qc_prep,
508 .qc_issue = nv_swncq_qc_issue,
029cfd6b 509
f140f0f1
KL		 510 .freeze = nv_mcp55_freeze,
511 .thaw = nv_mcp55_thaw,
512 .error_handler = nv_swncq_error_handler,
029cfd6b 513
f140f0f1
KL		 514#ifdef CONFIG_PM
515 .port_suspend = nv_swncq_port_suspend,
516 .port_resume = nv_swncq_port_resume,
517#endif
518 .port_start = nv_swncq_port_start,
519};
520
95947193
TH		 521struct nv_pi_priv {
522 irq_handler_t irq_handler;
25df73d9 523 const struct scsi_host_template *sht;
95947193
TH		 524};
525
526#define NV_PI_PRIV(_irq_handler, _sht) \
527 &(struct nv_pi_priv){ .irq_handler = _irq_handler, .sht = _sht }
528
1626aeb8 529static const struct ata_port_info nv_port_info[] = {
ada364e8
TH		 530 /* generic */
531 {
9cbe056f 532 .flags = ATA_FLAG_SATA,
ada364e8
TH		 533 .pio_mask = NV_PIO_MASK,
534 .mwdma_mask = NV_MWDMA_MASK,
535 .udma_mask = NV_UDMA_MASK,
536 .port_ops = &nv_generic_ops,
95947193 537 .private_data = NV_PI_PRIV(nv_generic_interrupt, &nv_sht),
ada364e8
TH		 538 },
539 /* nforce2/3 */
540 {
9cbe056f 541 .flags = ATA_FLAG_SATA,
ada364e8
TH		 542 .pio_mask = NV_PIO_MASK,
543 .mwdma_mask = NV_MWDMA_MASK,
544 .udma_mask = NV_UDMA_MASK,
545 .port_ops = &nv_nf2_ops,
95947193 546 .private_data = NV_PI_PRIV(nv_nf2_interrupt, &nv_sht),
ada364e8
TH		 547 },
548 /* ck804 */
549 {
9cbe056f 550 .flags = ATA_FLAG_SATA,
ada364e8
TH		 551 .pio_mask = NV_PIO_MASK,
552 .mwdma_mask = NV_MWDMA_MASK,
553 .udma_mask = NV_UDMA_MASK,
554 .port_ops = &nv_ck804_ops,
95947193 555 .private_data = NV_PI_PRIV(nv_ck804_interrupt, &nv_sht),
ada364e8 556 },
fbbb262d
RH		 557 /* ADMA */
558 {
9cbe056f 559 .flags = ATA_FLAG_SATA | ATA_FLAG_NCQ,
fbbb262d
RH		 560 .pio_mask = NV_PIO_MASK,
561 .mwdma_mask = NV_MWDMA_MASK,
562 .udma_mask = NV_UDMA_MASK,
563 .port_ops = &nv_adma_ops,
95947193 564 .private_data = NV_PI_PRIV(nv_adma_interrupt, &nv_adma_sht),
fbbb262d 565 },
2d775708
TH		 566 /* MCP5x */
567 {
9cbe056f 568 .flags = ATA_FLAG_SATA,
2d775708
TH		 569 .pio_mask = NV_PIO_MASK,
570 .mwdma_mask = NV_MWDMA_MASK,
571 .udma_mask = NV_UDMA_MASK,
7f4774b3 572 .port_ops = &nv_generic_ops,
2d775708
TH		 573 .private_data = NV_PI_PRIV(nv_generic_interrupt, &nv_sht),
574 },
f140f0f1
KL		 575 /* SWNCQ */
576 {
9cbe056f 577 .flags = ATA_FLAG_SATA | ATA_FLAG_NCQ,
f140f0f1
KL		 578 .pio_mask = NV_PIO_MASK,
579 .mwdma_mask = NV_MWDMA_MASK,
580 .udma_mask = NV_UDMA_MASK,
581 .port_ops = &nv_swncq_ops,
95947193 582 .private_data = NV_PI_PRIV(nv_swncq_interrupt, &nv_swncq_sht),
f140f0f1 583 },
1da177e4
LT		 584};
585
586MODULE_AUTHOR("NVIDIA");
587MODULE_DESCRIPTION("low-level driver for NVIDIA nForce SATA controller");
588MODULE_LICENSE("GPL");
589MODULE_DEVICE_TABLE(pci, nv_pci_tbl);
590MODULE_VERSION(DRV_VERSION);
591
90ab5ee9 592static bool adma_enabled;
c13aff32 593static bool swncq_enabled = true;
90ab5ee9 594static bool msi_enabled;
fbbb262d 595
2dec7555
RH		 596static void nv_adma_register_mode(struct ata_port *ap)
597{
2dec7555 598 struct nv_adma_port_priv *pp = ap->private_data;
cdf56bcf 599 void __iomem *mmio = pp->ctl_block;
a2cfe81a
RH		 600 u16 tmp, status;
601 int count = 0;
2dec7555
RH		 602
603 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
604 return;
605
a2cfe81a 606 status = readw(mmio + NV_ADMA_STAT);
2dcb407e 607 while (!(status & NV_ADMA_STAT_IDLE) && count < 20) {
a2cfe81a
RH		 608 ndelay(50);
609 status = readw(mmio + NV_ADMA_STAT);
610 count++;
611 }
2dcb407e 612 if (count == 20)
a9a79dfe
JP		 613 ata_port_warn(ap, "timeout waiting for ADMA IDLE, stat=0x%hx\n",
614 status);
a2cfe81a 615
2dec7555
RH		 616 tmp = readw(mmio + NV_ADMA_CTL);
617 writew(tmp & ~NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
618
a2cfe81a
RH		 619 count = 0;
620 status = readw(mmio + NV_ADMA_STAT);
2dcb407e 621 while (!(status & NV_ADMA_STAT_LEGACY) && count < 20) {
a2cfe81a
RH		 622 ndelay(50);
623 status = readw(mmio + NV_ADMA_STAT);
624 count++;
625 }
2dcb407e 626 if (count == 20)
a9a79dfe
JP		 627 ata_port_warn(ap,
628 "timeout waiting for ADMA LEGACY, stat=0x%hx\n",
629 status);
a2cfe81a 630
2dec7555
RH		 631 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
632}
633
634static void nv_adma_mode(struct ata_port *ap)
635{
2dec7555 636 struct nv_adma_port_priv *pp = ap->private_data;
cdf56bcf 637 void __iomem *mmio = pp->ctl_block;
a2cfe81a
RH		 638 u16 tmp, status;
639 int count = 0;
2dec7555
RH		 640
641 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE))
642 return;
f20b16ff 643
2dec7555
RH		 644 WARN_ON(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
645
646 tmp = readw(mmio + NV_ADMA_CTL);
647 writew(tmp | NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
648
a2cfe81a 649 status = readw(mmio + NV_ADMA_STAT);
2dcb407e 650 while (((status & NV_ADMA_STAT_LEGACY) ||
a2cfe81a
RH		 651 !(status & NV_ADMA_STAT_IDLE)) && count < 20) {
652 ndelay(50);
653 status = readw(mmio + NV_ADMA_STAT);
654 count++;
655 }
2dcb407e 656 if (count == 20)
a9a79dfe 657 ata_port_warn(ap,
a2cfe81a
RH		 658 "timeout waiting for ADMA LEGACY clear and IDLE, stat=0x%hx\n",
659 status);
660
2dec7555
RH		 661 pp->flags &= ~NV_ADMA_PORT_REGISTER_MODE;
662}
663
fbbb262d
RH		 664static int nv_adma_slave_config(struct scsi_device *sdev)
665{
666 struct ata_port *ap = ata_shost_to_port(sdev->host);
2dec7555 667 struct nv_adma_port_priv *pp = ap->private_data;
8959d300 668 struct nv_adma_port_priv *port0, *port1;
2dec7555 669 struct pci_dev *pdev = to_pci_dev(ap->host->dev);
8959d300 670 unsigned long segment_boundary, flags;
fbbb262d
RH		 671 unsigned short sg_tablesize;
672 int rc;
2dec7555
RH		 673 int adma_enable;
674 u32 current_reg, new_reg, config_mask;
fbbb262d
RH		 675
676 rc = ata_scsi_slave_config(sdev);
677
678 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
679 /* Not a proper libata device, ignore */
680 return rc;
681
8959d300
RH		 682 spin_lock_irqsave(ap->lock, flags);
683
9af5c9c9 684 if (ap->link.device[sdev->id].class == ATA_DEV_ATAPI) {
fbbb262d
RH		 685 /*
686 * NVIDIA reports that ADMA mode does not support ATAPI commands.
687 * Therefore ATAPI commands are sent through the legacy interface.
688 * However, the legacy interface only supports 32-bit DMA.
689 * Restrict DMA parameters as required by the legacy interface
690 * when an ATAPI device is connected.
691 */
fbbb262d
RH		 692 segment_boundary = ATA_DMA_BOUNDARY;
693 /* Subtract 1 since an extra entry may be needed for padding, see
694 libata-scsi.c */
695 sg_tablesize = LIBATA_MAX_PRD - 1;
f20b16ff 696
2dec7555
RH		 697 /* Since the legacy DMA engine is in use, we need to disable ADMA
698 on the port. */
699 adma_enable = 0;
700 nv_adma_register_mode(ap);
2dcb407e 701 } else {
fbbb262d
RH		 702 segment_boundary = NV_ADMA_DMA_BOUNDARY;
703 sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN;
2dec7555 704 adma_enable = 1;
fbbb262d 705 }
f20b16ff 706
2dec7555
RH		 707 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &current_reg);
708
2dcb407e 709 if (ap->port_no == 1)
2dec7555
RH		 710 config_mask = NV_MCP_SATA_CFG_20_PORT1_EN |
711 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
712 else
713 config_mask = NV_MCP_SATA_CFG_20_PORT0_EN |
714 NV_MCP_SATA_CFG_20_PORT0_PWB_EN;
f20b16ff 715
2dcb407e 716 if (adma_enable) {
2dec7555
RH		 717 new_reg = current_reg | config_mask;
718 pp->flags &= ~NV_ADMA_ATAPI_SETUP_COMPLETE;
2dcb407e 719 } else {
2dec7555
RH		 720 new_reg = current_reg & ~config_mask;
721 pp->flags |= NV_ADMA_ATAPI_SETUP_COMPLETE;
722 }
f20b16ff 723
2dcb407e 724 if (current_reg != new_reg)
2dec7555 725 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, new_reg);
f20b16ff 726
8959d300
RH		 727 port0 = ap->host->ports[0]->private_data;
728 port1 = ap->host->ports[1]->private_data;
8959d300
RH		 729 if ((port0->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
730 (port1->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)) {
258c9fde
CH		 731 /*
732 * We have to set the DMA mask to 32-bit if either port is in
733 * ATAPI mode, since they are on the same PCI device which is
734 * used for DMA mapping. If either SCSI device is not allocated
735 * yet, it's OK since that port will discover its correct
736 * setting when it does get allocated.
737 */
738 rc = dma_set_mask(&pdev->dev, ATA_DMA_MASK);
8959d300 739 } else {
258c9fde 740 rc = dma_set_mask(&pdev->dev, pp->adma_dma_mask);
8959d300
RH		 741 }
742
fbbb262d 743 blk_queue_segment_boundary(sdev->request_queue, segment_boundary);
8a78362c 744 blk_queue_max_segments(sdev->request_queue, sg_tablesize);
a9a79dfe
JP		 745 ata_port_info(ap,
746 "DMA mask 0x%llX, segment boundary 0x%lX, hw segs %hu\n",
747 (unsigned long long)*ap->host->dev->dma_mask,
748 segment_boundary, sg_tablesize);
8959d300
RH		 749
750 spin_unlock_irqrestore(ap->lock, flags);
751
fbbb262d
RH		 752 return rc;
753}
754
2dec7555
RH		 755static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc)
756{
757 struct nv_adma_port_priv *pp = qc->ap->private_data;
758 return !(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
759}
760
f2fb344b
RH		 761static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf)
762{
3f3debdb
RH		 763 /* Other than when internal or pass-through commands are executed,
764 the only time this function will be called in ADMA mode will be
765 if a command fails. In the failure case we don't care about going
766 into register mode with ADMA commands pending, as the commands will
767 all shortly be aborted anyway. We assume that NCQ commands are not
768 issued via passthrough, which is the only way that switching into
769 ADMA mode could abort outstanding commands. */
f2fb344b
RH		 770 nv_adma_register_mode(ap);
771
9363c382 772 ata_sff_tf_read(ap, tf);
f2fb344b
RH		 773}
774
2dec7555 775static unsigned int nv_adma_tf_to_cpb(struct ata_taskfile *tf, __le16 *cpb)
fbbb262d
RH		 776{
777 unsigned int idx = 0;
778
2dcb407e 779 if (tf->flags & ATA_TFLAG_ISADDR) {
ac3d6b86
RH		 780 if (tf->flags & ATA_TFLAG_LBA48) {
781 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->hob_feature | WNB);
782 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->hob_nsect);
783 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->hob_lbal);
784 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->hob_lbam);
785 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->hob_lbah);
786 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature);
787 } else
788 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature | WNB);
a84471fe 789
ac3d6b86
RH		 790 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->nsect);
791 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->lbal);
792 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->lbam);
793 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->lbah);
fbbb262d 794 }
a84471fe 795
2dcb407e 796 if (tf->flags & ATA_TFLAG_DEVICE)
ac3d6b86 797 cpb[idx++] = cpu_to_le16((ATA_REG_DEVICE << 8) | tf->device);
fbbb262d
RH		 798
799 cpb[idx++] = cpu_to_le16((ATA_REG_CMD << 8) | tf->command | CMDEND);
a84471fe 800
2dcb407e 801 while (idx < 12)
ac3d6b86 802 cpb[idx++] = cpu_to_le16(IGN);
fbbb262d
RH		 803
804 return idx;
805}
806
5bd28a4b 807static int nv_adma_check_cpb(struct ata_port *ap, int cpb_num, int force_err)
fbbb262d
RH		 808{
809 struct nv_adma_port_priv *pp = ap->private_data;
2dec7555 810 u8 flags = pp->cpb[cpb_num].resp_flags;
fbbb262d 811
47013c58 812 ata_port_dbg(ap, "CPB %d, flags=0x%x\n", cpb_num, flags);
fbbb262d 813
5bd28a4b
RH		 814 if (unlikely((force_err ||
815 flags & (NV_CPB_RESP_ATA_ERR |
816 NV_CPB_RESP_CMD_ERR |
817 NV_CPB_RESP_CPB_ERR)))) {
9af5c9c9 818 struct ata_eh_info *ehi = &ap->link.eh_info;
5bd28a4b
RH		 819 int freeze = 0;
820
821 ata_ehi_clear_desc(ehi);
2dcb407e 822 __ata_ehi_push_desc(ehi, "CPB resp_flags 0x%x: ", flags);
5bd28a4b 823 if (flags & NV_CPB_RESP_ATA_ERR) {
b64bbc39 824 ata_ehi_push_desc(ehi, "ATA error");
5bd28a4b
RH		 825 ehi->err_mask |= AC_ERR_DEV;
826 } else if (flags & NV_CPB_RESP_CMD_ERR) {
b64bbc39 827 ata_ehi_push_desc(ehi, "CMD error");
5bd28a4b
RH		 828 ehi->err_mask |= AC_ERR_DEV;
829 } else if (flags & NV_CPB_RESP_CPB_ERR) {
b64bbc39 830 ata_ehi_push_desc(ehi, "CPB error");
5bd28a4b
RH		 831 ehi->err_mask |= AC_ERR_SYSTEM;
832 freeze = 1;
833 } else {
834 /* notifier error, but no error in CPB flags? */
b64bbc39 835 ata_ehi_push_desc(ehi, "unknown");
5bd28a4b
RH		 836 ehi->err_mask |= AC_ERR_OTHER;
837 freeze = 1;
838 }
839 /* Kill all commands. EH will determine what actually failed. */
840 if (freeze)
841 ata_port_freeze(ap);
842 else
843 ata_port_abort(ap);
1aadf5c3 844 return -1;
fbbb262d 845 }
5bd28a4b 846
1aadf5c3
TH		 847 if (likely(flags & NV_CPB_RESP_DONE))
848 return 1;
5bd28a4b 849 return 0;
fbbb262d
RH		 850}
851
2dec7555
RH		 852static int nv_host_intr(struct ata_port *ap, u8 irq_stat)
853{
9af5c9c9 854 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, ap->link.active_tag);
2dec7555
RH		 855
856 /* freeze if hotplugged */
857 if (unlikely(irq_stat & (NV_INT_ADDED | NV_INT_REMOVED))) {
858 ata_port_freeze(ap);
859 return 1;
860 }
861
862 /* bail out if not our interrupt */
863 if (!(irq_stat & NV_INT_DEV))
864 return 0;
865
866 /* DEV interrupt w/ no active qc? */
867 if (unlikely(!qc || (qc->tf.flags & ATA_TFLAG_POLLING))) {
9363c382 868 ata_sff_check_status(ap);
2dec7555
RH		 869 return 1;
870 }
871
872 /* handle interrupt */
c3b28894 873 return ata_bmdma_port_intr(ap, qc);
2dec7555
RH		 874}
875
fbbb262d
RH		 876static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance)
877{
878 struct ata_host *host = dev_instance;
879 int i, handled = 0;
2dec7555 880 u32 notifier_clears[2];
fbbb262d
RH		 881
882 spin_lock(&host->lock);
883
884 for (i = 0; i < host->n_ports; i++) {
885 struct ata_port *ap = host->ports[i];
3e4ec344
TH		 886 struct nv_adma_port_priv *pp = ap->private_data;
887 void __iomem *mmio = pp->ctl_block;
888 u16 status;
889 u32 gen_ctl;
890 u32 notifier, notifier_error;
891
2dec7555 892 notifier_clears[i] = 0;
fbbb262d 893
3e4ec344
TH		 894 /* if ADMA is disabled, use standard ata interrupt handler */
895 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
896 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
897 >> (NV_INT_PORT_SHIFT * i);
898 handled += nv_host_intr(ap, irq_stat);
899 continue;
900 }
fbbb262d 901
3e4ec344
TH		 902 /* if in ATA register mode, check for standard interrupts */
903 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE) {
904 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
905 >> (NV_INT_PORT_SHIFT * i);
906 if (ata_tag_valid(ap->link.active_tag))
907 /** NV_INT_DEV indication seems unreliable
908 at times at least in ADMA mode. Force it
909 on always when a command is active, to
910 prevent losing interrupts. */
911 irq_stat |= NV_INT_DEV;
912 handled += nv_host_intr(ap, irq_stat);
913 }
914
915 notifier = readl(mmio + NV_ADMA_NOTIFIER);
916 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
917 notifier_clears[i] = notifier | notifier_error;
918
919 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
920
921 if (!NV_ADMA_CHECK_INTR(gen_ctl, ap->port_no) && !notifier &&
922 !notifier_error)
923 /* Nothing to do */
924 continue;
925
926 status = readw(mmio + NV_ADMA_STAT);
927
928 /*
929 * Clear status. Ensure the controller sees the
930 * clearing before we start looking at any of the CPB
931 * statuses, so that any CPB completions after this
932 * point in the handler will raise another interrupt.
933 */
934 writew(status, mmio + NV_ADMA_STAT);
935 readw(mmio + NV_ADMA_STAT); /* flush posted write */
936 rmb();
fbbb262d 937
3e4ec344
TH		 938 handled++; /* irq handled if we got here */
939
940 /* freeze if hotplugged or controller error */
941 if (unlikely(status & (NV_ADMA_STAT_HOTPLUG |
942 NV_ADMA_STAT_HOTUNPLUG |
943 NV_ADMA_STAT_TIMEOUT |
944 NV_ADMA_STAT_SERROR))) {
945 struct ata_eh_info *ehi = &ap->link.eh_info;
946
947 ata_ehi_clear_desc(ehi);
948 __ata_ehi_push_desc(ehi, "ADMA status 0x%08x: ", status);
949 if (status & NV_ADMA_STAT_TIMEOUT) {
950 ehi->err_mask |= AC_ERR_SYSTEM;
951 ata_ehi_push_desc(ehi, "timeout");
952 } else if (status & NV_ADMA_STAT_HOTPLUG) {
953 ata_ehi_hotplugged(ehi);
954 ata_ehi_push_desc(ehi, "hotplug");
955 } else if (status & NV_ADMA_STAT_HOTUNPLUG) {
956 ata_ehi_hotplugged(ehi);
957 ata_ehi_push_desc(ehi, "hot unplug");
958 } else if (status & NV_ADMA_STAT_SERROR) {
959 /* let EH analyze SError and figure out cause */
960 ata_ehi_push_desc(ehi, "SError");
961 } else
962 ata_ehi_push_desc(ehi, "unknown");
963 ata_port_freeze(ap);
964 continue;
965 }
966
967 if (status & (NV_ADMA_STAT_DONE |
968 NV_ADMA_STAT_CPBERR |
969 NV_ADMA_STAT_CMD_COMPLETE)) {
970 u32 check_commands = notifier_clears[i];
1aadf5c3 971 u32 done_mask = 0;
752e386c 972 int pos, rc;
3e4ec344
TH		 973
974 if (status & NV_ADMA_STAT_CPBERR) {
975 /* check all active commands */
976 if (ata_tag_valid(ap->link.active_tag))
977 check_commands = 1 <<
978 ap->link.active_tag;
979 else
980 check_commands = ap->link.sactive;
fbbb262d
RH		 981 }
982
3e4ec344 983 /* check CPBs for completed commands */
752e386c 984 while ((pos = ffs(check_commands))) {
3e4ec344 985 pos--;
752e386c 986 rc = nv_adma_check_cpb(ap, pos,
5796d1c4 987 notifier_error & (1 << pos));
1aadf5c3
TH		 988 if (rc > 0)
989 done_mask |= 1 << pos;
990 else if (unlikely(rc < 0))
752e386c 991 check_commands = 0;
3e4ec344 992 check_commands &= ~(1 << pos);
fbbb262d 993 }
8385d756 994 ata_qc_complete_multiple(ap, ata_qc_get_active(ap) ^ done_mask);
fbbb262d
RH		 995 }
996 }
f20b16ff 997
b447916e 998 if (notifier_clears[0] || notifier_clears[1]) {
2dec7555
RH		 999 /* Note: Both notifier clear registers must be written
1000 if either is set, even if one is zero, according to NVIDIA. */
cdf56bcf
RH		 1001 struct nv_adma_port_priv *pp = host->ports[0]->private_data;
1002 writel(notifier_clears[0], pp->notifier_clear_block);
1003 pp = host->ports[1]->private_data;
1004 writel(notifier_clears[1], pp->notifier_clear_block);
2dec7555 1005 }
fbbb262d
RH		 1006
1007 spin_unlock(&host->lock);
1008
1009 return IRQ_RETVAL(handled);
1010}
1011
53014e25
RH		 1012static void nv_adma_freeze(struct ata_port *ap)
1013{
1014 struct nv_adma_port_priv *pp = ap->private_data;
1015 void __iomem *mmio = pp->ctl_block;
1016 u16 tmp;
1017
1018 nv_ck804_freeze(ap);
1019
1020 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1021 return;
1022
1023 /* clear any outstanding CK804 notifications */
2dcb407e 1024 writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
53014e25
RH		 1025 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1026
1027 /* Disable interrupt */
1028 tmp = readw(mmio + NV_ADMA_CTL);
2dcb407e 1029 writew(tmp & ~(NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
53014e25 1030 mmio + NV_ADMA_CTL);
5796d1c4 1031 readw(mmio + NV_ADMA_CTL); /* flush posted write */
53014e25
RH		 1032}
1033
1034static void nv_adma_thaw(struct ata_port *ap)
1035{
1036 struct nv_adma_port_priv *pp = ap->private_data;
1037 void __iomem *mmio = pp->ctl_block;
1038 u16 tmp;
1039
1040 nv_ck804_thaw(ap);
1041
1042 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1043 return;
1044
1045 /* Enable interrupt */
1046 tmp = readw(mmio + NV_ADMA_CTL);
2dcb407e 1047 writew(tmp | (NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
53014e25 1048 mmio + NV_ADMA_CTL);
5796d1c4 1049 readw(mmio + NV_ADMA_CTL); /* flush posted write */
53014e25
RH		 1050}
1051
fbbb262d
RH		 1052static void nv_adma_irq_clear(struct ata_port *ap)
1053{
cdf56bcf
RH		 1054 struct nv_adma_port_priv *pp = ap->private_data;
1055 void __iomem *mmio = pp->ctl_block;
53014e25 1056 u32 notifier_clears[2];
fbbb262d 1057
53014e25 1058 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
37f65b8b 1059 ata_bmdma_irq_clear(ap);
53014e25
RH		 1060 return;
1061 }
1062
1063 /* clear any outstanding CK804 notifications */
2dcb407e 1064 writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
53014e25 1065 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
fbbb262d 1066
53014e25
RH		 1067 /* clear ADMA status */
1068 writew(0xffff, mmio + NV_ADMA_STAT);
a617c09f 1069
53014e25
RH		 1070 /* clear notifiers - note both ports need to be written with
1071 something even though we are only clearing on one */
1072 if (ap->port_no == 0) {
1073 notifier_clears[0] = 0xFFFFFFFF;
1074 notifier_clears[1] = 0;
1075 } else {
1076 notifier_clears[0] = 0;
1077 notifier_clears[1] = 0xFFFFFFFF;
1078 }
1079 pp = ap->host->ports[0]->private_data;
1080 writel(notifier_clears[0], pp->notifier_clear_block);
1081 pp = ap->host->ports[1]->private_data;
1082 writel(notifier_clears[1], pp->notifier_clear_block);
fbbb262d
RH		 1083}
1084
f5ecac2d 1085static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc)
fbbb262d 1086{
f5ecac2d 1087 struct nv_adma_port_priv *pp = qc->ap->private_data;
fbbb262d 1088
b447916e 1089 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
fe06e5f9 1090 ata_bmdma_post_internal_cmd(qc);
fbbb262d
RH		 1091}
1092
1093static int nv_adma_port_start(struct ata_port *ap)
1094{
1095 struct device *dev = ap->host->dev;
1096 struct nv_adma_port_priv *pp;
1097 int rc;
1098 void *mem;
1099 dma_addr_t mem_dma;
cdf56bcf 1100 void __iomem *mmio;
8959d300 1101 struct pci_dev *pdev = to_pci_dev(dev);
fbbb262d
RH		 1102 u16 tmp;
1103
258c9fde
CH		 1104 /*
1105 * Ensure DMA mask is set to 32-bit before allocating legacy PRD and
1106 * pad buffers.
1107 */
1108 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
8959d300
RH		 1109 if (rc)
1110 return rc;
1111
c7087652
TH		 1112 /* we might fallback to bmdma, allocate bmdma resources */
1113 rc = ata_bmdma_port_start(ap);
fbbb262d
RH		 1114 if (rc)
1115 return rc;
1116
24dc5f33
TH		 1117 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1118 if (!pp)
1119 return -ENOMEM;
fbbb262d 1120
0d5ff566 1121 mmio = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_PORT +
cdf56bcf
RH		 1122 ap->port_no * NV_ADMA_PORT_SIZE;
1123 pp->ctl_block = mmio;
0d5ff566 1124 pp->gen_block = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_GEN;
cdf56bcf
RH		 1125 pp->notifier_clear_block = pp->gen_block +
1126 NV_ADMA_NOTIFIER_CLEAR + (4 * ap->port_no);
1127
258c9fde
CH		 1128 /*
1129 * Now that the legacy PRD and padding buffer are allocated we can
51872b66 1130 * raise the DMA mask to allocate the CPB/APRD table.
258c9fde 1131 */
51872b66
CH		 1132 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1133
8959d300
RH		 1134 pp->adma_dma_mask = *dev->dma_mask;
1135
24dc5f33
TH		 1136 mem = dmam_alloc_coherent(dev, NV_ADMA_PORT_PRIV_DMA_SZ,
1137 &mem_dma, GFP_KERNEL);
1138 if (!mem)
1139 return -ENOMEM;
fbbb262d
RH		 1140
1141 /*
1142 * First item in chunk of DMA memory:
1143 * 128-byte command parameter block (CPB)
1144 * one for each command tag
1145 */
1146 pp->cpb = mem;
1147 pp->cpb_dma = mem_dma;
1148
1149 writel(mem_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
5796d1c4 1150 writel((mem_dma >> 16) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
fbbb262d
RH		 1151
1152 mem += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1153 mem_dma += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1154
1155 /*
1156 * Second item: block of ADMA_SGTBL_LEN s/g entries
1157 */
1158 pp->aprd = mem;
1159 pp->aprd_dma = mem_dma;
1160
1161 ap->private_data = pp;
1162
1163 /* clear any outstanding interrupt conditions */
1164 writew(0xffff, mmio + NV_ADMA_STAT);
1165
1166 /* initialize port variables */
1167 pp->flags = NV_ADMA_PORT_REGISTER_MODE;
1168
1169 /* clear CPB fetch count */
1170 writew(0, mmio + NV_ADMA_CPB_COUNT);
1171
cdf56bcf 1172 /* clear GO for register mode, enable interrupt */
fbbb262d 1173 tmp = readw(mmio + NV_ADMA_CTL);
5796d1c4
JG		 1174 writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1175 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
fbbb262d
RH		 1176
1177 tmp = readw(mmio + NV_ADMA_CTL);
1178 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
5796d1c4 1179 readw(mmio + NV_ADMA_CTL); /* flush posted write */
fbbb262d
RH		 1180 udelay(1);
1181 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
5796d1c4 1182 readw(mmio + NV_ADMA_CTL); /* flush posted write */
fbbb262d
RH		 1183
1184 return 0;
fbbb262d
RH		 1185}
1186
1187static void nv_adma_port_stop(struct ata_port *ap)
1188{
fbbb262d 1189 struct nv_adma_port_priv *pp = ap->private_data;
cdf56bcf 1190 void __iomem *mmio = pp->ctl_block;
fbbb262d 1191
fbbb262d 1192 writew(0, mmio + NV_ADMA_CTL);
fbbb262d
RH		 1193}
1194
438ac6d5 1195#ifdef CONFIG_PM
cdf56bcf
RH		 1196static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg)
1197{
1198 struct nv_adma_port_priv *pp = ap->private_data;
1199 void __iomem *mmio = pp->ctl_block;
1200
1201 /* Go to register mode - clears GO */
1202 nv_adma_register_mode(ap);
1203
1204 /* clear CPB fetch count */
1205 writew(0, mmio + NV_ADMA_CPB_COUNT);
1206
1207 /* disable interrupt, shut down port */
1208 writew(0, mmio + NV_ADMA_CTL);
1209
1210 return 0;
1211}
1212
1213static int nv_adma_port_resume(struct ata_port *ap)
1214{
1215 struct nv_adma_port_priv *pp = ap->private_data;
1216 void __iomem *mmio = pp->ctl_block;
1217 u16 tmp;
1218
1219 /* set CPB block location */
1220 writel(pp->cpb_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
5796d1c4 1221 writel((pp->cpb_dma >> 16) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
cdf56bcf
RH		 1222
1223 /* clear any outstanding interrupt conditions */
1224 writew(0xffff, mmio + NV_ADMA_STAT);
1225
1226 /* initialize port variables */
1227 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
1228
1229 /* clear CPB fetch count */
1230 writew(0, mmio + NV_ADMA_CPB_COUNT);
1231
1232 /* clear GO for register mode, enable interrupt */
1233 tmp = readw(mmio + NV_ADMA_CTL);
5796d1c4
JG		 1234 writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1235 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
cdf56bcf
RH		 1236
1237 tmp = readw(mmio + NV_ADMA_CTL);
1238 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
5796d1c4 1239 readw(mmio + NV_ADMA_CTL); /* flush posted write */
cdf56bcf
RH		 1240 udelay(1);
1241 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
5796d1c4 1242 readw(mmio + NV_ADMA_CTL); /* flush posted write */
cdf56bcf
RH		 1243
1244 return 0;
1245}
438ac6d5 1246#endif
fbbb262d 1247
9a829ccf 1248static void nv_adma_setup_port(struct ata_port *ap)
fbbb262d 1249{
9a829ccf
TH		 1250 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1251 struct ata_ioports *ioport = &ap->ioaddr;
fbbb262d 1252
9a829ccf 1253 mmio += NV_ADMA_PORT + ap->port_no * NV_ADMA_PORT_SIZE;
fbbb262d 1254
0d5ff566
TH		 1255 ioport->cmd_addr = mmio;
1256 ioport->data_addr = mmio + (ATA_REG_DATA * 4);
fbbb262d 1257 ioport->error_addr =
0d5ff566
TH		 1258 ioport->feature_addr = mmio + (ATA_REG_ERR * 4);
1259 ioport->nsect_addr = mmio + (ATA_REG_NSECT * 4);
1260 ioport->lbal_addr = mmio + (ATA_REG_LBAL * 4);
1261 ioport->lbam_addr = mmio + (ATA_REG_LBAM * 4);
1262 ioport->lbah_addr = mmio + (ATA_REG_LBAH * 4);
1263 ioport->device_addr = mmio + (ATA_REG_DEVICE * 4);
fbbb262d 1264 ioport->status_addr =
0d5ff566 1265 ioport->command_addr = mmio + (ATA_REG_STATUS * 4);
fbbb262d 1266 ioport->altstatus_addr =
0d5ff566 1267 ioport->ctl_addr = mmio + 0x20;
fbbb262d
RH		 1268}
1269
9a829ccf 1270static int nv_adma_host_init(struct ata_host *host)
fbbb262d 1271{
9a829ccf 1272 struct pci_dev *pdev = to_pci_dev(host->dev);
fbbb262d
RH		 1273 unsigned int i;
1274 u32 tmp32;
1275
fbbb262d
RH		 1276 /* enable ADMA on the ports */
1277 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
1278 tmp32 |= NV_MCP_SATA_CFG_20_PORT0_EN |
1279 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
1280 NV_MCP_SATA_CFG_20_PORT1_EN |
1281 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
1282
1283 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
1284
9a829ccf
TH		 1285 for (i = 0; i < host->n_ports; i++)
1286 nv_adma_setup_port(host->ports[i]);
fbbb262d 1287
fbbb262d
RH		 1288 return 0;
1289}
1290
1291static void nv_adma_fill_aprd(struct ata_queued_cmd *qc,
1292 struct scatterlist *sg,
1293 int idx,
1294 struct nv_adma_prd *aprd)
1295{
41949ed5 1296 u8 flags = 0;
fbbb262d
RH		 1297 if (qc->tf.flags & ATA_TFLAG_WRITE)
1298 flags |= NV_APRD_WRITE;
1299 if (idx == qc->n_elem - 1)
1300 flags |= NV_APRD_END;
1301 else if (idx != 4)
1302 flags |= NV_APRD_CONT;
1303
1304 aprd->addr = cpu_to_le64(((u64)sg_dma_address(sg)));
1305 aprd->len = cpu_to_le32(((u32)sg_dma_len(sg))); /* len in bytes */
2dec7555 1306 aprd->flags = flags;
41949ed5 1307 aprd->packet_len = 0;
fbbb262d
RH		 1308}
1309
1310static void nv_adma_fill_sg(struct ata_queued_cmd *qc, struct nv_adma_cpb *cpb)
1311{
1312 struct nv_adma_port_priv *pp = qc->ap->private_data;
fbbb262d
RH		 1313 struct nv_adma_prd *aprd;
1314 struct scatterlist *sg;
ff2aeb1e 1315 unsigned int si;
fbbb262d 1316
ff2aeb1e
TH		 1317 for_each_sg(qc->sg, sg, qc->n_elem, si) {
1318 aprd = (si < 5) ? &cpb->aprd[si] :
4e5b6260 1319 &pp->aprd[NV_ADMA_SGTBL_LEN * qc->hw_tag + (si-5)];
ff2aeb1e 1320 nv_adma_fill_aprd(qc, sg, si, aprd);
fbbb262d 1321 }
ff2aeb1e 1322 if (si > 5)
4e5b6260 1323 cpb->next_aprd = cpu_to_le64(((u64)(pp->aprd_dma + NV_ADMA_SGTBL_SZ * qc->hw_tag)));
41949ed5
RH		 1324 else
1325 cpb->next_aprd = cpu_to_le64(0);
fbbb262d
RH		 1326}
1327
382a6652
RH		 1328static int nv_adma_use_reg_mode(struct ata_queued_cmd *qc)
1329{
1330 struct nv_adma_port_priv *pp = qc->ap->private_data;
1331
1332 /* ADMA engine can only be used for non-ATAPI DMA commands,
3f3debdb 1333 or interrupt-driven no-data commands. */
b447916e 1334 if ((pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
3f3debdb 1335 (qc->tf.flags & ATA_TFLAG_POLLING))
382a6652
RH		 1336 return 1;
1337
b447916e 1338 if ((qc->flags & ATA_QCFLAG_DMAMAP) ||
382a6652
RH		 1339 (qc->tf.protocol == ATA_PROT_NODATA))
1340 return 0;
1341
1342 return 1;
1343}
1344
95364f36 1345static enum ata_completion_errors nv_adma_qc_prep(struct ata_queued_cmd *qc)
fbbb262d
RH		 1346{
1347 struct nv_adma_port_priv *pp = qc->ap->private_data;
4e5b6260 1348 struct nv_adma_cpb *cpb = &pp->cpb[qc->hw_tag];
fbbb262d 1349 u8 ctl_flags = NV_CPB_CTL_CPB_VALID |
fbbb262d
RH		 1350 NV_CPB_CTL_IEN;
1351
382a6652 1352 if (nv_adma_use_reg_mode(qc)) {
3f3debdb
RH		 1353 BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&
1354 (qc->flags & ATA_QCFLAG_DMAMAP));
2dec7555 1355 nv_adma_register_mode(qc->ap);
f47451c4 1356 ata_bmdma_qc_prep(qc);
95364f36 1357 return AC_ERR_OK;
fbbb262d
RH		 1358 }
1359
41949ed5
RH		 1360 cpb->resp_flags = NV_CPB_RESP_DONE;
1361 wmb();
1362 cpb->ctl_flags = 0;
1363 wmb();
fbbb262d
RH		 1364
1365 cpb->len = 3;
4e5b6260 1366 cpb->tag = qc->hw_tag;
fbbb262d
RH		 1367 cpb->next_cpb_idx = 0;
1368
1369 /* turn on NCQ flags for NCQ commands */
1370 if (qc->tf.protocol == ATA_PROT_NCQ)
1371 ctl_flags |= NV_CPB_CTL_QUEUE | NV_CPB_CTL_FPDMA;
1372
1373 nv_adma_tf_to_cpb(&qc->tf, cpb->tf);
1374
b447916e 1375 if (qc->flags & ATA_QCFLAG_DMAMAP) {
382a6652
RH		 1376 nv_adma_fill_sg(qc, cpb);
1377 ctl_flags |= NV_CPB_CTL_APRD_VALID;
1378 } else
1379 memset(&cpb->aprd[0], 0, sizeof(struct nv_adma_prd) * 5);
fbbb262d 1380
5796d1c4
JG		 1381 /* Be paranoid and don't let the device see NV_CPB_CTL_CPB_VALID
1382 until we are finished filling in all of the contents */
fbbb262d
RH		 1383 wmb();
1384 cpb->ctl_flags = ctl_flags;
41949ed5
RH		 1385 wmb();
1386 cpb->resp_flags = 0;
95364f36
JS		 1387
1388 return AC_ERR_OK;
fbbb262d
RH		 1389}
1390
1391static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc)
1392{
2dec7555 1393 struct nv_adma_port_priv *pp = qc->ap->private_data;
cdf56bcf 1394 void __iomem *mmio = pp->ctl_block;
5e5c74a5 1395 int curr_ncq = (qc->tf.protocol == ATA_PROT_NCQ);
fbbb262d 1396
3f3debdb
RH		 1397 /* We can't handle result taskfile with NCQ commands, since
1398 retrieving the taskfile switches us out of ADMA mode and would abort
1399 existing commands. */
1400 if (unlikely(qc->tf.protocol == ATA_PROT_NCQ &&
1401 (qc->flags & ATA_QCFLAG_RESULT_TF))) {
a9a79dfe 1402 ata_dev_err(qc->dev, "NCQ w/ RESULT_TF not allowed\n");
3f3debdb
RH		 1403 return AC_ERR_SYSTEM;
1404 }
1405
382a6652 1406 if (nv_adma_use_reg_mode(qc)) {
fbbb262d 1407 /* use ATA register mode */
3f3debdb
RH		 1408 BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&
1409 (qc->flags & ATA_QCFLAG_DMAMAP));
fbbb262d 1410 nv_adma_register_mode(qc->ap);
360ff783 1411 return ata_bmdma_qc_issue(qc);
fbbb262d
RH		 1412 } else
1413 nv_adma_mode(qc->ap);
1414
1415 /* write append register, command tag in lower 8 bits
1416 and (number of cpbs to append -1) in top 8 bits */
1417 wmb();
5e5c74a5 1418
b447916e 1419 if (curr_ncq != pp->last_issue_ncq) {
5796d1c4
JG		 1420 /* Seems to need some delay before switching between NCQ and
1421 non-NCQ commands, else we get command timeouts and such. */
5e5c74a5
RH		 1422 udelay(20);
1423 pp->last_issue_ncq = curr_ncq;
1424 }
1425
4e5b6260 1426 writew(qc->hw_tag, mmio + NV_ADMA_APPEND);
fbbb262d 1427
fbbb262d
RH		 1428 return 0;
1429}
1430
7d12e780 1431static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance)
1da177e4 1432{
cca3974e 1433 struct ata_host *host = dev_instance;
1da177e4
LT		 1434 unsigned int i;
1435 unsigned int handled = 0;
1436 unsigned long flags;
1437
cca3974e 1438 spin_lock_irqsave(&host->lock, flags);
1da177e4 1439
cca3974e 1440 for (i = 0; i < host->n_ports; i++) {
3e4ec344
TH		 1441 struct ata_port *ap = host->ports[i];
1442 struct ata_queued_cmd *qc;
1da177e4 1443
3e4ec344
TH		 1444 qc = ata_qc_from_tag(ap, ap->link.active_tag);
1445 if (qc && (!(qc->tf.flags & ATA_TFLAG_POLLING))) {
c3b28894 1446 handled += ata_bmdma_port_intr(ap, qc);
3e4ec344
TH		 1447 } else {
1448 /*
1449 * No request pending? Clear interrupt status
1450 * anyway, in case there's one pending.
1451 */
1452 ap->ops->sff_check_status(ap);
1453 }
1da177e4
LT		 1454 }
1455
cca3974e 1456 spin_unlock_irqrestore(&host->lock, flags);
1da177e4
LT		 1457
1458 return IRQ_RETVAL(handled);
1459}
1460
cca3974e 1461static irqreturn_t nv_do_interrupt(struct ata_host *host, u8 irq_stat)
ada364e8
TH		 1462{
1463 int i, handled = 0;
1464
cca3974e 1465 for (i = 0; i < host->n_ports; i++) {
3e4ec344 1466 handled += nv_host_intr(host->ports[i], irq_stat);
ada364e8
TH		 1467 irq_stat >>= NV_INT_PORT_SHIFT;
1468 }
1469
1470 return IRQ_RETVAL(handled);
1471}
1472
7d12e780 1473static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance)
ada364e8 1474{
cca3974e 1475 struct ata_host *host = dev_instance;
ada364e8
TH		 1476 u8 irq_stat;
1477 irqreturn_t ret;
1478
cca3974e 1479 spin_lock(&host->lock);
0d5ff566 1480 irq_stat = ioread8(host->ports[0]->ioaddr.scr_addr + NV_INT_STATUS);
cca3974e
JG		 1481 ret = nv_do_interrupt(host, irq_stat);
1482 spin_unlock(&host->lock);
ada364e8
TH		 1483
1484 return ret;
1485}
1486
7d12e780 1487static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance)
ada364e8 1488{
cca3974e 1489 struct ata_host *host = dev_instance;
ada364e8
TH		 1490 u8 irq_stat;
1491 irqreturn_t ret;
1492
cca3974e 1493 spin_lock(&host->lock);
0d5ff566 1494 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
cca3974e
JG		 1495 ret = nv_do_interrupt(host, irq_stat);
1496 spin_unlock(&host->lock);
ada364e8
TH		 1497
1498 return ret;
1499}
1500
82ef04fb 1501static int nv_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val)
1da177e4 1502{
1da177e4 1503 if (sc_reg > SCR_CONTROL)
da3dbb17 1504 return -EINVAL;
1da177e4 1505
82ef04fb 1506 *val = ioread32(link->ap->ioaddr.scr_addr + (sc_reg * 4));
da3dbb17 1507 return 0;
1da177e4
LT		 1508}
1509
82ef04fb 1510static int nv_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val)
1da177e4 1511{
1da177e4 1512 if (sc_reg > SCR_CONTROL)
da3dbb17 1513 return -EINVAL;
1da177e4 1514
82ef04fb 1515 iowrite32(val, link->ap->ioaddr.scr_addr + (sc_reg * 4));
da3dbb17 1516 return 0;
1da177e4
LT		 1517}
1518
7f4774b3
TH		 1519static int nv_hardreset(struct ata_link *link, unsigned int *class,
1520 unsigned long deadline)
e8caa3c7 1521{
7f4774b3 1522 struct ata_eh_context *ehc = &link->eh_context;
e8caa3c7 1523
7f4774b3
TH		 1524 /* Do hardreset iff it's post-boot probing, please read the
1525 * comment above port ops for details.
1526 */
1527 if (!(link->ap->pflags & ATA_PFLAG_LOADING) &&
1528 !ata_dev_enabled(link->device))
1529 sata_link_hardreset(link, sata_deb_timing_hotplug, deadline,
1530 NULL, NULL);
6489e326
TH		 1531 else {
1532 const unsigned long *timing = sata_ehc_deb_timing(ehc);
1533 int rc;
1534
1535 if (!(ehc->i.flags & ATA_EHI_QUIET))
a9a79dfe
JP		 1536 ata_link_info(link,
1537 "nv: skipping hardreset on occupied port\n");
6489e326
TH		 1538
1539 /* make sure the link is online */
1540 rc = sata_link_resume(link, timing, deadline);
1541 /* whine about phy resume failure but proceed */
1542 if (rc && rc != -EOPNOTSUPP)
a9a79dfe
JP		 1543 ata_link_warn(link, "failed to resume link (errno=%d)\n",
1544 rc);
6489e326 1545 }
7f4774b3
TH		 1546
1547 /* device signature acquisition is unreliable */
1548 return -EAGAIN;
e8caa3c7
TH		 1549}
1550
39f87582
TH		 1551static void nv_nf2_freeze(struct ata_port *ap)
1552{
0d5ff566 1553 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
39f87582
TH		 1554 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1555 u8 mask;
1556
0d5ff566 1557 mask = ioread8(scr_addr + NV_INT_ENABLE);
39f87582 1558 mask &= ~(NV_INT_ALL << shift);
0d5ff566 1559 iowrite8(mask, scr_addr + NV_INT_ENABLE);
39f87582
TH		 1560}
1561
1562static void nv_nf2_thaw(struct ata_port *ap)
1563{
0d5ff566 1564 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
39f87582
TH		 1565 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1566 u8 mask;
1567
0d5ff566 1568 iowrite8(NV_INT_ALL << shift, scr_addr + NV_INT_STATUS);
39f87582 1569
0d5ff566 1570 mask = ioread8(scr_addr + NV_INT_ENABLE);
39f87582 1571 mask |= (NV_INT_MASK << shift);
0d5ff566 1572 iowrite8(mask, scr_addr + NV_INT_ENABLE);
39f87582
TH		 1573}
1574
1575static void nv_ck804_freeze(struct ata_port *ap)
1576{
0d5ff566 1577 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
39f87582
TH		 1578 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1579 u8 mask;
1580
1581 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1582 mask &= ~(NV_INT_ALL << shift);
1583 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1584}
1585
1586static void nv_ck804_thaw(struct ata_port *ap)
1587{
0d5ff566 1588 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
39f87582
TH		 1589 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1590 u8 mask;
1591
1592 writeb(NV_INT_ALL << shift, mmio_base + NV_INT_STATUS_CK804);
1593
1594 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1595 mask |= (NV_INT_MASK << shift);
1596 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1597}
1598
f140f0f1
KL		 1599static void nv_mcp55_freeze(struct ata_port *ap)
1600{
1601 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1602 int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1603 u32 mask;
1604
1605 writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1606
1607 mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1608 mask &= ~(NV_INT_ALL_MCP55 << shift);
1609 writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
f140f0f1
KL		 1610}
1611
1612static void nv_mcp55_thaw(struct ata_port *ap)
1613{
1614 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1615 int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1616 u32 mask;
1617
1618 writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1619
1620 mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1621 mask |= (NV_INT_MASK_MCP55 << shift);
1622 writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
f140f0f1
KL		 1623}
1624
fbbb262d
RH		 1625static void nv_adma_error_handler(struct ata_port *ap)
1626{
1627 struct nv_adma_port_priv *pp = ap->private_data;
b447916e 1628 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) {
cdf56bcf 1629 void __iomem *mmio = pp->ctl_block;
fbbb262d
RH		 1630 int i;
1631 u16 tmp;
a84471fe 1632
b447916e 1633 if (ata_tag_valid(ap->link.active_tag) || ap->link.sactive) {
2cb27853
RH		 1634 u32 notifier = readl(mmio + NV_ADMA_NOTIFIER);
1635 u32 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
1636 u32 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
1637 u32 status = readw(mmio + NV_ADMA_STAT);
08af7414
RH		 1638 u8 cpb_count = readb(mmio + NV_ADMA_CPB_COUNT);
1639 u8 next_cpb_idx = readb(mmio + NV_ADMA_NEXT_CPB_IDX);
2cb27853 1640
a9a79dfe 1641 ata_port_err(ap,
5796d1c4 1642 "EH in ADMA mode, notifier 0x%X "
08af7414
RH		 1643 "notifier_error 0x%X gen_ctl 0x%X status 0x%X "
1644 "next cpb count 0x%X next cpb idx 0x%x\n",
1645 notifier, notifier_error, gen_ctl, status,
1646 cpb_count, next_cpb_idx);
2cb27853 1647
b447916e 1648 for (i = 0; i < NV_ADMA_MAX_CPBS; i++) {
2cb27853 1649 struct nv_adma_cpb *cpb = &pp->cpb[i];
b447916e 1650 if ((ata_tag_valid(ap->link.active_tag) && i == ap->link.active_tag) ||
5796d1c4 1651 ap->link.sactive & (1 << i))
a9a79dfe 1652 ata_port_err(ap,
2cb27853
RH		 1653 "CPB %d: ctl_flags 0x%x, resp_flags 0x%x\n",
1654 i, cpb->ctl_flags, cpb->resp_flags);
1655 }
1656 }
fbbb262d 1657
fbbb262d
RH		 1658 /* Push us back into port register mode for error handling. */
1659 nv_adma_register_mode(ap);
1660
5796d1c4
JG		 1661 /* Mark all of the CPBs as invalid to prevent them from
1662 being executed */
b447916e 1663 for (i = 0; i < NV_ADMA_MAX_CPBS; i++)
fbbb262d
RH		 1664 pp->cpb[i].ctl_flags &= ~NV_CPB_CTL_CPB_VALID;
1665
1666 /* clear CPB fetch count */
1667 writew(0, mmio + NV_ADMA_CPB_COUNT);
1668
1669 /* Reset channel */
1670 tmp = readw(mmio + NV_ADMA_CTL);
1671 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
b447916e 1672 readw(mmio + NV_ADMA_CTL); /* flush posted write */
fbbb262d
RH		 1673 udelay(1);
1674 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
b447916e 1675 readw(mmio + NV_ADMA_CTL); /* flush posted write */
fbbb262d
RH		 1676 }
1677
fe06e5f9 1678 ata_bmdma_error_handler(ap);
fbbb262d
RH		 1679}
1680
f140f0f1
KL		 1681static void nv_swncq_qc_to_dq(struct ata_port *ap, struct ata_queued_cmd *qc)
1682{
1683 struct nv_swncq_port_priv *pp = ap->private_data;
1684 struct defer_queue *dq = &pp->defer_queue;
1685
1686 /* queue is full */
1687 WARN_ON(dq->tail - dq->head == ATA_MAX_QUEUE);
4e5b6260
JA		 1688 dq->defer_bits |= (1 << qc->hw_tag);
1689 dq->tag[dq->tail++ & (ATA_MAX_QUEUE - 1)] = qc->hw_tag;
f140f0f1
KL		 1690}
1691
1692static struct ata_queued_cmd *nv_swncq_qc_from_dq(struct ata_port *ap)
1693{
1694 struct nv_swncq_port_priv *pp = ap->private_data;
1695 struct defer_queue *dq = &pp->defer_queue;
1696 unsigned int tag;
1697
1698 if (dq->head == dq->tail) /* null queue */
1699 return NULL;
1700
1701 tag = dq->tag[dq->head & (ATA_MAX_QUEUE - 1)];
1702 dq->tag[dq->head++ & (ATA_MAX_QUEUE - 1)] = ATA_TAG_POISON;
1703 WARN_ON(!(dq->defer_bits & (1 << tag)));
1704 dq->defer_bits &= ~(1 << tag);
1705
1706 return ata_qc_from_tag(ap, tag);
1707}
1708
1709static void nv_swncq_fis_reinit(struct ata_port *ap)
1710{
1711 struct nv_swncq_port_priv *pp = ap->private_data;
1712
1713 pp->dhfis_bits = 0;
1714 pp->dmafis_bits = 0;
1715 pp->sdbfis_bits = 0;
1716 pp->ncq_flags = 0;
1717}
1718
1719static void nv_swncq_pp_reinit(struct ata_port *ap)
1720{
1721 struct nv_swncq_port_priv *pp = ap->private_data;
1722 struct defer_queue *dq = &pp->defer_queue;
1723
1724 dq->head = 0;
1725 dq->tail = 0;
1726 dq->defer_bits = 0;
1727 pp->qc_active = 0;
1728 pp->last_issue_tag = ATA_TAG_POISON;
1729 nv_swncq_fis_reinit(ap);
1730}
1731
1732static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis)
1733{
1734 struct nv_swncq_port_priv *pp = ap->private_data;
1735
1736 writew(fis, pp->irq_block);
1737}
1738
1739static void __ata_bmdma_stop(struct ata_port *ap)
1740{
1741 struct ata_queued_cmd qc;
1742
1743 qc.ap = ap;
1744 ata_bmdma_stop(&qc);
1745}
1746
1747static void nv_swncq_ncq_stop(struct ata_port *ap)
1748{
1749 struct nv_swncq_port_priv *pp = ap->private_data;
1750 unsigned int i;
1751 u32 sactive;
1752 u32 done_mask;
1753
e3ed8939 1754 ata_port_err(ap, "EH in SWNCQ mode,QC:qc_active 0x%llX sactive 0x%X\n",
a9a79dfe
JP		 1755 ap->qc_active, ap->link.sactive);
1756 ata_port_err(ap,
f140f0f1
KL		 1757 "SWNCQ:qc_active 0x%X defer_bits 0x%X last_issue_tag 0x%x\n "
1758 "dhfis 0x%X dmafis 0x%X sdbfis 0x%X\n",
1759 pp->qc_active, pp->defer_queue.defer_bits, pp->last_issue_tag,
1760 pp->dhfis_bits, pp->dmafis_bits, pp->sdbfis_bits);
1761
a9a79dfe
JP		 1762 ata_port_err(ap, "ATA_REG 0x%X ERR_REG 0x%X\n",
1763 ap->ops->sff_check_status(ap),
1764 ioread8(ap->ioaddr.error_addr));
f140f0f1
KL		 1765
1766 sactive = readl(pp->sactive_block);
1767 done_mask = pp->qc_active ^ sactive;
1768
a9a79dfe 1769 ata_port_err(ap, "tag : dhfis dmafis sdbfis sactive\n");
f140f0f1
KL		 1770 for (i = 0; i < ATA_MAX_QUEUE; i++) {
1771 u8 err = 0;
1772 if (pp->qc_active & (1 << i))
1773 err = 0;
1774 else if (done_mask & (1 << i))
1775 err = 1;
1776 else
1777 continue;
1778
a9a79dfe
JP		 1779 ata_port_err(ap,
1780 "tag 0x%x: %01x %01x %01x %01x %s\n", i,
1781 (pp->dhfis_bits >> i) & 0x1,
1782 (pp->dmafis_bits >> i) & 0x1,
1783 (pp->sdbfis_bits >> i) & 0x1,
1784 (sactive >> i) & 0x1,
1785 (err ? "error! tag doesn't exit" : " "));
f140f0f1
KL		 1786 }
1787
1788 nv_swncq_pp_reinit(ap);
5682ed33 1789 ap->ops->sff_irq_clear(ap);
f140f0f1
KL		 1790 __ata_bmdma_stop(ap);
1791 nv_swncq_irq_clear(ap, 0xffff);
1792}
1793
1794static void nv_swncq_error_handler(struct ata_port *ap)
1795{
1796 struct ata_eh_context *ehc = &ap->link.eh_context;
1797
1798 if (ap->link.sactive) {
1799 nv_swncq_ncq_stop(ap);
cf480626 1800 ehc->i.action |= ATA_EH_RESET;
f140f0f1
KL		 1801 }
1802
fe06e5f9 1803 ata_bmdma_error_handler(ap);
f140f0f1
KL		 1804}
1805
1806#ifdef CONFIG_PM
1807static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg)
1808{
1809 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1810 u32 tmp;
1811
1812 /* clear irq */
1813 writel(~0, mmio + NV_INT_STATUS_MCP55);
1814
1815 /* disable irq */
1816 writel(0, mmio + NV_INT_ENABLE_MCP55);
1817
1818 /* disable swncq */
1819 tmp = readl(mmio + NV_CTL_MCP55);
1820 tmp &= ~(NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ);
1821 writel(tmp, mmio + NV_CTL_MCP55);
1822
1823 return 0;
1824}
1825
1826static int nv_swncq_port_resume(struct ata_port *ap)
1827{
1828 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1829 u32 tmp;
1830
1831 /* clear irq */
1832 writel(~0, mmio + NV_INT_STATUS_MCP55);
1833
1834 /* enable irq */
1835 writel(0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1836
1837 /* enable swncq */
1838 tmp = readl(mmio + NV_CTL_MCP55);
1839 writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1840
1841 return 0;
1842}
1843#endif
1844
1845static void nv_swncq_host_init(struct ata_host *host)
1846{
1847 u32 tmp;
1848 void __iomem *mmio = host->iomap[NV_MMIO_BAR];
1849 struct pci_dev *pdev = to_pci_dev(host->dev);
1850 u8 regval;
1851
1852 /* disable ECO 398 */
1853 pci_read_config_byte(pdev, 0x7f, &regval);
1854 regval &= ~(1 << 7);
1855 pci_write_config_byte(pdev, 0x7f, regval);
1856
1857 /* enable swncq */
1858 tmp = readl(mmio + NV_CTL_MCP55);
47013c58 1859 dev_dbg(&pdev->dev, "HOST_CTL:0x%X\n", tmp);
f140f0f1
KL		 1860 writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1861
1862 /* enable irq intr */
1863 tmp = readl(mmio + NV_INT_ENABLE_MCP55);
47013c58 1864 dev_dbg(&pdev->dev, "HOST_ENABLE:0x%X\n", tmp);
f140f0f1
KL		 1865 writel(tmp | 0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1866
1867 /* clear port irq */
1868 writel(~0x0, mmio + NV_INT_STATUS_MCP55);
1869}
1870
1871static int nv_swncq_slave_config(struct scsi_device *sdev)
1872{
1873 struct ata_port *ap = ata_shost_to_port(sdev->host);
1874 struct pci_dev *pdev = to_pci_dev(ap->host->dev);
1875 struct ata_device *dev;
1876 int rc;
1877 u8 rev;
1878 u8 check_maxtor = 0;
1879 unsigned char model_num[ATA_ID_PROD_LEN + 1];
1880
1881 rc = ata_scsi_slave_config(sdev);
1882 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
1883 /* Not a proper libata device, ignore */
1884 return rc;
1885
1886 dev = &ap->link.device[sdev->id];
1887 if (!(ap->flags & ATA_FLAG_NCQ) || dev->class == ATA_DEV_ATAPI)
1888 return rc;
1889
1890 /* if MCP51 and Maxtor, then disable ncq */
1891 if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA ||
1892 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2)
1893 check_maxtor = 1;
1894
1895 /* if MCP55 and rev <= a2 and Maxtor, then disable ncq */
1896 if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA ||
1897 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2) {
1898 pci_read_config_byte(pdev, 0x8, &rev);
1899 if (rev <= 0xa2)
1900 check_maxtor = 1;
1901 }
1902
1903 if (!check_maxtor)
1904 return rc;
1905
1906 ata_id_c_string(dev->id, model_num, ATA_ID_PROD, sizeof(model_num));
1907
1908 if (strncmp(model_num, "Maxtor", 6) == 0) {
db5ed4df 1909 ata_scsi_change_queue_depth(sdev, 1);
a9a79dfe
JP		 1910 ata_dev_notice(dev, "Disabling SWNCQ mode (depth %x)\n",
1911 sdev->queue_depth);
f140f0f1
KL		 1912 }
1913
1914 return rc;
1915}
1916
1917static int nv_swncq_port_start(struct ata_port *ap)
1918{
1919 struct device *dev = ap->host->dev;
1920 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1921 struct nv_swncq_port_priv *pp;
1922 int rc;
1923
c7087652
TH		 1924 /* we might fallback to bmdma, allocate bmdma resources */
1925 rc = ata_bmdma_port_start(ap);
f140f0f1
KL		 1926 if (rc)
1927 return rc;
1928
1929 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1930 if (!pp)
1931 return -ENOMEM;
1932
1933 pp->prd = dmam_alloc_coherent(dev, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE,
1934 &pp->prd_dma, GFP_KERNEL);
1935 if (!pp->prd)
1936 return -ENOMEM;
f140f0f1
KL		 1937
1938 ap->private_data = pp;
1939 pp->sactive_block = ap->ioaddr.scr_addr + 4 * SCR_ACTIVE;
1940 pp->irq_block = mmio + NV_INT_STATUS_MCP55 + ap->port_no * 2;
1941 pp->tag_block = mmio + NV_NCQ_REG_MCP55 + ap->port_no * 2;
1942
1943 return 0;
1944}
1945
95364f36 1946static enum ata_completion_errors nv_swncq_qc_prep(struct ata_queued_cmd *qc)
f140f0f1
KL		 1947{
1948 if (qc->tf.protocol != ATA_PROT_NCQ) {
f47451c4 1949 ata_bmdma_qc_prep(qc);
95364f36 1950 return AC_ERR_OK;
f140f0f1
KL		 1951 }
1952
1953 if (!(qc->flags & ATA_QCFLAG_DMAMAP))
95364f36 1954 return AC_ERR_OK;
f140f0f1
KL		 1955
1956 nv_swncq_fill_sg(qc);
95364f36
JS		 1957
1958 return AC_ERR_OK;
f140f0f1
KL		 1959}
1960
1961static void nv_swncq_fill_sg(struct ata_queued_cmd *qc)
1962{
1963 struct ata_port *ap = qc->ap;
1964 struct scatterlist *sg;
f140f0f1 1965 struct nv_swncq_port_priv *pp = ap->private_data;
f60d7011 1966 struct ata_bmdma_prd *prd;
ff2aeb1e 1967 unsigned int si, idx;
f140f0f1 1968
4e5b6260 1969 prd = pp->prd + ATA_MAX_PRD * qc->hw_tag;
f140f0f1
KL		 1970
1971 idx = 0;
ff2aeb1e 1972 for_each_sg(qc->sg, sg, qc->n_elem, si) {
f140f0f1
KL		 1973 u32 addr, offset;
1974 u32 sg_len, len;
1975
1976 addr = (u32)sg_dma_address(sg);
1977 sg_len = sg_dma_len(sg);
1978
1979 while (sg_len) {
1980 offset = addr & 0xffff;
1981 len = sg_len;
1982 if ((offset + sg_len) > 0x10000)
1983 len = 0x10000 - offset;
1984
1985 prd[idx].addr = cpu_to_le32(addr);
1986 prd[idx].flags_len = cpu_to_le32(len & 0xffff);
1987
1988 idx++;
1989 sg_len -= len;
1990 addr += len;
1991 }
1992 }
1993
ff2aeb1e 1994 prd[idx - 1].flags_len |= cpu_to_le32(ATA_PRD_EOT);
f140f0f1
KL		 1995}
1996
1997static unsigned int nv_swncq_issue_atacmd(struct ata_port *ap,
1998 struct ata_queued_cmd *qc)
1999{
2000 struct nv_swncq_port_priv *pp = ap->private_data;
2001
2002 if (qc == NULL)
2003 return 0;
2004
4e5b6260
JA		 2005 writel((1 << qc->hw_tag), pp->sactive_block);
2006 pp->last_issue_tag = qc->hw_tag;
2007 pp->dhfis_bits &= ~(1 << qc->hw_tag);
2008 pp->dmafis_bits &= ~(1 << qc->hw_tag);
2009 pp->qc_active |= (0x1 << qc->hw_tag);
f140f0f1 2010
c206a389 2011 trace_ata_tf_load(ap, &qc->tf);
5682ed33 2012 ap->ops->sff_tf_load(ap, &qc->tf); /* load tf registers */
c206a389 2013 trace_ata_exec_command(ap, &qc->tf, qc->hw_tag);
5682ed33 2014 ap->ops->sff_exec_command(ap, &qc->tf);
f140f0f1 2015
f140f0f1
KL		 2016 return 0;
2017}
2018
2019static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc)
2020{
2021 struct ata_port *ap = qc->ap;
2022 struct nv_swncq_port_priv *pp = ap->private_data;
2023
2024 if (qc->tf.protocol != ATA_PROT_NCQ)
360ff783 2025 return ata_bmdma_qc_issue(qc);
f140f0f1 2026
f140f0f1
KL		 2027 if (!pp->qc_active)
2028 nv_swncq_issue_atacmd(ap, qc);
2029 else
2030 nv_swncq_qc_to_dq(ap, qc); /* add qc to defer queue */
2031
2032 return 0;
2033}
2034
2035static void nv_swncq_hotplug(struct ata_port *ap, u32 fis)
2036{
2037 u32 serror;
2038 struct ata_eh_info *ehi = &ap->link.eh_info;
2039
2040 ata_ehi_clear_desc(ehi);
2041
2042 /* AHCI needs SError cleared; otherwise, it might lock up */
2043 sata_scr_read(&ap->link, SCR_ERROR, &serror);
2044 sata_scr_write(&ap->link, SCR_ERROR, serror);
2045
2046 /* analyze @irq_stat */
2047 if (fis & NV_SWNCQ_IRQ_ADDED)
2048 ata_ehi_push_desc(ehi, "hot plug");
2049 else if (fis & NV_SWNCQ_IRQ_REMOVED)
2050 ata_ehi_push_desc(ehi, "hot unplug");
2051
2052 ata_ehi_hotplugged(ehi);
2053
2054 /* okay, let's hand over to EH */
2055 ehi->serror |= serror;
2056
2057 ata_port_freeze(ap);
2058}
2059
2060static int nv_swncq_sdbfis(struct ata_port *ap)
2061{
2062 struct ata_queued_cmd *qc;
2063 struct nv_swncq_port_priv *pp = ap->private_data;
2064 struct ata_eh_info *ehi = &ap->link.eh_info;
2065 u32 sactive;
f140f0f1 2066 u32 done_mask;
f140f0f1
KL		 2067 u8 host_stat;
2068 u8 lack_dhfis = 0;
2069
2070 host_stat = ap->ops->bmdma_status(ap);
c206a389 2071 trace_ata_bmdma_status(ap, host_stat);
f140f0f1 2072 if (unlikely(host_stat & ATA_DMA_ERR)) {
25985edc 2073 /* error when transferring data to/from memory */
f140f0f1
KL		 2074 ata_ehi_clear_desc(ehi);
2075 ata_ehi_push_desc(ehi, "BMDMA stat 0x%x", host_stat);
2076 ehi->err_mask |= AC_ERR_HOST_BUS;
cf480626 2077 ehi->action |= ATA_EH_RESET;
f140f0f1
KL		 2078 return -EINVAL;
2079 }
2080
5682ed33 2081 ap->ops->sff_irq_clear(ap);
f140f0f1
KL		 2082 __ata_bmdma_stop(ap);
2083
2084 sactive = readl(pp->sactive_block);
2085 done_mask = pp->qc_active ^ sactive;
2086
1aadf5c3
TH		 2087 pp->qc_active &= ~done_mask;
2088 pp->dhfis_bits &= ~done_mask;
2089 pp->dmafis_bits &= ~done_mask;
2090 pp->sdbfis_bits |= done_mask;
8e4c309f 2091 ata_qc_complete_multiple(ap, ata_qc_get_active(ap) ^ done_mask);
f140f0f1
KL		 2092
2093 if (!ap->qc_active) {
47013c58 2094 ata_port_dbg(ap, "over\n");
f140f0f1 2095 nv_swncq_pp_reinit(ap);
752e386c 2096 return 0;
f140f0f1
KL		 2097 }
2098
2099 if (pp->qc_active & pp->dhfis_bits)
752e386c 2100 return 0;
f140f0f1
KL		 2101
2102 if ((pp->ncq_flags & ncq_saw_backout) ||
2103 (pp->qc_active ^ pp->dhfis_bits))
752e386c 2104 /* if the controller can't get a device to host register FIS,
f140f0f1
KL		 2105 * The driver needs to reissue the new command.
2106 */
2107 lack_dhfis = 1;
2108
47013c58
HR		 2109 ata_port_dbg(ap, "QC: qc_active 0x%llx,"
2110 "SWNCQ:qc_active 0x%X defer_bits %X "
2111 "dhfis 0x%X dmafis 0x%X last_issue_tag %x\n",
2112 ap->qc_active, pp->qc_active,
2113 pp->defer_queue.defer_bits, pp->dhfis_bits,
2114 pp->dmafis_bits, pp->last_issue_tag);
f140f0f1
KL		 2115
2116 nv_swncq_fis_reinit(ap);
2117
2118 if (lack_dhfis) {
2119 qc = ata_qc_from_tag(ap, pp->last_issue_tag);
2120 nv_swncq_issue_atacmd(ap, qc);
752e386c 2121 return 0;
f140f0f1
KL		 2122 }
2123
2124 if (pp->defer_queue.defer_bits) {
2125 /* send deferral queue command */
2126 qc = nv_swncq_qc_from_dq(ap);
2127 WARN_ON(qc == NULL);
2128 nv_swncq_issue_atacmd(ap, qc);
2129 }
2130
752e386c 2131 return 0;
f140f0f1
KL		 2132}
2133
2134static inline u32 nv_swncq_tag(struct ata_port *ap)
2135{
2136 struct nv_swncq_port_priv *pp = ap->private_data;
2137 u32 tag;
2138
2139 tag = readb(pp->tag_block) >> 2;
2140 return (tag & 0x1f);
2141}
2142
752e386c 2143static void nv_swncq_dmafis(struct ata_port *ap)
f140f0f1
KL		 2144{
2145 struct ata_queued_cmd *qc;
2146 unsigned int rw;
2147 u8 dmactl;
2148 u32 tag;
2149 struct nv_swncq_port_priv *pp = ap->private_data;
2150
2151 __ata_bmdma_stop(ap);
2152 tag = nv_swncq_tag(ap);
2153
47013c58 2154 ata_port_dbg(ap, "dma setup tag 0x%x\n", tag);
f140f0f1
KL		 2155 qc = ata_qc_from_tag(ap, tag);
2156
2157 if (unlikely(!qc))
752e386c 2158 return;
f140f0f1
KL		 2159
2160 rw = qc->tf.flags & ATA_TFLAG_WRITE;
2161
2162 /* load PRD table addr. */
4e5b6260 2163 iowrite32(pp->prd_dma + ATA_PRD_TBL_SZ * qc->hw_tag,
f140f0f1
KL		 2164 ap->ioaddr.bmdma_addr + ATA_DMA_TABLE_OFS);
2165
2166 /* specify data direction, triple-check start bit is clear */
2167 dmactl = ioread8(ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
2168 dmactl &= ~ATA_DMA_WR;
2169 if (!rw)
2170 dmactl |= ATA_DMA_WR;
2171
2172 iowrite8(dmactl | ATA_DMA_START, ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
f140f0f1
KL		 2173}
2174
2175static void nv_swncq_host_interrupt(struct ata_port *ap, u16 fis)
2176{
2177 struct nv_swncq_port_priv *pp = ap->private_data;
2178 struct ata_queued_cmd *qc;
2179 struct ata_eh_info *ehi = &ap->link.eh_info;
2180 u32 serror;
2181 u8 ata_stat;
f140f0f1 2182
5682ed33 2183 ata_stat = ap->ops->sff_check_status(ap);
f140f0f1
KL		 2184 nv_swncq_irq_clear(ap, fis);
2185 if (!fis)
2186 return;
2187
4cb7c6f1 2188 if (ata_port_is_frozen(ap))
f140f0f1
KL		 2189 return;
2190
2191 if (fis & NV_SWNCQ_IRQ_HOTPLUG) {
2192 nv_swncq_hotplug(ap, fis);
2193 return;
2194 }
2195
2196 if (!pp->qc_active)
2197 return;
2198
82ef04fb 2199 if (ap->ops->scr_read(&ap->link, SCR_ERROR, &serror))
f140f0f1 2200 return;
82ef04fb 2201 ap->ops->scr_write(&ap->link, SCR_ERROR, serror);
f140f0f1
KL		 2202
2203 if (ata_stat & ATA_ERR) {
2204 ata_ehi_clear_desc(ehi);
2205 ata_ehi_push_desc(ehi, "Ata error. fis:0x%X", fis);
2206 ehi->err_mask |= AC_ERR_DEV;
2207 ehi->serror |= serror;
cf480626 2208 ehi->action |= ATA_EH_RESET;
f140f0f1
KL		 2209 ata_port_freeze(ap);
2210 return;
2211 }
2212
2213 if (fis & NV_SWNCQ_IRQ_BACKOUT) {
2214 /* If the IRQ is backout, driver must issue
2215 * the new command again some time later.
2216 */
2217 pp->ncq_flags |= ncq_saw_backout;
2218 }
2219
2220 if (fis & NV_SWNCQ_IRQ_SDBFIS) {
2221 pp->ncq_flags |= ncq_saw_sdb;
47013c58 2222 ata_port_dbg(ap, "SWNCQ: qc_active 0x%X "
f140f0f1 2223 "dhfis 0x%X dmafis 0x%X sactive 0x%X\n",
47013c58 2224 pp->qc_active, pp->dhfis_bits,
f140f0f1 2225 pp->dmafis_bits, readl(pp->sactive_block));
752e386c 2226 if (nv_swncq_sdbfis(ap) < 0)
f140f0f1
KL		 2227 goto irq_error;
2228 }
2229
2230 if (fis & NV_SWNCQ_IRQ_DHREGFIS) {
2231 /* The interrupt indicates the new command
2232 * was transmitted correctly to the drive.
2233 */
2234 pp->dhfis_bits |= (0x1 << pp->last_issue_tag);
2235 pp->ncq_flags |= ncq_saw_d2h;
2236 if (pp->ncq_flags & (ncq_saw_sdb | ncq_saw_backout)) {
2237 ata_ehi_push_desc(ehi, "illegal fis transaction");
2238 ehi->err_mask |= AC_ERR_HSM;
cf480626 2239 ehi->action |= ATA_EH_RESET;
f140f0f1
KL		 2240 goto irq_error;
2241 }
2242
2243 if (!(fis & NV_SWNCQ_IRQ_DMASETUP) &&
2244 !(pp->ncq_flags & ncq_saw_dmas)) {
5682ed33 2245 ata_stat = ap->ops->sff_check_status(ap);
f140f0f1
KL		 2246 if (ata_stat & ATA_BUSY)
2247 goto irq_exit;
2248
2249 if (pp->defer_queue.defer_bits) {
47013c58 2250 ata_port_dbg(ap, "send next command\n");
f140f0f1
KL		 2251 qc = nv_swncq_qc_from_dq(ap);
2252 nv_swncq_issue_atacmd(ap, qc);
2253 }
2254 }
2255 }
2256
2257 if (fis & NV_SWNCQ_IRQ_DMASETUP) {
2258 /* program the dma controller with appropriate PRD buffers
2259 * and start the DMA transfer for requested command.
2260 */
2261 pp->dmafis_bits |= (0x1 << nv_swncq_tag(ap));
2262 pp->ncq_flags |= ncq_saw_dmas;
752e386c 2263 nv_swncq_dmafis(ap);
f140f0f1
KL		 2264 }
2265
2266irq_exit:
2267 return;
2268irq_error:
2269 ata_ehi_push_desc(ehi, "fis:0x%x", fis);
2270 ata_port_freeze(ap);
2271 return;
2272}
2273
2274static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance)
2275{
2276 struct ata_host *host = dev_instance;
2277 unsigned int i;
2278 unsigned int handled = 0;
2279 unsigned long flags;
2280 u32 irq_stat;
2281
2282 spin_lock_irqsave(&host->lock, flags);
2283
2284 irq_stat = readl(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_MCP55);
2285
2286 for (i = 0; i < host->n_ports; i++) {
2287 struct ata_port *ap = host->ports[i];
2288
3e4ec344
TH		 2289 if (ap->link.sactive) {
2290 nv_swncq_host_interrupt(ap, (u16)irq_stat);
2291 handled = 1;
2292 } else {
2293 if (irq_stat) /* reserve Hotplug */
2294 nv_swncq_irq_clear(ap, 0xfff0);
f140f0f1 2295
3e4ec344 2296 handled += nv_host_intr(ap, (u8)irq_stat);
f140f0f1
KL		 2297 }
2298 irq_stat >>= NV_INT_PORT_SHIFT_MCP55;
2299 }
2300
2301 spin_unlock_irqrestore(&host->lock, flags);
2302
2303 return IRQ_RETVAL(handled);
2304}
2305
5796d1c4 2306static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
1da177e4 2307{
1626aeb8 2308 const struct ata_port_info *ppi[] = { NULL, NULL };
95947193 2309 struct nv_pi_priv *ipriv;
9a829ccf 2310 struct ata_host *host;
cdf56bcf 2311 struct nv_host_priv *hpriv;
1da177e4
LT		 2312 int rc;
2313 u32 bar;
0d5ff566 2314 void __iomem *base;
fbbb262d 2315 unsigned long type = ent->driver_data;
1da177e4
LT		 2316
2317 // Make sure this is a SATA controller by counting the number of bars
2318 // (NVIDIA SATA controllers will always have six bars). Otherwise,
2319 // it's an IDE controller and we ignore it.
c9c13ba4 2320 for (bar = 0; bar < PCI_STD_NUM_BARS; bar++)
1da177e4
LT		 2321 if (pci_resource_start(pdev, bar) == 0)
2322 return -ENODEV;
2323
06296a1e 2324 ata_print_version_once(&pdev->dev, DRV_VERSION);
1da177e4 2325
24dc5f33 2326 rc = pcim_enable_device(pdev);
1da177e4 2327 if (rc)
24dc5f33 2328 return rc;
1da177e4 2329
9a829ccf 2330 /* determine type and allocate host */
f140f0f1 2331 if (type == CK804 && adma_enabled) {
a44fec1f 2332 dev_notice(&pdev->dev, "Using ADMA mode\n");
fbbb262d 2333 type = ADMA;
2d775708 2334 } else if (type == MCP5x && swncq_enabled) {
a44fec1f 2335 dev_notice(&pdev->dev, "Using SWNCQ mode\n");
2d775708 2336 type = SWNCQ;
360737a9
JG		 2337 }
2338
1626aeb8 2339 ppi[0] = &nv_port_info[type];
95947193 2340 ipriv = ppi[0]->private_data;
1c5afdf7 2341 rc = ata_pci_bmdma_prepare_host(pdev, ppi, &host);
9a829ccf
TH		 2342 if (rc)
2343 return rc;
1da177e4 2344
24dc5f33 2345 hpriv = devm_kzalloc(&pdev->dev, sizeof(*hpriv), GFP_KERNEL);
cdf56bcf 2346 if (!hpriv)
24dc5f33 2347 return -ENOMEM;
9a829ccf
TH		 2348 hpriv->type = type;
2349 host->private_data = hpriv;
cdf56bcf 2350
9a829ccf
TH		 2351 /* request and iomap NV_MMIO_BAR */
2352 rc = pcim_iomap_regions(pdev, 1 << NV_MMIO_BAR, DRV_NAME);
2353 if (rc)
2354 return rc;
1da177e4 2355
9a829ccf
TH		 2356 /* configure SCR access */
2357 base = host->iomap[NV_MMIO_BAR];
2358 host->ports[0]->ioaddr.scr_addr = base + NV_PORT0_SCR_REG_OFFSET;
2359 host->ports[1]->ioaddr.scr_addr = base + NV_PORT1_SCR_REG_OFFSET;
1da177e4 2360
ada364e8 2361 /* enable SATA space for CK804 */
fbbb262d 2362 if (type >= CK804) {
ada364e8
TH		 2363 u8 regval;
2364
2365 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2366 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2367 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2368 }
2369
9a829ccf 2370 /* init ADMA */
fbbb262d 2371 if (type == ADMA) {
9a829ccf 2372 rc = nv_adma_host_init(host);
fbbb262d 2373 if (rc)
24dc5f33 2374 return rc;
360737a9 2375 } else if (type == SWNCQ)
f140f0f1 2376 nv_swncq_host_init(host);
fbbb262d 2377
51c89499 2378 if (msi_enabled) {
a44fec1f 2379 dev_notice(&pdev->dev, "Using MSI\n");
51c89499
TV		 2380 pci_enable_msi(pdev);
2381 }
2382
9a829ccf 2383 pci_set_master(pdev);
95cc2c70 2384 return ata_pci_sff_activate_host(host, ipriv->irq_handler, ipriv->sht);
1da177e4
LT		 2385}
2386
58eb8cd5 2387#ifdef CONFIG_PM_SLEEP
cdf56bcf
RH		 2388static int nv_pci_device_resume(struct pci_dev *pdev)
2389{
0a86e1c8 2390 struct ata_host *host = pci_get_drvdata(pdev);
cdf56bcf 2391 struct nv_host_priv *hpriv = host->private_data;
ce053fa8 2392 int rc;
cdf56bcf 2393
ce053fa8 2394 rc = ata_pci_device_do_resume(pdev);
b447916e 2395 if (rc)
ce053fa8 2396 return rc;
cdf56bcf
RH		 2397
2398 if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
b447916e 2399 if (hpriv->type >= CK804) {
cdf56bcf
RH		 2400 u8 regval;
2401
2402 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2403 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2404 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2405 }
b447916e 2406 if (hpriv->type == ADMA) {
cdf56bcf
RH		 2407 u32 tmp32;
2408 struct nv_adma_port_priv *pp;
2409 /* enable/disable ADMA on the ports appropriately */
2410 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2411
2412 pp = host->ports[0]->private_data;
b447916e 2413 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
cdf56bcf 2414 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
5796d1c4 2415 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
cdf56bcf
RH		 2416 else
2417 tmp32 |= (NV_MCP_SATA_CFG_20_PORT0_EN |
5796d1c4 2418 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
cdf56bcf 2419 pp = host->ports[1]->private_data;
b447916e 2420 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
cdf56bcf 2421 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT1_EN |
5796d1c4 2422 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
cdf56bcf
RH		 2423 else
2424 tmp32 |= (NV_MCP_SATA_CFG_20_PORT1_EN |
5796d1c4 2425 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
cdf56bcf
RH		 2426
2427 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2428 }
2429 }
2430
2431 ata_host_resume(host);
2432
2433 return 0;
2434}
438ac6d5 2435#endif
cdf56bcf 2436
cca3974e 2437static void nv_ck804_host_stop(struct ata_host *host)
ada364e8 2438{
cca3974e 2439 struct pci_dev *pdev = to_pci_dev(host->dev);
ada364e8
TH		 2440 u8 regval;
2441
2442 /* disable SATA space for CK804 */
2443 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2444 regval &= ~NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2445 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
ada364e8
TH		 2446}
2447
fbbb262d
RH		 2448static void nv_adma_host_stop(struct ata_host *host)
2449{
2450 struct pci_dev *pdev = to_pci_dev(host->dev);
fbbb262d
RH		 2451 u32 tmp32;
2452
fbbb262d
RH		 2453 /* disable ADMA on the ports */
2454 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2455 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
2456 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
2457 NV_MCP_SATA_CFG_20_PORT1_EN |
2458 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2459
2460 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2461
2462 nv_ck804_host_stop(host);
2463}
2464
2fc75da0 2465module_pci_driver(nv_pci_driver);
1da177e4 2466
fbbb262d 2467module_param_named(adma, adma_enabled, bool, 0444);
55f784c8 2468MODULE_PARM_DESC(adma, "Enable use of ADMA (Default: false)");
f140f0f1 2469module_param_named(swncq, swncq_enabled, bool, 0444);
d21279f4 2470MODULE_PARM_DESC(swncq, "Enable use of SWNCQ (Default: true)");
51c89499
TV		 2471module_param_named(msi, msi_enabled, bool, 0444);
2472MODULE_PARM_DESC(msi, "Enable use of MSI (Default: false)");
Linux 6.x block layer and io_uring tree(s)