projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
convert ATM drivers to netdev_tx_t
[linux-block.git] / drivers / ata / ahci.c
CommitLineData
1da177e4
LT		 1/*
2 * ahci.c - AHCI SATA support
3 *
af36d7f0
JG		 4 * Maintained by: Jeff Garzik <jgarzik@pobox.com>
5 * Please ALWAYS copy linux-ide@vger.kernel.org
6 * on emails.
7 *
8 * Copyright 2004-2005 Red Hat, Inc.
9 *
10 *
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2, or (at your option)
14 * any later version.
15 *
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
20 *
21 * You should have received a copy of the GNU General Public License
22 * along with this program; see the file COPYING. If not, write to
23 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
24 *
25 *
26 * libata documentation is available via 'make {ps|pdf}docs',
27 * as Documentation/DocBook/libata.*
28 *
29 * AHCI hardware documentation:
1da177e4 30 * http://www.intel.com/technology/serialata/pdf/rev1_0.pdf
af36d7f0 31 * http://www.intel.com/technology/serialata/pdf/rev1_1.pdf
1da177e4
LT		 32 *
33 */
34
35#include <linux/kernel.h>
36#include <linux/module.h>
37#include <linux/pci.h>
38#include <linux/init.h>
39#include <linux/blkdev.h>
40#include <linux/delay.h>
41#include <linux/interrupt.h>
87507cfd 42#include <linux/dma-mapping.h>
a9524a76 43#include <linux/device.h>
edc93052 44#include <linux/dmi.h>
1da177e4 45#include <scsi/scsi_host.h>
193515d5 46#include <scsi/scsi_cmnd.h>
1da177e4 47#include <linux/libata.h>
1da177e4
LT		 48
49#define DRV_NAME "ahci"
7d50b60b 50#define DRV_VERSION "3.0"
1da177e4 51
87943acf
DM		 52/* Enclosure Management Control */
53#define EM_CTRL_MSG_TYPE 0x000f0000
54
55/* Enclosure Management LED Message Type */
56#define EM_MSG_LED_HBA_PORT 0x0000000f
57#define EM_MSG_LED_PMP_SLOT 0x0000ff00
58#define EM_MSG_LED_VALUE 0xffff0000
59#define EM_MSG_LED_VALUE_ACTIVITY 0x00070000
60#define EM_MSG_LED_VALUE_OFF 0xfff80000
61#define EM_MSG_LED_VALUE_ON 0x00010000
62
a22e6444 63static int ahci_skip_host_reset;
f3d7f23f
AV		 64static int ahci_ignore_sss;
65
a22e6444
TH		 66module_param_named(skip_host_reset, ahci_skip_host_reset, int, 0444);
67MODULE_PARM_DESC(skip_host_reset, "skip global host reset (0=don't skip, 1=skip)");
68
f3d7f23f
AV		 69module_param_named(ignore_sss, ahci_ignore_sss, int, 0444);
70MODULE_PARM_DESC(ignore_sss, "Ignore staggered spinup flag (0=don't ignore, 1=ignore)");
71
31556594
KCA		 72static int ahci_enable_alpm(struct ata_port *ap,
73 enum link_pm policy);
74static void ahci_disable_alpm(struct ata_port *ap);
18f7ba4c
KCA		 75static ssize_t ahci_led_show(struct ata_port *ap, char *buf);
76static ssize_t ahci_led_store(struct ata_port *ap, const char *buf,
77 size_t size);
78static ssize_t ahci_transmit_led_message(struct ata_port *ap, u32 state,
79 ssize_t size);
1da177e4
LT		 80
81enum {
82 AHCI_PCI_BAR = 5,
648a88be 83 AHCI_MAX_PORTS = 32,
1da177e4
LT		 84 AHCI_MAX_SG = 168, /* hardware max is 64K */
85 AHCI_DMA_BOUNDARY = 0xffffffff,
12fad3f9 86 AHCI_MAX_CMDS = 32,
dd410ff1 87 AHCI_CMD_SZ = 32,
12fad3f9 88 AHCI_CMD_SLOT_SZ = AHCI_MAX_CMDS * AHCI_CMD_SZ,
1da177e4 89 AHCI_RX_FIS_SZ = 256,
a0ea7328 90 AHCI_CMD_TBL_CDB = 0x40,
dd410ff1
TH		 91 AHCI_CMD_TBL_HDR_SZ = 0x80,
92 AHCI_CMD_TBL_SZ = AHCI_CMD_TBL_HDR_SZ + (AHCI_MAX_SG * 16),
93 AHCI_CMD_TBL_AR_SZ = AHCI_CMD_TBL_SZ * AHCI_MAX_CMDS,
94 AHCI_PORT_PRIV_DMA_SZ = AHCI_CMD_SLOT_SZ + AHCI_CMD_TBL_AR_SZ +
1da177e4
LT		 95 AHCI_RX_FIS_SZ,
96 AHCI_IRQ_ON_SG = (1 << 31),
97 AHCI_CMD_ATAPI = (1 << 5),
98 AHCI_CMD_WRITE = (1 << 6),
4b10e559 99 AHCI_CMD_PREFETCH = (1 << 7),
22b49985
TH		 100 AHCI_CMD_RESET = (1 << 8),
101 AHCI_CMD_CLR_BUSY = (1 << 10),
1da177e4
LT		 102
103 RX_FIS_D2H_REG = 0x40, /* offset of D2H Register FIS data */
0291f95f 104 RX_FIS_SDB = 0x58, /* offset of SDB FIS data */
78cd52d0 105 RX_FIS_UNK = 0x60, /* offset of Unknown FIS data */
1da177e4
LT		 106
107 board_ahci = 0,
7a234aff
TH		 108 board_ahci_vt8251 = 1,
109 board_ahci_ign_iferr = 2,
110 board_ahci_sb600 = 3,
111 board_ahci_mv = 4,
e427fe04 112 board_ahci_sb700 = 5, /* for SB700 and SB800 */
e297d99e 113 board_ahci_mcp65 = 6,
9a3b103c 114 board_ahci_nopmp = 7,
aa431dd3 115 board_ahci_yesncq = 8,
1da177e4
LT		 116
117 /* global controller registers */
118 HOST_CAP = 0x00, /* host capabilities */
119 HOST_CTL = 0x04, /* global host control */
120 HOST_IRQ_STAT = 0x08, /* interrupt status */
121 HOST_PORTS_IMPL = 0x0c, /* bitmap of implemented ports */
122 HOST_VERSION = 0x10, /* AHCI spec. version compliancy */
18f7ba4c
KCA		 123 HOST_EM_LOC = 0x1c, /* Enclosure Management location */
124 HOST_EM_CTL = 0x20, /* Enclosure Management Control */
1da177e4
LT		 125
126 /* HOST_CTL bits */
127 HOST_RESET = (1 << 0), /* reset controller; self-clear */
128 HOST_IRQ_EN = (1 << 1), /* global IRQ enable */
129 HOST_AHCI_EN = (1 << 31), /* AHCI enabled */
130
131 /* HOST_CAP bits */
18f7ba4c 132 HOST_CAP_EMS = (1 << 6), /* Enclosure Management support */
0be0aa98 133 HOST_CAP_SSC = (1 << 14), /* Slumber capable */
7d50b60b 134 HOST_CAP_PMP = (1 << 17), /* Port Multiplier support */
22b49985 135 HOST_CAP_CLO = (1 << 24), /* Command List Override support */
31556594 136 HOST_CAP_ALPM = (1 << 26), /* Aggressive Link PM support */
0be0aa98 137 HOST_CAP_SSS = (1 << 27), /* Staggered Spin-up */
203ef6c4 138 HOST_CAP_SNTF = (1 << 29), /* SNotification register */
979db803 139 HOST_CAP_NCQ = (1 << 30), /* Native Command Queueing */
dd410ff1 140 HOST_CAP_64 = (1 << 31), /* PCI DAC (64-bit DMA) support */
1da177e4
LT		 141
142 /* registers for each SATA port */
143 PORT_LST_ADDR = 0x00, /* command list DMA addr */
144 PORT_LST_ADDR_HI = 0x04, /* command list DMA addr hi */
145 PORT_FIS_ADDR = 0x08, /* FIS rx buf addr */
146 PORT_FIS_ADDR_HI = 0x0c, /* FIS rx buf addr hi */
147 PORT_IRQ_STAT = 0x10, /* interrupt status */
148 PORT_IRQ_MASK = 0x14, /* interrupt enable/disable mask */
149 PORT_CMD = 0x18, /* port command */
150 PORT_TFDATA = 0x20, /* taskfile data */
151 PORT_SIG = 0x24, /* device TF signature */
152 PORT_CMD_ISSUE = 0x38, /* command issue */
1da177e4
LT		 153 PORT_SCR_STAT = 0x28, /* SATA phy register: SStatus */
154 PORT_SCR_CTL = 0x2c, /* SATA phy register: SControl */
155 PORT_SCR_ERR = 0x30, /* SATA phy register: SError */
156 PORT_SCR_ACT = 0x34, /* SATA phy register: SActive */
203ef6c4 157 PORT_SCR_NTF = 0x3c, /* SATA phy register: SNotification */
1da177e4
LT		 158
159 /* PORT_IRQ_{STAT,MASK} bits */
160 PORT_IRQ_COLD_PRES = (1 << 31), /* cold presence detect */
161 PORT_IRQ_TF_ERR = (1 << 30), /* task file error */
162 PORT_IRQ_HBUS_ERR = (1 << 29), /* host bus fatal error */
163 PORT_IRQ_HBUS_DATA_ERR = (1 << 28), /* host bus data error */
164 PORT_IRQ_IF_ERR = (1 << 27), /* interface fatal error */
165 PORT_IRQ_IF_NONFATAL = (1 << 26), /* interface non-fatal error */
166 PORT_IRQ_OVERFLOW = (1 << 24), /* xfer exhausted available S/G */
167 PORT_IRQ_BAD_PMP = (1 << 23), /* incorrect port multiplier */
168
169 PORT_IRQ_PHYRDY = (1 << 22), /* PhyRdy changed */
170 PORT_IRQ_DEV_ILCK = (1 << 7), /* device interlock */
171 PORT_IRQ_CONNECT = (1 << 6), /* port connect change status */
172 PORT_IRQ_SG_DONE = (1 << 5), /* descriptor processed */
173 PORT_IRQ_UNK_FIS = (1 << 4), /* unknown FIS rx'd */
174 PORT_IRQ_SDB_FIS = (1 << 3), /* Set Device Bits FIS rx'd */
175 PORT_IRQ_DMAS_FIS = (1 << 2), /* DMA Setup FIS rx'd */
176 PORT_IRQ_PIOS_FIS = (1 << 1), /* PIO Setup FIS rx'd */
177 PORT_IRQ_D2H_REG_FIS = (1 << 0), /* D2H Register FIS rx'd */
178
78cd52d0
TH		 179 PORT_IRQ_FREEZE = PORT_IRQ_HBUS_ERR |
180 PORT_IRQ_IF_ERR |
181 PORT_IRQ_CONNECT |
4296971d 182 PORT_IRQ_PHYRDY |
7d50b60b
TH		 183 PORT_IRQ_UNK_FIS |
184 PORT_IRQ_BAD_PMP,
78cd52d0
TH		 185 PORT_IRQ_ERROR = PORT_IRQ_FREEZE |
186 PORT_IRQ_TF_ERR |
187 PORT_IRQ_HBUS_DATA_ERR,
188 DEF_PORT_IRQ = PORT_IRQ_ERROR | PORT_IRQ_SG_DONE |
189 PORT_IRQ_SDB_FIS | PORT_IRQ_DMAS_FIS |
190 PORT_IRQ_PIOS_FIS | PORT_IRQ_D2H_REG_FIS,
1da177e4
LT		 191
192 /* PORT_CMD bits */
31556594
KCA		 193 PORT_CMD_ASP = (1 << 27), /* Aggressive Slumber/Partial */
194 PORT_CMD_ALPE = (1 << 26), /* Aggressive Link PM enable */
02eaa666 195 PORT_CMD_ATAPI = (1 << 24), /* Device is ATAPI */
7d50b60b 196 PORT_CMD_PMP = (1 << 17), /* PMP attached */
1da177e4
LT		 197 PORT_CMD_LIST_ON = (1 << 15), /* cmd list DMA engine running */
198 PORT_CMD_FIS_ON = (1 << 14), /* FIS DMA engine running */
199 PORT_CMD_FIS_RX = (1 << 4), /* Enable FIS receive DMA engine */
22b49985 200 PORT_CMD_CLO = (1 << 3), /* Command list override */
1da177e4
LT		 201 PORT_CMD_POWER_ON = (1 << 2), /* Power up device */
202 PORT_CMD_SPIN_UP = (1 << 1), /* Spin up device */
203 PORT_CMD_START = (1 << 0), /* Enable port DMA engine */
204
0be0aa98 205 PORT_CMD_ICC_MASK = (0xf << 28), /* i/f ICC state mask */
1da177e4
LT		 206 PORT_CMD_ICC_ACTIVE = (0x1 << 28), /* Put i/f in active state */
207 PORT_CMD_ICC_PARTIAL = (0x2 << 28), /* Put i/f in partial state */
208 PORT_CMD_ICC_SLUMBER = (0x6 << 28), /* Put i/f in slumber state */
4b0060f4 209
417a1a6d
TH		 210 /* hpriv->flags bits */
211 AHCI_HFLAG_NO_NCQ = (1 << 0),
212 AHCI_HFLAG_IGN_IRQ_IF_ERR = (1 << 1), /* ignore IRQ_IF_ERR */
213 AHCI_HFLAG_IGN_SERR_INTERNAL = (1 << 2), /* ignore SERR_INTERNAL */
214 AHCI_HFLAG_32BIT_ONLY = (1 << 3), /* force 32bit */
215 AHCI_HFLAG_MV_PATA = (1 << 4), /* PATA port */
216 AHCI_HFLAG_NO_MSI = (1 << 5), /* no PCI MSI */
6949b914 217 AHCI_HFLAG_NO_PMP = (1 << 6), /* no PMP */
31556594 218 AHCI_HFLAG_NO_HOTPLUG = (1 << 7), /* ignore PxSERR.DIAG.N */
a878539e 219 AHCI_HFLAG_SECT255 = (1 << 8), /* max 255 sectors */
e297d99e 220 AHCI_HFLAG_YES_NCQ = (1 << 9), /* force NCQ cap on */
9b10ae86 221 AHCI_HFLAG_NO_SUSPEND = (1 << 10), /* don't suspend */
417a1a6d 222
bf2af2a2 223 /* ap->flags bits */
1188c0d8
TH		 224
225 AHCI_FLAG_COMMON = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
226 ATA_FLAG_MMIO | ATA_FLAG_PIO_DMA |
31556594
KCA		 227 ATA_FLAG_ACPI_SATA | ATA_FLAG_AN |
228 ATA_FLAG_IPM,
c4f7792c
TH		 229
230 ICH_MAP = 0x90, /* ICH MAP register */
18f7ba4c 231
d50ce07d
TH		 232 /* em constants */
233 EM_MAX_SLOTS = 8,
234 EM_MAX_RETRY = 5,
235
18f7ba4c
KCA		 236 /* em_ctl bits */
237 EM_CTL_RST = (1 << 9), /* Reset */
238 EM_CTL_TM = (1 << 8), /* Transmit Message */
239 EM_CTL_ALHD = (1 << 26), /* Activity LED */
1da177e4
LT		 240};
241
242struct ahci_cmd_hdr {
4ca4e439
AV		 243 __le32 opts;
244 __le32 status;
245 __le32 tbl_addr;
246 __le32 tbl_addr_hi;
247 __le32 reserved[4];
1da177e4
LT		 248};
249
250struct ahci_sg {
4ca4e439
AV		 251 __le32 addr;
252 __le32 addr_hi;
253 __le32 reserved;
254 __le32 flags_size;
1da177e4
LT		 255};
256
18f7ba4c
KCA		 257struct ahci_em_priv {
258 enum sw_activity blink_policy;
259 struct timer_list timer;
260 unsigned long saved_activity;
261 unsigned long activity;
262 unsigned long led_state;
263};
264
1da177e4 265struct ahci_host_priv {
417a1a6d 266 unsigned int flags; /* AHCI_HFLAG_* */
d447df14
TH		 267 u32 cap; /* cap to use */
268 u32 port_map; /* port map to use */
269 u32 saved_cap; /* saved initial cap */
270 u32 saved_port_map; /* saved initial port_map */
18f7ba4c 271 u32 em_loc; /* enclosure management location */
1da177e4
LT		 272};
273
274struct ahci_port_priv {
7d50b60b 275 struct ata_link *active_link;
1da177e4
LT		 276 struct ahci_cmd_hdr *cmd_slot;
277 dma_addr_t cmd_slot_dma;
278 void *cmd_tbl;
279 dma_addr_t cmd_tbl_dma;
1da177e4
LT		 280 void *rx_fis;
281 dma_addr_t rx_fis_dma;
0291f95f 282 /* for NCQ spurious interrupt analysis */
0291f95f
TH		 283 unsigned int ncq_saw_d2h:1;
284 unsigned int ncq_saw_dmas:1;
afb2d552 285 unsigned int ncq_saw_sdb:1;
a7384925 286 u32 intr_mask; /* interrupts to enable */
d50ce07d
TH		 287 /* enclosure management info per PM slot */
288 struct ahci_em_priv em_priv[EM_MAX_SLOTS];
1da177e4
LT		 289};
290
82ef04fb
TH		 291static int ahci_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val);
292static int ahci_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val);
2dcb407e 293static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
9a3d9eb0 294static unsigned int ahci_qc_issue(struct ata_queued_cmd *qc);
4c9bf4e7 295static bool ahci_qc_fill_rtf(struct ata_queued_cmd *qc);
1da177e4
LT		 296static int ahci_port_start(struct ata_port *ap);
297static void ahci_port_stop(struct ata_port *ap);
1da177e4 298static void ahci_qc_prep(struct ata_queued_cmd *qc);
78cd52d0
TH		 299static void ahci_freeze(struct ata_port *ap);
300static void ahci_thaw(struct ata_port *ap);
7d50b60b
TH		 301static void ahci_pmp_attach(struct ata_port *ap);
302static void ahci_pmp_detach(struct ata_port *ap);
a1efdaba
TH		 303static int ahci_softreset(struct ata_link *link, unsigned int *class,
304 unsigned long deadline);
bd17243a
SH		 305static int ahci_sb600_softreset(struct ata_link *link, unsigned int *class,
306 unsigned long deadline);
a1efdaba
TH		 307static int ahci_hardreset(struct ata_link *link, unsigned int *class,
308 unsigned long deadline);
309static int ahci_vt8251_hardreset(struct ata_link *link, unsigned int *class,
310 unsigned long deadline);
311static int ahci_p5wdh_hardreset(struct ata_link *link, unsigned int *class,
312 unsigned long deadline);
313static void ahci_postreset(struct ata_link *link, unsigned int *class);
78cd52d0
TH		 314static void ahci_error_handler(struct ata_port *ap);
315static void ahci_post_internal_cmd(struct ata_queued_cmd *qc);
df69c9c5 316static int ahci_port_resume(struct ata_port *ap);
a878539e 317static void ahci_dev_config(struct ata_device *dev);
dab632e8
JG		 318static void ahci_fill_cmd_slot(struct ahci_port_priv *pp, unsigned int tag,
319 u32 opts);
438ac6d5 320#ifdef CONFIG_PM
c1332875 321static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg);
c1332875
TH		 322static int ahci_pci_device_suspend(struct pci_dev *pdev, pm_message_t mesg);
323static int ahci_pci_device_resume(struct pci_dev *pdev);
438ac6d5 324#endif
18f7ba4c
KCA		 325static ssize_t ahci_activity_show(struct ata_device *dev, char *buf);
326static ssize_t ahci_activity_store(struct ata_device *dev,
327 enum sw_activity val);
328static void ahci_init_sw_activity(struct ata_link *link);
1da177e4 329
ee959b00
TJ		 330static struct device_attribute *ahci_shost_attrs[] = {
331 &dev_attr_link_power_management_policy,
18f7ba4c
KCA		 332 &dev_attr_em_message_type,
333 &dev_attr_em_message,
334 NULL
335};
336
337static struct device_attribute *ahci_sdev_attrs[] = {
338 &dev_attr_sw_activity,
45fabbb7 339 &dev_attr_unload_heads,
31556594
KCA		 340 NULL
341};
342
193515d5 343static struct scsi_host_template ahci_sht = {
68d1d07b 344 ATA_NCQ_SHT(DRV_NAME),
12fad3f9 345 .can_queue = AHCI_MAX_CMDS - 1,
1da177e4 346 .sg_tablesize = AHCI_MAX_SG,
1da177e4 347 .dma_boundary = AHCI_DMA_BOUNDARY,
31556594 348 .shost_attrs = ahci_shost_attrs,
18f7ba4c 349 .sdev_attrs = ahci_sdev_attrs,
1da177e4
LT		 350};
351
029cfd6b
TH		 352static struct ata_port_operations ahci_ops = {
353 .inherits = &sata_pmp_port_ops,
354
7d50b60b 355 .qc_defer = sata_pmp_qc_defer_cmd_switch,
1da177e4
LT		 356 .qc_prep = ahci_qc_prep,
357 .qc_issue = ahci_qc_issue,
4c9bf4e7 358 .qc_fill_rtf = ahci_qc_fill_rtf,
1da177e4 359
78cd52d0
TH		 360 .freeze = ahci_freeze,
361 .thaw = ahci_thaw,
a1efdaba
TH		 362 .softreset = ahci_softreset,
363 .hardreset = ahci_hardreset,
364 .postreset = ahci_postreset,
071f44b1 365 .pmp_softreset = ahci_softreset,
78cd52d0
TH		 366 .error_handler = ahci_error_handler,
367 .post_internal_cmd = ahci_post_internal_cmd,
6bd99b4e
TH		 368 .dev_config = ahci_dev_config,
369
ad616ffb
TH		 370 .scr_read = ahci_scr_read,
371 .scr_write = ahci_scr_write,
7d50b60b
TH		 372 .pmp_attach = ahci_pmp_attach,
373 .pmp_detach = ahci_pmp_detach,
7d50b60b 374
029cfd6b
TH		 375 .enable_pm = ahci_enable_alpm,
376 .disable_pm = ahci_disable_alpm,
18f7ba4c
KCA		 377 .em_show = ahci_led_show,
378 .em_store = ahci_led_store,
379 .sw_activity_show = ahci_activity_show,
380 .sw_activity_store = ahci_activity_store,
438ac6d5 381#ifdef CONFIG_PM
ad616ffb
TH		 382 .port_suspend = ahci_port_suspend,
383 .port_resume = ahci_port_resume,
438ac6d5 384#endif
ad616ffb
TH		 385 .port_start = ahci_port_start,
386 .port_stop = ahci_port_stop,
387};
388
029cfd6b
TH		 389static struct ata_port_operations ahci_vt8251_ops = {
390 .inherits = &ahci_ops,
a1efdaba 391 .hardreset = ahci_vt8251_hardreset,
029cfd6b 392};
edc93052 393
029cfd6b
TH		 394static struct ata_port_operations ahci_p5wdh_ops = {
395 .inherits = &ahci_ops,
a1efdaba 396 .hardreset = ahci_p5wdh_hardreset,
edc93052
TH		 397};
398
bd17243a
SH		 399static struct ata_port_operations ahci_sb600_ops = {
400 .inherits = &ahci_ops,
401 .softreset = ahci_sb600_softreset,
402 .pmp_softreset = ahci_sb600_softreset,
403};
404
417a1a6d
TH		 405#define AHCI_HFLAGS(flags) .private_data = (void *)(flags)
406
98ac62de 407static const struct ata_port_info ahci_port_info[] = {
4da646b7 408 [board_ahci] =
1da177e4 409 {
1188c0d8 410 .flags = AHCI_FLAG_COMMON,
14bdef98 411 .pio_mask = ATA_PIO4,
469248ab 412 .udma_mask = ATA_UDMA6,
1da177e4
LT		 413 .port_ops = &ahci_ops,
414 },
4da646b7 415 [board_ahci_vt8251] =
bf2af2a2 416 {
6949b914 417 AHCI_HFLAGS (AHCI_HFLAG_NO_NCQ | AHCI_HFLAG_NO_PMP),
417a1a6d 418 .flags = AHCI_FLAG_COMMON,
14bdef98 419 .pio_mask = ATA_PIO4,
469248ab 420 .udma_mask = ATA_UDMA6,
ad616ffb 421 .port_ops = &ahci_vt8251_ops,
bf2af2a2 422 },
4da646b7 423 [board_ahci_ign_iferr] =
41669553 424 {
417a1a6d
TH		 425 AHCI_HFLAGS (AHCI_HFLAG_IGN_IRQ_IF_ERR),
426 .flags = AHCI_FLAG_COMMON,
14bdef98 427 .pio_mask = ATA_PIO4,
469248ab 428 .udma_mask = ATA_UDMA6,
41669553
TH		 429 .port_ops = &ahci_ops,
430 },
4da646b7 431 [board_ahci_sb600] =
55a61604 432 {
417a1a6d 433 AHCI_HFLAGS (AHCI_HFLAG_IGN_SERR_INTERNAL |
58a09b38 434 AHCI_HFLAG_NO_MSI | AHCI_HFLAG_SECT255),
417a1a6d 435 .flags = AHCI_FLAG_COMMON,
14bdef98 436 .pio_mask = ATA_PIO4,
469248ab 437 .udma_mask = ATA_UDMA6,
bd17243a 438 .port_ops = &ahci_sb600_ops,
55a61604 439 },
4da646b7 440 [board_ahci_mv] =
cd70c266 441 {
417a1a6d 442 AHCI_HFLAGS (AHCI_HFLAG_NO_NCQ | AHCI_HFLAG_NO_MSI |
17248461 443 AHCI_HFLAG_MV_PATA | AHCI_HFLAG_NO_PMP),
cd70c266 444 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
417a1a6d 445 ATA_FLAG_MMIO | ATA_FLAG_PIO_DMA,
14bdef98 446 .pio_mask = ATA_PIO4,
cd70c266
JG		 447 .udma_mask = ATA_UDMA6,
448 .port_ops = &ahci_ops,
449 },
4da646b7 450 [board_ahci_sb700] = /* for SB700 and SB800 */
e39fc8c9 451 {
bd17243a 452 AHCI_HFLAGS (AHCI_HFLAG_IGN_SERR_INTERNAL),
e39fc8c9 453 .flags = AHCI_FLAG_COMMON,
14bdef98 454 .pio_mask = ATA_PIO4,
e39fc8c9 455 .udma_mask = ATA_UDMA6,
bd17243a 456 .port_ops = &ahci_sb600_ops,
e39fc8c9 457 },
4da646b7 458 [board_ahci_mcp65] =
e297d99e
TH		 459 {
460 AHCI_HFLAGS (AHCI_HFLAG_YES_NCQ),
461 .flags = AHCI_FLAG_COMMON,
14bdef98 462 .pio_mask = ATA_PIO4,
e297d99e
TH		 463 .udma_mask = ATA_UDMA6,
464 .port_ops = &ahci_ops,
465 },
4da646b7 466 [board_ahci_nopmp] =
9a3b103c
TH		 467 {
468 AHCI_HFLAGS (AHCI_HFLAG_NO_PMP),
469 .flags = AHCI_FLAG_COMMON,
14bdef98 470 .pio_mask = ATA_PIO4,
9a3b103c
TH		 471 .udma_mask = ATA_UDMA6,
472 .port_ops = &ahci_ops,
473 },
aa431dd3
TH		 474 /* board_ahci_yesncq */
475 {
476 AHCI_HFLAGS (AHCI_HFLAG_YES_NCQ),
477 .flags = AHCI_FLAG_COMMON,
478 .pio_mask = ATA_PIO4,
479 .udma_mask = ATA_UDMA6,
480 .port_ops = &ahci_ops,
481 },
1da177e4
LT		 482};
483
3b7d697d 484static const struct pci_device_id ahci_pci_tbl[] = {
fe7fa31a 485 /* Intel */
54bb3a94
JG		 486 { PCI_VDEVICE(INTEL, 0x2652), board_ahci }, /* ICH6 */
487 { PCI_VDEVICE(INTEL, 0x2653), board_ahci }, /* ICH6M */
488 { PCI_VDEVICE(INTEL, 0x27c1), board_ahci }, /* ICH7 */
489 { PCI_VDEVICE(INTEL, 0x27c5), board_ahci }, /* ICH7M */
490 { PCI_VDEVICE(INTEL, 0x27c3), board_ahci }, /* ICH7R */
82490c09 491 { PCI_VDEVICE(AL, 0x5288), board_ahci_ign_iferr }, /* ULi M5288 */
54bb3a94
JG		 492 { PCI_VDEVICE(INTEL, 0x2681), board_ahci }, /* ESB2 */
493 { PCI_VDEVICE(INTEL, 0x2682), board_ahci }, /* ESB2 */
494 { PCI_VDEVICE(INTEL, 0x2683), board_ahci }, /* ESB2 */
495 { PCI_VDEVICE(INTEL, 0x27c6), board_ahci }, /* ICH7-M DH */
7a234aff
TH		 496 { PCI_VDEVICE(INTEL, 0x2821), board_ahci }, /* ICH8 */
497 { PCI_VDEVICE(INTEL, 0x2822), board_ahci }, /* ICH8 */
498 { PCI_VDEVICE(INTEL, 0x2824), board_ahci }, /* ICH8 */
499 { PCI_VDEVICE(INTEL, 0x2829), board_ahci }, /* ICH8M */
500 { PCI_VDEVICE(INTEL, 0x282a), board_ahci }, /* ICH8M */
501 { PCI_VDEVICE(INTEL, 0x2922), board_ahci }, /* ICH9 */
502 { PCI_VDEVICE(INTEL, 0x2923), board_ahci }, /* ICH9 */
503 { PCI_VDEVICE(INTEL, 0x2924), board_ahci }, /* ICH9 */
504 { PCI_VDEVICE(INTEL, 0x2925), board_ahci }, /* ICH9 */
505 { PCI_VDEVICE(INTEL, 0x2927), board_ahci }, /* ICH9 */
506 { PCI_VDEVICE(INTEL, 0x2929), board_ahci }, /* ICH9M */
507 { PCI_VDEVICE(INTEL, 0x292a), board_ahci }, /* ICH9M */
508 { PCI_VDEVICE(INTEL, 0x292b), board_ahci }, /* ICH9M */
509 { PCI_VDEVICE(INTEL, 0x292c), board_ahci }, /* ICH9M */
510 { PCI_VDEVICE(INTEL, 0x292f), board_ahci }, /* ICH9M */
511 { PCI_VDEVICE(INTEL, 0x294d), board_ahci }, /* ICH9 */
512 { PCI_VDEVICE(INTEL, 0x294e), board_ahci }, /* ICH9M */
d4155e6f
JG		 513 { PCI_VDEVICE(INTEL, 0x502a), board_ahci }, /* Tolapai */
514 { PCI_VDEVICE(INTEL, 0x502b), board_ahci }, /* Tolapai */
16ad1ad9 515 { PCI_VDEVICE(INTEL, 0x3a05), board_ahci }, /* ICH10 */
b2dde6af 516 { PCI_VDEVICE(INTEL, 0x3a22), board_ahci }, /* ICH10 */
16ad1ad9 517 { PCI_VDEVICE(INTEL, 0x3a25), board_ahci }, /* ICH10 */
c1f57d9b
DM		 518 { PCI_VDEVICE(INTEL, 0x3b22), board_ahci }, /* PCH AHCI */
519 { PCI_VDEVICE(INTEL, 0x3b23), board_ahci }, /* PCH AHCI */
adcb5308 520 { PCI_VDEVICE(INTEL, 0x3b24), board_ahci }, /* PCH RAID */
8e48b6b3 521 { PCI_VDEVICE(INTEL, 0x3b25), board_ahci }, /* PCH RAID */
c1f57d9b 522 { PCI_VDEVICE(INTEL, 0x3b29), board_ahci }, /* PCH AHCI */
adcb5308 523 { PCI_VDEVICE(INTEL, 0x3b2b), board_ahci }, /* PCH RAID */
8e48b6b3 524 { PCI_VDEVICE(INTEL, 0x3b2c), board_ahci }, /* PCH RAID */
c1f57d9b 525 { PCI_VDEVICE(INTEL, 0x3b2f), board_ahci }, /* PCH AHCI */
fe7fa31a 526
e34bb370
TH		 527 /* JMicron 360/1/3/5/6, match class to avoid IDE function */
528 { PCI_VENDOR_ID_JMICRON, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID,
529 PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff, board_ahci_ign_iferr },
fe7fa31a
JG		 530
531 /* ATI */
c65ec1c2 532 { PCI_VDEVICE(ATI, 0x4380), board_ahci_sb600 }, /* ATI SB600 */
e39fc8c9
SH		 533 { PCI_VDEVICE(ATI, 0x4390), board_ahci_sb700 }, /* ATI SB700/800 */
534 { PCI_VDEVICE(ATI, 0x4391), board_ahci_sb700 }, /* ATI SB700/800 */
535 { PCI_VDEVICE(ATI, 0x4392), board_ahci_sb700 }, /* ATI SB700/800 */
536 { PCI_VDEVICE(ATI, 0x4393), board_ahci_sb700 }, /* ATI SB700/800 */
537 { PCI_VDEVICE(ATI, 0x4394), board_ahci_sb700 }, /* ATI SB700/800 */
538 { PCI_VDEVICE(ATI, 0x4395), board_ahci_sb700 }, /* ATI SB700/800 */
fe7fa31a
JG		 539
540 /* VIA */
54bb3a94 541 { PCI_VDEVICE(VIA, 0x3349), board_ahci_vt8251 }, /* VIA VT8251 */
bf335542 542 { PCI_VDEVICE(VIA, 0x6287), board_ahci_vt8251 }, /* VIA VT8251 */
fe7fa31a
JG		 543
544 /* NVIDIA */
e297d99e
TH		 545 { PCI_VDEVICE(NVIDIA, 0x044c), board_ahci_mcp65 }, /* MCP65 */
546 { PCI_VDEVICE(NVIDIA, 0x044d), board_ahci_mcp65 }, /* MCP65 */
547 { PCI_VDEVICE(NVIDIA, 0x044e), board_ahci_mcp65 }, /* MCP65 */
548 { PCI_VDEVICE(NVIDIA, 0x044f), board_ahci_mcp65 }, /* MCP65 */
549 { PCI_VDEVICE(NVIDIA, 0x045c), board_ahci_mcp65 }, /* MCP65 */
550 { PCI_VDEVICE(NVIDIA, 0x045d), board_ahci_mcp65 }, /* MCP65 */
551 { PCI_VDEVICE(NVIDIA, 0x045e), board_ahci_mcp65 }, /* MCP65 */
552 { PCI_VDEVICE(NVIDIA, 0x045f), board_ahci_mcp65 }, /* MCP65 */
aa431dd3
TH		 553 { PCI_VDEVICE(NVIDIA, 0x0550), board_ahci_yesncq }, /* MCP67 */
554 { PCI_VDEVICE(NVIDIA, 0x0551), board_ahci_yesncq }, /* MCP67 */
555 { PCI_VDEVICE(NVIDIA, 0x0552), board_ahci_yesncq }, /* MCP67 */
556 { PCI_VDEVICE(NVIDIA, 0x0553), board_ahci_yesncq }, /* MCP67 */
557 { PCI_VDEVICE(NVIDIA, 0x0554), board_ahci_yesncq }, /* MCP67 */
558 { PCI_VDEVICE(NVIDIA, 0x0555), board_ahci_yesncq }, /* MCP67 */
559 { PCI_VDEVICE(NVIDIA, 0x0556), board_ahci_yesncq }, /* MCP67 */
560 { PCI_VDEVICE(NVIDIA, 0x0557), board_ahci_yesncq }, /* MCP67 */
561 { PCI_VDEVICE(NVIDIA, 0x0558), board_ahci_yesncq }, /* MCP67 */
562 { PCI_VDEVICE(NVIDIA, 0x0559), board_ahci_yesncq }, /* MCP67 */
563 { PCI_VDEVICE(NVIDIA, 0x055a), board_ahci_yesncq }, /* MCP67 */
564 { PCI_VDEVICE(NVIDIA, 0x055b), board_ahci_yesncq }, /* MCP67 */
565 { PCI_VDEVICE(NVIDIA, 0x07f0), board_ahci_yesncq }, /* MCP73 */
566 { PCI_VDEVICE(NVIDIA, 0x07f1), board_ahci_yesncq }, /* MCP73 */
567 { PCI_VDEVICE(NVIDIA, 0x07f2), board_ahci_yesncq }, /* MCP73 */
568 { PCI_VDEVICE(NVIDIA, 0x07f3), board_ahci_yesncq }, /* MCP73 */
569 { PCI_VDEVICE(NVIDIA, 0x07f4), board_ahci_yesncq }, /* MCP73 */
570 { PCI_VDEVICE(NVIDIA, 0x07f5), board_ahci_yesncq }, /* MCP73 */
571 { PCI_VDEVICE(NVIDIA, 0x07f6), board_ahci_yesncq }, /* MCP73 */
572 { PCI_VDEVICE(NVIDIA, 0x07f7), board_ahci_yesncq }, /* MCP73 */
573 { PCI_VDEVICE(NVIDIA, 0x07f8), board_ahci_yesncq }, /* MCP73 */
574 { PCI_VDEVICE(NVIDIA, 0x07f9), board_ahci_yesncq }, /* MCP73 */
575 { PCI_VDEVICE(NVIDIA, 0x07fa), board_ahci_yesncq }, /* MCP73 */
576 { PCI_VDEVICE(NVIDIA, 0x07fb), board_ahci_yesncq }, /* MCP73 */
0522b286
PC		 577 { PCI_VDEVICE(NVIDIA, 0x0ad0), board_ahci }, /* MCP77 */
578 { PCI_VDEVICE(NVIDIA, 0x0ad1), board_ahci }, /* MCP77 */
579 { PCI_VDEVICE(NVIDIA, 0x0ad2), board_ahci }, /* MCP77 */
580 { PCI_VDEVICE(NVIDIA, 0x0ad3), board_ahci }, /* MCP77 */
581 { PCI_VDEVICE(NVIDIA, 0x0ad4), board_ahci }, /* MCP77 */
582 { PCI_VDEVICE(NVIDIA, 0x0ad5), board_ahci }, /* MCP77 */
583 { PCI_VDEVICE(NVIDIA, 0x0ad6), board_ahci }, /* MCP77 */
584 { PCI_VDEVICE(NVIDIA, 0x0ad7), board_ahci }, /* MCP77 */
585 { PCI_VDEVICE(NVIDIA, 0x0ad8), board_ahci }, /* MCP77 */
586 { PCI_VDEVICE(NVIDIA, 0x0ad9), board_ahci }, /* MCP77 */
587 { PCI_VDEVICE(NVIDIA, 0x0ada), board_ahci }, /* MCP77 */
588 { PCI_VDEVICE(NVIDIA, 0x0adb), board_ahci }, /* MCP77 */
6ba86958 589 { PCI_VDEVICE(NVIDIA, 0x0ab4), board_ahci }, /* MCP79 */
590 { PCI_VDEVICE(NVIDIA, 0x0ab5), board_ahci }, /* MCP79 */
591 { PCI_VDEVICE(NVIDIA, 0x0ab6), board_ahci }, /* MCP79 */
592 { PCI_VDEVICE(NVIDIA, 0x0ab7), board_ahci }, /* MCP79 */
7100819f
PC		 593 { PCI_VDEVICE(NVIDIA, 0x0ab8), board_ahci }, /* MCP79 */
594 { PCI_VDEVICE(NVIDIA, 0x0ab9), board_ahci }, /* MCP79 */
595 { PCI_VDEVICE(NVIDIA, 0x0aba), board_ahci }, /* MCP79 */
596 { PCI_VDEVICE(NVIDIA, 0x0abb), board_ahci }, /* MCP79 */
597 { PCI_VDEVICE(NVIDIA, 0x0abc), board_ahci }, /* MCP79 */
598 { PCI_VDEVICE(NVIDIA, 0x0abd), board_ahci }, /* MCP79 */
599 { PCI_VDEVICE(NVIDIA, 0x0abe), board_ahci }, /* MCP79 */
600 { PCI_VDEVICE(NVIDIA, 0x0abf), board_ahci }, /* MCP79 */
7adbe46b 601 { PCI_VDEVICE(NVIDIA, 0x0d84), board_ahci }, /* MCP89 */
602 { PCI_VDEVICE(NVIDIA, 0x0d85), board_ahci }, /* MCP89 */
603 { PCI_VDEVICE(NVIDIA, 0x0d86), board_ahci }, /* MCP89 */
604 { PCI_VDEVICE(NVIDIA, 0x0d87), board_ahci }, /* MCP89 */
605 { PCI_VDEVICE(NVIDIA, 0x0d88), board_ahci }, /* MCP89 */
606 { PCI_VDEVICE(NVIDIA, 0x0d89), board_ahci }, /* MCP89 */
607 { PCI_VDEVICE(NVIDIA, 0x0d8a), board_ahci }, /* MCP89 */
608 { PCI_VDEVICE(NVIDIA, 0x0d8b), board_ahci }, /* MCP89 */
609 { PCI_VDEVICE(NVIDIA, 0x0d8c), board_ahci }, /* MCP89 */
610 { PCI_VDEVICE(NVIDIA, 0x0d8d), board_ahci }, /* MCP89 */
611 { PCI_VDEVICE(NVIDIA, 0x0d8e), board_ahci }, /* MCP89 */
612 { PCI_VDEVICE(NVIDIA, 0x0d8f), board_ahci }, /* MCP89 */
fe7fa31a 613
95916edd 614 /* SiS */
20e2de4a
TH		 615 { PCI_VDEVICE(SI, 0x1184), board_ahci }, /* SiS 966 */
616 { PCI_VDEVICE(SI, 0x1185), board_ahci }, /* SiS 968 */
617 { PCI_VDEVICE(SI, 0x0186), board_ahci }, /* SiS 968 */
95916edd 618
cd70c266
JG		 619 /* Marvell */
620 { PCI_VDEVICE(MARVELL, 0x6145), board_ahci_mv }, /* 6145 */
c40e7cb8 621 { PCI_VDEVICE(MARVELL, 0x6121), board_ahci_mv }, /* 6121 */
cd70c266 622
c77a036b
MN		 623 /* Promise */
624 { PCI_VDEVICE(PROMISE, 0x3f20), board_ahci }, /* PDC42819 */
625
415ae2b5
JG		 626 /* Generic, PCI class code for AHCI */
627 { PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID,
c9f89475 628 PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff, board_ahci },
415ae2b5 629
1da177e4
LT		 630 { } /* terminate list */
631};
632
633
634static struct pci_driver ahci_pci_driver = {
635 .name = DRV_NAME,
636 .id_table = ahci_pci_tbl,
637 .probe = ahci_init_one,
24dc5f33 638 .remove = ata_pci_remove_one,
438ac6d5 639#ifdef CONFIG_PM
c1332875
TH		 640 .suspend = ahci_pci_device_suspend,
641 .resume = ahci_pci_device_resume,
438ac6d5 642#endif
1da177e4
LT		 643};
644
18f7ba4c
KCA		 645static int ahci_em_messages = 1;
646module_param(ahci_em_messages, int, 0444);
647/* add other LED protocol types when they become supported */
648MODULE_PARM_DESC(ahci_em_messages,
649 "Set AHCI Enclosure Management Message type (0 = disabled, 1 = LED");
1da177e4 650
5b66c829
AC		 651#if defined(CONFIG_PATA_MARVELL) || defined(CONFIG_PATA_MARVELL_MODULE)
652static int marvell_enable;
653#else
654static int marvell_enable = 1;
655#endif
656module_param(marvell_enable, int, 0644);
657MODULE_PARM_DESC(marvell_enable, "Marvell SATA via AHCI (1 = enabled)");
658
659
98fa4b60
TH		 660static inline int ahci_nr_ports(u32 cap)
661{
662 return (cap & 0x1f) + 1;
663}
664
dab632e8
JG		 665static inline void __iomem *__ahci_port_base(struct ata_host *host,
666 unsigned int port_no)
1da177e4 667{
dab632e8 668 void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
4447d351 669
dab632e8
JG		 670 return mmio + 0x100 + (port_no * 0x80);
671}
672
673static inline void __iomem *ahci_port_base(struct ata_port *ap)
674{
675 return __ahci_port_base(ap->host, ap->port_no);
1da177e4
LT		 676}
677
b710a1f4
TH		 678static void ahci_enable_ahci(void __iomem *mmio)
679{
15fe982e 680 int i;
b710a1f4
TH		 681 u32 tmp;
682
683 /* turn on AHCI_EN */
684 tmp = readl(mmio + HOST_CTL);
15fe982e
TH		 685 if (tmp & HOST_AHCI_EN)
686 return;
687
688 /* Some controllers need AHCI_EN to be written multiple times.
689 * Try a few times before giving up.
690 */
691 for (i = 0; i < 5; i++) {
b710a1f4
TH		 692 tmp |= HOST_AHCI_EN;
693 writel(tmp, mmio + HOST_CTL);
694 tmp = readl(mmio + HOST_CTL); /* flush && sanity check */
15fe982e
TH		 695 if (tmp & HOST_AHCI_EN)
696 return;
697 msleep(10);
b710a1f4 698 }
15fe982e
TH		 699
700 WARN_ON(1);
b710a1f4
TH		 701}
702
d447df14
TH		 703/**
704 * ahci_save_initial_config - Save and fixup initial config values
4447d351 705 * @pdev: target PCI device
4447d351 706 * @hpriv: host private area to store config values
d447df14
TH		 707 *
708 * Some registers containing configuration info might be setup by
709 * BIOS and might be cleared on reset. This function saves the
710 * initial values of those registers into @hpriv such that they
711 * can be restored after controller reset.
712 *
713 * If inconsistent, config values are fixed up by this function.
714 *
715 * LOCKING:
716 * None.
717 */
4447d351 718static void ahci_save_initial_config(struct pci_dev *pdev,
4447d351 719 struct ahci_host_priv *hpriv)
d447df14 720{
4447d351 721 void __iomem *mmio = pcim_iomap_table(pdev)[AHCI_PCI_BAR];
d447df14 722 u32 cap, port_map;
17199b18 723 int i;
c40e7cb8 724 int mv;
d447df14 725
b710a1f4
TH		 726 /* make sure AHCI mode is enabled before accessing CAP */
727 ahci_enable_ahci(mmio);
728
d447df14
TH		 729 /* Values prefixed with saved_ are written back to host after
730 * reset. Values without are used for driver operation.
731 */
732 hpriv->saved_cap = cap = readl(mmio + HOST_CAP);
733 hpriv->saved_port_map = port_map = readl(mmio + HOST_PORTS_IMPL);
734
274c1fde 735 /* some chips have errata preventing 64bit use */
417a1a6d 736 if ((cap & HOST_CAP_64) && (hpriv->flags & AHCI_HFLAG_32BIT_ONLY)) {
c7a42156
TH		 737 dev_printk(KERN_INFO, &pdev->dev,
738 "controller can't do 64bit DMA, forcing 32bit\n");
739 cap &= ~HOST_CAP_64;
740 }
741
417a1a6d 742 if ((cap & HOST_CAP_NCQ) && (hpriv->flags & AHCI_HFLAG_NO_NCQ)) {
274c1fde
TH		 743 dev_printk(KERN_INFO, &pdev->dev,
744 "controller can't do NCQ, turning off CAP_NCQ\n");
745 cap &= ~HOST_CAP_NCQ;
746 }
747
e297d99e
TH		 748 if (!(cap & HOST_CAP_NCQ) && (hpriv->flags & AHCI_HFLAG_YES_NCQ)) {
749 dev_printk(KERN_INFO, &pdev->dev,
750 "controller can do NCQ, turning on CAP_NCQ\n");
751 cap |= HOST_CAP_NCQ;
752 }
753
258cd846 754 if ((cap & HOST_CAP_PMP) && (hpriv->flags & AHCI_HFLAG_NO_PMP)) {
6949b914
TH		 755 dev_printk(KERN_INFO, &pdev->dev,
756 "controller can't do PMP, turning off CAP_PMP\n");
757 cap &= ~HOST_CAP_PMP;
758 }
759
d799e083
TH		 760 if (pdev->vendor == PCI_VENDOR_ID_JMICRON && pdev->device == 0x2361 &&
761 port_map != 1) {
762 dev_printk(KERN_INFO, &pdev->dev,
763 "JMB361 has only one port, port_map 0x%x -> 0x%x\n",
764 port_map, 1);
765 port_map = 1;
766 }
767
cd70c266
JG		 768 /*
769 * Temporary Marvell 6145 hack: PATA port presence
770 * is asserted through the standard AHCI port
771 * presence register, as bit 4 (counting from 0)
772 */
417a1a6d 773 if (hpriv->flags & AHCI_HFLAG_MV_PATA) {
c40e7cb8
JAR		 774 if (pdev->device == 0x6121)
775 mv = 0x3;
776 else
777 mv = 0xf;
cd70c266
JG		 778 dev_printk(KERN_ERR, &pdev->dev,
779 "MV_AHCI HACK: port_map %x -> %x\n",
c40e7cb8
JAR		 780 port_map,
781 port_map & mv);
5b66c829
AC		 782 dev_printk(KERN_ERR, &pdev->dev,
783 "Disabling your PATA port. Use the boot option 'ahci.marvell_enable=0' to avoid this.\n");
cd70c266 784
c40e7cb8 785 port_map &= mv;
cd70c266
JG		 786 }
787
17199b18 788 /* cross check port_map and cap.n_ports */
7a234aff 789 if (port_map) {
837f5f8f 790 int map_ports = 0;
17199b18 791
837f5f8f
TH		 792 for (i = 0; i < AHCI_MAX_PORTS; i++)
793 if (port_map & (1 << i))
794 map_ports++;
17199b18 795
837f5f8f
TH		 796 /* If PI has more ports than n_ports, whine, clear
797 * port_map and let it be generated from n_ports.
17199b18 798 */
837f5f8f 799 if (map_ports > ahci_nr_ports(cap)) {
4447d351 800 dev_printk(KERN_WARNING, &pdev->dev,
837f5f8f
TH		 801 "implemented port map (0x%x) contains more "
802 "ports than nr_ports (%u), using nr_ports\n",
803 port_map, ahci_nr_ports(cap));
7a234aff
TH		 804 port_map = 0;
805 }
806 }
807
808 /* fabricate port_map from cap.nr_ports */
809 if (!port_map) {
17199b18 810 port_map = (1 << ahci_nr_ports(cap)) - 1;
7a234aff
TH		 811 dev_printk(KERN_WARNING, &pdev->dev,
812 "forcing PORTS_IMPL to 0x%x\n", port_map);
813
814 /* write the fixed up value to the PI register */
815 hpriv->saved_port_map = port_map;
17199b18
TH		 816 }
817
d447df14
TH		 818 /* record values to use during operation */
819 hpriv->cap = cap;
820 hpriv->port_map = port_map;
821}
822
823/**
824 * ahci_restore_initial_config - Restore initial config
4447d351 825 * @host: target ATA host
d447df14
TH		 826 *
827 * Restore initial config stored by ahci_save_initial_config().
828 *
829 * LOCKING:
830 * None.
831 */
4447d351 832static void ahci_restore_initial_config(struct ata_host *host)
d447df14 833{
4447d351
TH		 834 struct ahci_host_priv *hpriv = host->private_data;
835 void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
836
d447df14
TH		 837 writel(hpriv->saved_cap, mmio + HOST_CAP);
838 writel(hpriv->saved_port_map, mmio + HOST_PORTS_IMPL);
839 (void) readl(mmio + HOST_PORTS_IMPL); /* flush */
840}
841
203ef6c4 842static unsigned ahci_scr_offset(struct ata_port *ap, unsigned int sc_reg)
1da177e4 843{
203ef6c4
TH		 844 static const int offset[] = {
845 [SCR_STATUS] = PORT_SCR_STAT,
846 [SCR_CONTROL] = PORT_SCR_CTL,
847 [SCR_ERROR] = PORT_SCR_ERR,
848 [SCR_ACTIVE] = PORT_SCR_ACT,
849 [SCR_NOTIFICATION] = PORT_SCR_NTF,
850 };
851 struct ahci_host_priv *hpriv = ap->host->private_data;
1da177e4 852
203ef6c4
TH		 853 if (sc_reg < ARRAY_SIZE(offset) &&
854 (sc_reg != SCR_NOTIFICATION || (hpriv->cap & HOST_CAP_SNTF)))
855 return offset[sc_reg];
da3dbb17 856 return 0;
1da177e4
LT		 857}
858
82ef04fb 859static int ahci_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val)
1da177e4 860{
82ef04fb
TH		 861 void __iomem *port_mmio = ahci_port_base(link->ap);
862 int offset = ahci_scr_offset(link->ap, sc_reg);
203ef6c4
TH		 863
864 if (offset) {
865 *val = readl(port_mmio + offset);
866 return 0;
1da177e4 867 }
203ef6c4
TH		 868 return -EINVAL;
869}
1da177e4 870
82ef04fb 871static int ahci_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val)
203ef6c4 872{
82ef04fb
TH		 873 void __iomem *port_mmio = ahci_port_base(link->ap);
874 int offset = ahci_scr_offset(link->ap, sc_reg);
203ef6c4
TH		 875
876 if (offset) {
877 writel(val, port_mmio + offset);
878 return 0;
879 }
880 return -EINVAL;
1da177e4
LT		 881}
882
4447d351 883static void ahci_start_engine(struct ata_port *ap)
7c76d1e8 884{
4447d351 885 void __iomem *port_mmio = ahci_port_base(ap);
7c76d1e8
TH		 886 u32 tmp;
887
d8fcd116 888 /* start DMA */
9f592056 889 tmp = readl(port_mmio + PORT_CMD);
7c76d1e8
TH		 890 tmp |= PORT_CMD_START;
891 writel(tmp, port_mmio + PORT_CMD);
892 readl(port_mmio + PORT_CMD); /* flush */
893}
894
4447d351 895static int ahci_stop_engine(struct ata_port *ap)
254950cd 896{
4447d351 897 void __iomem *port_mmio = ahci_port_base(ap);
254950cd
TH		 898 u32 tmp;
899
900 tmp = readl(port_mmio + PORT_CMD);
901
d8fcd116 902 /* check if the HBA is idle */
254950cd
TH		 903 if ((tmp & (PORT_CMD_START | PORT_CMD_LIST_ON)) == 0)
904 return 0;
905
d8fcd116 906 /* setting HBA to idle */
254950cd
TH		 907 tmp &= ~PORT_CMD_START;
908 writel(tmp, port_mmio + PORT_CMD);
909
d8fcd116 910 /* wait for engine to stop. This could be as long as 500 msec */
254950cd 911 tmp = ata_wait_register(port_mmio + PORT_CMD,
2dcb407e 912 PORT_CMD_LIST_ON, PORT_CMD_LIST_ON, 1, 500);
d8fcd116 913 if (tmp & PORT_CMD_LIST_ON)
254950cd
TH		 914 return -EIO;
915
916 return 0;
917}
918
4447d351 919static void ahci_start_fis_rx(struct ata_port *ap)
0be0aa98 920{
4447d351
TH		 921 void __iomem *port_mmio = ahci_port_base(ap);
922 struct ahci_host_priv *hpriv = ap->host->private_data;
923 struct ahci_port_priv *pp = ap->private_data;
0be0aa98
TH		 924 u32 tmp;
925
926 /* set FIS registers */
4447d351
TH		 927 if (hpriv->cap & HOST_CAP_64)
928 writel((pp->cmd_slot_dma >> 16) >> 16,
929 port_mmio + PORT_LST_ADDR_HI);
930 writel(pp->cmd_slot_dma & 0xffffffff, port_mmio + PORT_LST_ADDR);
0be0aa98 931
4447d351
TH		 932 if (hpriv->cap & HOST_CAP_64)
933 writel((pp->rx_fis_dma >> 16) >> 16,
934 port_mmio + PORT_FIS_ADDR_HI);
935 writel(pp->rx_fis_dma & 0xffffffff, port_mmio + PORT_FIS_ADDR);
0be0aa98
TH		 936
937 /* enable FIS reception */
938 tmp = readl(port_mmio + PORT_CMD);
939 tmp |= PORT_CMD_FIS_RX;
940 writel(tmp, port_mmio + PORT_CMD);
941
942 /* flush */
943 readl(port_mmio + PORT_CMD);
944}
945
4447d351 946static int ahci_stop_fis_rx(struct ata_port *ap)
0be0aa98 947{
4447d351 948 void __iomem *port_mmio = ahci_port_base(ap);
0be0aa98
TH		 949 u32 tmp;
950
951 /* disable FIS reception */
952 tmp = readl(port_mmio + PORT_CMD);
953 tmp &= ~PORT_CMD_FIS_RX;
954 writel(tmp, port_mmio + PORT_CMD);
955
956 /* wait for completion, spec says 500ms, give it 1000 */
957 tmp = ata_wait_register(port_mmio + PORT_CMD, PORT_CMD_FIS_ON,
958 PORT_CMD_FIS_ON, 10, 1000);
959 if (tmp & PORT_CMD_FIS_ON)
960 return -EBUSY;
961
962 return 0;
963}
964
4447d351 965static void ahci_power_up(struct ata_port *ap)
0be0aa98 966{
4447d351
TH		 967 struct ahci_host_priv *hpriv = ap->host->private_data;
968 void __iomem *port_mmio = ahci_port_base(ap);
0be0aa98
TH		 969 u32 cmd;
970
971 cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;
972
973 /* spin up device */
4447d351 974 if (hpriv->cap & HOST_CAP_SSS) {
0be0aa98
TH		 975 cmd |= PORT_CMD_SPIN_UP;
976 writel(cmd, port_mmio + PORT_CMD);
977 }
978
979 /* wake up link */
980 writel(cmd | PORT_CMD_ICC_ACTIVE, port_mmio + PORT_CMD);
981}
982
31556594
KCA		 983static void ahci_disable_alpm(struct ata_port *ap)
984{
985 struct ahci_host_priv *hpriv = ap->host->private_data;
986 void __iomem *port_mmio = ahci_port_base(ap);
987 u32 cmd;
988 struct ahci_port_priv *pp = ap->private_data;
989
990 /* IPM bits should be disabled by libata-core */
991 /* get the existing command bits */
992 cmd = readl(port_mmio + PORT_CMD);
993
994 /* disable ALPM and ASP */
995 cmd &= ~PORT_CMD_ASP;
996 cmd &= ~PORT_CMD_ALPE;
997
998 /* force the interface back to active */
999 cmd |= PORT_CMD_ICC_ACTIVE;
1000
1001 /* write out new cmd value */
1002 writel(cmd, port_mmio + PORT_CMD);
1003 cmd = readl(port_mmio + PORT_CMD);
1004
1005 /* wait 10ms to be sure we've come out of any low power state */
1006 msleep(10);
1007
1008 /* clear out any PhyRdy stuff from interrupt status */
1009 writel(PORT_IRQ_PHYRDY, port_mmio + PORT_IRQ_STAT);
1010
1011 /* go ahead and clean out PhyRdy Change from Serror too */
82ef04fb 1012 ahci_scr_write(&ap->link, SCR_ERROR, ((1 << 16) | (1 << 18)));
31556594
KCA		 1013
1014 /*
1015 * Clear flag to indicate that we should ignore all PhyRdy
1016 * state changes
1017 */
1018 hpriv->flags &= ~AHCI_HFLAG_NO_HOTPLUG;
1019
1020 /*
1021 * Enable interrupts on Phy Ready.
1022 */
1023 pp->intr_mask |= PORT_IRQ_PHYRDY;
1024 writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
1025
1026 /*
1027 * don't change the link pm policy - we can be called
1028 * just to turn of link pm temporarily
1029 */
1030}
1031
1032static int ahci_enable_alpm(struct ata_port *ap,
1033 enum link_pm policy)
1034{
1035 struct ahci_host_priv *hpriv = ap->host->private_data;
1036 void __iomem *port_mmio = ahci_port_base(ap);
1037 u32 cmd;
1038 struct ahci_port_priv *pp = ap->private_data;
1039 u32 asp;
1040
1041 /* Make sure the host is capable of link power management */
1042 if (!(hpriv->cap & HOST_CAP_ALPM))
1043 return -EINVAL;
1044
1045 switch (policy) {
1046 case MAX_PERFORMANCE:
1047 case NOT_AVAILABLE:
1048 /*
1049 * if we came here with NOT_AVAILABLE,
1050 * it just means this is the first time we
1051 * have tried to enable - default to max performance,
1052 * and let the user go to lower power modes on request.
1053 */
1054 ahci_disable_alpm(ap);
1055 return 0;
1056 case MIN_POWER:
1057 /* configure HBA to enter SLUMBER */
1058 asp = PORT_CMD_ASP;
1059 break;
1060 case MEDIUM_POWER:
1061 /* configure HBA to enter PARTIAL */
1062 asp = 0;
1063 break;
1064 default:
1065 return -EINVAL;
1066 }
1067
1068 /*
1069 * Disable interrupts on Phy Ready. This keeps us from
1070 * getting woken up due to spurious phy ready interrupts
1071 * TBD - Hot plug should be done via polling now, is
1072 * that even supported?
1073 */
1074 pp->intr_mask &= ~PORT_IRQ_PHYRDY;
1075 writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
1076
1077 /*
1078 * Set a flag to indicate that we should ignore all PhyRdy
1079 * state changes since these can happen now whenever we
1080 * change link state
1081 */
1082 hpriv->flags |= AHCI_HFLAG_NO_HOTPLUG;
1083
1084 /* get the existing command bits */
1085 cmd = readl(port_mmio + PORT_CMD);
1086
1087 /*
1088 * Set ASP based on Policy
1089 */
1090 cmd |= asp;
1091
1092 /*
1093 * Setting this bit will instruct the HBA to aggressively
1094 * enter a lower power link state when it's appropriate and
1095 * based on the value set above for ASP
1096 */
1097 cmd |= PORT_CMD_ALPE;
1098
1099 /* write out new cmd value */
1100 writel(cmd, port_mmio + PORT_CMD);
1101 cmd = readl(port_mmio + PORT_CMD);
1102
1103 /* IPM bits should be set by libata-core */
1104 return 0;
1105}
1106
438ac6d5 1107#ifdef CONFIG_PM
4447d351 1108static void ahci_power_down(struct ata_port *ap)
0be0aa98 1109{
4447d351
TH		 1110 struct ahci_host_priv *hpriv = ap->host->private_data;
1111 void __iomem *port_mmio = ahci_port_base(ap);
0be0aa98
TH		 1112 u32 cmd, scontrol;
1113
4447d351 1114 if (!(hpriv->cap & HOST_CAP_SSS))
07c53dac 1115 return;
0be0aa98 1116
07c53dac
TH		 1117 /* put device into listen mode, first set PxSCTL.DET to 0 */
1118 scontrol = readl(port_mmio + PORT_SCR_CTL);
1119 scontrol &= ~0xf;
1120 writel(scontrol, port_mmio + PORT_SCR_CTL);
0be0aa98 1121
07c53dac
TH		 1122 /* then set PxCMD.SUD to 0 */
1123 cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;
1124 cmd &= ~PORT_CMD_SPIN_UP;
1125 writel(cmd, port_mmio + PORT_CMD);
0be0aa98 1126}
438ac6d5 1127#endif
0be0aa98 1128
df69c9c5 1129static void ahci_start_port(struct ata_port *ap)
0be0aa98 1130{
18f7ba4c
KCA		 1131 struct ahci_port_priv *pp = ap->private_data;
1132 struct ata_link *link;
1133 struct ahci_em_priv *emp;
4c1e9aa4
DM		 1134 ssize_t rc;
1135 int i;
18f7ba4c 1136
0be0aa98 1137 /* enable FIS reception */
4447d351 1138 ahci_start_fis_rx(ap);
0be0aa98
TH		 1139
1140 /* enable DMA */
4447d351 1141 ahci_start_engine(ap);
18f7ba4c
KCA		 1142
1143 /* turn on LEDs */
1144 if (ap->flags & ATA_FLAG_EM) {
1eca4365 1145 ata_for_each_link(link, ap, EDGE) {
18f7ba4c 1146 emp = &pp->em_priv[link->pmp];
4c1e9aa4
DM		 1147
1148 /* EM Transmit bit maybe busy during init */
d50ce07d 1149 for (i = 0; i < EM_MAX_RETRY; i++) {
4c1e9aa4
DM		 1150 rc = ahci_transmit_led_message(ap,
1151 emp->led_state,
1152 4);
1153 if (rc == -EBUSY)
d50ce07d 1154 msleep(1);
4c1e9aa4
DM		 1155 else
1156 break;
1157 }
18f7ba4c
KCA		 1158 }
1159 }
1160
1161 if (ap->flags & ATA_FLAG_SW_ACTIVITY)
1eca4365 1162 ata_for_each_link(link, ap, EDGE)
18f7ba4c
KCA		 1163 ahci_init_sw_activity(link);
1164
0be0aa98
TH		 1165}
1166
4447d351 1167static int ahci_deinit_port(struct ata_port *ap, const char **emsg)
0be0aa98
TH		 1168{
1169 int rc;
1170
1171 /* disable DMA */
4447d351 1172 rc = ahci_stop_engine(ap);
0be0aa98
TH		 1173 if (rc) {
1174 *emsg = "failed to stop engine";
1175 return rc;
1176 }
1177
1178 /* disable FIS reception */
4447d351 1179 rc = ahci_stop_fis_rx(ap);
0be0aa98
TH		 1180 if (rc) {
1181 *emsg = "failed stop FIS RX";
1182 return rc;
1183 }
1184
0be0aa98
TH		 1185 return 0;
1186}
1187
4447d351 1188static int ahci_reset_controller(struct ata_host *host)
d91542c1 1189{
4447d351 1190 struct pci_dev *pdev = to_pci_dev(host->dev);
49f29090 1191 struct ahci_host_priv *hpriv = host->private_data;
4447d351 1192 void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
d447df14 1193 u32 tmp;
d91542c1 1194
3cc3eb11
JG		 1195 /* we must be in AHCI mode, before using anything
1196 * AHCI-specific, such as HOST_RESET.
1197 */
b710a1f4 1198 ahci_enable_ahci(mmio);
3cc3eb11
JG		 1199
1200 /* global controller reset */
a22e6444
TH		 1201 if (!ahci_skip_host_reset) {
1202 tmp = readl(mmio + HOST_CTL);
1203 if ((tmp & HOST_RESET) == 0) {
1204 writel(tmp | HOST_RESET, mmio + HOST_CTL);
1205 readl(mmio + HOST_CTL); /* flush */
1206 }
d91542c1 1207
24920c8a
ZR		 1208 /*
1209 * to perform host reset, OS should set HOST_RESET
1210 * and poll until this bit is read to be "0".
1211 * reset must complete within 1 second, or
a22e6444
TH		 1212 * the hardware should be considered fried.
1213 */
24920c8a
ZR		 1214 tmp = ata_wait_register(mmio + HOST_CTL, HOST_RESET,
1215 HOST_RESET, 10, 1000);
d91542c1 1216
a22e6444
TH		 1217 if (tmp & HOST_RESET) {
1218 dev_printk(KERN_ERR, host->dev,
1219 "controller reset failed (0x%x)\n", tmp);
1220 return -EIO;
1221 }
d91542c1 1222
a22e6444
TH		 1223 /* turn on AHCI mode */
1224 ahci_enable_ahci(mmio);
98fa4b60 1225
a22e6444
TH		 1226 /* Some registers might be cleared on reset. Restore
1227 * initial values.
1228 */
1229 ahci_restore_initial_config(host);
1230 } else
1231 dev_printk(KERN_INFO, host->dev,
1232 "skipping global host reset\n");
d91542c1
TH		 1233
1234 if (pdev->vendor == PCI_VENDOR_ID_INTEL) {
1235 u16 tmp16;
1236
1237 /* configure PCS */
1238 pci_read_config_word(pdev, 0x92, &tmp16);
49f29090
TH		 1239 if ((tmp16 & hpriv->port_map) != hpriv->port_map) {
1240 tmp16 |= hpriv->port_map;
1241 pci_write_config_word(pdev, 0x92, tmp16);
1242 }
d91542c1
TH		 1243 }
1244
1245 return 0;
1246}
1247
18f7ba4c
KCA		 1248static void ahci_sw_activity(struct ata_link *link)
1249{
1250 struct ata_port *ap = link->ap;
1251 struct ahci_port_priv *pp = ap->private_data;
1252 struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
1253
1254 if (!(link->flags & ATA_LFLAG_SW_ACTIVITY))
1255 return;
1256
1257 emp->activity++;
1258 if (!timer_pending(&emp->timer))
1259 mod_timer(&emp->timer, jiffies + msecs_to_jiffies(10));
1260}
1261
1262static void ahci_sw_activity_blink(unsigned long arg)
1263{
1264 struct ata_link *link = (struct ata_link *)arg;
1265 struct ata_port *ap = link->ap;
1266 struct ahci_port_priv *pp = ap->private_data;
1267 struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
1268 unsigned long led_message = emp->led_state;
1269 u32 activity_led_state;
eb40963c 1270 unsigned long flags;
18f7ba4c 1271
87943acf 1272 led_message &= EM_MSG_LED_VALUE;
18f7ba4c
KCA		 1273 led_message |= ap->port_no | (link->pmp << 8);
1274
1275 /* check to see if we've had activity. If so,
1276 * toggle state of LED and reset timer. If not,
1277 * turn LED to desired idle state.
1278 */
eb40963c 1279 spin_lock_irqsave(ap->lock, flags);
18f7ba4c
KCA		 1280 if (emp->saved_activity != emp->activity) {
1281 emp->saved_activity = emp->activity;
1282 /* get the current LED state */
87943acf 1283 activity_led_state = led_message & EM_MSG_LED_VALUE_ON;
18f7ba4c
KCA		 1284
1285 if (activity_led_state)
1286 activity_led_state = 0;
1287 else
1288 activity_led_state = 1;
1289
1290 /* clear old state */
87943acf 1291 led_message &= ~EM_MSG_LED_VALUE_ACTIVITY;
18f7ba4c
KCA		 1292
1293 /* toggle state */
1294 led_message |= (activity_led_state << 16);
1295 mod_timer(&emp->timer, jiffies + msecs_to_jiffies(100));
1296 } else {
1297 /* switch to idle */
87943acf 1298 led_message &= ~EM_MSG_LED_VALUE_ACTIVITY;
18f7ba4c
KCA		 1299 if (emp->blink_policy == BLINK_OFF)
1300 led_message |= (1 << 16);
1301 }
eb40963c 1302 spin_unlock_irqrestore(ap->lock, flags);
18f7ba4c
KCA		 1303 ahci_transmit_led_message(ap, led_message, 4);
1304}
1305
1306static void ahci_init_sw_activity(struct ata_link *link)
1307{
1308 struct ata_port *ap = link->ap;
1309 struct ahci_port_priv *pp = ap->private_data;
1310 struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
1311
1312 /* init activity stats, setup timer */
1313 emp->saved_activity = emp->activity = 0;
1314 setup_timer(&emp->timer, ahci_sw_activity_blink, (unsigned long)link);
1315
1316 /* check our blink policy and set flag for link if it's enabled */
1317 if (emp->blink_policy)
1318 link->flags |= ATA_LFLAG_SW_ACTIVITY;
1319}
1320
1321static int ahci_reset_em(struct ata_host *host)
1322{
1323 void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
1324 u32 em_ctl;
1325
1326 em_ctl = readl(mmio + HOST_EM_CTL);
1327 if ((em_ctl & EM_CTL_TM) || (em_ctl & EM_CTL_RST))
1328 return -EINVAL;
1329
1330 writel(em_ctl | EM_CTL_RST, mmio + HOST_EM_CTL);
1331 return 0;
1332}
1333
1334static ssize_t ahci_transmit_led_message(struct ata_port *ap, u32 state,
1335 ssize_t size)
1336{
1337 struct ahci_host_priv *hpriv = ap->host->private_data;
1338 struct ahci_port_priv *pp = ap->private_data;
1339 void __iomem *mmio = ap->host->iomap[AHCI_PCI_BAR];
1340 u32 em_ctl;
1341 u32 message[] = {0, 0};
93082f0b 1342 unsigned long flags;
18f7ba4c
KCA		 1343 int pmp;
1344 struct ahci_em_priv *emp;
1345
1346 /* get the slot number from the message */
87943acf 1347 pmp = (state & EM_MSG_LED_PMP_SLOT) >> 8;
d50ce07d 1348 if (pmp < EM_MAX_SLOTS)
18f7ba4c
KCA		 1349 emp = &pp->em_priv[pmp];
1350 else
1351 return -EINVAL;
1352
1353 spin_lock_irqsave(ap->lock, flags);
1354
1355 /*
1356 * if we are still busy transmitting a previous message,
1357 * do not allow
1358 */
1359 em_ctl = readl(mmio + HOST_EM_CTL);
1360 if (em_ctl & EM_CTL_TM) {
1361 spin_unlock_irqrestore(ap->lock, flags);
4c1e9aa4 1362 return -EBUSY;
18f7ba4c
KCA		 1363 }
1364
1365 /*
1366 * create message header - this is all zero except for
1367 * the message size, which is 4 bytes.
1368 */
1369 message[0] |= (4 << 8);
1370
1371 /* ignore 0:4 of byte zero, fill in port info yourself */
87943acf 1372 message[1] = ((state & ~EM_MSG_LED_HBA_PORT) | ap->port_no);
18f7ba4c
KCA		 1373
1374 /* write message to EM_LOC */
1375 writel(message[0], mmio + hpriv->em_loc);
1376 writel(message[1], mmio + hpriv->em_loc+4);
1377
1378 /* save off new led state for port/slot */
208f2a88 1379 emp->led_state = state;
18f7ba4c
KCA		 1380
1381 /*
1382 * tell hardware to transmit the message
1383 */
1384 writel(em_ctl | EM_CTL_TM, mmio + HOST_EM_CTL);
1385
1386 spin_unlock_irqrestore(ap->lock, flags);
1387 return size;
1388}
1389
1390static ssize_t ahci_led_show(struct ata_port *ap, char *buf)
1391{
1392 struct ahci_port_priv *pp = ap->private_data;
1393 struct ata_link *link;
1394 struct ahci_em_priv *emp;
1395 int rc = 0;
1396
1eca4365 1397 ata_for_each_link(link, ap, EDGE) {
18f7ba4c
KCA		 1398 emp = &pp->em_priv[link->pmp];
1399 rc += sprintf(buf, "%lx\n", emp->led_state);
1400 }
1401 return rc;
1402}
1403
1404static ssize_t ahci_led_store(struct ata_port *ap, const char *buf,
1405 size_t size)
1406{
1407 int state;
1408 int pmp;
1409 struct ahci_port_priv *pp = ap->private_data;
1410 struct ahci_em_priv *emp;
1411
1412 state = simple_strtoul(buf, NULL, 0);
1413
1414 /* get the slot number from the message */
87943acf 1415 pmp = (state & EM_MSG_LED_PMP_SLOT) >> 8;
d50ce07d 1416 if (pmp < EM_MAX_SLOTS)
18f7ba4c
KCA		 1417 emp = &pp->em_priv[pmp];
1418 else
1419 return -EINVAL;
1420
1421 /* mask off the activity bits if we are in sw_activity
1422 * mode, user should turn off sw_activity before setting
1423 * activity led through em_message
1424 */
1425 if (emp->blink_policy)
87943acf 1426 state &= ~EM_MSG_LED_VALUE_ACTIVITY;
18f7ba4c
KCA		 1427
1428 return ahci_transmit_led_message(ap, state, size);
1429}
1430
1431static ssize_t ahci_activity_store(struct ata_device *dev, enum sw_activity val)
1432{
1433 struct ata_link *link = dev->link;
1434 struct ata_port *ap = link->ap;
1435 struct ahci_port_priv *pp = ap->private_data;
1436 struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
1437 u32 port_led_state = emp->led_state;
1438
1439 /* save the desired Activity LED behavior */
1440 if (val == OFF) {
1441 /* clear LFLAG */
1442 link->flags &= ~(ATA_LFLAG_SW_ACTIVITY);
1443
1444 /* set the LED to OFF */
87943acf 1445 port_led_state &= EM_MSG_LED_VALUE_OFF;
18f7ba4c
KCA		 1446 port_led_state |= (ap->port_no | (link->pmp << 8));
1447 ahci_transmit_led_message(ap, port_led_state, 4);
1448 } else {
1449 link->flags |= ATA_LFLAG_SW_ACTIVITY;
1450 if (val == BLINK_OFF) {
1451 /* set LED to ON for idle */
87943acf 1452 port_led_state &= EM_MSG_LED_VALUE_OFF;
18f7ba4c 1453 port_led_state |= (ap->port_no | (link->pmp << 8));
87943acf 1454 port_led_state |= EM_MSG_LED_VALUE_ON; /* check this */
18f7ba4c
KCA		 1455 ahci_transmit_led_message(ap, port_led_state, 4);
1456 }
1457 }
1458 emp->blink_policy = val;
1459 return 0;
1460}
1461
1462static ssize_t ahci_activity_show(struct ata_device *dev, char *buf)
1463{
1464 struct ata_link *link = dev->link;
1465 struct ata_port *ap = link->ap;
1466 struct ahci_port_priv *pp = ap->private_data;
1467 struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
1468
1469 /* display the saved value of activity behavior for this
1470 * disk.
1471 */
1472 return sprintf(buf, "%d\n", emp->blink_policy);
1473}
1474
2bcd866b
JG		 1475static void ahci_port_init(struct pci_dev *pdev, struct ata_port *ap,
1476 int port_no, void __iomem *mmio,
1477 void __iomem *port_mmio)
1478{
1479 const char *emsg = NULL;
1480 int rc;
1481 u32 tmp;
1482
1483 /* make sure port is not active */
1484 rc = ahci_deinit_port(ap, &emsg);
1485 if (rc)
1486 dev_printk(KERN_WARNING, &pdev->dev,
1487 "%s (%d)\n", emsg, rc);
1488
1489 /* clear SError */
1490 tmp = readl(port_mmio + PORT_SCR_ERR);
1491 VPRINTK("PORT_SCR_ERR 0x%x\n", tmp);
1492 writel(tmp, port_mmio + PORT_SCR_ERR);
1493
1494 /* clear port IRQ */
1495 tmp = readl(port_mmio + PORT_IRQ_STAT);
1496 VPRINTK("PORT_IRQ_STAT 0x%x\n", tmp);
1497 if (tmp)
1498 writel(tmp, port_mmio + PORT_IRQ_STAT);
1499
1500 writel(1 << port_no, mmio + HOST_IRQ_STAT);
1501}
1502
4447d351 1503static void ahci_init_controller(struct ata_host *host)
d91542c1 1504{
417a1a6d 1505 struct ahci_host_priv *hpriv = host->private_data;
4447d351
TH		 1506 struct pci_dev *pdev = to_pci_dev(host->dev);
1507 void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
2bcd866b 1508 int i;
cd70c266 1509 void __iomem *port_mmio;
d91542c1 1510 u32 tmp;
c40e7cb8 1511 int mv;
d91542c1 1512
417a1a6d 1513 if (hpriv->flags & AHCI_HFLAG_MV_PATA) {
c40e7cb8
JAR		 1514 if (pdev->device == 0x6121)
1515 mv = 2;
1516 else
1517 mv = 4;
1518 port_mmio = __ahci_port_base(host, mv);
cd70c266
JG		 1519
1520 writel(0, port_mmio + PORT_IRQ_MASK);
1521
1522 /* clear port IRQ */
1523 tmp = readl(port_mmio + PORT_IRQ_STAT);
1524 VPRINTK("PORT_IRQ_STAT 0x%x\n", tmp);
1525 if (tmp)
1526 writel(tmp, port_mmio + PORT_IRQ_STAT);
1527 }
1528
4447d351
TH		 1529 for (i = 0; i < host->n_ports; i++) {
1530 struct ata_port *ap = host->ports[i];
d91542c1 1531
cd70c266 1532 port_mmio = ahci_port_base(ap);
4447d351 1533 if (ata_port_is_dummy(ap))
d91542c1 1534 continue;
d91542c1 1535
2bcd866b 1536 ahci_port_init(pdev, ap, i, mmio, port_mmio);
d91542c1
TH		 1537 }
1538
1539 tmp = readl(mmio + HOST_CTL);
1540 VPRINTK("HOST_CTL 0x%x\n", tmp);
1541 writel(tmp | HOST_IRQ_EN, mmio + HOST_CTL);
1542 tmp = readl(mmio + HOST_CTL);
1543 VPRINTK("HOST_CTL 0x%x\n", tmp);
1544}
1545
a878539e
JG		 1546static void ahci_dev_config(struct ata_device *dev)
1547{
1548 struct ahci_host_priv *hpriv = dev->link->ap->host->private_data;
1549
4cde32fc 1550 if (hpriv->flags & AHCI_HFLAG_SECT255) {
a878539e 1551 dev->max_sectors = 255;
4cde32fc
JG		 1552 ata_dev_printk(dev, KERN_INFO,
1553 "SB600 AHCI: limiting to 255 sectors per cmd\n");
1554 }
a878539e
JG		 1555}
1556
422b7595 1557static unsigned int ahci_dev_classify(struct ata_port *ap)
1da177e4 1558{
4447d351 1559 void __iomem *port_mmio = ahci_port_base(ap);
1da177e4 1560 struct ata_taskfile tf;
422b7595
TH		 1561 u32 tmp;
1562
1563 tmp = readl(port_mmio + PORT_SIG);
1564 tf.lbah = (tmp >> 24) & 0xff;
1565 tf.lbam = (tmp >> 16) & 0xff;
1566 tf.lbal = (tmp >> 8) & 0xff;
1567 tf.nsect = (tmp) & 0xff;
1568
1569 return ata_dev_classify(&tf);
1570}
1571
12fad3f9
TH		 1572static void ahci_fill_cmd_slot(struct ahci_port_priv *pp, unsigned int tag,
1573 u32 opts)
cc9278ed 1574{
12fad3f9
TH		 1575 dma_addr_t cmd_tbl_dma;
1576
1577 cmd_tbl_dma = pp->cmd_tbl_dma + tag * AHCI_CMD_TBL_SZ;
1578
1579 pp->cmd_slot[tag].opts = cpu_to_le32(opts);
1580 pp->cmd_slot[tag].status = 0;
1581 pp->cmd_slot[tag].tbl_addr = cpu_to_le32(cmd_tbl_dma & 0xffffffff);
1582 pp->cmd_slot[tag].tbl_addr_hi = cpu_to_le32((cmd_tbl_dma >> 16) >> 16);
cc9278ed
TH		 1583}
1584
d2e75dff 1585static int ahci_kick_engine(struct ata_port *ap, int force_restart)
4658f79b 1586{
350756f6 1587 void __iomem *port_mmio = ahci_port_base(ap);
cca3974e 1588 struct ahci_host_priv *hpriv = ap->host->private_data;
520d06f9 1589 u8 status = readl(port_mmio + PORT_TFDATA) & 0xFF;
bf2af2a2 1590 u32 tmp;
d2e75dff 1591 int busy, rc;
bf2af2a2 1592
d2e75dff 1593 /* do we need to kick the port? */
520d06f9 1594 busy = status & (ATA_BUSY | ATA_DRQ);
d2e75dff
TH		 1595 if (!busy && !force_restart)
1596 return 0;
1597
1598 /* stop engine */
1599 rc = ahci_stop_engine(ap);
1600 if (rc)
1601 goto out_restart;
1602
1603 /* need to do CLO? */
1604 if (!busy) {
1605 rc = 0;
1606 goto out_restart;
1607 }
1608
1609 if (!(hpriv->cap & HOST_CAP_CLO)) {
1610 rc = -EOPNOTSUPP;
1611 goto out_restart;
1612 }
bf2af2a2 1613
d2e75dff 1614 /* perform CLO */
bf2af2a2
BJ		 1615 tmp = readl(port_mmio + PORT_CMD);
1616 tmp |= PORT_CMD_CLO;
1617 writel(tmp, port_mmio + PORT_CMD);
1618
d2e75dff 1619 rc = 0;
bf2af2a2
BJ		 1620 tmp = ata_wait_register(port_mmio + PORT_CMD,
1621 PORT_CMD_CLO, PORT_CMD_CLO, 1, 500);
1622 if (tmp & PORT_CMD_CLO)
d2e75dff 1623 rc = -EIO;
bf2af2a2 1624
d2e75dff
TH		 1625 /* restart engine */
1626 out_restart:
1627 ahci_start_engine(ap);
1628 return rc;
bf2af2a2
BJ		 1629}
1630
91c4a2e0
TH		 1631static int ahci_exec_polled_cmd(struct ata_port *ap, int pmp,
1632 struct ata_taskfile *tf, int is_cmd, u16 flags,
1633 unsigned long timeout_msec)
bf2af2a2 1634{
91c4a2e0 1635 const u32 cmd_fis_len = 5; /* five dwords */
4658f79b 1636 struct ahci_port_priv *pp = ap->private_data;
4447d351 1637 void __iomem *port_mmio = ahci_port_base(ap);
91c4a2e0
TH		 1638 u8 *fis = pp->cmd_tbl;
1639 u32 tmp;
1640
1641 /* prep the command */
1642 ata_tf_to_fis(tf, pmp, is_cmd, fis);
1643 ahci_fill_cmd_slot(pp, 0, cmd_fis_len | flags | (pmp << 12));
1644
1645 /* issue & wait */
1646 writel(1, port_mmio + PORT_CMD_ISSUE);
1647
1648 if (timeout_msec) {
1649 tmp = ata_wait_register(port_mmio + PORT_CMD_ISSUE, 0x1, 0x1,
1650 1, timeout_msec);
1651 if (tmp & 0x1) {
1652 ahci_kick_engine(ap, 1);
1653 return -EBUSY;
1654 }
1655 } else
1656 readl(port_mmio + PORT_CMD_ISSUE); /* flush */
1657
1658 return 0;
1659}
1660
bd17243a
SH		 1661static int ahci_do_softreset(struct ata_link *link, unsigned int *class,
1662 int pmp, unsigned long deadline,
1663 int (*check_ready)(struct ata_link *link))
91c4a2e0 1664{
cc0680a5 1665 struct ata_port *ap = link->ap;
4658f79b 1666 const char *reason = NULL;
2cbb79eb 1667 unsigned long now, msecs;
4658f79b 1668 struct ata_taskfile tf;
4658f79b
TH		 1669 int rc;
1670
1671 DPRINTK("ENTER\n");
1672
1673 /* prepare for SRST (AHCI-1.1 10.4.1) */
d2e75dff 1674 rc = ahci_kick_engine(ap, 1);
994056d7 1675 if (rc && rc != -EOPNOTSUPP)
cc0680a5 1676 ata_link_printk(link, KERN_WARNING,
994056d7 1677 "failed to reset engine (errno=%d)\n", rc);
4658f79b 1678
cc0680a5 1679 ata_tf_init(link->device, &tf);
4658f79b
TH		 1680
1681 /* issue the first D2H Register FIS */
2cbb79eb
TH		 1682 msecs = 0;
1683 now = jiffies;
1684 if (time_after(now, deadline))
1685 msecs = jiffies_to_msecs(deadline - now);
1686
4658f79b 1687 tf.ctl |= ATA_SRST;
a9cf5e85 1688 if (ahci_exec_polled_cmd(ap, pmp, &tf, 0,
91c4a2e0 1689 AHCI_CMD_RESET | AHCI_CMD_CLR_BUSY, msecs)) {
4658f79b
TH		 1690 rc = -EIO;
1691 reason = "1st FIS failed";
1692 goto fail;
1693 }
1694
1695 /* spec says at least 5us, but be generous and sleep for 1ms */
1696 msleep(1);
1697
1698 /* issue the second D2H Register FIS */
4658f79b 1699 tf.ctl &= ~ATA_SRST;
a9cf5e85 1700 ahci_exec_polled_cmd(ap, pmp, &tf, 0, 0, 0);
4658f79b 1701
705e76be 1702 /* wait for link to become ready */
bd17243a 1703 rc = ata_wait_after_reset(link, deadline, check_ready);
9b89391c
TH		 1704 /* link occupied, -ENODEV too is an error */
1705 if (rc) {
1706 reason = "device not ready";
1707 goto fail;
4658f79b 1708 }
9b89391c 1709 *class = ahci_dev_classify(ap);
4658f79b
TH		 1710
1711 DPRINTK("EXIT, class=%u\n", *class);
1712 return 0;
1713
4658f79b 1714 fail:
cc0680a5 1715 ata_link_printk(link, KERN_ERR, "softreset failed (%s)\n", reason);
4658f79b
TH		 1716 return rc;
1717}
1718
bd17243a
SH		 1719static int ahci_check_ready(struct ata_link *link)
1720{
1721 void __iomem *port_mmio = ahci_port_base(link->ap);
1722 u8 status = readl(port_mmio + PORT_TFDATA) & 0xFF;
1723
1724 return ata_check_ready(status);
1725}
1726
1727static int ahci_softreset(struct ata_link *link, unsigned int *class,
1728 unsigned long deadline)
1729{
1730 int pmp = sata_srst_pmp(link);
1731
1732 DPRINTK("ENTER\n");
1733
1734 return ahci_do_softreset(link, class, pmp, deadline, ahci_check_ready);
1735}
1736
1737static int ahci_sb600_check_ready(struct ata_link *link)
1738{
1739 void __iomem *port_mmio = ahci_port_base(link->ap);
1740 u8 status = readl(port_mmio + PORT_TFDATA) & 0xFF;
1741 u32 irq_status = readl(port_mmio + PORT_IRQ_STAT);
1742
1743 /*
1744 * There is no need to check TFDATA if BAD PMP is found due to HW bug,
1745 * which can save timeout delay.
1746 */
1747 if (irq_status & PORT_IRQ_BAD_PMP)
1748 return -EIO;
1749
1750 return ata_check_ready(status);
1751}
1752
1753static int ahci_sb600_softreset(struct ata_link *link, unsigned int *class,
1754 unsigned long deadline)
1755{
1756 struct ata_port *ap = link->ap;
1757 void __iomem *port_mmio = ahci_port_base(ap);
1758 int pmp = sata_srst_pmp(link);
1759 int rc;
1760 u32 irq_sts;
1761
1762 DPRINTK("ENTER\n");
1763
1764 rc = ahci_do_softreset(link, class, pmp, deadline,
1765 ahci_sb600_check_ready);
1766
1767 /*
1768 * Soft reset fails on some ATI chips with IPMS set when PMP
1769 * is enabled but SATA HDD/ODD is connected to SATA port,
1770 * do soft reset again to port 0.
1771 */
1772 if (rc == -EIO) {
1773 irq_sts = readl(port_mmio + PORT_IRQ_STAT);
1774 if (irq_sts & PORT_IRQ_BAD_PMP) {
1775 ata_link_printk(link, KERN_WARNING,
1776 "failed due to HW bug, retry pmp=0\n");
1777 rc = ahci_do_softreset(link, class, 0, deadline,
1778 ahci_check_ready);
1779 }
1780 }
1781
1782 return rc;
1783}
1784
cc0680a5 1785static int ahci_hardreset(struct ata_link *link, unsigned int *class,
d4b2bab4 1786 unsigned long deadline)
422b7595 1787{
9dadd45b 1788 const unsigned long *timing = sata_ehc_deb_timing(&link->eh_context);
cc0680a5 1789 struct ata_port *ap = link->ap;
4296971d
TH		 1790 struct ahci_port_priv *pp = ap->private_data;
1791 u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;
1792 struct ata_taskfile tf;
9dadd45b 1793 bool online;
4bd00f6a
TH		 1794 int rc;
1795
1796 DPRINTK("ENTER\n");
1da177e4 1797
4447d351 1798 ahci_stop_engine(ap);
4296971d
TH		 1799
1800 /* clear D2H reception area to properly wait for D2H FIS */
cc0680a5 1801 ata_tf_init(link->device, &tf);
dfd7a3db 1802 tf.command = 0x80;
9977126c 1803 ata_tf_to_fis(&tf, 0, 0, d2h_fis);
4296971d 1804
9dadd45b
TH		 1805 rc = sata_link_hardreset(link, timing, deadline, &online,
1806 ahci_check_ready);
4296971d 1807
4447d351 1808 ahci_start_engine(ap);
1da177e4 1809
9dadd45b 1810 if (online)
4bd00f6a 1811 *class = ahci_dev_classify(ap);
1da177e4 1812
4bd00f6a
TH		 1813 DPRINTK("EXIT, rc=%d, class=%u\n", rc, *class);
1814 return rc;
1815}
1816
cc0680a5 1817static int ahci_vt8251_hardreset(struct ata_link *link, unsigned int *class,
d4b2bab4 1818 unsigned long deadline)
ad616ffb 1819{
cc0680a5 1820 struct ata_port *ap = link->ap;
9dadd45b 1821 bool online;
ad616ffb
TH		 1822 int rc;
1823
1824 DPRINTK("ENTER\n");
1825
4447d351 1826 ahci_stop_engine(ap);
ad616ffb 1827
cc0680a5 1828 rc = sata_link_hardreset(link, sata_ehc_deb_timing(&link->eh_context),
9dadd45b 1829 deadline, &online, NULL);
ad616ffb 1830
4447d351 1831 ahci_start_engine(ap);
ad616ffb
TH		 1832
1833 DPRINTK("EXIT, rc=%d, class=%u\n", rc, *class);
1834
1835 /* vt8251 doesn't clear BSY on signature FIS reception,
1836 * request follow-up softreset.
1837 */
9dadd45b 1838 return online ? -EAGAIN : rc;
ad616ffb
TH		 1839}
1840
edc93052
TH		 1841static int ahci_p5wdh_hardreset(struct ata_link *link, unsigned int *class,
1842 unsigned long deadline)
1843{
1844 struct ata_port *ap = link->ap;
1845 struct ahci_port_priv *pp = ap->private_data;
1846 u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;
1847 struct ata_taskfile tf;
9dadd45b 1848 bool online;
edc93052
TH		 1849 int rc;
1850
1851 ahci_stop_engine(ap);
1852
1853 /* clear D2H reception area to properly wait for D2H FIS */
1854 ata_tf_init(link->device, &tf);
1855 tf.command = 0x80;
1856 ata_tf_to_fis(&tf, 0, 0, d2h_fis);
1857
1858 rc = sata_link_hardreset(link, sata_ehc_deb_timing(&link->eh_context),
9dadd45b 1859 deadline, &online, NULL);
edc93052
TH		 1860
1861 ahci_start_engine(ap);
1862
edc93052
TH		 1863 /* The pseudo configuration device on SIMG4726 attached to
1864 * ASUS P5W-DH Deluxe doesn't send signature FIS after
1865 * hardreset if no device is attached to the first downstream
1866 * port && the pseudo device locks up on SRST w/ PMP==0. To
1867 * work around this, wait for !BSY only briefly. If BSY isn't
1868 * cleared, perform CLO and proceed to IDENTIFY (achieved by
1869 * ATA_LFLAG_NO_SRST and ATA_LFLAG_ASSUME_ATA).
1870 *
1871 * Wait for two seconds. Devices attached to downstream port
1872 * which can't process the following IDENTIFY after this will
1873 * have to be reset again. For most cases, this should
1874 * suffice while making probing snappish enough.
1875 */
9dadd45b
TH		 1876 if (online) {
1877 rc = ata_wait_after_reset(link, jiffies + 2 * HZ,
1878 ahci_check_ready);
1879 if (rc)
1880 ahci_kick_engine(ap, 0);
1881 }
9dadd45b 1882 return rc;
edc93052
TH		 1883}
1884
cc0680a5 1885static void ahci_postreset(struct ata_link *link, unsigned int *class)
4bd00f6a 1886{
cc0680a5 1887 struct ata_port *ap = link->ap;
4447d351 1888 void __iomem *port_mmio = ahci_port_base(ap);
4bd00f6a
TH		 1889 u32 new_tmp, tmp;
1890
203c75b8 1891 ata_std_postreset(link, class);
02eaa666
JG		 1892
1893 /* Make sure port's ATAPI bit is set appropriately */
1894 new_tmp = tmp = readl(port_mmio + PORT_CMD);
4bd00f6a 1895 if (*class == ATA_DEV_ATAPI)
02eaa666
JG		 1896 new_tmp |= PORT_CMD_ATAPI;
1897 else
1898 new_tmp &= ~PORT_CMD_ATAPI;
1899 if (new_tmp != tmp) {
1900 writel(new_tmp, port_mmio + PORT_CMD);
1901 readl(port_mmio + PORT_CMD); /* flush */
1902 }
1da177e4
LT		 1903}
1904
12fad3f9 1905static unsigned int ahci_fill_sg(struct ata_queued_cmd *qc, void *cmd_tbl)
1da177e4 1906{
cedc9a47 1907 struct scatterlist *sg;
ff2aeb1e
TH		 1908 struct ahci_sg *ahci_sg = cmd_tbl + AHCI_CMD_TBL_HDR_SZ;
1909 unsigned int si;
1da177e4
LT		 1910
1911 VPRINTK("ENTER\n");
1912
1913 /*
1914 * Next, the S/G list.
1915 */
ff2aeb1e 1916 for_each_sg(qc->sg, sg, qc->n_elem, si) {
cedc9a47
JG		 1917 dma_addr_t addr = sg_dma_address(sg);
1918 u32 sg_len = sg_dma_len(sg);
1919
ff2aeb1e
TH		 1920 ahci_sg[si].addr = cpu_to_le32(addr & 0xffffffff);
1921 ahci_sg[si].addr_hi = cpu_to_le32((addr >> 16) >> 16);
1922 ahci_sg[si].flags_size = cpu_to_le32(sg_len - 1);
1da177e4 1923 }
828d09de 1924
ff2aeb1e 1925 return si;
1da177e4
LT		 1926}
1927
1928static void ahci_qc_prep(struct ata_queued_cmd *qc)
1929{
a0ea7328
JG		 1930 struct ata_port *ap = qc->ap;
1931 struct ahci_port_priv *pp = ap->private_data;
405e66b3 1932 int is_atapi = ata_is_atapi(qc->tf.protocol);
12fad3f9 1933 void *cmd_tbl;
1da177e4
LT		 1934 u32 opts;
1935 const u32 cmd_fis_len = 5; /* five dwords */
828d09de 1936 unsigned int n_elem;
1da177e4 1937
1da177e4
LT		 1938 /*
1939 * Fill in command table information. First, the header,
1940 * a SATA Register - Host to Device command FIS.
1941 */
12fad3f9
TH		 1942 cmd_tbl = pp->cmd_tbl + qc->tag * AHCI_CMD_TBL_SZ;
1943
7d50b60b 1944 ata_tf_to_fis(&qc->tf, qc->dev->link->pmp, 1, cmd_tbl);
cc9278ed 1945 if (is_atapi) {
12fad3f9
TH		 1946 memset(cmd_tbl + AHCI_CMD_TBL_CDB, 0, 32);
1947 memcpy(cmd_tbl + AHCI_CMD_TBL_CDB, qc->cdb, qc->dev->cdb_len);
a0ea7328 1948 }
1da177e4 1949
cc9278ed
TH		 1950 n_elem = 0;
1951 if (qc->flags & ATA_QCFLAG_DMAMAP)
12fad3f9 1952 n_elem = ahci_fill_sg(qc, cmd_tbl);
1da177e4 1953
cc9278ed
TH		 1954 /*
1955 * Fill in command slot information.
1956 */
7d50b60b 1957 opts = cmd_fis_len | n_elem << 16 | (qc->dev->link->pmp << 12);
cc9278ed
TH		 1958 if (qc->tf.flags & ATA_TFLAG_WRITE)
1959 opts |= AHCI_CMD_WRITE;
1960 if (is_atapi)
4b10e559 1961 opts |= AHCI_CMD_ATAPI | AHCI_CMD_PREFETCH;
828d09de 1962
12fad3f9 1963 ahci_fill_cmd_slot(pp, qc->tag, opts);
1da177e4
LT		 1964}
1965
78cd52d0 1966static void ahci_error_intr(struct ata_port *ap, u32 irq_stat)
1da177e4 1967{
417a1a6d 1968 struct ahci_host_priv *hpriv = ap->host->private_data;
78cd52d0 1969 struct ahci_port_priv *pp = ap->private_data;
7d50b60b
TH		 1970 struct ata_eh_info *host_ehi = &ap->link.eh_info;
1971 struct ata_link *link = NULL;
1972 struct ata_queued_cmd *active_qc;
1973 struct ata_eh_info *active_ehi;
78cd52d0 1974 u32 serror;
1da177e4 1975
7d50b60b 1976 /* determine active link */
1eca4365 1977 ata_for_each_link(link, ap, EDGE)
7d50b60b
TH		 1978 if (ata_link_active(link))
1979 break;
1980 if (!link)
1981 link = &ap->link;
1982
1983 active_qc = ata_qc_from_tag(ap, link->active_tag);
1984 active_ehi = &link->eh_info;
1985
1986 /* record irq stat */
1987 ata_ehi_clear_desc(host_ehi);
1988 ata_ehi_push_desc(host_ehi, "irq_stat 0x%08x", irq_stat);
1da177e4 1989
78cd52d0 1990 /* AHCI needs SError cleared; otherwise, it might lock up */
82ef04fb
TH		 1991 ahci_scr_read(&ap->link, SCR_ERROR, &serror);
1992 ahci_scr_write(&ap->link, SCR_ERROR, serror);
7d50b60b 1993 host_ehi->serror |= serror;
78cd52d0 1994
41669553 1995 /* some controllers set IRQ_IF_ERR on device errors, ignore it */
417a1a6d 1996 if (hpriv->flags & AHCI_HFLAG_IGN_IRQ_IF_ERR)
41669553
TH		 1997 irq_stat &= ~PORT_IRQ_IF_ERR;
1998
55a61604 1999 if (irq_stat & PORT_IRQ_TF_ERR) {
7d50b60b
TH		 2000 /* If qc is active, charge it; otherwise, the active
2001 * link. There's no active qc on NCQ errors. It will
2002 * be determined by EH by reading log page 10h.
2003 */
2004 if (active_qc)
2005 active_qc->err_mask |= AC_ERR_DEV;
2006 else
2007 active_ehi->err_mask |= AC_ERR_DEV;
2008
417a1a6d 2009 if (hpriv->flags & AHCI_HFLAG_IGN_SERR_INTERNAL)
7d50b60b
TH		 2010 host_ehi->serror &= ~SERR_INTERNAL;
2011 }
2012
2013 if (irq_stat & PORT_IRQ_UNK_FIS) {
2014 u32 *unk = (u32 *)(pp->rx_fis + RX_FIS_UNK);
2015
2016 active_ehi->err_mask |= AC_ERR_HSM;
cf480626 2017 active_ehi->action |= ATA_EH_RESET;
7d50b60b
TH		 2018 ata_ehi_push_desc(active_ehi,
2019 "unknown FIS %08x %08x %08x %08x" ,
2020 unk[0], unk[1], unk[2], unk[3]);
2021 }
2022
071f44b1 2023 if (sata_pmp_attached(ap) && (irq_stat & PORT_IRQ_BAD_PMP)) {
7d50b60b 2024 active_ehi->err_mask |= AC_ERR_HSM;
cf480626 2025 active_ehi->action |= ATA_EH_RESET;
7d50b60b 2026 ata_ehi_push_desc(active_ehi, "incorrect PMP");
55a61604 2027 }
78cd52d0
TH		 2028
2029 if (irq_stat & (PORT_IRQ_HBUS_ERR | PORT_IRQ_HBUS_DATA_ERR)) {
7d50b60b 2030 host_ehi->err_mask |= AC_ERR_HOST_BUS;
cf480626 2031 host_ehi->action |= ATA_EH_RESET;
7d50b60b 2032 ata_ehi_push_desc(host_ehi, "host bus error");
1da177e4
LT		 2033 }
2034
78cd52d0 2035 if (irq_stat & PORT_IRQ_IF_ERR) {
7d50b60b 2036 host_ehi->err_mask |= AC_ERR_ATA_BUS;
cf480626 2037 host_ehi->action |= ATA_EH_RESET;
7d50b60b 2038 ata_ehi_push_desc(host_ehi, "interface fatal error");
78cd52d0 2039 }
1da177e4 2040
78cd52d0 2041 if (irq_stat & (PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY)) {
7d50b60b
TH		 2042 ata_ehi_hotplugged(host_ehi);
2043 ata_ehi_push_desc(host_ehi, "%s",
2044 irq_stat & PORT_IRQ_CONNECT ?
78cd52d0
TH		 2045 "connection status changed" : "PHY RDY changed");
2046 }
2047
78cd52d0 2048 /* okay, let's hand over to EH */
a72ec4ce 2049
78cd52d0
TH		 2050 if (irq_stat & PORT_IRQ_FREEZE)
2051 ata_port_freeze(ap);
2052 else
2053 ata_port_abort(ap);
1da177e4
LT		 2054}
2055
df69c9c5 2056static void ahci_port_intr(struct ata_port *ap)
1da177e4 2057{
350756f6 2058 void __iomem *port_mmio = ahci_port_base(ap);
9af5c9c9 2059 struct ata_eh_info *ehi = &ap->link.eh_info;
0291f95f 2060 struct ahci_port_priv *pp = ap->private_data;
5f226c6b 2061 struct ahci_host_priv *hpriv = ap->host->private_data;
b06ce3e5 2062 int resetting = !!(ap->pflags & ATA_PFLAG_RESETTING);
12fad3f9 2063 u32 status, qc_active;
459ad688 2064 int rc;
1da177e4
LT		 2065
2066 status = readl(port_mmio + PORT_IRQ_STAT);
2067 writel(status, port_mmio + PORT_IRQ_STAT);
2068
b06ce3e5
TH		 2069 /* ignore BAD_PMP while resetting */
2070 if (unlikely(resetting))
2071 status &= ~PORT_IRQ_BAD_PMP;
2072
31556594
KCA		 2073 /* If we are getting PhyRdy, this is
2074 * just a power state change, we should
2075 * clear out this, plus the PhyRdy/Comm
2076 * Wake bits from Serror
2077 */
2078 if ((hpriv->flags & AHCI_HFLAG_NO_HOTPLUG) &&
2079 (status & PORT_IRQ_PHYRDY)) {
2080 status &= ~PORT_IRQ_PHYRDY;
82ef04fb 2081 ahci_scr_write(&ap->link, SCR_ERROR, ((1 << 16) | (1 << 18)));
31556594
KCA		 2082 }
2083
78cd52d0
TH		 2084 if (unlikely(status & PORT_IRQ_ERROR)) {
2085 ahci_error_intr(ap, status);
2086 return;
1da177e4
LT		 2087 }
2088
2f294968 2089 if (status & PORT_IRQ_SDB_FIS) {
5f226c6b
TH		 2090 /* If SNotification is available, leave notification
2091 * handling to sata_async_notification(). If not,
2092 * emulate it by snooping SDB FIS RX area.
2093 *
2094 * Snooping FIS RX area is probably cheaper than
2095 * poking SNotification but some constrollers which
2096 * implement SNotification, ICH9 for example, don't
2097 * store AN SDB FIS into receive area.
2f294968 2098 */
5f226c6b 2099 if (hpriv->cap & HOST_CAP_SNTF)
7d77b247 2100 sata_async_notification(ap);
5f226c6b
TH		 2101 else {
2102 /* If the 'N' bit in word 0 of the FIS is set,
2103 * we just received asynchronous notification.
2104 * Tell libata about it.
2105 */
2106 const __le32 *f = pp->rx_fis + RX_FIS_SDB;
2107 u32 f0 = le32_to_cpu(f[0]);
2108
2109 if (f0 & (1 << 15))
2110 sata_async_notification(ap);
2111 }
2f294968
KCA		 2112 }
2113
7d50b60b
TH		 2114 /* pp->active_link is valid iff any command is in flight */
2115 if (ap->qc_active && pp->active_link->sactive)
12fad3f9
TH		 2116 qc_active = readl(port_mmio + PORT_SCR_ACT);
2117 else
2118 qc_active = readl(port_mmio + PORT_CMD_ISSUE);
2119
79f97dad 2120 rc = ata_qc_complete_multiple(ap, qc_active);
b06ce3e5 2121
459ad688
TH		 2122 /* while resetting, invalid completions are expected */
2123 if (unlikely(rc < 0 && !resetting)) {
12fad3f9 2124 ehi->err_mask |= AC_ERR_HSM;
cf480626 2125 ehi->action |= ATA_EH_RESET;
12fad3f9 2126 ata_port_freeze(ap);
1da177e4 2127 }
1da177e4
LT		 2128}
2129
7d12e780 2130static irqreturn_t ahci_interrupt(int irq, void *dev_instance)
1da177e4 2131{
cca3974e 2132 struct ata_host *host = dev_instance;
1da177e4
LT		 2133 struct ahci_host_priv *hpriv;
2134 unsigned int i, handled = 0;
ea6ba10b 2135 void __iomem *mmio;
d28f87aa 2136 u32 irq_stat, irq_masked;
1da177e4
LT		 2137
2138 VPRINTK("ENTER\n");
2139
cca3974e 2140 hpriv = host->private_data;
0d5ff566 2141 mmio = host->iomap[AHCI_PCI_BAR];
1da177e4
LT		 2142
2143 /* sigh. 0xffffffff is a valid return from h/w */
2144 irq_stat = readl(mmio + HOST_IRQ_STAT);
1da177e4
LT		 2145 if (!irq_stat)
2146 return IRQ_NONE;
2147
d28f87aa
TH		 2148 irq_masked = irq_stat & hpriv->port_map;
2149
2dcb407e 2150 spin_lock(&host->lock);
1da177e4 2151
2dcb407e 2152 for (i = 0; i < host->n_ports; i++) {
1da177e4 2153 struct ata_port *ap;
1da177e4 2154
d28f87aa 2155 if (!(irq_masked & (1 << i)))
67846b30
JG		 2156 continue;
2157
cca3974e 2158 ap = host->ports[i];
67846b30 2159 if (ap) {
df69c9c5 2160 ahci_port_intr(ap);
67846b30
JG		 2161 VPRINTK("port %u\n", i);
2162 } else {
2163 VPRINTK("port %u (no irq)\n", i);
6971ed1f 2164 if (ata_ratelimit())
cca3974e 2165 dev_printk(KERN_WARNING, host->dev,
a9524a76 2166 "interrupt on disabled port %u\n", i);
1da177e4 2167 }
67846b30 2168
1da177e4
LT		 2169 handled = 1;
2170 }
2171
d28f87aa
TH		 2172 /* HOST_IRQ_STAT behaves as level triggered latch meaning that
2173 * it should be cleared after all the port events are cleared;
2174 * otherwise, it will raise a spurious interrupt after each
2175 * valid one. Please read section 10.6.2 of ahci 1.1 for more
2176 * information.
2177 *
2178 * Also, use the unmasked value to clear interrupt as spurious
2179 * pending event on a dummy port might cause screaming IRQ.
2180 */
ea0c62f7
TH		 2181 writel(irq_stat, mmio + HOST_IRQ_STAT);
2182
cca3974e 2183 spin_unlock(&host->lock);
1da177e4
LT		 2184
2185 VPRINTK("EXIT\n");
2186
2187 return IRQ_RETVAL(handled);
2188}
2189
9a3d9eb0 2190static unsigned int ahci_qc_issue(struct ata_queued_cmd *qc)
1da177e4
LT		 2191{
2192 struct ata_port *ap = qc->ap;
4447d351 2193 void __iomem *port_mmio = ahci_port_base(ap);
7d50b60b
TH		 2194 struct ahci_port_priv *pp = ap->private_data;
2195
2196 /* Keep track of the currently active link. It will be used
2197 * in completion path to determine whether NCQ phase is in
2198 * progress.
2199 */
2200 pp->active_link = qc->dev->link;
1da177e4 2201
12fad3f9
TH		 2202 if (qc->tf.protocol == ATA_PROT_NCQ)
2203 writel(1 << qc->tag, port_mmio + PORT_SCR_ACT);
2204 writel(1 << qc->tag, port_mmio + PORT_CMD_ISSUE);
1da177e4 2205
18f7ba4c
KCA		 2206 ahci_sw_activity(qc->dev->link);
2207
1da177e4
LT		 2208 return 0;
2209}
2210
4c9bf4e7
TH		 2211static bool ahci_qc_fill_rtf(struct ata_queued_cmd *qc)
2212{
2213 struct ahci_port_priv *pp = qc->ap->private_data;
2214 u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;
2215
2216 ata_tf_from_fis(d2h_fis, &qc->result_tf);
2217 return true;
2218}
2219
78cd52d0
TH		 2220static void ahci_freeze(struct ata_port *ap)
2221{
4447d351 2222 void __iomem *port_mmio = ahci_port_base(ap);
78cd52d0
TH		 2223
2224 /* turn IRQ off */
2225 writel(0, port_mmio + PORT_IRQ_MASK);
2226}
2227
2228static void ahci_thaw(struct ata_port *ap)
2229{
0d5ff566 2230 void __iomem *mmio = ap->host->iomap[AHCI_PCI_BAR];
4447d351 2231 void __iomem *port_mmio = ahci_port_base(ap);
78cd52d0 2232 u32 tmp;
a7384925 2233 struct ahci_port_priv *pp = ap->private_data;
78cd52d0
TH		 2234
2235 /* clear IRQ */
2236 tmp = readl(port_mmio + PORT_IRQ_STAT);
2237 writel(tmp, port_mmio + PORT_IRQ_STAT);
a718728f 2238 writel(1 << ap->port_no, mmio + HOST_IRQ_STAT);
78cd52d0 2239
1c954a4d
TH		 2240 /* turn IRQ back on */
2241 writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
78cd52d0
TH		 2242}
2243
2244static void ahci_error_handler(struct ata_port *ap)
2245{
b51e9e5d 2246 if (!(ap->pflags & ATA_PFLAG_FROZEN)) {
78cd52d0 2247 /* restart engine */
4447d351
TH		 2248 ahci_stop_engine(ap);
2249 ahci_start_engine(ap);
78cd52d0
TH		 2250 }
2251
a1efdaba 2252 sata_pmp_error_handler(ap);
edc93052
TH		 2253}
2254
78cd52d0
TH		 2255static void ahci_post_internal_cmd(struct ata_queued_cmd *qc)
2256{
2257 struct ata_port *ap = qc->ap;
2258
d2e75dff
TH		 2259 /* make DMA engine forget about the failed command */
2260 if (qc->flags & ATA_QCFLAG_FAILED)
2261 ahci_kick_engine(ap, 1);
78cd52d0
TH		 2262}
2263
7d50b60b
TH		 2264static void ahci_pmp_attach(struct ata_port *ap)
2265{
2266 void __iomem *port_mmio = ahci_port_base(ap);
1c954a4d 2267 struct ahci_port_priv *pp = ap->private_data;
7d50b60b
TH		 2268 u32 cmd;
2269
2270 cmd = readl(port_mmio + PORT_CMD);
2271 cmd |= PORT_CMD_PMP;
2272 writel(cmd, port_mmio + PORT_CMD);
1c954a4d
TH		 2273
2274 pp->intr_mask |= PORT_IRQ_BAD_PMP;
2275 writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
7d50b60b
TH		 2276}
2277
2278static void ahci_pmp_detach(struct ata_port *ap)
2279{
2280 void __iomem *port_mmio = ahci_port_base(ap);
1c954a4d 2281 struct ahci_port_priv *pp = ap->private_data;
7d50b60b
TH		 2282 u32 cmd;
2283
2284 cmd = readl(port_mmio + PORT_CMD);
2285 cmd &= ~PORT_CMD_PMP;
2286 writel(cmd, port_mmio + PORT_CMD);
1c954a4d
TH		 2287
2288 pp->intr_mask &= ~PORT_IRQ_BAD_PMP;
2289 writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
7d50b60b
TH		 2290}
2291
028a2596
AD		 2292static int ahci_port_resume(struct ata_port *ap)
2293{
2294 ahci_power_up(ap);
2295 ahci_start_port(ap);
2296
071f44b1 2297 if (sata_pmp_attached(ap))
7d50b60b
TH		 2298 ahci_pmp_attach(ap);
2299 else
2300 ahci_pmp_detach(ap);
2301
028a2596
AD		 2302 return 0;
2303}
2304
438ac6d5 2305#ifdef CONFIG_PM
c1332875
TH		 2306static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg)
2307{
c1332875
TH		 2308 const char *emsg = NULL;
2309 int rc;
2310
4447d351 2311 rc = ahci_deinit_port(ap, &emsg);
8e16f941 2312 if (rc == 0)
4447d351 2313 ahci_power_down(ap);
8e16f941 2314 else {
c1332875 2315 ata_port_printk(ap, KERN_ERR, "%s (%d)\n", emsg, rc);
df69c9c5 2316 ahci_start_port(ap);
c1332875
TH		 2317 }
2318
2319 return rc;
2320}
2321
c1332875
TH		 2322static int ahci_pci_device_suspend(struct pci_dev *pdev, pm_message_t mesg)
2323{
cca3974e 2324 struct ata_host *host = dev_get_drvdata(&pdev->dev);
9b10ae86 2325 struct ahci_host_priv *hpriv = host->private_data;
0d5ff566 2326 void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
c1332875
TH		 2327 u32 ctl;
2328
9b10ae86
TH		 2329 if (mesg.event & PM_EVENT_SUSPEND &&
2330 hpriv->flags & AHCI_HFLAG_NO_SUSPEND) {
2331 dev_printk(KERN_ERR, &pdev->dev,
2332 "BIOS update required for suspend/resume\n");
2333 return -EIO;
2334 }
2335
3a2d5b70 2336 if (mesg.event & PM_EVENT_SLEEP) {
c1332875
TH		 2337 /* AHCI spec rev1.1 section 8.3.3:
2338 * Software must disable interrupts prior to requesting a
2339 * transition of the HBA to D3 state.
2340 */
2341 ctl = readl(mmio + HOST_CTL);
2342 ctl &= ~HOST_IRQ_EN;
2343 writel(ctl, mmio + HOST_CTL);
2344 readl(mmio + HOST_CTL); /* flush */
2345 }
2346
2347 return ata_pci_device_suspend(pdev, mesg);
2348}
2349
2350static int ahci_pci_device_resume(struct pci_dev *pdev)
2351{
cca3974e 2352 struct ata_host *host = dev_get_drvdata(&pdev->dev);
c1332875
TH		 2353 int rc;
2354
553c4aa6
TH		 2355 rc = ata_pci_device_do_resume(pdev);
2356 if (rc)
2357 return rc;
c1332875
TH		 2358
2359 if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
4447d351 2360 rc = ahci_reset_controller(host);
c1332875
TH		 2361 if (rc)
2362 return rc;
2363
4447d351 2364 ahci_init_controller(host);
c1332875
TH		 2365 }
2366
cca3974e 2367 ata_host_resume(host);
c1332875
TH		 2368
2369 return 0;
2370}
438ac6d5 2371#endif
c1332875 2372
254950cd
TH		 2373static int ahci_port_start(struct ata_port *ap)
2374{
cca3974e 2375 struct device *dev = ap->host->dev;
254950cd 2376 struct ahci_port_priv *pp;
254950cd
TH		 2377 void *mem;
2378 dma_addr_t mem_dma;
254950cd 2379
24dc5f33 2380 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
254950cd
TH		 2381 if (!pp)
2382 return -ENOMEM;
254950cd 2383
24dc5f33
TH		 2384 mem = dmam_alloc_coherent(dev, AHCI_PORT_PRIV_DMA_SZ, &mem_dma,
2385 GFP_KERNEL);
2386 if (!mem)
254950cd 2387 return -ENOMEM;
254950cd
TH		 2388 memset(mem, 0, AHCI_PORT_PRIV_DMA_SZ);
2389
2390 /*
2391 * First item in chunk of DMA memory: 32-slot command table,
2392 * 32 bytes each in size
2393 */
2394 pp->cmd_slot = mem;
2395 pp->cmd_slot_dma = mem_dma;
2396
2397 mem += AHCI_CMD_SLOT_SZ;
2398 mem_dma += AHCI_CMD_SLOT_SZ;
2399
2400 /*
2401 * Second item: Received-FIS area
2402 */
2403 pp->rx_fis = mem;
2404 pp->rx_fis_dma = mem_dma;
2405
2406 mem += AHCI_RX_FIS_SZ;
2407 mem_dma += AHCI_RX_FIS_SZ;
2408
2409 /*
2410 * Third item: data area for storing a single command
2411 * and its scatter-gather table
2412 */
2413 pp->cmd_tbl = mem;
2414 pp->cmd_tbl_dma = mem_dma;
2415
a7384925 2416 /*
2dcb407e
JG		 2417 * Save off initial list of interrupts to be enabled.
2418 * This could be changed later
2419 */
a7384925
KCA		 2420 pp->intr_mask = DEF_PORT_IRQ;
2421
254950cd
TH		 2422 ap->private_data = pp;
2423
df69c9c5
JG		 2424 /* engage engines, captain */
2425 return ahci_port_resume(ap);
254950cd
TH		 2426}
2427
2428static void ahci_port_stop(struct ata_port *ap)
2429{
0be0aa98
TH		 2430 const char *emsg = NULL;
2431 int rc;
254950cd 2432
0be0aa98 2433 /* de-initialize port */
4447d351 2434 rc = ahci_deinit_port(ap, &emsg);
0be0aa98
TH		 2435 if (rc)
2436 ata_port_printk(ap, KERN_WARNING, "%s (%d)\n", emsg, rc);
254950cd
TH		 2437}
2438
4447d351 2439static int ahci_configure_dma_masks(struct pci_dev *pdev, int using_dac)
1da177e4 2440{
1da177e4 2441 int rc;
1da177e4 2442
1da177e4 2443 if (using_dac &&
6a35528a
YH		 2444 !pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
2445 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1da177e4 2446 if (rc) {
284901a9 2447 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1da177e4 2448 if (rc) {
a9524a76
JG		 2449 dev_printk(KERN_ERR, &pdev->dev,
2450 "64-bit DMA enable failed\n");
1da177e4
LT		 2451 return rc;
2452 }
2453 }
1da177e4 2454 } else {
284901a9 2455 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1da177e4 2456 if (rc) {
a9524a76
JG		 2457 dev_printk(KERN_ERR, &pdev->dev,
2458 "32-bit DMA enable failed\n");
1da177e4
LT		 2459 return rc;
2460 }
284901a9 2461 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1da177e4 2462 if (rc) {
a9524a76
JG		 2463 dev_printk(KERN_ERR, &pdev->dev,
2464 "32-bit consistent DMA enable failed\n");
1da177e4
LT		 2465 return rc;
2466 }
2467 }
1da177e4
LT		 2468 return 0;
2469}
2470
4447d351 2471static void ahci_print_info(struct ata_host *host)
1da177e4 2472{
4447d351
TH		 2473 struct ahci_host_priv *hpriv = host->private_data;
2474 struct pci_dev *pdev = to_pci_dev(host->dev);
2475 void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
1da177e4
LT		 2476 u32 vers, cap, impl, speed;
2477 const char *speed_s;
2478 u16 cc;
2479 const char *scc_s;
2480
2481 vers = readl(mmio + HOST_VERSION);
2482 cap = hpriv->cap;
2483 impl = hpriv->port_map;
2484
2485 speed = (cap >> 20) & 0xf;
2486 if (speed == 1)
2487 speed_s = "1.5";
2488 else if (speed == 2)
2489 speed_s = "3";
8522ee25
SH		 2490 else if (speed == 3)
2491 speed_s = "6";
1da177e4
LT		 2492 else
2493 speed_s = "?";
2494
2495 pci_read_config_word(pdev, 0x0a, &cc);
c9f89475 2496 if (cc == PCI_CLASS_STORAGE_IDE)
1da177e4 2497 scc_s = "IDE";
c9f89475 2498 else if (cc == PCI_CLASS_STORAGE_SATA)
1da177e4 2499 scc_s = "SATA";
c9f89475 2500 else if (cc == PCI_CLASS_STORAGE_RAID)
1da177e4
LT		 2501 scc_s = "RAID";
2502 else
2503 scc_s = "unknown";
2504
a9524a76
JG		 2505 dev_printk(KERN_INFO, &pdev->dev,
2506 "AHCI %02x%02x.%02x%02x "
1da177e4 2507 "%u slots %u ports %s Gbps 0x%x impl %s mode\n"
2dcb407e 2508 ,
1da177e4 2509
2dcb407e
JG		 2510 (vers >> 24) & 0xff,
2511 (vers >> 16) & 0xff,
2512 (vers >> 8) & 0xff,
2513 vers & 0xff,
1da177e4
LT		 2514
2515 ((cap >> 8) & 0x1f) + 1,
2516 (cap & 0x1f) + 1,
2517 speed_s,
2518 impl,
2519 scc_s);
2520
a9524a76
JG		 2521 dev_printk(KERN_INFO, &pdev->dev,
2522 "flags: "
203ef6c4 2523 "%s%s%s%s%s%s%s"
18f7ba4c
KCA		 2524 "%s%s%s%s%s%s%s"
2525 "%s\n"
2dcb407e 2526 ,
1da177e4
LT		 2527
2528 cap & (1 << 31) ? "64bit " : "",
2529 cap & (1 << 30) ? "ncq " : "",
203ef6c4 2530 cap & (1 << 29) ? "sntf " : "",
1da177e4
LT		 2531 cap & (1 << 28) ? "ilck " : "",
2532 cap & (1 << 27) ? "stag " : "",
2533 cap & (1 << 26) ? "pm " : "",
2534 cap & (1 << 25) ? "led " : "",
2535
2536 cap & (1 << 24) ? "clo " : "",
2537 cap & (1 << 19) ? "nz " : "",
2538 cap & (1 << 18) ? "only " : "",
2539 cap & (1 << 17) ? "pmp " : "",
2540 cap & (1 << 15) ? "pio " : "",
2541 cap & (1 << 14) ? "slum " : "",
18f7ba4c
KCA		 2542 cap & (1 << 13) ? "part " : "",
2543 cap & (1 << 6) ? "ems ": ""
1da177e4
LT		 2544 );
2545}
2546
edc93052
TH		 2547/* On ASUS P5W DH Deluxe, the second port of PCI device 00:1f.2 is
2548 * hardwired to on-board SIMG 4726. The chipset is ICH8 and doesn't
2549 * support PMP and the 4726 either directly exports the device
2550 * attached to the first downstream port or acts as a hardware storage
2551 * controller and emulate a single ATA device (can be RAID 0/1 or some
2552 * other configuration).
2553 *
2554 * When there's no device attached to the first downstream port of the
2555 * 4726, "Config Disk" appears, which is a pseudo ATA device to
2556 * configure the 4726. However, ATA emulation of the device is very
2557 * lame. It doesn't send signature D2H Reg FIS after the initial
2558 * hardreset, pukes on SRST w/ PMP==0 and has bunch of other issues.
2559 *
2560 * The following function works around the problem by always using
2561 * hardreset on the port and not depending on receiving signature FIS
2562 * afterward. If signature FIS isn't received soon, ATA class is
2563 * assumed without follow-up softreset.
2564 */
2565static void ahci_p5wdh_workaround(struct ata_host *host)
2566{
2567 static struct dmi_system_id sysids[] = {
2568 {
2569 .ident = "P5W DH Deluxe",
2570 .matches = {
2571 DMI_MATCH(DMI_SYS_VENDOR,
2572 "ASUSTEK COMPUTER INC"),
2573 DMI_MATCH(DMI_PRODUCT_NAME, "P5W DH Deluxe"),
2574 },
2575 },
2576 { }
2577 };
2578 struct pci_dev *pdev = to_pci_dev(host->dev);
2579
2580 if (pdev->bus->number == 0 && pdev->devfn == PCI_DEVFN(0x1f, 2) &&
2581 dmi_check_system(sysids)) {
2582 struct ata_port *ap = host->ports[1];
2583
2584 dev_printk(KERN_INFO, &pdev->dev, "enabling ASUS P5W DH "
2585 "Deluxe on-board SIMG4726 workaround\n");
2586
2587 ap->ops = &ahci_p5wdh_ops;
2588 ap->link.flags |= ATA_LFLAG_NO_SRST | ATA_LFLAG_ASSUME_ATA;
2589 }
2590}
2591
58a09b38
SH		 2592/*
2593 * SB600 ahci controller on ASUS M2A-VM can't do 64bit DMA with older
2594 * BIOS. The oldest version known to be broken is 0901 and working is
2595 * 1501 which was released on 2007-10-26. Force 32bit DMA on anything
2596 * older than 1501. Please read bko#9412 for more info.
2597 */
2598static bool ahci_asus_m2a_vm_32bit_only(struct pci_dev *pdev)
2599{
2600 static const struct dmi_system_id sysids[] = {
2601 {
2602 .ident = "ASUS M2A-VM",
2603 .matches = {
2604 DMI_MATCH(DMI_BOARD_VENDOR,
2605 "ASUSTeK Computer INC."),
2606 DMI_MATCH(DMI_BOARD_NAME, "M2A-VM"),
2607 },
2608 },
2609 { }
2610 };
2611 const char *cutoff_mmdd = "10/26";
2612 const char *date;
2613 int year;
2614
2615 if (pdev->bus->number != 0 || pdev->devfn != PCI_DEVFN(0x12, 0) ||
2616 !dmi_check_system(sysids))
2617 return false;
2618
2619 /*
2620 * Argh.... both version and date are free form strings.
2621 * Let's hope they're using the same date format across
2622 * different versions.
2623 */
2624 date = dmi_get_system_info(DMI_BIOS_DATE);
2625 year = dmi_get_year(DMI_BIOS_DATE);
2626 if (date && strlen(date) >= 10 && date[2] == '/' && date[5] == '/' &&
2627 (year > 2007 ||
2628 (year == 2007 && strncmp(date, cutoff_mmdd, 5) >= 0)))
2629 return false;
2630
2631 dev_printk(KERN_WARNING, &pdev->dev, "ASUS M2A-VM: BIOS too old, "
2632 "forcing 32bit DMA, update BIOS\n");
2633
2634 return true;
2635}
2636
1fd68434
RW		 2637static bool ahci_broken_system_poweroff(struct pci_dev *pdev)
2638{
2639 static const struct dmi_system_id broken_systems[] = {
2640 {
2641 .ident = "HP Compaq nx6310",
2642 .matches = {
2643 DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"),
2644 DMI_MATCH(DMI_PRODUCT_NAME, "HP Compaq nx6310"),
2645 },
2646 /* PCI slot number of the controller */
2647 .driver_data = (void *)0x1FUL,
2648 },
d2f9c061
MR		 2649 {
2650 .ident = "HP Compaq 6720s",
2651 .matches = {
2652 DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"),
2653 DMI_MATCH(DMI_PRODUCT_NAME, "HP Compaq 6720s"),
2654 },
2655 /* PCI slot number of the controller */
2656 .driver_data = (void *)0x1FUL,
2657 },
1fd68434
RW		 2658
2659 { } /* terminate list */
2660 };
2661 const struct dmi_system_id *dmi = dmi_first_match(broken_systems);
2662
2663 if (dmi) {
2664 unsigned long slot = (unsigned long)dmi->driver_data;
2665 /* apply the quirk only to on-board controllers */
2666 return slot == PCI_SLOT(pdev->devfn);
2667 }
2668
2669 return false;
2670}
2671
9b10ae86
TH		 2672static bool ahci_broken_suspend(struct pci_dev *pdev)
2673{
2674 static const struct dmi_system_id sysids[] = {
2675 /*
2676 * On HP dv[4-6] and HDX18 with earlier BIOSen, link
2677 * to the harddisk doesn't become online after
2678 * resuming from STR. Warn and fail suspend.
2679 */
2680 {
2681 .ident = "dv4",
2682 .matches = {
2683 DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"),
2684 DMI_MATCH(DMI_PRODUCT_NAME,
2685 "HP Pavilion dv4 Notebook PC"),
2686 },
2687 .driver_data = "F.30", /* cutoff BIOS version */
2688 },
2689 {
2690 .ident = "dv5",
2691 .matches = {
2692 DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"),
2693 DMI_MATCH(DMI_PRODUCT_NAME,
2694 "HP Pavilion dv5 Notebook PC"),
2695 },
2696 .driver_data = "F.16", /* cutoff BIOS version */
2697 },
2698 {
2699 .ident = "dv6",
2700 .matches = {
2701 DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"),
2702 DMI_MATCH(DMI_PRODUCT_NAME,
2703 "HP Pavilion dv6 Notebook PC"),
2704 },
2705 .driver_data = "F.21", /* cutoff BIOS version */
2706 },
2707 {
2708 .ident = "HDX18",
2709 .matches = {
2710 DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"),
2711 DMI_MATCH(DMI_PRODUCT_NAME,
2712 "HP HDX18 Notebook PC"),
2713 },
2714 .driver_data = "F.23", /* cutoff BIOS version */
2715 },
2716 { } /* terminate list */
2717 };
2718 const struct dmi_system_id *dmi = dmi_first_match(sysids);
2719 const char *ver;
2720
2721 if (!dmi || pdev->bus->number || pdev->devfn != PCI_DEVFN(0x1f, 2))
2722 return false;
2723
2724 ver = dmi_get_system_info(DMI_BIOS_VERSION);
2725
2726 return !ver || strcmp(ver, dmi->driver_data) < 0;
2727}
2728
24dc5f33 2729static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
1da177e4
LT		 2730{
2731 static int printed_version;
e297d99e
TH		 2732 unsigned int board_id = ent->driver_data;
2733 struct ata_port_info pi = ahci_port_info[board_id];
4447d351 2734 const struct ata_port_info *ppi[] = { &pi, NULL };
24dc5f33 2735 struct device *dev = &pdev->dev;
1da177e4 2736 struct ahci_host_priv *hpriv;
4447d351 2737 struct ata_host *host;
837f5f8f 2738 int n_ports, i, rc;
1da177e4
LT		 2739
2740 VPRINTK("ENTER\n");
2741
12fad3f9
TH		 2742 WARN_ON(ATA_MAX_QUEUE > AHCI_MAX_CMDS);
2743
1da177e4 2744 if (!printed_version++)
a9524a76 2745 dev_printk(KERN_DEBUG, &pdev->dev, "version " DRV_VERSION "\n");
1da177e4 2746
5b66c829
AC		 2747 /* The AHCI driver can only drive the SATA ports, the PATA driver
2748 can drive them all so if both drivers are selected make sure
2749 AHCI stays out of the way */
2750 if (pdev->vendor == PCI_VENDOR_ID_MARVELL && !marvell_enable)
2751 return -ENODEV;
2752
4447d351 2753 /* acquire resources */
24dc5f33 2754 rc = pcim_enable_device(pdev);
1da177e4
LT		 2755 if (rc)
2756 return rc;
2757
dea55137
TH		 2758 /* AHCI controllers often implement SFF compatible interface.
2759 * Grab all PCI BARs just in case.
2760 */
2761 rc = pcim_iomap_regions_request_all(pdev, 1 << AHCI_PCI_BAR, DRV_NAME);
0d5ff566 2762 if (rc == -EBUSY)
24dc5f33 2763 pcim_pin_device(pdev);
0d5ff566 2764 if (rc)
24dc5f33 2765 return rc;
1da177e4 2766
c4f7792c
TH		 2767 if (pdev->vendor == PCI_VENDOR_ID_INTEL &&
2768 (pdev->device == 0x2652 || pdev->device == 0x2653)) {
2769 u8 map;
2770
2771 /* ICH6s share the same PCI ID for both piix and ahci
2772 * modes. Enabling ahci mode while MAP indicates
2773 * combined mode is a bad idea. Yield to ata_piix.
2774 */
2775 pci_read_config_byte(pdev, ICH_MAP, &map);
2776 if (map & 0x3) {
2777 dev_printk(KERN_INFO, &pdev->dev, "controller is in "
2778 "combined mode, can't enable AHCI mode\n");
2779 return -ENODEV;
2780 }
2781 }
2782
24dc5f33
TH		 2783 hpriv = devm_kzalloc(dev, sizeof(*hpriv), GFP_KERNEL);
2784 if (!hpriv)
2785 return -ENOMEM;
417a1a6d
TH		 2786 hpriv->flags |= (unsigned long)pi.private_data;
2787
e297d99e
TH		 2788 /* MCP65 revision A1 and A2 can't do MSI */
2789 if (board_id == board_ahci_mcp65 &&
2790 (pdev->revision == 0xa1 || pdev->revision == 0xa2))
2791 hpriv->flags |= AHCI_HFLAG_NO_MSI;
2792
e427fe04
SH		 2793 /* SB800 does NOT need the workaround to ignore SERR_INTERNAL */
2794 if (board_id == board_ahci_sb700 && pdev->revision >= 0x40)
2795 hpriv->flags &= ~AHCI_HFLAG_IGN_SERR_INTERNAL;
2796
58a09b38
SH		 2797 /* apply ASUS M2A_VM quirk */
2798 if (ahci_asus_m2a_vm_32bit_only(pdev))
2799 hpriv->flags |= AHCI_HFLAG_32BIT_ONLY;
2800
a5bfc471
TH		 2801 if (!(hpriv->flags & AHCI_HFLAG_NO_MSI))
2802 pci_enable_msi(pdev);
1da177e4 2803
4447d351 2804 /* save initial config */
417a1a6d 2805 ahci_save_initial_config(pdev, hpriv);
1da177e4 2806
4447d351 2807 /* prepare host */
274c1fde 2808 if (hpriv->cap & HOST_CAP_NCQ)
4447d351 2809 pi.flags |= ATA_FLAG_NCQ;
1da177e4 2810
7d50b60b
TH		 2811 if (hpriv->cap & HOST_CAP_PMP)
2812 pi.flags |= ATA_FLAG_PMP;
2813
18f7ba4c
KCA		 2814 if (ahci_em_messages && (hpriv->cap & HOST_CAP_EMS)) {
2815 u8 messages;
2816 void __iomem *mmio = pcim_iomap_table(pdev)[AHCI_PCI_BAR];
2817 u32 em_loc = readl(mmio + HOST_EM_LOC);
2818 u32 em_ctl = readl(mmio + HOST_EM_CTL);
2819
87943acf 2820 messages = (em_ctl & EM_CTRL_MSG_TYPE) >> 16;
18f7ba4c
KCA		 2821
2822 /* we only support LED message type right now */
2823 if ((messages & 0x01) && (ahci_em_messages == 1)) {
2824 /* store em_loc */
2825 hpriv->em_loc = ((em_loc >> 16) * 4);
2826 pi.flags |= ATA_FLAG_EM;
2827 if (!(em_ctl & EM_CTL_ALHD))
2828 pi.flags |= ATA_FLAG_SW_ACTIVITY;
2829 }
2830 }
2831
1fd68434
RW		 2832 if (ahci_broken_system_poweroff(pdev)) {
2833 pi.flags |= ATA_FLAG_NO_POWEROFF_SPINDOWN;
2834 dev_info(&pdev->dev,
2835 "quirky BIOS, skipping spindown on poweroff\n");
2836 }
2837
9b10ae86
TH		 2838 if (ahci_broken_suspend(pdev)) {
2839 hpriv->flags |= AHCI_HFLAG_NO_SUSPEND;
2840 dev_printk(KERN_WARNING, &pdev->dev,
2841 "BIOS update required for suspend/resume\n");
2842 }
2843
837f5f8f
TH		 2844 /* CAP.NP sometimes indicate the index of the last enabled
2845 * port, at other times, that of the last possible port, so
2846 * determining the maximum port number requires looking at
2847 * both CAP.NP and port_map.
2848 */
2849 n_ports = max(ahci_nr_ports(hpriv->cap), fls(hpriv->port_map));
2850
2851 host = ata_host_alloc_pinfo(&pdev->dev, ppi, n_ports);
4447d351
TH		 2852 if (!host)
2853 return -ENOMEM;
2854 host->iomap = pcim_iomap_table(pdev);
2855 host->private_data = hpriv;
2856
f3d7f23f 2857 if (!(hpriv->cap & HOST_CAP_SSS) || ahci_ignore_sss)
886ad09f 2858 host->flags |= ATA_HOST_PARALLEL_SCAN;
f3d7f23f
AV		 2859 else
2860 printk(KERN_INFO "ahci: SSS flag set, parallel bus scan disabled\n");
886ad09f 2861
18f7ba4c
KCA		 2862 if (pi.flags & ATA_FLAG_EM)
2863 ahci_reset_em(host);
2864
4447d351 2865 for (i = 0; i < host->n_ports; i++) {
dab632e8 2866 struct ata_port *ap = host->ports[i];
4447d351 2867
cbcdd875
TH		 2868 ata_port_pbar_desc(ap, AHCI_PCI_BAR, -1, "abar");
2869 ata_port_pbar_desc(ap, AHCI_PCI_BAR,
2870 0x100 + ap->port_no * 0x80, "port");
2871
31556594
KCA		 2872 /* set initial link pm policy */
2873 ap->pm_policy = NOT_AVAILABLE;
2874
18f7ba4c
KCA		 2875 /* set enclosure management message type */
2876 if (ap->flags & ATA_FLAG_EM)
2877 ap->em_message_type = ahci_em_messages;
2878
2879
dab632e8 2880 /* disabled/not-implemented port */
350756f6 2881 if (!(hpriv->port_map & (1 << i)))
dab632e8 2882 ap->ops = &ata_dummy_port_ops;
4447d351 2883 }
d447df14 2884
edc93052
TH		 2885 /* apply workaround for ASUS P5W DH Deluxe mainboard */
2886 ahci_p5wdh_workaround(host);
2887
4447d351
TH		 2888 /* initialize adapter */
2889 rc = ahci_configure_dma_masks(pdev, hpriv->cap & HOST_CAP_64);
1da177e4 2890 if (rc)
24dc5f33 2891 return rc;
1da177e4 2892
4447d351
TH		 2893 rc = ahci_reset_controller(host);
2894 if (rc)
2895 return rc;
1da177e4 2896
4447d351
TH		 2897 ahci_init_controller(host);
2898 ahci_print_info(host);
1da177e4 2899
4447d351
TH		 2900 pci_set_master(pdev);
2901 return ata_host_activate(host, pdev->irq, ahci_interrupt, IRQF_SHARED,
2902 &ahci_sht);
907f4678 2903}
1da177e4
LT		 2904
2905static int __init ahci_init(void)
2906{
b7887196 2907 return pci_register_driver(&ahci_pci_driver);
1da177e4
LT		 2908}
2909
1da177e4
LT		 2910static void __exit ahci_exit(void)
2911{
2912 pci_unregister_driver(&ahci_pci_driver);
2913}
2914
2915
2916MODULE_AUTHOR("Jeff Garzik");
2917MODULE_DESCRIPTION("AHCI SATA low-level driver");
2918MODULE_LICENSE("GPL");
2919MODULE_DEVICE_TABLE(pci, ahci_pci_tbl);
6885433c 2920MODULE_VERSION(DRV_VERSION);
1da177e4
LT		 2921
2922module_init(ahci_init);
2923module_exit(ahci_exit);
Linux 6.x block layer and io_uring tree(s)