Commit | Line | Data |
---|---|---|
7d715a6c SL |
1 | /* |
2 | * File: drivers/pci/pcie/aspm.c | |
45e829ea | 3 | * Enabling PCIe link L0s/L1 state and Clock Power Management |
7d715a6c SL |
4 | * |
5 | * Copyright (C) 2007 Intel | |
6 | * Copyright (C) Zhang Yanmin (yanmin.zhang@intel.com) | |
7 | * Copyright (C) Shaohua Li (shaohua.li@intel.com) | |
8 | */ | |
9 | ||
10 | #include <linux/kernel.h> | |
11 | #include <linux/module.h> | |
12 | #include <linux/moduleparam.h> | |
13 | #include <linux/pci.h> | |
14 | #include <linux/pci_regs.h> | |
15 | #include <linux/errno.h> | |
16 | #include <linux/pm.h> | |
17 | #include <linux/init.h> | |
18 | #include <linux/slab.h> | |
2a42d9db | 19 | #include <linux/jiffies.h> |
987a4c78 | 20 | #include <linux/delay.h> |
7d715a6c SL |
21 | #include <linux/pci-aspm.h> |
22 | #include "../pci.h" | |
23 | ||
24 | #ifdef MODULE_PARAM_PREFIX | |
25 | #undef MODULE_PARAM_PREFIX | |
26 | #endif | |
27 | #define MODULE_PARAM_PREFIX "pcie_aspm." | |
28 | ||
ac18018a KK |
29 | /* Note: those are not register definitions */ |
30 | #define ASPM_STATE_L0S_UP (1) /* Upstream direction L0s state */ | |
31 | #define ASPM_STATE_L0S_DW (2) /* Downstream direction L0s state */ | |
32 | #define ASPM_STATE_L1 (4) /* L1 state */ | |
b2103ccb RJ |
33 | #define ASPM_STATE_L1_1 (8) /* ASPM L1.1 state */ |
34 | #define ASPM_STATE_L1_2 (0x10) /* ASPM L1.2 state */ | |
35 | #define ASPM_STATE_L1_1_PCIPM (0x20) /* PCI PM L1.1 state */ | |
36 | #define ASPM_STATE_L1_2_PCIPM (0x40) /* PCI PM L1.2 state */ | |
37 | #define ASPM_STATE_L1_SS_PCIPM (ASPM_STATE_L1_1_PCIPM | ASPM_STATE_L1_2_PCIPM) | |
38 | #define ASPM_STATE_L1_2_MASK (ASPM_STATE_L1_2 | ASPM_STATE_L1_2_PCIPM) | |
39 | #define ASPM_STATE_L1SS (ASPM_STATE_L1_1 | ASPM_STATE_L1_1_PCIPM |\ | |
40 | ASPM_STATE_L1_2_MASK) | |
ac18018a | 41 | #define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW) |
b2103ccb RJ |
42 | #define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1 | \ |
43 | ASPM_STATE_L1SS) | |
ac18018a | 44 | |
f1f0366d RJ |
45 | /* |
46 | * When L1 substates are enabled, the LTR L1.2 threshold is a timing parameter | |
47 | * that decides whether L1.1 or L1.2 is entered (Refer PCIe spec for details). | |
48 | * Not sure is there is a way to "calculate" this on the fly, but maybe we | |
49 | * could turn it into a parameter in future. This value has been taken from | |
50 | * the following files from Intel's coreboot (which is the only code I found | |
51 | * to have used this): | |
52 | * https://www.coreboot.org/pipermail/coreboot-gerrit/2015-March/021134.html | |
53 | * https://review.coreboot.org/#/c/8832/ | |
54 | */ | |
55 | #define LTR_L1_2_THRESHOLD_BITS ((1 << 21) | (1 << 23) | (1 << 30)) | |
ac18018a | 56 | |
b6c2e54d KK |
57 | struct aspm_latency { |
58 | u32 l0s; /* L0s latency (nsec) */ | |
59 | u32 l1; /* L1 latency (nsec) */ | |
7d715a6c SL |
60 | }; |
61 | ||
62 | struct pcie_link_state { | |
5cde89d8 | 63 | struct pci_dev *pdev; /* Upstream component of the Link */ |
b5a0a9b5 | 64 | struct pci_dev *downstream; /* Downstream component, function 0 */ |
5c92ffb1 | 65 | struct pcie_link_state *root; /* pointer to the root port link */ |
5cde89d8 KK |
66 | struct pcie_link_state *parent; /* pointer to the parent Link state */ |
67 | struct list_head sibling; /* node in link_list */ | |
68 | struct list_head children; /* list of child link states */ | |
69 | struct list_head link; /* node in parent's children list */ | |
7d715a6c SL |
70 | |
71 | /* ASPM state */ | |
b2103ccb RJ |
72 | u32 aspm_support:7; /* Supported ASPM state */ |
73 | u32 aspm_enabled:7; /* Enabled ASPM state */ | |
74 | u32 aspm_capable:7; /* Capable ASPM state with latency */ | |
75 | u32 aspm_default:7; /* Default ASPM state by BIOS */ | |
76 | u32 aspm_disable:7; /* Disabled ASPM state */ | |
80bfdbe3 | 77 | |
4d246e45 KK |
78 | /* Clock PM state */ |
79 | u32 clkpm_capable:1; /* Clock PM capable? */ | |
80 | u32 clkpm_enabled:1; /* Current Clock PM state */ | |
81 | u32 clkpm_default:1; /* Default Clock PM state by BIOS */ | |
82 | ||
ac18018a KK |
83 | /* Exit latencies */ |
84 | struct aspm_latency latency_up; /* Upstream direction exit latency */ | |
85 | struct aspm_latency latency_dw; /* Downstream direction exit latency */ | |
7d715a6c | 86 | /* |
b6c2e54d KK |
87 | * Endpoint acceptable latencies. A pcie downstream port only |
88 | * has one slot under it, so at most there are 8 functions. | |
7d715a6c | 89 | */ |
b6c2e54d | 90 | struct aspm_latency acceptable[8]; |
f1f0366d RJ |
91 | |
92 | /* L1 PM Substate info */ | |
93 | struct { | |
94 | u32 up_cap_ptr; /* L1SS cap ptr in upstream dev */ | |
95 | u32 dw_cap_ptr; /* L1SS cap ptr in downstream dev */ | |
96 | u32 ctl1; /* value to be programmed in ctl1 */ | |
97 | u32 ctl2; /* value to be programmed in ctl2 */ | |
98 | } l1ss; | |
7d715a6c SL |
99 | }; |
100 | ||
3c076351 | 101 | static int aspm_disabled, aspm_force; |
8b8bae90 | 102 | static bool aspm_support_enabled = true; |
7d715a6c SL |
103 | static DEFINE_MUTEX(aspm_lock); |
104 | static LIST_HEAD(link_list); | |
105 | ||
106 | #define POLICY_DEFAULT 0 /* BIOS default setting */ | |
107 | #define POLICY_PERFORMANCE 1 /* high performance */ | |
108 | #define POLICY_POWERSAVE 2 /* high power saving */ | |
b2103ccb | 109 | #define POLICY_POWER_SUPERSAVE 3 /* possibly even more power saving */ |
ad71c962 MG |
110 | |
111 | #ifdef CONFIG_PCIEASPM_PERFORMANCE | |
112 | static int aspm_policy = POLICY_PERFORMANCE; | |
113 | #elif defined CONFIG_PCIEASPM_POWERSAVE | |
114 | static int aspm_policy = POLICY_POWERSAVE; | |
b2103ccb RJ |
115 | #elif defined CONFIG_PCIEASPM_POWER_SUPERSAVE |
116 | static int aspm_policy = POLICY_POWER_SUPERSAVE; | |
ad71c962 | 117 | #else |
7d715a6c | 118 | static int aspm_policy; |
ad71c962 MG |
119 | #endif |
120 | ||
7d715a6c SL |
121 | static const char *policy_str[] = { |
122 | [POLICY_DEFAULT] = "default", | |
123 | [POLICY_PERFORMANCE] = "performance", | |
b2103ccb RJ |
124 | [POLICY_POWERSAVE] = "powersave", |
125 | [POLICY_POWER_SUPERSAVE] = "powersupersave" | |
7d715a6c SL |
126 | }; |
127 | ||
987a4c78 AP |
128 | #define LINK_RETRAIN_TIMEOUT HZ |
129 | ||
5aa63583 | 130 | static int policy_to_aspm_state(struct pcie_link_state *link) |
7d715a6c | 131 | { |
7d715a6c SL |
132 | switch (aspm_policy) { |
133 | case POLICY_PERFORMANCE: | |
134 | /* Disable ASPM and Clock PM */ | |
135 | return 0; | |
136 | case POLICY_POWERSAVE: | |
137 | /* Enable ASPM L0s/L1 */ | |
b2103ccb RJ |
138 | return (ASPM_STATE_L0S | ASPM_STATE_L1); |
139 | case POLICY_POWER_SUPERSAVE: | |
140 | /* Enable Everything */ | |
ac18018a | 141 | return ASPM_STATE_ALL; |
7d715a6c | 142 | case POLICY_DEFAULT: |
5aa63583 | 143 | return link->aspm_default; |
7d715a6c SL |
144 | } |
145 | return 0; | |
146 | } | |
147 | ||
5aa63583 | 148 | static int policy_to_clkpm_state(struct pcie_link_state *link) |
7d715a6c | 149 | { |
7d715a6c SL |
150 | switch (aspm_policy) { |
151 | case POLICY_PERFORMANCE: | |
152 | /* Disable ASPM and Clock PM */ | |
153 | return 0; | |
154 | case POLICY_POWERSAVE: | |
b2103ccb RJ |
155 | case POLICY_POWER_SUPERSAVE: |
156 | /* Enable Clock PM */ | |
7d715a6c SL |
157 | return 1; |
158 | case POLICY_DEFAULT: | |
5aa63583 | 159 | return link->clkpm_default; |
7d715a6c SL |
160 | } |
161 | return 0; | |
162 | } | |
163 | ||
430842e2 | 164 | static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable) |
7d715a6c | 165 | { |
5aa63583 KK |
166 | struct pci_dev *child; |
167 | struct pci_bus *linkbus = link->pdev->subordinate; | |
0c0cbb6c | 168 | u32 val = enable ? PCI_EXP_LNKCTL_CLKREQ_EN : 0; |
7d715a6c | 169 | |
0c0cbb6c BH |
170 | list_for_each_entry(child, &linkbus->devices, bus_list) |
171 | pcie_capability_clear_and_set_word(child, PCI_EXP_LNKCTL, | |
172 | PCI_EXP_LNKCTL_CLKREQ_EN, | |
173 | val); | |
5aa63583 | 174 | link->clkpm_enabled = !!enable; |
7d715a6c SL |
175 | } |
176 | ||
430842e2 KK |
177 | static void pcie_set_clkpm(struct pcie_link_state *link, int enable) |
178 | { | |
179 | /* Don't enable Clock PM if the link is not Clock PM capable */ | |
a6c1c6f3 | 180 | if (!link->clkpm_capable) |
2f671e2d | 181 | enable = 0; |
430842e2 KK |
182 | /* Need nothing if the specified equals to current state */ |
183 | if (link->clkpm_enabled == enable) | |
184 | return; | |
185 | pcie_set_clkpm_nocheck(link, enable); | |
186 | } | |
187 | ||
8d349ace | 188 | static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist) |
7d715a6c | 189 | { |
f12eb72a | 190 | int capable = 1, enabled = 1; |
7d715a6c SL |
191 | u32 reg32; |
192 | u16 reg16; | |
5aa63583 KK |
193 | struct pci_dev *child; |
194 | struct pci_bus *linkbus = link->pdev->subordinate; | |
7d715a6c SL |
195 | |
196 | /* All functions should have the same cap and state, take the worst */ | |
5aa63583 | 197 | list_for_each_entry(child, &linkbus->devices, bus_list) { |
f12eb72a | 198 | pcie_capability_read_dword(child, PCI_EXP_LNKCAP, ®32); |
7d715a6c SL |
199 | if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) { |
200 | capable = 0; | |
201 | enabled = 0; | |
202 | break; | |
203 | } | |
f12eb72a | 204 | pcie_capability_read_word(child, PCI_EXP_LNKCTL, ®16); |
7d715a6c SL |
205 | if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN)) |
206 | enabled = 0; | |
207 | } | |
5aa63583 KK |
208 | link->clkpm_enabled = enabled; |
209 | link->clkpm_default = enabled; | |
8d349ace | 210 | link->clkpm_capable = (blacklist) ? 0 : capable; |
46bbdfa4 SL |
211 | } |
212 | ||
7d715a6c SL |
213 | /* |
214 | * pcie_aspm_configure_common_clock: check if the 2 ends of a link | |
215 | * could use common clock. If they are, configure them to use the | |
216 | * common clock. That will reduce the ASPM state exit latency. | |
217 | */ | |
5aa63583 | 218 | static void pcie_aspm_configure_common_clock(struct pcie_link_state *link) |
7d715a6c | 219 | { |
f12eb72a | 220 | int same_clock = 1; |
5aa63583 | 221 | u16 reg16, parent_reg, child_reg[8]; |
2a42d9db | 222 | unsigned long start_jiffies; |
5aa63583 KK |
223 | struct pci_dev *child, *parent = link->pdev; |
224 | struct pci_bus *linkbus = parent->subordinate; | |
7d715a6c | 225 | /* |
5aa63583 | 226 | * All functions of a slot should have the same Slot Clock |
7d715a6c | 227 | * Configuration, so just check one function |
5aa63583 KK |
228 | */ |
229 | child = list_entry(linkbus->devices.next, struct pci_dev, bus_list); | |
8b06477d | 230 | BUG_ON(!pci_is_pcie(child)); |
7d715a6c SL |
231 | |
232 | /* Check downstream component if bit Slot Clock Configuration is 1 */ | |
f12eb72a | 233 | pcie_capability_read_word(child, PCI_EXP_LNKSTA, ®16); |
7d715a6c SL |
234 | if (!(reg16 & PCI_EXP_LNKSTA_SLC)) |
235 | same_clock = 0; | |
236 | ||
237 | /* Check upstream component if bit Slot Clock Configuration is 1 */ | |
f12eb72a | 238 | pcie_capability_read_word(parent, PCI_EXP_LNKSTA, ®16); |
7d715a6c SL |
239 | if (!(reg16 & PCI_EXP_LNKSTA_SLC)) |
240 | same_clock = 0; | |
241 | ||
242 | /* Configure downstream component, all functions */ | |
5aa63583 | 243 | list_for_each_entry(child, &linkbus->devices, bus_list) { |
f12eb72a | 244 | pcie_capability_read_word(child, PCI_EXP_LNKCTL, ®16); |
5aa63583 | 245 | child_reg[PCI_FUNC(child->devfn)] = reg16; |
7d715a6c SL |
246 | if (same_clock) |
247 | reg16 |= PCI_EXP_LNKCTL_CCC; | |
248 | else | |
249 | reg16 &= ~PCI_EXP_LNKCTL_CCC; | |
f12eb72a | 250 | pcie_capability_write_word(child, PCI_EXP_LNKCTL, reg16); |
7d715a6c SL |
251 | } |
252 | ||
253 | /* Configure upstream component */ | |
f12eb72a | 254 | pcie_capability_read_word(parent, PCI_EXP_LNKCTL, ®16); |
2a42d9db | 255 | parent_reg = reg16; |
7d715a6c SL |
256 | if (same_clock) |
257 | reg16 |= PCI_EXP_LNKCTL_CCC; | |
258 | else | |
259 | reg16 &= ~PCI_EXP_LNKCTL_CCC; | |
f12eb72a | 260 | pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16); |
7d715a6c | 261 | |
5aa63583 | 262 | /* Retrain link */ |
7d715a6c | 263 | reg16 |= PCI_EXP_LNKCTL_RL; |
f12eb72a | 264 | pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16); |
7d715a6c | 265 | |
5aa63583 | 266 | /* Wait for link training end. Break out after waiting for timeout */ |
2a42d9db | 267 | start_jiffies = jiffies; |
987a4c78 | 268 | for (;;) { |
f12eb72a | 269 | pcie_capability_read_word(parent, PCI_EXP_LNKSTA, ®16); |
7d715a6c SL |
270 | if (!(reg16 & PCI_EXP_LNKSTA_LT)) |
271 | break; | |
987a4c78 AP |
272 | if (time_after(jiffies, start_jiffies + LINK_RETRAIN_TIMEOUT)) |
273 | break; | |
274 | msleep(1); | |
7d715a6c | 275 | } |
5aa63583 KK |
276 | if (!(reg16 & PCI_EXP_LNKSTA_LT)) |
277 | return; | |
278 | ||
279 | /* Training failed. Restore common clock configurations */ | |
438be3c6 | 280 | dev_err(&parent->dev, "ASPM: Could not configure common clock\n"); |
f12eb72a JL |
281 | list_for_each_entry(child, &linkbus->devices, bus_list) |
282 | pcie_capability_write_word(child, PCI_EXP_LNKCTL, | |
283 | child_reg[PCI_FUNC(child->devfn)]); | |
284 | pcie_capability_write_word(parent, PCI_EXP_LNKCTL, parent_reg); | |
7d715a6c SL |
285 | } |
286 | ||
5e0eaa7d KK |
287 | /* Convert L0s latency encoding to ns */ |
288 | static u32 calc_l0s_latency(u32 encoding) | |
7d715a6c | 289 | { |
5e0eaa7d KK |
290 | if (encoding == 0x7) |
291 | return (5 * 1000); /* > 4us */ | |
292 | return (64 << encoding); | |
293 | } | |
7d715a6c | 294 | |
5e0eaa7d KK |
295 | /* Convert L0s acceptable latency encoding to ns */ |
296 | static u32 calc_l0s_acceptable(u32 encoding) | |
297 | { | |
298 | if (encoding == 0x7) | |
299 | return -1U; | |
300 | return (64 << encoding); | |
7d715a6c SL |
301 | } |
302 | ||
5e0eaa7d KK |
303 | /* Convert L1 latency encoding to ns */ |
304 | static u32 calc_l1_latency(u32 encoding) | |
7d715a6c | 305 | { |
5e0eaa7d KK |
306 | if (encoding == 0x7) |
307 | return (65 * 1000); /* > 64us */ | |
308 | return (1000 << encoding); | |
309 | } | |
7d715a6c | 310 | |
5e0eaa7d KK |
311 | /* Convert L1 acceptable latency encoding to ns */ |
312 | static u32 calc_l1_acceptable(u32 encoding) | |
313 | { | |
314 | if (encoding == 0x7) | |
315 | return -1U; | |
316 | return (1000 << encoding); | |
7d715a6c SL |
317 | } |
318 | ||
f1f0366d RJ |
319 | /* Convert L1SS T_pwr encoding to usec */ |
320 | static u32 calc_l1ss_pwron(struct pci_dev *pdev, u32 scale, u32 val) | |
321 | { | |
322 | switch (scale) { | |
323 | case 0: | |
324 | return val * 2; | |
325 | case 1: | |
326 | return val * 10; | |
327 | case 2: | |
328 | return val * 100; | |
329 | } | |
330 | dev_err(&pdev->dev, "%s: Invalid T_PwrOn scale: %u\n", | |
331 | __func__, scale); | |
332 | return 0; | |
333 | } | |
334 | ||
ac18018a KK |
335 | struct aspm_register_info { |
336 | u32 support:2; | |
337 | u32 enabled:2; | |
338 | u32 latency_encoding_l0s; | |
339 | u32 latency_encoding_l1; | |
b5a0a9b5 RJ |
340 | |
341 | /* L1 substates */ | |
342 | u32 l1ss_cap_ptr; | |
343 | u32 l1ss_cap; | |
344 | u32 l1ss_ctl1; | |
345 | u32 l1ss_ctl2; | |
ac18018a KK |
346 | }; |
347 | ||
348 | static void pcie_get_aspm_reg(struct pci_dev *pdev, | |
349 | struct aspm_register_info *info) | |
7d715a6c | 350 | { |
7d715a6c | 351 | u16 reg16; |
ac18018a | 352 | u32 reg32; |
7d715a6c | 353 | |
f12eb72a | 354 | pcie_capability_read_dword(pdev, PCI_EXP_LNKCAP, ®32); |
ac18018a | 355 | info->support = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10; |
ac18018a KK |
356 | info->latency_encoding_l0s = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12; |
357 | info->latency_encoding_l1 = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15; | |
f12eb72a | 358 | pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, ®16); |
ac18018a | 359 | info->enabled = reg16 & PCI_EXP_LNKCTL_ASPMC; |
b5a0a9b5 RJ |
360 | |
361 | /* Read L1 PM substate capabilities */ | |
362 | info->l1ss_cap = info->l1ss_ctl1 = info->l1ss_ctl2 = 0; | |
363 | info->l1ss_cap_ptr = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_L1SS); | |
364 | if (!info->l1ss_cap_ptr) | |
365 | return; | |
366 | pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CAP, | |
367 | &info->l1ss_cap); | |
368 | if (!(info->l1ss_cap & PCI_L1SS_CAP_L1_PM_SS)) { | |
369 | info->l1ss_cap = 0; | |
370 | return; | |
371 | } | |
372 | pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CTL1, | |
373 | &info->l1ss_ctl1); | |
374 | pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CTL2, | |
375 | &info->l1ss_ctl2); | |
7d715a6c SL |
376 | } |
377 | ||
07d92760 KK |
378 | static void pcie_aspm_check_latency(struct pci_dev *endpoint) |
379 | { | |
ac18018a | 380 | u32 latency, l1_switch_latency = 0; |
07d92760 KK |
381 | struct aspm_latency *acceptable; |
382 | struct pcie_link_state *link; | |
383 | ||
384 | /* Device not in D0 doesn't need latency check */ | |
385 | if ((endpoint->current_state != PCI_D0) && | |
386 | (endpoint->current_state != PCI_UNKNOWN)) | |
387 | return; | |
388 | ||
389 | link = endpoint->bus->self->link_state; | |
390 | acceptable = &link->acceptable[PCI_FUNC(endpoint->devfn)]; | |
391 | ||
392 | while (link) { | |
ac18018a KK |
393 | /* Check upstream direction L0s latency */ |
394 | if ((link->aspm_capable & ASPM_STATE_L0S_UP) && | |
395 | (link->latency_up.l0s > acceptable->l0s)) | |
396 | link->aspm_capable &= ~ASPM_STATE_L0S_UP; | |
397 | ||
398 | /* Check downstream direction L0s latency */ | |
399 | if ((link->aspm_capable & ASPM_STATE_L0S_DW) && | |
400 | (link->latency_dw.l0s > acceptable->l0s)) | |
401 | link->aspm_capable &= ~ASPM_STATE_L0S_DW; | |
07d92760 KK |
402 | /* |
403 | * Check L1 latency. | |
404 | * Every switch on the path to root complex need 1 | |
405 | * more microsecond for L1. Spec doesn't mention L0s. | |
a142f4d3 RJ |
406 | * |
407 | * The exit latencies for L1 substates are not advertised | |
408 | * by a device. Since the spec also doesn't mention a way | |
409 | * to determine max latencies introduced by enabling L1 | |
410 | * substates on the components, it is not clear how to do | |
411 | * a L1 substate exit latency check. We assume that the | |
412 | * L1 exit latencies advertised by a device include L1 | |
413 | * substate latencies (and hence do not do any check). | |
07d92760 | 414 | */ |
ac18018a KK |
415 | latency = max_t(u32, link->latency_up.l1, link->latency_dw.l1); |
416 | if ((link->aspm_capable & ASPM_STATE_L1) && | |
417 | (latency + l1_switch_latency > acceptable->l1)) | |
418 | link->aspm_capable &= ~ASPM_STATE_L1; | |
07d92760 KK |
419 | l1_switch_latency += 1000; |
420 | ||
421 | link = link->parent; | |
422 | } | |
423 | } | |
424 | ||
b5a0a9b5 RJ |
425 | /* |
426 | * The L1 PM substate capability is only implemented in function 0 in a | |
427 | * multi function device. | |
428 | */ | |
429 | static struct pci_dev *pci_function_0(struct pci_bus *linkbus) | |
430 | { | |
431 | struct pci_dev *child; | |
432 | ||
433 | list_for_each_entry(child, &linkbus->devices, bus_list) | |
434 | if (PCI_FUNC(child->devfn) == 0) | |
435 | return child; | |
436 | return NULL; | |
437 | } | |
438 | ||
f1f0366d RJ |
439 | /* Calculate L1.2 PM substate timing parameters */ |
440 | static void aspm_calc_l1ss_info(struct pcie_link_state *link, | |
441 | struct aspm_register_info *upreg, | |
442 | struct aspm_register_info *dwreg) | |
443 | { | |
444 | u32 val1, val2, scale1, scale2; | |
445 | ||
446 | link->l1ss.up_cap_ptr = upreg->l1ss_cap_ptr; | |
447 | link->l1ss.dw_cap_ptr = dwreg->l1ss_cap_ptr; | |
448 | link->l1ss.ctl1 = link->l1ss.ctl2 = 0; | |
449 | ||
450 | if (!(link->aspm_support & ASPM_STATE_L1_2_MASK)) | |
451 | return; | |
452 | ||
453 | /* Choose the greater of the two T_cmn_mode_rstr_time */ | |
454 | val1 = (upreg->l1ss_cap >> 8) & 0xFF; | |
455 | val2 = (upreg->l1ss_cap >> 8) & 0xFF; | |
456 | if (val1 > val2) | |
457 | link->l1ss.ctl1 |= val1 << 8; | |
458 | else | |
459 | link->l1ss.ctl1 |= val2 << 8; | |
460 | /* | |
461 | * We currently use LTR L1.2 threshold to be fixed constant picked from | |
462 | * Intel's coreboot. | |
463 | */ | |
464 | link->l1ss.ctl1 |= LTR_L1_2_THRESHOLD_BITS; | |
465 | ||
466 | /* Choose the greater of the two T_pwr_on */ | |
467 | val1 = (upreg->l1ss_cap >> 19) & 0x1F; | |
468 | scale1 = (upreg->l1ss_cap >> 16) & 0x03; | |
469 | val2 = (dwreg->l1ss_cap >> 19) & 0x1F; | |
470 | scale2 = (dwreg->l1ss_cap >> 16) & 0x03; | |
471 | ||
472 | if (calc_l1ss_pwron(link->pdev, scale1, val1) > | |
473 | calc_l1ss_pwron(link->downstream, scale2, val2)) | |
474 | link->l1ss.ctl2 |= scale1 | (val1 << 3); | |
475 | else | |
476 | link->l1ss.ctl2 |= scale2 | (val2 << 3); | |
477 | } | |
478 | ||
8d349ace | 479 | static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) |
7d715a6c | 480 | { |
3bd7db63 | 481 | struct pci_dev *child = link->downstream, *parent = link->pdev; |
5aa63583 | 482 | struct pci_bus *linkbus = parent->subordinate; |
ac18018a | 483 | struct aspm_register_info upreg, dwreg; |
7d715a6c | 484 | |
8d349ace | 485 | if (blacklist) { |
f1c0ca29 | 486 | /* Set enabled/disable so that we will disable ASPM later */ |
ac18018a KK |
487 | link->aspm_enabled = ASPM_STATE_ALL; |
488 | link->aspm_disable = ASPM_STATE_ALL; | |
8d349ace KK |
489 | return; |
490 | } | |
491 | ||
e53f9a28 DD |
492 | /* Get upstream/downstream components' register state */ |
493 | pcie_get_aspm_reg(parent, &upreg); | |
e53f9a28 DD |
494 | pcie_get_aspm_reg(child, &dwreg); |
495 | ||
496 | /* | |
497 | * If ASPM not supported, don't mess with the clocks and link, | |
498 | * bail out now. | |
499 | */ | |
500 | if (!(upreg.support & dwreg.support)) | |
501 | return; | |
502 | ||
8d349ace KK |
503 | /* Configure common clock before checking latencies */ |
504 | pcie_aspm_configure_common_clock(link); | |
505 | ||
e53f9a28 DD |
506 | /* |
507 | * Re-read upstream/downstream components' register state | |
508 | * after clock configuration | |
509 | */ | |
ac18018a | 510 | pcie_get_aspm_reg(parent, &upreg); |
ac18018a KK |
511 | pcie_get_aspm_reg(child, &dwreg); |
512 | ||
513 | /* | |
514 | * Setup L0s state | |
515 | * | |
516 | * Note that we must not enable L0s in either direction on a | |
517 | * given link unless components on both sides of the link each | |
518 | * support L0s. | |
519 | */ | |
520 | if (dwreg.support & upreg.support & PCIE_LINK_STATE_L0S) | |
521 | link->aspm_support |= ASPM_STATE_L0S; | |
522 | if (dwreg.enabled & PCIE_LINK_STATE_L0S) | |
523 | link->aspm_enabled |= ASPM_STATE_L0S_UP; | |
524 | if (upreg.enabled & PCIE_LINK_STATE_L0S) | |
525 | link->aspm_enabled |= ASPM_STATE_L0S_DW; | |
526 | link->latency_up.l0s = calc_l0s_latency(upreg.latency_encoding_l0s); | |
527 | link->latency_dw.l0s = calc_l0s_latency(dwreg.latency_encoding_l0s); | |
528 | ||
529 | /* Setup L1 state */ | |
530 | if (upreg.support & dwreg.support & PCIE_LINK_STATE_L1) | |
531 | link->aspm_support |= ASPM_STATE_L1; | |
532 | if (upreg.enabled & dwreg.enabled & PCIE_LINK_STATE_L1) | |
533 | link->aspm_enabled |= ASPM_STATE_L1; | |
534 | link->latency_up.l1 = calc_l1_latency(upreg.latency_encoding_l1); | |
535 | link->latency_dw.l1 = calc_l1_latency(dwreg.latency_encoding_l1); | |
5aa63583 | 536 | |
b5a0a9b5 RJ |
537 | /* Setup L1 substate */ |
538 | if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1) | |
539 | link->aspm_support |= ASPM_STATE_L1_1; | |
540 | if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2) | |
541 | link->aspm_support |= ASPM_STATE_L1_2; | |
542 | if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1) | |
543 | link->aspm_support |= ASPM_STATE_L1_1_PCIPM; | |
544 | if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2) | |
545 | link->aspm_support |= ASPM_STATE_L1_2_PCIPM; | |
546 | ||
547 | if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1) | |
548 | link->aspm_enabled |= ASPM_STATE_L1_1; | |
549 | if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2) | |
550 | link->aspm_enabled |= ASPM_STATE_L1_2; | |
551 | if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1) | |
552 | link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM; | |
553 | if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2) | |
554 | link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM; | |
555 | ||
f1f0366d RJ |
556 | if (link->aspm_support & ASPM_STATE_L1SS) |
557 | aspm_calc_l1ss_info(link, &upreg, &dwreg); | |
558 | ||
b127bd55 KK |
559 | /* Save default state */ |
560 | link->aspm_default = link->aspm_enabled; | |
07d92760 KK |
561 | |
562 | /* Setup initial capable state. Will be updated later */ | |
563 | link->aspm_capable = link->aspm_support; | |
f1c0ca29 KK |
564 | /* |
565 | * If the downstream component has pci bridge function, don't | |
566 | * do ASPM for now. | |
567 | */ | |
568 | list_for_each_entry(child, &linkbus->devices, bus_list) { | |
62f87c0e | 569 | if (pci_pcie_type(child) == PCI_EXP_TYPE_PCI_BRIDGE) { |
ac18018a | 570 | link->aspm_disable = ASPM_STATE_ALL; |
f1c0ca29 KK |
571 | break; |
572 | } | |
573 | } | |
b127bd55 | 574 | |
b7206cbf | 575 | /* Get and check endpoint acceptable latencies */ |
5aa63583 | 576 | list_for_each_entry(child, &linkbus->devices, bus_list) { |
5e0eaa7d | 577 | u32 reg32, encoding; |
b6c2e54d | 578 | struct aspm_latency *acceptable = |
5aa63583 | 579 | &link->acceptable[PCI_FUNC(child->devfn)]; |
7d715a6c | 580 | |
62f87c0e YW |
581 | if (pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT && |
582 | pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END) | |
7d715a6c SL |
583 | continue; |
584 | ||
f12eb72a | 585 | pcie_capability_read_dword(child, PCI_EXP_DEVCAP, ®32); |
07d92760 | 586 | /* Calculate endpoint L0s acceptable latency */ |
5e0eaa7d KK |
587 | encoding = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6; |
588 | acceptable->l0s = calc_l0s_acceptable(encoding); | |
07d92760 KK |
589 | /* Calculate endpoint L1 acceptable latency */ |
590 | encoding = (reg32 & PCI_EXP_DEVCAP_L1) >> 9; | |
591 | acceptable->l1 = calc_l1_acceptable(encoding); | |
592 | ||
593 | pcie_aspm_check_latency(child); | |
7d715a6c SL |
594 | } |
595 | } | |
596 | ||
aeda9ade RJ |
597 | static void pci_clear_and_set_dword(struct pci_dev *pdev, int pos, |
598 | u32 clear, u32 set) | |
599 | { | |
600 | u32 val; | |
601 | ||
602 | pci_read_config_dword(pdev, pos, &val); | |
603 | val &= ~clear; | |
604 | val |= set; | |
605 | pci_write_config_dword(pdev, pos, val); | |
606 | } | |
607 | ||
608 | /* Configure the ASPM L1 substates */ | |
609 | static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state) | |
610 | { | |
611 | u32 val, enable_req; | |
612 | struct pci_dev *child = link->downstream, *parent = link->pdev; | |
613 | u32 up_cap_ptr = link->l1ss.up_cap_ptr; | |
614 | u32 dw_cap_ptr = link->l1ss.dw_cap_ptr; | |
615 | ||
616 | enable_req = (link->aspm_enabled ^ state) & state; | |
617 | ||
618 | /* | |
619 | * Here are the rules specified in the PCIe spec for enabling L1SS: | |
620 | * - When enabling L1.x, enable bit at parent first, then at child | |
621 | * - When disabling L1.x, disable bit at child first, then at parent | |
622 | * - When enabling ASPM L1.x, need to disable L1 | |
623 | * (at child followed by parent). | |
624 | * - The ASPM/PCIPM L1.2 must be disabled while programming timing | |
625 | * parameters | |
626 | * | |
627 | * To keep it simple, disable all L1SS bits first, and later enable | |
628 | * what is needed. | |
629 | */ | |
630 | ||
631 | /* Disable all L1 substates */ | |
632 | pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1, | |
633 | PCI_L1SS_CTL1_L1SS_MASK, 0); | |
634 | pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1, | |
635 | PCI_L1SS_CTL1_L1SS_MASK, 0); | |
636 | /* | |
637 | * If needed, disable L1, and it gets enabled later | |
638 | * in pcie_config_aspm_link(). | |
639 | */ | |
640 | if (enable_req & (ASPM_STATE_L1_1 | ASPM_STATE_L1_2)) { | |
641 | pcie_capability_clear_and_set_word(child, PCI_EXP_LNKCTL, | |
642 | PCI_EXP_LNKCTL_ASPM_L1, 0); | |
643 | pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL, | |
644 | PCI_EXP_LNKCTL_ASPM_L1, 0); | |
645 | } | |
646 | ||
647 | if (enable_req & ASPM_STATE_L1_2_MASK) { | |
648 | ||
649 | /* Program T_pwr_on in both ports */ | |
650 | pci_write_config_dword(parent, up_cap_ptr + PCI_L1SS_CTL2, | |
651 | link->l1ss.ctl2); | |
652 | pci_write_config_dword(child, dw_cap_ptr + PCI_L1SS_CTL2, | |
653 | link->l1ss.ctl2); | |
654 | ||
655 | /* Program T_cmn_mode in parent */ | |
656 | pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1, | |
657 | 0xFF00, link->l1ss.ctl1); | |
658 | ||
659 | /* Program LTR L1.2 threshold in both ports */ | |
660 | pci_clear_and_set_dword(parent, dw_cap_ptr + PCI_L1SS_CTL1, | |
661 | 0xE3FF0000, link->l1ss.ctl1); | |
662 | pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1, | |
663 | 0xE3FF0000, link->l1ss.ctl1); | |
664 | } | |
665 | ||
666 | val = 0; | |
667 | if (state & ASPM_STATE_L1_1) | |
668 | val |= PCI_L1SS_CTL1_ASPM_L1_1; | |
669 | if (state & ASPM_STATE_L1_2) | |
670 | val |= PCI_L1SS_CTL1_ASPM_L1_2; | |
671 | if (state & ASPM_STATE_L1_1_PCIPM) | |
672 | val |= PCI_L1SS_CTL1_PCIPM_L1_1; | |
673 | if (state & ASPM_STATE_L1_2_PCIPM) | |
674 | val |= PCI_L1SS_CTL1_PCIPM_L1_2; | |
675 | ||
676 | /* Enable what we need to enable */ | |
677 | pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1, | |
678 | PCI_L1SS_CAP_L1_PM_SS, val); | |
679 | pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1, | |
680 | PCI_L1SS_CAP_L1_PM_SS, val); | |
681 | } | |
682 | ||
ac18018a | 683 | static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val) |
7d715a6c | 684 | { |
75083206 BH |
685 | pcie_capability_clear_and_set_word(pdev, PCI_EXP_LNKCTL, |
686 | PCI_EXP_LNKCTL_ASPMC, val); | |
7d715a6c SL |
687 | } |
688 | ||
b7206cbf | 689 | static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state) |
7d715a6c | 690 | { |
ac18018a | 691 | u32 upstream = 0, dwstream = 0; |
aeda9ade | 692 | struct pci_dev *child = link->downstream, *parent = link->pdev; |
5aa63583 | 693 | struct pci_bus *linkbus = parent->subordinate; |
7d715a6c | 694 | |
aeda9ade | 695 | /* Enable only the states that were not explicitly disabled */ |
b7206cbf | 696 | state &= (link->aspm_capable & ~link->aspm_disable); |
aeda9ade RJ |
697 | |
698 | /* Can't enable any substates if L1 is not enabled */ | |
699 | if (!(state & ASPM_STATE_L1)) | |
700 | state &= ~ASPM_STATE_L1SS; | |
701 | ||
702 | /* Spec says both ports must be in D0 before enabling PCI PM substates*/ | |
703 | if (parent->current_state != PCI_D0 || child->current_state != PCI_D0) { | |
704 | state &= ~ASPM_STATE_L1_SS_PCIPM; | |
705 | state |= (link->aspm_enabled & ASPM_STATE_L1_SS_PCIPM); | |
706 | } | |
707 | ||
708 | /* Nothing to do if the link is already in the requested state */ | |
f1c0ca29 KK |
709 | if (link->aspm_enabled == state) |
710 | return; | |
ac18018a KK |
711 | /* Convert ASPM state to upstream/downstream ASPM register state */ |
712 | if (state & ASPM_STATE_L0S_UP) | |
75083206 | 713 | dwstream |= PCI_EXP_LNKCTL_ASPM_L0S; |
ac18018a | 714 | if (state & ASPM_STATE_L0S_DW) |
75083206 | 715 | upstream |= PCI_EXP_LNKCTL_ASPM_L0S; |
ac18018a | 716 | if (state & ASPM_STATE_L1) { |
75083206 BH |
717 | upstream |= PCI_EXP_LNKCTL_ASPM_L1; |
718 | dwstream |= PCI_EXP_LNKCTL_ASPM_L1; | |
ac18018a | 719 | } |
aeda9ade RJ |
720 | |
721 | if (link->aspm_capable & ASPM_STATE_L1SS) | |
722 | pcie_config_aspm_l1ss(link, state); | |
723 | ||
7d715a6c | 724 | /* |
5aa63583 KK |
725 | * Spec 2.0 suggests all functions should be configured the |
726 | * same setting for ASPM. Enabling ASPM L1 should be done in | |
727 | * upstream component first and then downstream, and vice | |
728 | * versa for disabling ASPM L1. Spec doesn't mention L0S. | |
7d715a6c | 729 | */ |
ac18018a KK |
730 | if (state & ASPM_STATE_L1) |
731 | pcie_config_aspm_dev(parent, upstream); | |
5aa63583 | 732 | list_for_each_entry(child, &linkbus->devices, bus_list) |
ac18018a KK |
733 | pcie_config_aspm_dev(child, dwstream); |
734 | if (!(state & ASPM_STATE_L1)) | |
735 | pcie_config_aspm_dev(parent, upstream); | |
7d715a6c | 736 | |
5aa63583 | 737 | link->aspm_enabled = state; |
7d715a6c SL |
738 | } |
739 | ||
b7206cbf | 740 | static void pcie_config_aspm_path(struct pcie_link_state *link) |
7d715a6c | 741 | { |
b7206cbf KK |
742 | while (link) { |
743 | pcie_config_aspm_link(link, policy_to_aspm_state(link)); | |
744 | link = link->parent; | |
46bbdfa4 | 745 | } |
7d715a6c SL |
746 | } |
747 | ||
5aa63583 | 748 | static void free_link_state(struct pcie_link_state *link) |
7d715a6c | 749 | { |
5aa63583 KK |
750 | link->pdev->link_state = NULL; |
751 | kfree(link); | |
7d715a6c SL |
752 | } |
753 | ||
ddc9753f SL |
754 | static int pcie_aspm_sanity_check(struct pci_dev *pdev) |
755 | { | |
3647584d | 756 | struct pci_dev *child; |
149e1637 | 757 | u32 reg32; |
2f671e2d | 758 | |
ddc9753f | 759 | /* |
45e829ea | 760 | * Some functions in a slot might not all be PCIe functions, |
3647584d | 761 | * very strange. Disable ASPM for the whole slot |
ddc9753f | 762 | */ |
3647584d | 763 | list_for_each_entry(child, &pdev->subordinate->devices, bus_list) { |
f12eb72a | 764 | if (!pci_is_pcie(child)) |
ddc9753f | 765 | return -EINVAL; |
c9651e70 MG |
766 | |
767 | /* | |
768 | * If ASPM is disabled then we're not going to change | |
769 | * the BIOS state. It's safe to continue even if it's a | |
770 | * pre-1.1 device | |
771 | */ | |
772 | ||
773 | if (aspm_disabled) | |
774 | continue; | |
775 | ||
149e1637 SL |
776 | /* |
777 | * Disable ASPM for pre-1.1 PCIe device, we follow MS to use | |
778 | * RBER bit to determine if a function is 1.1 version device | |
779 | */ | |
f12eb72a | 780 | pcie_capability_read_dword(child, PCI_EXP_DEVCAP, ®32); |
e1f4f59d | 781 | if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) { |
438be3c6 | 782 | dev_info(&child->dev, "disabling ASPM on pre-1.1 PCIe device. You can enable it with 'pcie_aspm=force'\n"); |
149e1637 SL |
783 | return -EINVAL; |
784 | } | |
ddc9753f SL |
785 | } |
786 | return 0; | |
787 | } | |
788 | ||
b7206cbf | 789 | static struct pcie_link_state *alloc_pcie_link_state(struct pci_dev *pdev) |
8d349ace KK |
790 | { |
791 | struct pcie_link_state *link; | |
8d349ace KK |
792 | |
793 | link = kzalloc(sizeof(*link), GFP_KERNEL); | |
794 | if (!link) | |
795 | return NULL; | |
030305d6 | 796 | |
8d349ace KK |
797 | INIT_LIST_HEAD(&link->sibling); |
798 | INIT_LIST_HEAD(&link->children); | |
799 | INIT_LIST_HEAD(&link->link); | |
800 | link->pdev = pdev; | |
3bd7db63 | 801 | link->downstream = pci_function_0(pdev->subordinate); |
030305d6 BH |
802 | |
803 | /* | |
804 | * Root Ports and PCI/PCI-X to PCIe Bridges are roots of PCIe | |
805 | * hierarchies. | |
806 | */ | |
807 | if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT || | |
808 | pci_pcie_type(pdev) == PCI_EXP_TYPE_PCIE_BRIDGE) { | |
809 | link->root = link; | |
810 | } else { | |
8d349ace | 811 | struct pcie_link_state *parent; |
030305d6 | 812 | |
8d349ace KK |
813 | parent = pdev->bus->parent->self->link_state; |
814 | if (!parent) { | |
815 | kfree(link); | |
816 | return NULL; | |
817 | } | |
030305d6 | 818 | |
8d349ace | 819 | link->parent = parent; |
030305d6 | 820 | link->root = link->parent->root; |
8d349ace KK |
821 | list_add(&link->link, &parent->children); |
822 | } | |
5c92ffb1 | 823 | |
8d349ace | 824 | list_add(&link->sibling, &link_list); |
8d349ace | 825 | pdev->link_state = link; |
8d349ace KK |
826 | return link; |
827 | } | |
828 | ||
7d715a6c SL |
829 | /* |
830 | * pcie_aspm_init_link_state: Initiate PCI express link state. | |
f7625980 | 831 | * It is called after the pcie and its children devices are scanned. |
7d715a6c SL |
832 | * @pdev: the root port or switch downstream port |
833 | */ | |
834 | void pcie_aspm_init_link_state(struct pci_dev *pdev) | |
835 | { | |
8d349ace | 836 | struct pcie_link_state *link; |
b7206cbf | 837 | int blacklist = !!pcie_aspm_sanity_check(pdev); |
7d715a6c | 838 | |
a26d5ecb JL |
839 | if (!aspm_support_enabled) |
840 | return; | |
841 | ||
c8fc9339 | 842 | if (pdev->link_state) |
7d715a6c | 843 | return; |
c8fc9339 YW |
844 | |
845 | /* | |
846 | * We allocate pcie_link_state for the component on the upstream | |
847 | * end of a Link, so there's nothing to do unless this device has a | |
848 | * Link on its secondary side. | |
849 | */ | |
850 | if (!pdev->has_secondary_link) | |
7d715a6c | 851 | return; |
8d349ace | 852 | |
8e822df7 | 853 | /* VIA has a strange chipset, root port is under a bridge */ |
62f87c0e | 854 | if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT && |
8d349ace | 855 | pdev->bus->self) |
8e822df7 | 856 | return; |
8d349ace | 857 | |
7d715a6c SL |
858 | down_read(&pci_bus_sem); |
859 | if (list_empty(&pdev->subordinate->devices)) | |
860 | goto out; | |
861 | ||
862 | mutex_lock(&aspm_lock); | |
b7206cbf | 863 | link = alloc_pcie_link_state(pdev); |
8d349ace KK |
864 | if (!link) |
865 | goto unlock; | |
866 | /* | |
b7206cbf KK |
867 | * Setup initial ASPM state. Note that we need to configure |
868 | * upstream links also because capable state of them can be | |
869 | * update through pcie_aspm_cap_init(). | |
8d349ace | 870 | */ |
b7206cbf | 871 | pcie_aspm_cap_init(link, blacklist); |
7d715a6c | 872 | |
8d349ace | 873 | /* Setup initial Clock PM state */ |
b7206cbf | 874 | pcie_clkpm_cap_init(link, blacklist); |
41cd766b MG |
875 | |
876 | /* | |
877 | * At this stage drivers haven't had an opportunity to change the | |
878 | * link policy setting. Enabling ASPM on broken hardware can cripple | |
879 | * it even before the driver has had a chance to disable ASPM, so | |
880 | * default to a safe level right now. If we're enabling ASPM beyond | |
881 | * the BIOS's expectation, we'll do so once pci_enable_device() is | |
882 | * called. | |
883 | */ | |
b2103ccb RJ |
884 | if (aspm_policy != POLICY_POWERSAVE && |
885 | aspm_policy != POLICY_POWER_SUPERSAVE) { | |
41cd766b MG |
886 | pcie_config_aspm_path(link); |
887 | pcie_set_clkpm(link, policy_to_clkpm_state(link)); | |
888 | } | |
889 | ||
8d349ace | 890 | unlock: |
7d715a6c SL |
891 | mutex_unlock(&aspm_lock); |
892 | out: | |
893 | up_read(&pci_bus_sem); | |
894 | } | |
895 | ||
07d92760 KK |
896 | /* Recheck latencies and update aspm_capable for links under the root */ |
897 | static void pcie_update_aspm_capable(struct pcie_link_state *root) | |
898 | { | |
899 | struct pcie_link_state *link; | |
900 | BUG_ON(root->parent); | |
901 | list_for_each_entry(link, &link_list, sibling) { | |
902 | if (link->root != root) | |
903 | continue; | |
904 | link->aspm_capable = link->aspm_support; | |
905 | } | |
906 | list_for_each_entry(link, &link_list, sibling) { | |
907 | struct pci_dev *child; | |
908 | struct pci_bus *linkbus = link->pdev->subordinate; | |
909 | if (link->root != root) | |
910 | continue; | |
911 | list_for_each_entry(child, &linkbus->devices, bus_list) { | |
62f87c0e YW |
912 | if ((pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT) && |
913 | (pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END)) | |
07d92760 KK |
914 | continue; |
915 | pcie_aspm_check_latency(child); | |
916 | } | |
917 | } | |
918 | } | |
919 | ||
7d715a6c SL |
920 | /* @pdev: the endpoint device */ |
921 | void pcie_aspm_exit_link_state(struct pci_dev *pdev) | |
922 | { | |
923 | struct pci_dev *parent = pdev->bus->self; | |
b7206cbf | 924 | struct pcie_link_state *link, *root, *parent_link; |
7d715a6c | 925 | |
84fb913c | 926 | if (!parent || !parent->link_state) |
7d715a6c | 927 | return; |
fc87e919 | 928 | |
7d715a6c SL |
929 | down_read(&pci_bus_sem); |
930 | mutex_lock(&aspm_lock); | |
7d715a6c SL |
931 | /* |
932 | * All PCIe functions are in one slot, remove one function will remove | |
3419c75e | 933 | * the whole slot, so just wait until we are the last function left. |
7d715a6c | 934 | */ |
3419c75e | 935 | if (!list_is_last(&pdev->bus_list, &parent->subordinate->devices)) |
7d715a6c SL |
936 | goto out; |
937 | ||
fc87e919 | 938 | link = parent->link_state; |
07d92760 | 939 | root = link->root; |
b7206cbf | 940 | parent_link = link->parent; |
fc87e919 | 941 | |
7d715a6c | 942 | /* All functions are removed, so just disable ASPM for the link */ |
b7206cbf | 943 | pcie_config_aspm_link(link, 0); |
fc87e919 KK |
944 | list_del(&link->sibling); |
945 | list_del(&link->link); | |
7d715a6c | 946 | /* Clock PM is for endpoint device */ |
fc87e919 | 947 | free_link_state(link); |
07d92760 KK |
948 | |
949 | /* Recheck latencies and configure upstream links */ | |
b26a34aa KK |
950 | if (parent_link) { |
951 | pcie_update_aspm_capable(root); | |
952 | pcie_config_aspm_path(parent_link); | |
953 | } | |
7d715a6c SL |
954 | out: |
955 | mutex_unlock(&aspm_lock); | |
956 | up_read(&pci_bus_sem); | |
957 | } | |
958 | ||
959 | /* @pdev: the root port or switch downstream port */ | |
960 | void pcie_aspm_pm_state_change(struct pci_dev *pdev) | |
961 | { | |
07d92760 | 962 | struct pcie_link_state *link = pdev->link_state; |
7d715a6c | 963 | |
f9b8cd7c | 964 | if (aspm_disabled || !link) |
7d715a6c SL |
965 | return; |
966 | /* | |
07d92760 KK |
967 | * Devices changed PM state, we should recheck if latency |
968 | * meets all functions' requirement | |
7d715a6c | 969 | */ |
07d92760 KK |
970 | down_read(&pci_bus_sem); |
971 | mutex_lock(&aspm_lock); | |
972 | pcie_update_aspm_capable(link->root); | |
b7206cbf | 973 | pcie_config_aspm_path(link); |
07d92760 KK |
974 | mutex_unlock(&aspm_lock); |
975 | up_read(&pci_bus_sem); | |
7d715a6c SL |
976 | } |
977 | ||
1a680b7c NC |
978 | void pcie_aspm_powersave_config_link(struct pci_dev *pdev) |
979 | { | |
980 | struct pcie_link_state *link = pdev->link_state; | |
981 | ||
f9b8cd7c | 982 | if (aspm_disabled || !link) |
1a680b7c NC |
983 | return; |
984 | ||
b2103ccb RJ |
985 | if (aspm_policy != POLICY_POWERSAVE && |
986 | aspm_policy != POLICY_POWER_SUPERSAVE) | |
1a680b7c NC |
987 | return; |
988 | ||
1a680b7c NC |
989 | down_read(&pci_bus_sem); |
990 | mutex_lock(&aspm_lock); | |
991 | pcie_config_aspm_path(link); | |
992 | pcie_set_clkpm(link, policy_to_clkpm_state(link)); | |
993 | mutex_unlock(&aspm_lock); | |
994 | up_read(&pci_bus_sem); | |
995 | } | |
996 | ||
e127a04f | 997 | static void __pci_disable_link_state(struct pci_dev *pdev, int state, bool sem) |
7d715a6c SL |
998 | { |
999 | struct pci_dev *parent = pdev->bus->self; | |
f1c0ca29 | 1000 | struct pcie_link_state *link; |
7d715a6c | 1001 | |
3c076351 | 1002 | if (!pci_is_pcie(pdev)) |
7d715a6c | 1003 | return; |
3c076351 | 1004 | |
c8fc9339 | 1005 | if (pdev->has_secondary_link) |
7d715a6c SL |
1006 | parent = pdev; |
1007 | if (!parent || !parent->link_state) | |
1008 | return; | |
1009 | ||
2add0ec1 BH |
1010 | /* |
1011 | * A driver requested that ASPM be disabled on this device, but | |
1012 | * if we don't have permission to manage ASPM (e.g., on ACPI | |
1013 | * systems we have to observe the FADT ACPI_FADT_NO_ASPM bit and | |
1014 | * the _OSC method), we can't honor that request. Windows has | |
1015 | * a similar mechanism using "PciASPMOptOut", which is also | |
1016 | * ignored in this situation. | |
1017 | */ | |
e127a04f | 1018 | if (aspm_disabled) { |
2add0ec1 BH |
1019 | dev_warn(&pdev->dev, "can't disable ASPM; OS doesn't have ASPM control\n"); |
1020 | return; | |
1021 | } | |
1022 | ||
9f728f53 YL |
1023 | if (sem) |
1024 | down_read(&pci_bus_sem); | |
7d715a6c | 1025 | mutex_lock(&aspm_lock); |
f1c0ca29 | 1026 | link = parent->link_state; |
ac18018a KK |
1027 | if (state & PCIE_LINK_STATE_L0S) |
1028 | link->aspm_disable |= ASPM_STATE_L0S; | |
1029 | if (state & PCIE_LINK_STATE_L1) | |
1030 | link->aspm_disable |= ASPM_STATE_L1; | |
b7206cbf KK |
1031 | pcie_config_aspm_link(link, policy_to_aspm_state(link)); |
1032 | ||
430842e2 | 1033 | if (state & PCIE_LINK_STATE_CLKPM) { |
f1c0ca29 KK |
1034 | link->clkpm_capable = 0; |
1035 | pcie_set_clkpm(link, 0); | |
430842e2 | 1036 | } |
7d715a6c | 1037 | mutex_unlock(&aspm_lock); |
9f728f53 YL |
1038 | if (sem) |
1039 | up_read(&pci_bus_sem); | |
1040 | } | |
1041 | ||
1042 | void pci_disable_link_state_locked(struct pci_dev *pdev, int state) | |
1043 | { | |
e127a04f | 1044 | __pci_disable_link_state(pdev, state, false); |
9f728f53 YL |
1045 | } |
1046 | EXPORT_SYMBOL(pci_disable_link_state_locked); | |
1047 | ||
2dfca877 YW |
1048 | /** |
1049 | * pci_disable_link_state - Disable device's link state, so the link will | |
1050 | * never enter specific states. Note that if the BIOS didn't grant ASPM | |
1051 | * control to the OS, this does nothing because we can't touch the LNKCTL | |
1052 | * register. | |
1053 | * | |
1054 | * @pdev: PCI device | |
1055 | * @state: ASPM link state to disable | |
1056 | */ | |
9f728f53 YL |
1057 | void pci_disable_link_state(struct pci_dev *pdev, int state) |
1058 | { | |
e127a04f | 1059 | __pci_disable_link_state(pdev, state, true); |
7d715a6c SL |
1060 | } |
1061 | EXPORT_SYMBOL(pci_disable_link_state); | |
1062 | ||
1063 | static int pcie_aspm_set_policy(const char *val, struct kernel_param *kp) | |
1064 | { | |
1065 | int i; | |
b7206cbf | 1066 | struct pcie_link_state *link; |
7d715a6c | 1067 | |
bbfa306a NC |
1068 | if (aspm_disabled) |
1069 | return -EPERM; | |
7d715a6c SL |
1070 | for (i = 0; i < ARRAY_SIZE(policy_str); i++) |
1071 | if (!strncmp(val, policy_str[i], strlen(policy_str[i]))) | |
1072 | break; | |
1073 | if (i >= ARRAY_SIZE(policy_str)) | |
1074 | return -EINVAL; | |
1075 | if (i == aspm_policy) | |
1076 | return 0; | |
1077 | ||
1078 | down_read(&pci_bus_sem); | |
1079 | mutex_lock(&aspm_lock); | |
1080 | aspm_policy = i; | |
b7206cbf KK |
1081 | list_for_each_entry(link, &link_list, sibling) { |
1082 | pcie_config_aspm_link(link, policy_to_aspm_state(link)); | |
1083 | pcie_set_clkpm(link, policy_to_clkpm_state(link)); | |
7d715a6c SL |
1084 | } |
1085 | mutex_unlock(&aspm_lock); | |
1086 | up_read(&pci_bus_sem); | |
1087 | return 0; | |
1088 | } | |
1089 | ||
1090 | static int pcie_aspm_get_policy(char *buffer, struct kernel_param *kp) | |
1091 | { | |
1092 | int i, cnt = 0; | |
1093 | for (i = 0; i < ARRAY_SIZE(policy_str); i++) | |
1094 | if (i == aspm_policy) | |
1095 | cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]); | |
1096 | else | |
1097 | cnt += sprintf(buffer + cnt, "%s ", policy_str[i]); | |
1098 | return cnt; | |
1099 | } | |
1100 | ||
1101 | module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy, | |
1102 | NULL, 0644); | |
1103 | ||
1104 | #ifdef CONFIG_PCIEASPM_DEBUG | |
1105 | static ssize_t link_state_show(struct device *dev, | |
1106 | struct device_attribute *attr, | |
1107 | char *buf) | |
1108 | { | |
1109 | struct pci_dev *pci_device = to_pci_dev(dev); | |
1110 | struct pcie_link_state *link_state = pci_device->link_state; | |
1111 | ||
80bfdbe3 | 1112 | return sprintf(buf, "%d\n", link_state->aspm_enabled); |
7d715a6c SL |
1113 | } |
1114 | ||
1115 | static ssize_t link_state_store(struct device *dev, | |
1116 | struct device_attribute *attr, | |
1117 | const char *buf, | |
1118 | size_t n) | |
1119 | { | |
5aa63583 | 1120 | struct pci_dev *pdev = to_pci_dev(dev); |
b7206cbf | 1121 | struct pcie_link_state *link, *root = pdev->link_state->root; |
57d86a04 | 1122 | u32 state; |
7d715a6c | 1123 | |
bbfa306a NC |
1124 | if (aspm_disabled) |
1125 | return -EPERM; | |
7d715a6c | 1126 | |
57d86a04 AL |
1127 | if (kstrtouint(buf, 10, &state)) |
1128 | return -EINVAL; | |
1129 | if ((state & ~ASPM_STATE_ALL) != 0) | |
1130 | return -EINVAL; | |
ac18018a | 1131 | |
b7206cbf KK |
1132 | down_read(&pci_bus_sem); |
1133 | mutex_lock(&aspm_lock); | |
1134 | list_for_each_entry(link, &link_list, sibling) { | |
1135 | if (link->root != root) | |
1136 | continue; | |
1137 | pcie_config_aspm_link(link, state); | |
1138 | } | |
1139 | mutex_unlock(&aspm_lock); | |
1140 | up_read(&pci_bus_sem); | |
1141 | return n; | |
7d715a6c SL |
1142 | } |
1143 | ||
1144 | static ssize_t clk_ctl_show(struct device *dev, | |
1145 | struct device_attribute *attr, | |
1146 | char *buf) | |
1147 | { | |
1148 | struct pci_dev *pci_device = to_pci_dev(dev); | |
1149 | struct pcie_link_state *link_state = pci_device->link_state; | |
1150 | ||
4d246e45 | 1151 | return sprintf(buf, "%d\n", link_state->clkpm_enabled); |
7d715a6c SL |
1152 | } |
1153 | ||
1154 | static ssize_t clk_ctl_store(struct device *dev, | |
1155 | struct device_attribute *attr, | |
1156 | const char *buf, | |
1157 | size_t n) | |
1158 | { | |
430842e2 | 1159 | struct pci_dev *pdev = to_pci_dev(dev); |
94a90312 | 1160 | bool state; |
7d715a6c | 1161 | |
94a90312 | 1162 | if (strtobool(buf, &state)) |
7d715a6c | 1163 | return -EINVAL; |
7d715a6c SL |
1164 | |
1165 | down_read(&pci_bus_sem); | |
1166 | mutex_lock(&aspm_lock); | |
94a90312 | 1167 | pcie_set_clkpm_nocheck(pdev->link_state, state); |
7d715a6c SL |
1168 | mutex_unlock(&aspm_lock); |
1169 | up_read(&pci_bus_sem); | |
1170 | ||
1171 | return n; | |
1172 | } | |
1173 | ||
fc4f57fa JL |
1174 | static DEVICE_ATTR_RW(link_state); |
1175 | static DEVICE_ATTR_RW(clk_ctl); | |
7d715a6c SL |
1176 | |
1177 | static char power_group[] = "power"; | |
1178 | void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev) | |
1179 | { | |
1180 | struct pcie_link_state *link_state = pdev->link_state; | |
1181 | ||
f9b8cd7c | 1182 | if (!link_state) |
7d715a6c SL |
1183 | return; |
1184 | ||
80bfdbe3 | 1185 | if (link_state->aspm_support) |
7d715a6c SL |
1186 | sysfs_add_file_to_group(&pdev->dev.kobj, |
1187 | &dev_attr_link_state.attr, power_group); | |
4d246e45 | 1188 | if (link_state->clkpm_capable) |
7d715a6c SL |
1189 | sysfs_add_file_to_group(&pdev->dev.kobj, |
1190 | &dev_attr_clk_ctl.attr, power_group); | |
1191 | } | |
1192 | ||
1193 | void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev) | |
1194 | { | |
1195 | struct pcie_link_state *link_state = pdev->link_state; | |
1196 | ||
f9b8cd7c | 1197 | if (!link_state) |
7d715a6c SL |
1198 | return; |
1199 | ||
80bfdbe3 | 1200 | if (link_state->aspm_support) |
7d715a6c SL |
1201 | sysfs_remove_file_from_group(&pdev->dev.kobj, |
1202 | &dev_attr_link_state.attr, power_group); | |
4d246e45 | 1203 | if (link_state->clkpm_capable) |
7d715a6c SL |
1204 | sysfs_remove_file_from_group(&pdev->dev.kobj, |
1205 | &dev_attr_clk_ctl.attr, power_group); | |
1206 | } | |
1207 | #endif | |
1208 | ||
1209 | static int __init pcie_aspm_disable(char *str) | |
1210 | { | |
d6d38574 | 1211 | if (!strcmp(str, "off")) { |
3c076351 | 1212 | aspm_policy = POLICY_DEFAULT; |
d6d38574 | 1213 | aspm_disabled = 1; |
8b8bae90 | 1214 | aspm_support_enabled = false; |
d6d38574 SL |
1215 | printk(KERN_INFO "PCIe ASPM is disabled\n"); |
1216 | } else if (!strcmp(str, "force")) { | |
1217 | aspm_force = 1; | |
8072ba1b | 1218 | printk(KERN_INFO "PCIe ASPM is forcibly enabled\n"); |
d6d38574 | 1219 | } |
7d715a6c SL |
1220 | return 1; |
1221 | } | |
1222 | ||
d6d38574 | 1223 | __setup("pcie_aspm=", pcie_aspm_disable); |
7d715a6c | 1224 | |
5fde244d SL |
1225 | void pcie_no_aspm(void) |
1226 | { | |
3c076351 MG |
1227 | /* |
1228 | * Disabling ASPM is intended to prevent the kernel from modifying | |
1229 | * existing hardware state, not to clear existing state. To that end: | |
1230 | * (a) set policy to POLICY_DEFAULT in order to avoid changing state | |
1231 | * (b) prevent userspace from changing policy | |
1232 | */ | |
1233 | if (!aspm_force) { | |
1234 | aspm_policy = POLICY_DEFAULT; | |
d6d38574 | 1235 | aspm_disabled = 1; |
3c076351 | 1236 | } |
5fde244d SL |
1237 | } |
1238 | ||
8b8bae90 RW |
1239 | bool pcie_aspm_support_enabled(void) |
1240 | { | |
1241 | return aspm_support_enabled; | |
1242 | } | |
1243 | EXPORT_SYMBOL(pcie_aspm_support_enabled); |