Commit | Line | Data |
---|---|---|
7d715a6c SL |
1 | /* |
2 | * File: drivers/pci/pcie/aspm.c | |
3 | * Enabling PCIE link L0s/L1 state and Clock Power Management | |
4 | * | |
5 | * Copyright (C) 2007 Intel | |
6 | * Copyright (C) Zhang Yanmin (yanmin.zhang@intel.com) | |
7 | * Copyright (C) Shaohua Li (shaohua.li@intel.com) | |
8 | */ | |
9 | ||
10 | #include <linux/kernel.h> | |
11 | #include <linux/module.h> | |
12 | #include <linux/moduleparam.h> | |
13 | #include <linux/pci.h> | |
14 | #include <linux/pci_regs.h> | |
15 | #include <linux/errno.h> | |
16 | #include <linux/pm.h> | |
17 | #include <linux/init.h> | |
18 | #include <linux/slab.h> | |
2a42d9db | 19 | #include <linux/jiffies.h> |
7d715a6c SL |
20 | #include <linux/pci-aspm.h> |
21 | #include "../pci.h" | |
22 | ||
23 | #ifdef MODULE_PARAM_PREFIX | |
24 | #undef MODULE_PARAM_PREFIX | |
25 | #endif | |
26 | #define MODULE_PARAM_PREFIX "pcie_aspm." | |
27 | ||
28 | struct endpoint_state { | |
29 | unsigned int l0s_acceptable_latency; | |
30 | unsigned int l1_acceptable_latency; | |
31 | }; | |
32 | ||
33 | struct pcie_link_state { | |
34 | struct list_head sibiling; | |
35 | struct pci_dev *pdev; | |
36 | ||
37 | /* ASPM state */ | |
38 | unsigned int support_state; | |
39 | unsigned int enabled_state; | |
40 | unsigned int bios_aspm_state; | |
41 | /* upstream component */ | |
42 | unsigned int l0s_upper_latency; | |
43 | unsigned int l1_upper_latency; | |
44 | /* downstream component */ | |
45 | unsigned int l0s_down_latency; | |
46 | unsigned int l1_down_latency; | |
47 | /* Clock PM state*/ | |
48 | unsigned int clk_pm_capable; | |
49 | unsigned int clk_pm_enabled; | |
50 | unsigned int bios_clk_state; | |
51 | ||
52 | /* | |
53 | * A pcie downstream port only has one slot under it, so at most there | |
54 | * are 8 functions | |
55 | */ | |
56 | struct endpoint_state endpoints[8]; | |
57 | }; | |
58 | ||
d6d38574 | 59 | static int aspm_disabled, aspm_force; |
7d715a6c SL |
60 | static DEFINE_MUTEX(aspm_lock); |
61 | static LIST_HEAD(link_list); | |
62 | ||
63 | #define POLICY_DEFAULT 0 /* BIOS default setting */ | |
64 | #define POLICY_PERFORMANCE 1 /* high performance */ | |
65 | #define POLICY_POWERSAVE 2 /* high power saving */ | |
66 | static int aspm_policy; | |
67 | static const char *policy_str[] = { | |
68 | [POLICY_DEFAULT] = "default", | |
69 | [POLICY_PERFORMANCE] = "performance", | |
70 | [POLICY_POWERSAVE] = "powersave" | |
71 | }; | |
72 | ||
73 | static int policy_to_aspm_state(struct pci_dev *pdev) | |
74 | { | |
75 | struct pcie_link_state *link_state = pdev->link_state; | |
76 | ||
77 | switch (aspm_policy) { | |
78 | case POLICY_PERFORMANCE: | |
79 | /* Disable ASPM and Clock PM */ | |
80 | return 0; | |
81 | case POLICY_POWERSAVE: | |
82 | /* Enable ASPM L0s/L1 */ | |
83 | return PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1; | |
84 | case POLICY_DEFAULT: | |
85 | return link_state->bios_aspm_state; | |
86 | } | |
87 | return 0; | |
88 | } | |
89 | ||
90 | static int policy_to_clkpm_state(struct pci_dev *pdev) | |
91 | { | |
92 | struct pcie_link_state *link_state = pdev->link_state; | |
93 | ||
94 | switch (aspm_policy) { | |
95 | case POLICY_PERFORMANCE: | |
96 | /* Disable ASPM and Clock PM */ | |
97 | return 0; | |
98 | case POLICY_POWERSAVE: | |
99 | /* Disable Clock PM */ | |
100 | return 1; | |
101 | case POLICY_DEFAULT: | |
102 | return link_state->bios_clk_state; | |
103 | } | |
104 | return 0; | |
105 | } | |
106 | ||
107 | static void pcie_set_clock_pm(struct pci_dev *pdev, int enable) | |
108 | { | |
109 | struct pci_dev *child_dev; | |
110 | int pos; | |
111 | u16 reg16; | |
112 | struct pcie_link_state *link_state = pdev->link_state; | |
113 | ||
114 | list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) { | |
115 | pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP); | |
116 | if (!pos) | |
117 | return; | |
118 | pci_read_config_word(child_dev, pos + PCI_EXP_LNKCTL, ®16); | |
119 | if (enable) | |
120 | reg16 |= PCI_EXP_LNKCTL_CLKREQ_EN; | |
121 | else | |
122 | reg16 &= ~PCI_EXP_LNKCTL_CLKREQ_EN; | |
123 | pci_write_config_word(child_dev, pos + PCI_EXP_LNKCTL, reg16); | |
124 | } | |
125 | link_state->clk_pm_enabled = !!enable; | |
126 | } | |
127 | ||
128 | static void pcie_check_clock_pm(struct pci_dev *pdev) | |
129 | { | |
130 | int pos; | |
131 | u32 reg32; | |
132 | u16 reg16; | |
133 | int capable = 1, enabled = 1; | |
134 | struct pci_dev *child_dev; | |
135 | struct pcie_link_state *link_state = pdev->link_state; | |
136 | ||
137 | /* All functions should have the same cap and state, take the worst */ | |
138 | list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) { | |
139 | pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP); | |
140 | if (!pos) | |
141 | return; | |
142 | pci_read_config_dword(child_dev, pos + PCI_EXP_LNKCAP, ®32); | |
143 | if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) { | |
144 | capable = 0; | |
145 | enabled = 0; | |
146 | break; | |
147 | } | |
148 | pci_read_config_word(child_dev, pos + PCI_EXP_LNKCTL, ®16); | |
149 | if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN)) | |
150 | enabled = 0; | |
151 | } | |
152 | link_state->clk_pm_capable = capable; | |
153 | link_state->clk_pm_enabled = enabled; | |
154 | link_state->bios_clk_state = enabled; | |
155 | pcie_set_clock_pm(pdev, policy_to_clkpm_state(pdev)); | |
156 | } | |
157 | ||
158 | /* | |
159 | * pcie_aspm_configure_common_clock: check if the 2 ends of a link | |
160 | * could use common clock. If they are, configure them to use the | |
161 | * common clock. That will reduce the ASPM state exit latency. | |
162 | */ | |
163 | static void pcie_aspm_configure_common_clock(struct pci_dev *pdev) | |
164 | { | |
2a42d9db | 165 | int pos, child_pos, i = 0; |
7d715a6c SL |
166 | u16 reg16 = 0; |
167 | struct pci_dev *child_dev; | |
168 | int same_clock = 1; | |
2a42d9db TR |
169 | unsigned long start_jiffies; |
170 | u16 child_regs[8], parent_reg; | |
7d715a6c SL |
171 | /* |
172 | * all functions of a slot should have the same Slot Clock | |
173 | * Configuration, so just check one function | |
174 | * */ | |
175 | child_dev = list_entry(pdev->subordinate->devices.next, struct pci_dev, | |
176 | bus_list); | |
177 | BUG_ON(!child_dev->is_pcie); | |
178 | ||
179 | /* Check downstream component if bit Slot Clock Configuration is 1 */ | |
180 | child_pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP); | |
181 | pci_read_config_word(child_dev, child_pos + PCI_EXP_LNKSTA, ®16); | |
182 | if (!(reg16 & PCI_EXP_LNKSTA_SLC)) | |
183 | same_clock = 0; | |
184 | ||
185 | /* Check upstream component if bit Slot Clock Configuration is 1 */ | |
186 | pos = pci_find_capability(pdev, PCI_CAP_ID_EXP); | |
187 | pci_read_config_word(pdev, pos + PCI_EXP_LNKSTA, ®16); | |
188 | if (!(reg16 & PCI_EXP_LNKSTA_SLC)) | |
189 | same_clock = 0; | |
190 | ||
191 | /* Configure downstream component, all functions */ | |
192 | list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) { | |
193 | child_pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP); | |
194 | pci_read_config_word(child_dev, child_pos + PCI_EXP_LNKCTL, | |
195 | ®16); | |
2a42d9db | 196 | child_regs[i] = reg16; |
7d715a6c SL |
197 | if (same_clock) |
198 | reg16 |= PCI_EXP_LNKCTL_CCC; | |
199 | else | |
200 | reg16 &= ~PCI_EXP_LNKCTL_CCC; | |
201 | pci_write_config_word(child_dev, child_pos + PCI_EXP_LNKCTL, | |
202 | reg16); | |
2a42d9db | 203 | i++; |
7d715a6c SL |
204 | } |
205 | ||
206 | /* Configure upstream component */ | |
207 | pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16); | |
2a42d9db | 208 | parent_reg = reg16; |
7d715a6c SL |
209 | if (same_clock) |
210 | reg16 |= PCI_EXP_LNKCTL_CCC; | |
211 | else | |
212 | reg16 &= ~PCI_EXP_LNKCTL_CCC; | |
213 | pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16); | |
214 | ||
215 | /* retrain link */ | |
216 | reg16 |= PCI_EXP_LNKCTL_RL; | |
217 | pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16); | |
218 | ||
219 | /* Wait for link training end */ | |
2a42d9db TR |
220 | /* break out after waiting for 1 second */ |
221 | start_jiffies = jiffies; | |
222 | while ((jiffies - start_jiffies) < HZ) { | |
7d715a6c SL |
223 | pci_read_config_word(pdev, pos + PCI_EXP_LNKSTA, ®16); |
224 | if (!(reg16 & PCI_EXP_LNKSTA_LT)) | |
225 | break; | |
226 | cpu_relax(); | |
227 | } | |
2a42d9db TR |
228 | /* training failed -> recover */ |
229 | if ((jiffies - start_jiffies) >= HZ) { | |
230 | dev_printk (KERN_ERR, &pdev->dev, "ASPM: Could not configure" | |
231 | " common clock\n"); | |
232 | i = 0; | |
233 | list_for_each_entry(child_dev, &pdev->subordinate->devices, | |
234 | bus_list) { | |
235 | child_pos = pci_find_capability(child_dev, | |
236 | PCI_CAP_ID_EXP); | |
237 | pci_write_config_word(child_dev, | |
238 | child_pos + PCI_EXP_LNKCTL, | |
239 | child_regs[i]); | |
240 | i++; | |
241 | } | |
242 | pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, parent_reg); | |
243 | } | |
7d715a6c SL |
244 | } |
245 | ||
246 | /* | |
247 | * calc_L0S_latency: Convert L0s latency encoding to ns | |
248 | */ | |
249 | static unsigned int calc_L0S_latency(unsigned int latency_encoding, int ac) | |
250 | { | |
251 | unsigned int ns = 64; | |
252 | ||
253 | if (latency_encoding == 0x7) { | |
254 | if (ac) | |
255 | ns = -1U; | |
256 | else | |
257 | ns = 5*1000; /* > 4us */ | |
258 | } else | |
259 | ns *= (1 << latency_encoding); | |
260 | return ns; | |
261 | } | |
262 | ||
263 | /* | |
264 | * calc_L1_latency: Convert L1 latency encoding to ns | |
265 | */ | |
266 | static unsigned int calc_L1_latency(unsigned int latency_encoding, int ac) | |
267 | { | |
268 | unsigned int ns = 1000; | |
269 | ||
270 | if (latency_encoding == 0x7) { | |
271 | if (ac) | |
272 | ns = -1U; | |
273 | else | |
274 | ns = 65*1000; /* > 64us */ | |
275 | } else | |
276 | ns *= (1 << latency_encoding); | |
277 | return ns; | |
278 | } | |
279 | ||
280 | static void pcie_aspm_get_cap_device(struct pci_dev *pdev, u32 *state, | |
281 | unsigned int *l0s, unsigned int *l1, unsigned int *enabled) | |
282 | { | |
283 | int pos; | |
284 | u16 reg16; | |
285 | u32 reg32; | |
286 | unsigned int latency; | |
287 | ||
288 | pos = pci_find_capability(pdev, PCI_CAP_ID_EXP); | |
289 | pci_read_config_dword(pdev, pos + PCI_EXP_LNKCAP, ®32); | |
290 | *state = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10; | |
291 | if (*state != PCIE_LINK_STATE_L0S && | |
292 | *state != (PCIE_LINK_STATE_L1|PCIE_LINK_STATE_L0S)) | |
293 | *state = 0; | |
294 | if (*state == 0) | |
295 | return; | |
296 | ||
297 | latency = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12; | |
298 | *l0s = calc_L0S_latency(latency, 0); | |
299 | if (*state & PCIE_LINK_STATE_L1) { | |
300 | latency = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15; | |
301 | *l1 = calc_L1_latency(latency, 0); | |
302 | } | |
303 | pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16); | |
304 | *enabled = reg16 & (PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1); | |
305 | } | |
306 | ||
307 | static void pcie_aspm_cap_init(struct pci_dev *pdev) | |
308 | { | |
309 | struct pci_dev *child_dev; | |
310 | u32 state, tmp; | |
311 | struct pcie_link_state *link_state = pdev->link_state; | |
312 | ||
313 | /* upstream component states */ | |
314 | pcie_aspm_get_cap_device(pdev, &link_state->support_state, | |
315 | &link_state->l0s_upper_latency, | |
316 | &link_state->l1_upper_latency, | |
317 | &link_state->enabled_state); | |
318 | /* downstream component states, all functions have the same setting */ | |
319 | child_dev = list_entry(pdev->subordinate->devices.next, struct pci_dev, | |
320 | bus_list); | |
321 | pcie_aspm_get_cap_device(child_dev, &state, | |
322 | &link_state->l0s_down_latency, | |
323 | &link_state->l1_down_latency, | |
324 | &tmp); | |
325 | link_state->support_state &= state; | |
326 | if (!link_state->support_state) | |
327 | return; | |
328 | link_state->enabled_state &= link_state->support_state; | |
329 | link_state->bios_aspm_state = link_state->enabled_state; | |
330 | ||
331 | /* ENDPOINT states*/ | |
332 | list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) { | |
333 | int pos; | |
334 | u32 reg32; | |
335 | unsigned int latency; | |
336 | struct endpoint_state *ep_state = | |
337 | &link_state->endpoints[PCI_FUNC(child_dev->devfn)]; | |
338 | ||
339 | if (child_dev->pcie_type != PCI_EXP_TYPE_ENDPOINT && | |
340 | child_dev->pcie_type != PCI_EXP_TYPE_LEG_END) | |
341 | continue; | |
342 | ||
343 | pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP); | |
344 | pci_read_config_dword(child_dev, pos + PCI_EXP_DEVCAP, ®32); | |
345 | latency = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6; | |
346 | latency = calc_L0S_latency(latency, 1); | |
347 | ep_state->l0s_acceptable_latency = latency; | |
348 | if (link_state->support_state & PCIE_LINK_STATE_L1) { | |
349 | latency = (reg32 & PCI_EXP_DEVCAP_L1) >> 9; | |
350 | latency = calc_L1_latency(latency, 1); | |
351 | ep_state->l1_acceptable_latency = latency; | |
352 | } | |
353 | } | |
354 | } | |
355 | ||
356 | static unsigned int __pcie_aspm_check_state_one(struct pci_dev *pdev, | |
357 | unsigned int state) | |
358 | { | |
359 | struct pci_dev *parent_dev, *tmp_dev; | |
360 | unsigned int latency, l1_latency = 0; | |
361 | struct pcie_link_state *link_state; | |
362 | struct endpoint_state *ep_state; | |
363 | ||
364 | parent_dev = pdev->bus->self; | |
365 | link_state = parent_dev->link_state; | |
366 | state &= link_state->support_state; | |
367 | if (state == 0) | |
368 | return 0; | |
369 | ep_state = &link_state->endpoints[PCI_FUNC(pdev->devfn)]; | |
370 | ||
371 | /* | |
372 | * Check latency for endpoint device. | |
373 | * TBD: The latency from the endpoint to root complex vary per | |
374 | * switch's upstream link state above the device. Here we just do a | |
375 | * simple check which assumes all links above the device can be in L1 | |
376 | * state, that is we just consider the worst case. If switch's upstream | |
377 | * link can't be put into L0S/L1, then our check is too strictly. | |
378 | */ | |
379 | tmp_dev = pdev; | |
380 | while (state & (PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1)) { | |
381 | parent_dev = tmp_dev->bus->self; | |
382 | link_state = parent_dev->link_state; | |
383 | if (state & PCIE_LINK_STATE_L0S) { | |
384 | latency = max_t(unsigned int, | |
385 | link_state->l0s_upper_latency, | |
386 | link_state->l0s_down_latency); | |
387 | if (latency > ep_state->l0s_acceptable_latency) | |
388 | state &= ~PCIE_LINK_STATE_L0S; | |
389 | } | |
390 | if (state & PCIE_LINK_STATE_L1) { | |
391 | latency = max_t(unsigned int, | |
392 | link_state->l1_upper_latency, | |
393 | link_state->l1_down_latency); | |
394 | if (latency + l1_latency > | |
395 | ep_state->l1_acceptable_latency) | |
396 | state &= ~PCIE_LINK_STATE_L1; | |
397 | } | |
398 | if (!parent_dev->bus->self) /* parent_dev is a root port */ | |
399 | break; | |
400 | else { | |
401 | /* | |
402 | * parent_dev is the downstream port of a switch, make | |
403 | * tmp_dev the upstream port of the switch | |
404 | */ | |
405 | tmp_dev = parent_dev->bus->self; | |
406 | /* | |
407 | * every switch on the path to root complex need 1 more | |
408 | * microsecond for L1. Spec doesn't mention L0S. | |
409 | */ | |
410 | if (state & PCIE_LINK_STATE_L1) | |
411 | l1_latency += 1000; | |
412 | } | |
413 | } | |
414 | return state; | |
415 | } | |
416 | ||
417 | static unsigned int pcie_aspm_check_state(struct pci_dev *pdev, | |
418 | unsigned int state) | |
419 | { | |
420 | struct pci_dev *child_dev; | |
421 | ||
422 | /* If no child, disable the link */ | |
423 | if (list_empty(&pdev->subordinate->devices)) | |
424 | return 0; | |
425 | list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) { | |
426 | if (child_dev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE) { | |
427 | /* | |
428 | * If downstream component of a link is pci bridge, we | |
429 | * disable ASPM for now for the link | |
430 | * */ | |
431 | state = 0; | |
432 | break; | |
433 | } | |
434 | if ((child_dev->pcie_type != PCI_EXP_TYPE_ENDPOINT && | |
435 | child_dev->pcie_type != PCI_EXP_TYPE_LEG_END)) | |
436 | continue; | |
437 | /* Device not in D0 doesn't need check latency */ | |
438 | if (child_dev->current_state == PCI_D1 || | |
439 | child_dev->current_state == PCI_D2 || | |
440 | child_dev->current_state == PCI_D3hot || | |
441 | child_dev->current_state == PCI_D3cold) | |
442 | continue; | |
443 | state = __pcie_aspm_check_state_one(child_dev, state); | |
444 | } | |
445 | return state; | |
446 | } | |
447 | ||
448 | static void __pcie_aspm_config_one_dev(struct pci_dev *pdev, unsigned int state) | |
449 | { | |
450 | u16 reg16; | |
451 | int pos = pci_find_capability(pdev, PCI_CAP_ID_EXP); | |
452 | ||
453 | pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16); | |
454 | reg16 &= ~0x3; | |
455 | reg16 |= state; | |
456 | pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16); | |
457 | } | |
458 | ||
459 | static void __pcie_aspm_config_link(struct pci_dev *pdev, unsigned int state) | |
460 | { | |
461 | struct pci_dev *child_dev; | |
462 | int valid = 1; | |
463 | struct pcie_link_state *link_state = pdev->link_state; | |
464 | ||
465 | /* | |
466 | * if the downstream component has pci bridge function, don't do ASPM | |
467 | * now | |
468 | */ | |
469 | list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) { | |
470 | if (child_dev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE) { | |
471 | valid = 0; | |
472 | break; | |
473 | } | |
474 | } | |
475 | if (!valid) | |
476 | return; | |
477 | ||
478 | /* | |
479 | * spec 2.0 suggests all functions should be configured the same | |
480 | * setting for ASPM. Enabling ASPM L1 should be done in upstream | |
481 | * component first and then downstream, and vice versa for disabling | |
482 | * ASPM L1. Spec doesn't mention L0S. | |
483 | */ | |
484 | if (state & PCIE_LINK_STATE_L1) | |
485 | __pcie_aspm_config_one_dev(pdev, state); | |
486 | ||
487 | list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) | |
488 | __pcie_aspm_config_one_dev(child_dev, state); | |
489 | ||
490 | if (!(state & PCIE_LINK_STATE_L1)) | |
491 | __pcie_aspm_config_one_dev(pdev, state); | |
492 | ||
493 | link_state->enabled_state = state; | |
494 | } | |
495 | ||
496 | static void __pcie_aspm_configure_link_state(struct pci_dev *pdev, | |
497 | unsigned int state) | |
498 | { | |
499 | struct pcie_link_state *link_state = pdev->link_state; | |
500 | ||
501 | if (link_state->support_state == 0) | |
502 | return; | |
503 | state &= PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1; | |
504 | ||
505 | /* state 0 means disabling aspm */ | |
506 | state = pcie_aspm_check_state(pdev, state); | |
507 | if (link_state->enabled_state == state) | |
508 | return; | |
509 | __pcie_aspm_config_link(pdev, state); | |
510 | } | |
511 | ||
512 | /* | |
513 | * pcie_aspm_configure_link_state: enable/disable PCI express link state | |
514 | * @pdev: the root port or switch downstream port | |
515 | */ | |
516 | static void pcie_aspm_configure_link_state(struct pci_dev *pdev, | |
517 | unsigned int state) | |
518 | { | |
519 | down_read(&pci_bus_sem); | |
520 | mutex_lock(&aspm_lock); | |
521 | __pcie_aspm_configure_link_state(pdev, state); | |
522 | mutex_unlock(&aspm_lock); | |
523 | up_read(&pci_bus_sem); | |
524 | } | |
525 | ||
526 | static void free_link_state(struct pci_dev *pdev) | |
527 | { | |
528 | kfree(pdev->link_state); | |
529 | pdev->link_state = NULL; | |
530 | } | |
531 | ||
ddc9753f SL |
532 | static int pcie_aspm_sanity_check(struct pci_dev *pdev) |
533 | { | |
534 | struct pci_dev *child_dev; | |
535 | int child_pos; | |
149e1637 | 536 | u32 reg32; |
ddc9753f SL |
537 | |
538 | /* | |
539 | * Some functions in a slot might not all be PCIE functions, very | |
540 | * strange. Disable ASPM for the whole slot | |
541 | */ | |
542 | list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) { | |
543 | child_pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP); | |
544 | if (!child_pos) | |
545 | return -EINVAL; | |
149e1637 SL |
546 | |
547 | /* | |
548 | * Disable ASPM for pre-1.1 PCIe device, we follow MS to use | |
549 | * RBER bit to determine if a function is 1.1 version device | |
550 | */ | |
551 | pci_read_config_dword(child_dev, child_pos + PCI_EXP_DEVCAP, | |
552 | ®32); | |
e1f4f59d | 553 | if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) { |
f393d9b1 VL |
554 | dev_printk(KERN_INFO, &child_dev->dev, "disabling ASPM" |
555 | " on pre-1.1 PCIe device. You can enable it" | |
556 | " with 'pcie_aspm=force'\n"); | |
149e1637 SL |
557 | return -EINVAL; |
558 | } | |
ddc9753f SL |
559 | } |
560 | return 0; | |
561 | } | |
562 | ||
7d715a6c SL |
563 | /* |
564 | * pcie_aspm_init_link_state: Initiate PCI express link state. | |
565 | * It is called after the pcie and its children devices are scaned. | |
566 | * @pdev: the root port or switch downstream port | |
567 | */ | |
568 | void pcie_aspm_init_link_state(struct pci_dev *pdev) | |
569 | { | |
570 | unsigned int state; | |
571 | struct pcie_link_state *link_state; | |
572 | int error = 0; | |
573 | ||
574 | if (aspm_disabled || !pdev->is_pcie || pdev->link_state) | |
575 | return; | |
576 | if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT && | |
577 | pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) | |
578 | return; | |
579 | down_read(&pci_bus_sem); | |
580 | if (list_empty(&pdev->subordinate->devices)) | |
581 | goto out; | |
582 | ||
ddc9753f SL |
583 | if (pcie_aspm_sanity_check(pdev)) |
584 | goto out; | |
585 | ||
7d715a6c SL |
586 | mutex_lock(&aspm_lock); |
587 | ||
588 | link_state = kzalloc(sizeof(*link_state), GFP_KERNEL); | |
589 | if (!link_state) | |
590 | goto unlock_out; | |
591 | pdev->link_state = link_state; | |
592 | ||
593 | pcie_aspm_configure_common_clock(pdev); | |
594 | ||
595 | pcie_aspm_cap_init(pdev); | |
596 | ||
597 | /* config link state to avoid BIOS error */ | |
598 | state = pcie_aspm_check_state(pdev, policy_to_aspm_state(pdev)); | |
599 | __pcie_aspm_config_link(pdev, state); | |
600 | ||
601 | pcie_check_clock_pm(pdev); | |
602 | ||
603 | link_state->pdev = pdev; | |
604 | list_add(&link_state->sibiling, &link_list); | |
605 | ||
606 | unlock_out: | |
607 | if (error) | |
608 | free_link_state(pdev); | |
609 | mutex_unlock(&aspm_lock); | |
610 | out: | |
611 | up_read(&pci_bus_sem); | |
612 | } | |
613 | ||
614 | /* @pdev: the endpoint device */ | |
615 | void pcie_aspm_exit_link_state(struct pci_dev *pdev) | |
616 | { | |
617 | struct pci_dev *parent = pdev->bus->self; | |
618 | struct pcie_link_state *link_state = parent->link_state; | |
619 | ||
620 | if (aspm_disabled || !pdev->is_pcie || !parent || !link_state) | |
621 | return; | |
622 | if (parent->pcie_type != PCI_EXP_TYPE_ROOT_PORT && | |
623 | parent->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) | |
624 | return; | |
625 | down_read(&pci_bus_sem); | |
626 | mutex_lock(&aspm_lock); | |
627 | ||
628 | /* | |
629 | * All PCIe functions are in one slot, remove one function will remove | |
630 | * the the whole slot, so just wait | |
631 | */ | |
632 | if (!list_empty(&parent->subordinate->devices)) | |
633 | goto out; | |
634 | ||
635 | /* All functions are removed, so just disable ASPM for the link */ | |
636 | __pcie_aspm_config_one_dev(parent, 0); | |
637 | list_del(&link_state->sibiling); | |
638 | /* Clock PM is for endpoint device */ | |
639 | ||
640 | free_link_state(parent); | |
641 | out: | |
642 | mutex_unlock(&aspm_lock); | |
643 | up_read(&pci_bus_sem); | |
644 | } | |
645 | ||
646 | /* @pdev: the root port or switch downstream port */ | |
647 | void pcie_aspm_pm_state_change(struct pci_dev *pdev) | |
648 | { | |
649 | struct pcie_link_state *link_state = pdev->link_state; | |
650 | ||
651 | if (aspm_disabled || !pdev->is_pcie || !pdev->link_state) | |
652 | return; | |
653 | if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT && | |
654 | pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) | |
655 | return; | |
656 | /* | |
657 | * devices changed PM state, we should recheck if latency meets all | |
658 | * functions' requirement | |
659 | */ | |
660 | pcie_aspm_configure_link_state(pdev, link_state->enabled_state); | |
661 | } | |
662 | ||
663 | /* | |
664 | * pci_disable_link_state - disable pci device's link state, so the link will | |
665 | * never enter specific states | |
666 | */ | |
667 | void pci_disable_link_state(struct pci_dev *pdev, int state) | |
668 | { | |
669 | struct pci_dev *parent = pdev->bus->self; | |
670 | struct pcie_link_state *link_state; | |
671 | ||
672 | if (aspm_disabled || !pdev->is_pcie) | |
673 | return; | |
674 | if (pdev->pcie_type == PCI_EXP_TYPE_ROOT_PORT || | |
675 | pdev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM) | |
676 | parent = pdev; | |
677 | if (!parent || !parent->link_state) | |
678 | return; | |
679 | ||
680 | down_read(&pci_bus_sem); | |
681 | mutex_lock(&aspm_lock); | |
682 | link_state = parent->link_state; | |
683 | link_state->support_state &= | |
684 | ~(state & (PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1)); | |
685 | if (state & PCIE_LINK_STATE_CLKPM) | |
686 | link_state->clk_pm_capable = 0; | |
687 | ||
688 | __pcie_aspm_configure_link_state(parent, link_state->enabled_state); | |
689 | if (!link_state->clk_pm_capable && link_state->clk_pm_enabled) | |
690 | pcie_set_clock_pm(parent, 0); | |
691 | mutex_unlock(&aspm_lock); | |
692 | up_read(&pci_bus_sem); | |
693 | } | |
694 | EXPORT_SYMBOL(pci_disable_link_state); | |
695 | ||
696 | static int pcie_aspm_set_policy(const char *val, struct kernel_param *kp) | |
697 | { | |
698 | int i; | |
699 | struct pci_dev *pdev; | |
700 | struct pcie_link_state *link_state; | |
701 | ||
702 | for (i = 0; i < ARRAY_SIZE(policy_str); i++) | |
703 | if (!strncmp(val, policy_str[i], strlen(policy_str[i]))) | |
704 | break; | |
705 | if (i >= ARRAY_SIZE(policy_str)) | |
706 | return -EINVAL; | |
707 | if (i == aspm_policy) | |
708 | return 0; | |
709 | ||
710 | down_read(&pci_bus_sem); | |
711 | mutex_lock(&aspm_lock); | |
712 | aspm_policy = i; | |
713 | list_for_each_entry(link_state, &link_list, sibiling) { | |
714 | pdev = link_state->pdev; | |
715 | __pcie_aspm_configure_link_state(pdev, | |
716 | policy_to_aspm_state(pdev)); | |
717 | if (link_state->clk_pm_capable && | |
718 | link_state->clk_pm_enabled != policy_to_clkpm_state(pdev)) | |
719 | pcie_set_clock_pm(pdev, policy_to_clkpm_state(pdev)); | |
720 | ||
721 | } | |
722 | mutex_unlock(&aspm_lock); | |
723 | up_read(&pci_bus_sem); | |
724 | return 0; | |
725 | } | |
726 | ||
727 | static int pcie_aspm_get_policy(char *buffer, struct kernel_param *kp) | |
728 | { | |
729 | int i, cnt = 0; | |
730 | for (i = 0; i < ARRAY_SIZE(policy_str); i++) | |
731 | if (i == aspm_policy) | |
732 | cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]); | |
733 | else | |
734 | cnt += sprintf(buffer + cnt, "%s ", policy_str[i]); | |
735 | return cnt; | |
736 | } | |
737 | ||
738 | module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy, | |
739 | NULL, 0644); | |
740 | ||
741 | #ifdef CONFIG_PCIEASPM_DEBUG | |
742 | static ssize_t link_state_show(struct device *dev, | |
743 | struct device_attribute *attr, | |
744 | char *buf) | |
745 | { | |
746 | struct pci_dev *pci_device = to_pci_dev(dev); | |
747 | struct pcie_link_state *link_state = pci_device->link_state; | |
748 | ||
749 | return sprintf(buf, "%d\n", link_state->enabled_state); | |
750 | } | |
751 | ||
752 | static ssize_t link_state_store(struct device *dev, | |
753 | struct device_attribute *attr, | |
754 | const char *buf, | |
755 | size_t n) | |
756 | { | |
757 | struct pci_dev *pci_device = to_pci_dev(dev); | |
758 | int state; | |
759 | ||
760 | if (n < 1) | |
761 | return -EINVAL; | |
762 | state = buf[0]-'0'; | |
763 | if (state >= 0 && state <= 3) { | |
764 | /* setup link aspm state */ | |
765 | pcie_aspm_configure_link_state(pci_device, state); | |
766 | return n; | |
767 | } | |
768 | ||
769 | return -EINVAL; | |
770 | } | |
771 | ||
772 | static ssize_t clk_ctl_show(struct device *dev, | |
773 | struct device_attribute *attr, | |
774 | char *buf) | |
775 | { | |
776 | struct pci_dev *pci_device = to_pci_dev(dev); | |
777 | struct pcie_link_state *link_state = pci_device->link_state; | |
778 | ||
779 | return sprintf(buf, "%d\n", link_state->clk_pm_enabled); | |
780 | } | |
781 | ||
782 | static ssize_t clk_ctl_store(struct device *dev, | |
783 | struct device_attribute *attr, | |
784 | const char *buf, | |
785 | size_t n) | |
786 | { | |
787 | struct pci_dev *pci_device = to_pci_dev(dev); | |
788 | int state; | |
789 | ||
790 | if (n < 1) | |
791 | return -EINVAL; | |
792 | state = buf[0]-'0'; | |
793 | ||
794 | down_read(&pci_bus_sem); | |
795 | mutex_lock(&aspm_lock); | |
796 | pcie_set_clock_pm(pci_device, !!state); | |
797 | mutex_unlock(&aspm_lock); | |
798 | up_read(&pci_bus_sem); | |
799 | ||
800 | return n; | |
801 | } | |
802 | ||
803 | static DEVICE_ATTR(link_state, 0644, link_state_show, link_state_store); | |
804 | static DEVICE_ATTR(clk_ctl, 0644, clk_ctl_show, clk_ctl_store); | |
805 | ||
806 | static char power_group[] = "power"; | |
807 | void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev) | |
808 | { | |
809 | struct pcie_link_state *link_state = pdev->link_state; | |
810 | ||
811 | if (!pdev->is_pcie || (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT && | |
812 | pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state) | |
813 | return; | |
814 | ||
815 | if (link_state->support_state) | |
816 | sysfs_add_file_to_group(&pdev->dev.kobj, | |
817 | &dev_attr_link_state.attr, power_group); | |
818 | if (link_state->clk_pm_capable) | |
819 | sysfs_add_file_to_group(&pdev->dev.kobj, | |
820 | &dev_attr_clk_ctl.attr, power_group); | |
821 | } | |
822 | ||
823 | void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev) | |
824 | { | |
825 | struct pcie_link_state *link_state = pdev->link_state; | |
826 | ||
827 | if (!pdev->is_pcie || (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT && | |
828 | pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state) | |
829 | return; | |
830 | ||
831 | if (link_state->support_state) | |
832 | sysfs_remove_file_from_group(&pdev->dev.kobj, | |
833 | &dev_attr_link_state.attr, power_group); | |
834 | if (link_state->clk_pm_capable) | |
835 | sysfs_remove_file_from_group(&pdev->dev.kobj, | |
836 | &dev_attr_clk_ctl.attr, power_group); | |
837 | } | |
838 | #endif | |
839 | ||
840 | static int __init pcie_aspm_disable(char *str) | |
841 | { | |
d6d38574 SL |
842 | if (!strcmp(str, "off")) { |
843 | aspm_disabled = 1; | |
844 | printk(KERN_INFO "PCIe ASPM is disabled\n"); | |
845 | } else if (!strcmp(str, "force")) { | |
846 | aspm_force = 1; | |
847 | printk(KERN_INFO "PCIe ASPM is forcedly enabled\n"); | |
848 | } | |
7d715a6c SL |
849 | return 1; |
850 | } | |
851 | ||
d6d38574 | 852 | __setup("pcie_aspm=", pcie_aspm_disable); |
7d715a6c | 853 | |
5fde244d SL |
854 | void pcie_no_aspm(void) |
855 | { | |
d6d38574 SL |
856 | if (!aspm_force) |
857 | aspm_disabled = 1; | |
5fde244d SL |
858 | } |
859 | ||
7d715a6c SL |
860 | #ifdef CONFIG_ACPI |
861 | #include <acpi/acpi_bus.h> | |
862 | #include <linux/pci-acpi.h> | |
863 | static void pcie_aspm_platform_init(void) | |
864 | { | |
865 | pcie_osc_support_set(OSC_ACTIVE_STATE_PWR_SUPPORT| | |
866 | OSC_CLOCK_PWR_CAPABILITY_SUPPORT); | |
867 | } | |
868 | #else | |
869 | static inline void pcie_aspm_platform_init(void) { } | |
870 | #endif | |
871 | ||
872 | static int __init pcie_aspm_init(void) | |
873 | { | |
874 | if (aspm_disabled) | |
875 | return 0; | |
876 | pcie_aspm_platform_init(); | |
877 | return 0; | |
878 | } | |
879 | ||
880 | fs_initcall(pcie_aspm_init); |