Commit | Line | Data |
---|---|---|
8cfab3cf | 1 | // SPDX-License-Identifier: GPL-2.0 |
425c1b22 AW |
2 | /* |
3 | * PCI Virtual Channel support | |
4 | * | |
5 | * Copyright (C) 2013 Red Hat, Inc. All rights reserved. | |
6 | * Author: Alex Williamson <alex.williamson@redhat.com> | |
425c1b22 AW |
7 | */ |
8 | ||
9 | #include <linux/device.h> | |
10 | #include <linux/kernel.h> | |
11 | #include <linux/module.h> | |
12 | #include <linux/pci.h> | |
13 | #include <linux/pci_regs.h> | |
14 | #include <linux/types.h> | |
15 | ||
16 | /** | |
17 | * pci_vc_save_restore_dwords - Save or restore a series of dwords | |
18 | * @dev: device | |
19 | * @pos: starting config space position | |
20 | * @buf: buffer to save to or restore from | |
21 | * @dwords: number of dwords to save/restore | |
22 | * @save: whether to save or restore | |
23 | */ | |
24 | static void pci_vc_save_restore_dwords(struct pci_dev *dev, int pos, | |
25 | u32 *buf, int dwords, bool save) | |
26 | { | |
27 | int i; | |
28 | ||
29 | for (i = 0; i < dwords; i++, buf++) { | |
30 | if (save) | |
31 | pci_read_config_dword(dev, pos + (i * 4), buf); | |
32 | else | |
33 | pci_write_config_dword(dev, pos + (i * 4), *buf); | |
34 | } | |
35 | } | |
36 | ||
37 | /** | |
38 | * pci_vc_load_arb_table - load and wait for VC arbitration table | |
39 | * @dev: device | |
40 | * @pos: starting position of VC capability (VC/VC9/MFVC) | |
41 | * | |
42 | * Set Load VC Arbitration Table bit requesting hardware to apply the VC | |
43 | * Arbitration Table (previously loaded). When the VC Arbitration Table | |
44 | * Status clears, hardware has latched the table into VC arbitration logic. | |
45 | */ | |
46 | static void pci_vc_load_arb_table(struct pci_dev *dev, int pos) | |
47 | { | |
48 | u16 ctrl; | |
49 | ||
50 | pci_read_config_word(dev, pos + PCI_VC_PORT_CTRL, &ctrl); | |
51 | pci_write_config_word(dev, pos + PCI_VC_PORT_CTRL, | |
52 | ctrl | PCI_VC_PORT_CTRL_LOAD_TABLE); | |
53 | if (pci_wait_for_pending(dev, pos + PCI_VC_PORT_STATUS, | |
54 | PCI_VC_PORT_STATUS_TABLE)) | |
55 | return; | |
56 | ||
7506dc79 | 57 | pci_err(dev, "VC arbitration table failed to load\n"); |
425c1b22 AW |
58 | } |
59 | ||
60 | /** | |
61 | * pci_vc_load_port_arb_table - Load and wait for VC port arbitration table | |
62 | * @dev: device | |
63 | * @pos: starting position of VC capability (VC/VC9/MFVC) | |
64 | * @res: VC resource number, ie. VCn (0-7) | |
65 | * | |
66 | * Set Load Port Arbitration Table bit requesting hardware to apply the Port | |
67 | * Arbitration Table (previously loaded). When the Port Arbitration Table | |
68 | * Status clears, hardware has latched the table into port arbitration logic. | |
69 | */ | |
70 | static void pci_vc_load_port_arb_table(struct pci_dev *dev, int pos, int res) | |
71 | { | |
72 | int ctrl_pos, status_pos; | |
73 | u32 ctrl; | |
74 | ||
75 | ctrl_pos = pos + PCI_VC_RES_CTRL + (res * PCI_CAP_VC_PER_VC_SIZEOF); | |
76 | status_pos = pos + PCI_VC_RES_STATUS + (res * PCI_CAP_VC_PER_VC_SIZEOF); | |
77 | ||
78 | pci_read_config_dword(dev, ctrl_pos, &ctrl); | |
79 | pci_write_config_dword(dev, ctrl_pos, | |
80 | ctrl | PCI_VC_RES_CTRL_LOAD_TABLE); | |
81 | ||
82 | if (pci_wait_for_pending(dev, status_pos, PCI_VC_RES_STATUS_TABLE)) | |
83 | return; | |
84 | ||
7506dc79 | 85 | pci_err(dev, "VC%d port arbitration table failed to load\n", res); |
425c1b22 AW |
86 | } |
87 | ||
88 | /** | |
89 | * pci_vc_enable - Enable virtual channel | |
90 | * @dev: device | |
91 | * @pos: starting position of VC capability (VC/VC9/MFVC) | |
92 | * @res: VC res number, ie. VCn (0-7) | |
93 | * | |
94 | * A VC is enabled by setting the enable bit in matching resource control | |
95 | * registers on both sides of a link. We therefore need to find the opposite | |
96 | * end of the link. To keep this simple we enable from the downstream device. | |
97 | * RC devices do not have an upstream device, nor does it seem that VC9 do | |
98 | * (spec is unclear). Once we find the upstream device, match the VC ID to | |
99 | * get the correct resource, disable and enable on both ends. | |
100 | */ | |
101 | static void pci_vc_enable(struct pci_dev *dev, int pos, int res) | |
102 | { | |
103 | int ctrl_pos, status_pos, id, pos2, evcc, i, ctrl_pos2, status_pos2; | |
274127a1 | 104 | u32 ctrl, header, cap1, ctrl2; |
425c1b22 AW |
105 | struct pci_dev *link = NULL; |
106 | ||
107 | /* Enable VCs from the downstream device */ | |
777e61ea | 108 | if (!dev->has_secondary_link) |
425c1b22 AW |
109 | return; |
110 | ||
111 | ctrl_pos = pos + PCI_VC_RES_CTRL + (res * PCI_CAP_VC_PER_VC_SIZEOF); | |
112 | status_pos = pos + PCI_VC_RES_STATUS + (res * PCI_CAP_VC_PER_VC_SIZEOF); | |
113 | ||
114 | pci_read_config_dword(dev, ctrl_pos, &ctrl); | |
115 | id = ctrl & PCI_VC_RES_CTRL_ID; | |
116 | ||
117 | pci_read_config_dword(dev, pos, &header); | |
118 | ||
119 | /* If there is no opposite end of the link, skip to enable */ | |
120 | if (PCI_EXT_CAP_ID(header) == PCI_EXT_CAP_ID_VC9 || | |
121 | pci_is_root_bus(dev->bus)) | |
122 | goto enable; | |
123 | ||
124 | pos2 = pci_find_ext_capability(dev->bus->self, PCI_EXT_CAP_ID_VC); | |
125 | if (!pos2) | |
126 | goto enable; | |
127 | ||
274127a1 AW |
128 | pci_read_config_dword(dev->bus->self, pos2 + PCI_VC_PORT_CAP1, &cap1); |
129 | evcc = cap1 & PCI_VC_CAP1_EVCC; | |
425c1b22 AW |
130 | |
131 | /* VC0 is hardwired enabled, so we can start with 1 */ | |
132 | for (i = 1; i < evcc + 1; i++) { | |
133 | ctrl_pos2 = pos2 + PCI_VC_RES_CTRL + | |
134 | (i * PCI_CAP_VC_PER_VC_SIZEOF); | |
135 | status_pos2 = pos2 + PCI_VC_RES_STATUS + | |
136 | (i * PCI_CAP_VC_PER_VC_SIZEOF); | |
137 | pci_read_config_dword(dev->bus->self, ctrl_pos2, &ctrl2); | |
138 | if ((ctrl2 & PCI_VC_RES_CTRL_ID) == id) { | |
139 | link = dev->bus->self; | |
140 | break; | |
141 | } | |
142 | } | |
143 | ||
144 | if (!link) | |
145 | goto enable; | |
146 | ||
147 | /* Disable if enabled */ | |
148 | if (ctrl2 & PCI_VC_RES_CTRL_ENABLE) { | |
149 | ctrl2 &= ~PCI_VC_RES_CTRL_ENABLE; | |
150 | pci_write_config_dword(link, ctrl_pos2, ctrl2); | |
151 | } | |
152 | ||
153 | /* Enable on both ends */ | |
154 | ctrl2 |= PCI_VC_RES_CTRL_ENABLE; | |
155 | pci_write_config_dword(link, ctrl_pos2, ctrl2); | |
156 | enable: | |
157 | ctrl |= PCI_VC_RES_CTRL_ENABLE; | |
158 | pci_write_config_dword(dev, ctrl_pos, ctrl); | |
159 | ||
160 | if (!pci_wait_for_pending(dev, status_pos, PCI_VC_RES_STATUS_NEGO)) | |
7506dc79 | 161 | pci_err(dev, "VC%d negotiation stuck pending\n", id); |
425c1b22 AW |
162 | |
163 | if (link && !pci_wait_for_pending(link, status_pos2, | |
164 | PCI_VC_RES_STATUS_NEGO)) | |
7506dc79 | 165 | pci_err(link, "VC%d negotiation stuck pending\n", id); |
425c1b22 AW |
166 | } |
167 | ||
168 | /** | |
169 | * pci_vc_do_save_buffer - Size, save, or restore VC state | |
170 | * @dev: device | |
171 | * @pos: starting position of VC capability (VC/VC9/MFVC) | |
172 | * @save_state: buffer for save/restore | |
173 | * @name: for error message | |
174 | * @save: if provided a buffer, this indicates what to do with it | |
175 | * | |
176 | * Walking Virtual Channel config space to size, save, or restore it | |
177 | * is complicated, so we do it all from one function to reduce code and | |
178 | * guarantee ordering matches in the buffer. When called with NULL | |
179 | * @save_state, return the size of the necessary save buffer. When called | |
180 | * with a non-NULL @save_state, @save determines whether we save to the | |
181 | * buffer or restore from it. | |
182 | */ | |
183 | static int pci_vc_do_save_buffer(struct pci_dev *dev, int pos, | |
184 | struct pci_cap_saved_state *save_state, | |
185 | bool save) | |
186 | { | |
274127a1 | 187 | u32 cap1; |
425c1b22 AW |
188 | char evcc, lpevcc, parb_size; |
189 | int i, len = 0; | |
190 | u8 *buf = save_state ? (u8 *)save_state->cap.data : NULL; | |
191 | ||
192 | /* Sanity check buffer size for save/restore */ | |
193 | if (buf && save_state->cap.size != | |
194 | pci_vc_do_save_buffer(dev, pos, NULL, save)) { | |
7506dc79 | 195 | pci_err(dev, "VC save buffer size does not match @0x%x\n", pos); |
425c1b22 AW |
196 | return -ENOMEM; |
197 | } | |
198 | ||
274127a1 | 199 | pci_read_config_dword(dev, pos + PCI_VC_PORT_CAP1, &cap1); |
425c1b22 | 200 | /* Extended VC Count (not counting VC0) */ |
274127a1 | 201 | evcc = cap1 & PCI_VC_CAP1_EVCC; |
425c1b22 | 202 | /* Low Priority Extended VC Count (not counting VC0) */ |
274127a1 | 203 | lpevcc = (cap1 & PCI_VC_CAP1_LPEVCC) >> 4; |
425c1b22 | 204 | /* Port Arbitration Table Entry Size (bits) */ |
274127a1 | 205 | parb_size = 1 << ((cap1 & PCI_VC_CAP1_ARB_SIZE) >> 10); |
425c1b22 AW |
206 | |
207 | /* | |
208 | * Port VC Control Register contains VC Arbitration Select, which | |
209 | * cannot be modified when more than one LPVC is in operation. We | |
210 | * therefore save/restore it first, as only VC0 should be enabled | |
211 | * after device reset. | |
212 | */ | |
213 | if (buf) { | |
214 | if (save) | |
215 | pci_read_config_word(dev, pos + PCI_VC_PORT_CTRL, | |
216 | (u16 *)buf); | |
217 | else | |
218 | pci_write_config_word(dev, pos + PCI_VC_PORT_CTRL, | |
219 | *(u16 *)buf); | |
ef0dab4a | 220 | buf += 4; |
425c1b22 | 221 | } |
ef0dab4a | 222 | len += 4; |
425c1b22 AW |
223 | |
224 | /* | |
225 | * If we have any Low Priority VCs and a VC Arbitration Table Offset | |
226 | * in Port VC Capability Register 2 then save/restore it next. | |
227 | */ | |
228 | if (lpevcc) { | |
274127a1 | 229 | u32 cap2; |
425c1b22 AW |
230 | int vcarb_offset; |
231 | ||
274127a1 AW |
232 | pci_read_config_dword(dev, pos + PCI_VC_PORT_CAP2, &cap2); |
233 | vcarb_offset = ((cap2 & PCI_VC_CAP2_ARB_OFF) >> 24) * 16; | |
425c1b22 AW |
234 | |
235 | if (vcarb_offset) { | |
236 | int size, vcarb_phases = 0; | |
237 | ||
274127a1 | 238 | if (cap2 & PCI_VC_CAP2_128_PHASE) |
425c1b22 | 239 | vcarb_phases = 128; |
274127a1 | 240 | else if (cap2 & PCI_VC_CAP2_64_PHASE) |
425c1b22 | 241 | vcarb_phases = 64; |
274127a1 | 242 | else if (cap2 & PCI_VC_CAP2_32_PHASE) |
425c1b22 AW |
243 | vcarb_phases = 32; |
244 | ||
245 | /* Fixed 4 bits per phase per lpevcc (plus VC0) */ | |
246 | size = ((lpevcc + 1) * vcarb_phases * 4) / 8; | |
247 | ||
248 | if (size && buf) { | |
249 | pci_vc_save_restore_dwords(dev, | |
250 | pos + vcarb_offset, | |
251 | (u32 *)buf, | |
252 | size / 4, save); | |
253 | /* | |
254 | * On restore, we need to signal hardware to | |
255 | * re-load the VC Arbitration Table. | |
256 | */ | |
257 | if (!save) | |
258 | pci_vc_load_arb_table(dev, pos); | |
259 | ||
260 | buf += size; | |
261 | } | |
262 | len += size; | |
263 | } | |
264 | } | |
265 | ||
266 | /* | |
267 | * In addition to each VC Resource Control Register, we may have a | |
268 | * Port Arbitration Table attached to each VC. The Port Arbitration | |
269 | * Table Offset in each VC Resource Capability Register tells us if | |
270 | * it exists. The entry size is global from the Port VC Capability | |
271 | * Register1 above. The number of phases is determined per VC. | |
272 | */ | |
273 | for (i = 0; i < evcc + 1; i++) { | |
274 | u32 cap; | |
275 | int parb_offset; | |
276 | ||
277 | pci_read_config_dword(dev, pos + PCI_VC_RES_CAP + | |
278 | (i * PCI_CAP_VC_PER_VC_SIZEOF), &cap); | |
279 | parb_offset = ((cap & PCI_VC_RES_CAP_ARB_OFF) >> 24) * 16; | |
280 | if (parb_offset) { | |
281 | int size, parb_phases = 0; | |
282 | ||
283 | if (cap & PCI_VC_RES_CAP_256_PHASE) | |
284 | parb_phases = 256; | |
285 | else if (cap & (PCI_VC_RES_CAP_128_PHASE | | |
286 | PCI_VC_RES_CAP_128_PHASE_TB)) | |
287 | parb_phases = 128; | |
288 | else if (cap & PCI_VC_RES_CAP_64_PHASE) | |
289 | parb_phases = 64; | |
290 | else if (cap & PCI_VC_RES_CAP_32_PHASE) | |
291 | parb_phases = 32; | |
292 | ||
293 | size = (parb_size * parb_phases) / 8; | |
294 | ||
295 | if (size && buf) { | |
296 | pci_vc_save_restore_dwords(dev, | |
297 | pos + parb_offset, | |
298 | (u32 *)buf, | |
299 | size / 4, save); | |
300 | buf += size; | |
301 | } | |
302 | len += size; | |
303 | } | |
304 | ||
305 | /* VC Resource Control Register */ | |
306 | if (buf) { | |
307 | int ctrl_pos = pos + PCI_VC_RES_CTRL + | |
308 | (i * PCI_CAP_VC_PER_VC_SIZEOF); | |
309 | if (save) | |
310 | pci_read_config_dword(dev, ctrl_pos, | |
311 | (u32 *)buf); | |
312 | else { | |
313 | u32 tmp, ctrl = *(u32 *)buf; | |
314 | /* | |
315 | * For an FLR case, the VC config may remain. | |
316 | * Preserve enable bit, restore the rest. | |
317 | */ | |
318 | pci_read_config_dword(dev, ctrl_pos, &tmp); | |
319 | tmp &= PCI_VC_RES_CTRL_ENABLE; | |
320 | tmp |= ctrl & ~PCI_VC_RES_CTRL_ENABLE; | |
321 | pci_write_config_dword(dev, ctrl_pos, tmp); | |
322 | /* Load port arbitration table if used */ | |
323 | if (ctrl & PCI_VC_RES_CTRL_ARB_SELECT) | |
324 | pci_vc_load_port_arb_table(dev, pos, i); | |
325 | /* Re-enable if needed */ | |
326 | if ((ctrl ^ tmp) & PCI_VC_RES_CTRL_ENABLE) | |
327 | pci_vc_enable(dev, pos, i); | |
328 | } | |
329 | buf += 4; | |
330 | } | |
331 | len += 4; | |
332 | } | |
333 | ||
334 | return buf ? 0 : len; | |
335 | } | |
336 | ||
337 | static struct { | |
338 | u16 id; | |
339 | const char *name; | |
340 | } vc_caps[] = { { PCI_EXT_CAP_ID_MFVC, "MFVC" }, | |
341 | { PCI_EXT_CAP_ID_VC, "VC" }, | |
342 | { PCI_EXT_CAP_ID_VC9, "VC9" } }; | |
343 | ||
344 | /** | |
345 | * pci_save_vc_state - Save VC state to pre-allocate save buffer | |
346 | * @dev: device | |
347 | * | |
348 | * For each type of VC capability, VC/VC9/MFVC, find the capability and | |
349 | * save it to the pre-allocated save buffer. | |
350 | */ | |
351 | int pci_save_vc_state(struct pci_dev *dev) | |
352 | { | |
353 | int i; | |
354 | ||
355 | for (i = 0; i < ARRAY_SIZE(vc_caps); i++) { | |
356 | int pos, ret; | |
357 | struct pci_cap_saved_state *save_state; | |
358 | ||
359 | pos = pci_find_ext_capability(dev, vc_caps[i].id); | |
360 | if (!pos) | |
361 | continue; | |
362 | ||
363 | save_state = pci_find_saved_ext_cap(dev, vc_caps[i].id); | |
364 | if (!save_state) { | |
7506dc79 | 365 | pci_err(dev, "%s buffer not found in %s\n", |
425c1b22 AW |
366 | vc_caps[i].name, __func__); |
367 | return -ENOMEM; | |
368 | } | |
369 | ||
370 | ret = pci_vc_do_save_buffer(dev, pos, save_state, true); | |
371 | if (ret) { | |
7506dc79 | 372 | pci_err(dev, "%s save unsuccessful %s\n", |
425c1b22 AW |
373 | vc_caps[i].name, __func__); |
374 | return ret; | |
375 | } | |
376 | } | |
377 | ||
378 | return 0; | |
379 | } | |
380 | ||
381 | /** | |
382 | * pci_restore_vc_state - Restore VC state from save buffer | |
383 | * @dev: device | |
384 | * | |
385 | * For each type of VC capability, VC/VC9/MFVC, find the capability and | |
386 | * restore it from the previously saved buffer. | |
387 | */ | |
388 | void pci_restore_vc_state(struct pci_dev *dev) | |
389 | { | |
390 | int i; | |
391 | ||
392 | for (i = 0; i < ARRAY_SIZE(vc_caps); i++) { | |
393 | int pos; | |
394 | struct pci_cap_saved_state *save_state; | |
395 | ||
396 | pos = pci_find_ext_capability(dev, vc_caps[i].id); | |
397 | save_state = pci_find_saved_ext_cap(dev, vc_caps[i].id); | |
398 | if (!save_state || !pos) | |
399 | continue; | |
400 | ||
401 | pci_vc_do_save_buffer(dev, pos, save_state, false); | |
402 | } | |
403 | } | |
404 | ||
405 | /** | |
406 | * pci_allocate_vc_save_buffers - Allocate save buffers for VC caps | |
407 | * @dev: device | |
408 | * | |
409 | * For each type of VC capability, VC/VC9/MFVC, find the capability, size | |
410 | * it, and allocate a buffer for save/restore. | |
411 | */ | |
412 | ||
413 | void pci_allocate_vc_save_buffers(struct pci_dev *dev) | |
414 | { | |
415 | int i; | |
416 | ||
417 | for (i = 0; i < ARRAY_SIZE(vc_caps); i++) { | |
418 | int len, pos = pci_find_ext_capability(dev, vc_caps[i].id); | |
419 | ||
420 | if (!pos) | |
421 | continue; | |
422 | ||
423 | len = pci_vc_do_save_buffer(dev, pos, NULL, false); | |
424 | if (pci_add_ext_cap_save_buffer(dev, vc_caps[i].id, len)) | |
7506dc79 | 425 | pci_err(dev, "unable to preallocate %s save buffer\n", |
425c1b22 AW |
426 | vc_caps[i].name); |
427 | } | |
428 | } |