Commit | Line | Data |
---|---|---|
5fd54ace | 1 | // SPDX-License-Identifier: GPL-2.0 |
aeb9dd1d LB |
2 | /** |
3 | * xhci-dbc.c - xHCI debug capability early driver | |
4 | * | |
5 | * Copyright (C) 2016 Intel Corporation | |
6 | * | |
7 | * Author: Lu Baolu <baolu.lu@linux.intel.com> | |
aeb9dd1d LB |
8 | */ |
9 | ||
10 | #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ | |
11 | ||
12 | #include <linux/console.h> | |
13 | #include <linux/pci_regs.h> | |
14 | #include <linux/pci_ids.h> | |
2013288f | 15 | #include <linux/memblock.h> |
aeb9dd1d LB |
16 | #include <linux/io.h> |
17 | #include <asm/pci-direct.h> | |
18 | #include <asm/fixmap.h> | |
19 | #include <linux/bcd.h> | |
20 | #include <linux/export.h> | |
21 | #include <linux/version.h> | |
22 | #include <linux/module.h> | |
23 | #include <linux/delay.h> | |
24 | #include <linux/kthread.h> | |
25 | ||
26 | #include "../host/xhci.h" | |
27 | #include "xhci-dbc.h" | |
28 | ||
29 | static struct xdbc_state xdbc; | |
30 | static bool early_console_keep; | |
31 | ||
aeb9dd1d LB |
32 | #ifdef XDBC_TRACE |
33 | #define xdbc_trace trace_printk | |
34 | #else | |
35 | static inline void xdbc_trace(const char *fmt, ...) { } | |
36 | #endif /* XDBC_TRACE */ | |
37 | ||
38 | static void __iomem * __init xdbc_map_pci_mmio(u32 bus, u32 dev, u32 func) | |
39 | { | |
40 | u64 val64, sz64, mask64; | |
41 | void __iomem *base; | |
42 | u32 val, sz; | |
43 | u8 byte; | |
44 | ||
45 | val = read_pci_config(bus, dev, func, PCI_BASE_ADDRESS_0); | |
46 | write_pci_config(bus, dev, func, PCI_BASE_ADDRESS_0, ~0); | |
47 | sz = read_pci_config(bus, dev, func, PCI_BASE_ADDRESS_0); | |
48 | write_pci_config(bus, dev, func, PCI_BASE_ADDRESS_0, val); | |
49 | ||
50 | if (val == 0xffffffff || sz == 0xffffffff) { | |
51 | pr_notice("invalid mmio bar\n"); | |
52 | return NULL; | |
53 | } | |
54 | ||
55 | val64 = val & PCI_BASE_ADDRESS_MEM_MASK; | |
56 | sz64 = sz & PCI_BASE_ADDRESS_MEM_MASK; | |
57 | mask64 = PCI_BASE_ADDRESS_MEM_MASK; | |
58 | ||
59 | if ((val & PCI_BASE_ADDRESS_MEM_TYPE_MASK) == PCI_BASE_ADDRESS_MEM_TYPE_64) { | |
60 | val = read_pci_config(bus, dev, func, PCI_BASE_ADDRESS_0 + 4); | |
61 | write_pci_config(bus, dev, func, PCI_BASE_ADDRESS_0 + 4, ~0); | |
62 | sz = read_pci_config(bus, dev, func, PCI_BASE_ADDRESS_0 + 4); | |
63 | write_pci_config(bus, dev, func, PCI_BASE_ADDRESS_0 + 4, val); | |
64 | ||
65 | val64 |= (u64)val << 32; | |
66 | sz64 |= (u64)sz << 32; | |
67 | mask64 |= ~0ULL << 32; | |
68 | } | |
69 | ||
70 | sz64 &= mask64; | |
71 | ||
72 | if (!sz64) { | |
73 | pr_notice("invalid mmio address\n"); | |
74 | return NULL; | |
75 | } | |
76 | ||
77 | sz64 = 1ULL << __ffs64(sz64); | |
78 | ||
79 | /* Check if the mem space is enabled: */ | |
80 | byte = read_pci_config_byte(bus, dev, func, PCI_COMMAND); | |
81 | if (!(byte & PCI_COMMAND_MEMORY)) { | |
82 | byte |= PCI_COMMAND_MEMORY; | |
83 | write_pci_config_byte(bus, dev, func, PCI_COMMAND, byte); | |
84 | } | |
85 | ||
86 | xdbc.xhci_start = val64; | |
87 | xdbc.xhci_length = sz64; | |
88 | base = early_ioremap(val64, sz64); | |
89 | ||
90 | return base; | |
91 | } | |
92 | ||
93 | static void * __init xdbc_get_page(dma_addr_t *dma_addr) | |
94 | { | |
95 | void *virt; | |
96 | ||
238997e5 | 97 | virt = memblock_alloc_nopanic(PAGE_SIZE, PAGE_SIZE); |
aeb9dd1d LB |
98 | if (!virt) |
99 | return NULL; | |
100 | ||
101 | if (dma_addr) | |
102 | *dma_addr = (dma_addr_t)__pa(virt); | |
103 | ||
104 | return virt; | |
105 | } | |
106 | ||
107 | static u32 __init xdbc_find_dbgp(int xdbc_num, u32 *b, u32 *d, u32 *f) | |
108 | { | |
109 | u32 bus, dev, func, class; | |
110 | ||
111 | for (bus = 0; bus < XDBC_PCI_MAX_BUSES; bus++) { | |
112 | for (dev = 0; dev < XDBC_PCI_MAX_DEVICES; dev++) { | |
113 | for (func = 0; func < XDBC_PCI_MAX_FUNCTION; func++) { | |
114 | ||
115 | class = read_pci_config(bus, dev, func, PCI_CLASS_REVISION); | |
116 | if ((class >> 8) != PCI_CLASS_SERIAL_USB_XHCI) | |
117 | continue; | |
118 | ||
119 | if (xdbc_num-- != 0) | |
120 | continue; | |
121 | ||
122 | *b = bus; | |
123 | *d = dev; | |
124 | *f = func; | |
125 | ||
126 | return 0; | |
127 | } | |
128 | } | |
129 | } | |
130 | ||
131 | return -1; | |
132 | } | |
133 | ||
134 | static int handshake(void __iomem *ptr, u32 mask, u32 done, int wait, int delay) | |
135 | { | |
136 | u32 result; | |
137 | ||
138 | do { | |
139 | result = readl(ptr); | |
140 | result &= mask; | |
141 | if (result == done) | |
142 | return 0; | |
143 | udelay(delay); | |
144 | wait -= delay; | |
145 | } while (wait > 0); | |
146 | ||
147 | return -ETIMEDOUT; | |
148 | } | |
149 | ||
150 | static void __init xdbc_bios_handoff(void) | |
151 | { | |
152 | int offset, timeout; | |
153 | u32 val; | |
154 | ||
155 | offset = xhci_find_next_ext_cap(xdbc.xhci_base, 0, XHCI_EXT_CAPS_LEGACY); | |
156 | val = readl(xdbc.xhci_base + offset); | |
157 | ||
158 | if (val & XHCI_HC_BIOS_OWNED) { | |
159 | writel(val | XHCI_HC_OS_OWNED, xdbc.xhci_base + offset); | |
160 | timeout = handshake(xdbc.xhci_base + offset, XHCI_HC_BIOS_OWNED, 0, 5000, 10); | |
161 | ||
162 | if (timeout) { | |
163 | pr_notice("failed to hand over xHCI control from BIOS\n"); | |
164 | writel(val & ~XHCI_HC_BIOS_OWNED, xdbc.xhci_base + offset); | |
165 | } | |
166 | } | |
167 | ||
168 | /* Disable BIOS SMIs and clear all SMI events: */ | |
169 | val = readl(xdbc.xhci_base + offset + XHCI_LEGACY_CONTROL_OFFSET); | |
170 | val &= XHCI_LEGACY_DISABLE_SMI; | |
171 | val |= XHCI_LEGACY_SMI_EVENTS; | |
172 | writel(val, xdbc.xhci_base + offset + XHCI_LEGACY_CONTROL_OFFSET); | |
173 | } | |
174 | ||
175 | static int __init | |
176 | xdbc_alloc_ring(struct xdbc_segment *seg, struct xdbc_ring *ring) | |
177 | { | |
178 | seg->trbs = xdbc_get_page(&seg->dma); | |
179 | if (!seg->trbs) | |
180 | return -ENOMEM; | |
181 | ||
182 | ring->segment = seg; | |
183 | ||
184 | return 0; | |
185 | } | |
186 | ||
187 | static void __init xdbc_free_ring(struct xdbc_ring *ring) | |
188 | { | |
189 | struct xdbc_segment *seg = ring->segment; | |
190 | ||
191 | if (!seg) | |
192 | return; | |
193 | ||
2013288f | 194 | memblock_free(seg->dma, PAGE_SIZE); |
aeb9dd1d LB |
195 | ring->segment = NULL; |
196 | } | |
197 | ||
198 | static void xdbc_reset_ring(struct xdbc_ring *ring) | |
199 | { | |
200 | struct xdbc_segment *seg = ring->segment; | |
201 | struct xdbc_trb *link_trb; | |
202 | ||
203 | memset(seg->trbs, 0, PAGE_SIZE); | |
204 | ||
205 | ring->enqueue = seg->trbs; | |
206 | ring->dequeue = seg->trbs; | |
207 | ring->cycle_state = 1; | |
208 | ||
209 | if (ring != &xdbc.evt_ring) { | |
210 | link_trb = &seg->trbs[XDBC_TRBS_PER_SEGMENT - 1]; | |
211 | link_trb->field[0] = cpu_to_le32(lower_32_bits(seg->dma)); | |
212 | link_trb->field[1] = cpu_to_le32(upper_32_bits(seg->dma)); | |
213 | link_trb->field[3] = cpu_to_le32(TRB_TYPE(TRB_LINK)) | cpu_to_le32(LINK_TOGGLE); | |
214 | } | |
215 | } | |
216 | ||
217 | static inline void xdbc_put_utf16(u16 *s, const char *c, size_t size) | |
218 | { | |
219 | int i; | |
220 | ||
221 | for (i = 0; i < size; i++) | |
222 | s[i] = cpu_to_le16(c[i]); | |
223 | } | |
224 | ||
225 | static void xdbc_mem_init(void) | |
226 | { | |
227 | struct xdbc_ep_context *ep_in, *ep_out; | |
228 | struct usb_string_descriptor *s_desc; | |
229 | struct xdbc_erst_entry *entry; | |
230 | struct xdbc_strings *strings; | |
231 | struct xdbc_context *ctx; | |
232 | unsigned int max_burst; | |
233 | u32 string_length; | |
234 | int index = 0; | |
235 | u32 dev_info; | |
236 | ||
237 | xdbc_reset_ring(&xdbc.evt_ring); | |
238 | xdbc_reset_ring(&xdbc.in_ring); | |
239 | xdbc_reset_ring(&xdbc.out_ring); | |
240 | memset(xdbc.table_base, 0, PAGE_SIZE); | |
241 | memset(xdbc.out_buf, 0, PAGE_SIZE); | |
242 | ||
243 | /* Initialize event ring segment table: */ | |
244 | xdbc.erst_size = 16; | |
245 | xdbc.erst_base = xdbc.table_base + index * XDBC_TABLE_ENTRY_SIZE; | |
246 | xdbc.erst_dma = xdbc.table_dma + index * XDBC_TABLE_ENTRY_SIZE; | |
247 | ||
248 | index += XDBC_ERST_ENTRY_NUM; | |
249 | entry = (struct xdbc_erst_entry *)xdbc.erst_base; | |
250 | ||
251 | entry->seg_addr = cpu_to_le64(xdbc.evt_seg.dma); | |
252 | entry->seg_size = cpu_to_le32(XDBC_TRBS_PER_SEGMENT); | |
253 | entry->__reserved_0 = 0; | |
254 | ||
255 | /* Initialize ERST registers: */ | |
256 | writel(1, &xdbc.xdbc_reg->ersts); | |
257 | xdbc_write64(xdbc.erst_dma, &xdbc.xdbc_reg->erstba); | |
258 | xdbc_write64(xdbc.evt_seg.dma, &xdbc.xdbc_reg->erdp); | |
259 | ||
260 | /* Debug capability contexts: */ | |
261 | xdbc.dbcc_size = 64 * 3; | |
262 | xdbc.dbcc_base = xdbc.table_base + index * XDBC_TABLE_ENTRY_SIZE; | |
263 | xdbc.dbcc_dma = xdbc.table_dma + index * XDBC_TABLE_ENTRY_SIZE; | |
264 | ||
265 | index += XDBC_DBCC_ENTRY_NUM; | |
266 | ||
267 | /* Popluate the strings: */ | |
268 | xdbc.string_size = sizeof(struct xdbc_strings); | |
269 | xdbc.string_base = xdbc.table_base + index * XDBC_TABLE_ENTRY_SIZE; | |
270 | xdbc.string_dma = xdbc.table_dma + index * XDBC_TABLE_ENTRY_SIZE; | |
271 | strings = (struct xdbc_strings *)xdbc.string_base; | |
272 | ||
273 | index += XDBC_STRING_ENTRY_NUM; | |
274 | ||
275 | /* Serial string: */ | |
276 | s_desc = (struct usb_string_descriptor *)strings->serial; | |
277 | s_desc->bLength = (strlen(XDBC_STRING_SERIAL) + 1) * 2; | |
278 | s_desc->bDescriptorType = USB_DT_STRING; | |
279 | ||
280 | xdbc_put_utf16(s_desc->wData, XDBC_STRING_SERIAL, strlen(XDBC_STRING_SERIAL)); | |
281 | string_length = s_desc->bLength; | |
282 | string_length <<= 8; | |
283 | ||
284 | /* Product string: */ | |
285 | s_desc = (struct usb_string_descriptor *)strings->product; | |
286 | s_desc->bLength = (strlen(XDBC_STRING_PRODUCT) + 1) * 2; | |
287 | s_desc->bDescriptorType = USB_DT_STRING; | |
288 | ||
289 | xdbc_put_utf16(s_desc->wData, XDBC_STRING_PRODUCT, strlen(XDBC_STRING_PRODUCT)); | |
290 | string_length += s_desc->bLength; | |
291 | string_length <<= 8; | |
292 | ||
293 | /* Manufacture string: */ | |
294 | s_desc = (struct usb_string_descriptor *)strings->manufacturer; | |
295 | s_desc->bLength = (strlen(XDBC_STRING_MANUFACTURER) + 1) * 2; | |
296 | s_desc->bDescriptorType = USB_DT_STRING; | |
297 | ||
298 | xdbc_put_utf16(s_desc->wData, XDBC_STRING_MANUFACTURER, strlen(XDBC_STRING_MANUFACTURER)); | |
299 | string_length += s_desc->bLength; | |
300 | string_length <<= 8; | |
301 | ||
302 | /* String0: */ | |
303 | strings->string0[0] = 4; | |
304 | strings->string0[1] = USB_DT_STRING; | |
305 | strings->string0[2] = 0x09; | |
306 | strings->string0[3] = 0x04; | |
307 | ||
308 | string_length += 4; | |
309 | ||
310 | /* Populate info Context: */ | |
311 | ctx = (struct xdbc_context *)xdbc.dbcc_base; | |
312 | ||
313 | ctx->info.string0 = cpu_to_le64(xdbc.string_dma); | |
314 | ctx->info.manufacturer = cpu_to_le64(xdbc.string_dma + XDBC_MAX_STRING_LENGTH); | |
315 | ctx->info.product = cpu_to_le64(xdbc.string_dma + XDBC_MAX_STRING_LENGTH * 2); | |
316 | ctx->info.serial = cpu_to_le64(xdbc.string_dma + XDBC_MAX_STRING_LENGTH * 3); | |
317 | ctx->info.length = cpu_to_le32(string_length); | |
318 | ||
319 | /* Populate bulk out endpoint context: */ | |
320 | max_burst = DEBUG_MAX_BURST(readl(&xdbc.xdbc_reg->control)); | |
321 | ep_out = (struct xdbc_ep_context *)&ctx->out; | |
322 | ||
323 | ep_out->ep_info1 = 0; | |
324 | ep_out->ep_info2 = cpu_to_le32(EP_TYPE(BULK_OUT_EP) | MAX_PACKET(1024) | MAX_BURST(max_burst)); | |
325 | ep_out->deq = cpu_to_le64(xdbc.out_seg.dma | xdbc.out_ring.cycle_state); | |
326 | ||
327 | /* Populate bulk in endpoint context: */ | |
328 | ep_in = (struct xdbc_ep_context *)&ctx->in; | |
329 | ||
330 | ep_in->ep_info1 = 0; | |
4bda35a0 | 331 | ep_in->ep_info2 = cpu_to_le32(EP_TYPE(BULK_IN_EP) | MAX_PACKET(1024) | MAX_BURST(max_burst)); |
aeb9dd1d LB |
332 | ep_in->deq = cpu_to_le64(xdbc.in_seg.dma | xdbc.in_ring.cycle_state); |
333 | ||
334 | /* Set DbC context and info registers: */ | |
335 | xdbc_write64(xdbc.dbcc_dma, &xdbc.xdbc_reg->dccp); | |
336 | ||
337 | dev_info = cpu_to_le32((XDBC_VENDOR_ID << 16) | XDBC_PROTOCOL); | |
338 | writel(dev_info, &xdbc.xdbc_reg->devinfo1); | |
339 | ||
340 | dev_info = cpu_to_le32((XDBC_DEVICE_REV << 16) | XDBC_PRODUCT_ID); | |
341 | writel(dev_info, &xdbc.xdbc_reg->devinfo2); | |
342 | ||
343 | xdbc.in_buf = xdbc.out_buf + XDBC_MAX_PACKET; | |
344 | xdbc.in_dma = xdbc.out_dma + XDBC_MAX_PACKET; | |
345 | } | |
346 | ||
347 | static void xdbc_do_reset_debug_port(u32 id, u32 count) | |
348 | { | |
349 | void __iomem *ops_reg; | |
350 | void __iomem *portsc; | |
351 | u32 val, cap_length; | |
352 | int i; | |
353 | ||
354 | cap_length = readl(xdbc.xhci_base) & 0xff; | |
355 | ops_reg = xdbc.xhci_base + cap_length; | |
356 | ||
357 | id--; | |
358 | for (i = id; i < (id + count); i++) { | |
359 | portsc = ops_reg + 0x400 + i * 0x10; | |
360 | val = readl(portsc); | |
361 | if (!(val & PORT_CONNECT)) | |
362 | writel(val | PORT_RESET, portsc); | |
363 | } | |
364 | } | |
365 | ||
366 | static void xdbc_reset_debug_port(void) | |
367 | { | |
368 | u32 val, port_offset, port_count; | |
369 | int offset = 0; | |
370 | ||
371 | do { | |
372 | offset = xhci_find_next_ext_cap(xdbc.xhci_base, offset, XHCI_EXT_CAPS_PROTOCOL); | |
373 | if (!offset) | |
374 | break; | |
375 | ||
376 | val = readl(xdbc.xhci_base + offset); | |
377 | if (XHCI_EXT_PORT_MAJOR(val) != 0x3) | |
378 | continue; | |
379 | ||
380 | val = readl(xdbc.xhci_base + offset + 8); | |
381 | port_offset = XHCI_EXT_PORT_OFF(val); | |
382 | port_count = XHCI_EXT_PORT_COUNT(val); | |
383 | ||
384 | xdbc_do_reset_debug_port(port_offset, port_count); | |
385 | } while (1); | |
386 | } | |
387 | ||
388 | static void | |
389 | xdbc_queue_trb(struct xdbc_ring *ring, u32 field1, u32 field2, u32 field3, u32 field4) | |
390 | { | |
391 | struct xdbc_trb *trb, *link_trb; | |
392 | ||
393 | trb = ring->enqueue; | |
394 | trb->field[0] = cpu_to_le32(field1); | |
395 | trb->field[1] = cpu_to_le32(field2); | |
396 | trb->field[2] = cpu_to_le32(field3); | |
397 | trb->field[3] = cpu_to_le32(field4); | |
398 | ||
399 | ++(ring->enqueue); | |
400 | if (ring->enqueue >= &ring->segment->trbs[TRBS_PER_SEGMENT - 1]) { | |
401 | link_trb = ring->enqueue; | |
402 | if (ring->cycle_state) | |
403 | link_trb->field[3] |= cpu_to_le32(TRB_CYCLE); | |
404 | else | |
405 | link_trb->field[3] &= cpu_to_le32(~TRB_CYCLE); | |
406 | ||
407 | ring->enqueue = ring->segment->trbs; | |
408 | ring->cycle_state ^= 1; | |
409 | } | |
410 | } | |
411 | ||
412 | static void xdbc_ring_doorbell(int target) | |
413 | { | |
414 | writel(DOOR_BELL_TARGET(target), &xdbc.xdbc_reg->doorbell); | |
415 | } | |
416 | ||
417 | static int xdbc_start(void) | |
418 | { | |
419 | u32 ctrl, status; | |
420 | int ret; | |
421 | ||
422 | ctrl = readl(&xdbc.xdbc_reg->control); | |
423 | writel(ctrl | CTRL_DBC_ENABLE | CTRL_PORT_ENABLE, &xdbc.xdbc_reg->control); | |
424 | ret = handshake(&xdbc.xdbc_reg->control, CTRL_DBC_ENABLE, CTRL_DBC_ENABLE, 100000, 100); | |
425 | if (ret) { | |
426 | xdbc_trace("failed to initialize hardware\n"); | |
427 | return ret; | |
428 | } | |
429 | ||
430 | /* Reset port to avoid bus hang: */ | |
431 | if (xdbc.vendor == PCI_VENDOR_ID_INTEL) | |
432 | xdbc_reset_debug_port(); | |
433 | ||
434 | /* Wait for port connection: */ | |
435 | ret = handshake(&xdbc.xdbc_reg->portsc, PORTSC_CONN_STATUS, PORTSC_CONN_STATUS, 5000000, 100); | |
436 | if (ret) { | |
437 | xdbc_trace("waiting for connection timed out\n"); | |
438 | return ret; | |
439 | } | |
440 | ||
441 | /* Wait for debug device to be configured: */ | |
442 | ret = handshake(&xdbc.xdbc_reg->control, CTRL_DBC_RUN, CTRL_DBC_RUN, 5000000, 100); | |
443 | if (ret) { | |
444 | xdbc_trace("waiting for device configuration timed out\n"); | |
445 | return ret; | |
446 | } | |
447 | ||
448 | /* Check port number: */ | |
449 | status = readl(&xdbc.xdbc_reg->status); | |
450 | if (!DCST_DEBUG_PORT(status)) { | |
451 | xdbc_trace("invalid root hub port number\n"); | |
452 | return -ENODEV; | |
453 | } | |
454 | ||
455 | xdbc.port_number = DCST_DEBUG_PORT(status); | |
456 | ||
457 | xdbc_trace("DbC is running now, control 0x%08x port ID %d\n", | |
458 | readl(&xdbc.xdbc_reg->control), xdbc.port_number); | |
459 | ||
460 | return 0; | |
461 | } | |
462 | ||
463 | static int xdbc_bulk_transfer(void *data, int size, bool read) | |
464 | { | |
465 | struct xdbc_ring *ring; | |
466 | struct xdbc_trb *trb; | |
467 | u32 length, control; | |
468 | u32 cycle; | |
469 | u64 addr; | |
470 | ||
471 | if (size > XDBC_MAX_PACKET) { | |
472 | xdbc_trace("bad parameter, size %d\n", size); | |
473 | return -EINVAL; | |
474 | } | |
475 | ||
476 | if (!(xdbc.flags & XDBC_FLAGS_INITIALIZED) || | |
477 | !(xdbc.flags & XDBC_FLAGS_CONFIGURED) || | |
478 | (!read && (xdbc.flags & XDBC_FLAGS_OUT_STALL)) || | |
479 | (read && (xdbc.flags & XDBC_FLAGS_IN_STALL))) { | |
480 | ||
481 | xdbc_trace("connection not ready, flags %08x\n", xdbc.flags); | |
482 | return -EIO; | |
483 | } | |
484 | ||
485 | ring = (read ? &xdbc.in_ring : &xdbc.out_ring); | |
486 | trb = ring->enqueue; | |
487 | cycle = ring->cycle_state; | |
488 | length = TRB_LEN(size); | |
489 | control = TRB_TYPE(TRB_NORMAL) | TRB_IOC; | |
490 | ||
491 | if (cycle) | |
492 | control &= cpu_to_le32(~TRB_CYCLE); | |
493 | else | |
494 | control |= cpu_to_le32(TRB_CYCLE); | |
495 | ||
496 | if (read) { | |
497 | memset(xdbc.in_buf, 0, XDBC_MAX_PACKET); | |
498 | addr = xdbc.in_dma; | |
499 | xdbc.flags |= XDBC_FLAGS_IN_PROCESS; | |
500 | } else { | |
501 | memset(xdbc.out_buf, 0, XDBC_MAX_PACKET); | |
502 | memcpy(xdbc.out_buf, data, size); | |
503 | addr = xdbc.out_dma; | |
504 | xdbc.flags |= XDBC_FLAGS_OUT_PROCESS; | |
505 | } | |
506 | ||
507 | xdbc_queue_trb(ring, lower_32_bits(addr), upper_32_bits(addr), length, control); | |
508 | ||
509 | /* | |
510 | * Add a barrier between writes of trb fields and flipping | |
511 | * the cycle bit: | |
512 | */ | |
513 | wmb(); | |
514 | if (cycle) | |
515 | trb->field[3] |= cpu_to_le32(cycle); | |
516 | else | |
517 | trb->field[3] &= cpu_to_le32(~TRB_CYCLE); | |
518 | ||
519 | xdbc_ring_doorbell(read ? IN_EP_DOORBELL : OUT_EP_DOORBELL); | |
520 | ||
521 | return size; | |
522 | } | |
523 | ||
524 | static int xdbc_handle_external_reset(void) | |
525 | { | |
526 | int ret = 0; | |
527 | ||
528 | xdbc.flags = 0; | |
529 | writel(0, &xdbc.xdbc_reg->control); | |
530 | ret = handshake(&xdbc.xdbc_reg->control, CTRL_DBC_ENABLE, 0, 100000, 10); | |
531 | if (ret) | |
532 | goto reset_out; | |
533 | ||
534 | xdbc_mem_init(); | |
535 | ||
536 | mmiowb(); | |
537 | ||
538 | ret = xdbc_start(); | |
539 | if (ret < 0) | |
540 | goto reset_out; | |
541 | ||
542 | xdbc_trace("dbc recovered\n"); | |
543 | ||
544 | xdbc.flags |= XDBC_FLAGS_INITIALIZED | XDBC_FLAGS_CONFIGURED; | |
545 | ||
546 | xdbc_bulk_transfer(NULL, XDBC_MAX_PACKET, true); | |
547 | ||
548 | return 0; | |
549 | ||
550 | reset_out: | |
551 | xdbc_trace("failed to recover from external reset\n"); | |
552 | return ret; | |
553 | } | |
554 | ||
555 | static int __init xdbc_early_setup(void) | |
556 | { | |
557 | int ret; | |
558 | ||
559 | writel(0, &xdbc.xdbc_reg->control); | |
560 | ret = handshake(&xdbc.xdbc_reg->control, CTRL_DBC_ENABLE, 0, 100000, 100); | |
561 | if (ret) | |
562 | return ret; | |
563 | ||
564 | /* Allocate the table page: */ | |
565 | xdbc.table_base = xdbc_get_page(&xdbc.table_dma); | |
566 | if (!xdbc.table_base) | |
567 | return -ENOMEM; | |
568 | ||
569 | /* Get and store the transfer buffer: */ | |
570 | xdbc.out_buf = xdbc_get_page(&xdbc.out_dma); | |
571 | if (!xdbc.out_buf) | |
572 | return -ENOMEM; | |
573 | ||
574 | /* Allocate the event ring: */ | |
575 | ret = xdbc_alloc_ring(&xdbc.evt_seg, &xdbc.evt_ring); | |
576 | if (ret < 0) | |
577 | return ret; | |
578 | ||
579 | /* Allocate IN/OUT endpoint transfer rings: */ | |
580 | ret = xdbc_alloc_ring(&xdbc.in_seg, &xdbc.in_ring); | |
581 | if (ret < 0) | |
582 | return ret; | |
583 | ||
584 | ret = xdbc_alloc_ring(&xdbc.out_seg, &xdbc.out_ring); | |
585 | if (ret < 0) | |
586 | return ret; | |
587 | ||
588 | xdbc_mem_init(); | |
589 | ||
590 | mmiowb(); | |
591 | ||
592 | ret = xdbc_start(); | |
593 | if (ret < 0) { | |
594 | writel(0, &xdbc.xdbc_reg->control); | |
595 | return ret; | |
596 | } | |
597 | ||
598 | xdbc.flags |= XDBC_FLAGS_INITIALIZED | XDBC_FLAGS_CONFIGURED; | |
599 | ||
600 | xdbc_bulk_transfer(NULL, XDBC_MAX_PACKET, true); | |
601 | ||
602 | return 0; | |
603 | } | |
604 | ||
605 | int __init early_xdbc_parse_parameter(char *s) | |
606 | { | |
607 | unsigned long dbgp_num = 0; | |
608 | u32 bus, dev, func, offset; | |
609 | int ret; | |
610 | ||
611 | if (!early_pci_allowed()) | |
612 | return -EPERM; | |
613 | ||
614 | if (strstr(s, "keep")) | |
615 | early_console_keep = true; | |
616 | ||
617 | if (xdbc.xdbc_reg) | |
618 | return 0; | |
619 | ||
620 | if (*s && kstrtoul(s, 0, &dbgp_num)) | |
621 | dbgp_num = 0; | |
622 | ||
623 | pr_notice("dbgp_num: %lu\n", dbgp_num); | |
624 | ||
625 | /* Locate the host controller: */ | |
626 | ret = xdbc_find_dbgp(dbgp_num, &bus, &dev, &func); | |
627 | if (ret) { | |
628 | pr_notice("failed to locate xhci host\n"); | |
629 | return -ENODEV; | |
630 | } | |
631 | ||
632 | xdbc.vendor = read_pci_config_16(bus, dev, func, PCI_VENDOR_ID); | |
633 | xdbc.device = read_pci_config_16(bus, dev, func, PCI_DEVICE_ID); | |
634 | xdbc.bus = bus; | |
635 | xdbc.dev = dev; | |
636 | xdbc.func = func; | |
637 | ||
638 | /* Map the IO memory: */ | |
639 | xdbc.xhci_base = xdbc_map_pci_mmio(bus, dev, func); | |
640 | if (!xdbc.xhci_base) | |
641 | return -EINVAL; | |
642 | ||
643 | /* Locate DbC registers: */ | |
644 | offset = xhci_find_next_ext_cap(xdbc.xhci_base, 0, XHCI_EXT_CAPS_DEBUG); | |
645 | if (!offset) { | |
646 | pr_notice("xhci host doesn't support debug capability\n"); | |
647 | early_iounmap(xdbc.xhci_base, xdbc.xhci_length); | |
648 | xdbc.xhci_base = NULL; | |
649 | xdbc.xhci_length = 0; | |
650 | ||
651 | return -ENODEV; | |
652 | } | |
653 | xdbc.xdbc_reg = (struct xdbc_regs __iomem *)(xdbc.xhci_base + offset); | |
654 | ||
655 | return 0; | |
656 | } | |
657 | ||
658 | int __init early_xdbc_setup_hardware(void) | |
659 | { | |
660 | int ret; | |
661 | ||
662 | if (!xdbc.xdbc_reg) | |
663 | return -ENODEV; | |
664 | ||
665 | xdbc_bios_handoff(); | |
666 | ||
667 | raw_spin_lock_init(&xdbc.lock); | |
668 | ||
669 | ret = xdbc_early_setup(); | |
670 | if (ret) { | |
671 | pr_notice("failed to setup the connection to host\n"); | |
672 | ||
673 | xdbc_free_ring(&xdbc.evt_ring); | |
674 | xdbc_free_ring(&xdbc.out_ring); | |
675 | xdbc_free_ring(&xdbc.in_ring); | |
676 | ||
677 | if (xdbc.table_dma) | |
2013288f | 678 | memblock_free(xdbc.table_dma, PAGE_SIZE); |
aeb9dd1d LB |
679 | |
680 | if (xdbc.out_dma) | |
2013288f | 681 | memblock_free(xdbc.out_dma, PAGE_SIZE); |
aeb9dd1d LB |
682 | |
683 | xdbc.table_base = NULL; | |
684 | xdbc.out_buf = NULL; | |
685 | } | |
686 | ||
687 | return ret; | |
688 | } | |
689 | ||
690 | static void xdbc_handle_port_status(struct xdbc_trb *evt_trb) | |
691 | { | |
692 | u32 port_reg; | |
693 | ||
694 | port_reg = readl(&xdbc.xdbc_reg->portsc); | |
695 | if (port_reg & PORTSC_CONN_CHANGE) { | |
696 | xdbc_trace("connect status change event\n"); | |
697 | ||
698 | /* Check whether cable unplugged: */ | |
699 | if (!(port_reg & PORTSC_CONN_STATUS)) { | |
700 | xdbc.flags = 0; | |
701 | xdbc_trace("cable unplugged\n"); | |
702 | } | |
703 | } | |
704 | ||
705 | if (port_reg & PORTSC_RESET_CHANGE) | |
706 | xdbc_trace("port reset change event\n"); | |
707 | ||
708 | if (port_reg & PORTSC_LINK_CHANGE) | |
709 | xdbc_trace("port link status change event\n"); | |
710 | ||
711 | if (port_reg & PORTSC_CONFIG_CHANGE) | |
712 | xdbc_trace("config error change\n"); | |
713 | ||
714 | /* Write back the value to clear RW1C bits: */ | |
715 | writel(port_reg, &xdbc.xdbc_reg->portsc); | |
716 | } | |
717 | ||
718 | static void xdbc_handle_tx_event(struct xdbc_trb *evt_trb) | |
719 | { | |
aeb9dd1d LB |
720 | u32 comp_code; |
721 | int ep_id; | |
722 | ||
723 | comp_code = GET_COMP_CODE(le32_to_cpu(evt_trb->field[2])); | |
aeb9dd1d LB |
724 | ep_id = TRB_TO_EP_ID(le32_to_cpu(evt_trb->field[3])); |
725 | ||
726 | switch (comp_code) { | |
727 | case COMP_SUCCESS: | |
aeb9dd1d LB |
728 | case COMP_SHORT_PACKET: |
729 | break; | |
730 | case COMP_TRB_ERROR: | |
731 | case COMP_BABBLE_DETECTED_ERROR: | |
732 | case COMP_USB_TRANSACTION_ERROR: | |
733 | case COMP_STALL_ERROR: | |
734 | default: | |
735 | if (ep_id == XDBC_EPID_OUT) | |
736 | xdbc.flags |= XDBC_FLAGS_OUT_STALL; | |
737 | if (ep_id == XDBC_EPID_IN) | |
738 | xdbc.flags |= XDBC_FLAGS_IN_STALL; | |
739 | ||
740 | xdbc_trace("endpoint %d stalled\n", ep_id); | |
741 | break; | |
742 | } | |
743 | ||
744 | if (ep_id == XDBC_EPID_IN) { | |
745 | xdbc.flags &= ~XDBC_FLAGS_IN_PROCESS; | |
746 | xdbc_bulk_transfer(NULL, XDBC_MAX_PACKET, true); | |
747 | } else if (ep_id == XDBC_EPID_OUT) { | |
748 | xdbc.flags &= ~XDBC_FLAGS_OUT_PROCESS; | |
749 | } else { | |
750 | xdbc_trace("invalid endpoint id %d\n", ep_id); | |
751 | } | |
752 | } | |
753 | ||
754 | static void xdbc_handle_events(void) | |
755 | { | |
756 | struct xdbc_trb *evt_trb; | |
757 | bool update_erdp = false; | |
758 | u32 reg; | |
759 | u8 cmd; | |
760 | ||
761 | cmd = read_pci_config_byte(xdbc.bus, xdbc.dev, xdbc.func, PCI_COMMAND); | |
762 | if (!(cmd & PCI_COMMAND_MASTER)) { | |
763 | cmd |= PCI_COMMAND_MASTER | PCI_COMMAND_MEMORY; | |
764 | write_pci_config_byte(xdbc.bus, xdbc.dev, xdbc.func, PCI_COMMAND, cmd); | |
765 | } | |
766 | ||
767 | if (!(xdbc.flags & XDBC_FLAGS_INITIALIZED)) | |
768 | return; | |
769 | ||
770 | /* Handle external reset events: */ | |
771 | reg = readl(&xdbc.xdbc_reg->control); | |
772 | if (!(reg & CTRL_DBC_ENABLE)) { | |
773 | if (xdbc_handle_external_reset()) { | |
774 | xdbc_trace("failed to recover connection\n"); | |
775 | return; | |
776 | } | |
777 | } | |
778 | ||
779 | /* Handle configure-exit event: */ | |
780 | reg = readl(&xdbc.xdbc_reg->control); | |
781 | if (reg & CTRL_DBC_RUN_CHANGE) { | |
782 | writel(reg, &xdbc.xdbc_reg->control); | |
783 | if (reg & CTRL_DBC_RUN) | |
784 | xdbc.flags |= XDBC_FLAGS_CONFIGURED; | |
785 | else | |
786 | xdbc.flags &= ~XDBC_FLAGS_CONFIGURED; | |
787 | } | |
788 | ||
789 | /* Handle endpoint stall event: */ | |
790 | reg = readl(&xdbc.xdbc_reg->control); | |
791 | if (reg & CTRL_HALT_IN_TR) { | |
792 | xdbc.flags |= XDBC_FLAGS_IN_STALL; | |
793 | } else { | |
794 | xdbc.flags &= ~XDBC_FLAGS_IN_STALL; | |
795 | if (!(xdbc.flags & XDBC_FLAGS_IN_PROCESS)) | |
796 | xdbc_bulk_transfer(NULL, XDBC_MAX_PACKET, true); | |
797 | } | |
798 | ||
799 | if (reg & CTRL_HALT_OUT_TR) | |
800 | xdbc.flags |= XDBC_FLAGS_OUT_STALL; | |
801 | else | |
802 | xdbc.flags &= ~XDBC_FLAGS_OUT_STALL; | |
803 | ||
804 | /* Handle the events in the event ring: */ | |
805 | evt_trb = xdbc.evt_ring.dequeue; | |
806 | while ((le32_to_cpu(evt_trb->field[3]) & TRB_CYCLE) == xdbc.evt_ring.cycle_state) { | |
807 | /* | |
808 | * Add a barrier between reading the cycle flag and any | |
809 | * reads of the event's flags/data below: | |
810 | */ | |
811 | rmb(); | |
812 | ||
813 | switch ((le32_to_cpu(evt_trb->field[3]) & TRB_TYPE_BITMASK)) { | |
814 | case TRB_TYPE(TRB_PORT_STATUS): | |
815 | xdbc_handle_port_status(evt_trb); | |
816 | break; | |
817 | case TRB_TYPE(TRB_TRANSFER): | |
818 | xdbc_handle_tx_event(evt_trb); | |
819 | break; | |
820 | default: | |
821 | break; | |
822 | } | |
823 | ||
824 | ++(xdbc.evt_ring.dequeue); | |
825 | if (xdbc.evt_ring.dequeue == &xdbc.evt_seg.trbs[TRBS_PER_SEGMENT]) { | |
826 | xdbc.evt_ring.dequeue = xdbc.evt_seg.trbs; | |
827 | xdbc.evt_ring.cycle_state ^= 1; | |
828 | } | |
829 | ||
830 | evt_trb = xdbc.evt_ring.dequeue; | |
831 | update_erdp = true; | |
832 | } | |
833 | ||
834 | /* Update event ring dequeue pointer: */ | |
835 | if (update_erdp) | |
836 | xdbc_write64(__pa(xdbc.evt_ring.dequeue), &xdbc.xdbc_reg->erdp); | |
837 | } | |
838 | ||
839 | static int xdbc_bulk_write(const char *bytes, int size) | |
840 | { | |
841 | int ret, timeout = 0; | |
842 | unsigned long flags; | |
843 | ||
844 | retry: | |
845 | if (in_nmi()) { | |
846 | if (!raw_spin_trylock_irqsave(&xdbc.lock, flags)) | |
847 | return -EAGAIN; | |
848 | } else { | |
849 | raw_spin_lock_irqsave(&xdbc.lock, flags); | |
850 | } | |
851 | ||
852 | xdbc_handle_events(); | |
853 | ||
854 | /* Check completion of the previous request: */ | |
855 | if ((xdbc.flags & XDBC_FLAGS_OUT_PROCESS) && (timeout < 2000000)) { | |
856 | raw_spin_unlock_irqrestore(&xdbc.lock, flags); | |
857 | udelay(100); | |
858 | timeout += 100; | |
859 | goto retry; | |
860 | } | |
861 | ||
862 | if (xdbc.flags & XDBC_FLAGS_OUT_PROCESS) { | |
863 | raw_spin_unlock_irqrestore(&xdbc.lock, flags); | |
864 | xdbc_trace("previous transfer not completed yet\n"); | |
865 | ||
866 | return -ETIMEDOUT; | |
867 | } | |
868 | ||
869 | ret = xdbc_bulk_transfer((void *)bytes, size, false); | |
870 | raw_spin_unlock_irqrestore(&xdbc.lock, flags); | |
871 | ||
872 | return ret; | |
873 | } | |
874 | ||
875 | static void early_xdbc_write(struct console *con, const char *str, u32 n) | |
876 | { | |
877 | static char buf[XDBC_MAX_PACKET]; | |
878 | int chunk, ret; | |
879 | int use_cr = 0; | |
880 | ||
881 | if (!xdbc.xdbc_reg) | |
882 | return; | |
883 | memset(buf, 0, XDBC_MAX_PACKET); | |
884 | while (n > 0) { | |
885 | for (chunk = 0; chunk < XDBC_MAX_PACKET && n > 0; str++, chunk++, n--) { | |
886 | ||
887 | if (!use_cr && *str == '\n') { | |
888 | use_cr = 1; | |
889 | buf[chunk] = '\r'; | |
890 | str--; | |
891 | n++; | |
892 | continue; | |
893 | } | |
894 | ||
895 | if (use_cr) | |
896 | use_cr = 0; | |
897 | buf[chunk] = *str; | |
898 | } | |
899 | ||
900 | if (chunk > 0) { | |
901 | ret = xdbc_bulk_write(buf, chunk); | |
902 | if (ret < 0) | |
903 | xdbc_trace("missed message {%s}\n", buf); | |
904 | } | |
905 | } | |
906 | } | |
907 | ||
908 | static struct console early_xdbc_console = { | |
909 | .name = "earlyxdbc", | |
910 | .write = early_xdbc_write, | |
911 | .flags = CON_PRINTBUFFER, | |
912 | .index = -1, | |
913 | }; | |
914 | ||
915 | void __init early_xdbc_register_console(void) | |
916 | { | |
917 | if (early_console) | |
918 | return; | |
919 | ||
920 | early_console = &early_xdbc_console; | |
921 | if (early_console_keep) | |
922 | early_console->flags &= ~CON_BOOT; | |
923 | else | |
924 | early_console->flags |= CON_BOOT; | |
925 | register_console(early_console); | |
926 | } | |
927 | ||
928 | static void xdbc_unregister_console(void) | |
929 | { | |
930 | if (early_xdbc_console.flags & CON_ENABLED) | |
931 | unregister_console(&early_xdbc_console); | |
932 | } | |
933 | ||
934 | static int xdbc_scrub_function(void *ptr) | |
935 | { | |
936 | unsigned long flags; | |
937 | ||
938 | while (true) { | |
939 | raw_spin_lock_irqsave(&xdbc.lock, flags); | |
940 | xdbc_handle_events(); | |
941 | ||
942 | if (!(xdbc.flags & XDBC_FLAGS_INITIALIZED)) { | |
943 | raw_spin_unlock_irqrestore(&xdbc.lock, flags); | |
944 | break; | |
945 | } | |
946 | ||
947 | raw_spin_unlock_irqrestore(&xdbc.lock, flags); | |
948 | schedule_timeout_interruptible(1); | |
949 | } | |
950 | ||
951 | xdbc_unregister_console(); | |
952 | writel(0, &xdbc.xdbc_reg->control); | |
953 | xdbc_trace("dbc scrub function exits\n"); | |
954 | ||
955 | return 0; | |
956 | } | |
957 | ||
958 | static int __init xdbc_init(void) | |
959 | { | |
960 | unsigned long flags; | |
961 | void __iomem *base; | |
962 | int ret = 0; | |
963 | u32 offset; | |
964 | ||
965 | if (!(xdbc.flags & XDBC_FLAGS_INITIALIZED)) | |
966 | return 0; | |
967 | ||
968 | /* | |
969 | * It's time to shut down the DbC, so that the debug | |
970 | * port can be reused by the host controller: | |
971 | */ | |
972 | if (early_xdbc_console.index == -1 || | |
973 | (early_xdbc_console.flags & CON_BOOT)) { | |
974 | xdbc_trace("hardware not used anymore\n"); | |
975 | goto free_and_quit; | |
976 | } | |
977 | ||
978 | base = ioremap_nocache(xdbc.xhci_start, xdbc.xhci_length); | |
979 | if (!base) { | |
980 | xdbc_trace("failed to remap the io address\n"); | |
981 | ret = -ENOMEM; | |
982 | goto free_and_quit; | |
983 | } | |
984 | ||
985 | raw_spin_lock_irqsave(&xdbc.lock, flags); | |
986 | early_iounmap(xdbc.xhci_base, xdbc.xhci_length); | |
987 | xdbc.xhci_base = base; | |
988 | offset = xhci_find_next_ext_cap(xdbc.xhci_base, 0, XHCI_EXT_CAPS_DEBUG); | |
989 | xdbc.xdbc_reg = (struct xdbc_regs __iomem *)(xdbc.xhci_base + offset); | |
990 | raw_spin_unlock_irqrestore(&xdbc.lock, flags); | |
991 | ||
992 | kthread_run(xdbc_scrub_function, NULL, "%s", "xdbc"); | |
993 | ||
994 | return 0; | |
995 | ||
996 | free_and_quit: | |
997 | xdbc_free_ring(&xdbc.evt_ring); | |
998 | xdbc_free_ring(&xdbc.out_ring); | |
999 | xdbc_free_ring(&xdbc.in_ring); | |
2013288f MR |
1000 | memblock_free(xdbc.table_dma, PAGE_SIZE); |
1001 | memblock_free(xdbc.out_dma, PAGE_SIZE); | |
aeb9dd1d LB |
1002 | writel(0, &xdbc.xdbc_reg->control); |
1003 | early_iounmap(xdbc.xhci_base, xdbc.xhci_length); | |
1004 | ||
1005 | return ret; | |
1006 | } | |
1007 | subsys_initcall(xdbc_init); |