Commit | Line | Data |
---|---|---|
5fd54ace | 1 | // SPDX-License-Identifier: GPL-2.0 |
aeb9dd1d LB |
2 | /** |
3 | * xhci-dbc.c - xHCI debug capability early driver | |
4 | * | |
5 | * Copyright (C) 2016 Intel Corporation | |
6 | * | |
7 | * Author: Lu Baolu <baolu.lu@linux.intel.com> | |
aeb9dd1d LB |
8 | */ |
9 | ||
10 | #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ | |
11 | ||
12 | #include <linux/console.h> | |
13 | #include <linux/pci_regs.h> | |
14 | #include <linux/pci_ids.h> | |
15 | #include <linux/bootmem.h> | |
2013288f | 16 | #include <linux/memblock.h> |
aeb9dd1d LB |
17 | #include <linux/io.h> |
18 | #include <asm/pci-direct.h> | |
19 | #include <asm/fixmap.h> | |
20 | #include <linux/bcd.h> | |
21 | #include <linux/export.h> | |
22 | #include <linux/version.h> | |
23 | #include <linux/module.h> | |
24 | #include <linux/delay.h> | |
25 | #include <linux/kthread.h> | |
26 | ||
27 | #include "../host/xhci.h" | |
28 | #include "xhci-dbc.h" | |
29 | ||
30 | static struct xdbc_state xdbc; | |
31 | static bool early_console_keep; | |
32 | ||
aeb9dd1d LB |
33 | #ifdef XDBC_TRACE |
34 | #define xdbc_trace trace_printk | |
35 | #else | |
36 | static inline void xdbc_trace(const char *fmt, ...) { } | |
37 | #endif /* XDBC_TRACE */ | |
38 | ||
39 | static void __iomem * __init xdbc_map_pci_mmio(u32 bus, u32 dev, u32 func) | |
40 | { | |
41 | u64 val64, sz64, mask64; | |
42 | void __iomem *base; | |
43 | u32 val, sz; | |
44 | u8 byte; | |
45 | ||
46 | val = read_pci_config(bus, dev, func, PCI_BASE_ADDRESS_0); | |
47 | write_pci_config(bus, dev, func, PCI_BASE_ADDRESS_0, ~0); | |
48 | sz = read_pci_config(bus, dev, func, PCI_BASE_ADDRESS_0); | |
49 | write_pci_config(bus, dev, func, PCI_BASE_ADDRESS_0, val); | |
50 | ||
51 | if (val == 0xffffffff || sz == 0xffffffff) { | |
52 | pr_notice("invalid mmio bar\n"); | |
53 | return NULL; | |
54 | } | |
55 | ||
56 | val64 = val & PCI_BASE_ADDRESS_MEM_MASK; | |
57 | sz64 = sz & PCI_BASE_ADDRESS_MEM_MASK; | |
58 | mask64 = PCI_BASE_ADDRESS_MEM_MASK; | |
59 | ||
60 | if ((val & PCI_BASE_ADDRESS_MEM_TYPE_MASK) == PCI_BASE_ADDRESS_MEM_TYPE_64) { | |
61 | val = read_pci_config(bus, dev, func, PCI_BASE_ADDRESS_0 + 4); | |
62 | write_pci_config(bus, dev, func, PCI_BASE_ADDRESS_0 + 4, ~0); | |
63 | sz = read_pci_config(bus, dev, func, PCI_BASE_ADDRESS_0 + 4); | |
64 | write_pci_config(bus, dev, func, PCI_BASE_ADDRESS_0 + 4, val); | |
65 | ||
66 | val64 |= (u64)val << 32; | |
67 | sz64 |= (u64)sz << 32; | |
68 | mask64 |= ~0ULL << 32; | |
69 | } | |
70 | ||
71 | sz64 &= mask64; | |
72 | ||
73 | if (!sz64) { | |
74 | pr_notice("invalid mmio address\n"); | |
75 | return NULL; | |
76 | } | |
77 | ||
78 | sz64 = 1ULL << __ffs64(sz64); | |
79 | ||
80 | /* Check if the mem space is enabled: */ | |
81 | byte = read_pci_config_byte(bus, dev, func, PCI_COMMAND); | |
82 | if (!(byte & PCI_COMMAND_MEMORY)) { | |
83 | byte |= PCI_COMMAND_MEMORY; | |
84 | write_pci_config_byte(bus, dev, func, PCI_COMMAND, byte); | |
85 | } | |
86 | ||
87 | xdbc.xhci_start = val64; | |
88 | xdbc.xhci_length = sz64; | |
89 | base = early_ioremap(val64, sz64); | |
90 | ||
91 | return base; | |
92 | } | |
93 | ||
94 | static void * __init xdbc_get_page(dma_addr_t *dma_addr) | |
95 | { | |
96 | void *virt; | |
97 | ||
238997e5 | 98 | virt = memblock_alloc_nopanic(PAGE_SIZE, PAGE_SIZE); |
aeb9dd1d LB |
99 | if (!virt) |
100 | return NULL; | |
101 | ||
102 | if (dma_addr) | |
103 | *dma_addr = (dma_addr_t)__pa(virt); | |
104 | ||
105 | return virt; | |
106 | } | |
107 | ||
108 | static u32 __init xdbc_find_dbgp(int xdbc_num, u32 *b, u32 *d, u32 *f) | |
109 | { | |
110 | u32 bus, dev, func, class; | |
111 | ||
112 | for (bus = 0; bus < XDBC_PCI_MAX_BUSES; bus++) { | |
113 | for (dev = 0; dev < XDBC_PCI_MAX_DEVICES; dev++) { | |
114 | for (func = 0; func < XDBC_PCI_MAX_FUNCTION; func++) { | |
115 | ||
116 | class = read_pci_config(bus, dev, func, PCI_CLASS_REVISION); | |
117 | if ((class >> 8) != PCI_CLASS_SERIAL_USB_XHCI) | |
118 | continue; | |
119 | ||
120 | if (xdbc_num-- != 0) | |
121 | continue; | |
122 | ||
123 | *b = bus; | |
124 | *d = dev; | |
125 | *f = func; | |
126 | ||
127 | return 0; | |
128 | } | |
129 | } | |
130 | } | |
131 | ||
132 | return -1; | |
133 | } | |
134 | ||
135 | static int handshake(void __iomem *ptr, u32 mask, u32 done, int wait, int delay) | |
136 | { | |
137 | u32 result; | |
138 | ||
139 | do { | |
140 | result = readl(ptr); | |
141 | result &= mask; | |
142 | if (result == done) | |
143 | return 0; | |
144 | udelay(delay); | |
145 | wait -= delay; | |
146 | } while (wait > 0); | |
147 | ||
148 | return -ETIMEDOUT; | |
149 | } | |
150 | ||
151 | static void __init xdbc_bios_handoff(void) | |
152 | { | |
153 | int offset, timeout; | |
154 | u32 val; | |
155 | ||
156 | offset = xhci_find_next_ext_cap(xdbc.xhci_base, 0, XHCI_EXT_CAPS_LEGACY); | |
157 | val = readl(xdbc.xhci_base + offset); | |
158 | ||
159 | if (val & XHCI_HC_BIOS_OWNED) { | |
160 | writel(val | XHCI_HC_OS_OWNED, xdbc.xhci_base + offset); | |
161 | timeout = handshake(xdbc.xhci_base + offset, XHCI_HC_BIOS_OWNED, 0, 5000, 10); | |
162 | ||
163 | if (timeout) { | |
164 | pr_notice("failed to hand over xHCI control from BIOS\n"); | |
165 | writel(val & ~XHCI_HC_BIOS_OWNED, xdbc.xhci_base + offset); | |
166 | } | |
167 | } | |
168 | ||
169 | /* Disable BIOS SMIs and clear all SMI events: */ | |
170 | val = readl(xdbc.xhci_base + offset + XHCI_LEGACY_CONTROL_OFFSET); | |
171 | val &= XHCI_LEGACY_DISABLE_SMI; | |
172 | val |= XHCI_LEGACY_SMI_EVENTS; | |
173 | writel(val, xdbc.xhci_base + offset + XHCI_LEGACY_CONTROL_OFFSET); | |
174 | } | |
175 | ||
176 | static int __init | |
177 | xdbc_alloc_ring(struct xdbc_segment *seg, struct xdbc_ring *ring) | |
178 | { | |
179 | seg->trbs = xdbc_get_page(&seg->dma); | |
180 | if (!seg->trbs) | |
181 | return -ENOMEM; | |
182 | ||
183 | ring->segment = seg; | |
184 | ||
185 | return 0; | |
186 | } | |
187 | ||
188 | static void __init xdbc_free_ring(struct xdbc_ring *ring) | |
189 | { | |
190 | struct xdbc_segment *seg = ring->segment; | |
191 | ||
192 | if (!seg) | |
193 | return; | |
194 | ||
2013288f | 195 | memblock_free(seg->dma, PAGE_SIZE); |
aeb9dd1d LB |
196 | ring->segment = NULL; |
197 | } | |
198 | ||
199 | static void xdbc_reset_ring(struct xdbc_ring *ring) | |
200 | { | |
201 | struct xdbc_segment *seg = ring->segment; | |
202 | struct xdbc_trb *link_trb; | |
203 | ||
204 | memset(seg->trbs, 0, PAGE_SIZE); | |
205 | ||
206 | ring->enqueue = seg->trbs; | |
207 | ring->dequeue = seg->trbs; | |
208 | ring->cycle_state = 1; | |
209 | ||
210 | if (ring != &xdbc.evt_ring) { | |
211 | link_trb = &seg->trbs[XDBC_TRBS_PER_SEGMENT - 1]; | |
212 | link_trb->field[0] = cpu_to_le32(lower_32_bits(seg->dma)); | |
213 | link_trb->field[1] = cpu_to_le32(upper_32_bits(seg->dma)); | |
214 | link_trb->field[3] = cpu_to_le32(TRB_TYPE(TRB_LINK)) | cpu_to_le32(LINK_TOGGLE); | |
215 | } | |
216 | } | |
217 | ||
218 | static inline void xdbc_put_utf16(u16 *s, const char *c, size_t size) | |
219 | { | |
220 | int i; | |
221 | ||
222 | for (i = 0; i < size; i++) | |
223 | s[i] = cpu_to_le16(c[i]); | |
224 | } | |
225 | ||
226 | static void xdbc_mem_init(void) | |
227 | { | |
228 | struct xdbc_ep_context *ep_in, *ep_out; | |
229 | struct usb_string_descriptor *s_desc; | |
230 | struct xdbc_erst_entry *entry; | |
231 | struct xdbc_strings *strings; | |
232 | struct xdbc_context *ctx; | |
233 | unsigned int max_burst; | |
234 | u32 string_length; | |
235 | int index = 0; | |
236 | u32 dev_info; | |
237 | ||
238 | xdbc_reset_ring(&xdbc.evt_ring); | |
239 | xdbc_reset_ring(&xdbc.in_ring); | |
240 | xdbc_reset_ring(&xdbc.out_ring); | |
241 | memset(xdbc.table_base, 0, PAGE_SIZE); | |
242 | memset(xdbc.out_buf, 0, PAGE_SIZE); | |
243 | ||
244 | /* Initialize event ring segment table: */ | |
245 | xdbc.erst_size = 16; | |
246 | xdbc.erst_base = xdbc.table_base + index * XDBC_TABLE_ENTRY_SIZE; | |
247 | xdbc.erst_dma = xdbc.table_dma + index * XDBC_TABLE_ENTRY_SIZE; | |
248 | ||
249 | index += XDBC_ERST_ENTRY_NUM; | |
250 | entry = (struct xdbc_erst_entry *)xdbc.erst_base; | |
251 | ||
252 | entry->seg_addr = cpu_to_le64(xdbc.evt_seg.dma); | |
253 | entry->seg_size = cpu_to_le32(XDBC_TRBS_PER_SEGMENT); | |
254 | entry->__reserved_0 = 0; | |
255 | ||
256 | /* Initialize ERST registers: */ | |
257 | writel(1, &xdbc.xdbc_reg->ersts); | |
258 | xdbc_write64(xdbc.erst_dma, &xdbc.xdbc_reg->erstba); | |
259 | xdbc_write64(xdbc.evt_seg.dma, &xdbc.xdbc_reg->erdp); | |
260 | ||
261 | /* Debug capability contexts: */ | |
262 | xdbc.dbcc_size = 64 * 3; | |
263 | xdbc.dbcc_base = xdbc.table_base + index * XDBC_TABLE_ENTRY_SIZE; | |
264 | xdbc.dbcc_dma = xdbc.table_dma + index * XDBC_TABLE_ENTRY_SIZE; | |
265 | ||
266 | index += XDBC_DBCC_ENTRY_NUM; | |
267 | ||
268 | /* Popluate the strings: */ | |
269 | xdbc.string_size = sizeof(struct xdbc_strings); | |
270 | xdbc.string_base = xdbc.table_base + index * XDBC_TABLE_ENTRY_SIZE; | |
271 | xdbc.string_dma = xdbc.table_dma + index * XDBC_TABLE_ENTRY_SIZE; | |
272 | strings = (struct xdbc_strings *)xdbc.string_base; | |
273 | ||
274 | index += XDBC_STRING_ENTRY_NUM; | |
275 | ||
276 | /* Serial string: */ | |
277 | s_desc = (struct usb_string_descriptor *)strings->serial; | |
278 | s_desc->bLength = (strlen(XDBC_STRING_SERIAL) + 1) * 2; | |
279 | s_desc->bDescriptorType = USB_DT_STRING; | |
280 | ||
281 | xdbc_put_utf16(s_desc->wData, XDBC_STRING_SERIAL, strlen(XDBC_STRING_SERIAL)); | |
282 | string_length = s_desc->bLength; | |
283 | string_length <<= 8; | |
284 | ||
285 | /* Product string: */ | |
286 | s_desc = (struct usb_string_descriptor *)strings->product; | |
287 | s_desc->bLength = (strlen(XDBC_STRING_PRODUCT) + 1) * 2; | |
288 | s_desc->bDescriptorType = USB_DT_STRING; | |
289 | ||
290 | xdbc_put_utf16(s_desc->wData, XDBC_STRING_PRODUCT, strlen(XDBC_STRING_PRODUCT)); | |
291 | string_length += s_desc->bLength; | |
292 | string_length <<= 8; | |
293 | ||
294 | /* Manufacture string: */ | |
295 | s_desc = (struct usb_string_descriptor *)strings->manufacturer; | |
296 | s_desc->bLength = (strlen(XDBC_STRING_MANUFACTURER) + 1) * 2; | |
297 | s_desc->bDescriptorType = USB_DT_STRING; | |
298 | ||
299 | xdbc_put_utf16(s_desc->wData, XDBC_STRING_MANUFACTURER, strlen(XDBC_STRING_MANUFACTURER)); | |
300 | string_length += s_desc->bLength; | |
301 | string_length <<= 8; | |
302 | ||
303 | /* String0: */ | |
304 | strings->string0[0] = 4; | |
305 | strings->string0[1] = USB_DT_STRING; | |
306 | strings->string0[2] = 0x09; | |
307 | strings->string0[3] = 0x04; | |
308 | ||
309 | string_length += 4; | |
310 | ||
311 | /* Populate info Context: */ | |
312 | ctx = (struct xdbc_context *)xdbc.dbcc_base; | |
313 | ||
314 | ctx->info.string0 = cpu_to_le64(xdbc.string_dma); | |
315 | ctx->info.manufacturer = cpu_to_le64(xdbc.string_dma + XDBC_MAX_STRING_LENGTH); | |
316 | ctx->info.product = cpu_to_le64(xdbc.string_dma + XDBC_MAX_STRING_LENGTH * 2); | |
317 | ctx->info.serial = cpu_to_le64(xdbc.string_dma + XDBC_MAX_STRING_LENGTH * 3); | |
318 | ctx->info.length = cpu_to_le32(string_length); | |
319 | ||
320 | /* Populate bulk out endpoint context: */ | |
321 | max_burst = DEBUG_MAX_BURST(readl(&xdbc.xdbc_reg->control)); | |
322 | ep_out = (struct xdbc_ep_context *)&ctx->out; | |
323 | ||
324 | ep_out->ep_info1 = 0; | |
325 | ep_out->ep_info2 = cpu_to_le32(EP_TYPE(BULK_OUT_EP) | MAX_PACKET(1024) | MAX_BURST(max_burst)); | |
326 | ep_out->deq = cpu_to_le64(xdbc.out_seg.dma | xdbc.out_ring.cycle_state); | |
327 | ||
328 | /* Populate bulk in endpoint context: */ | |
329 | ep_in = (struct xdbc_ep_context *)&ctx->in; | |
330 | ||
331 | ep_in->ep_info1 = 0; | |
4bda35a0 | 332 | ep_in->ep_info2 = cpu_to_le32(EP_TYPE(BULK_IN_EP) | MAX_PACKET(1024) | MAX_BURST(max_burst)); |
aeb9dd1d LB |
333 | ep_in->deq = cpu_to_le64(xdbc.in_seg.dma | xdbc.in_ring.cycle_state); |
334 | ||
335 | /* Set DbC context and info registers: */ | |
336 | xdbc_write64(xdbc.dbcc_dma, &xdbc.xdbc_reg->dccp); | |
337 | ||
338 | dev_info = cpu_to_le32((XDBC_VENDOR_ID << 16) | XDBC_PROTOCOL); | |
339 | writel(dev_info, &xdbc.xdbc_reg->devinfo1); | |
340 | ||
341 | dev_info = cpu_to_le32((XDBC_DEVICE_REV << 16) | XDBC_PRODUCT_ID); | |
342 | writel(dev_info, &xdbc.xdbc_reg->devinfo2); | |
343 | ||
344 | xdbc.in_buf = xdbc.out_buf + XDBC_MAX_PACKET; | |
345 | xdbc.in_dma = xdbc.out_dma + XDBC_MAX_PACKET; | |
346 | } | |
347 | ||
348 | static void xdbc_do_reset_debug_port(u32 id, u32 count) | |
349 | { | |
350 | void __iomem *ops_reg; | |
351 | void __iomem *portsc; | |
352 | u32 val, cap_length; | |
353 | int i; | |
354 | ||
355 | cap_length = readl(xdbc.xhci_base) & 0xff; | |
356 | ops_reg = xdbc.xhci_base + cap_length; | |
357 | ||
358 | id--; | |
359 | for (i = id; i < (id + count); i++) { | |
360 | portsc = ops_reg + 0x400 + i * 0x10; | |
361 | val = readl(portsc); | |
362 | if (!(val & PORT_CONNECT)) | |
363 | writel(val | PORT_RESET, portsc); | |
364 | } | |
365 | } | |
366 | ||
367 | static void xdbc_reset_debug_port(void) | |
368 | { | |
369 | u32 val, port_offset, port_count; | |
370 | int offset = 0; | |
371 | ||
372 | do { | |
373 | offset = xhci_find_next_ext_cap(xdbc.xhci_base, offset, XHCI_EXT_CAPS_PROTOCOL); | |
374 | if (!offset) | |
375 | break; | |
376 | ||
377 | val = readl(xdbc.xhci_base + offset); | |
378 | if (XHCI_EXT_PORT_MAJOR(val) != 0x3) | |
379 | continue; | |
380 | ||
381 | val = readl(xdbc.xhci_base + offset + 8); | |
382 | port_offset = XHCI_EXT_PORT_OFF(val); | |
383 | port_count = XHCI_EXT_PORT_COUNT(val); | |
384 | ||
385 | xdbc_do_reset_debug_port(port_offset, port_count); | |
386 | } while (1); | |
387 | } | |
388 | ||
389 | static void | |
390 | xdbc_queue_trb(struct xdbc_ring *ring, u32 field1, u32 field2, u32 field3, u32 field4) | |
391 | { | |
392 | struct xdbc_trb *trb, *link_trb; | |
393 | ||
394 | trb = ring->enqueue; | |
395 | trb->field[0] = cpu_to_le32(field1); | |
396 | trb->field[1] = cpu_to_le32(field2); | |
397 | trb->field[2] = cpu_to_le32(field3); | |
398 | trb->field[3] = cpu_to_le32(field4); | |
399 | ||
400 | ++(ring->enqueue); | |
401 | if (ring->enqueue >= &ring->segment->trbs[TRBS_PER_SEGMENT - 1]) { | |
402 | link_trb = ring->enqueue; | |
403 | if (ring->cycle_state) | |
404 | link_trb->field[3] |= cpu_to_le32(TRB_CYCLE); | |
405 | else | |
406 | link_trb->field[3] &= cpu_to_le32(~TRB_CYCLE); | |
407 | ||
408 | ring->enqueue = ring->segment->trbs; | |
409 | ring->cycle_state ^= 1; | |
410 | } | |
411 | } | |
412 | ||
413 | static void xdbc_ring_doorbell(int target) | |
414 | { | |
415 | writel(DOOR_BELL_TARGET(target), &xdbc.xdbc_reg->doorbell); | |
416 | } | |
417 | ||
418 | static int xdbc_start(void) | |
419 | { | |
420 | u32 ctrl, status; | |
421 | int ret; | |
422 | ||
423 | ctrl = readl(&xdbc.xdbc_reg->control); | |
424 | writel(ctrl | CTRL_DBC_ENABLE | CTRL_PORT_ENABLE, &xdbc.xdbc_reg->control); | |
425 | ret = handshake(&xdbc.xdbc_reg->control, CTRL_DBC_ENABLE, CTRL_DBC_ENABLE, 100000, 100); | |
426 | if (ret) { | |
427 | xdbc_trace("failed to initialize hardware\n"); | |
428 | return ret; | |
429 | } | |
430 | ||
431 | /* Reset port to avoid bus hang: */ | |
432 | if (xdbc.vendor == PCI_VENDOR_ID_INTEL) | |
433 | xdbc_reset_debug_port(); | |
434 | ||
435 | /* Wait for port connection: */ | |
436 | ret = handshake(&xdbc.xdbc_reg->portsc, PORTSC_CONN_STATUS, PORTSC_CONN_STATUS, 5000000, 100); | |
437 | if (ret) { | |
438 | xdbc_trace("waiting for connection timed out\n"); | |
439 | return ret; | |
440 | } | |
441 | ||
442 | /* Wait for debug device to be configured: */ | |
443 | ret = handshake(&xdbc.xdbc_reg->control, CTRL_DBC_RUN, CTRL_DBC_RUN, 5000000, 100); | |
444 | if (ret) { | |
445 | xdbc_trace("waiting for device configuration timed out\n"); | |
446 | return ret; | |
447 | } | |
448 | ||
449 | /* Check port number: */ | |
450 | status = readl(&xdbc.xdbc_reg->status); | |
451 | if (!DCST_DEBUG_PORT(status)) { | |
452 | xdbc_trace("invalid root hub port number\n"); | |
453 | return -ENODEV; | |
454 | } | |
455 | ||
456 | xdbc.port_number = DCST_DEBUG_PORT(status); | |
457 | ||
458 | xdbc_trace("DbC is running now, control 0x%08x port ID %d\n", | |
459 | readl(&xdbc.xdbc_reg->control), xdbc.port_number); | |
460 | ||
461 | return 0; | |
462 | } | |
463 | ||
464 | static int xdbc_bulk_transfer(void *data, int size, bool read) | |
465 | { | |
466 | struct xdbc_ring *ring; | |
467 | struct xdbc_trb *trb; | |
468 | u32 length, control; | |
469 | u32 cycle; | |
470 | u64 addr; | |
471 | ||
472 | if (size > XDBC_MAX_PACKET) { | |
473 | xdbc_trace("bad parameter, size %d\n", size); | |
474 | return -EINVAL; | |
475 | } | |
476 | ||
477 | if (!(xdbc.flags & XDBC_FLAGS_INITIALIZED) || | |
478 | !(xdbc.flags & XDBC_FLAGS_CONFIGURED) || | |
479 | (!read && (xdbc.flags & XDBC_FLAGS_OUT_STALL)) || | |
480 | (read && (xdbc.flags & XDBC_FLAGS_IN_STALL))) { | |
481 | ||
482 | xdbc_trace("connection not ready, flags %08x\n", xdbc.flags); | |
483 | return -EIO; | |
484 | } | |
485 | ||
486 | ring = (read ? &xdbc.in_ring : &xdbc.out_ring); | |
487 | trb = ring->enqueue; | |
488 | cycle = ring->cycle_state; | |
489 | length = TRB_LEN(size); | |
490 | control = TRB_TYPE(TRB_NORMAL) | TRB_IOC; | |
491 | ||
492 | if (cycle) | |
493 | control &= cpu_to_le32(~TRB_CYCLE); | |
494 | else | |
495 | control |= cpu_to_le32(TRB_CYCLE); | |
496 | ||
497 | if (read) { | |
498 | memset(xdbc.in_buf, 0, XDBC_MAX_PACKET); | |
499 | addr = xdbc.in_dma; | |
500 | xdbc.flags |= XDBC_FLAGS_IN_PROCESS; | |
501 | } else { | |
502 | memset(xdbc.out_buf, 0, XDBC_MAX_PACKET); | |
503 | memcpy(xdbc.out_buf, data, size); | |
504 | addr = xdbc.out_dma; | |
505 | xdbc.flags |= XDBC_FLAGS_OUT_PROCESS; | |
506 | } | |
507 | ||
508 | xdbc_queue_trb(ring, lower_32_bits(addr), upper_32_bits(addr), length, control); | |
509 | ||
510 | /* | |
511 | * Add a barrier between writes of trb fields and flipping | |
512 | * the cycle bit: | |
513 | */ | |
514 | wmb(); | |
515 | if (cycle) | |
516 | trb->field[3] |= cpu_to_le32(cycle); | |
517 | else | |
518 | trb->field[3] &= cpu_to_le32(~TRB_CYCLE); | |
519 | ||
520 | xdbc_ring_doorbell(read ? IN_EP_DOORBELL : OUT_EP_DOORBELL); | |
521 | ||
522 | return size; | |
523 | } | |
524 | ||
525 | static int xdbc_handle_external_reset(void) | |
526 | { | |
527 | int ret = 0; | |
528 | ||
529 | xdbc.flags = 0; | |
530 | writel(0, &xdbc.xdbc_reg->control); | |
531 | ret = handshake(&xdbc.xdbc_reg->control, CTRL_DBC_ENABLE, 0, 100000, 10); | |
532 | if (ret) | |
533 | goto reset_out; | |
534 | ||
535 | xdbc_mem_init(); | |
536 | ||
537 | mmiowb(); | |
538 | ||
539 | ret = xdbc_start(); | |
540 | if (ret < 0) | |
541 | goto reset_out; | |
542 | ||
543 | xdbc_trace("dbc recovered\n"); | |
544 | ||
545 | xdbc.flags |= XDBC_FLAGS_INITIALIZED | XDBC_FLAGS_CONFIGURED; | |
546 | ||
547 | xdbc_bulk_transfer(NULL, XDBC_MAX_PACKET, true); | |
548 | ||
549 | return 0; | |
550 | ||
551 | reset_out: | |
552 | xdbc_trace("failed to recover from external reset\n"); | |
553 | return ret; | |
554 | } | |
555 | ||
556 | static int __init xdbc_early_setup(void) | |
557 | { | |
558 | int ret; | |
559 | ||
560 | writel(0, &xdbc.xdbc_reg->control); | |
561 | ret = handshake(&xdbc.xdbc_reg->control, CTRL_DBC_ENABLE, 0, 100000, 100); | |
562 | if (ret) | |
563 | return ret; | |
564 | ||
565 | /* Allocate the table page: */ | |
566 | xdbc.table_base = xdbc_get_page(&xdbc.table_dma); | |
567 | if (!xdbc.table_base) | |
568 | return -ENOMEM; | |
569 | ||
570 | /* Get and store the transfer buffer: */ | |
571 | xdbc.out_buf = xdbc_get_page(&xdbc.out_dma); | |
572 | if (!xdbc.out_buf) | |
573 | return -ENOMEM; | |
574 | ||
575 | /* Allocate the event ring: */ | |
576 | ret = xdbc_alloc_ring(&xdbc.evt_seg, &xdbc.evt_ring); | |
577 | if (ret < 0) | |
578 | return ret; | |
579 | ||
580 | /* Allocate IN/OUT endpoint transfer rings: */ | |
581 | ret = xdbc_alloc_ring(&xdbc.in_seg, &xdbc.in_ring); | |
582 | if (ret < 0) | |
583 | return ret; | |
584 | ||
585 | ret = xdbc_alloc_ring(&xdbc.out_seg, &xdbc.out_ring); | |
586 | if (ret < 0) | |
587 | return ret; | |
588 | ||
589 | xdbc_mem_init(); | |
590 | ||
591 | mmiowb(); | |
592 | ||
593 | ret = xdbc_start(); | |
594 | if (ret < 0) { | |
595 | writel(0, &xdbc.xdbc_reg->control); | |
596 | return ret; | |
597 | } | |
598 | ||
599 | xdbc.flags |= XDBC_FLAGS_INITIALIZED | XDBC_FLAGS_CONFIGURED; | |
600 | ||
601 | xdbc_bulk_transfer(NULL, XDBC_MAX_PACKET, true); | |
602 | ||
603 | return 0; | |
604 | } | |
605 | ||
606 | int __init early_xdbc_parse_parameter(char *s) | |
607 | { | |
608 | unsigned long dbgp_num = 0; | |
609 | u32 bus, dev, func, offset; | |
610 | int ret; | |
611 | ||
612 | if (!early_pci_allowed()) | |
613 | return -EPERM; | |
614 | ||
615 | if (strstr(s, "keep")) | |
616 | early_console_keep = true; | |
617 | ||
618 | if (xdbc.xdbc_reg) | |
619 | return 0; | |
620 | ||
621 | if (*s && kstrtoul(s, 0, &dbgp_num)) | |
622 | dbgp_num = 0; | |
623 | ||
624 | pr_notice("dbgp_num: %lu\n", dbgp_num); | |
625 | ||
626 | /* Locate the host controller: */ | |
627 | ret = xdbc_find_dbgp(dbgp_num, &bus, &dev, &func); | |
628 | if (ret) { | |
629 | pr_notice("failed to locate xhci host\n"); | |
630 | return -ENODEV; | |
631 | } | |
632 | ||
633 | xdbc.vendor = read_pci_config_16(bus, dev, func, PCI_VENDOR_ID); | |
634 | xdbc.device = read_pci_config_16(bus, dev, func, PCI_DEVICE_ID); | |
635 | xdbc.bus = bus; | |
636 | xdbc.dev = dev; | |
637 | xdbc.func = func; | |
638 | ||
639 | /* Map the IO memory: */ | |
640 | xdbc.xhci_base = xdbc_map_pci_mmio(bus, dev, func); | |
641 | if (!xdbc.xhci_base) | |
642 | return -EINVAL; | |
643 | ||
644 | /* Locate DbC registers: */ | |
645 | offset = xhci_find_next_ext_cap(xdbc.xhci_base, 0, XHCI_EXT_CAPS_DEBUG); | |
646 | if (!offset) { | |
647 | pr_notice("xhci host doesn't support debug capability\n"); | |
648 | early_iounmap(xdbc.xhci_base, xdbc.xhci_length); | |
649 | xdbc.xhci_base = NULL; | |
650 | xdbc.xhci_length = 0; | |
651 | ||
652 | return -ENODEV; | |
653 | } | |
654 | xdbc.xdbc_reg = (struct xdbc_regs __iomem *)(xdbc.xhci_base + offset); | |
655 | ||
656 | return 0; | |
657 | } | |
658 | ||
659 | int __init early_xdbc_setup_hardware(void) | |
660 | { | |
661 | int ret; | |
662 | ||
663 | if (!xdbc.xdbc_reg) | |
664 | return -ENODEV; | |
665 | ||
666 | xdbc_bios_handoff(); | |
667 | ||
668 | raw_spin_lock_init(&xdbc.lock); | |
669 | ||
670 | ret = xdbc_early_setup(); | |
671 | if (ret) { | |
672 | pr_notice("failed to setup the connection to host\n"); | |
673 | ||
674 | xdbc_free_ring(&xdbc.evt_ring); | |
675 | xdbc_free_ring(&xdbc.out_ring); | |
676 | xdbc_free_ring(&xdbc.in_ring); | |
677 | ||
678 | if (xdbc.table_dma) | |
2013288f | 679 | memblock_free(xdbc.table_dma, PAGE_SIZE); |
aeb9dd1d LB |
680 | |
681 | if (xdbc.out_dma) | |
2013288f | 682 | memblock_free(xdbc.out_dma, PAGE_SIZE); |
aeb9dd1d LB |
683 | |
684 | xdbc.table_base = NULL; | |
685 | xdbc.out_buf = NULL; | |
686 | } | |
687 | ||
688 | return ret; | |
689 | } | |
690 | ||
691 | static void xdbc_handle_port_status(struct xdbc_trb *evt_trb) | |
692 | { | |
693 | u32 port_reg; | |
694 | ||
695 | port_reg = readl(&xdbc.xdbc_reg->portsc); | |
696 | if (port_reg & PORTSC_CONN_CHANGE) { | |
697 | xdbc_trace("connect status change event\n"); | |
698 | ||
699 | /* Check whether cable unplugged: */ | |
700 | if (!(port_reg & PORTSC_CONN_STATUS)) { | |
701 | xdbc.flags = 0; | |
702 | xdbc_trace("cable unplugged\n"); | |
703 | } | |
704 | } | |
705 | ||
706 | if (port_reg & PORTSC_RESET_CHANGE) | |
707 | xdbc_trace("port reset change event\n"); | |
708 | ||
709 | if (port_reg & PORTSC_LINK_CHANGE) | |
710 | xdbc_trace("port link status change event\n"); | |
711 | ||
712 | if (port_reg & PORTSC_CONFIG_CHANGE) | |
713 | xdbc_trace("config error change\n"); | |
714 | ||
715 | /* Write back the value to clear RW1C bits: */ | |
716 | writel(port_reg, &xdbc.xdbc_reg->portsc); | |
717 | } | |
718 | ||
719 | static void xdbc_handle_tx_event(struct xdbc_trb *evt_trb) | |
720 | { | |
aeb9dd1d LB |
721 | u32 comp_code; |
722 | int ep_id; | |
723 | ||
724 | comp_code = GET_COMP_CODE(le32_to_cpu(evt_trb->field[2])); | |
aeb9dd1d LB |
725 | ep_id = TRB_TO_EP_ID(le32_to_cpu(evt_trb->field[3])); |
726 | ||
727 | switch (comp_code) { | |
728 | case COMP_SUCCESS: | |
aeb9dd1d LB |
729 | case COMP_SHORT_PACKET: |
730 | break; | |
731 | case COMP_TRB_ERROR: | |
732 | case COMP_BABBLE_DETECTED_ERROR: | |
733 | case COMP_USB_TRANSACTION_ERROR: | |
734 | case COMP_STALL_ERROR: | |
735 | default: | |
736 | if (ep_id == XDBC_EPID_OUT) | |
737 | xdbc.flags |= XDBC_FLAGS_OUT_STALL; | |
738 | if (ep_id == XDBC_EPID_IN) | |
739 | xdbc.flags |= XDBC_FLAGS_IN_STALL; | |
740 | ||
741 | xdbc_trace("endpoint %d stalled\n", ep_id); | |
742 | break; | |
743 | } | |
744 | ||
745 | if (ep_id == XDBC_EPID_IN) { | |
746 | xdbc.flags &= ~XDBC_FLAGS_IN_PROCESS; | |
747 | xdbc_bulk_transfer(NULL, XDBC_MAX_PACKET, true); | |
748 | } else if (ep_id == XDBC_EPID_OUT) { | |
749 | xdbc.flags &= ~XDBC_FLAGS_OUT_PROCESS; | |
750 | } else { | |
751 | xdbc_trace("invalid endpoint id %d\n", ep_id); | |
752 | } | |
753 | } | |
754 | ||
755 | static void xdbc_handle_events(void) | |
756 | { | |
757 | struct xdbc_trb *evt_trb; | |
758 | bool update_erdp = false; | |
759 | u32 reg; | |
760 | u8 cmd; | |
761 | ||
762 | cmd = read_pci_config_byte(xdbc.bus, xdbc.dev, xdbc.func, PCI_COMMAND); | |
763 | if (!(cmd & PCI_COMMAND_MASTER)) { | |
764 | cmd |= PCI_COMMAND_MASTER | PCI_COMMAND_MEMORY; | |
765 | write_pci_config_byte(xdbc.bus, xdbc.dev, xdbc.func, PCI_COMMAND, cmd); | |
766 | } | |
767 | ||
768 | if (!(xdbc.flags & XDBC_FLAGS_INITIALIZED)) | |
769 | return; | |
770 | ||
771 | /* Handle external reset events: */ | |
772 | reg = readl(&xdbc.xdbc_reg->control); | |
773 | if (!(reg & CTRL_DBC_ENABLE)) { | |
774 | if (xdbc_handle_external_reset()) { | |
775 | xdbc_trace("failed to recover connection\n"); | |
776 | return; | |
777 | } | |
778 | } | |
779 | ||
780 | /* Handle configure-exit event: */ | |
781 | reg = readl(&xdbc.xdbc_reg->control); | |
782 | if (reg & CTRL_DBC_RUN_CHANGE) { | |
783 | writel(reg, &xdbc.xdbc_reg->control); | |
784 | if (reg & CTRL_DBC_RUN) | |
785 | xdbc.flags |= XDBC_FLAGS_CONFIGURED; | |
786 | else | |
787 | xdbc.flags &= ~XDBC_FLAGS_CONFIGURED; | |
788 | } | |
789 | ||
790 | /* Handle endpoint stall event: */ | |
791 | reg = readl(&xdbc.xdbc_reg->control); | |
792 | if (reg & CTRL_HALT_IN_TR) { | |
793 | xdbc.flags |= XDBC_FLAGS_IN_STALL; | |
794 | } else { | |
795 | xdbc.flags &= ~XDBC_FLAGS_IN_STALL; | |
796 | if (!(xdbc.flags & XDBC_FLAGS_IN_PROCESS)) | |
797 | xdbc_bulk_transfer(NULL, XDBC_MAX_PACKET, true); | |
798 | } | |
799 | ||
800 | if (reg & CTRL_HALT_OUT_TR) | |
801 | xdbc.flags |= XDBC_FLAGS_OUT_STALL; | |
802 | else | |
803 | xdbc.flags &= ~XDBC_FLAGS_OUT_STALL; | |
804 | ||
805 | /* Handle the events in the event ring: */ | |
806 | evt_trb = xdbc.evt_ring.dequeue; | |
807 | while ((le32_to_cpu(evt_trb->field[3]) & TRB_CYCLE) == xdbc.evt_ring.cycle_state) { | |
808 | /* | |
809 | * Add a barrier between reading the cycle flag and any | |
810 | * reads of the event's flags/data below: | |
811 | */ | |
812 | rmb(); | |
813 | ||
814 | switch ((le32_to_cpu(evt_trb->field[3]) & TRB_TYPE_BITMASK)) { | |
815 | case TRB_TYPE(TRB_PORT_STATUS): | |
816 | xdbc_handle_port_status(evt_trb); | |
817 | break; | |
818 | case TRB_TYPE(TRB_TRANSFER): | |
819 | xdbc_handle_tx_event(evt_trb); | |
820 | break; | |
821 | default: | |
822 | break; | |
823 | } | |
824 | ||
825 | ++(xdbc.evt_ring.dequeue); | |
826 | if (xdbc.evt_ring.dequeue == &xdbc.evt_seg.trbs[TRBS_PER_SEGMENT]) { | |
827 | xdbc.evt_ring.dequeue = xdbc.evt_seg.trbs; | |
828 | xdbc.evt_ring.cycle_state ^= 1; | |
829 | } | |
830 | ||
831 | evt_trb = xdbc.evt_ring.dequeue; | |
832 | update_erdp = true; | |
833 | } | |
834 | ||
835 | /* Update event ring dequeue pointer: */ | |
836 | if (update_erdp) | |
837 | xdbc_write64(__pa(xdbc.evt_ring.dequeue), &xdbc.xdbc_reg->erdp); | |
838 | } | |
839 | ||
840 | static int xdbc_bulk_write(const char *bytes, int size) | |
841 | { | |
842 | int ret, timeout = 0; | |
843 | unsigned long flags; | |
844 | ||
845 | retry: | |
846 | if (in_nmi()) { | |
847 | if (!raw_spin_trylock_irqsave(&xdbc.lock, flags)) | |
848 | return -EAGAIN; | |
849 | } else { | |
850 | raw_spin_lock_irqsave(&xdbc.lock, flags); | |
851 | } | |
852 | ||
853 | xdbc_handle_events(); | |
854 | ||
855 | /* Check completion of the previous request: */ | |
856 | if ((xdbc.flags & XDBC_FLAGS_OUT_PROCESS) && (timeout < 2000000)) { | |
857 | raw_spin_unlock_irqrestore(&xdbc.lock, flags); | |
858 | udelay(100); | |
859 | timeout += 100; | |
860 | goto retry; | |
861 | } | |
862 | ||
863 | if (xdbc.flags & XDBC_FLAGS_OUT_PROCESS) { | |
864 | raw_spin_unlock_irqrestore(&xdbc.lock, flags); | |
865 | xdbc_trace("previous transfer not completed yet\n"); | |
866 | ||
867 | return -ETIMEDOUT; | |
868 | } | |
869 | ||
870 | ret = xdbc_bulk_transfer((void *)bytes, size, false); | |
871 | raw_spin_unlock_irqrestore(&xdbc.lock, flags); | |
872 | ||
873 | return ret; | |
874 | } | |
875 | ||
876 | static void early_xdbc_write(struct console *con, const char *str, u32 n) | |
877 | { | |
878 | static char buf[XDBC_MAX_PACKET]; | |
879 | int chunk, ret; | |
880 | int use_cr = 0; | |
881 | ||
882 | if (!xdbc.xdbc_reg) | |
883 | return; | |
884 | memset(buf, 0, XDBC_MAX_PACKET); | |
885 | while (n > 0) { | |
886 | for (chunk = 0; chunk < XDBC_MAX_PACKET && n > 0; str++, chunk++, n--) { | |
887 | ||
888 | if (!use_cr && *str == '\n') { | |
889 | use_cr = 1; | |
890 | buf[chunk] = '\r'; | |
891 | str--; | |
892 | n++; | |
893 | continue; | |
894 | } | |
895 | ||
896 | if (use_cr) | |
897 | use_cr = 0; | |
898 | buf[chunk] = *str; | |
899 | } | |
900 | ||
901 | if (chunk > 0) { | |
902 | ret = xdbc_bulk_write(buf, chunk); | |
903 | if (ret < 0) | |
904 | xdbc_trace("missed message {%s}\n", buf); | |
905 | } | |
906 | } | |
907 | } | |
908 | ||
909 | static struct console early_xdbc_console = { | |
910 | .name = "earlyxdbc", | |
911 | .write = early_xdbc_write, | |
912 | .flags = CON_PRINTBUFFER, | |
913 | .index = -1, | |
914 | }; | |
915 | ||
916 | void __init early_xdbc_register_console(void) | |
917 | { | |
918 | if (early_console) | |
919 | return; | |
920 | ||
921 | early_console = &early_xdbc_console; | |
922 | if (early_console_keep) | |
923 | early_console->flags &= ~CON_BOOT; | |
924 | else | |
925 | early_console->flags |= CON_BOOT; | |
926 | register_console(early_console); | |
927 | } | |
928 | ||
929 | static void xdbc_unregister_console(void) | |
930 | { | |
931 | if (early_xdbc_console.flags & CON_ENABLED) | |
932 | unregister_console(&early_xdbc_console); | |
933 | } | |
934 | ||
935 | static int xdbc_scrub_function(void *ptr) | |
936 | { | |
937 | unsigned long flags; | |
938 | ||
939 | while (true) { | |
940 | raw_spin_lock_irqsave(&xdbc.lock, flags); | |
941 | xdbc_handle_events(); | |
942 | ||
943 | if (!(xdbc.flags & XDBC_FLAGS_INITIALIZED)) { | |
944 | raw_spin_unlock_irqrestore(&xdbc.lock, flags); | |
945 | break; | |
946 | } | |
947 | ||
948 | raw_spin_unlock_irqrestore(&xdbc.lock, flags); | |
949 | schedule_timeout_interruptible(1); | |
950 | } | |
951 | ||
952 | xdbc_unregister_console(); | |
953 | writel(0, &xdbc.xdbc_reg->control); | |
954 | xdbc_trace("dbc scrub function exits\n"); | |
955 | ||
956 | return 0; | |
957 | } | |
958 | ||
959 | static int __init xdbc_init(void) | |
960 | { | |
961 | unsigned long flags; | |
962 | void __iomem *base; | |
963 | int ret = 0; | |
964 | u32 offset; | |
965 | ||
966 | if (!(xdbc.flags & XDBC_FLAGS_INITIALIZED)) | |
967 | return 0; | |
968 | ||
969 | /* | |
970 | * It's time to shut down the DbC, so that the debug | |
971 | * port can be reused by the host controller: | |
972 | */ | |
973 | if (early_xdbc_console.index == -1 || | |
974 | (early_xdbc_console.flags & CON_BOOT)) { | |
975 | xdbc_trace("hardware not used anymore\n"); | |
976 | goto free_and_quit; | |
977 | } | |
978 | ||
979 | base = ioremap_nocache(xdbc.xhci_start, xdbc.xhci_length); | |
980 | if (!base) { | |
981 | xdbc_trace("failed to remap the io address\n"); | |
982 | ret = -ENOMEM; | |
983 | goto free_and_quit; | |
984 | } | |
985 | ||
986 | raw_spin_lock_irqsave(&xdbc.lock, flags); | |
987 | early_iounmap(xdbc.xhci_base, xdbc.xhci_length); | |
988 | xdbc.xhci_base = base; | |
989 | offset = xhci_find_next_ext_cap(xdbc.xhci_base, 0, XHCI_EXT_CAPS_DEBUG); | |
990 | xdbc.xdbc_reg = (struct xdbc_regs __iomem *)(xdbc.xhci_base + offset); | |
991 | raw_spin_unlock_irqrestore(&xdbc.lock, flags); | |
992 | ||
993 | kthread_run(xdbc_scrub_function, NULL, "%s", "xdbc"); | |
994 | ||
995 | return 0; | |
996 | ||
997 | free_and_quit: | |
998 | xdbc_free_ring(&xdbc.evt_ring); | |
999 | xdbc_free_ring(&xdbc.out_ring); | |
1000 | xdbc_free_ring(&xdbc.in_ring); | |
2013288f MR |
1001 | memblock_free(xdbc.table_dma, PAGE_SIZE); |
1002 | memblock_free(xdbc.out_dma, PAGE_SIZE); | |
aeb9dd1d LB |
1003 | writel(0, &xdbc.xdbc_reg->control); |
1004 | early_iounmap(xdbc.xhci_base, xdbc.xhci_length); | |
1005 | ||
1006 | return ret; | |
1007 | } | |
1008 | subsys_initcall(xdbc_init); |