projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
USB: xhci: drop spinlock in xhci_urb_enqueue() error path.
[linux-2.6-block.git] / drivers / usb / host / xhci-mem.c
CommitLineData
66d4eadd
SS		 1/*
2 * xHCI host controller driver
3 *
4 * Copyright (C) 2008 Intel Corp.
5 *
6 * Author: Sarah Sharp
7 * Some code borrowed from the Linux EHCI driver.
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
12 *
13 * This program is distributed in the hope that it will be useful, but
14 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
15 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 * for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with this program; if not, write to the Free Software Foundation,
20 * Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
21 */
22
23#include <linux/usb.h>
0ebbab37 24#include <linux/pci.h>
66d4eadd
SS		 25
26#include "xhci.h"
27
0ebbab37
SS		 28/*
29 * Allocates a generic ring segment from the ring pool, sets the dma address,
30 * initializes the segment to zero, and sets the private next pointer to NULL.
31 *
32 * Section 4.11.1.1:
33 * "All components of all Command and Transfer TRBs shall be initialized to '0'"
34 */
35static struct xhci_segment *xhci_segment_alloc(struct xhci_hcd *xhci, gfp_t flags)
36{
37 struct xhci_segment *seg;
38 dma_addr_t dma;
39
40 seg = kzalloc(sizeof *seg, flags);
41 if (!seg)
42 return 0;
700e2052 43 xhci_dbg(xhci, "Allocating priv segment structure at %p\n", seg);
0ebbab37
SS		 44
45 seg->trbs = dma_pool_alloc(xhci->segment_pool, flags, &dma);
46 if (!seg->trbs) {
47 kfree(seg);
48 return 0;
49 }
700e2052
GKH		 50 xhci_dbg(xhci, "// Allocating segment at %p (virtual) 0x%llx (DMA)\n",
51 seg->trbs, (unsigned long long)dma);
0ebbab37
SS		 52
53 memset(seg->trbs, 0, SEGMENT_SIZE);
54 seg->dma = dma;
55 seg->next = NULL;
56
57 return seg;
58}
59
60static void xhci_segment_free(struct xhci_hcd *xhci, struct xhci_segment *seg)
61{
62 if (!seg)
63 return;
64 if (seg->trbs) {
700e2052
GKH		 65 xhci_dbg(xhci, "Freeing DMA segment at %p (virtual) 0x%llx (DMA)\n",
66 seg->trbs, (unsigned long long)seg->dma);
0ebbab37
SS		 67 dma_pool_free(xhci->segment_pool, seg->trbs, seg->dma);
68 seg->trbs = NULL;
69 }
700e2052 70 xhci_dbg(xhci, "Freeing priv segment structure at %p\n", seg);
0ebbab37
SS		 71 kfree(seg);
72}
73
74/*
75 * Make the prev segment point to the next segment.
76 *
77 * Change the last TRB in the prev segment to be a Link TRB which points to the
78 * DMA address of the next segment. The caller needs to set any Link TRB
79 * related flags, such as End TRB, Toggle Cycle, and no snoop.
80 */
81static void xhci_link_segments(struct xhci_hcd *xhci, struct xhci_segment *prev,
82 struct xhci_segment *next, bool link_trbs)
83{
84 u32 val;
85
86 if (!prev || !next)
87 return;
88 prev->next = next;
89 if (link_trbs) {
90 prev->trbs[TRBS_PER_SEGMENT-1].link.segment_ptr[0] = next->dma;
91
92 /* Set the last TRB in the segment to have a TRB type ID of Link TRB */
93 val = prev->trbs[TRBS_PER_SEGMENT-1].link.control;
94 val &= ~TRB_TYPE_BITMASK;
95 val |= TRB_TYPE(TRB_LINK);
96 prev->trbs[TRBS_PER_SEGMENT-1].link.control = val;
97 }
700e2052
GKH		 98 xhci_dbg(xhci, "Linking segment 0x%llx to segment 0x%llx (DMA)\n",
99 (unsigned long long)prev->dma,
100 (unsigned long long)next->dma);
0ebbab37
SS		 101}
102
103/* XXX: Do we need the hcd structure in all these functions? */
f94e0186 104void xhci_ring_free(struct xhci_hcd *xhci, struct xhci_ring *ring)
0ebbab37
SS		 105{
106 struct xhci_segment *seg;
107 struct xhci_segment *first_seg;
108
109 if (!ring || !ring->first_seg)
110 return;
111 first_seg = ring->first_seg;
112 seg = first_seg->next;
700e2052 113 xhci_dbg(xhci, "Freeing ring at %p\n", ring);
0ebbab37
SS		 114 while (seg != first_seg) {
115 struct xhci_segment *next = seg->next;
116 xhci_segment_free(xhci, seg);
117 seg = next;
118 }
119 xhci_segment_free(xhci, first_seg);
120 ring->first_seg = NULL;
121 kfree(ring);
122}
123
124/**
125 * Create a new ring with zero or more segments.
126 *
127 * Link each segment together into a ring.
128 * Set the end flag and the cycle toggle bit on the last segment.
129 * See section 4.9.1 and figures 15 and 16.
130 */
131static struct xhci_ring *xhci_ring_alloc(struct xhci_hcd *xhci,
132 unsigned int num_segs, bool link_trbs, gfp_t flags)
133{
134 struct xhci_ring *ring;
135 struct xhci_segment *prev;
136
137 ring = kzalloc(sizeof *(ring), flags);
700e2052 138 xhci_dbg(xhci, "Allocating ring at %p\n", ring);
0ebbab37
SS		 139 if (!ring)
140 return 0;
141
d0e96f5a 142 INIT_LIST_HEAD(&ring->td_list);
ae636747 143 INIT_LIST_HEAD(&ring->cancelled_td_list);
0ebbab37
SS		 144 if (num_segs == 0)
145 return ring;
146
147 ring->first_seg = xhci_segment_alloc(xhci, flags);
148 if (!ring->first_seg)
149 goto fail;
150 num_segs--;
151
152 prev = ring->first_seg;
153 while (num_segs > 0) {
154 struct xhci_segment *next;
155
156 next = xhci_segment_alloc(xhci, flags);
157 if (!next)
158 goto fail;
159 xhci_link_segments(xhci, prev, next, link_trbs);
160
161 prev = next;
162 num_segs--;
163 }
164 xhci_link_segments(xhci, prev, ring->first_seg, link_trbs);
165
166 if (link_trbs) {
167 /* See section 4.9.2.1 and 6.4.4.1 */
168 prev->trbs[TRBS_PER_SEGMENT-1].link.control |= (LINK_TOGGLE);
169 xhci_dbg(xhci, "Wrote link toggle flag to"
700e2052
GKH		 170 " segment %p (virtual), 0x%llx (DMA)\n",
171 prev, (unsigned long long)prev->dma);
0ebbab37
SS		 172 }
173 /* The ring is empty, so the enqueue pointer == dequeue pointer */
174 ring->enqueue = ring->first_seg->trbs;
7f84eef0 175 ring->enq_seg = ring->first_seg;
0ebbab37 176 ring->dequeue = ring->enqueue;
7f84eef0 177 ring->deq_seg = ring->first_seg;
0ebbab37
SS		 178 /* The ring is initialized to 0. The producer must write 1 to the cycle
179 * bit to handover ownership of the TRB, so PCS = 1. The consumer must
180 * compare CCS to the cycle bit to check ownership, so CCS = 1.
181 */
182 ring->cycle_state = 1;
183
184 return ring;
185
186fail:
187 xhci_ring_free(xhci, ring);
188 return 0;
189}
190
d0e96f5a 191/* All the xhci_tds in the ring's TD list should be freed at this point */
3ffbba95
SS		 192void xhci_free_virt_device(struct xhci_hcd *xhci, int slot_id)
193{
194 struct xhci_virt_device *dev;
195 int i;
196
197 /* Slot ID 0 is reserved */
198 if (slot_id == 0 || !xhci->devs[slot_id])
199 return;
200
201 dev = xhci->devs[slot_id];
202 xhci->dcbaa->dev_context_ptrs[2*slot_id] = 0;
203 xhci->dcbaa->dev_context_ptrs[2*slot_id + 1] = 0;
204 if (!dev)
205 return;
206
207 for (i = 0; i < 31; ++i)
208 if (dev->ep_rings[i])
209 xhci_ring_free(xhci, dev->ep_rings[i]);
210
211 if (dev->in_ctx)
212 dma_pool_free(xhci->device_pool,
213 dev->in_ctx, dev->in_ctx_dma);
214 if (dev->out_ctx)
215 dma_pool_free(xhci->device_pool,
216 dev->out_ctx, dev->out_ctx_dma);
217 kfree(xhci->devs[slot_id]);
218 xhci->devs[slot_id] = 0;
219}
220
221int xhci_alloc_virt_device(struct xhci_hcd *xhci, int slot_id,
222 struct usb_device *udev, gfp_t flags)
223{
224 dma_addr_t dma;
225 struct xhci_virt_device *dev;
226
227 /* Slot ID 0 is reserved */
228 if (slot_id == 0 || xhci->devs[slot_id]) {
229 xhci_warn(xhci, "Bad Slot ID %d\n", slot_id);
230 return 0;
231 }
232
233 xhci->devs[slot_id] = kzalloc(sizeof(*xhci->devs[slot_id]), flags);
234 if (!xhci->devs[slot_id])
235 return 0;
236 dev = xhci->devs[slot_id];
237
238 /* Allocate the (output) device context that will be used in the HC */
239 dev->out_ctx = dma_pool_alloc(xhci->device_pool, flags, &dma);
240 if (!dev->out_ctx)
241 goto fail;
242 dev->out_ctx_dma = dma;
700e2052
GKH		 243 xhci_dbg(xhci, "Slot %d output ctx = 0x%llx (dma)\n", slot_id,
244 (unsigned long long)dma);
3ffbba95
SS		 245 memset(dev->out_ctx, 0, sizeof(*dev->out_ctx));
246
247 /* Allocate the (input) device context for address device command */
248 dev->in_ctx = dma_pool_alloc(xhci->device_pool, flags, &dma);
249 if (!dev->in_ctx)
250 goto fail;
251 dev->in_ctx_dma = dma;
700e2052
GKH		 252 xhci_dbg(xhci, "Slot %d input ctx = 0x%llx (dma)\n", slot_id,
253 (unsigned long long)dma);
3ffbba95
SS		 254 memset(dev->in_ctx, 0, sizeof(*dev->in_ctx));
255
256 /* Allocate endpoint 0 ring */
257 dev->ep_rings[0] = xhci_ring_alloc(xhci, 1, true, flags);
258 if (!dev->ep_rings[0])
259 goto fail;
260
f94e0186
SS		 261 init_completion(&dev->cmd_completion);
262
3ffbba95
SS		 263 /*
264 * Point to output device context in dcbaa; skip the output control
265 * context, which is eight 32 bit fields (or 32 bytes long)
266 */
267 xhci->dcbaa->dev_context_ptrs[2*slot_id] =
268 (u32) dev->out_ctx_dma + (32);
700e2052 269 xhci_dbg(xhci, "Set slot id %d dcbaa entry %p to 0x%llx\n",
3ffbba95 270 slot_id,
700e2052
GKH		 271 &xhci->dcbaa->dev_context_ptrs[2*slot_id],
272 (unsigned long long)dev->out_ctx_dma);
3ffbba95
SS		 273 xhci->dcbaa->dev_context_ptrs[2*slot_id + 1] = 0;
274
275 return 1;
276fail:
277 xhci_free_virt_device(xhci, slot_id);
278 return 0;
279}
280
281/* Setup an xHCI virtual device for a Set Address command */
282int xhci_setup_addressable_virt_dev(struct xhci_hcd *xhci, struct usb_device *udev)
283{
284 struct xhci_virt_device *dev;
285 struct xhci_ep_ctx *ep0_ctx;
286 struct usb_device *top_dev;
287
288 dev = xhci->devs[udev->slot_id];
289 /* Slot ID 0 is reserved */
290 if (udev->slot_id == 0 || !dev) {
291 xhci_warn(xhci, "Slot ID %d is not assigned to this device\n",
292 udev->slot_id);
293 return -EINVAL;
294 }
295 ep0_ctx = &dev->in_ctx->ep[0];
296
297 /* 2) New slot context and endpoint 0 context are valid*/
298 dev->in_ctx->add_flags = SLOT_FLAG | EP0_FLAG;
299
300 /* 3) Only the control endpoint is valid - one endpoint context */
301 dev->in_ctx->slot.dev_info |= LAST_CTX(1);
302
303 switch (udev->speed) {
304 case USB_SPEED_SUPER:
305 dev->in_ctx->slot.dev_info |= (u32) udev->route;
306 dev->in_ctx->slot.dev_info |= (u32) SLOT_SPEED_SS;
307 break;
308 case USB_SPEED_HIGH:
309 dev->in_ctx->slot.dev_info |= (u32) SLOT_SPEED_HS;
310 break;
311 case USB_SPEED_FULL:
312 dev->in_ctx->slot.dev_info |= (u32) SLOT_SPEED_FS;
313 break;
314 case USB_SPEED_LOW:
315 dev->in_ctx->slot.dev_info |= (u32) SLOT_SPEED_LS;
316 break;
317 case USB_SPEED_VARIABLE:
318 xhci_dbg(xhci, "FIXME xHCI doesn't support wireless speeds\n");
319 return -EINVAL;
320 break;
321 default:
322 /* Speed was set earlier, this shouldn't happen. */
323 BUG();
324 }
325 /* Find the root hub port this device is under */
326 for (top_dev = udev; top_dev->parent && top_dev->parent->parent;
327 top_dev = top_dev->parent)
328 /* Found device below root hub */;
329 dev->in_ctx->slot.dev_info2 |= (u32) ROOT_HUB_PORT(top_dev->portnum);
330 xhci_dbg(xhci, "Set root hub portnum to %d\n", top_dev->portnum);
331
332 /* Is this a LS/FS device under a HS hub? */
333 /*
334 * FIXME: I don't think this is right, where does the TT info for the
335 * roothub or parent hub come from?
336 */
337 if ((udev->speed == USB_SPEED_LOW || udev->speed == USB_SPEED_FULL) &&
338 udev->tt) {
339 dev->in_ctx->slot.tt_info = udev->tt->hub->slot_id;
340 dev->in_ctx->slot.tt_info |= udev->ttport << 8;
341 }
700e2052 342 xhci_dbg(xhci, "udev->tt = %p\n", udev->tt);
3ffbba95
SS		 343 xhci_dbg(xhci, "udev->ttport = 0x%x\n", udev->ttport);
344
345 /* Step 4 - ring already allocated */
346 /* Step 5 */
347 ep0_ctx->ep_info2 = EP_TYPE(CTRL_EP);
348 /*
349 * See section 4.3 bullet 6:
350 * The default Max Packet size for ep0 is "8 bytes for a USB2
351 * LS/FS/HS device or 512 bytes for a USB3 SS device"
352 * XXX: Not sure about wireless USB devices.
353 */
354 if (udev->speed == USB_SPEED_SUPER)
355 ep0_ctx->ep_info2 |= MAX_PACKET(512);
356 else
357 ep0_ctx->ep_info2 |= MAX_PACKET(8);
358 /* EP 0 can handle "burst" sizes of 1, so Max Burst Size field is 0 */
359 ep0_ctx->ep_info2 |= MAX_BURST(0);
360 ep0_ctx->ep_info2 |= ERROR_COUNT(3);
361
362 ep0_ctx->deq[0] =
363 dev->ep_rings[0]->first_seg->dma;
364 ep0_ctx->deq[0] |= dev->ep_rings[0]->cycle_state;
365 ep0_ctx->deq[1] = 0;
366
367 /* Steps 7 and 8 were done in xhci_alloc_virt_device() */
368
369 return 0;
370}
371
f94e0186
SS		 372/* Return the polling or NAK interval.
373 *
374 * The polling interval is expressed in "microframes". If xHCI's Interval field
375 * is set to N, it will service the endpoint every 2^(Interval)*125us.
376 *
377 * The NAK interval is one NAK per 1 to 255 microframes, or no NAKs if interval
378 * is set to 0.
379 */
380static inline unsigned int xhci_get_endpoint_interval(struct usb_device *udev,
381 struct usb_host_endpoint *ep)
382{
383 unsigned int interval = 0;
384
385 switch (udev->speed) {
386 case USB_SPEED_HIGH:
387 /* Max NAK rate */
388 if (usb_endpoint_xfer_control(&ep->desc) ||
389 usb_endpoint_xfer_bulk(&ep->desc))
390 interval = ep->desc.bInterval;
391 /* Fall through - SS and HS isoc/int have same decoding */
392 case USB_SPEED_SUPER:
393 if (usb_endpoint_xfer_int(&ep->desc) ||
394 usb_endpoint_xfer_isoc(&ep->desc)) {
395 if (ep->desc.bInterval == 0)
396 interval = 0;
397 else
398 interval = ep->desc.bInterval - 1;
399 if (interval > 15)
400 interval = 15;
401 if (interval != ep->desc.bInterval + 1)
402 dev_warn(&udev->dev, "ep %#x - rounding interval to %d microframes\n",
403 ep->desc.bEndpointAddress, 1 << interval);
404 }
405 break;
406 /* Convert bInterval (in 1-255 frames) to microframes and round down to
407 * nearest power of 2.
408 */
409 case USB_SPEED_FULL:
410 case USB_SPEED_LOW:
411 if (usb_endpoint_xfer_int(&ep->desc) ||
412 usb_endpoint_xfer_isoc(&ep->desc)) {
413 interval = fls(8*ep->desc.bInterval) - 1;
414 if (interval > 10)
415 interval = 10;
416 if (interval < 3)
417 interval = 3;
418 if ((1 << interval) != 8*ep->desc.bInterval)
419 dev_warn(&udev->dev, "ep %#x - rounding interval to %d microframes\n",
420 ep->desc.bEndpointAddress, 1 << interval);
421 }
422 break;
423 default:
424 BUG();
425 }
426 return EP_INTERVAL(interval);
427}
428
429static inline u32 xhci_get_endpoint_type(struct usb_device *udev,
430 struct usb_host_endpoint *ep)
431{
432 int in;
433 u32 type;
434
435 in = usb_endpoint_dir_in(&ep->desc);
436 if (usb_endpoint_xfer_control(&ep->desc)) {
437 type = EP_TYPE(CTRL_EP);
438 } else if (usb_endpoint_xfer_bulk(&ep->desc)) {
439 if (in)
440 type = EP_TYPE(BULK_IN_EP);
441 else
442 type = EP_TYPE(BULK_OUT_EP);
443 } else if (usb_endpoint_xfer_isoc(&ep->desc)) {
444 if (in)
445 type = EP_TYPE(ISOC_IN_EP);
446 else
447 type = EP_TYPE(ISOC_OUT_EP);
448 } else if (usb_endpoint_xfer_int(&ep->desc)) {
449 if (in)
450 type = EP_TYPE(INT_IN_EP);
451 else
452 type = EP_TYPE(INT_OUT_EP);
453 } else {
454 BUG();
455 }
456 return type;
457}
458
459int xhci_endpoint_init(struct xhci_hcd *xhci,
460 struct xhci_virt_device *virt_dev,
461 struct usb_device *udev,
462 struct usb_host_endpoint *ep)
463{
464 unsigned int ep_index;
465 struct xhci_ep_ctx *ep_ctx;
466 struct xhci_ring *ep_ring;
467 unsigned int max_packet;
468 unsigned int max_burst;
469
470 ep_index = xhci_get_endpoint_index(&ep->desc);
471 ep_ctx = &virt_dev->in_ctx->ep[ep_index];
472
473 /* Set up the endpoint ring */
474 virt_dev->new_ep_rings[ep_index] = xhci_ring_alloc(xhci, 1, true, GFP_KERNEL);
475 if (!virt_dev->new_ep_rings[ep_index])
476 return -ENOMEM;
477 ep_ring = virt_dev->new_ep_rings[ep_index];
f94e0186 478 ep_ctx->deq[0] = ep_ring->first_seg->dma | ep_ring->cycle_state;
3841d56e 479 ep_ctx->deq[1] = 0;
f94e0186
SS		 480
481 ep_ctx->ep_info = xhci_get_endpoint_interval(udev, ep);
482
483 /* FIXME dig Mult and streams info out of ep companion desc */
484
485 /* Allow 3 retries for everything but isoc */
486 if (!usb_endpoint_xfer_isoc(&ep->desc))
487 ep_ctx->ep_info2 = ERROR_COUNT(3);
488 else
489 ep_ctx->ep_info2 = ERROR_COUNT(0);
490
491 ep_ctx->ep_info2 |= xhci_get_endpoint_type(udev, ep);
492
493 /* Set the max packet size and max burst */
494 switch (udev->speed) {
495 case USB_SPEED_SUPER:
496 max_packet = ep->desc.wMaxPacketSize;
497 ep_ctx->ep_info2 |= MAX_PACKET(max_packet);
b10de142 498 /* dig out max burst from ep companion desc */
f0058c62 499 max_packet = ep->ss_ep_comp->desc.bMaxBurst;
b10de142 500 ep_ctx->ep_info2 |= MAX_BURST(max_packet);
f94e0186
SS		 501 break;
502 case USB_SPEED_HIGH:
503 /* bits 11:12 specify the number of additional transaction
504 * opportunities per microframe (USB 2.0, section 9.6.6)
505 */
506 if (usb_endpoint_xfer_isoc(&ep->desc) ||
507 usb_endpoint_xfer_int(&ep->desc)) {
508 max_burst = (ep->desc.wMaxPacketSize & 0x1800) >> 11;
509 ep_ctx->ep_info2 |= MAX_BURST(max_burst);
510 }
511 /* Fall through */
512 case USB_SPEED_FULL:
513 case USB_SPEED_LOW:
514 max_packet = ep->desc.wMaxPacketSize & 0x3ff;
515 ep_ctx->ep_info2 |= MAX_PACKET(max_packet);
516 break;
517 default:
518 BUG();
519 }
520 /* FIXME Debug endpoint context */
521 return 0;
522}
523
524void xhci_endpoint_zero(struct xhci_hcd *xhci,
525 struct xhci_virt_device *virt_dev,
526 struct usb_host_endpoint *ep)
527{
528 unsigned int ep_index;
529 struct xhci_ep_ctx *ep_ctx;
530
531 ep_index = xhci_get_endpoint_index(&ep->desc);
532 ep_ctx = &virt_dev->in_ctx->ep[ep_index];
533
534 ep_ctx->ep_info = 0;
535 ep_ctx->ep_info2 = 0;
f94e0186 536 ep_ctx->deq[0] = 0;
3841d56e 537 ep_ctx->deq[1] = 0;
f94e0186
SS		 538 ep_ctx->tx_info = 0;
539 /* Don't free the endpoint ring until the set interface or configuration
540 * request succeeds.
541 */
542}
543
66d4eadd
SS		 544void xhci_mem_cleanup(struct xhci_hcd *xhci)
545{
0ebbab37
SS		 546 struct pci_dev *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);
547 int size;
3ffbba95 548 int i;
0ebbab37
SS		 549
550 /* Free the Event Ring Segment Table and the actual Event Ring */
551 xhci_writel(xhci, 0, &xhci->ir_set->erst_size);
0ebbab37 552 xhci_writel(xhci, 0, &xhci->ir_set->erst_base[0]);
3841d56e 553 xhci_writel(xhci, 0, &xhci->ir_set->erst_base[1]);
0ebbab37 554 xhci_writel(xhci, 0, &xhci->ir_set->erst_dequeue[0]);
3841d56e 555 xhci_writel(xhci, 0, &xhci->ir_set->erst_dequeue[1]);
0ebbab37
SS		 556 size = sizeof(struct xhci_erst_entry)*(xhci->erst.num_entries);
557 if (xhci->erst.entries)
558 pci_free_consistent(pdev, size,
559 xhci->erst.entries, xhci->erst.erst_dma_addr);
560 xhci->erst.entries = NULL;
561 xhci_dbg(xhci, "Freed ERST\n");
562 if (xhci->event_ring)
563 xhci_ring_free(xhci, xhci->event_ring);
564 xhci->event_ring = NULL;
565 xhci_dbg(xhci, "Freed event ring\n");
566
0ebbab37 567 xhci_writel(xhci, 0, &xhci->op_regs->cmd_ring[0]);
3841d56e 568 xhci_writel(xhci, 0, &xhci->op_regs->cmd_ring[1]);
0ebbab37
SS		 569 if (xhci->cmd_ring)
570 xhci_ring_free(xhci, xhci->cmd_ring);
571 xhci->cmd_ring = NULL;
572 xhci_dbg(xhci, "Freed command ring\n");
3ffbba95
SS		 573
574 for (i = 1; i < MAX_HC_SLOTS; ++i)
575 xhci_free_virt_device(xhci, i);
576
0ebbab37
SS		 577 if (xhci->segment_pool)
578 dma_pool_destroy(xhci->segment_pool);
579 xhci->segment_pool = NULL;
580 xhci_dbg(xhci, "Freed segment pool\n");
3ffbba95
SS		 581
582 if (xhci->device_pool)
583 dma_pool_destroy(xhci->device_pool);
584 xhci->device_pool = NULL;
585 xhci_dbg(xhci, "Freed device context pool\n");
586
a74588f9 587 xhci_writel(xhci, 0, &xhci->op_regs->dcbaa_ptr[0]);
3841d56e 588 xhci_writel(xhci, 0, &xhci->op_regs->dcbaa_ptr[1]);
a74588f9
SS		 589 if (xhci->dcbaa)
590 pci_free_consistent(pdev, sizeof(*xhci->dcbaa),
591 xhci->dcbaa, xhci->dcbaa->dma);
592 xhci->dcbaa = NULL;
3ffbba95 593
66d4eadd
SS		 594 xhci->page_size = 0;
595 xhci->page_shift = 0;
596}
597
598int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags)
599{
0ebbab37
SS		 600 dma_addr_t dma;
601 struct device *dev = xhci_to_hcd(xhci)->self.controller;
66d4eadd 602 unsigned int val, val2;
0ebbab37 603 struct xhci_segment *seg;
66d4eadd
SS		 604 u32 page_size;
605 int i;
606
607 page_size = xhci_readl(xhci, &xhci->op_regs->page_size);
608 xhci_dbg(xhci, "Supported page size register = 0x%x\n", page_size);
609 for (i = 0; i < 16; i++) {
610 if ((0x1 & page_size) != 0)
611 break;
612 page_size = page_size >> 1;
613 }
614 if (i < 16)
615 xhci_dbg(xhci, "Supported page size of %iK\n", (1 << (i+12)) / 1024);
616 else
617 xhci_warn(xhci, "WARN: no supported page size\n");
618 /* Use 4K pages, since that's common and the minimum the HC supports */
619 xhci->page_shift = 12;
620 xhci->page_size = 1 << xhci->page_shift;
621 xhci_dbg(xhci, "HCD page size set to %iK\n", xhci->page_size / 1024);
622
623 /*
624 * Program the Number of Device Slots Enabled field in the CONFIG
625 * register with the max value of slots the HC can handle.
626 */
627 val = HCS_MAX_SLOTS(xhci_readl(xhci, &xhci->cap_regs->hcs_params1));
628 xhci_dbg(xhci, "// xHC can handle at most %d device slots.\n",
629 (unsigned int) val);
630 val2 = xhci_readl(xhci, &xhci->op_regs->config_reg);
631 val |= (val2 & ~HCS_SLOTS_MASK);
632 xhci_dbg(xhci, "// Setting Max device slots reg = 0x%x.\n",
633 (unsigned int) val);
634 xhci_writel(xhci, val, &xhci->op_regs->config_reg);
635
a74588f9
SS		 636 /*
637 * Section 5.4.8 - doorbell array must be
638 * "physically contiguous and 64-byte (cache line) aligned".
639 */
640 xhci->dcbaa = pci_alloc_consistent(to_pci_dev(dev),
641 sizeof(*xhci->dcbaa), &dma);
642 if (!xhci->dcbaa)
643 goto fail;
644 memset(xhci->dcbaa, 0, sizeof *(xhci->dcbaa));
645 xhci->dcbaa->dma = dma;
700e2052
GKH		 646 xhci_dbg(xhci, "// Device context base array address = 0x%llx (DMA), %p (virt)\n",
647 (unsigned long long)xhci->dcbaa->dma, xhci->dcbaa);
a74588f9 648 xhci_writel(xhci, dma, &xhci->op_regs->dcbaa_ptr[0]);
3841d56e 649 xhci_writel(xhci, (u32) 0, &xhci->op_regs->dcbaa_ptr[1]);
a74588f9 650
0ebbab37
SS		 651 /*
652 * Initialize the ring segment pool. The ring must be a contiguous
653 * structure comprised of TRBs. The TRBs must be 16 byte aligned,
654 * however, the command ring segment needs 64-byte aligned segments,
655 * so we pick the greater alignment need.
656 */
657 xhci->segment_pool = dma_pool_create("xHCI ring segments", dev,
658 SEGMENT_SIZE, 64, xhci->page_size);
3ffbba95
SS		 659 /* See Table 46 and Note on Figure 55 */
660 /* FIXME support 64-byte contexts */
661 xhci->device_pool = dma_pool_create("xHCI input/output contexts", dev,
662 sizeof(struct xhci_device_control),
663 64, xhci->page_size);
664 if (!xhci->segment_pool || !xhci->device_pool)
0ebbab37
SS		 665 goto fail;
666
667 /* Set up the command ring to have one segments for now. */
668 xhci->cmd_ring = xhci_ring_alloc(xhci, 1, true, flags);
669 if (!xhci->cmd_ring)
670 goto fail;
700e2052
GKH		 671 xhci_dbg(xhci, "Allocated command ring at %p\n", xhci->cmd_ring);
672 xhci_dbg(xhci, "First segment DMA is 0x%llx\n",
673 (unsigned long long)xhci->cmd_ring->first_seg->dma);
0ebbab37
SS		 674
675 /* Set the address in the Command Ring Control register */
676 val = xhci_readl(xhci, &xhci->op_regs->cmd_ring[0]);
677 val = (val & ~CMD_RING_ADDR_MASK) |
678 (xhci->cmd_ring->first_seg->dma & CMD_RING_ADDR_MASK) |
679 xhci->cmd_ring->cycle_state;
0ebbab37
SS		 680 xhci_dbg(xhci, "// Setting command ring address low bits to 0x%x\n", val);
681 xhci_writel(xhci, val, &xhci->op_regs->cmd_ring[0]);
3841d56e
SS		 682 xhci_dbg(xhci, "// Setting command ring address high bits to 0x0\n");
683 xhci_writel(xhci, (u32) 0, &xhci->op_regs->cmd_ring[1]);
0ebbab37
SS		 684 xhci_dbg_cmd_ptrs(xhci);
685
686 val = xhci_readl(xhci, &xhci->cap_regs->db_off);
687 val &= DBOFF_MASK;
688 xhci_dbg(xhci, "// Doorbell array is located at offset 0x%x"
689 " from cap regs base addr\n", val);
690 xhci->dba = (void *) xhci->cap_regs + val;
691 xhci_dbg_regs(xhci);
692 xhci_print_run_regs(xhci);
693 /* Set ir_set to interrupt register set 0 */
694 xhci->ir_set = (void *) xhci->run_regs->ir_set;
695
696 /*
697 * Event ring setup: Allocate a normal ring, but also setup
698 * the event ring segment table (ERST). Section 4.9.3.
699 */
700 xhci_dbg(xhci, "// Allocating event ring\n");
701 xhci->event_ring = xhci_ring_alloc(xhci, ERST_NUM_SEGS, false, flags);
702 if (!xhci->event_ring)
703 goto fail;
704
705 xhci->erst.entries = pci_alloc_consistent(to_pci_dev(dev),
706 sizeof(struct xhci_erst_entry)*ERST_NUM_SEGS, &dma);
707 if (!xhci->erst.entries)
708 goto fail;
700e2052
GKH		 709 xhci_dbg(xhci, "// Allocated event ring segment table at 0x%llx\n",
710 (unsigned long long)dma);
0ebbab37
SS		 711
712 memset(xhci->erst.entries, 0, sizeof(struct xhci_erst_entry)*ERST_NUM_SEGS);
713 xhci->erst.num_entries = ERST_NUM_SEGS;
714 xhci->erst.erst_dma_addr = dma;
700e2052 715 xhci_dbg(xhci, "Set ERST to 0; private num segs = %i, virt addr = %p, dma addr = 0x%llx\n",
0ebbab37 716 xhci->erst.num_entries,
700e2052
GKH		 717 xhci->erst.entries,
718 (unsigned long long)xhci->erst.erst_dma_addr);
0ebbab37
SS		 719
720 /* set ring base address and size for each segment table entry */
721 for (val = 0, seg = xhci->event_ring->first_seg; val < ERST_NUM_SEGS; val++) {
722 struct xhci_erst_entry *entry = &xhci->erst.entries[val];
0ebbab37 723 entry->seg_addr[0] = seg->dma;
3841d56e 724 entry->seg_addr[1] = 0;
0ebbab37
SS		 725 entry->seg_size = TRBS_PER_SEGMENT;
726 entry->rsvd = 0;
727 seg = seg->next;
728 }
729
730 /* set ERST count with the number of entries in the segment table */
731 val = xhci_readl(xhci, &xhci->ir_set->erst_size);
732 val &= ERST_SIZE_MASK;
733 val |= ERST_NUM_SEGS;
734 xhci_dbg(xhci, "// Write ERST size = %i to ir_set 0 (some bits preserved)\n",
735 val);
736 xhci_writel(xhci, val, &xhci->ir_set->erst_size);
737
738 xhci_dbg(xhci, "// Set ERST entries to point to event ring.\n");
739 /* set the segment table base address */
700e2052
GKH		 740 xhci_dbg(xhci, "// Set ERST base address for ir_set 0 = 0x%llx\n",
741 (unsigned long long)xhci->erst.erst_dma_addr);
0ebbab37
SS		 742 val = xhci_readl(xhci, &xhci->ir_set->erst_base[0]);
743 val &= ERST_PTR_MASK;
744 val |= (xhci->erst.erst_dma_addr & ~ERST_PTR_MASK);
745 xhci_writel(xhci, val, &xhci->ir_set->erst_base[0]);
3841d56e 746 xhci_writel(xhci, 0, &xhci->ir_set->erst_base[1]);
0ebbab37
SS		 747
748 /* Set the event ring dequeue address */
23e3be11 749 xhci_set_hc_event_deq(xhci);
0ebbab37
SS		 750 xhci_dbg(xhci, "Wrote ERST address to ir_set 0.\n");
751 xhci_print_ir_set(xhci, xhci->ir_set, 0);
752
753 /*
754 * XXX: Might need to set the Interrupter Moderation Register to
755 * something other than the default (~1ms minimum between interrupts).
756 * See section 5.5.1.2.
757 */
3ffbba95
SS		 758 init_completion(&xhci->addr_dev);
759 for (i = 0; i < MAX_HC_SLOTS; ++i)
760 xhci->devs[i] = 0;
66d4eadd
SS		 761
762 return 0;
763fail:
764 xhci_warn(xhci, "Couldn't initialize memory\n");
765 xhci_mem_cleanup(xhci);
766 return -ENOMEM;
767}
Linux 6.x block layer and io_uring tree(s)