projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
nvme: optimise io_uring passthrough completion
[linux-block.git] / drivers / usb / cdns3 / cdnsp-gadget.c
CommitLineData
3d829045
PL		 1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Cadence CDNSP DRD Driver.
4 *
5 * Copyright (C) 2020 Cadence.
6 *
7 * Author: Pawel Laszczak <pawell@cadence.com>
8 *
9 */
10
11#include <linux/moduleparam.h>
12#include <linux/dma-mapping.h>
13#include <linux/module.h>
14#include <linux/iopoll.h>
15#include <linux/delay.h>
16#include <linux/log2.h>
17#include <linux/slab.h>
18#include <linux/pci.h>
19#include <linux/irq.h>
20#include <linux/dmi.h>
21
22#include "core.h"
23#include "gadget-export.h"
24#include "drd.h"
25#include "cdnsp-gadget.h"
118b2a32 26#include "cdnsp-trace.h"
3d829045
PL		 27
28unsigned int cdnsp_port_speed(unsigned int port_status)
29{
30 /*Detect gadget speed based on PORTSC register*/
31 if (DEV_SUPERSPEEDPLUS(port_status))
32 return USB_SPEED_SUPER_PLUS;
33 else if (DEV_SUPERSPEED(port_status))
34 return USB_SPEED_SUPER;
35 else if (DEV_HIGHSPEED(port_status))
36 return USB_SPEED_HIGH;
37 else if (DEV_FULLSPEED(port_status))
38 return USB_SPEED_FULL;
39
40 /* If device is detached then speed will be USB_SPEED_UNKNOWN.*/
41 return USB_SPEED_UNKNOWN;
42}
43
44/*
45 * Given a port state, this function returns a value that would result in the
46 * port being in the same state, if the value was written to the port status
47 * control register.
48 * Save Read Only (RO) bits and save read/write bits where
49 * writing a 0 clears the bit and writing a 1 sets the bit (RWS).
50 * For all other types (RW1S, RW1CS, RW, and RZ), writing a '0' has no effect.
51 */
52u32 cdnsp_port_state_to_neutral(u32 state)
53{
54 /* Save read-only status and port state. */
55 return (state & CDNSP_PORT_RO) | (state & CDNSP_PORT_RWS);
56}
57
58/**
632d234b
LJ		 59 * cdnsp_find_next_ext_cap - Find the offset of the extended capabilities
60 * with capability ID id.
3d829045
PL		 61 * @base: PCI MMIO registers base address.
62 * @start: Address at which to start looking, (0 or HCC_PARAMS to start at
63 * beginning of list)
64 * @id: Extended capability ID to search for.
65 *
66 * Returns the offset of the next matching extended capability structure.
67 * Some capabilities can occur several times,
68 * e.g., the EXT_CAPS_PROTOCOL, and this provides a way to find them all.
69 */
70int cdnsp_find_next_ext_cap(void __iomem *base, u32 start, int id)
71{
72 u32 offset = start;
73 u32 next;
74 u32 val;
75
76 if (!start || start == HCC_PARAMS_OFFSET) {
77 val = readl(base + HCC_PARAMS_OFFSET);
78 if (val == ~0)
79 return 0;
80
81 offset = HCC_EXT_CAPS(val) << 2;
82 if (!offset)
83 return 0;
9c8846c7 84 }
3d829045
PL		 85
86 do {
87 val = readl(base + offset);
88 if (val == ~0)
89 return 0;
90
91 if (EXT_CAPS_ID(val) == id && offset != start)
92 return offset;
93
94 next = EXT_CAPS_NEXT(val);
95 offset += next << 2;
96 } while (next);
97
98 return 0;
99}
100
101void cdnsp_set_link_state(struct cdnsp_device *pdev,
102 __le32 __iomem *port_regs,
103 u32 link_state)
104{
118b2a32 105 int port_num = 0xFF;
3d829045
PL		 106 u32 temp;
107
108 temp = readl(port_regs);
109 temp = cdnsp_port_state_to_neutral(temp);
110 temp |= PORT_WKCONN_E | PORT_WKDISC_E;
111 writel(temp, port_regs);
112
113 temp &= ~PORT_PLS_MASK;
114 temp |= PORT_LINK_STROBE | link_state;
115
118b2a32
PL		 116 if (pdev->active_port)
117 port_num = pdev->active_port->port_num;
118
119 trace_cdnsp_handle_port_status(port_num, readl(port_regs));
3d829045 120 writel(temp, port_regs);
118b2a32 121 trace_cdnsp_link_state_changed(port_num, readl(port_regs));
3d829045
PL		 122}
123
124static void cdnsp_disable_port(struct cdnsp_device *pdev,
125 __le32 __iomem *port_regs)
126{
127 u32 temp = cdnsp_port_state_to_neutral(readl(port_regs));
128
129 writel(temp | PORT_PED, port_regs);
130}
131
132static void cdnsp_clear_port_change_bit(struct cdnsp_device *pdev,
133 __le32 __iomem *port_regs)
134{
135 u32 portsc = readl(port_regs);
136
137 writel(cdnsp_port_state_to_neutral(portsc) |
138 (portsc & PORT_CHANGE_BITS), port_regs);
139}
140
141static void cdnsp_set_chicken_bits_2(struct cdnsp_device *pdev, u32 bit)
142{
143 __le32 __iomem *reg;
144 void __iomem *base;
145 u32 offset = 0;
146
147 base = &pdev->cap_regs->hc_capbase;
148 offset = cdnsp_find_next_ext_cap(base, offset, D_XEC_PRE_REGS_CAP);
149 reg = base + offset + REG_CHICKEN_BITS_2_OFFSET;
150
151 bit = readl(reg) | bit;
152 writel(bit, reg);
153}
154
155static void cdnsp_clear_chicken_bits_2(struct cdnsp_device *pdev, u32 bit)
156{
157 __le32 __iomem *reg;
158 void __iomem *base;
159 u32 offset = 0;
160
161 base = &pdev->cap_regs->hc_capbase;
162 offset = cdnsp_find_next_ext_cap(base, offset, D_XEC_PRE_REGS_CAP);
163 reg = base + offset + REG_CHICKEN_BITS_2_OFFSET;
164
165 bit = readl(reg) & ~bit;
166 writel(bit, reg);
167}
168
169/*
170 * Disable interrupts and begin the controller halting process.
171 */
172static void cdnsp_quiesce(struct cdnsp_device *pdev)
173{
174 u32 halted;
175 u32 mask;
176 u32 cmd;
177
178 mask = ~(u32)(CDNSP_IRQS);
179
180 halted = readl(&pdev->op_regs->status) & STS_HALT;
181 if (!halted)
182 mask &= ~(CMD_R_S | CMD_DEVEN);
183
184 cmd = readl(&pdev->op_regs->command);
185 cmd &= mask;
186 writel(cmd, &pdev->op_regs->command);
187}
188
189/*
190 * Force controller into halt state.
191 *
192 * Disable any IRQs and clear the run/stop bit.
193 * Controller will complete any current and actively pipelined transactions, and
194 * should halt within 16 ms of the run/stop bit being cleared.
195 * Read controller Halted bit in the status register to see when the
196 * controller is finished.
197 */
198int cdnsp_halt(struct cdnsp_device *pdev)
199{
200 int ret;
201 u32 val;
202
203 cdnsp_quiesce(pdev);
204
205 ret = readl_poll_timeout_atomic(&pdev->op_regs->status, val,
206 val & STS_HALT, 1,
207 CDNSP_MAX_HALT_USEC);
208 if (ret) {
209 dev_err(pdev->dev, "ERROR: Device halt failed\n");
210 return ret;
211 }
212
213 pdev->cdnsp_state |= CDNSP_STATE_HALTED;
214
215 return 0;
216}
217
218/*
219 * device controller died, register read returns 0xffffffff, or command never
220 * ends.
221 */
222void cdnsp_died(struct cdnsp_device *pdev)
223{
224 dev_err(pdev->dev, "ERROR: CDNSP controller not responding\n");
225 pdev->cdnsp_state |= CDNSP_STATE_DYING;
226 cdnsp_halt(pdev);
227}
228
229/*
230 * Set the run bit and wait for the device to be running.
231 */
232static int cdnsp_start(struct cdnsp_device *pdev)
233{
234 u32 temp;
235 int ret;
236
237 temp = readl(&pdev->op_regs->command);
238 temp |= (CMD_R_S | CMD_DEVEN);
239 writel(temp, &pdev->op_regs->command);
240
241 pdev->cdnsp_state = 0;
242
243 /*
244 * Wait for the STS_HALT Status bit to be 0 to indicate the device is
245 * running.
246 */
247 ret = readl_poll_timeout_atomic(&pdev->op_regs->status, temp,
248 !(temp & STS_HALT), 1,
249 CDNSP_MAX_HALT_USEC);
250 if (ret) {
251 pdev->cdnsp_state = CDNSP_STATE_DYING;
252 dev_err(pdev->dev, "ERROR: Controller run failed\n");
253 }
254
255 return ret;
256}
257
258/*
259 * Reset a halted controller.
260 *
261 * This resets pipelines, timers, counters, state machines, etc.
262 * Transactions will be terminated immediately, and operational registers
263 * will be set to their defaults.
264 */
265int cdnsp_reset(struct cdnsp_device *pdev)
266{
267 u32 command;
268 u32 temp;
269 int ret;
270
271 temp = readl(&pdev->op_regs->status);
272
273 if (temp == ~(u32)0) {
274 dev_err(pdev->dev, "Device not accessible, reset failed.\n");
275 return -ENODEV;
276 }
277
278 if ((temp & STS_HALT) == 0) {
279 dev_err(pdev->dev, "Controller not halted, aborting reset.\n");
280 return -EINVAL;
281 }
282
283 command = readl(&pdev->op_regs->command);
284 command |= CMD_RESET;
285 writel(command, &pdev->op_regs->command);
286
287 ret = readl_poll_timeout_atomic(&pdev->op_regs->command, temp,
288 !(temp & CMD_RESET), 1,
289 10 * 1000);
290 if (ret) {
291 dev_err(pdev->dev, "ERROR: Controller reset failed\n");
292 return ret;
293 }
294
295 /*
296 * CDNSP cannot write any doorbells or operational registers other
297 * than status until the "Controller Not Ready" flag is cleared.
298 */
299 ret = readl_poll_timeout_atomic(&pdev->op_regs->status, temp,
300 !(temp & STS_CNR), 1,
301 10 * 1000);
302
303 if (ret) {
304 dev_err(pdev->dev, "ERROR: Controller not ready to work\n");
305 return ret;
306 }
307
308 dev_dbg(pdev->dev, "Controller ready to work");
309
310 return ret;
311}
312
313/*
314 * cdnsp_get_endpoint_index - Find the index for an endpoint given its
315 * descriptor.Use the return value to right shift 1 for the bitmask.
316 *
317 * Index = (epnum * 2) + direction - 1,
318 * where direction = 0 for OUT, 1 for IN.
319 * For control endpoints, the IN index is used (OUT index is unused), so
320 * index = (epnum * 2) + direction - 1 = (epnum * 2) + 1 - 1 = (epnum * 2)
321 */
322static unsigned int
323 cdnsp_get_endpoint_index(const struct usb_endpoint_descriptor *desc)
324{
325 unsigned int index = (unsigned int)usb_endpoint_num(desc);
326
327 if (usb_endpoint_xfer_control(desc))
328 return index * 2;
329
330 return (index * 2) + (usb_endpoint_dir_in(desc) ? 1 : 0) - 1;
331}
332
333/*
334 * Find the flag for this endpoint (for use in the control context). Use the
335 * endpoint index to create a bitmask. The slot context is bit 0, endpoint 0 is
336 * bit 1, etc.
337 */
338static unsigned int
339 cdnsp_get_endpoint_flag(const struct usb_endpoint_descriptor *desc)
340{
341 return 1 << (cdnsp_get_endpoint_index(desc) + 1);
342}
343
344int cdnsp_ep_enqueue(struct cdnsp_ep *pep, struct cdnsp_request *preq)
345{
346 struct cdnsp_device *pdev = pep->pdev;
347 struct usb_request *request;
348 int ret;
349
118b2a32
PL		 350 if (preq->epnum == 0 && !list_empty(&pep->pending_list)) {
351 trace_cdnsp_request_enqueue_busy(preq);
3d829045 352 return -EBUSY;
118b2a32 353 }
3d829045
PL		 354
355 request = &preq->request;
356 request->actual = 0;
357 request->status = -EINPROGRESS;
358 preq->direction = pep->direction;
359 preq->epnum = pep->number;
360 preq->td.drbl = 0;
361
362 ret = usb_gadget_map_request_by_dev(pdev->dev, request, pep->direction);
118b2a32
PL		 363 if (ret) {
364 trace_cdnsp_request_enqueue_error(preq);
3d829045 365 return ret;
118b2a32 366 }
3d829045
PL		 367
368 list_add_tail(&preq->list, &pep->pending_list);
369
118b2a32
PL		 370 trace_cdnsp_request_enqueue(preq);
371
3d829045
PL		 372 switch (usb_endpoint_type(pep->endpoint.desc)) {
373 case USB_ENDPOINT_XFER_CONTROL:
374 ret = cdnsp_queue_ctrl_tx(pdev, preq);
375 break;
376 case USB_ENDPOINT_XFER_BULK:
377 case USB_ENDPOINT_XFER_INT:
378 ret = cdnsp_queue_bulk_tx(pdev, preq);
379 break;
380 case USB_ENDPOINT_XFER_ISOC:
82b0417e 381 ret = cdnsp_queue_isoc_tx(pdev, preq);
3d829045
PL		 382 }
383
384 if (ret)
385 goto unmap;
386
387 return 0;
388
389unmap:
390 usb_gadget_unmap_request_by_dev(pdev->dev, &preq->request,
391 pep->direction);
392 list_del(&preq->list);
118b2a32 393 trace_cdnsp_request_enqueue_error(preq);
3d829045
PL		 394
395 return ret;
396}
397
398/*
399 * Remove the request's TD from the endpoint ring. This may cause the
400 * controller to stop USB transfers, potentially stopping in the middle of a
401 * TRB buffer. The controller should pick up where it left off in the TD,
402 * unless a Set Transfer Ring Dequeue Pointer is issued.
403 *
404 * The TRBs that make up the buffers for the canceled request will be "removed"
405 * from the ring. Since the ring is a contiguous structure, they can't be
406 * physically removed. Instead, there are two options:
407 *
408 * 1) If the controller is in the middle of processing the request to be
409 * canceled, we simply move the ring's dequeue pointer past those TRBs
410 * using the Set Transfer Ring Dequeue Pointer command. This will be
411 * the common case, when drivers timeout on the last submitted request
412 * and attempt to cancel.
413 *
414 * 2) If the controller is in the middle of a different TD, we turn the TRBs
415 * into a series of 1-TRB transfer no-op TDs. No-ops shouldn't be chained.
416 * The controller will need to invalidate the any TRBs it has cached after
417 * the stop endpoint command.
418 *
419 * 3) The TD may have completed by the time the Stop Endpoint Command
420 * completes, so software needs to handle that case too.
421 *
422 */
423int cdnsp_ep_dequeue(struct cdnsp_ep *pep, struct cdnsp_request *preq)
424{
425 struct cdnsp_device *pdev = pep->pdev;
3b414d1b
PL		 426 int ret_stop = 0;
427 int ret_rem;
3d829045 428
118b2a32
PL		 429 trace_cdnsp_request_dequeue(preq);
430
3b414d1b
PL		 431 if (GET_EP_CTX_STATE(pep->out_ctx) == EP_STATE_RUNNING)
432 ret_stop = cdnsp_cmd_stop_ep(pdev, pep);
433
434 ret_rem = cdnsp_remove_request(pdev, preq, pep);
3d829045 435
3b414d1b 436 return ret_rem ? ret_rem : ret_stop;
3d829045
PL		 437}
438
439static void cdnsp_zero_in_ctx(struct cdnsp_device *pdev)
440{
441 struct cdnsp_input_control_ctx *ctrl_ctx;
442 struct cdnsp_slot_ctx *slot_ctx;
443 struct cdnsp_ep_ctx *ep_ctx;
444 int i;
445
446 ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
447
448 /*
449 * When a device's add flag and drop flag are zero, any subsequent
450 * configure endpoint command will leave that endpoint's state
451 * untouched. Make sure we don't leave any old state in the input
452 * endpoint contexts.
453 */
454 ctrl_ctx->drop_flags = 0;
455 ctrl_ctx->add_flags = 0;
456 slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
457 slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
458
459 /* Endpoint 0 is always valid */
460 slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1));
461 for (i = 1; i < CDNSP_ENDPOINTS_NUM; ++i) {
462 ep_ctx = cdnsp_get_ep_ctx(&pdev->in_ctx, i);
463 ep_ctx->ep_info = 0;
464 ep_ctx->ep_info2 = 0;
465 ep_ctx->deq = 0;
466 ep_ctx->tx_info = 0;
467 }
468}
469
470/* Issue a configure endpoint command and wait for it to finish. */
471static int cdnsp_configure_endpoint(struct cdnsp_device *pdev)
472{
473 int ret;
474
475 cdnsp_queue_configure_endpoint(pdev, pdev->cmd.in_ctx->dma);
476 cdnsp_ring_cmd_db(pdev);
477 ret = cdnsp_wait_for_cmd_compl(pdev);
478 if (ret) {
479 dev_err(pdev->dev,
480 "ERR: unexpected command completion code 0x%x.\n", ret);
481 return -EINVAL;
482 }
483
484 return ret;
485}
486
487static void cdnsp_invalidate_ep_events(struct cdnsp_device *pdev,
488 struct cdnsp_ep *pep)
489{
490 struct cdnsp_segment *segment;
491 union cdnsp_trb *event;
492 u32 cycle_state;
16e36101 493 u32 data;
3d829045
PL		 494
495 event = pdev->event_ring->dequeue;
496 segment = pdev->event_ring->deq_seg;
497 cycle_state = pdev->event_ring->cycle_state;
498
499 while (1) {
500 data = le32_to_cpu(event->trans_event.flags);
501
502 /* Check the owner of the TRB. */
503 if ((data & TRB_CYCLE) != cycle_state)
504 break;
505
506 if (TRB_FIELD_TO_TYPE(data) == TRB_TRANSFER &&
507 TRB_TO_EP_ID(data) == (pep->idx + 1)) {
508 data |= TRB_EVENT_INVALIDATE;
509 event->trans_event.flags = cpu_to_le32(data);
510 }
511
512 if (cdnsp_last_trb_on_seg(segment, event)) {
513 cycle_state ^= 1;
514 segment = pdev->event_ring->deq_seg->next;
515 event = segment->trbs;
516 } else {
517 event++;
518 }
519 }
520}
521
522int cdnsp_wait_for_cmd_compl(struct cdnsp_device *pdev)
523{
524 struct cdnsp_segment *event_deq_seg;
525 union cdnsp_trb *cmd_trb;
526 dma_addr_t cmd_deq_dma;
527 union cdnsp_trb *event;
528 u32 cycle_state;
3d829045
PL		 529 int ret, val;
530 u64 cmd_dma;
16e36101 531 u32 flags;
3d829045
PL		 532
533 cmd_trb = pdev->cmd.command_trb;
534 pdev->cmd.status = 0;
535
118b2a32
PL		 536 trace_cdnsp_cmd_wait_for_compl(pdev->cmd_ring, &cmd_trb->generic);
537
3d829045
PL		 538 ret = readl_poll_timeout_atomic(&pdev->op_regs->cmd_ring, val,
539 !CMD_RING_BUSY(val), 1,
540 CDNSP_CMD_TIMEOUT);
541 if (ret) {
542 dev_err(pdev->dev, "ERR: Timeout while waiting for command\n");
118b2a32 543 trace_cdnsp_cmd_timeout(pdev->cmd_ring, &cmd_trb->generic);
3d829045
PL		 544 pdev->cdnsp_state = CDNSP_STATE_DYING;
545 return -ETIMEDOUT;
546 }
547
548 event = pdev->event_ring->dequeue;
549 event_deq_seg = pdev->event_ring->deq_seg;
550 cycle_state = pdev->event_ring->cycle_state;
551
552 cmd_deq_dma = cdnsp_trb_virt_to_dma(pdev->cmd_ring->deq_seg, cmd_trb);
553 if (!cmd_deq_dma)
554 return -EINVAL;
555
556 while (1) {
557 flags = le32_to_cpu(event->event_cmd.flags);
558
559 /* Check the owner of the TRB. */
560 if ((flags & TRB_CYCLE) != cycle_state)
561 return -EINVAL;
562
563 cmd_dma = le64_to_cpu(event->event_cmd.cmd_trb);
564
565 /*
566 * Check whether the completion event is for last queued
567 * command.
568 */
569 if (TRB_FIELD_TO_TYPE(flags) != TRB_COMPLETION ||
570 cmd_dma != (u64)cmd_deq_dma) {
571 if (!cdnsp_last_trb_on_seg(event_deq_seg, event)) {
572 event++;
573 continue;
574 }
575
576 if (cdnsp_last_trb_on_ring(pdev->event_ring,
577 event_deq_seg, event))
578 cycle_state ^= 1;
579
580 event_deq_seg = event_deq_seg->next;
581 event = event_deq_seg->trbs;
582 continue;
583 }
584
118b2a32
PL		 585 trace_cdnsp_handle_command(pdev->cmd_ring, &cmd_trb->generic);
586
3d829045
PL		 587 pdev->cmd.status = GET_COMP_CODE(le32_to_cpu(event->event_cmd.status));
588 if (pdev->cmd.status == COMP_SUCCESS)
589 return 0;
590
591 return -pdev->cmd.status;
592 }
593}
594
595int cdnsp_halt_endpoint(struct cdnsp_device *pdev,
596 struct cdnsp_ep *pep,
597 int value)
598{
599 int ret;
600
118b2a32
PL		 601 trace_cdnsp_ep_halt(value ? "Set" : "Clear");
602
b25264f2
PL		 603 ret = cdnsp_cmd_stop_ep(pdev, pep);
604 if (ret)
605 return ret;
3d829045 606
b25264f2 607 if (value) {
3d829045
PL		 608 if (GET_EP_CTX_STATE(pep->out_ctx) == EP_STATE_STOPPED) {
609 cdnsp_queue_halt_endpoint(pdev, pep->idx);
610 cdnsp_ring_cmd_db(pdev);
611 ret = cdnsp_wait_for_cmd_compl(pdev);
612 }
613
614 pep->ep_state |= EP_HALTED;
615 } else {
3d829045
PL		 616 cdnsp_queue_reset_ep(pdev, pep->idx);
617 cdnsp_ring_cmd_db(pdev);
618 ret = cdnsp_wait_for_cmd_compl(pdev);
118b2a32
PL		 619 trace_cdnsp_handle_cmd_reset_ep(pep->out_ctx);
620
3d829045
PL		 621 if (ret)
622 return ret;
623
624 pep->ep_state &= ~EP_HALTED;
625
626 if (pep->idx != 0 && !(pep->ep_state & EP_WEDGE))
627 cdnsp_ring_doorbell_for_active_rings(pdev, pep);
628
629 pep->ep_state &= ~EP_WEDGE;
630 }
631
632 return 0;
633}
634
635static int cdnsp_update_eps_configuration(struct cdnsp_device *pdev,
636 struct cdnsp_ep *pep)
637{
638 struct cdnsp_input_control_ctx *ctrl_ctx;
639 struct cdnsp_slot_ctx *slot_ctx;
640 int ret = 0;
641 u32 ep_sts;
642 int i;
643
644 ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
645
646 /* Don't issue the command if there's no endpoints to update. */
647 if (ctrl_ctx->add_flags == 0 && ctrl_ctx->drop_flags == 0)
648 return 0;
649
650 ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
651 ctrl_ctx->add_flags &= cpu_to_le32(~EP0_FLAG);
652 ctrl_ctx->drop_flags &= cpu_to_le32(~(SLOT_FLAG | EP0_FLAG));
653
654 /* Fix up Context Entries field. Minimum value is EP0 == BIT(1). */
655 slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
656 for (i = CDNSP_ENDPOINTS_NUM; i >= 1; i--) {
657 __le32 le32 = cpu_to_le32(BIT(i));
658
659 if ((pdev->eps[i - 1].ring && !(ctrl_ctx->drop_flags & le32)) ||
660 (ctrl_ctx->add_flags & le32) || i == 1) {
661 slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
662 slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(i));
663 break;
664 }
665 }
666
667 ep_sts = GET_EP_CTX_STATE(pep->out_ctx);
668
669 if ((ctrl_ctx->add_flags != cpu_to_le32(SLOT_FLAG) &&
670 ep_sts == EP_STATE_DISABLED) ||
671 (ep_sts != EP_STATE_DISABLED && ctrl_ctx->drop_flags))
672 ret = cdnsp_configure_endpoint(pdev);
673
118b2a32
PL		 674 trace_cdnsp_configure_endpoint(cdnsp_get_slot_ctx(&pdev->out_ctx));
675 trace_cdnsp_handle_cmd_config_ep(pep->out_ctx);
676
3d829045
PL		 677 cdnsp_zero_in_ctx(pdev);
678
679 return ret;
680}
681
682/*
683 * This submits a Reset Device Command, which will set the device state to 0,
684 * set the device address to 0, and disable all the endpoints except the default
685 * control endpoint. The USB core should come back and call
686 * cdnsp_setup_device(), and then re-set up the configuration.
687 */
688int cdnsp_reset_device(struct cdnsp_device *pdev)
689{
690 struct cdnsp_slot_ctx *slot_ctx;
691 int slot_state;
692 int ret, i;
693
694 slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
695 slot_ctx->dev_info = 0;
696 pdev->device_address = 0;
697
698 /* If device is not setup, there is no point in resetting it. */
699 slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx);
700 slot_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state));
118b2a32 701 trace_cdnsp_reset_device(slot_ctx);
3d829045
PL		 702
703 if (slot_state <= SLOT_STATE_DEFAULT &&
704 pdev->eps[0].ep_state & EP_HALTED) {
705 cdnsp_halt_endpoint(pdev, &pdev->eps[0], 0);
706 }
707
708 /*
709 * During Reset Device command controller shall transition the
710 * endpoint ep0 to the Running State.
711 */
712 pdev->eps[0].ep_state &= ~(EP_STOPPED | EP_HALTED);
713 pdev->eps[0].ep_state |= EP_ENABLED;
714
715 if (slot_state <= SLOT_STATE_DEFAULT)
716 return 0;
717
718 cdnsp_queue_reset_device(pdev);
719 cdnsp_ring_cmd_db(pdev);
720 ret = cdnsp_wait_for_cmd_compl(pdev);
721
722 /*
723 * After Reset Device command all not default endpoints
724 * are in Disabled state.
725 */
726 for (i = 1; i < CDNSP_ENDPOINTS_NUM; ++i)
10076de3 727 pdev->eps[i].ep_state |= EP_STOPPED | EP_UNCONFIGURED;
3d829045 728
118b2a32
PL		 729 trace_cdnsp_handle_cmd_reset_dev(slot_ctx);
730
3d829045
PL		 731 if (ret)
732 dev_err(pdev->dev, "Reset device failed with error code %d",
733 ret);
734
735 return ret;
736}
737
738/*
739 * Sets the MaxPStreams field and the Linear Stream Array field.
740 * Sets the dequeue pointer to the stream context array.
741 */
742static void cdnsp_setup_streams_ep_input_ctx(struct cdnsp_device *pdev,
743 struct cdnsp_ep_ctx *ep_ctx,
744 struct cdnsp_stream_info *stream_info)
745{
746 u32 max_primary_streams;
747
748 /* MaxPStreams is the number of stream context array entries, not the
749 * number we're actually using. Must be in 2^(MaxPstreams + 1) format.
750 * fls(0) = 0, fls(0x1) = 1, fls(0x10) = 2, fls(0x100) = 3, etc.
751 */
752 max_primary_streams = fls(stream_info->num_stream_ctxs) - 2;
753 ep_ctx->ep_info &= cpu_to_le32(~EP_MAXPSTREAMS_MASK);
754 ep_ctx->ep_info |= cpu_to_le32(EP_MAXPSTREAMS(max_primary_streams)
755 | EP_HAS_LSA);
756 ep_ctx->deq = cpu_to_le64(stream_info->ctx_array_dma);
757}
758
759/*
760 * The drivers use this function to prepare a bulk endpoints to use streams.
761 *
762 * Don't allow the call to succeed if endpoint only supports one stream
763 * (which means it doesn't support streams at all).
764 */
765int cdnsp_alloc_streams(struct cdnsp_device *pdev, struct cdnsp_ep *pep)
766{
767 unsigned int num_streams = usb_ss_max_streams(pep->endpoint.comp_desc);
768 unsigned int num_stream_ctxs;
769 int ret;
770
771 if (num_streams == 0)
772 return 0;
773
774 if (num_streams > STREAM_NUM_STREAMS)
775 return -EINVAL;
776
777 /*
778 * Add two to the number of streams requested to account for
779 * stream 0 that is reserved for controller usage and one additional
780 * for TASK SET FULL response.
781 */
782 num_streams += 2;
783
784 /* The stream context array size must be a power of two */
785 num_stream_ctxs = roundup_pow_of_two(num_streams);
786
118b2a32
PL		 787 trace_cdnsp_stream_number(pep, num_stream_ctxs, num_streams);
788
3d829045
PL		 789 ret = cdnsp_alloc_stream_info(pdev, pep, num_stream_ctxs, num_streams);
790 if (ret)
791 return ret;
792
793 cdnsp_setup_streams_ep_input_ctx(pdev, pep->in_ctx, &pep->stream_info);
794
795 pep->ep_state |= EP_HAS_STREAMS;
796 pep->stream_info.td_count = 0;
797 pep->stream_info.first_prime_det = 0;
798
799 /* Subtract 1 for stream 0, which drivers can't use. */
800 return num_streams - 1;
801}
802
803int cdnsp_disable_slot(struct cdnsp_device *pdev)
804{
805 int ret;
806
807 cdnsp_queue_slot_control(pdev, TRB_DISABLE_SLOT);
808 cdnsp_ring_cmd_db(pdev);
809 ret = cdnsp_wait_for_cmd_compl(pdev);
810
811 pdev->slot_id = 0;
812 pdev->active_port = NULL;
813
118b2a32
PL		 814 trace_cdnsp_handle_cmd_disable_slot(cdnsp_get_slot_ctx(&pdev->out_ctx));
815
3d829045
PL		 816 memset(pdev->in_ctx.bytes, 0, CDNSP_CTX_SIZE);
817 memset(pdev->out_ctx.bytes, 0, CDNSP_CTX_SIZE);
818
819 return ret;
820}
821
822int cdnsp_enable_slot(struct cdnsp_device *pdev)
823{
824 struct cdnsp_slot_ctx *slot_ctx;
825 int slot_state;
826 int ret;
827
828 /* If device is not setup, there is no point in resetting it */
829 slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx);
830 slot_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state));
831
832 if (slot_state != SLOT_STATE_DISABLED)
833 return 0;
834
835 cdnsp_queue_slot_control(pdev, TRB_ENABLE_SLOT);
836 cdnsp_ring_cmd_db(pdev);
837 ret = cdnsp_wait_for_cmd_compl(pdev);
838 if (ret)
118b2a32 839 goto show_trace;
3d829045
PL		 840
841 pdev->slot_id = 1;
842
118b2a32
PL		 843show_trace:
844 trace_cdnsp_handle_cmd_enable_slot(cdnsp_get_slot_ctx(&pdev->out_ctx));
845
846 return ret;
3d829045
PL		 847}
848
849/*
850 * Issue an Address Device command with BSR=0 if setup is SETUP_CONTEXT_ONLY
851 * or with BSR = 1 if set_address is SETUP_CONTEXT_ADDRESS.
852 */
853int cdnsp_setup_device(struct cdnsp_device *pdev, enum cdnsp_setup_dev setup)
854{
855 struct cdnsp_input_control_ctx *ctrl_ctx;
856 struct cdnsp_slot_ctx *slot_ctx;
857 int dev_state = 0;
858 int ret;
859
118b2a32
PL		 860 if (!pdev->slot_id) {
861 trace_cdnsp_slot_id("incorrect");
3d829045 862 return -EINVAL;
118b2a32 863 }
3d829045
PL		 864
865 if (!pdev->active_port->port_num)
866 return -EINVAL;
867
868 slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx);
869 dev_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state));
870
118b2a32
PL		 871 if (setup == SETUP_CONTEXT_ONLY && dev_state == SLOT_STATE_DEFAULT) {
872 trace_cdnsp_slot_already_in_default(slot_ctx);
3d829045 873 return 0;
118b2a32 874 }
3d829045
PL		 875
876 slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
877 ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
878
879 if (!slot_ctx->dev_info || dev_state == SLOT_STATE_DEFAULT) {
880 ret = cdnsp_setup_addressable_priv_dev(pdev);
881 if (ret)
882 return ret;
883 }
884
885 cdnsp_copy_ep0_dequeue_into_input_ctx(pdev);
886
887 ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG | EP0_FLAG);
888 ctrl_ctx->drop_flags = 0;
889
118b2a32
PL		 890 trace_cdnsp_setup_device_slot(slot_ctx);
891
3d829045
PL		 892 cdnsp_queue_address_device(pdev, pdev->in_ctx.dma, setup);
893 cdnsp_ring_cmd_db(pdev);
894 ret = cdnsp_wait_for_cmd_compl(pdev);
895
118b2a32
PL		 896 trace_cdnsp_handle_cmd_addr_dev(cdnsp_get_slot_ctx(&pdev->out_ctx));
897
3d829045
PL		 898 /* Zero the input context control for later use. */
899 ctrl_ctx->add_flags = 0;
900 ctrl_ctx->drop_flags = 0;
901
902 return ret;
903}
904
905void cdnsp_set_usb2_hardware_lpm(struct cdnsp_device *pdev,
906 struct usb_request *req,
907 int enable)
908{
909 if (pdev->active_port != &pdev->usb2_port || !pdev->gadget.lpm_capable)
910 return;
911
118b2a32
PL		 912 trace_cdnsp_lpm(enable);
913
3d829045
PL		 914 if (enable)
915 writel(PORT_BESL(CDNSP_DEFAULT_BESL) | PORT_L1S_NYET | PORT_HLE,
916 &pdev->active_port->regs->portpmsc);
917 else
918 writel(PORT_L1S_NYET, &pdev->active_port->regs->portpmsc);
919}
920
921static int cdnsp_get_frame(struct cdnsp_device *pdev)
922{
923 return readl(&pdev->run_regs->microframe_index) >> 3;
924}
925
926static int cdnsp_gadget_ep_enable(struct usb_ep *ep,
927 const struct usb_endpoint_descriptor *desc)
928{
929 struct cdnsp_input_control_ctx *ctrl_ctx;
930 struct cdnsp_device *pdev;
931 struct cdnsp_ep *pep;
932 unsigned long flags;
933 u32 added_ctxs;
934 int ret;
935
936 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT ||
937 !desc->wMaxPacketSize)
938 return -EINVAL;
939
940 pep = to_cdnsp_ep(ep);
941 pdev = pep->pdev;
10076de3 942 pep->ep_state &= ~EP_UNCONFIGURED;
3d829045
PL		 943
944 if (dev_WARN_ONCE(pdev->dev, pep->ep_state & EP_ENABLED,
945 "%s is already enabled\n", pep->name))
946 return 0;
947
948 spin_lock_irqsave(&pdev->lock, flags);
949
950 added_ctxs = cdnsp_get_endpoint_flag(desc);
951 if (added_ctxs == SLOT_FLAG || added_ctxs == EP0_FLAG) {
952 dev_err(pdev->dev, "ERROR: Bad endpoint number\n");
953 ret = -EINVAL;
954 goto unlock;
955 }
956
957 pep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
958
959 if (pdev->gadget.speed == USB_SPEED_FULL) {
960 if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_INT)
961 pep->interval = desc->bInterval << 3;
962 if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_ISOC)
963 pep->interval = BIT(desc->bInterval - 1) << 3;
964 }
965
966 if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_ISOC) {
967 if (pep->interval > BIT(12)) {
968 dev_err(pdev->dev, "bInterval %d not supported\n",
969 desc->bInterval);
970 ret = -EINVAL;
971 goto unlock;
972 }
973 cdnsp_set_chicken_bits_2(pdev, CHICKEN_XDMA_2_TP_CACHE_DIS);
974 }
975
976 ret = cdnsp_endpoint_init(pdev, pep, GFP_ATOMIC);
977 if (ret)
978 goto unlock;
979
980 ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
981 ctrl_ctx->add_flags = cpu_to_le32(added_ctxs);
982 ctrl_ctx->drop_flags = 0;
983
984 ret = cdnsp_update_eps_configuration(pdev, pep);
985 if (ret) {
986 cdnsp_free_endpoint_rings(pdev, pep);
987 goto unlock;
988 }
989
990 pep->ep_state |= EP_ENABLED;
991 pep->ep_state &= ~EP_STOPPED;
992
993unlock:
118b2a32 994 trace_cdnsp_ep_enable_end(pep, 0);
3d829045
PL		 995 spin_unlock_irqrestore(&pdev->lock, flags);
996
997 return ret;
998}
999
1000static int cdnsp_gadget_ep_disable(struct usb_ep *ep)
1001{
1002 struct cdnsp_input_control_ctx *ctrl_ctx;
1003 struct cdnsp_request *preq;
1004 struct cdnsp_device *pdev;
1005 struct cdnsp_ep *pep;
1006 unsigned long flags;
1007 u32 drop_flag;
1008 int ret = 0;
1009
1010 if (!ep)
1011 return -EINVAL;
1012
1013 pep = to_cdnsp_ep(ep);
1014 pdev = pep->pdev;
1015
1016 spin_lock_irqsave(&pdev->lock, flags);
1017
1018 if (!(pep->ep_state & EP_ENABLED)) {
1019 dev_err(pdev->dev, "%s is already disabled\n", pep->name);
1020 ret = -EINVAL;
1021 goto finish;
1022 }
1023
3d829045 1024 pep->ep_state |= EP_DIS_IN_RROGRESS;
10076de3
PL		 1025
1026 /* Endpoint was unconfigured by Reset Device command. */
1027 if (!(pep->ep_state & EP_UNCONFIGURED)) {
1028 cdnsp_cmd_stop_ep(pdev, pep);
1029 cdnsp_cmd_flush_ep(pdev, pep);
1030 }
3d829045
PL		 1031
1032 /* Remove all queued USB requests. */
1033 while (!list_empty(&pep->pending_list)) {
1034 preq = next_request(&pep->pending_list);
1035 cdnsp_ep_dequeue(pep, preq);
1036 }
1037
1038 cdnsp_invalidate_ep_events(pdev, pep);
1039
1040 pep->ep_state &= ~EP_DIS_IN_RROGRESS;
1041 drop_flag = cdnsp_get_endpoint_flag(pep->endpoint.desc);
1042 ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
1043 ctrl_ctx->drop_flags = cpu_to_le32(drop_flag);
1044 ctrl_ctx->add_flags = 0;
1045
1046 cdnsp_endpoint_zero(pdev, pep);
1047
10076de3
PL		 1048 if (!(pep->ep_state & EP_UNCONFIGURED))
1049 ret = cdnsp_update_eps_configuration(pdev, pep);
1050
3d829045
PL		 1051 cdnsp_free_endpoint_rings(pdev, pep);
1052
10076de3 1053 pep->ep_state &= ~(EP_ENABLED | EP_UNCONFIGURED);
3d829045
PL		 1054 pep->ep_state |= EP_STOPPED;
1055
1056finish:
118b2a32 1057 trace_cdnsp_ep_disable_end(pep, 0);
3d829045
PL		 1058 spin_unlock_irqrestore(&pdev->lock, flags);
1059
1060 return ret;
1061}
1062
1063static struct usb_request *cdnsp_gadget_ep_alloc_request(struct usb_ep *ep,
1064 gfp_t gfp_flags)
1065{
1066 struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1067 struct cdnsp_request *preq;
1068
1069 preq = kzalloc(sizeof(*preq), gfp_flags);
1070 if (!preq)
1071 return NULL;
1072
1073 preq->epnum = pep->number;
1074 preq->pep = pep;
1075
118b2a32
PL		 1076 trace_cdnsp_alloc_request(preq);
1077
3d829045
PL		 1078 return &preq->request;
1079}
1080
1081static void cdnsp_gadget_ep_free_request(struct usb_ep *ep,
1082 struct usb_request *request)
1083{
1084 struct cdnsp_request *preq = to_cdnsp_request(request);
1085
118b2a32 1086 trace_cdnsp_free_request(preq);
3d829045
PL		 1087 kfree(preq);
1088}
1089
1090static int cdnsp_gadget_ep_queue(struct usb_ep *ep,
1091 struct usb_request *request,
1092 gfp_t gfp_flags)
1093{
1094 struct cdnsp_request *preq;
1095 struct cdnsp_device *pdev;
1096 struct cdnsp_ep *pep;
1097 unsigned long flags;
1098 int ret;
1099
1100 if (!request || !ep)
1101 return -EINVAL;
1102
1103 pep = to_cdnsp_ep(ep);
1104 pdev = pep->pdev;
1105
1106 if (!(pep->ep_state & EP_ENABLED)) {
1107 dev_err(pdev->dev, "%s: can't queue to disabled endpoint\n",
1108 pep->name);
1109 return -EINVAL;
1110 }
1111
1112 preq = to_cdnsp_request(request);
1113 spin_lock_irqsave(&pdev->lock, flags);
1114 ret = cdnsp_ep_enqueue(pep, preq);
1115 spin_unlock_irqrestore(&pdev->lock, flags);
1116
1117 return ret;
1118}
1119
1120static int cdnsp_gadget_ep_dequeue(struct usb_ep *ep,
1121 struct usb_request *request)
1122{
1123 struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1124 struct cdnsp_device *pdev = pep->pdev;
1125 unsigned long flags;
1126 int ret;
1127
1128 if (!pep->endpoint.desc) {
1129 dev_err(pdev->dev,
1130 "%s: can't dequeue to disabled endpoint\n",
1131 pep->name);
1132 return -ESHUTDOWN;
1133 }
1134
cf97d7af
PL		 1135 /* Requests has been dequeued during disabling endpoint. */
1136 if (!(pep->ep_state & EP_ENABLED))
1137 return 0;
1138
3d829045
PL		 1139 spin_lock_irqsave(&pdev->lock, flags);
1140 ret = cdnsp_ep_dequeue(pep, to_cdnsp_request(request));
1141 spin_unlock_irqrestore(&pdev->lock, flags);
1142
1143 return ret;
1144}
1145
1146static int cdnsp_gadget_ep_set_halt(struct usb_ep *ep, int value)
1147{
1148 struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1149 struct cdnsp_device *pdev = pep->pdev;
1150 struct cdnsp_request *preq;
18538a50 1151 unsigned long flags;
3d829045
PL		 1152 int ret;
1153
1154 spin_lock_irqsave(&pdev->lock, flags);
1155
1156 preq = next_request(&pep->pending_list);
1157 if (value) {
1158 if (preq) {
118b2a32 1159 trace_cdnsp_ep_busy_try_halt_again(pep, 0);
3d829045
PL		 1160 ret = -EAGAIN;
1161 goto done;
1162 }
1163 }
1164
1165 ret = cdnsp_halt_endpoint(pdev, pep, value);
1166
1167done:
1168 spin_unlock_irqrestore(&pdev->lock, flags);
1169 return ret;
1170}
1171
1172static int cdnsp_gadget_ep_set_wedge(struct usb_ep *ep)
1173{
1174 struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1175 struct cdnsp_device *pdev = pep->pdev;
18538a50 1176 unsigned long flags;
3d829045
PL		 1177 int ret;
1178
1179 spin_lock_irqsave(&pdev->lock, flags);
1180 pep->ep_state |= EP_WEDGE;
1181 ret = cdnsp_halt_endpoint(pdev, pep, 1);
1182 spin_unlock_irqrestore(&pdev->lock, flags);
1183
1184 return ret;
1185}
1186
1187static const struct usb_ep_ops cdnsp_gadget_ep0_ops = {
1188 .enable = cdnsp_gadget_ep_enable,
1189 .disable = cdnsp_gadget_ep_disable,
1190 .alloc_request = cdnsp_gadget_ep_alloc_request,
1191 .free_request = cdnsp_gadget_ep_free_request,
1192 .queue = cdnsp_gadget_ep_queue,
1193 .dequeue = cdnsp_gadget_ep_dequeue,
1194 .set_halt = cdnsp_gadget_ep_set_halt,
1195 .set_wedge = cdnsp_gadget_ep_set_wedge,
1196};
1197
1198static const struct usb_ep_ops cdnsp_gadget_ep_ops = {
1199 .enable = cdnsp_gadget_ep_enable,
1200 .disable = cdnsp_gadget_ep_disable,
1201 .alloc_request = cdnsp_gadget_ep_alloc_request,
1202 .free_request = cdnsp_gadget_ep_free_request,
1203 .queue = cdnsp_gadget_ep_queue,
1204 .dequeue = cdnsp_gadget_ep_dequeue,
1205 .set_halt = cdnsp_gadget_ep_set_halt,
1206 .set_wedge = cdnsp_gadget_ep_set_wedge,
1207};
1208
1209void cdnsp_gadget_giveback(struct cdnsp_ep *pep,
1210 struct cdnsp_request *preq,
1211 int status)
1212{
1213 struct cdnsp_device *pdev = pep->pdev;
1214
1215 list_del(&preq->list);
1216
1217 if (preq->request.status == -EINPROGRESS)
1218 preq->request.status = status;
1219
1220 usb_gadget_unmap_request_by_dev(pdev->dev, &preq->request,
1221 preq->direction);
1222
118b2a32
PL		 1223 trace_cdnsp_request_giveback(preq);
1224
3d829045
PL		 1225 if (preq != &pdev->ep0_preq) {
1226 spin_unlock(&pdev->lock);
1227 usb_gadget_giveback_request(&pep->endpoint, &preq->request);
1228 spin_lock(&pdev->lock);
1229 }
1230}
1231
1232static struct usb_endpoint_descriptor cdnsp_gadget_ep0_desc = {
1233 .bLength = USB_DT_ENDPOINT_SIZE,
1234 .bDescriptorType = USB_DT_ENDPOINT,
1235 .bmAttributes = USB_ENDPOINT_XFER_CONTROL,
1236};
1237
1238static int cdnsp_run(struct cdnsp_device *pdev,
1239 enum usb_device_speed speed)
1240{
1241 u32 fs_speed = 0;
3d829045
PL		 1242 u32 temp;
1243 int ret;
1244
3d829045
PL		 1245 temp = readl(&pdev->ir_set->irq_control);
1246 temp &= ~IMOD_INTERVAL_MASK;
1247 temp |= ((IMOD_DEFAULT_INTERVAL / 250) & IMOD_INTERVAL_MASK);
1248 writel(temp, &pdev->ir_set->irq_control);
1249
1250 temp = readl(&pdev->port3x_regs->mode_addr);
1251
1252 switch (speed) {
1253 case USB_SPEED_SUPER_PLUS:
1254 temp |= CFG_3XPORT_SSP_SUPPORT;
1255 break;
1256 case USB_SPEED_SUPER:
1257 temp &= ~CFG_3XPORT_SSP_SUPPORT;
1258 break;
1259 case USB_SPEED_HIGH:
1260 break;
1261 case USB_SPEED_FULL:
1262 fs_speed = PORT_REG6_FORCE_FS;
1263 break;
1264 default:
1265 dev_err(pdev->dev, "invalid maximum_speed parameter %d\n",
1266 speed);
1267 fallthrough;
1268 case USB_SPEED_UNKNOWN:
1269 /* Default to superspeed. */
1270 speed = USB_SPEED_SUPER;
1271 break;
1272 }
1273
1274 if (speed >= USB_SPEED_SUPER) {
1275 writel(temp, &pdev->port3x_regs->mode_addr);
1276 cdnsp_set_link_state(pdev, &pdev->usb3_port.regs->portsc,
1277 XDEV_RXDETECT);
1278 } else {
1279 cdnsp_disable_port(pdev, &pdev->usb3_port.regs->portsc);
1280 }
1281
1282 cdnsp_set_link_state(pdev, &pdev->usb2_port.regs->portsc,
1283 XDEV_RXDETECT);
1284
1285 cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1286
1287 writel(PORT_REG6_L1_L0_HW_EN | fs_speed, &pdev->port20_regs->port_reg6);
1288
1289 ret = cdnsp_start(pdev);
1290 if (ret) {
1291 ret = -ENODEV;
1292 goto err;
1293 }
1294
1295 temp = readl(&pdev->op_regs->command);
1296 temp |= (CMD_INTE);
1297 writel(temp, &pdev->op_regs->command);
1298
1299 temp = readl(&pdev->ir_set->irq_pending);
1300 writel(IMAN_IE_SET(temp), &pdev->ir_set->irq_pending);
1301
118b2a32 1302 trace_cdnsp_init("Controller ready to work");
3d829045
PL		 1303 return 0;
1304err:
1305 cdnsp_halt(pdev);
1306 return ret;
1307}
1308
1309static int cdnsp_gadget_udc_start(struct usb_gadget *g,
1310 struct usb_gadget_driver *driver)
1311{
1312 enum usb_device_speed max_speed = driver->max_speed;
1313 struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1314 unsigned long flags;
1315 int ret;
1316
1317 spin_lock_irqsave(&pdev->lock, flags);
1318 pdev->gadget_driver = driver;
1319
1320 /* limit speed if necessary */
1321 max_speed = min(driver->max_speed, g->max_speed);
1322 ret = cdnsp_run(pdev, max_speed);
1323
1324 spin_unlock_irqrestore(&pdev->lock, flags);
1325
1326 return ret;
1327}
1328
1329/*
1330 * Update Event Ring Dequeue Pointer:
1331 * - When all events have finished
1332 * - To avoid "Event Ring Full Error" condition
1333 */
1334void cdnsp_update_erst_dequeue(struct cdnsp_device *pdev,
1335 union cdnsp_trb *event_ring_deq,
1336 u8 clear_ehb)
1337{
1338 u64 temp_64;
1339 dma_addr_t deq;
1340
1341 temp_64 = cdnsp_read_64(&pdev->ir_set->erst_dequeue);
1342
1343 /* If necessary, update the HW's version of the event ring deq ptr. */
1344 if (event_ring_deq != pdev->event_ring->dequeue) {
1345 deq = cdnsp_trb_virt_to_dma(pdev->event_ring->deq_seg,
1346 pdev->event_ring->dequeue);
1347 temp_64 &= ERST_PTR_MASK;
1348 temp_64 |= ((u64)deq & (u64)~ERST_PTR_MASK);
1349 }
1350
1351 /* Clear the event handler busy flag (RW1C). */
1352 if (clear_ehb)
1353 temp_64 |= ERST_EHB;
1354 else
1355 temp_64 &= ~ERST_EHB;
1356
1357 cdnsp_write_64(temp_64, &pdev->ir_set->erst_dequeue);
1358}
1359
1360static void cdnsp_clear_cmd_ring(struct cdnsp_device *pdev)
1361{
1362 struct cdnsp_segment *seg;
1363 u64 val_64;
1364 int i;
1365
1366 cdnsp_initialize_ring_info(pdev->cmd_ring);
1367
1368 seg = pdev->cmd_ring->first_seg;
1369 for (i = 0; i < pdev->cmd_ring->num_segs; i++) {
1370 memset(seg->trbs, 0,
1371 sizeof(union cdnsp_trb) * (TRBS_PER_SEGMENT - 1));
1372 seg = seg->next;
1373 }
1374
1375 /* Set the address in the Command Ring Control register. */
1376 val_64 = cdnsp_read_64(&pdev->op_regs->cmd_ring);
1377 val_64 = (val_64 & (u64)CMD_RING_RSVD_BITS) |
1378 (pdev->cmd_ring->first_seg->dma & (u64)~CMD_RING_RSVD_BITS) |
1379 pdev->cmd_ring->cycle_state;
1380 cdnsp_write_64(val_64, &pdev->op_regs->cmd_ring);
1381}
1382
1383static void cdnsp_consume_all_events(struct cdnsp_device *pdev)
1384{
1385 struct cdnsp_segment *event_deq_seg;
1386 union cdnsp_trb *event_ring_deq;
1387 union cdnsp_trb *event;
1388 u32 cycle_bit;
1389
1390 event_ring_deq = pdev->event_ring->dequeue;
1391 event_deq_seg = pdev->event_ring->deq_seg;
1392 event = pdev->event_ring->dequeue;
1393
1394 /* Update ring dequeue pointer. */
1395 while (1) {
1396 cycle_bit = (le32_to_cpu(event->event_cmd.flags) & TRB_CYCLE);
1397
1398 /* Does the controller or driver own the TRB? */
1399 if (cycle_bit != pdev->event_ring->cycle_state)
1400 break;
1401
1402 cdnsp_inc_deq(pdev, pdev->event_ring);
1403
1404 if (!cdnsp_last_trb_on_seg(event_deq_seg, event)) {
1405 event++;
1406 continue;
1407 }
1408
1409 if (cdnsp_last_trb_on_ring(pdev->event_ring, event_deq_seg,
1410 event))
1411 cycle_bit ^= 1;
1412
1413 event_deq_seg = event_deq_seg->next;
1414 event = event_deq_seg->trbs;
1415 }
1416
1417 cdnsp_update_erst_dequeue(pdev, event_ring_deq, 1);
1418}
1419
1420static void cdnsp_stop(struct cdnsp_device *pdev)
1421{
1422 u32 temp;
1423
1424 cdnsp_cmd_flush_ep(pdev, &pdev->eps[0]);
1425
1426 /* Remove internally queued request for ep0. */
1427 if (!list_empty(&pdev->eps[0].pending_list)) {
1428 struct cdnsp_request *req;
1429
1430 req = next_request(&pdev->eps[0].pending_list);
1431 if (req == &pdev->ep0_preq)
1432 cdnsp_ep_dequeue(&pdev->eps[0], req);
1433 }
1434
1435 cdnsp_disable_port(pdev, &pdev->usb2_port.regs->portsc);
1436 cdnsp_disable_port(pdev, &pdev->usb3_port.regs->portsc);
1437 cdnsp_disable_slot(pdev);
1438 cdnsp_halt(pdev);
1439
1440 temp = readl(&pdev->op_regs->status);
1441 writel((temp & ~0x1fff) | STS_EINT, &pdev->op_regs->status);
1442 temp = readl(&pdev->ir_set->irq_pending);
1443 writel(IMAN_IE_CLEAR(temp), &pdev->ir_set->irq_pending);
1444
1445 cdnsp_clear_port_change_bit(pdev, &pdev->usb2_port.regs->portsc);
1446 cdnsp_clear_port_change_bit(pdev, &pdev->usb3_port.regs->portsc);
1447
1448 /* Clear interrupt line */
1449 temp = readl(&pdev->ir_set->irq_pending);
1450 temp |= IMAN_IP;
1451 writel(temp, &pdev->ir_set->irq_pending);
1452
1453 cdnsp_consume_all_events(pdev);
1454 cdnsp_clear_cmd_ring(pdev);
118b2a32
PL		 1455
1456 trace_cdnsp_exit("Controller stopped.");
3d829045
PL		 1457}
1458
1459/*
1460 * Stop controller.
1461 * This function is called by the gadget core when the driver is removed.
1462 * Disable slot, disable IRQs, and quiesce the controller.
1463 */
1464static int cdnsp_gadget_udc_stop(struct usb_gadget *g)
1465{
1466 struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1467 unsigned long flags;
1468
1469 spin_lock_irqsave(&pdev->lock, flags);
1470 cdnsp_stop(pdev);
1471 pdev->gadget_driver = NULL;
1472 spin_unlock_irqrestore(&pdev->lock, flags);
1473
1474 return 0;
1475}
1476
1477static int cdnsp_gadget_get_frame(struct usb_gadget *g)
1478{
1479 struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1480
1481 return cdnsp_get_frame(pdev);
1482}
1483
1484static void __cdnsp_gadget_wakeup(struct cdnsp_device *pdev)
1485{
1486 struct cdnsp_port_regs __iomem *port_regs;
1487 u32 portpm, portsc;
1488
1489 port_regs = pdev->active_port->regs;
1490 portsc = readl(&port_regs->portsc) & PORT_PLS_MASK;
1491
1492 /* Remote wakeup feature is not enabled by host. */
1493 if (pdev->gadget.speed < USB_SPEED_SUPER && portsc == XDEV_U2) {
1494 portpm = readl(&port_regs->portpmsc);
1495
1496 if (!(portpm & PORT_RWE))
1497 return;
1498 }
1499
1500 if (portsc == XDEV_U3 && !pdev->may_wakeup)
1501 return;
1502
1503 cdnsp_set_link_state(pdev, &port_regs->portsc, XDEV_U0);
1504
1505 pdev->cdnsp_state |= CDNSP_WAKEUP_PENDING;
1506}
1507
1508static int cdnsp_gadget_wakeup(struct usb_gadget *g)
1509{
1510 struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1511 unsigned long flags;
1512
1513 spin_lock_irqsave(&pdev->lock, flags);
1514 __cdnsp_gadget_wakeup(pdev);
1515 spin_unlock_irqrestore(&pdev->lock, flags);
1516
1517 return 0;
1518}
1519
1520static int cdnsp_gadget_set_selfpowered(struct usb_gadget *g,
1521 int is_selfpowered)
1522{
1523 struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1524 unsigned long flags;
1525
1526 spin_lock_irqsave(&pdev->lock, flags);
1527 g->is_selfpowered = !!is_selfpowered;
1528 spin_unlock_irqrestore(&pdev->lock, flags);
1529
1530 return 0;
1531}
1532
1533static int cdnsp_gadget_pullup(struct usb_gadget *gadget, int is_on)
1534{
1535 struct cdnsp_device *pdev = gadget_to_cdnsp(gadget);
1536 struct cdns *cdns = dev_get_drvdata(pdev->dev);
4c4e162d 1537 unsigned long flags;
3d829045 1538
118b2a32
PL		 1539 trace_cdnsp_pullup(is_on);
1540
4c4e162d
PL		 1541 /*
1542 * Disable events handling while controller is being
1543 * enabled/disabled.
1544 */
1545 disable_irq(cdns->dev_irq);
1546 spin_lock_irqsave(&pdev->lock, flags);
1547
3d829045
PL		 1548 if (!is_on) {
1549 cdnsp_reset_device(pdev);
1550 cdns_clear_vbus(cdns);
1551 } else {
1552 cdns_set_vbus(cdns);
1553 }
4c4e162d
PL		 1554
1555 spin_unlock_irqrestore(&pdev->lock, flags);
1556 enable_irq(cdns->dev_irq);
1557
3d829045
PL		 1558 return 0;
1559}
1560
7650778e 1561static const struct usb_gadget_ops cdnsp_gadget_ops = {
3d829045
PL		 1562 .get_frame = cdnsp_gadget_get_frame,
1563 .wakeup = cdnsp_gadget_wakeup,
1564 .set_selfpowered = cdnsp_gadget_set_selfpowered,
1565 .pullup = cdnsp_gadget_pullup,
1566 .udc_start = cdnsp_gadget_udc_start,
1567 .udc_stop = cdnsp_gadget_udc_stop,
1568};
1569
1570static void cdnsp_get_ep_buffering(struct cdnsp_device *pdev,
1571 struct cdnsp_ep *pep)
1572{
1573 void __iomem *reg = &pdev->cap_regs->hc_capbase;
1574 int endpoints;
1575
1576 reg += cdnsp_find_next_ext_cap(reg, 0, XBUF_CAP_ID);
1577
1578 if (!pep->direction) {
1579 pep->buffering = readl(reg + XBUF_RX_TAG_MASK_0_OFFSET);
1580 pep->buffering_period = readl(reg + XBUF_RX_TAG_MASK_1_OFFSET);
1581 pep->buffering = (pep->buffering + 1) / 2;
1582 pep->buffering_period = (pep->buffering_period + 1) / 2;
1583 return;
1584 }
1585
16e36101 1586 endpoints = HCS_ENDPOINTS(pdev->hcs_params1) / 2;
3d829045
PL		 1587
1588 /* Set to XBUF_TX_TAG_MASK_0 register. */
1589 reg += XBUF_TX_CMD_OFFSET + (endpoints * 2 + 2) * sizeof(u32);
1590 /* Set reg to XBUF_TX_TAG_MASK_N related with this endpoint. */
1591 reg += pep->number * sizeof(u32) * 2;
1592
1593 pep->buffering = (readl(reg) + 1) / 2;
1594 pep->buffering_period = pep->buffering;
1595}
1596
1597static int cdnsp_gadget_init_endpoints(struct cdnsp_device *pdev)
1598{
1599 int max_streams = HCC_MAX_PSA(pdev->hcc_params);
1600 struct cdnsp_ep *pep;
1601 int i;
1602
1603 INIT_LIST_HEAD(&pdev->gadget.ep_list);
1604
1605 if (max_streams < STREAM_LOG_STREAMS) {
1606 dev_err(pdev->dev, "Stream size %d not supported\n",
1607 max_streams);
1608 return -EINVAL;
1609 }
1610
1611 max_streams = STREAM_LOG_STREAMS;
1612
1613 for (i = 0; i < CDNSP_ENDPOINTS_NUM; i++) {
1614 bool direction = !(i & 1); /* Start from OUT endpoint. */
1615 u8 epnum = ((i + 1) >> 1);
1616
1617 if (!CDNSP_IF_EP_EXIST(pdev, epnum, direction))
1618 continue;
1619
1620 pep = &pdev->eps[i];
1621 pep->pdev = pdev;
1622 pep->number = epnum;
1623 pep->direction = direction; /* 0 for OUT, 1 for IN. */
1624
1625 /*
1626 * Ep0 is bidirectional, so ep0in and ep0out are represented by
1627 * pdev->eps[0]
1628 */
1629 if (epnum == 0) {
1630 snprintf(pep->name, sizeof(pep->name), "ep%d%s",
1631 epnum, "BiDir");
1632
1633 pep->idx = 0;
1634 usb_ep_set_maxpacket_limit(&pep->endpoint, 512);
1635 pep->endpoint.maxburst = 1;
1636 pep->endpoint.ops = &cdnsp_gadget_ep0_ops;
1637 pep->endpoint.desc = &cdnsp_gadget_ep0_desc;
1638 pep->endpoint.comp_desc = NULL;
1639 pep->endpoint.caps.type_control = true;
1640 pep->endpoint.caps.dir_in = true;
1641 pep->endpoint.caps.dir_out = true;
1642
1643 pdev->ep0_preq.epnum = pep->number;
1644 pdev->ep0_preq.pep = pep;
1645 pdev->gadget.ep0 = &pep->endpoint;
1646 } else {
1647 snprintf(pep->name, sizeof(pep->name), "ep%d%s",
1648 epnum, (pep->direction) ? "in" : "out");
1649
1650 pep->idx = (epnum * 2 + (direction ? 1 : 0)) - 1;
1651 usb_ep_set_maxpacket_limit(&pep->endpoint, 1024);
1652
1653 pep->endpoint.max_streams = max_streams;
1654 pep->endpoint.ops = &cdnsp_gadget_ep_ops;
1655 list_add_tail(&pep->endpoint.ep_list,
1656 &pdev->gadget.ep_list);
1657
1658 pep->endpoint.caps.type_iso = true;
1659 pep->endpoint.caps.type_bulk = true;
1660 pep->endpoint.caps.type_int = true;
1661
1662 pep->endpoint.caps.dir_in = direction;
1663 pep->endpoint.caps.dir_out = !direction;
1664 }
1665
1666 pep->endpoint.name = pep->name;
1667 pep->in_ctx = cdnsp_get_ep_ctx(&pdev->in_ctx, pep->idx);
1668 pep->out_ctx = cdnsp_get_ep_ctx(&pdev->out_ctx, pep->idx);
1669 cdnsp_get_ep_buffering(pdev, pep);
1670
1671 dev_dbg(pdev->dev, "Init %s, MPS: %04x SupType: "
1672 "CTRL: %s, INT: %s, BULK: %s, ISOC %s, "
1673 "SupDir IN: %s, OUT: %s\n",
1674 pep->name, 1024,
1675 (pep->endpoint.caps.type_control) ? "yes" : "no",
1676 (pep->endpoint.caps.type_int) ? "yes" : "no",
1677 (pep->endpoint.caps.type_bulk) ? "yes" : "no",
1678 (pep->endpoint.caps.type_iso) ? "yes" : "no",
1679 (pep->endpoint.caps.dir_in) ? "yes" : "no",
1680 (pep->endpoint.caps.dir_out) ? "yes" : "no");
1681
1682 INIT_LIST_HEAD(&pep->pending_list);
1683 }
1684
1685 return 0;
1686}
1687
1688static void cdnsp_gadget_free_endpoints(struct cdnsp_device *pdev)
1689{
1690 struct cdnsp_ep *pep;
1691 int i;
1692
1693 for (i = 0; i < CDNSP_ENDPOINTS_NUM; i++) {
1694 pep = &pdev->eps[i];
1695 if (pep->number != 0 && pep->out_ctx)
1696 list_del(&pep->endpoint.ep_list);
1697 }
1698}
1699
1700void cdnsp_disconnect_gadget(struct cdnsp_device *pdev)
1701{
1702 pdev->cdnsp_state |= CDNSP_STATE_DISCONNECT_PENDING;
1703
1704 if (pdev->gadget_driver && pdev->gadget_driver->disconnect) {
1705 spin_unlock(&pdev->lock);
1706 pdev->gadget_driver->disconnect(&pdev->gadget);
1707 spin_lock(&pdev->lock);
1708 }
1709
1710 pdev->gadget.speed = USB_SPEED_UNKNOWN;
1711 usb_gadget_set_state(&pdev->gadget, USB_STATE_NOTATTACHED);
1712
1713 pdev->cdnsp_state &= ~CDNSP_STATE_DISCONNECT_PENDING;
1714}
1715
1716void cdnsp_suspend_gadget(struct cdnsp_device *pdev)
1717{
1718 if (pdev->gadget_driver && pdev->gadget_driver->suspend) {
1719 spin_unlock(&pdev->lock);
1720 pdev->gadget_driver->suspend(&pdev->gadget);
1721 spin_lock(&pdev->lock);
1722 }
1723}
1724
1725void cdnsp_resume_gadget(struct cdnsp_device *pdev)
1726{
1727 if (pdev->gadget_driver && pdev->gadget_driver->resume) {
1728 spin_unlock(&pdev->lock);
1729 pdev->gadget_driver->resume(&pdev->gadget);
1730 spin_lock(&pdev->lock);
1731 }
1732}
1733
1734void cdnsp_irq_reset(struct cdnsp_device *pdev)
1735{
1736 struct cdnsp_port_regs __iomem *port_regs;
1737
1738 cdnsp_reset_device(pdev);
1739
1740 port_regs = pdev->active_port->regs;
1741 pdev->gadget.speed = cdnsp_port_speed(readl(port_regs));
1742
1743 spin_unlock(&pdev->lock);
1744 usb_gadget_udc_reset(&pdev->gadget, pdev->gadget_driver);
1745 spin_lock(&pdev->lock);
1746
1747 switch (pdev->gadget.speed) {
1748 case USB_SPEED_SUPER_PLUS:
1749 case USB_SPEED_SUPER:
1750 cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1751 pdev->gadget.ep0->maxpacket = 512;
1752 break;
1753 case USB_SPEED_HIGH:
1754 case USB_SPEED_FULL:
1755 cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
1756 pdev->gadget.ep0->maxpacket = 64;
1757 break;
1758 default:
1759 /* Low speed is not supported. */
1760 dev_err(pdev->dev, "Unknown device speed\n");
1761 break;
1762 }
1763
1764 cdnsp_clear_chicken_bits_2(pdev, CHICKEN_XDMA_2_TP_CACHE_DIS);
1765 cdnsp_setup_device(pdev, SETUP_CONTEXT_ONLY);
1766 usb_gadget_set_state(&pdev->gadget, USB_STATE_DEFAULT);
1767}
1768
1769static void cdnsp_get_rev_cap(struct cdnsp_device *pdev)
1770{
1771 void __iomem *reg = &pdev->cap_regs->hc_capbase;
3d829045
PL		 1772
1773 reg += cdnsp_find_next_ext_cap(reg, 0, RTL_REV_CAP);
16e36101 1774 pdev->rev_cap = reg;
3d829045
PL		 1775
1776 dev_info(pdev->dev, "Rev: %08x/%08x, eps: %08x, buff: %08x/%08x\n",
16e36101
PL		 1777 readl(&pdev->rev_cap->ctrl_revision),
1778 readl(&pdev->rev_cap->rtl_revision),
1779 readl(&pdev->rev_cap->ep_supported),
1780 readl(&pdev->rev_cap->rx_buff_size),
1781 readl(&pdev->rev_cap->tx_buff_size));
3d829045
PL		 1782}
1783
1784static int cdnsp_gen_setup(struct cdnsp_device *pdev)
1785{
1786 int ret;
1787 u32 reg;
1788
1789 pdev->cap_regs = pdev->regs;
1790 pdev->op_regs = pdev->regs +
1791 HC_LENGTH(readl(&pdev->cap_regs->hc_capbase));
1792 pdev->run_regs = pdev->regs +
1793 (readl(&pdev->cap_regs->run_regs_off) & RTSOFF_MASK);
1794
1795 /* Cache read-only capability registers */
1796 pdev->hcs_params1 = readl(&pdev->cap_regs->hcs_params1);
1797 pdev->hcc_params = readl(&pdev->cap_regs->hc_capbase);
1798 pdev->hci_version = HC_VERSION(pdev->hcc_params);
1799 pdev->hcc_params = readl(&pdev->cap_regs->hcc_params);
1800
1801 cdnsp_get_rev_cap(pdev);
1802
1803 /* Make sure the Device Controller is halted. */
1804 ret = cdnsp_halt(pdev);
1805 if (ret)
1806 return ret;
1807
1808 /* Reset the internal controller memory state and registers. */
1809 ret = cdnsp_reset(pdev);
1810 if (ret)
1811 return ret;
1812
1813 /*
1814 * Set dma_mask and coherent_dma_mask to 64-bits,
1815 * if controller supports 64-bit addressing.
1816 */
1817 if (HCC_64BIT_ADDR(pdev->hcc_params) &&
1818 !dma_set_mask(pdev->dev, DMA_BIT_MASK(64))) {
1819 dev_dbg(pdev->dev, "Enabling 64-bit DMA addresses.\n");
1820 dma_set_coherent_mask(pdev->dev, DMA_BIT_MASK(64));
1821 } else {
1822 /*
1823 * This is to avoid error in cases where a 32-bit USB
1824 * controller is used on a 64-bit capable system.
1825 */
1826 ret = dma_set_mask(pdev->dev, DMA_BIT_MASK(32));
1827 if (ret)
1828 return ret;
1829
1830 dev_dbg(pdev->dev, "Enabling 32-bit DMA addresses.\n");
1831 dma_set_coherent_mask(pdev->dev, DMA_BIT_MASK(32));
1832 }
1833
1834 spin_lock_init(&pdev->lock);
1835
dc68ba6c 1836 ret = cdnsp_mem_init(pdev);
3d829045
PL		 1837 if (ret)
1838 return ret;
1839
1840 /*
1841 * Software workaround for U1: after transition
1842 * to U1 the controller starts gating clock, and in some cases,
1843 * it causes that controller stack.
1844 */
1845 reg = readl(&pdev->port3x_regs->mode_2);
1846 reg &= ~CFG_3XPORT_U1_PIPE_CLK_GATE_EN;
1847 writel(reg, &pdev->port3x_regs->mode_2);
1848
1849 return 0;
1850}
1851
1852static int __cdnsp_gadget_init(struct cdns *cdns)
1853{
1854 struct cdnsp_device *pdev;
1855 u32 max_speed;
1856 int ret = -ENOMEM;
1857
1858 cdns_drd_gadget_on(cdns);
1859
1860 pdev = kzalloc(sizeof(*pdev), GFP_KERNEL);
1861 if (!pdev)
1862 return -ENOMEM;
1863
1864 pm_runtime_get_sync(cdns->dev);
1865
1866 cdns->gadget_dev = pdev;
1867 pdev->dev = cdns->dev;
1868 pdev->regs = cdns->dev_regs;
1869 max_speed = usb_get_maximum_speed(cdns->dev);
1870
1871 switch (max_speed) {
1872 case USB_SPEED_FULL:
1873 case USB_SPEED_HIGH:
1874 case USB_SPEED_SUPER:
1875 case USB_SPEED_SUPER_PLUS:
1876 break;
1877 default:
1878 dev_err(cdns->dev, "invalid speed parameter %d\n", max_speed);
1879 fallthrough;
1880 case USB_SPEED_UNKNOWN:
1881 /* Default to SSP */
1882 max_speed = USB_SPEED_SUPER_PLUS;
1883 break;
1884 }
1885
1886 pdev->gadget.ops = &cdnsp_gadget_ops;
1887 pdev->gadget.name = "cdnsp-gadget";
1888 pdev->gadget.speed = USB_SPEED_UNKNOWN;
1889 pdev->gadget.sg_supported = 1;
aa82f94e 1890 pdev->gadget.max_speed = max_speed;
3d829045
PL		 1891 pdev->gadget.lpm_capable = 1;
1892
1893 pdev->setup_buf = kzalloc(CDNSP_EP0_SETUP_SIZE, GFP_KERNEL);
1894 if (!pdev->setup_buf)
1895 goto free_pdev;
1896
1897 /*
1898 * Controller supports not aligned buffer but it should improve
1899 * performance.
1900 */
1901 pdev->gadget.quirk_ep_out_aligned_size = true;
1902
1903 ret = cdnsp_gen_setup(pdev);
1904 if (ret) {
1905 dev_err(pdev->dev, "Generic initialization failed %d\n", ret);
1906 goto free_setup;
1907 }
1908
1909 ret = cdnsp_gadget_init_endpoints(pdev);
1910 if (ret) {
1911 dev_err(pdev->dev, "failed to initialize endpoints\n");
1912 goto halt_pdev;
1913 }
1914
1915 ret = usb_add_gadget_udc(pdev->dev, &pdev->gadget);
1916 if (ret) {
1917 dev_err(pdev->dev, "failed to register udc\n");
1918 goto free_endpoints;
1919 }
1920
1921 ret = devm_request_threaded_irq(pdev->dev, cdns->dev_irq,
1922 cdnsp_irq_handler,
1923 cdnsp_thread_irq_handler, IRQF_SHARED,
1924 dev_name(pdev->dev), pdev);
1925 if (ret)
1926 goto del_gadget;
1927
1928 return 0;
1929
1930del_gadget:
1931 usb_del_gadget_udc(&pdev->gadget);
1932free_endpoints:
1933 cdnsp_gadget_free_endpoints(pdev);
1934halt_pdev:
1935 cdnsp_halt(pdev);
1936 cdnsp_reset(pdev);
1937 cdnsp_mem_cleanup(pdev);
1938free_setup:
1939 kfree(pdev->setup_buf);
1940free_pdev:
1941 kfree(pdev);
1942
1943 return ret;
1944}
1945
1946static void cdnsp_gadget_exit(struct cdns *cdns)
1947{
1948 struct cdnsp_device *pdev = cdns->gadget_dev;
1949
1950 devm_free_irq(pdev->dev, cdns->dev_irq, pdev);
1951 pm_runtime_mark_last_busy(cdns->dev);
1952 pm_runtime_put_autosuspend(cdns->dev);
1953 usb_del_gadget_udc(&pdev->gadget);
1954 cdnsp_gadget_free_endpoints(pdev);
1955 cdnsp_mem_cleanup(pdev);
1956 kfree(pdev);
1957 cdns->gadget_dev = NULL;
1958 cdns_drd_gadget_off(cdns);
1959}
1960
1961static int cdnsp_gadget_suspend(struct cdns *cdns, bool do_wakeup)
1962{
1963 struct cdnsp_device *pdev = cdns->gadget_dev;
1964 unsigned long flags;
1965
1966 if (pdev->link_state == XDEV_U3)
1967 return 0;
1968
1969 spin_lock_irqsave(&pdev->lock, flags);
1970 cdnsp_disconnect_gadget(pdev);
1971 cdnsp_stop(pdev);
1972 spin_unlock_irqrestore(&pdev->lock, flags);
1973
1974 return 0;
1975}
1976
1977static int cdnsp_gadget_resume(struct cdns *cdns, bool hibernated)
1978{
1979 struct cdnsp_device *pdev = cdns->gadget_dev;
1980 enum usb_device_speed max_speed;
1981 unsigned long flags;
1982 int ret;
1983
1984 if (!pdev->gadget_driver)
1985 return 0;
1986
1987 spin_lock_irqsave(&pdev->lock, flags);
1988 max_speed = pdev->gadget_driver->max_speed;
1989
1990 /* Limit speed if necessary. */
1991 max_speed = min(max_speed, pdev->gadget.max_speed);
1992
1993 ret = cdnsp_run(pdev, max_speed);
1994
1995 if (pdev->link_state == XDEV_U3)
1996 __cdnsp_gadget_wakeup(pdev);
1997
1998 spin_unlock_irqrestore(&pdev->lock, flags);
1999
2000 return ret;
2001}
2002
2003/**
2004 * cdnsp_gadget_init - initialize device structure
2005 * @cdns: cdnsp instance
2006 *
2007 * This function initializes the gadget.
2008 */
2009int cdnsp_gadget_init(struct cdns *cdns)
2010{
2011 struct cdns_role_driver *rdrv;
2012
2013 rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
2014 if (!rdrv)
2015 return -ENOMEM;
2016
2017 rdrv->start = __cdnsp_gadget_init;
2018 rdrv->stop = cdnsp_gadget_exit;
2019 rdrv->suspend = cdnsp_gadget_suspend;
2020 rdrv->resume = cdnsp_gadget_resume;
2021 rdrv->state = CDNS_ROLE_STATE_INACTIVE;
2022 rdrv->name = "gadget";
2023 cdns->roles[USB_ROLE_DEVICE] = rdrv;
2024
2025 return 0;
2026}
Linux 6.x block layer and io_uring tree(s)