projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'mm-hotfixes-stable-2025-07-11-16-16' of git://git.kernel.org/pub/scm/linux...
[linux-block.git] / drivers / usb / cdns3 / cdnsp-gadget.c
CommitLineData
3d829045
PL		 1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Cadence CDNSP DRD Driver.
4 *
5 * Copyright (C) 2020 Cadence.
6 *
7 * Author: Pawel Laszczak <pawell@cadence.com>
8 *
9 */
10
11#include <linux/moduleparam.h>
12#include <linux/dma-mapping.h>
13#include <linux/module.h>
14#include <linux/iopoll.h>
15#include <linux/delay.h>
16#include <linux/log2.h>
17#include <linux/slab.h>
f386bfad 18#include <linux/string_choices.h>
3d829045
PL		 19#include <linux/pci.h>
20#include <linux/irq.h>
21#include <linux/dmi.h>
22
23#include "core.h"
24#include "gadget-export.h"
25#include "drd.h"
26#include "cdnsp-gadget.h"
118b2a32 27#include "cdnsp-trace.h"
3d829045
PL		 28
29unsigned int cdnsp_port_speed(unsigned int port_status)
30{
31 /*Detect gadget speed based on PORTSC register*/
2852788c
PL		 32 if (DEV_SUPERSPEEDPLUS(port_status) ||
33 DEV_SSP_GEN1x2(port_status) || DEV_SSP_GEN2x2(port_status))
3d829045
PL		 34 return USB_SPEED_SUPER_PLUS;
35 else if (DEV_SUPERSPEED(port_status))
36 return USB_SPEED_SUPER;
37 else if (DEV_HIGHSPEED(port_status))
38 return USB_SPEED_HIGH;
39 else if (DEV_FULLSPEED(port_status))
40 return USB_SPEED_FULL;
41
42 /* If device is detached then speed will be USB_SPEED_UNKNOWN.*/
43 return USB_SPEED_UNKNOWN;
44}
45
46/*
47 * Given a port state, this function returns a value that would result in the
48 * port being in the same state, if the value was written to the port status
49 * control register.
50 * Save Read Only (RO) bits and save read/write bits where
51 * writing a 0 clears the bit and writing a 1 sets the bit (RWS).
52 * For all other types (RW1S, RW1CS, RW, and RZ), writing a '0' has no effect.
53 */
54u32 cdnsp_port_state_to_neutral(u32 state)
55{
56 /* Save read-only status and port state. */
57 return (state & CDNSP_PORT_RO) | (state & CDNSP_PORT_RWS);
58}
59
60/**
632d234b
LJ		 61 * cdnsp_find_next_ext_cap - Find the offset of the extended capabilities
62 * with capability ID id.
3d829045
PL		 63 * @base: PCI MMIO registers base address.
64 * @start: Address at which to start looking, (0 or HCC_PARAMS to start at
65 * beginning of list)
66 * @id: Extended capability ID to search for.
67 *
68 * Returns the offset of the next matching extended capability structure.
69 * Some capabilities can occur several times,
70 * e.g., the EXT_CAPS_PROTOCOL, and this provides a way to find them all.
71 */
72int cdnsp_find_next_ext_cap(void __iomem *base, u32 start, int id)
73{
74 u32 offset = start;
75 u32 next;
76 u32 val;
77
78 if (!start || start == HCC_PARAMS_OFFSET) {
79 val = readl(base + HCC_PARAMS_OFFSET);
80 if (val == ~0)
81 return 0;
82
83 offset = HCC_EXT_CAPS(val) << 2;
84 if (!offset)
85 return 0;
9c8846c7 86 }
3d829045
PL		 87
88 do {
89 val = readl(base + offset);
90 if (val == ~0)
91 return 0;
92
93 if (EXT_CAPS_ID(val) == id && offset != start)
94 return offset;
95
96 next = EXT_CAPS_NEXT(val);
97 offset += next << 2;
98 } while (next);
99
100 return 0;
101}
102
103void cdnsp_set_link_state(struct cdnsp_device *pdev,
104 __le32 __iomem *port_regs,
105 u32 link_state)
106{
118b2a32 107 int port_num = 0xFF;
3d829045
PL		 108 u32 temp;
109
110 temp = readl(port_regs);
111 temp = cdnsp_port_state_to_neutral(temp);
112 temp |= PORT_WKCONN_E | PORT_WKDISC_E;
113 writel(temp, port_regs);
114
115 temp &= ~PORT_PLS_MASK;
116 temp |= PORT_LINK_STROBE | link_state;
117
118b2a32
PL		 118 if (pdev->active_port)
119 port_num = pdev->active_port->port_num;
120
121 trace_cdnsp_handle_port_status(port_num, readl(port_regs));
3d829045 122 writel(temp, port_regs);
118b2a32 123 trace_cdnsp_link_state_changed(port_num, readl(port_regs));
3d829045
PL		 124}
125
126static void cdnsp_disable_port(struct cdnsp_device *pdev,
127 __le32 __iomem *port_regs)
128{
129 u32 temp = cdnsp_port_state_to_neutral(readl(port_regs));
130
131 writel(temp | PORT_PED, port_regs);
132}
133
134static void cdnsp_clear_port_change_bit(struct cdnsp_device *pdev,
135 __le32 __iomem *port_regs)
136{
137 u32 portsc = readl(port_regs);
138
139 writel(cdnsp_port_state_to_neutral(portsc) |
140 (portsc & PORT_CHANGE_BITS), port_regs);
141}
142
241e2ce8
PL		 143static void cdnsp_set_apb_timeout_value(struct cdnsp_device *pdev)
144{
145 struct cdns *cdns = dev_get_drvdata(pdev->dev);
146 __le32 __iomem *reg;
147 void __iomem *base;
148 u32 offset = 0;
149 u32 val;
150
151 if (!cdns->override_apb_timeout)
152 return;
153
154 base = &pdev->cap_regs->hc_capbase;
155 offset = cdnsp_find_next_ext_cap(base, offset, D_XEC_PRE_REGS_CAP);
156 reg = base + offset + REG_CHICKEN_BITS_3_OFFSET;
157
158 val = le32_to_cpu(readl(reg));
159 val = CHICKEN_APB_TIMEOUT_SET(val, cdns->override_apb_timeout);
160 writel(cpu_to_le32(val), reg);
161}
162
3d829045
PL		 163static void cdnsp_set_chicken_bits_2(struct cdnsp_device *pdev, u32 bit)
164{
165 __le32 __iomem *reg;
166 void __iomem *base;
167 u32 offset = 0;
168
169 base = &pdev->cap_regs->hc_capbase;
170 offset = cdnsp_find_next_ext_cap(base, offset, D_XEC_PRE_REGS_CAP);
171 reg = base + offset + REG_CHICKEN_BITS_2_OFFSET;
172
173 bit = readl(reg) | bit;
174 writel(bit, reg);
175}
176
177static void cdnsp_clear_chicken_bits_2(struct cdnsp_device *pdev, u32 bit)
178{
179 __le32 __iomem *reg;
180 void __iomem *base;
181 u32 offset = 0;
182
183 base = &pdev->cap_regs->hc_capbase;
184 offset = cdnsp_find_next_ext_cap(base, offset, D_XEC_PRE_REGS_CAP);
185 reg = base + offset + REG_CHICKEN_BITS_2_OFFSET;
186
187 bit = readl(reg) & ~bit;
188 writel(bit, reg);
189}
190
191/*
192 * Disable interrupts and begin the controller halting process.
193 */
194static void cdnsp_quiesce(struct cdnsp_device *pdev)
195{
196 u32 halted;
197 u32 mask;
198 u32 cmd;
199
200 mask = ~(u32)(CDNSP_IRQS);
201
202 halted = readl(&pdev->op_regs->status) & STS_HALT;
203 if (!halted)
204 mask &= ~(CMD_R_S | CMD_DEVEN);
205
206 cmd = readl(&pdev->op_regs->command);
207 cmd &= mask;
208 writel(cmd, &pdev->op_regs->command);
209}
210
211/*
212 * Force controller into halt state.
213 *
214 * Disable any IRQs and clear the run/stop bit.
215 * Controller will complete any current and actively pipelined transactions, and
216 * should halt within 16 ms of the run/stop bit being cleared.
217 * Read controller Halted bit in the status register to see when the
218 * controller is finished.
219 */
220int cdnsp_halt(struct cdnsp_device *pdev)
221{
222 int ret;
223 u32 val;
224
225 cdnsp_quiesce(pdev);
226
227 ret = readl_poll_timeout_atomic(&pdev->op_regs->status, val,
228 val & STS_HALT, 1,
229 CDNSP_MAX_HALT_USEC);
230 if (ret) {
231 dev_err(pdev->dev, "ERROR: Device halt failed\n");
232 return ret;
233 }
234
235 pdev->cdnsp_state |= CDNSP_STATE_HALTED;
236
237 return 0;
238}
239
240/*
241 * device controller died, register read returns 0xffffffff, or command never
242 * ends.
243 */
244void cdnsp_died(struct cdnsp_device *pdev)
245{
246 dev_err(pdev->dev, "ERROR: CDNSP controller not responding\n");
247 pdev->cdnsp_state |= CDNSP_STATE_DYING;
248 cdnsp_halt(pdev);
249}
250
251/*
252 * Set the run bit and wait for the device to be running.
253 */
254static int cdnsp_start(struct cdnsp_device *pdev)
255{
256 u32 temp;
257 int ret;
258
259 temp = readl(&pdev->op_regs->command);
260 temp |= (CMD_R_S | CMD_DEVEN);
261 writel(temp, &pdev->op_regs->command);
262
263 pdev->cdnsp_state = 0;
264
265 /*
266 * Wait for the STS_HALT Status bit to be 0 to indicate the device is
267 * running.
268 */
269 ret = readl_poll_timeout_atomic(&pdev->op_regs->status, temp,
270 !(temp & STS_HALT), 1,
271 CDNSP_MAX_HALT_USEC);
272 if (ret) {
273 pdev->cdnsp_state = CDNSP_STATE_DYING;
274 dev_err(pdev->dev, "ERROR: Controller run failed\n");
275 }
276
277 return ret;
278}
279
280/*
281 * Reset a halted controller.
282 *
283 * This resets pipelines, timers, counters, state machines, etc.
284 * Transactions will be terminated immediately, and operational registers
285 * will be set to their defaults.
286 */
287int cdnsp_reset(struct cdnsp_device *pdev)
288{
289 u32 command;
290 u32 temp;
291 int ret;
292
293 temp = readl(&pdev->op_regs->status);
294
295 if (temp == ~(u32)0) {
296 dev_err(pdev->dev, "Device not accessible, reset failed.\n");
297 return -ENODEV;
298 }
299
300 if ((temp & STS_HALT) == 0) {
301 dev_err(pdev->dev, "Controller not halted, aborting reset.\n");
302 return -EINVAL;
303 }
304
305 command = readl(&pdev->op_regs->command);
306 command |= CMD_RESET;
307 writel(command, &pdev->op_regs->command);
308
309 ret = readl_poll_timeout_atomic(&pdev->op_regs->command, temp,
310 !(temp & CMD_RESET), 1,
311 10 * 1000);
312 if (ret) {
313 dev_err(pdev->dev, "ERROR: Controller reset failed\n");
314 return ret;
315 }
316
317 /*
318 * CDNSP cannot write any doorbells or operational registers other
319 * than status until the "Controller Not Ready" flag is cleared.
320 */
321 ret = readl_poll_timeout_atomic(&pdev->op_regs->status, temp,
322 !(temp & STS_CNR), 1,
323 10 * 1000);
324
325 if (ret) {
326 dev_err(pdev->dev, "ERROR: Controller not ready to work\n");
327 return ret;
328 }
329
330 dev_dbg(pdev->dev, "Controller ready to work");
331
332 return ret;
333}
334
335/*
336 * cdnsp_get_endpoint_index - Find the index for an endpoint given its
337 * descriptor.Use the return value to right shift 1 for the bitmask.
338 *
339 * Index = (epnum * 2) + direction - 1,
340 * where direction = 0 for OUT, 1 for IN.
341 * For control endpoints, the IN index is used (OUT index is unused), so
342 * index = (epnum * 2) + direction - 1 = (epnum * 2) + 1 - 1 = (epnum * 2)
343 */
344static unsigned int
345 cdnsp_get_endpoint_index(const struct usb_endpoint_descriptor *desc)
346{
347 unsigned int index = (unsigned int)usb_endpoint_num(desc);
348
349 if (usb_endpoint_xfer_control(desc))
350 return index * 2;
351
352 return (index * 2) + (usb_endpoint_dir_in(desc) ? 1 : 0) - 1;
353}
354
355/*
356 * Find the flag for this endpoint (for use in the control context). Use the
357 * endpoint index to create a bitmask. The slot context is bit 0, endpoint 0 is
358 * bit 1, etc.
359 */
360static unsigned int
361 cdnsp_get_endpoint_flag(const struct usb_endpoint_descriptor *desc)
362{
363 return 1 << (cdnsp_get_endpoint_index(desc) + 1);
364}
365
366int cdnsp_ep_enqueue(struct cdnsp_ep *pep, struct cdnsp_request *preq)
367{
368 struct cdnsp_device *pdev = pep->pdev;
369 struct usb_request *request;
370 int ret;
371
118b2a32
PL		 372 if (preq->epnum == 0 && !list_empty(&pep->pending_list)) {
373 trace_cdnsp_request_enqueue_busy(preq);
3d829045 374 return -EBUSY;
118b2a32 375 }
3d829045
PL		 376
377 request = &preq->request;
378 request->actual = 0;
379 request->status = -EINPROGRESS;
380 preq->direction = pep->direction;
381 preq->epnum = pep->number;
382 preq->td.drbl = 0;
383
384 ret = usb_gadget_map_request_by_dev(pdev->dev, request, pep->direction);
118b2a32
PL		 385 if (ret) {
386 trace_cdnsp_request_enqueue_error(preq);
3d829045 387 return ret;
118b2a32 388 }
3d829045
PL		 389
390 list_add_tail(&preq->list, &pep->pending_list);
391
118b2a32
PL		 392 trace_cdnsp_request_enqueue(preq);
393
3d829045
PL		 394 switch (usb_endpoint_type(pep->endpoint.desc)) {
395 case USB_ENDPOINT_XFER_CONTROL:
396 ret = cdnsp_queue_ctrl_tx(pdev, preq);
397 break;
398 case USB_ENDPOINT_XFER_BULK:
399 case USB_ENDPOINT_XFER_INT:
400 ret = cdnsp_queue_bulk_tx(pdev, preq);
401 break;
402 case USB_ENDPOINT_XFER_ISOC:
82b0417e 403 ret = cdnsp_queue_isoc_tx(pdev, preq);
3d829045
PL		 404 }
405
406 if (ret)
407 goto unmap;
408
409 return 0;
410
411unmap:
412 usb_gadget_unmap_request_by_dev(pdev->dev, &preq->request,
413 pep->direction);
414 list_del(&preq->list);
118b2a32 415 trace_cdnsp_request_enqueue_error(preq);
3d829045
PL		 416
417 return ret;
418}
419
420/*
421 * Remove the request's TD from the endpoint ring. This may cause the
422 * controller to stop USB transfers, potentially stopping in the middle of a
423 * TRB buffer. The controller should pick up where it left off in the TD,
424 * unless a Set Transfer Ring Dequeue Pointer is issued.
425 *
426 * The TRBs that make up the buffers for the canceled request will be "removed"
427 * from the ring. Since the ring is a contiguous structure, they can't be
428 * physically removed. Instead, there are two options:
429 *
430 * 1) If the controller is in the middle of processing the request to be
431 * canceled, we simply move the ring's dequeue pointer past those TRBs
432 * using the Set Transfer Ring Dequeue Pointer command. This will be
433 * the common case, when drivers timeout on the last submitted request
434 * and attempt to cancel.
435 *
436 * 2) If the controller is in the middle of a different TD, we turn the TRBs
437 * into a series of 1-TRB transfer no-op TDs. No-ops shouldn't be chained.
438 * The controller will need to invalidate the any TRBs it has cached after
439 * the stop endpoint command.
440 *
441 * 3) The TD may have completed by the time the Stop Endpoint Command
442 * completes, so software needs to handle that case too.
443 *
444 */
445int cdnsp_ep_dequeue(struct cdnsp_ep *pep, struct cdnsp_request *preq)
446{
447 struct cdnsp_device *pdev = pep->pdev;
3b414d1b
PL		 448 int ret_stop = 0;
449 int ret_rem;
3d829045 450
118b2a32
PL		 451 trace_cdnsp_request_dequeue(preq);
452
3b414d1b
PL		 453 if (GET_EP_CTX_STATE(pep->out_ctx) == EP_STATE_RUNNING)
454 ret_stop = cdnsp_cmd_stop_ep(pdev, pep);
455
456 ret_rem = cdnsp_remove_request(pdev, preq, pep);
3d829045 457
3b414d1b 458 return ret_rem ? ret_rem : ret_stop;
3d829045
PL		 459}
460
461static void cdnsp_zero_in_ctx(struct cdnsp_device *pdev)
462{
463 struct cdnsp_input_control_ctx *ctrl_ctx;
464 struct cdnsp_slot_ctx *slot_ctx;
465 struct cdnsp_ep_ctx *ep_ctx;
466 int i;
467
468 ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
469
470 /*
471 * When a device's add flag and drop flag are zero, any subsequent
472 * configure endpoint command will leave that endpoint's state
473 * untouched. Make sure we don't leave any old state in the input
474 * endpoint contexts.
475 */
476 ctrl_ctx->drop_flags = 0;
477 ctrl_ctx->add_flags = 0;
478 slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
479 slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
480
481 /* Endpoint 0 is always valid */
482 slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1));
483 for (i = 1; i < CDNSP_ENDPOINTS_NUM; ++i) {
484 ep_ctx = cdnsp_get_ep_ctx(&pdev->in_ctx, i);
485 ep_ctx->ep_info = 0;
486 ep_ctx->ep_info2 = 0;
487 ep_ctx->deq = 0;
488 ep_ctx->tx_info = 0;
489 }
490}
491
492/* Issue a configure endpoint command and wait for it to finish. */
493static int cdnsp_configure_endpoint(struct cdnsp_device *pdev)
494{
495 int ret;
496
497 cdnsp_queue_configure_endpoint(pdev, pdev->cmd.in_ctx->dma);
498 cdnsp_ring_cmd_db(pdev);
499 ret = cdnsp_wait_for_cmd_compl(pdev);
500 if (ret) {
501 dev_err(pdev->dev,
502 "ERR: unexpected command completion code 0x%x.\n", ret);
503 return -EINVAL;
504 }
505
506 return ret;
507}
508
509static void cdnsp_invalidate_ep_events(struct cdnsp_device *pdev,
510 struct cdnsp_ep *pep)
511{
512 struct cdnsp_segment *segment;
513 union cdnsp_trb *event;
514 u32 cycle_state;
16e36101 515 u32 data;
3d829045
PL		 516
517 event = pdev->event_ring->dequeue;
518 segment = pdev->event_ring->deq_seg;
519 cycle_state = pdev->event_ring->cycle_state;
520
521 while (1) {
522 data = le32_to_cpu(event->trans_event.flags);
523
524 /* Check the owner of the TRB. */
525 if ((data & TRB_CYCLE) != cycle_state)
526 break;
527
528 if (TRB_FIELD_TO_TYPE(data) == TRB_TRANSFER &&
529 TRB_TO_EP_ID(data) == (pep->idx + 1)) {
530 data |= TRB_EVENT_INVALIDATE;
531 event->trans_event.flags = cpu_to_le32(data);
532 }
533
534 if (cdnsp_last_trb_on_seg(segment, event)) {
535 cycle_state ^= 1;
536 segment = pdev->event_ring->deq_seg->next;
537 event = segment->trbs;
538 } else {
539 event++;
540 }
541 }
542}
543
544int cdnsp_wait_for_cmd_compl(struct cdnsp_device *pdev)
545{
546 struct cdnsp_segment *event_deq_seg;
547 union cdnsp_trb *cmd_trb;
548 dma_addr_t cmd_deq_dma;
549 union cdnsp_trb *event;
550 u32 cycle_state;
f4ecdc35 551 u32 retry = 10;
3d829045
PL		 552 int ret, val;
553 u64 cmd_dma;
16e36101 554 u32 flags;
3d829045
PL		 555
556 cmd_trb = pdev->cmd.command_trb;
557 pdev->cmd.status = 0;
558
118b2a32
PL		 559 trace_cdnsp_cmd_wait_for_compl(pdev->cmd_ring, &cmd_trb->generic);
560
3d829045
PL		 561 ret = readl_poll_timeout_atomic(&pdev->op_regs->cmd_ring, val,
562 !CMD_RING_BUSY(val), 1,
563 CDNSP_CMD_TIMEOUT);
564 if (ret) {
565 dev_err(pdev->dev, "ERR: Timeout while waiting for command\n");
118b2a32 566 trace_cdnsp_cmd_timeout(pdev->cmd_ring, &cmd_trb->generic);
3d829045
PL		 567 pdev->cdnsp_state = CDNSP_STATE_DYING;
568 return -ETIMEDOUT;
569 }
570
571 event = pdev->event_ring->dequeue;
572 event_deq_seg = pdev->event_ring->deq_seg;
573 cycle_state = pdev->event_ring->cycle_state;
574
575 cmd_deq_dma = cdnsp_trb_virt_to_dma(pdev->cmd_ring->deq_seg, cmd_trb);
576 if (!cmd_deq_dma)
577 return -EINVAL;
578
579 while (1) {
580 flags = le32_to_cpu(event->event_cmd.flags);
581
582 /* Check the owner of the TRB. */
f4ecdc35
PL		 583 if ((flags & TRB_CYCLE) != cycle_state) {
584 /*
585 * Give some extra time to get chance controller
586 * to finish command before returning error code.
587 * Checking CMD_RING_BUSY is not sufficient because
588 * this bit is cleared to '0' when the Command
589 * Descriptor has been executed by controller
590 * and not when command completion event has
591 * be added to event ring.
592 */
593 if (retry--) {
594 udelay(20);
595 continue;
596 }
597
3d829045 598 return -EINVAL;
f4ecdc35 599 }
3d829045
PL		 600
601 cmd_dma = le64_to_cpu(event->event_cmd.cmd_trb);
602
603 /*
604 * Check whether the completion event is for last queued
605 * command.
606 */
607 if (TRB_FIELD_TO_TYPE(flags) != TRB_COMPLETION ||
608 cmd_dma != (u64)cmd_deq_dma) {
609 if (!cdnsp_last_trb_on_seg(event_deq_seg, event)) {
610 event++;
611 continue;
612 }
613
614 if (cdnsp_last_trb_on_ring(pdev->event_ring,
615 event_deq_seg, event))
616 cycle_state ^= 1;
617
618 event_deq_seg = event_deq_seg->next;
619 event = event_deq_seg->trbs;
620 continue;
621 }
622
118b2a32
PL		 623 trace_cdnsp_handle_command(pdev->cmd_ring, &cmd_trb->generic);
624
3d829045
PL		 625 pdev->cmd.status = GET_COMP_CODE(le32_to_cpu(event->event_cmd.status));
626 if (pdev->cmd.status == COMP_SUCCESS)
627 return 0;
628
629 return -pdev->cmd.status;
630 }
631}
632
633int cdnsp_halt_endpoint(struct cdnsp_device *pdev,
634 struct cdnsp_ep *pep,
635 int value)
636{
637 int ret;
638
118b2a32
PL		 639 trace_cdnsp_ep_halt(value ? "Set" : "Clear");
640
b25264f2
PL		 641 ret = cdnsp_cmd_stop_ep(pdev, pep);
642 if (ret)
643 return ret;
3d829045 644
b25264f2 645 if (value) {
3d829045
PL		 646 if (GET_EP_CTX_STATE(pep->out_ctx) == EP_STATE_STOPPED) {
647 cdnsp_queue_halt_endpoint(pdev, pep->idx);
648 cdnsp_ring_cmd_db(pdev);
649 ret = cdnsp_wait_for_cmd_compl(pdev);
650 }
651
652 pep->ep_state |= EP_HALTED;
653 } else {
3d829045
PL		 654 cdnsp_queue_reset_ep(pdev, pep->idx);
655 cdnsp_ring_cmd_db(pdev);
656 ret = cdnsp_wait_for_cmd_compl(pdev);
118b2a32
PL		 657 trace_cdnsp_handle_cmd_reset_ep(pep->out_ctx);
658
3d829045
PL		 659 if (ret)
660 return ret;
661
662 pep->ep_state &= ~EP_HALTED;
663
664 if (pep->idx != 0 && !(pep->ep_state & EP_WEDGE))
665 cdnsp_ring_doorbell_for_active_rings(pdev, pep);
666
667 pep->ep_state &= ~EP_WEDGE;
668 }
669
670 return 0;
671}
672
673static int cdnsp_update_eps_configuration(struct cdnsp_device *pdev,
674 struct cdnsp_ep *pep)
675{
676 struct cdnsp_input_control_ctx *ctrl_ctx;
677 struct cdnsp_slot_ctx *slot_ctx;
678 int ret = 0;
679 u32 ep_sts;
680 int i;
681
682 ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
683
684 /* Don't issue the command if there's no endpoints to update. */
685 if (ctrl_ctx->add_flags == 0 && ctrl_ctx->drop_flags == 0)
686 return 0;
687
688 ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
689 ctrl_ctx->add_flags &= cpu_to_le32(~EP0_FLAG);
690 ctrl_ctx->drop_flags &= cpu_to_le32(~(SLOT_FLAG | EP0_FLAG));
691
692 /* Fix up Context Entries field. Minimum value is EP0 == BIT(1). */
693 slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
694 for (i = CDNSP_ENDPOINTS_NUM; i >= 1; i--) {
695 __le32 le32 = cpu_to_le32(BIT(i));
696
697 if ((pdev->eps[i - 1].ring && !(ctrl_ctx->drop_flags & le32)) ||
698 (ctrl_ctx->add_flags & le32) || i == 1) {
699 slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
700 slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(i));
701 break;
702 }
703 }
704
705 ep_sts = GET_EP_CTX_STATE(pep->out_ctx);
706
707 if ((ctrl_ctx->add_flags != cpu_to_le32(SLOT_FLAG) &&
708 ep_sts == EP_STATE_DISABLED) ||
709 (ep_sts != EP_STATE_DISABLED && ctrl_ctx->drop_flags))
710 ret = cdnsp_configure_endpoint(pdev);
711
118b2a32
PL		 712 trace_cdnsp_configure_endpoint(cdnsp_get_slot_ctx(&pdev->out_ctx));
713 trace_cdnsp_handle_cmd_config_ep(pep->out_ctx);
714
3d829045
PL		 715 cdnsp_zero_in_ctx(pdev);
716
717 return ret;
718}
719
720/*
721 * This submits a Reset Device Command, which will set the device state to 0,
722 * set the device address to 0, and disable all the endpoints except the default
723 * control endpoint. The USB core should come back and call
724 * cdnsp_setup_device(), and then re-set up the configuration.
725 */
726int cdnsp_reset_device(struct cdnsp_device *pdev)
727{
728 struct cdnsp_slot_ctx *slot_ctx;
729 int slot_state;
730 int ret, i;
731
732 slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
733 slot_ctx->dev_info = 0;
734 pdev->device_address = 0;
735
736 /* If device is not setup, there is no point in resetting it. */
737 slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx);
738 slot_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state));
118b2a32 739 trace_cdnsp_reset_device(slot_ctx);
3d829045
PL		 740
741 if (slot_state <= SLOT_STATE_DEFAULT &&
742 pdev->eps[0].ep_state & EP_HALTED) {
743 cdnsp_halt_endpoint(pdev, &pdev->eps[0], 0);
744 }
745
746 /*
747 * During Reset Device command controller shall transition the
748 * endpoint ep0 to the Running State.
749 */
750 pdev->eps[0].ep_state &= ~(EP_STOPPED | EP_HALTED);
751 pdev->eps[0].ep_state |= EP_ENABLED;
752
753 if (slot_state <= SLOT_STATE_DEFAULT)
754 return 0;
755
756 cdnsp_queue_reset_device(pdev);
757 cdnsp_ring_cmd_db(pdev);
758 ret = cdnsp_wait_for_cmd_compl(pdev);
759
760 /*
761 * After Reset Device command all not default endpoints
762 * are in Disabled state.
763 */
764 for (i = 1; i < CDNSP_ENDPOINTS_NUM; ++i)
10076de3 765 pdev->eps[i].ep_state |= EP_STOPPED | EP_UNCONFIGURED;
3d829045 766
118b2a32
PL		 767 trace_cdnsp_handle_cmd_reset_dev(slot_ctx);
768
3d829045
PL		 769 if (ret)
770 dev_err(pdev->dev, "Reset device failed with error code %d",
771 ret);
772
773 return ret;
774}
775
776/*
777 * Sets the MaxPStreams field and the Linear Stream Array field.
778 * Sets the dequeue pointer to the stream context array.
779 */
780static void cdnsp_setup_streams_ep_input_ctx(struct cdnsp_device *pdev,
781 struct cdnsp_ep_ctx *ep_ctx,
782 struct cdnsp_stream_info *stream_info)
783{
784 u32 max_primary_streams;
785
786 /* MaxPStreams is the number of stream context array entries, not the
787 * number we're actually using. Must be in 2^(MaxPstreams + 1) format.
788 * fls(0) = 0, fls(0x1) = 1, fls(0x10) = 2, fls(0x100) = 3, etc.
789 */
790 max_primary_streams = fls(stream_info->num_stream_ctxs) - 2;
791 ep_ctx->ep_info &= cpu_to_le32(~EP_MAXPSTREAMS_MASK);
792 ep_ctx->ep_info |= cpu_to_le32(EP_MAXPSTREAMS(max_primary_streams)
793 | EP_HAS_LSA);
794 ep_ctx->deq = cpu_to_le64(stream_info->ctx_array_dma);
795}
796
797/*
798 * The drivers use this function to prepare a bulk endpoints to use streams.
799 *
800 * Don't allow the call to succeed if endpoint only supports one stream
801 * (which means it doesn't support streams at all).
802 */
803int cdnsp_alloc_streams(struct cdnsp_device *pdev, struct cdnsp_ep *pep)
804{
805 unsigned int num_streams = usb_ss_max_streams(pep->endpoint.comp_desc);
806 unsigned int num_stream_ctxs;
807 int ret;
808
809 if (num_streams == 0)
810 return 0;
811
812 if (num_streams > STREAM_NUM_STREAMS)
813 return -EINVAL;
814
815 /*
816 * Add two to the number of streams requested to account for
817 * stream 0 that is reserved for controller usage and one additional
818 * for TASK SET FULL response.
819 */
820 num_streams += 2;
821
822 /* The stream context array size must be a power of two */
823 num_stream_ctxs = roundup_pow_of_two(num_streams);
824
118b2a32
PL		 825 trace_cdnsp_stream_number(pep, num_stream_ctxs, num_streams);
826
3d829045
PL		 827 ret = cdnsp_alloc_stream_info(pdev, pep, num_stream_ctxs, num_streams);
828 if (ret)
829 return ret;
830
831 cdnsp_setup_streams_ep_input_ctx(pdev, pep->in_ctx, &pep->stream_info);
832
833 pep->ep_state |= EP_HAS_STREAMS;
834 pep->stream_info.td_count = 0;
835 pep->stream_info.first_prime_det = 0;
836
837 /* Subtract 1 for stream 0, which drivers can't use. */
838 return num_streams - 1;
839}
840
841int cdnsp_disable_slot(struct cdnsp_device *pdev)
842{
843 int ret;
844
845 cdnsp_queue_slot_control(pdev, TRB_DISABLE_SLOT);
846 cdnsp_ring_cmd_db(pdev);
847 ret = cdnsp_wait_for_cmd_compl(pdev);
848
849 pdev->slot_id = 0;
850 pdev->active_port = NULL;
851
118b2a32
PL		 852 trace_cdnsp_handle_cmd_disable_slot(cdnsp_get_slot_ctx(&pdev->out_ctx));
853
3d829045
PL		 854 memset(pdev->in_ctx.bytes, 0, CDNSP_CTX_SIZE);
855 memset(pdev->out_ctx.bytes, 0, CDNSP_CTX_SIZE);
856
857 return ret;
858}
859
860int cdnsp_enable_slot(struct cdnsp_device *pdev)
861{
862 struct cdnsp_slot_ctx *slot_ctx;
863 int slot_state;
864 int ret;
865
866 /* If device is not setup, there is no point in resetting it */
867 slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx);
868 slot_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state));
869
870 if (slot_state != SLOT_STATE_DISABLED)
871 return 0;
872
873 cdnsp_queue_slot_control(pdev, TRB_ENABLE_SLOT);
874 cdnsp_ring_cmd_db(pdev);
875 ret = cdnsp_wait_for_cmd_compl(pdev);
876 if (ret)
118b2a32 877 goto show_trace;
3d829045
PL		 878
879 pdev->slot_id = 1;
880
118b2a32
PL		 881show_trace:
882 trace_cdnsp_handle_cmd_enable_slot(cdnsp_get_slot_ctx(&pdev->out_ctx));
883
884 return ret;
3d829045
PL		 885}
886
887/*
888 * Issue an Address Device command with BSR=0 if setup is SETUP_CONTEXT_ONLY
889 * or with BSR = 1 if set_address is SETUP_CONTEXT_ADDRESS.
890 */
891int cdnsp_setup_device(struct cdnsp_device *pdev, enum cdnsp_setup_dev setup)
892{
893 struct cdnsp_input_control_ctx *ctrl_ctx;
894 struct cdnsp_slot_ctx *slot_ctx;
895 int dev_state = 0;
896 int ret;
897
118b2a32
PL		 898 if (!pdev->slot_id) {
899 trace_cdnsp_slot_id("incorrect");
3d829045 900 return -EINVAL;
118b2a32 901 }
3d829045
PL		 902
903 if (!pdev->active_port->port_num)
904 return -EINVAL;
905
906 slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx);
907 dev_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state));
908
118b2a32
PL		 909 if (setup == SETUP_CONTEXT_ONLY && dev_state == SLOT_STATE_DEFAULT) {
910 trace_cdnsp_slot_already_in_default(slot_ctx);
3d829045 911 return 0;
118b2a32 912 }
3d829045
PL		 913
914 slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
915 ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
916
917 if (!slot_ctx->dev_info || dev_state == SLOT_STATE_DEFAULT) {
918 ret = cdnsp_setup_addressable_priv_dev(pdev);
919 if (ret)
920 return ret;
921 }
922
923 cdnsp_copy_ep0_dequeue_into_input_ctx(pdev);
924
925 ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG | EP0_FLAG);
926 ctrl_ctx->drop_flags = 0;
927
118b2a32
PL		 928 trace_cdnsp_setup_device_slot(slot_ctx);
929
3d829045
PL		 930 cdnsp_queue_address_device(pdev, pdev->in_ctx.dma, setup);
931 cdnsp_ring_cmd_db(pdev);
932 ret = cdnsp_wait_for_cmd_compl(pdev);
933
118b2a32
PL		 934 trace_cdnsp_handle_cmd_addr_dev(cdnsp_get_slot_ctx(&pdev->out_ctx));
935
3d829045
PL		 936 /* Zero the input context control for later use. */
937 ctrl_ctx->add_flags = 0;
938 ctrl_ctx->drop_flags = 0;
939
940 return ret;
941}
942
943void cdnsp_set_usb2_hardware_lpm(struct cdnsp_device *pdev,
944 struct usb_request *req,
945 int enable)
946{
947 if (pdev->active_port != &pdev->usb2_port || !pdev->gadget.lpm_capable)
948 return;
949
118b2a32
PL		 950 trace_cdnsp_lpm(enable);
951
3d829045
PL		 952 if (enable)
953 writel(PORT_BESL(CDNSP_DEFAULT_BESL) | PORT_L1S_NYET | PORT_HLE,
954 &pdev->active_port->regs->portpmsc);
955 else
956 writel(PORT_L1S_NYET, &pdev->active_port->regs->portpmsc);
957}
958
959static int cdnsp_get_frame(struct cdnsp_device *pdev)
960{
961 return readl(&pdev->run_regs->microframe_index) >> 3;
962}
963
964static int cdnsp_gadget_ep_enable(struct usb_ep *ep,
965 const struct usb_endpoint_descriptor *desc)
966{
967 struct cdnsp_input_control_ctx *ctrl_ctx;
968 struct cdnsp_device *pdev;
969 struct cdnsp_ep *pep;
970 unsigned long flags;
971 u32 added_ctxs;
972 int ret;
973
974 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT ||
975 !desc->wMaxPacketSize)
976 return -EINVAL;
977
978 pep = to_cdnsp_ep(ep);
979 pdev = pep->pdev;
10076de3 980 pep->ep_state &= ~EP_UNCONFIGURED;
3d829045
PL		 981
982 if (dev_WARN_ONCE(pdev->dev, pep->ep_state & EP_ENABLED,
983 "%s is already enabled\n", pep->name))
984 return 0;
985
986 spin_lock_irqsave(&pdev->lock, flags);
987
988 added_ctxs = cdnsp_get_endpoint_flag(desc);
989 if (added_ctxs == SLOT_FLAG || added_ctxs == EP0_FLAG) {
990 dev_err(pdev->dev, "ERROR: Bad endpoint number\n");
991 ret = -EINVAL;
992 goto unlock;
993 }
994
995 pep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
996
997 if (pdev->gadget.speed == USB_SPEED_FULL) {
998 if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_INT)
999 pep->interval = desc->bInterval << 3;
1000 if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_ISOC)
1001 pep->interval = BIT(desc->bInterval - 1) << 3;
1002 }
1003
1004 if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_ISOC) {
1005 if (pep->interval > BIT(12)) {
1006 dev_err(pdev->dev, "bInterval %d not supported\n",
1007 desc->bInterval);
1008 ret = -EINVAL;
1009 goto unlock;
1010 }
1011 cdnsp_set_chicken_bits_2(pdev, CHICKEN_XDMA_2_TP_CACHE_DIS);
1012 }
1013
1014 ret = cdnsp_endpoint_init(pdev, pep, GFP_ATOMIC);
1015 if (ret)
1016 goto unlock;
1017
1018 ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
1019 ctrl_ctx->add_flags = cpu_to_le32(added_ctxs);
1020 ctrl_ctx->drop_flags = 0;
1021
1022 ret = cdnsp_update_eps_configuration(pdev, pep);
1023 if (ret) {
1024 cdnsp_free_endpoint_rings(pdev, pep);
1025 goto unlock;
1026 }
1027
1028 pep->ep_state |= EP_ENABLED;
1029 pep->ep_state &= ~EP_STOPPED;
1030
1031unlock:
118b2a32 1032 trace_cdnsp_ep_enable_end(pep, 0);
3d829045
PL		 1033 spin_unlock_irqrestore(&pdev->lock, flags);
1034
1035 return ret;
1036}
1037
1038static int cdnsp_gadget_ep_disable(struct usb_ep *ep)
1039{
1040 struct cdnsp_input_control_ctx *ctrl_ctx;
1041 struct cdnsp_request *preq;
1042 struct cdnsp_device *pdev;
1043 struct cdnsp_ep *pep;
1044 unsigned long flags;
1045 u32 drop_flag;
1046 int ret = 0;
1047
1048 if (!ep)
1049 return -EINVAL;
1050
1051 pep = to_cdnsp_ep(ep);
1052 pdev = pep->pdev;
1053
1054 spin_lock_irqsave(&pdev->lock, flags);
1055
1056 if (!(pep->ep_state & EP_ENABLED)) {
1057 dev_err(pdev->dev, "%s is already disabled\n", pep->name);
1058 ret = -EINVAL;
1059 goto finish;
1060 }
1061
3d829045 1062 pep->ep_state |= EP_DIS_IN_RROGRESS;
10076de3
PL		 1063
1064 /* Endpoint was unconfigured by Reset Device command. */
29988747 1065 if (!(pep->ep_state & EP_UNCONFIGURED))
10076de3 1066 cdnsp_cmd_stop_ep(pdev, pep);
3d829045
PL		 1067
1068 /* Remove all queued USB requests. */
1069 while (!list_empty(&pep->pending_list)) {
1070 preq = next_request(&pep->pending_list);
1071 cdnsp_ep_dequeue(pep, preq);
1072 }
1073
1074 cdnsp_invalidate_ep_events(pdev, pep);
1075
1076 pep->ep_state &= ~EP_DIS_IN_RROGRESS;
1077 drop_flag = cdnsp_get_endpoint_flag(pep->endpoint.desc);
1078 ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
1079 ctrl_ctx->drop_flags = cpu_to_le32(drop_flag);
1080 ctrl_ctx->add_flags = 0;
1081
1082 cdnsp_endpoint_zero(pdev, pep);
1083
10076de3
PL		 1084 if (!(pep->ep_state & EP_UNCONFIGURED))
1085 ret = cdnsp_update_eps_configuration(pdev, pep);
1086
3d829045
PL		 1087 cdnsp_free_endpoint_rings(pdev, pep);
1088
10076de3 1089 pep->ep_state &= ~(EP_ENABLED | EP_UNCONFIGURED);
3d829045
PL		 1090 pep->ep_state |= EP_STOPPED;
1091
1092finish:
118b2a32 1093 trace_cdnsp_ep_disable_end(pep, 0);
3d829045
PL		 1094 spin_unlock_irqrestore(&pdev->lock, flags);
1095
1096 return ret;
1097}
1098
1099static struct usb_request *cdnsp_gadget_ep_alloc_request(struct usb_ep *ep,
1100 gfp_t gfp_flags)
1101{
1102 struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1103 struct cdnsp_request *preq;
1104
1105 preq = kzalloc(sizeof(*preq), gfp_flags);
1106 if (!preq)
1107 return NULL;
1108
1109 preq->epnum = pep->number;
1110 preq->pep = pep;
1111
118b2a32
PL		 1112 trace_cdnsp_alloc_request(preq);
1113
3d829045
PL		 1114 return &preq->request;
1115}
1116
1117static void cdnsp_gadget_ep_free_request(struct usb_ep *ep,
1118 struct usb_request *request)
1119{
1120 struct cdnsp_request *preq = to_cdnsp_request(request);
1121
118b2a32 1122 trace_cdnsp_free_request(preq);
3d829045
PL		 1123 kfree(preq);
1124}
1125
1126static int cdnsp_gadget_ep_queue(struct usb_ep *ep,
1127 struct usb_request *request,
1128 gfp_t gfp_flags)
1129{
1130 struct cdnsp_request *preq;
1131 struct cdnsp_device *pdev;
1132 struct cdnsp_ep *pep;
1133 unsigned long flags;
1134 int ret;
1135
1136 if (!request || !ep)
1137 return -EINVAL;
1138
1139 pep = to_cdnsp_ep(ep);
1140 pdev = pep->pdev;
1141
1142 if (!(pep->ep_state & EP_ENABLED)) {
1143 dev_err(pdev->dev, "%s: can't queue to disabled endpoint\n",
1144 pep->name);
1145 return -EINVAL;
1146 }
1147
1148 preq = to_cdnsp_request(request);
1149 spin_lock_irqsave(&pdev->lock, flags);
1150 ret = cdnsp_ep_enqueue(pep, preq);
1151 spin_unlock_irqrestore(&pdev->lock, flags);
1152
1153 return ret;
1154}
1155
1156static int cdnsp_gadget_ep_dequeue(struct usb_ep *ep,
1157 struct usb_request *request)
1158{
1159 struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1160 struct cdnsp_device *pdev = pep->pdev;
1161 unsigned long flags;
1162 int ret;
1163
34f08eb0
PL		 1164 if (request->status != -EINPROGRESS)
1165 return 0;
1166
3d829045
PL		 1167 if (!pep->endpoint.desc) {
1168 dev_err(pdev->dev,
1169 "%s: can't dequeue to disabled endpoint\n",
1170 pep->name);
1171 return -ESHUTDOWN;
1172 }
1173
cf97d7af
PL		 1174 /* Requests has been dequeued during disabling endpoint. */
1175 if (!(pep->ep_state & EP_ENABLED))
1176 return 0;
1177
3d829045
PL		 1178 spin_lock_irqsave(&pdev->lock, flags);
1179 ret = cdnsp_ep_dequeue(pep, to_cdnsp_request(request));
1180 spin_unlock_irqrestore(&pdev->lock, flags);
1181
1182 return ret;
1183}
1184
1185static int cdnsp_gadget_ep_set_halt(struct usb_ep *ep, int value)
1186{
1187 struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1188 struct cdnsp_device *pdev = pep->pdev;
1189 struct cdnsp_request *preq;
18538a50 1190 unsigned long flags;
3d829045
PL		 1191 int ret;
1192
1193 spin_lock_irqsave(&pdev->lock, flags);
1194
1195 preq = next_request(&pep->pending_list);
1196 if (value) {
1197 if (preq) {
118b2a32 1198 trace_cdnsp_ep_busy_try_halt_again(pep, 0);
3d829045
PL		 1199 ret = -EAGAIN;
1200 goto done;
1201 }
1202 }
1203
1204 ret = cdnsp_halt_endpoint(pdev, pep, value);
1205
1206done:
1207 spin_unlock_irqrestore(&pdev->lock, flags);
1208 return ret;
1209}
1210
1211static int cdnsp_gadget_ep_set_wedge(struct usb_ep *ep)
1212{
1213 struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1214 struct cdnsp_device *pdev = pep->pdev;
18538a50 1215 unsigned long flags;
3d829045
PL		 1216 int ret;
1217
1218 spin_lock_irqsave(&pdev->lock, flags);
1219 pep->ep_state |= EP_WEDGE;
1220 ret = cdnsp_halt_endpoint(pdev, pep, 1);
1221 spin_unlock_irqrestore(&pdev->lock, flags);
1222
1223 return ret;
1224}
1225
1226static const struct usb_ep_ops cdnsp_gadget_ep0_ops = {
1227 .enable = cdnsp_gadget_ep_enable,
1228 .disable = cdnsp_gadget_ep_disable,
1229 .alloc_request = cdnsp_gadget_ep_alloc_request,
1230 .free_request = cdnsp_gadget_ep_free_request,
1231 .queue = cdnsp_gadget_ep_queue,
1232 .dequeue = cdnsp_gadget_ep_dequeue,
1233 .set_halt = cdnsp_gadget_ep_set_halt,
1234 .set_wedge = cdnsp_gadget_ep_set_wedge,
1235};
1236
1237static const struct usb_ep_ops cdnsp_gadget_ep_ops = {
1238 .enable = cdnsp_gadget_ep_enable,
1239 .disable = cdnsp_gadget_ep_disable,
1240 .alloc_request = cdnsp_gadget_ep_alloc_request,
1241 .free_request = cdnsp_gadget_ep_free_request,
1242 .queue = cdnsp_gadget_ep_queue,
1243 .dequeue = cdnsp_gadget_ep_dequeue,
1244 .set_halt = cdnsp_gadget_ep_set_halt,
1245 .set_wedge = cdnsp_gadget_ep_set_wedge,
1246};
1247
1248void cdnsp_gadget_giveback(struct cdnsp_ep *pep,
1249 struct cdnsp_request *preq,
1250 int status)
1251{
1252 struct cdnsp_device *pdev = pep->pdev;
1253
1254 list_del(&preq->list);
1255
1256 if (preq->request.status == -EINPROGRESS)
1257 preq->request.status = status;
1258
1259 usb_gadget_unmap_request_by_dev(pdev->dev, &preq->request,
1260 preq->direction);
1261
118b2a32
PL		 1262 trace_cdnsp_request_giveback(preq);
1263
3d829045
PL		 1264 if (preq != &pdev->ep0_preq) {
1265 spin_unlock(&pdev->lock);
1266 usb_gadget_giveback_request(&pep->endpoint, &preq->request);
1267 spin_lock(&pdev->lock);
1268 }
1269}
1270
1271static struct usb_endpoint_descriptor cdnsp_gadget_ep0_desc = {
1272 .bLength = USB_DT_ENDPOINT_SIZE,
1273 .bDescriptorType = USB_DT_ENDPOINT,
1274 .bmAttributes = USB_ENDPOINT_XFER_CONTROL,
1275};
1276
1277static int cdnsp_run(struct cdnsp_device *pdev,
1278 enum usb_device_speed speed)
1279{
1280 u32 fs_speed = 0;
3d829045
PL		 1281 u32 temp;
1282 int ret;
1283
3d829045
PL		 1284 temp = readl(&pdev->ir_set->irq_control);
1285 temp &= ~IMOD_INTERVAL_MASK;
1286 temp |= ((IMOD_DEFAULT_INTERVAL / 250) & IMOD_INTERVAL_MASK);
1287 writel(temp, &pdev->ir_set->irq_control);
1288
1289 temp = readl(&pdev->port3x_regs->mode_addr);
1290
1291 switch (speed) {
1292 case USB_SPEED_SUPER_PLUS:
1293 temp |= CFG_3XPORT_SSP_SUPPORT;
1294 break;
1295 case USB_SPEED_SUPER:
1296 temp &= ~CFG_3XPORT_SSP_SUPPORT;
1297 break;
1298 case USB_SPEED_HIGH:
1299 break;
1300 case USB_SPEED_FULL:
1301 fs_speed = PORT_REG6_FORCE_FS;
1302 break;
1303 default:
1304 dev_err(pdev->dev, "invalid maximum_speed parameter %d\n",
1305 speed);
1306 fallthrough;
1307 case USB_SPEED_UNKNOWN:
1308 /* Default to superspeed. */
1309 speed = USB_SPEED_SUPER;
1310 break;
1311 }
1312
1313 if (speed >= USB_SPEED_SUPER) {
1314 writel(temp, &pdev->port3x_regs->mode_addr);
1315 cdnsp_set_link_state(pdev, &pdev->usb3_port.regs->portsc,
1316 XDEV_RXDETECT);
1317 } else {
1318 cdnsp_disable_port(pdev, &pdev->usb3_port.regs->portsc);
1319 }
1320
1321 cdnsp_set_link_state(pdev, &pdev->usb2_port.regs->portsc,
1322 XDEV_RXDETECT);
1323
1324 cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1325
1326 writel(PORT_REG6_L1_L0_HW_EN | fs_speed, &pdev->port20_regs->port_reg6);
1327
1328 ret = cdnsp_start(pdev);
1329 if (ret) {
1330 ret = -ENODEV;
1331 goto err;
1332 }
1333
1334 temp = readl(&pdev->op_regs->command);
1335 temp |= (CMD_INTE);
1336 writel(temp, &pdev->op_regs->command);
1337
1338 temp = readl(&pdev->ir_set->irq_pending);
1339 writel(IMAN_IE_SET(temp), &pdev->ir_set->irq_pending);
1340
118b2a32 1341 trace_cdnsp_init("Controller ready to work");
3d829045
PL		 1342 return 0;
1343err:
1344 cdnsp_halt(pdev);
1345 return ret;
1346}
1347
1348static int cdnsp_gadget_udc_start(struct usb_gadget *g,
1349 struct usb_gadget_driver *driver)
1350{
1351 enum usb_device_speed max_speed = driver->max_speed;
1352 struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1353 unsigned long flags;
1354 int ret;
1355
1356 spin_lock_irqsave(&pdev->lock, flags);
1357 pdev->gadget_driver = driver;
1358
1359 /* limit speed if necessary */
1360 max_speed = min(driver->max_speed, g->max_speed);
1361 ret = cdnsp_run(pdev, max_speed);
1362
1363 spin_unlock_irqrestore(&pdev->lock, flags);
1364
1365 return ret;
1366}
1367
1368/*
1369 * Update Event Ring Dequeue Pointer:
1370 * - When all events have finished
1371 * - To avoid "Event Ring Full Error" condition
1372 */
1373void cdnsp_update_erst_dequeue(struct cdnsp_device *pdev,
1374 union cdnsp_trb *event_ring_deq,
1375 u8 clear_ehb)
1376{
1377 u64 temp_64;
1378 dma_addr_t deq;
1379
1380 temp_64 = cdnsp_read_64(&pdev->ir_set->erst_dequeue);
1381
1382 /* If necessary, update the HW's version of the event ring deq ptr. */
1383 if (event_ring_deq != pdev->event_ring->dequeue) {
1384 deq = cdnsp_trb_virt_to_dma(pdev->event_ring->deq_seg,
1385 pdev->event_ring->dequeue);
1386 temp_64 &= ERST_PTR_MASK;
1387 temp_64 |= ((u64)deq & (u64)~ERST_PTR_MASK);
1388 }
1389
1390 /* Clear the event handler busy flag (RW1C). */
1391 if (clear_ehb)
1392 temp_64 |= ERST_EHB;
1393 else
1394 temp_64 &= ~ERST_EHB;
1395
1396 cdnsp_write_64(temp_64, &pdev->ir_set->erst_dequeue);
1397}
1398
1399static void cdnsp_clear_cmd_ring(struct cdnsp_device *pdev)
1400{
1401 struct cdnsp_segment *seg;
1402 u64 val_64;
1403 int i;
1404
1405 cdnsp_initialize_ring_info(pdev->cmd_ring);
1406
1407 seg = pdev->cmd_ring->first_seg;
1408 for (i = 0; i < pdev->cmd_ring->num_segs; i++) {
1409 memset(seg->trbs, 0,
1410 sizeof(union cdnsp_trb) * (TRBS_PER_SEGMENT - 1));
1411 seg = seg->next;
1412 }
1413
1414 /* Set the address in the Command Ring Control register. */
1415 val_64 = cdnsp_read_64(&pdev->op_regs->cmd_ring);
1416 val_64 = (val_64 & (u64)CMD_RING_RSVD_BITS) |
1417 (pdev->cmd_ring->first_seg->dma & (u64)~CMD_RING_RSVD_BITS) |
1418 pdev->cmd_ring->cycle_state;
1419 cdnsp_write_64(val_64, &pdev->op_regs->cmd_ring);
1420}
1421
1422static void cdnsp_consume_all_events(struct cdnsp_device *pdev)
1423{
1424 struct cdnsp_segment *event_deq_seg;
1425 union cdnsp_trb *event_ring_deq;
1426 union cdnsp_trb *event;
1427 u32 cycle_bit;
1428
1429 event_ring_deq = pdev->event_ring->dequeue;
1430 event_deq_seg = pdev->event_ring->deq_seg;
1431 event = pdev->event_ring->dequeue;
1432
1433 /* Update ring dequeue pointer. */
1434 while (1) {
1435 cycle_bit = (le32_to_cpu(event->event_cmd.flags) & TRB_CYCLE);
1436
1437 /* Does the controller or driver own the TRB? */
1438 if (cycle_bit != pdev->event_ring->cycle_state)
1439 break;
1440
1441 cdnsp_inc_deq(pdev, pdev->event_ring);
1442
1443 if (!cdnsp_last_trb_on_seg(event_deq_seg, event)) {
1444 event++;
1445 continue;
1446 }
1447
1448 if (cdnsp_last_trb_on_ring(pdev->event_ring, event_deq_seg,
1449 event))
1450 cycle_bit ^= 1;
1451
1452 event_deq_seg = event_deq_seg->next;
1453 event = event_deq_seg->trbs;
1454 }
1455
1456 cdnsp_update_erst_dequeue(pdev, event_ring_deq, 1);
1457}
1458
1459static void cdnsp_stop(struct cdnsp_device *pdev)
1460{
1461 u32 temp;
1462
3d829045
PL		 1463 /* Remove internally queued request for ep0. */
1464 if (!list_empty(&pdev->eps[0].pending_list)) {
1465 struct cdnsp_request *req;
1466
1467 req = next_request(&pdev->eps[0].pending_list);
1468 if (req == &pdev->ep0_preq)
1469 cdnsp_ep_dequeue(&pdev->eps[0], req);
1470 }
1471
1472 cdnsp_disable_port(pdev, &pdev->usb2_port.regs->portsc);
1473 cdnsp_disable_port(pdev, &pdev->usb3_port.regs->portsc);
1474 cdnsp_disable_slot(pdev);
1475 cdnsp_halt(pdev);
1476
1477 temp = readl(&pdev->op_regs->status);
1478 writel((temp & ~0x1fff) | STS_EINT, &pdev->op_regs->status);
1479 temp = readl(&pdev->ir_set->irq_pending);
1480 writel(IMAN_IE_CLEAR(temp), &pdev->ir_set->irq_pending);
1481
1482 cdnsp_clear_port_change_bit(pdev, &pdev->usb2_port.regs->portsc);
1483 cdnsp_clear_port_change_bit(pdev, &pdev->usb3_port.regs->portsc);
1484
1485 /* Clear interrupt line */
1486 temp = readl(&pdev->ir_set->irq_pending);
1487 temp |= IMAN_IP;
1488 writel(temp, &pdev->ir_set->irq_pending);
1489
1490 cdnsp_consume_all_events(pdev);
1491 cdnsp_clear_cmd_ring(pdev);
118b2a32
PL		 1492
1493 trace_cdnsp_exit("Controller stopped.");
3d829045
PL		 1494}
1495
1496/*
1497 * Stop controller.
1498 * This function is called by the gadget core when the driver is removed.
1499 * Disable slot, disable IRQs, and quiesce the controller.
1500 */
1501static int cdnsp_gadget_udc_stop(struct usb_gadget *g)
1502{
1503 struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1504 unsigned long flags;
1505
1506 spin_lock_irqsave(&pdev->lock, flags);
1507 cdnsp_stop(pdev);
1508 pdev->gadget_driver = NULL;
1509 spin_unlock_irqrestore(&pdev->lock, flags);
1510
1511 return 0;
1512}
1513
1514static int cdnsp_gadget_get_frame(struct usb_gadget *g)
1515{
1516 struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1517
1518 return cdnsp_get_frame(pdev);
1519}
1520
1521static void __cdnsp_gadget_wakeup(struct cdnsp_device *pdev)
1522{
1523 struct cdnsp_port_regs __iomem *port_regs;
1524 u32 portpm, portsc;
1525
1526 port_regs = pdev->active_port->regs;
1527 portsc = readl(&port_regs->portsc) & PORT_PLS_MASK;
1528
1529 /* Remote wakeup feature is not enabled by host. */
1530 if (pdev->gadget.speed < USB_SPEED_SUPER && portsc == XDEV_U2) {
1531 portpm = readl(&port_regs->portpmsc);
1532
1533 if (!(portpm & PORT_RWE))
1534 return;
1535 }
1536
1537 if (portsc == XDEV_U3 && !pdev->may_wakeup)
1538 return;
1539
1540 cdnsp_set_link_state(pdev, &port_regs->portsc, XDEV_U0);
1541
1542 pdev->cdnsp_state |= CDNSP_WAKEUP_PENDING;
1543}
1544
1545static int cdnsp_gadget_wakeup(struct usb_gadget *g)
1546{
1547 struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1548 unsigned long flags;
1549
1550 spin_lock_irqsave(&pdev->lock, flags);
1551 __cdnsp_gadget_wakeup(pdev);
1552 spin_unlock_irqrestore(&pdev->lock, flags);
1553
1554 return 0;
1555}
1556
1557static int cdnsp_gadget_set_selfpowered(struct usb_gadget *g,
1558 int is_selfpowered)
1559{
1560 struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1561 unsigned long flags;
1562
1563 spin_lock_irqsave(&pdev->lock, flags);
1564 g->is_selfpowered = !!is_selfpowered;
1565 spin_unlock_irqrestore(&pdev->lock, flags);
1566
1567 return 0;
1568}
1569
1570static int cdnsp_gadget_pullup(struct usb_gadget *gadget, int is_on)
1571{
1572 struct cdnsp_device *pdev = gadget_to_cdnsp(gadget);
1573 struct cdns *cdns = dev_get_drvdata(pdev->dev);
4c4e162d 1574 unsigned long flags;
3d829045 1575
118b2a32
PL		 1576 trace_cdnsp_pullup(is_on);
1577
4c4e162d
PL		 1578 /*
1579 * Disable events handling while controller is being
1580 * enabled/disabled.
1581 */
1582 disable_irq(cdns->dev_irq);
1583 spin_lock_irqsave(&pdev->lock, flags);
1584
3d829045
PL		 1585 if (!is_on) {
1586 cdnsp_reset_device(pdev);
1587 cdns_clear_vbus(cdns);
1588 } else {
1589 cdns_set_vbus(cdns);
1590 }
4c4e162d
PL		 1591
1592 spin_unlock_irqrestore(&pdev->lock, flags);
1593 enable_irq(cdns->dev_irq);
1594
3d829045
PL		 1595 return 0;
1596}
1597
7650778e 1598static const struct usb_gadget_ops cdnsp_gadget_ops = {
3d829045
PL		 1599 .get_frame = cdnsp_gadget_get_frame,
1600 .wakeup = cdnsp_gadget_wakeup,
1601 .set_selfpowered = cdnsp_gadget_set_selfpowered,
1602 .pullup = cdnsp_gadget_pullup,
1603 .udc_start = cdnsp_gadget_udc_start,
1604 .udc_stop = cdnsp_gadget_udc_stop,
1605};
1606
1607static void cdnsp_get_ep_buffering(struct cdnsp_device *pdev,
1608 struct cdnsp_ep *pep)
1609{
1610 void __iomem *reg = &pdev->cap_regs->hc_capbase;
1611 int endpoints;
1612
1613 reg += cdnsp_find_next_ext_cap(reg, 0, XBUF_CAP_ID);
1614
1615 if (!pep->direction) {
1616 pep->buffering = readl(reg + XBUF_RX_TAG_MASK_0_OFFSET);
1617 pep->buffering_period = readl(reg + XBUF_RX_TAG_MASK_1_OFFSET);
1618 pep->buffering = (pep->buffering + 1) / 2;
1619 pep->buffering_period = (pep->buffering_period + 1) / 2;
1620 return;
1621 }
1622
16e36101 1623 endpoints = HCS_ENDPOINTS(pdev->hcs_params1) / 2;
3d829045
PL		 1624
1625 /* Set to XBUF_TX_TAG_MASK_0 register. */
1626 reg += XBUF_TX_CMD_OFFSET + (endpoints * 2 + 2) * sizeof(u32);
1627 /* Set reg to XBUF_TX_TAG_MASK_N related with this endpoint. */
1628 reg += pep->number * sizeof(u32) * 2;
1629
1630 pep->buffering = (readl(reg) + 1) / 2;
1631 pep->buffering_period = pep->buffering;
1632}
1633
1634static int cdnsp_gadget_init_endpoints(struct cdnsp_device *pdev)
1635{
1636 int max_streams = HCC_MAX_PSA(pdev->hcc_params);
1637 struct cdnsp_ep *pep;
1638 int i;
1639
1640 INIT_LIST_HEAD(&pdev->gadget.ep_list);
1641
1642 if (max_streams < STREAM_LOG_STREAMS) {
1643 dev_err(pdev->dev, "Stream size %d not supported\n",
1644 max_streams);
1645 return -EINVAL;
1646 }
1647
1648 max_streams = STREAM_LOG_STREAMS;
1649
1650 for (i = 0; i < CDNSP_ENDPOINTS_NUM; i++) {
1651 bool direction = !(i & 1); /* Start from OUT endpoint. */
1652 u8 epnum = ((i + 1) >> 1);
1653
1654 if (!CDNSP_IF_EP_EXIST(pdev, epnum, direction))
1655 continue;
1656
1657 pep = &pdev->eps[i];
1658 pep->pdev = pdev;
1659 pep->number = epnum;
1660 pep->direction = direction; /* 0 for OUT, 1 for IN. */
1661
1662 /*
1663 * Ep0 is bidirectional, so ep0in and ep0out are represented by
1664 * pdev->eps[0]
1665 */
1666 if (epnum == 0) {
1667 snprintf(pep->name, sizeof(pep->name), "ep%d%s",
1668 epnum, "BiDir");
1669
1670 pep->idx = 0;
1671 usb_ep_set_maxpacket_limit(&pep->endpoint, 512);
1672 pep->endpoint.maxburst = 1;
1673 pep->endpoint.ops = &cdnsp_gadget_ep0_ops;
1674 pep->endpoint.desc = &cdnsp_gadget_ep0_desc;
1675 pep->endpoint.comp_desc = NULL;
1676 pep->endpoint.caps.type_control = true;
1677 pep->endpoint.caps.dir_in = true;
1678 pep->endpoint.caps.dir_out = true;
1679
1680 pdev->ep0_preq.epnum = pep->number;
1681 pdev->ep0_preq.pep = pep;
1682 pdev->gadget.ep0 = &pep->endpoint;
1683 } else {
1684 snprintf(pep->name, sizeof(pep->name), "ep%d%s",
1685 epnum, (pep->direction) ? "in" : "out");
1686
1687 pep->idx = (epnum * 2 + (direction ? 1 : 0)) - 1;
1688 usb_ep_set_maxpacket_limit(&pep->endpoint, 1024);
1689
1690 pep->endpoint.max_streams = max_streams;
1691 pep->endpoint.ops = &cdnsp_gadget_ep_ops;
1692 list_add_tail(&pep->endpoint.ep_list,
1693 &pdev->gadget.ep_list);
1694
1695 pep->endpoint.caps.type_iso = true;
1696 pep->endpoint.caps.type_bulk = true;
1697 pep->endpoint.caps.type_int = true;
1698
1699 pep->endpoint.caps.dir_in = direction;
1700 pep->endpoint.caps.dir_out = !direction;
1701 }
1702
1703 pep->endpoint.name = pep->name;
1704 pep->in_ctx = cdnsp_get_ep_ctx(&pdev->in_ctx, pep->idx);
1705 pep->out_ctx = cdnsp_get_ep_ctx(&pdev->out_ctx, pep->idx);
1706 cdnsp_get_ep_buffering(pdev, pep);
1707
1708 dev_dbg(pdev->dev, "Init %s, MPS: %04x SupType: "
1709 "CTRL: %s, INT: %s, BULK: %s, ISOC %s, "
1710 "SupDir IN: %s, OUT: %s\n",
1711 pep->name, 1024,
f386bfad
KK		 1712 str_yes_no(pep->endpoint.caps.type_control),
1713 str_yes_no(pep->endpoint.caps.type_int),
1714 str_yes_no(pep->endpoint.caps.type_bulk),
1715 str_yes_no(pep->endpoint.caps.type_iso),
1716 str_yes_no(pep->endpoint.caps.dir_in),
1717 str_yes_no(pep->endpoint.caps.dir_out));
3d829045
PL		 1718
1719 INIT_LIST_HEAD(&pep->pending_list);
1720 }
1721
1722 return 0;
1723}
1724
1725static void cdnsp_gadget_free_endpoints(struct cdnsp_device *pdev)
1726{
1727 struct cdnsp_ep *pep;
1728 int i;
1729
1730 for (i = 0; i < CDNSP_ENDPOINTS_NUM; i++) {
1731 pep = &pdev->eps[i];
1732 if (pep->number != 0 && pep->out_ctx)
1733 list_del(&pep->endpoint.ep_list);
1734 }
1735}
1736
1737void cdnsp_disconnect_gadget(struct cdnsp_device *pdev)
1738{
1739 pdev->cdnsp_state |= CDNSP_STATE_DISCONNECT_PENDING;
1740
1741 if (pdev->gadget_driver && pdev->gadget_driver->disconnect) {
1742 spin_unlock(&pdev->lock);
1743 pdev->gadget_driver->disconnect(&pdev->gadget);
1744 spin_lock(&pdev->lock);
1745 }
1746
1747 pdev->gadget.speed = USB_SPEED_UNKNOWN;
1748 usb_gadget_set_state(&pdev->gadget, USB_STATE_NOTATTACHED);
1749
1750 pdev->cdnsp_state &= ~CDNSP_STATE_DISCONNECT_PENDING;
1751}
1752
1753void cdnsp_suspend_gadget(struct cdnsp_device *pdev)
1754{
1755 if (pdev->gadget_driver && pdev->gadget_driver->suspend) {
1756 spin_unlock(&pdev->lock);
1757 pdev->gadget_driver->suspend(&pdev->gadget);
1758 spin_lock(&pdev->lock);
1759 }
1760}
1761
1762void cdnsp_resume_gadget(struct cdnsp_device *pdev)
1763{
1764 if (pdev->gadget_driver && pdev->gadget_driver->resume) {
1765 spin_unlock(&pdev->lock);
1766 pdev->gadget_driver->resume(&pdev->gadget);
1767 spin_lock(&pdev->lock);
1768 }
1769}
1770
1771void cdnsp_irq_reset(struct cdnsp_device *pdev)
1772{
1773 struct cdnsp_port_regs __iomem *port_regs;
1774
1775 cdnsp_reset_device(pdev);
1776
1777 port_regs = pdev->active_port->regs;
1778 pdev->gadget.speed = cdnsp_port_speed(readl(port_regs));
1779
1780 spin_unlock(&pdev->lock);
1781 usb_gadget_udc_reset(&pdev->gadget, pdev->gadget_driver);
1782 spin_lock(&pdev->lock);
1783
1784 switch (pdev->gadget.speed) {
1785 case USB_SPEED_SUPER_PLUS:
1786 case USB_SPEED_SUPER:
1787 cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1788 pdev->gadget.ep0->maxpacket = 512;
1789 break;
1790 case USB_SPEED_HIGH:
1791 case USB_SPEED_FULL:
1792 cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
1793 pdev->gadget.ep0->maxpacket = 64;
1794 break;
1795 default:
1796 /* Low speed is not supported. */
1797 dev_err(pdev->dev, "Unknown device speed\n");
1798 break;
1799 }
1800
1801 cdnsp_clear_chicken_bits_2(pdev, CHICKEN_XDMA_2_TP_CACHE_DIS);
1802 cdnsp_setup_device(pdev, SETUP_CONTEXT_ONLY);
1803 usb_gadget_set_state(&pdev->gadget, USB_STATE_DEFAULT);
1804}
1805
1806static void cdnsp_get_rev_cap(struct cdnsp_device *pdev)
1807{
1808 void __iomem *reg = &pdev->cap_regs->hc_capbase;
3d829045
PL		 1809
1810 reg += cdnsp_find_next_ext_cap(reg, 0, RTL_REV_CAP);
16e36101 1811 pdev->rev_cap = reg;
3d829045 1812
8614ecdb
PL		 1813 pdev->rtl_revision = readl(&pdev->rev_cap->rtl_revision);
1814
3d829045 1815 dev_info(pdev->dev, "Rev: %08x/%08x, eps: %08x, buff: %08x/%08x\n",
16e36101
PL		 1816 readl(&pdev->rev_cap->ctrl_revision),
1817 readl(&pdev->rev_cap->rtl_revision),
1818 readl(&pdev->rev_cap->ep_supported),
1819 readl(&pdev->rev_cap->rx_buff_size),
1820 readl(&pdev->rev_cap->tx_buff_size));
3d829045
PL		 1821}
1822
1823static int cdnsp_gen_setup(struct cdnsp_device *pdev)
1824{
1825 int ret;
1826 u32 reg;
1827
1828 pdev->cap_regs = pdev->regs;
1829 pdev->op_regs = pdev->regs +
1830 HC_LENGTH(readl(&pdev->cap_regs->hc_capbase));
1831 pdev->run_regs = pdev->regs +
1832 (readl(&pdev->cap_regs->run_regs_off) & RTSOFF_MASK);
1833
1834 /* Cache read-only capability registers */
1835 pdev->hcs_params1 = readl(&pdev->cap_regs->hcs_params1);
1836 pdev->hcc_params = readl(&pdev->cap_regs->hc_capbase);
1837 pdev->hci_version = HC_VERSION(pdev->hcc_params);
1838 pdev->hcc_params = readl(&pdev->cap_regs->hcc_params);
1839
241e2ce8
PL		 1840 /*
1841 * Override the APB timeout value to give the controller more time for
1842 * enabling UTMI clock and synchronizing APB and UTMI clock domains.
1843 * This fix is platform specific and is required to fixes issue with
1844 * reading incorrect value from PORTSC register after resuming
1845 * from L1 state.
1846 */
1847 cdnsp_set_apb_timeout_value(pdev);
1848
3d829045
PL		 1849 cdnsp_get_rev_cap(pdev);
1850
1851 /* Make sure the Device Controller is halted. */
1852 ret = cdnsp_halt(pdev);
1853 if (ret)
1854 return ret;
1855
1856 /* Reset the internal controller memory state and registers. */
1857 ret = cdnsp_reset(pdev);
1858 if (ret)
1859 return ret;
1860
1861 /*
1862 * Set dma_mask and coherent_dma_mask to 64-bits,
1863 * if controller supports 64-bit addressing.
1864 */
1865 if (HCC_64BIT_ADDR(pdev->hcc_params) &&
1866 !dma_set_mask(pdev->dev, DMA_BIT_MASK(64))) {
1867 dev_dbg(pdev->dev, "Enabling 64-bit DMA addresses.\n");
1868 dma_set_coherent_mask(pdev->dev, DMA_BIT_MASK(64));
1869 } else {
1870 /*
1871 * This is to avoid error in cases where a 32-bit USB
1872 * controller is used on a 64-bit capable system.
1873 */
1874 ret = dma_set_mask(pdev->dev, DMA_BIT_MASK(32));
1875 if (ret)
1876 return ret;
1877
1878 dev_dbg(pdev->dev, "Enabling 32-bit DMA addresses.\n");
1879 dma_set_coherent_mask(pdev->dev, DMA_BIT_MASK(32));
1880 }
1881
1882 spin_lock_init(&pdev->lock);
1883
dc68ba6c 1884 ret = cdnsp_mem_init(pdev);
3d829045
PL		 1885 if (ret)
1886 return ret;
1887
1888 /*
1889 * Software workaround for U1: after transition
1890 * to U1 the controller starts gating clock, and in some cases,
1891 * it causes that controller stack.
1892 */
1893 reg = readl(&pdev->port3x_regs->mode_2);
1894 reg &= ~CFG_3XPORT_U1_PIPE_CLK_GATE_EN;
1895 writel(reg, &pdev->port3x_regs->mode_2);
1896
1897 return 0;
1898}
1899
1900static int __cdnsp_gadget_init(struct cdns *cdns)
1901{
1902 struct cdnsp_device *pdev;
1903 u32 max_speed;
1904 int ret = -ENOMEM;
1905
1906 cdns_drd_gadget_on(cdns);
1907
1908 pdev = kzalloc(sizeof(*pdev), GFP_KERNEL);
1909 if (!pdev)
1910 return -ENOMEM;
1911
1912 pm_runtime_get_sync(cdns->dev);
1913
1914 cdns->gadget_dev = pdev;
1915 pdev->dev = cdns->dev;
1916 pdev->regs = cdns->dev_regs;
1917 max_speed = usb_get_maximum_speed(cdns->dev);
1918
1919 switch (max_speed) {
1920 case USB_SPEED_FULL:
1921 case USB_SPEED_HIGH:
1922 case USB_SPEED_SUPER:
1923 case USB_SPEED_SUPER_PLUS:
1924 break;
1925 default:
1926 dev_err(cdns->dev, "invalid speed parameter %d\n", max_speed);
1927 fallthrough;
1928 case USB_SPEED_UNKNOWN:
1929 /* Default to SSP */
1930 max_speed = USB_SPEED_SUPER_PLUS;
1931 break;
1932 }
1933
1934 pdev->gadget.ops = &cdnsp_gadget_ops;
1935 pdev->gadget.name = "cdnsp-gadget";
1936 pdev->gadget.speed = USB_SPEED_UNKNOWN;
1937 pdev->gadget.sg_supported = 1;
aa82f94e 1938 pdev->gadget.max_speed = max_speed;
3d829045
PL		 1939 pdev->gadget.lpm_capable = 1;
1940
1941 pdev->setup_buf = kzalloc(CDNSP_EP0_SETUP_SIZE, GFP_KERNEL);
1942 if (!pdev->setup_buf)
1943 goto free_pdev;
1944
1945 /*
1946 * Controller supports not aligned buffer but it should improve
1947 * performance.
1948 */
1949 pdev->gadget.quirk_ep_out_aligned_size = true;
1950
1951 ret = cdnsp_gen_setup(pdev);
1952 if (ret) {
1953 dev_err(pdev->dev, "Generic initialization failed %d\n", ret);
1954 goto free_setup;
1955 }
1956
1957 ret = cdnsp_gadget_init_endpoints(pdev);
1958 if (ret) {
1959 dev_err(pdev->dev, "failed to initialize endpoints\n");
1960 goto halt_pdev;
1961 }
1962
1963 ret = usb_add_gadget_udc(pdev->dev, &pdev->gadget);
1964 if (ret) {
1965 dev_err(pdev->dev, "failed to register udc\n");
1966 goto free_endpoints;
1967 }
1968
1969 ret = devm_request_threaded_irq(pdev->dev, cdns->dev_irq,
1970 cdnsp_irq_handler,
1971 cdnsp_thread_irq_handler, IRQF_SHARED,
1972 dev_name(pdev->dev), pdev);
1973 if (ret)
1974 goto del_gadget;
1975
1976 return 0;
1977
1978del_gadget:
1979 usb_del_gadget_udc(&pdev->gadget);
1980free_endpoints:
1981 cdnsp_gadget_free_endpoints(pdev);
1982halt_pdev:
1983 cdnsp_halt(pdev);
1984 cdnsp_reset(pdev);
1985 cdnsp_mem_cleanup(pdev);
1986free_setup:
1987 kfree(pdev->setup_buf);
1988free_pdev:
1989 kfree(pdev);
1990
1991 return ret;
1992}
1993
1994static void cdnsp_gadget_exit(struct cdns *cdns)
1995{
1996 struct cdnsp_device *pdev = cdns->gadget_dev;
1997
1998 devm_free_irq(pdev->dev, cdns->dev_irq, pdev);
1999 pm_runtime_mark_last_busy(cdns->dev);
2000 pm_runtime_put_autosuspend(cdns->dev);
2001 usb_del_gadget_udc(&pdev->gadget);
2002 cdnsp_gadget_free_endpoints(pdev);
2003 cdnsp_mem_cleanup(pdev);
2004 kfree(pdev);
2005 cdns->gadget_dev = NULL;
2006 cdns_drd_gadget_off(cdns);
2007}
2008
2009static int cdnsp_gadget_suspend(struct cdns *cdns, bool do_wakeup)
2010{
2011 struct cdnsp_device *pdev = cdns->gadget_dev;
2012 unsigned long flags;
2013
2014 if (pdev->link_state == XDEV_U3)
2015 return 0;
2016
2017 spin_lock_irqsave(&pdev->lock, flags);
2018 cdnsp_disconnect_gadget(pdev);
2019 cdnsp_stop(pdev);
2020 spin_unlock_irqrestore(&pdev->lock, flags);
2021
2022 return 0;
2023}
2024
0bde749c 2025static int cdnsp_gadget_resume(struct cdns *cdns, bool lost_power)
3d829045
PL		 2026{
2027 struct cdnsp_device *pdev = cdns->gadget_dev;
2028 enum usb_device_speed max_speed;
2029 unsigned long flags;
2030 int ret;
2031
2032 if (!pdev->gadget_driver)
2033 return 0;
2034
2035 spin_lock_irqsave(&pdev->lock, flags);
2036 max_speed = pdev->gadget_driver->max_speed;
2037
2038 /* Limit speed if necessary. */
2039 max_speed = min(max_speed, pdev->gadget.max_speed);
2040
2041 ret = cdnsp_run(pdev, max_speed);
2042
2043 if (pdev->link_state == XDEV_U3)
2044 __cdnsp_gadget_wakeup(pdev);
2045
2046 spin_unlock_irqrestore(&pdev->lock, flags);
2047
2048 return ret;
2049}
2050
2051/**
2052 * cdnsp_gadget_init - initialize device structure
2053 * @cdns: cdnsp instance
2054 *
2055 * This function initializes the gadget.
2056 */
2057int cdnsp_gadget_init(struct cdns *cdns)
2058{
2059 struct cdns_role_driver *rdrv;
2060
2061 rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
2062 if (!rdrv)
2063 return -ENOMEM;
2064
2065 rdrv->start = __cdnsp_gadget_init;
2066 rdrv->stop = cdnsp_gadget_exit;
2067 rdrv->suspend = cdnsp_gadget_suspend;
2068 rdrv->resume = cdnsp_gadget_resume;
2069 rdrv->state = CDNS_ROLE_STATE_INACTIVE;
2070 rdrv->name = "gadget";
2071 cdns->roles[USB_ROLE_DEVICE] = rdrv;
2072
2073 return 0;
2074}
Linux 6.x block layer and io_uring tree(s)