projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
usb: dwc2: Fix lines over 80 characters
[linux-2.6-block.git] / drivers / usb / dwc2 / gadget.c
CommitLineData
8b9bc460 1/**
dfbc6fa3
AT		 2 * Copyright (c) 2011 Samsung Electronics Co., Ltd.
3 * http://www.samsung.com
5b7d70c6
BD		 4 *
5 * Copyright 2008 Openmoko, Inc.
6 * Copyright 2008 Simtec Electronics
7 * Ben Dooks <ben@simtec.co.uk>
8 * http://armlinux.simtec.co.uk/
9 *
10 * S3C USB2.0 High-speed / OtG driver
11 *
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License version 2 as
14 * published by the Free Software Foundation.
8b9bc460 15 */
5b7d70c6
BD		 16
17#include <linux/kernel.h>
18#include <linux/module.h>
19#include <linux/spinlock.h>
20#include <linux/interrupt.h>
21#include <linux/platform_device.h>
22#include <linux/dma-mapping.h>
7ad8096e 23#include <linux/mutex.h>
5b7d70c6
BD		 24#include <linux/seq_file.h>
25#include <linux/delay.h>
26#include <linux/io.h>
5a0e3ad6 27#include <linux/slab.h>
c50f056c 28#include <linux/of_platform.h>
5b7d70c6
BD		 29
30#include <linux/usb/ch9.h>
31#include <linux/usb/gadget.h>
b2e587db 32#include <linux/usb/phy.h>
5b7d70c6 33
f7c0b143 34#include "core.h"
941fcce4 35#include "hw.h"
5b7d70c6
BD		 36
37/* conversion functions */
1f91b4cc 38static inline struct dwc2_hsotg_req *our_req(struct usb_request *req)
5b7d70c6 39{
1f91b4cc 40 return container_of(req, struct dwc2_hsotg_req, req);
5b7d70c6
BD		 41}
42
1f91b4cc 43static inline struct dwc2_hsotg_ep *our_ep(struct usb_ep *ep)
5b7d70c6 44{
1f91b4cc 45 return container_of(ep, struct dwc2_hsotg_ep, ep);
5b7d70c6
BD		 46}
47
941fcce4 48static inline struct dwc2_hsotg *to_hsotg(struct usb_gadget *gadget)
5b7d70c6 49{
941fcce4 50 return container_of(gadget, struct dwc2_hsotg, gadget);
5b7d70c6
BD		 51}
52
53static inline void __orr32(void __iomem *ptr, u32 val)
54{
95c8bc36 55 dwc2_writel(dwc2_readl(ptr) | val, ptr);
5b7d70c6
BD		 56}
57
58static inline void __bic32(void __iomem *ptr, u32 val)
59{
95c8bc36 60 dwc2_writel(dwc2_readl(ptr) & ~val, ptr);
5b7d70c6
BD		 61}
62
1f91b4cc 63static inline struct dwc2_hsotg_ep *index_to_ep(struct dwc2_hsotg *hsotg,
c6f5c050
MYK		 64 u32 ep_index, u32 dir_in)
65{
66 if (dir_in)
67 return hsotg->eps_in[ep_index];
68 else
69 return hsotg->eps_out[ep_index];
70}
71
997f4f81 72/* forward declaration of functions */
1f91b4cc 73static void dwc2_hsotg_dump(struct dwc2_hsotg *hsotg);
5b7d70c6
BD		 74
75/**
76 * using_dma - return the DMA status of the driver.
77 * @hsotg: The driver state.
78 *
79 * Return true if we're using DMA.
80 *
81 * Currently, we have the DMA support code worked into everywhere
82 * that needs it, but the AMBA DMA implementation in the hardware can
83 * only DMA from 32bit aligned addresses. This means that gadgets such
84 * as the CDC Ethernet cannot work as they often pass packets which are
85 * not 32bit aligned.
86 *
87 * Unfortunately the choice to use DMA or not is global to the controller
88 * and seems to be only settable when the controller is being put through
89 * a core reset. This means we either need to fix the gadgets to take
90 * account of DMA alignment, or add bounce buffers (yuerk).
91 *
edd74be8 92 * g_using_dma is set depending on dts flag.
5b7d70c6 93 */
941fcce4 94static inline bool using_dma(struct dwc2_hsotg *hsotg)
5b7d70c6 95{
05ee799f 96 return hsotg->params.g_dma;
5b7d70c6
BD		 97}
98
dec4b556
VA		 99/*
100 * using_desc_dma - return the descriptor DMA status of the driver.
101 * @hsotg: The driver state.
102 *
103 * Return true if we're using descriptor DMA.
104 */
105static inline bool using_desc_dma(struct dwc2_hsotg *hsotg)
106{
107 return hsotg->params.g_dma_desc;
108}
109
92d1635d
VM		 110/**
111 * dwc2_gadget_incr_frame_num - Increments the targeted frame number.
112 * @hs_ep: The endpoint
113 * @increment: The value to increment by
114 *
115 * This function will also check if the frame number overruns DSTS_SOFFN_LIMIT.
116 * If an overrun occurs it will wrap the value and set the frame_overrun flag.
117 */
118static inline void dwc2_gadget_incr_frame_num(struct dwc2_hsotg_ep *hs_ep)
119{
120 hs_ep->target_frame += hs_ep->interval;
121 if (hs_ep->target_frame > DSTS_SOFFN_LIMIT) {
122 hs_ep->frame_overrun = 1;
123 hs_ep->target_frame &= DSTS_SOFFN_LIMIT;
124 } else {
125 hs_ep->frame_overrun = 0;
126 }
127}
128
5b7d70c6 129/**
1f91b4cc 130 * dwc2_hsotg_en_gsint - enable one or more of the general interrupt
5b7d70c6
BD		 131 * @hsotg: The device state
132 * @ints: A bitmask of the interrupts to enable
133 */
1f91b4cc 134static void dwc2_hsotg_en_gsint(struct dwc2_hsotg *hsotg, u32 ints)
5b7d70c6 135{
95c8bc36 136 u32 gsintmsk = dwc2_readl(hsotg->regs + GINTMSK);
5b7d70c6
BD		 137 u32 new_gsintmsk;
138
139 new_gsintmsk = gsintmsk | ints;
140
141 if (new_gsintmsk != gsintmsk) {
142 dev_dbg(hsotg->dev, "gsintmsk now 0x%08x\n", new_gsintmsk);
95c8bc36 143 dwc2_writel(new_gsintmsk, hsotg->regs + GINTMSK);
5b7d70c6
BD		 144 }
145}
146
147/**
1f91b4cc 148 * dwc2_hsotg_disable_gsint - disable one or more of the general interrupt
5b7d70c6
BD		 149 * @hsotg: The device state
150 * @ints: A bitmask of the interrupts to enable
151 */
1f91b4cc 152static void dwc2_hsotg_disable_gsint(struct dwc2_hsotg *hsotg, u32 ints)
5b7d70c6 153{
95c8bc36 154 u32 gsintmsk = dwc2_readl(hsotg->regs + GINTMSK);
5b7d70c6
BD		 155 u32 new_gsintmsk;
156
157 new_gsintmsk = gsintmsk & ~ints;
158
159 if (new_gsintmsk != gsintmsk)
95c8bc36 160 dwc2_writel(new_gsintmsk, hsotg->regs + GINTMSK);
5b7d70c6
BD		 161}
162
163/**
1f91b4cc 164 * dwc2_hsotg_ctrl_epint - enable/disable an endpoint irq
5b7d70c6
BD		 165 * @hsotg: The device state
166 * @ep: The endpoint index
167 * @dir_in: True if direction is in.
168 * @en: The enable value, true to enable
169 *
170 * Set or clear the mask for an individual endpoint's interrupt
171 * request.
172 */
1f91b4cc 173static void dwc2_hsotg_ctrl_epint(struct dwc2_hsotg *hsotg,
9da51974 174 unsigned int ep, unsigned int dir_in,
5b7d70c6
BD		 175 unsigned int en)
176{
177 unsigned long flags;
178 u32 bit = 1 << ep;
179 u32 daint;
180
181 if (!dir_in)
182 bit <<= 16;
183
184 local_irq_save(flags);
95c8bc36 185 daint = dwc2_readl(hsotg->regs + DAINTMSK);
5b7d70c6
BD		 186 if (en)
187 daint |= bit;
188 else
189 daint &= ~bit;
95c8bc36 190 dwc2_writel(daint, hsotg->regs + DAINTMSK);
5b7d70c6
BD		 191 local_irq_restore(flags);
192}
193
194/**
1f91b4cc 195 * dwc2_hsotg_init_fifo - initialise non-periodic FIFOs
5b7d70c6
BD		 196 * @hsotg: The device instance.
197 */
1f91b4cc 198static void dwc2_hsotg_init_fifo(struct dwc2_hsotg *hsotg)
5b7d70c6 199{
2317eacd 200 unsigned int ep;
0f002d20 201 unsigned int addr;
1703a6d3 202 int timeout;
0f002d20 203 u32 val;
05ee799f 204 u32 *txfsz = hsotg->params.g_tx_fifo_size;
0f002d20 205
7fcbc95c
GH		 206 /* Reset fifo map if not correctly cleared during previous session */
207 WARN_ON(hsotg->fifo_map);
208 hsotg->fifo_map = 0;
209
0a176279 210 /* set RX/NPTX FIFO sizes */
05ee799f
JY		 211 dwc2_writel(hsotg->params.g_rx_fifo_size, hsotg->regs + GRXFSIZ);
212 dwc2_writel((hsotg->params.g_rx_fifo_size << FIFOSIZE_STARTADDR_SHIFT) |
213 (hsotg->params.g_np_tx_fifo_size << FIFOSIZE_DEPTH_SHIFT),
214 hsotg->regs + GNPTXFSIZ);
0f002d20 215
8b9bc460
LM		 216 /*
217 * arange all the rest of the TX FIFOs, as some versions of this
0f002d20
BD		 218 * block have overlapping default addresses. This also ensures
219 * that if the settings have been changed, then they are set to
8b9bc460
LM		 220 * known values.
221 */
0f002d20
BD		 222
223 /* start at the end of the GNPTXFSIZ, rounded up */
05ee799f 224 addr = hsotg->params.g_rx_fifo_size + hsotg->params.g_np_tx_fifo_size;
0f002d20 225
8b9bc460 226 /*
0a176279 227 * Configure fifos sizes from provided configuration and assign
b203d0a2
RB		 228 * them to endpoints dynamically according to maxpacket size value of
229 * given endpoint.
8b9bc460 230 */
2317eacd 231 for (ep = 1; ep < MAX_EPS_CHANNELS; ep++) {
05ee799f 232 if (!txfsz[ep])
3fa95385
JY		 233 continue;
234 val = addr;
05ee799f
JY		 235 val |= txfsz[ep] << FIFOSIZE_DEPTH_SHIFT;
236 WARN_ONCE(addr + txfsz[ep] > hsotg->fifo_mem,
3fa95385 237 "insufficient fifo memory");
05ee799f 238 addr += txfsz[ep];
0f002d20 239
2317eacd 240 dwc2_writel(val, hsotg->regs + DPTXFSIZN(ep));
05ee799f 241 val = dwc2_readl(hsotg->regs + DPTXFSIZN(ep));
0f002d20 242 }
1703a6d3 243
8b9bc460
LM		 244 /*
245 * according to p428 of the design guide, we need to ensure that
246 * all fifos are flushed before continuing
247 */
1703a6d3 248
95c8bc36 249 dwc2_writel(GRSTCTL_TXFNUM(0x10) | GRSTCTL_TXFFLSH |
47a1685f 250 GRSTCTL_RXFFLSH, hsotg->regs + GRSTCTL);
1703a6d3
BD		 251
252 /* wait until the fifos are both flushed */
253 timeout = 100;
254 while (1) {
95c8bc36 255 val = dwc2_readl(hsotg->regs + GRSTCTL);
1703a6d3 256
47a1685f 257 if ((val & (GRSTCTL_TXFFLSH | GRSTCTL_RXFFLSH)) == 0)
1703a6d3
BD		 258 break;
259
260 if (--timeout == 0) {
261 dev_err(hsotg->dev,
262 "%s: timeout flushing fifos (GRSTCTL=%08x)\n",
263 __func__, val);
48b20bcb 264 break;
1703a6d3
BD		 265 }
266
267 udelay(1);
268 }
269
270 dev_dbg(hsotg->dev, "FIFOs reset, timeout at %d\n", timeout);
5b7d70c6
BD		 271}
272
273/**
274 * @ep: USB endpoint to allocate request for.
275 * @flags: Allocation flags
276 *
277 * Allocate a new USB request structure appropriate for the specified endpoint
278 */
1f91b4cc 279static struct usb_request *dwc2_hsotg_ep_alloc_request(struct usb_ep *ep,
9da51974 280 gfp_t flags)
5b7d70c6 281{
1f91b4cc 282 struct dwc2_hsotg_req *req;
5b7d70c6 283
1f91b4cc 284 req = kzalloc(sizeof(struct dwc2_hsotg_req), flags);
5b7d70c6
BD		 285 if (!req)
286 return NULL;
287
288 INIT_LIST_HEAD(&req->queue);
289
5b7d70c6
BD		 290 return &req->req;
291}
292
293/**
294 * is_ep_periodic - return true if the endpoint is in periodic mode.
295 * @hs_ep: The endpoint to query.
296 *
297 * Returns true if the endpoint is in periodic mode, meaning it is being
298 * used for an Interrupt or ISO transfer.
299 */
1f91b4cc 300static inline int is_ep_periodic(struct dwc2_hsotg_ep *hs_ep)
5b7d70c6
BD		 301{
302 return hs_ep->periodic;
303}
304
305/**
1f91b4cc 306 * dwc2_hsotg_unmap_dma - unmap the DMA memory being used for the request
5b7d70c6
BD		 307 * @hsotg: The device state.
308 * @hs_ep: The endpoint for the request
309 * @hs_req: The request being processed.
310 *
1f91b4cc 311 * This is the reverse of dwc2_hsotg_map_dma(), called for the completion
5b7d70c6 312 * of a request to ensure the buffer is ready for access by the caller.
8b9bc460 313 */
1f91b4cc 314static void dwc2_hsotg_unmap_dma(struct dwc2_hsotg *hsotg,
9da51974 315 struct dwc2_hsotg_ep *hs_ep,
1f91b4cc 316 struct dwc2_hsotg_req *hs_req)
5b7d70c6
BD		 317{
318 struct usb_request *req = &hs_req->req;
9da51974 319
17d966a3 320 usb_gadget_unmap_request(&hsotg->gadget, req, hs_ep->dir_in);
5b7d70c6
BD		 321}
322
0f6b80c0
VA		 323/*
324 * dwc2_gadget_alloc_ctrl_desc_chains - allocate DMA descriptor chains
325 * for Control endpoint
326 * @hsotg: The device state.
327 *
328 * This function will allocate 4 descriptor chains for EP 0: 2 for
329 * Setup stage, per one for IN and OUT data/status transactions.
330 */
331static int dwc2_gadget_alloc_ctrl_desc_chains(struct dwc2_hsotg *hsotg)
332{
333 hsotg->setup_desc[0] =
334 dmam_alloc_coherent(hsotg->dev,
335 sizeof(struct dwc2_dma_desc),
336 &hsotg->setup_desc_dma[0],
337 GFP_KERNEL);
338 if (!hsotg->setup_desc[0])
339 goto fail;
340
341 hsotg->setup_desc[1] =
342 dmam_alloc_coherent(hsotg->dev,
343 sizeof(struct dwc2_dma_desc),
344 &hsotg->setup_desc_dma[1],
345 GFP_KERNEL);
346 if (!hsotg->setup_desc[1])
347 goto fail;
348
349 hsotg->ctrl_in_desc =
350 dmam_alloc_coherent(hsotg->dev,
351 sizeof(struct dwc2_dma_desc),
352 &hsotg->ctrl_in_desc_dma,
353 GFP_KERNEL);
354 if (!hsotg->ctrl_in_desc)
355 goto fail;
356
357 hsotg->ctrl_out_desc =
358 dmam_alloc_coherent(hsotg->dev,
359 sizeof(struct dwc2_dma_desc),
360 &hsotg->ctrl_out_desc_dma,
361 GFP_KERNEL);
362 if (!hsotg->ctrl_out_desc)
363 goto fail;
364
365 return 0;
366
367fail:
368 return -ENOMEM;
369}
370
5b7d70c6 371/**
1f91b4cc 372 * dwc2_hsotg_write_fifo - write packet Data to the TxFIFO
5b7d70c6
BD		 373 * @hsotg: The controller state.
374 * @hs_ep: The endpoint we're going to write for.
375 * @hs_req: The request to write data for.
376 *
377 * This is called when the TxFIFO has some space in it to hold a new
378 * transmission and we have something to give it. The actual setup of
379 * the data size is done elsewhere, so all we have to do is to actually
380 * write the data.
381 *
382 * The return value is zero if there is more space (or nothing was done)
383 * otherwise -ENOSPC is returned if the FIFO space was used up.
384 *
385 * This routine is only needed for PIO
8b9bc460 386 */
1f91b4cc 387static int dwc2_hsotg_write_fifo(struct dwc2_hsotg *hsotg,
9da51974 388 struct dwc2_hsotg_ep *hs_ep,
1f91b4cc 389 struct dwc2_hsotg_req *hs_req)
5b7d70c6
BD		 390{
391 bool periodic = is_ep_periodic(hs_ep);
95c8bc36 392 u32 gnptxsts = dwc2_readl(hsotg->regs + GNPTXSTS);
5b7d70c6
BD		 393 int buf_pos = hs_req->req.actual;
394 int to_write = hs_ep->size_loaded;
395 void *data;
396 int can_write;
397 int pkt_round;
4fca54aa 398 int max_transfer;
5b7d70c6
BD		 399
400 to_write -= (buf_pos - hs_ep->last_load);
401
402 /* if there's nothing to write, get out early */
403 if (to_write == 0)
404 return 0;
405
10aebc77 406 if (periodic && !hsotg->dedicated_fifos) {
95c8bc36 407 u32 epsize = dwc2_readl(hsotg->regs + DIEPTSIZ(hs_ep->index));
5b7d70c6
BD		 408 int size_left;
409 int size_done;
410
8b9bc460
LM		 411 /*
412 * work out how much data was loaded so we can calculate
413 * how much data is left in the fifo.
414 */
5b7d70c6 415
47a1685f 416 size_left = DXEPTSIZ_XFERSIZE_GET(epsize);
5b7d70c6 417
8b9bc460
LM		 418 /*
419 * if shared fifo, we cannot write anything until the
e7a9ff54
BD		 420 * previous data has been completely sent.
421 */
422 if (hs_ep->fifo_load != 0) {
1f91b4cc 423 dwc2_hsotg_en_gsint(hsotg, GINTSTS_PTXFEMP);
e7a9ff54
BD		 424 return -ENOSPC;
425 }
426
5b7d70c6
BD		 427 dev_dbg(hsotg->dev, "%s: left=%d, load=%d, fifo=%d, size %d\n",
428 __func__, size_left,
429 hs_ep->size_loaded, hs_ep->fifo_load, hs_ep->fifo_size);
430
431 /* how much of the data has moved */
432 size_done = hs_ep->size_loaded - size_left;
433
434 /* how much data is left in the fifo */
435 can_write = hs_ep->fifo_load - size_done;
436 dev_dbg(hsotg->dev, "%s: => can_write1=%d\n",
437 __func__, can_write);
438
439 can_write = hs_ep->fifo_size - can_write;
440 dev_dbg(hsotg->dev, "%s: => can_write2=%d\n",
441 __func__, can_write);
442
443 if (can_write <= 0) {
1f91b4cc 444 dwc2_hsotg_en_gsint(hsotg, GINTSTS_PTXFEMP);
5b7d70c6
BD		 445 return -ENOSPC;
446 }
10aebc77 447 } else if (hsotg->dedicated_fifos && hs_ep->index != 0) {
ad674a15
RB		 448 can_write = dwc2_readl(hsotg->regs +
449 DTXFSTS(hs_ep->fifo_index));
10aebc77
BD		 450
451 can_write &= 0xffff;
452 can_write *= 4;
5b7d70c6 453 } else {
47a1685f 454 if (GNPTXSTS_NP_TXQ_SPC_AVAIL_GET(gnptxsts) == 0) {
5b7d70c6
BD		 455 dev_dbg(hsotg->dev,
456 "%s: no queue slots available (0x%08x)\n",
457 __func__, gnptxsts);
458
1f91b4cc 459 dwc2_hsotg_en_gsint(hsotg, GINTSTS_NPTXFEMP);
5b7d70c6
BD		 460 return -ENOSPC;
461 }
462
47a1685f 463 can_write = GNPTXSTS_NP_TXF_SPC_AVAIL_GET(gnptxsts);
679f9b7c 464 can_write *= 4; /* fifo size is in 32bit quantities. */
5b7d70c6
BD		 465 }
466
4fca54aa
RB		 467 max_transfer = hs_ep->ep.maxpacket * hs_ep->mc;
468
469 dev_dbg(hsotg->dev, "%s: GNPTXSTS=%08x, can=%d, to=%d, max_transfer %d\n",
9da51974 470 __func__, gnptxsts, can_write, to_write, max_transfer);
5b7d70c6 471
8b9bc460
LM		 472 /*
473 * limit to 512 bytes of data, it seems at least on the non-periodic
5b7d70c6
BD		 474 * FIFO, requests of >512 cause the endpoint to get stuck with a
475 * fragment of the end of the transfer in it.
476 */
811f3303 477 if (can_write > 512 && !periodic)
5b7d70c6
BD		 478 can_write = 512;
479
8b9bc460
LM		 480 /*
481 * limit the write to one max-packet size worth of data, but allow
03e10e5a 482 * the transfer to return that it did not run out of fifo space
8b9bc460
LM		 483 * doing it.
484 */
4fca54aa
RB		 485 if (to_write > max_transfer) {
486 to_write = max_transfer;
03e10e5a 487
5cb2ff0c
RB		 488 /* it's needed only when we do not use dedicated fifos */
489 if (!hsotg->dedicated_fifos)
1f91b4cc 490 dwc2_hsotg_en_gsint(hsotg,
9da51974 491 periodic ? GINTSTS_PTXFEMP :
47a1685f 492 GINTSTS_NPTXFEMP);
03e10e5a
BD		 493 }
494
5b7d70c6
BD		 495 /* see if we can write data */
496
497 if (to_write > can_write) {
498 to_write = can_write;
4fca54aa 499 pkt_round = to_write % max_transfer;
5b7d70c6 500
8b9bc460
LM		 501 /*
502 * Round the write down to an
5b7d70c6
BD		 503 * exact number of packets.
504 *
505 * Note, we do not currently check to see if we can ever
506 * write a full packet or not to the FIFO.
507 */
508
509 if (pkt_round)
510 to_write -= pkt_round;
511
8b9bc460
LM		 512 /*
513 * enable correct FIFO interrupt to alert us when there
514 * is more room left.
515 */
5b7d70c6 516
5cb2ff0c
RB		 517 /* it's needed only when we do not use dedicated fifos */
518 if (!hsotg->dedicated_fifos)
1f91b4cc 519 dwc2_hsotg_en_gsint(hsotg,
9da51974 520 periodic ? GINTSTS_PTXFEMP :
47a1685f 521 GINTSTS_NPTXFEMP);
5b7d70c6
BD		 522 }
523
524 dev_dbg(hsotg->dev, "write %d/%d, can_write %d, done %d\n",
9da51974 525 to_write, hs_req->req.length, can_write, buf_pos);
5b7d70c6
BD		 526
527 if (to_write <= 0)
528 return -ENOSPC;
529
530 hs_req->req.actual = buf_pos + to_write;
531 hs_ep->total_data += to_write;
532
533 if (periodic)
534 hs_ep->fifo_load += to_write;
535
536 to_write = DIV_ROUND_UP(to_write, 4);
537 data = hs_req->req.buf + buf_pos;
538
1a7ed5be 539 iowrite32_rep(hsotg->regs + EPFIFO(hs_ep->index), data, to_write);
5b7d70c6
BD		 540
541 return (to_write >= can_write) ? -ENOSPC : 0;
542}
543
544/**
545 * get_ep_limit - get the maximum data legnth for this endpoint
546 * @hs_ep: The endpoint
547 *
548 * Return the maximum data that can be queued in one go on a given endpoint
549 * so that transfers that are too long can be split.
550 */
9da51974 551static unsigned int get_ep_limit(struct dwc2_hsotg_ep *hs_ep)
5b7d70c6
BD		 552{
553 int index = hs_ep->index;
9da51974
JY		 554 unsigned int maxsize;
555 unsigned int maxpkt;
5b7d70c6
BD		 556
557 if (index != 0) {
47a1685f
DN		 558 maxsize = DXEPTSIZ_XFERSIZE_LIMIT + 1;
559 maxpkt = DXEPTSIZ_PKTCNT_LIMIT + 1;
5b7d70c6 560 } else {
9da51974 561 maxsize = 64 + 64;
66e5c643 562 if (hs_ep->dir_in)
47a1685f 563 maxpkt = DIEPTSIZ0_PKTCNT_LIMIT + 1;
66e5c643 564 else
5b7d70c6 565 maxpkt = 2;
5b7d70c6
BD		 566 }
567
568 /* we made the constant loading easier above by using +1 */
569 maxpkt--;
570 maxsize--;
571
8b9bc460
LM		 572 /*
573 * constrain by packet count if maxpkts*pktsize is greater
574 * than the length register size.
575 */
5b7d70c6
BD		 576
577 if ((maxpkt * hs_ep->ep.maxpacket) < maxsize)
578 maxsize = maxpkt * hs_ep->ep.maxpacket;
579
580 return maxsize;
581}
582
381fc8f8 583/**
38beaec6
JY		 584 * dwc2_hsotg_read_frameno - read current frame number
585 * @hsotg: The device instance
586 *
587 * Return the current frame number
588 */
381fc8f8
VM		 589static u32 dwc2_hsotg_read_frameno(struct dwc2_hsotg *hsotg)
590{
591 u32 dsts;
592
593 dsts = dwc2_readl(hsotg->regs + DSTS);
594 dsts &= DSTS_SOFFN_MASK;
595 dsts >>= DSTS_SOFFN_SHIFT;
596
597 return dsts;
598}
599
cf77b5fb
VA		 600/**
601 * dwc2_gadget_get_chain_limit - get the maximum data payload value of the
602 * DMA descriptor chain prepared for specific endpoint
603 * @hs_ep: The endpoint
604 *
605 * Return the maximum data that can be queued in one go on a given endpoint
606 * depending on its descriptor chain capacity so that transfers that
607 * are too long can be split.
608 */
609static unsigned int dwc2_gadget_get_chain_limit(struct dwc2_hsotg_ep *hs_ep)
610{
611 int is_isoc = hs_ep->isochronous;
612 unsigned int maxsize;
613
614 if (is_isoc)
615 maxsize = hs_ep->dir_in ? DEV_DMA_ISOC_TX_NBYTES_LIMIT :
616 DEV_DMA_ISOC_RX_NBYTES_LIMIT;
617 else
618 maxsize = DEV_DMA_NBYTES_LIMIT;
619
620 /* Above size of one descriptor was chosen, multiple it */
621 maxsize *= MAX_DMA_DESC_NUM_GENERIC;
622
623 return maxsize;
624}
625
e02f9aa6
VA		 626/*
627 * dwc2_gadget_get_desc_params - get DMA descriptor parameters.
628 * @hs_ep: The endpoint
629 * @mask: RX/TX bytes mask to be defined
630 *
631 * Returns maximum data payload for one descriptor after analyzing endpoint
632 * characteristics.
633 * DMA descriptor transfer bytes limit depends on EP type:
634 * Control out - MPS,
635 * Isochronous - descriptor rx/tx bytes bitfield limit,
636 * Control In/Bulk/Interrupt - multiple of mps. This will allow to not
637 * have concatenations from various descriptors within one packet.
638 *
639 * Selects corresponding mask for RX/TX bytes as well.
640 */
641static u32 dwc2_gadget_get_desc_params(struct dwc2_hsotg_ep *hs_ep, u32 *mask)
642{
643 u32 mps = hs_ep->ep.maxpacket;
644 int dir_in = hs_ep->dir_in;
645 u32 desc_size = 0;
646
647 if (!hs_ep->index && !dir_in) {
648 desc_size = mps;
649 *mask = DEV_DMA_NBYTES_MASK;
650 } else if (hs_ep->isochronous) {
651 if (dir_in) {
652 desc_size = DEV_DMA_ISOC_TX_NBYTES_LIMIT;
653 *mask = DEV_DMA_ISOC_TX_NBYTES_MASK;
654 } else {
655 desc_size = DEV_DMA_ISOC_RX_NBYTES_LIMIT;
656 *mask = DEV_DMA_ISOC_RX_NBYTES_MASK;
657 }
658 } else {
659 desc_size = DEV_DMA_NBYTES_LIMIT;
660 *mask = DEV_DMA_NBYTES_MASK;
661
662 /* Round down desc_size to be mps multiple */
663 desc_size -= desc_size % mps;
664 }
665
666 return desc_size;
667}
668
669/*
670 * dwc2_gadget_config_nonisoc_xfer_ddma - prepare non ISOC DMA desc chain.
671 * @hs_ep: The endpoint
672 * @dma_buff: DMA address to use
673 * @len: Length of the transfer
674 *
675 * This function will iterate over descriptor chain and fill its entries
676 * with corresponding information based on transfer data.
677 */
678static void dwc2_gadget_config_nonisoc_xfer_ddma(struct dwc2_hsotg_ep *hs_ep,
679 dma_addr_t dma_buff,
680 unsigned int len)
681{
682 struct dwc2_hsotg *hsotg = hs_ep->parent;
683 int dir_in = hs_ep->dir_in;
684 struct dwc2_dma_desc *desc = hs_ep->desc_list;
685 u32 mps = hs_ep->ep.maxpacket;
686 u32 maxsize = 0;
687 u32 offset = 0;
688 u32 mask = 0;
689 int i;
690
691 maxsize = dwc2_gadget_get_desc_params(hs_ep, &mask);
692
693 hs_ep->desc_count = (len / maxsize) +
694 ((len % maxsize) ? 1 : 0);
695 if (len == 0)
696 hs_ep->desc_count = 1;
697
698 for (i = 0; i < hs_ep->desc_count; ++i) {
699 desc->status = 0;
700 desc->status |= (DEV_DMA_BUFF_STS_HBUSY
701 << DEV_DMA_BUFF_STS_SHIFT);
702
703 if (len > maxsize) {
704 if (!hs_ep->index && !dir_in)
705 desc->status |= (DEV_DMA_L | DEV_DMA_IOC);
706
707 desc->status |= (maxsize <<
708 DEV_DMA_NBYTES_SHIFT & mask);
709 desc->buf = dma_buff + offset;
710
711 len -= maxsize;
712 offset += maxsize;
713 } else {
714 desc->status |= (DEV_DMA_L | DEV_DMA_IOC);
715
716 if (dir_in)
717 desc->status |= (len % mps) ? DEV_DMA_SHORT :
718 ((hs_ep->send_zlp) ? DEV_DMA_SHORT : 0);
719 if (len > maxsize)
720 dev_err(hsotg->dev, "wrong len %d\n", len);
721
722 desc->status |=
723 len << DEV_DMA_NBYTES_SHIFT & mask;
724 desc->buf = dma_buff + offset;
725 }
726
727 desc->status &= ~DEV_DMA_BUFF_STS_MASK;
728 desc->status |= (DEV_DMA_BUFF_STS_HREADY
729 << DEV_DMA_BUFF_STS_SHIFT);
730 desc++;
731 }
732}
733
540ccba0
VA		 734/*
735 * dwc2_gadget_fill_isoc_desc - fills next isochronous descriptor in chain.
736 * @hs_ep: The isochronous endpoint.
737 * @dma_buff: usb requests dma buffer.
738 * @len: usb request transfer length.
739 *
740 * Finds out index of first free entry either in the bottom or up half of
741 * descriptor chain depend on which is under SW control and not processed
742 * by HW. Then fills that descriptor with the data of the arrived usb request,
743 * frame info, sets Last and IOC bits increments next_desc. If filled
744 * descriptor is not the first one, removes L bit from the previous descriptor
745 * status.
746 */
747static int dwc2_gadget_fill_isoc_desc(struct dwc2_hsotg_ep *hs_ep,
748 dma_addr_t dma_buff, unsigned int len)
749{
750 struct dwc2_dma_desc *desc;
751 struct dwc2_hsotg *hsotg = hs_ep->parent;
752 u32 index;
753 u32 maxsize = 0;
754 u32 mask = 0;
755
756 maxsize = dwc2_gadget_get_desc_params(hs_ep, &mask);
757 if (len > maxsize) {
758 dev_err(hsotg->dev, "wrong len %d\n", len);
759 return -EINVAL;
760 }
761
762 /*
763 * If SW has already filled half of chain, then return and wait for
764 * the other chain to be processed by HW.
765 */
766 if (hs_ep->next_desc == MAX_DMA_DESC_NUM_GENERIC / 2)
767 return -EBUSY;
768
769 /* Increment frame number by interval for IN */
770 if (hs_ep->dir_in)
771 dwc2_gadget_incr_frame_num(hs_ep);
772
773 index = (MAX_DMA_DESC_NUM_GENERIC / 2) * hs_ep->isoc_chain_num +
774 hs_ep->next_desc;
775
776 /* Sanity check of calculated index */
777 if ((hs_ep->isoc_chain_num && index > MAX_DMA_DESC_NUM_GENERIC) ||
778 (!hs_ep->isoc_chain_num && index > MAX_DMA_DESC_NUM_GENERIC / 2)) {
779 dev_err(hsotg->dev, "wrong index %d for iso chain\n", index);
780 return -EINVAL;
781 }
782
783 desc = &hs_ep->desc_list[index];
784
785 /* Clear L bit of previous desc if more than one entries in the chain */
786 if (hs_ep->next_desc)
787 hs_ep->desc_list[index - 1].status &= ~DEV_DMA_L;
788
789 dev_dbg(hsotg->dev, "%s: Filling ep %d, dir %s isoc desc # %d\n",
790 __func__, hs_ep->index, hs_ep->dir_in ? "in" : "out", index);
791
792 desc->status = 0;
793 desc->status |= (DEV_DMA_BUFF_STS_HBUSY << DEV_DMA_BUFF_STS_SHIFT);
794
795 desc->buf = dma_buff;
796 desc->status |= (DEV_DMA_L | DEV_DMA_IOC |
797 ((len << DEV_DMA_NBYTES_SHIFT) & mask));
798
799 if (hs_ep->dir_in) {
800 desc->status |= ((hs_ep->mc << DEV_DMA_ISOC_PID_SHIFT) &
801 DEV_DMA_ISOC_PID_MASK) |
802 ((len % hs_ep->ep.maxpacket) ?
803 DEV_DMA_SHORT : 0) |
804 ((hs_ep->target_frame <<
805 DEV_DMA_ISOC_FRNUM_SHIFT) &
806 DEV_DMA_ISOC_FRNUM_MASK);
807 }
808
809 desc->status &= ~DEV_DMA_BUFF_STS_MASK;
810 desc->status |= (DEV_DMA_BUFF_STS_HREADY << DEV_DMA_BUFF_STS_SHIFT);
811
812 /* Update index of last configured entry in the chain */
813 hs_ep->next_desc++;
814
815 return 0;
816}
817
818/*
819 * dwc2_gadget_start_isoc_ddma - start isochronous transfer in DDMA
820 * @hs_ep: The isochronous endpoint.
821 *
822 * Prepare first descriptor chain for isochronous endpoints. Afterwards
823 * write DMA address to HW and enable the endpoint.
824 *
825 * Switch between descriptor chains via isoc_chain_num to give SW opportunity
826 * to prepare second descriptor chain while first one is being processed by HW.
827 */
828static void dwc2_gadget_start_isoc_ddma(struct dwc2_hsotg_ep *hs_ep)
829{
830 struct dwc2_hsotg *hsotg = hs_ep->parent;
831 struct dwc2_hsotg_req *hs_req, *treq;
832 int index = hs_ep->index;
833 int ret;
834 u32 dma_reg;
835 u32 depctl;
836 u32 ctrl;
837
838 if (list_empty(&hs_ep->queue)) {
839 dev_dbg(hsotg->dev, "%s: No requests in queue\n", __func__);
840 return;
841 }
842
843 list_for_each_entry_safe(hs_req, treq, &hs_ep->queue, queue) {
844 ret = dwc2_gadget_fill_isoc_desc(hs_ep, hs_req->req.dma,
845 hs_req->req.length);
846 if (ret) {
847 dev_dbg(hsotg->dev, "%s: desc chain full\n", __func__);
848 break;
849 }
850 }
851
852 depctl = hs_ep->dir_in ? DIEPCTL(index) : DOEPCTL(index);
853 dma_reg = hs_ep->dir_in ? DIEPDMA(index) : DOEPDMA(index);
854
855 /* write descriptor chain address to control register */
856 dwc2_writel(hs_ep->desc_list_dma, hsotg->regs + dma_reg);
857
858 ctrl = dwc2_readl(hsotg->regs + depctl);
859 ctrl |= DXEPCTL_EPENA | DXEPCTL_CNAK;
860 dwc2_writel(ctrl, hsotg->regs + depctl);
861
862 /* Switch ISOC descriptor chain number being processed by SW*/
863 hs_ep->isoc_chain_num = (hs_ep->isoc_chain_num ^ 1) & 0x1;
864 hs_ep->next_desc = 0;
865}
866
5b7d70c6 867/**
1f91b4cc 868 * dwc2_hsotg_start_req - start a USB request from an endpoint's queue
5b7d70c6
BD		 869 * @hsotg: The controller state.
870 * @hs_ep: The endpoint to process a request for
871 * @hs_req: The request to start.
872 * @continuing: True if we are doing more for the current request.
873 *
874 * Start the given request running by setting the endpoint registers
875 * appropriately, and writing any data to the FIFOs.
876 */
1f91b4cc 877static void dwc2_hsotg_start_req(struct dwc2_hsotg *hsotg,
9da51974 878 struct dwc2_hsotg_ep *hs_ep,
1f91b4cc 879 struct dwc2_hsotg_req *hs_req,
5b7d70c6
BD		 880 bool continuing)
881{
882 struct usb_request *ureq = &hs_req->req;
883 int index = hs_ep->index;
884 int dir_in = hs_ep->dir_in;
885 u32 epctrl_reg;
886 u32 epsize_reg;
887 u32 epsize;
888 u32 ctrl;
9da51974
JY		 889 unsigned int length;
890 unsigned int packets;
891 unsigned int maxreq;
aa3e8bc8 892 unsigned int dma_reg;
5b7d70c6
BD		 893
894 if (index != 0) {
895 if (hs_ep->req && !continuing) {
896 dev_err(hsotg->dev, "%s: active request\n", __func__);
897 WARN_ON(1);
898 return;
899 } else if (hs_ep->req != hs_req && continuing) {
900 dev_err(hsotg->dev,
901 "%s: continue different req\n", __func__);
902 WARN_ON(1);
903 return;
904 }
905 }
906
aa3e8bc8 907 dma_reg = dir_in ? DIEPDMA(index) : DOEPDMA(index);
94cb8fd6
LM		 908 epctrl_reg = dir_in ? DIEPCTL(index) : DOEPCTL(index);
909 epsize_reg = dir_in ? DIEPTSIZ(index) : DOEPTSIZ(index);
5b7d70c6
BD		 910
911 dev_dbg(hsotg->dev, "%s: DxEPCTL=0x%08x, ep %d, dir %s\n",
95c8bc36 912 __func__, dwc2_readl(hsotg->regs + epctrl_reg), index,
5b7d70c6
BD		 913 hs_ep->dir_in ? "in" : "out");
914
9c39ddc6 915 /* If endpoint is stalled, we will restart request later */
95c8bc36 916 ctrl = dwc2_readl(hsotg->regs + epctrl_reg);
9c39ddc6 917
b2d4c54e 918 if (index && ctrl & DXEPCTL_STALL) {
9c39ddc6
AT		 919 dev_warn(hsotg->dev, "%s: ep%d is stalled\n", __func__, index);
920 return;
921 }
922
5b7d70c6 923 length = ureq->length - ureq->actual;
71225bee
LM		 924 dev_dbg(hsotg->dev, "ureq->length:%d ureq->actual:%d\n",
925 ureq->length, ureq->actual);
5b7d70c6 926
cf77b5fb
VA		 927 if (!using_desc_dma(hsotg))
928 maxreq = get_ep_limit(hs_ep);
929 else
930 maxreq = dwc2_gadget_get_chain_limit(hs_ep);
931
5b7d70c6
BD		 932 if (length > maxreq) {
933 int round = maxreq % hs_ep->ep.maxpacket;
934
935 dev_dbg(hsotg->dev, "%s: length %d, max-req %d, r %d\n",
936 __func__, length, maxreq, round);
937
938 /* round down to multiple of packets */
939 if (round)
940 maxreq -= round;
941
942 length = maxreq;
943 }
944
945 if (length)
946 packets = DIV_ROUND_UP(length, hs_ep->ep.maxpacket);
947 else
948 packets = 1; /* send one packet if length is zero. */
949
4fca54aa
RB		 950 if (hs_ep->isochronous && length > (hs_ep->mc * hs_ep->ep.maxpacket)) {
951 dev_err(hsotg->dev, "req length > maxpacket*mc\n");
952 return;
953 }
954
5b7d70c6 955 if (dir_in && index != 0)
4fca54aa 956 if (hs_ep->isochronous)
47a1685f 957 epsize = DXEPTSIZ_MC(packets);
4fca54aa 958 else
47a1685f 959 epsize = DXEPTSIZ_MC(1);
5b7d70c6
BD		 960 else
961 epsize = 0;
962
f71b5e25
MYK		 963 /*
964 * zero length packet should be programmed on its own and should not
965 * be counted in DIEPTSIZ.PktCnt with other packets.
966 */
967 if (dir_in && ureq->zero && !continuing) {
968 /* Test if zlp is actually required. */
969 if ((ureq->length >= hs_ep->ep.maxpacket) &&
9da51974 970 !(ureq->length % hs_ep->ep.maxpacket))
8a20fa45 971 hs_ep->send_zlp = 1;
5b7d70c6
BD		 972 }
973
47a1685f
DN		 974 epsize |= DXEPTSIZ_PKTCNT(packets);
975 epsize |= DXEPTSIZ_XFERSIZE(length);
5b7d70c6
BD		 976
977 dev_dbg(hsotg->dev, "%s: %d@%d/%d, 0x%08x => 0x%08x\n",
978 __func__, packets, length, ureq->length, epsize, epsize_reg);
979
980 /* store the request as the current one we're doing */
981 hs_ep->req = hs_req;
982
aa3e8bc8
VA		 983 if (using_desc_dma(hsotg)) {
984 u32 offset = 0;
985 u32 mps = hs_ep->ep.maxpacket;
986
987 /* Adjust length: EP0 - MPS, other OUT EPs - multiple of MPS */
988 if (!dir_in) {
989 if (!index)
990 length = mps;
991 else if (length % mps)
992 length += (mps - (length % mps));
993 }
5b7d70c6 994
8b9bc460 995 /*
aa3e8bc8
VA		 996 * If more data to send, adjust DMA for EP0 out data stage.
997 * ureq->dma stays unchanged, hence increment it by already
998 * passed passed data count before starting new transaction.
8b9bc460 999 */
aa3e8bc8
VA		 1000 if (!index && hsotg->ep0_state == DWC2_EP0_DATA_OUT &&
1001 continuing)
1002 offset = ureq->actual;
1003
1004 /* Fill DDMA chain entries */
1005 dwc2_gadget_config_nonisoc_xfer_ddma(hs_ep, ureq->dma + offset,
1006 length);
1007
1008 /* write descriptor chain address to control register */
1009 dwc2_writel(hs_ep->desc_list_dma, hsotg->regs + dma_reg);
5b7d70c6 1010
aa3e8bc8
VA		 1011 dev_dbg(hsotg->dev, "%s: %08x pad => 0x%08x\n",
1012 __func__, (u32)hs_ep->desc_list_dma, dma_reg);
1013 } else {
1014 /* write size / packets */
1015 dwc2_writel(epsize, hsotg->regs + epsize_reg);
1016
729e6574 1017 if (using_dma(hsotg) && !continuing && (length != 0)) {
aa3e8bc8
VA		 1018 /*
1019 * write DMA address to control register, buffer
1020 * already synced by dwc2_hsotg_ep_queue().
1021 */
5b7d70c6 1022
aa3e8bc8
VA		 1023 dwc2_writel(ureq->dma, hsotg->regs + dma_reg);
1024
1025 dev_dbg(hsotg->dev, "%s: %pad => 0x%08x\n",
1026 __func__, &ureq->dma, dma_reg);
1027 }
5b7d70c6
BD		 1028 }
1029
837e9f00
VM		 1030 if (hs_ep->isochronous && hs_ep->interval == 1) {
1031 hs_ep->target_frame = dwc2_hsotg_read_frameno(hsotg);
1032 dwc2_gadget_incr_frame_num(hs_ep);
1033
1034 if (hs_ep->target_frame & 0x1)
1035 ctrl |= DXEPCTL_SETODDFR;
1036 else
1037 ctrl |= DXEPCTL_SETEVENFR;
1038 }
1039
47a1685f 1040 ctrl |= DXEPCTL_EPENA; /* ensure ep enabled */
71225bee 1041
fe0b94ab 1042 dev_dbg(hsotg->dev, "ep0 state:%d\n", hsotg->ep0_state);
71225bee
LM		 1043
1044 /* For Setup request do not clear NAK */
fe0b94ab 1045 if (!(index == 0 && hsotg->ep0_state == DWC2_EP0_SETUP))
47a1685f 1046 ctrl |= DXEPCTL_CNAK; /* clear NAK set by core */
71225bee 1047
5b7d70c6 1048 dev_dbg(hsotg->dev, "%s: DxEPCTL=0x%08x\n", __func__, ctrl);
95c8bc36 1049 dwc2_writel(ctrl, hsotg->regs + epctrl_reg);
5b7d70c6 1050
8b9bc460
LM		 1051 /*
1052 * set these, it seems that DMA support increments past the end
5b7d70c6 1053 * of the packet buffer so we need to calculate the length from
8b9bc460
LM		 1054 * this information.
1055 */
5b7d70c6
BD		 1056 hs_ep->size_loaded = length;
1057 hs_ep->last_load = ureq->actual;
1058
1059 if (dir_in && !using_dma(hsotg)) {
1060 /* set these anyway, we may need them for non-periodic in */
1061 hs_ep->fifo_load = 0;
1062
1f91b4cc 1063 dwc2_hsotg_write_fifo(hsotg, hs_ep, hs_req);
5b7d70c6
BD		 1064 }
1065
8b9bc460
LM		 1066 /*
1067 * Note, trying to clear the NAK here causes problems with transmit
1068 * on the S3C6400 ending up with the TXFIFO becoming full.
1069 */
5b7d70c6
BD		 1070
1071 /* check ep is enabled */
95c8bc36 1072 if (!(dwc2_readl(hsotg->regs + epctrl_reg) & DXEPCTL_EPENA))
1a0ed863 1073 dev_dbg(hsotg->dev,
9da51974 1074 "ep%d: failed to become enabled (DXEPCTL=0x%08x)?\n",
95c8bc36 1075 index, dwc2_readl(hsotg->regs + epctrl_reg));
5b7d70c6 1076
47a1685f 1077 dev_dbg(hsotg->dev, "%s: DXEPCTL=0x%08x\n",
95c8bc36 1078 __func__, dwc2_readl(hsotg->regs + epctrl_reg));
afcf4169
RB		 1079
1080 /* enable ep interrupts */
1f91b4cc 1081 dwc2_hsotg_ctrl_epint(hsotg, hs_ep->index, hs_ep->dir_in, 1);
5b7d70c6
BD		 1082}
1083
1084/**
1f91b4cc 1085 * dwc2_hsotg_map_dma - map the DMA memory being used for the request
5b7d70c6
BD		 1086 * @hsotg: The device state.
1087 * @hs_ep: The endpoint the request is on.
1088 * @req: The request being processed.
1089 *
1090 * We've been asked to queue a request, so ensure that the memory buffer
1091 * is correctly setup for DMA. If we've been passed an extant DMA address
1092 * then ensure the buffer has been synced to memory. If our buffer has no
1093 * DMA memory, then we map the memory and mark our request to allow us to
1094 * cleanup on completion.
8b9bc460 1095 */
1f91b4cc 1096static int dwc2_hsotg_map_dma(struct dwc2_hsotg *hsotg,
9da51974 1097 struct dwc2_hsotg_ep *hs_ep,
5b7d70c6
BD		 1098 struct usb_request *req)
1099{
e58ebcd1 1100 int ret;
5b7d70c6 1101
e58ebcd1
FB		 1102 ret = usb_gadget_map_request(&hsotg->gadget, req, hs_ep->dir_in);
1103 if (ret)
1104 goto dma_error;
5b7d70c6
BD		 1105
1106 return 0;
1107
1108dma_error:
1109 dev_err(hsotg->dev, "%s: failed to map buffer %p, %d bytes\n",
1110 __func__, req->buf, req->length);
1111
1112 return -EIO;
1113}
1114
1f91b4cc 1115static int dwc2_hsotg_handle_unaligned_buf_start(struct dwc2_hsotg *hsotg,
b98866c2
JY		 1116 struct dwc2_hsotg_ep *hs_ep,
1117 struct dwc2_hsotg_req *hs_req)
7d24c1b5
MYK		 1118{
1119 void *req_buf = hs_req->req.buf;
1120
1121 /* If dma is not being used or buffer is aligned */
1122 if (!using_dma(hsotg) || !((long)req_buf & 3))
1123 return 0;
1124
1125 WARN_ON(hs_req->saved_req_buf);
1126
1127 dev_dbg(hsotg->dev, "%s: %s: buf=%p length=%d\n", __func__,
9da51974 1128 hs_ep->ep.name, req_buf, hs_req->req.length);
7d24c1b5
MYK		 1129
1130 hs_req->req.buf = kmalloc(hs_req->req.length, GFP_ATOMIC);
1131 if (!hs_req->req.buf) {
1132 hs_req->req.buf = req_buf;
1133 dev_err(hsotg->dev,
1134 "%s: unable to allocate memory for bounce buffer\n",
1135 __func__);
1136 return -ENOMEM;
1137 }
1138
1139 /* Save actual buffer */
1140 hs_req->saved_req_buf = req_buf;
1141
1142 if (hs_ep->dir_in)
1143 memcpy(hs_req->req.buf, req_buf, hs_req->req.length);
1144 return 0;
1145}
1146
b98866c2
JY		 1147static void
1148dwc2_hsotg_handle_unaligned_buf_complete(struct dwc2_hsotg *hsotg,
1149 struct dwc2_hsotg_ep *hs_ep,
1150 struct dwc2_hsotg_req *hs_req)
7d24c1b5
MYK		 1151{
1152 /* If dma is not being used or buffer was aligned */
1153 if (!using_dma(hsotg) || !hs_req->saved_req_buf)
1154 return;
1155
1156 dev_dbg(hsotg->dev, "%s: %s: status=%d actual-length=%d\n", __func__,
1157 hs_ep->ep.name, hs_req->req.status, hs_req->req.actual);
1158
1159 /* Copy data from bounce buffer on successful out transfer */
1160 if (!hs_ep->dir_in && !hs_req->req.status)
1161 memcpy(hs_req->saved_req_buf, hs_req->req.buf,
9da51974 1162 hs_req->req.actual);
7d24c1b5
MYK		 1163
1164 /* Free bounce buffer */
1165 kfree(hs_req->req.buf);
1166
1167 hs_req->req.buf = hs_req->saved_req_buf;
1168 hs_req->saved_req_buf = NULL;
1169}
1170
381fc8f8
VM		 1171/**
1172 * dwc2_gadget_target_frame_elapsed - Checks target frame
1173 * @hs_ep: The driver endpoint to check
1174 *
1175 * Returns 1 if targeted frame elapsed. If returned 1 then we need to drop
1176 * corresponding transfer.
1177 */
1178static bool dwc2_gadget_target_frame_elapsed(struct dwc2_hsotg_ep *hs_ep)
1179{
1180 struct dwc2_hsotg *hsotg = hs_ep->parent;
1181 u32 target_frame = hs_ep->target_frame;
1182 u32 current_frame = dwc2_hsotg_read_frameno(hsotg);
1183 bool frame_overrun = hs_ep->frame_overrun;
1184
1185 if (!frame_overrun && current_frame >= target_frame)
1186 return true;
1187
1188 if (frame_overrun && current_frame >= target_frame &&
1189 ((current_frame - target_frame) < DSTS_SOFFN_LIMIT / 2))
1190 return true;
1191
1192 return false;
1193}
1194
e02f9aa6
VA		 1195/*
1196 * dwc2_gadget_set_ep0_desc_chain - Set EP's desc chain pointers
1197 * @hsotg: The driver state
1198 * @hs_ep: the ep descriptor chain is for
1199 *
1200 * Called to update EP0 structure's pointers depend on stage of
1201 * control transfer.
1202 */
1203static int dwc2_gadget_set_ep0_desc_chain(struct dwc2_hsotg *hsotg,
1204 struct dwc2_hsotg_ep *hs_ep)
1205{
1206 switch (hsotg->ep0_state) {
1207 case DWC2_EP0_SETUP:
1208 case DWC2_EP0_STATUS_OUT:
1209 hs_ep->desc_list = hsotg->setup_desc[0];
1210 hs_ep->desc_list_dma = hsotg->setup_desc_dma[0];
1211 break;
1212 case DWC2_EP0_DATA_IN:
1213 case DWC2_EP0_STATUS_IN:
1214 hs_ep->desc_list = hsotg->ctrl_in_desc;
1215 hs_ep->desc_list_dma = hsotg->ctrl_in_desc_dma;
1216 break;
1217 case DWC2_EP0_DATA_OUT:
1218 hs_ep->desc_list = hsotg->ctrl_out_desc;
1219 hs_ep->desc_list_dma = hsotg->ctrl_out_desc_dma;
1220 break;
1221 default:
1222 dev_err(hsotg->dev, "invalid EP 0 state in queue %d\n",
1223 hsotg->ep0_state);
1224 return -EINVAL;
1225 }
1226
1227 return 0;
1228}
1229
1f91b4cc 1230static int dwc2_hsotg_ep_queue(struct usb_ep *ep, struct usb_request *req,
9da51974 1231 gfp_t gfp_flags)
5b7d70c6 1232{
1f91b4cc
FB		 1233 struct dwc2_hsotg_req *hs_req = our_req(req);
1234 struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
941fcce4 1235 struct dwc2_hsotg *hs = hs_ep->parent;
5b7d70c6 1236 bool first;
7d24c1b5 1237 int ret;
5b7d70c6
BD		 1238
1239 dev_dbg(hs->dev, "%s: req %p: %d@%p, noi=%d, zero=%d, snok=%d\n",
1240 ep->name, req, req->length, req->buf, req->no_interrupt,
1241 req->zero, req->short_not_ok);
1242
7ababa92
GH		 1243 /* Prevent new request submission when controller is suspended */
1244 if (hs->lx_state == DWC2_L2) {
1245 dev_dbg(hs->dev, "%s: don't submit request while suspended\n",
9da51974 1246 __func__);
7ababa92
GH		 1247 return -EAGAIN;
1248 }
1249
5b7d70c6
BD		 1250 /* initialise status of the request */
1251 INIT_LIST_HEAD(&hs_req->queue);
1252 req->actual = 0;
1253 req->status = -EINPROGRESS;
1254
1f91b4cc 1255 ret = dwc2_hsotg_handle_unaligned_buf_start(hs, hs_ep, hs_req);
7d24c1b5
MYK		 1256 if (ret)
1257 return ret;
1258
5b7d70c6
BD		 1259 /* if we're using DMA, sync the buffers as necessary */
1260 if (using_dma(hs)) {
1f91b4cc 1261 ret = dwc2_hsotg_map_dma(hs, hs_ep, req);
5b7d70c6
BD		 1262 if (ret)
1263 return ret;
1264 }
e02f9aa6
VA		 1265 /* If using descriptor DMA configure EP0 descriptor chain pointers */
1266 if (using_desc_dma(hs) && !hs_ep->index) {
1267 ret = dwc2_gadget_set_ep0_desc_chain(hs, hs_ep);
1268 if (ret)
1269 return ret;
1270 }
5b7d70c6 1271
5b7d70c6
BD		 1272 first = list_empty(&hs_ep->queue);
1273 list_add_tail(&hs_req->queue, &hs_ep->queue);
1274
540ccba0
VA		 1275 /*
1276 * Handle DDMA isochronous transfers separately - just add new entry
1277 * to the half of descriptor chain that is not processed by HW.
1278 * Transfer will be started once SW gets either one of NAK or
1279 * OutTknEpDis interrupts.
1280 */
1281 if (using_desc_dma(hs) && hs_ep->isochronous &&
1282 hs_ep->target_frame != TARGET_FRAME_INITIAL) {
1283 ret = dwc2_gadget_fill_isoc_desc(hs_ep, hs_req->req.dma,
1284 hs_req->req.length);
1285 if (ret)
1286 dev_dbg(hs->dev, "%s: ISO desc chain full\n", __func__);
1287
1288 return 0;
1289 }
1290
837e9f00
VM		 1291 if (first) {
1292 if (!hs_ep->isochronous) {
1293 dwc2_hsotg_start_req(hs, hs_ep, hs_req, false);
1294 return 0;
1295 }
1296
1297 while (dwc2_gadget_target_frame_elapsed(hs_ep))
1298 dwc2_gadget_incr_frame_num(hs_ep);
5b7d70c6 1299
837e9f00
VM		 1300 if (hs_ep->target_frame != TARGET_FRAME_INITIAL)
1301 dwc2_hsotg_start_req(hs, hs_ep, hs_req, false);
1302 }
5b7d70c6
BD		 1303 return 0;
1304}
1305
1f91b4cc 1306static int dwc2_hsotg_ep_queue_lock(struct usb_ep *ep, struct usb_request *req,
9da51974 1307 gfp_t gfp_flags)
5ad1d316 1308{
1f91b4cc 1309 struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
941fcce4 1310 struct dwc2_hsotg *hs = hs_ep->parent;
5ad1d316
LM		 1311 unsigned long flags = 0;
1312 int ret = 0;
1313
1314 spin_lock_irqsave(&hs->lock, flags);
1f91b4cc 1315 ret = dwc2_hsotg_ep_queue(ep, req, gfp_flags);
5ad1d316
LM		 1316 spin_unlock_irqrestore(&hs->lock, flags);
1317
1318 return ret;
1319}
1320
1f91b4cc 1321static void dwc2_hsotg_ep_free_request(struct usb_ep *ep,
9da51974 1322 struct usb_request *req)
5b7d70c6 1323{
1f91b4cc 1324 struct dwc2_hsotg_req *hs_req = our_req(req);
5b7d70c6
BD		 1325
1326 kfree(hs_req);
1327}
1328
1329/**
1f91b4cc 1330 * dwc2_hsotg_complete_oursetup - setup completion callback
5b7d70c6
BD		 1331 * @ep: The endpoint the request was on.
1332 * @req: The request completed.
1333 *
1334 * Called on completion of any requests the driver itself
1335 * submitted that need cleaning up.
1336 */
1f91b4cc 1337static void dwc2_hsotg_complete_oursetup(struct usb_ep *ep,
9da51974 1338 struct usb_request *req)
5b7d70c6 1339{
1f91b4cc 1340 struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
941fcce4 1341 struct dwc2_hsotg *hsotg = hs_ep->parent;
5b7d70c6
BD		 1342
1343 dev_dbg(hsotg->dev, "%s: ep %p, req %p\n", __func__, ep, req);
1344
1f91b4cc 1345 dwc2_hsotg_ep_free_request(ep, req);
5b7d70c6
BD		 1346}
1347
1348/**
1349 * ep_from_windex - convert control wIndex value to endpoint
1350 * @hsotg: The driver state.
1351 * @windex: The control request wIndex field (in host order).
1352 *
1353 * Convert the given wIndex into a pointer to an driver endpoint
1354 * structure, or return NULL if it is not a valid endpoint.
8b9bc460 1355 */
1f91b4cc 1356static struct dwc2_hsotg_ep *ep_from_windex(struct dwc2_hsotg *hsotg,
9da51974 1357 u32 windex)
5b7d70c6 1358{
1f91b4cc 1359 struct dwc2_hsotg_ep *ep;
5b7d70c6
BD		 1360 int dir = (windex & USB_DIR_IN) ? 1 : 0;
1361 int idx = windex & 0x7F;
1362
1363 if (windex >= 0x100)
1364 return NULL;
1365
b3f489b2 1366 if (idx > hsotg->num_of_eps)
5b7d70c6
BD		 1367 return NULL;
1368
c6f5c050
MYK		 1369 ep = index_to_ep(hsotg, idx, dir);
1370
5b7d70c6
BD		 1371 if (idx && ep->dir_in != dir)
1372 return NULL;
1373
1374 return ep;
1375}
1376
9e14d0a5 1377/**
1f91b4cc 1378 * dwc2_hsotg_set_test_mode - Enable usb Test Modes
9e14d0a5
GH		 1379 * @hsotg: The driver state.
1380 * @testmode: requested usb test mode
1381 * Enable usb Test Mode requested by the Host.
1382 */
1f91b4cc 1383int dwc2_hsotg_set_test_mode(struct dwc2_hsotg *hsotg, int testmode)
9e14d0a5 1384{
95c8bc36 1385 int dctl = dwc2_readl(hsotg->regs + DCTL);
9e14d0a5
GH		 1386
1387 dctl &= ~DCTL_TSTCTL_MASK;
1388 switch (testmode) {
1389 case TEST_J:
1390 case TEST_K:
1391 case TEST_SE0_NAK:
1392 case TEST_PACKET:
1393 case TEST_FORCE_EN:
1394 dctl |= testmode << DCTL_TSTCTL_SHIFT;
1395 break;
1396 default:
1397 return -EINVAL;
1398 }
95c8bc36 1399 dwc2_writel(dctl, hsotg->regs + DCTL);
9e14d0a5
GH		 1400 return 0;
1401}
1402
5b7d70c6 1403/**
1f91b4cc 1404 * dwc2_hsotg_send_reply - send reply to control request
5b7d70c6
BD		 1405 * @hsotg: The device state
1406 * @ep: Endpoint 0
1407 * @buff: Buffer for request
1408 * @length: Length of reply.
1409 *
1410 * Create a request and queue it on the given endpoint. This is useful as
1411 * an internal method of sending replies to certain control requests, etc.
1412 */
1f91b4cc 1413static int dwc2_hsotg_send_reply(struct dwc2_hsotg *hsotg,
9da51974 1414 struct dwc2_hsotg_ep *ep,
5b7d70c6
BD		 1415 void *buff,
1416 int length)
1417{
1418 struct usb_request *req;
1419 int ret;
1420
1421 dev_dbg(hsotg->dev, "%s: buff %p, len %d\n", __func__, buff, length);
1422
1f91b4cc 1423 req = dwc2_hsotg_ep_alloc_request(&ep->ep, GFP_ATOMIC);
5b7d70c6
BD		 1424 hsotg->ep0_reply = req;
1425 if (!req) {
1426 dev_warn(hsotg->dev, "%s: cannot alloc req\n", __func__);
1427 return -ENOMEM;
1428 }
1429
1430 req->buf = hsotg->ep0_buff;
1431 req->length = length;
f71b5e25
MYK		 1432 /*
1433 * zero flag is for sending zlp in DATA IN stage. It has no impact on
1434 * STATUS stage.
1435 */
1436 req->zero = 0;
1f91b4cc 1437 req->complete = dwc2_hsotg_complete_oursetup;
5b7d70c6
BD		 1438
1439 if (length)
1440 memcpy(req->buf, buff, length);
5b7d70c6 1441
1f91b4cc 1442 ret = dwc2_hsotg_ep_queue(&ep->ep, req, GFP_ATOMIC);
5b7d70c6
BD		 1443 if (ret) {
1444 dev_warn(hsotg->dev, "%s: cannot queue req\n", __func__);
1445 return ret;
1446 }
1447
1448 return 0;
1449}
1450
1451/**
1f91b4cc 1452 * dwc2_hsotg_process_req_status - process request GET_STATUS
5b7d70c6
BD		 1453 * @hsotg: The device state
1454 * @ctrl: USB control request
1455 */
1f91b4cc 1456static int dwc2_hsotg_process_req_status(struct dwc2_hsotg *hsotg,
9da51974 1457 struct usb_ctrlrequest *ctrl)
5b7d70c6 1458{
1f91b4cc
FB		 1459 struct dwc2_hsotg_ep *ep0 = hsotg->eps_out[0];
1460 struct dwc2_hsotg_ep *ep;
5b7d70c6
BD		 1461 __le16 reply;
1462 int ret;
1463
1464 dev_dbg(hsotg->dev, "%s: USB_REQ_GET_STATUS\n", __func__);
1465
1466 if (!ep0->dir_in) {
1467 dev_warn(hsotg->dev, "%s: direction out?\n", __func__);
1468 return -EINVAL;
1469 }
1470
1471 switch (ctrl->bRequestType & USB_RECIP_MASK) {
1472 case USB_RECIP_DEVICE:
38beaec6
JY		 1473 /*
1474 * bit 0 => self powered
1475 * bit 1 => remote wakeup
1476 */
1477 reply = cpu_to_le16(0);
5b7d70c6
BD		 1478 break;
1479
1480 case USB_RECIP_INTERFACE:
1481 /* currently, the data result should be zero */
1482 reply = cpu_to_le16(0);
1483 break;
1484
1485 case USB_RECIP_ENDPOINT:
1486 ep = ep_from_windex(hsotg, le16_to_cpu(ctrl->wIndex));
1487 if (!ep)
1488 return -ENOENT;
1489
1490 reply = cpu_to_le16(ep->halted ? 1 : 0);
1491 break;
1492
1493 default:
1494 return 0;
1495 }
1496
1497 if (le16_to_cpu(ctrl->wLength) != 2)
1498 return -EINVAL;
1499
1f91b4cc 1500 ret = dwc2_hsotg_send_reply(hsotg, ep0, &reply, 2);
5b7d70c6
BD		 1501 if (ret) {
1502 dev_err(hsotg->dev, "%s: failed to send reply\n", __func__);
1503 return ret;
1504 }
1505
1506 return 1;
1507}
1508
51da43b5 1509static int dwc2_hsotg_ep_sethalt(struct usb_ep *ep, int value, bool now);
5b7d70c6 1510
9c39ddc6
AT		 1511/**
1512 * get_ep_head - return the first request on the endpoint
1513 * @hs_ep: The controller endpoint to get
1514 *
1515 * Get the first request on the endpoint.
1516 */
1f91b4cc 1517static struct dwc2_hsotg_req *get_ep_head(struct dwc2_hsotg_ep *hs_ep)
9c39ddc6 1518{
ffc4b406
MY		 1519 return list_first_entry_or_null(&hs_ep->queue, struct dwc2_hsotg_req,
1520 queue);
9c39ddc6
AT		 1521}
1522
41cc4cd2
VM		 1523/**
1524 * dwc2_gadget_start_next_request - Starts next request from ep queue
1525 * @hs_ep: Endpoint structure
1526 *
1527 * If queue is empty and EP is ISOC-OUT - unmasks OUTTKNEPDIS which is masked
1528 * in its handler. Hence we need to unmask it here to be able to do
1529 * resynchronization.
1530 */
1531static void dwc2_gadget_start_next_request(struct dwc2_hsotg_ep *hs_ep)
1532{
1533 u32 mask;
1534 struct dwc2_hsotg *hsotg = hs_ep->parent;
1535 int dir_in = hs_ep->dir_in;
1536 struct dwc2_hsotg_req *hs_req;
1537 u32 epmsk_reg = dir_in ? DIEPMSK : DOEPMSK;
1538
1539 if (!list_empty(&hs_ep->queue)) {
1540 hs_req = get_ep_head(hs_ep);
1541 dwc2_hsotg_start_req(hsotg, hs_ep, hs_req, false);
1542 return;
1543 }
1544 if (!hs_ep->isochronous)
1545 return;
1546
1547 if (dir_in) {
1548 dev_dbg(hsotg->dev, "%s: No more ISOC-IN requests\n",
1549 __func__);
1550 } else {
1551 dev_dbg(hsotg->dev, "%s: No more ISOC-OUT requests\n",
1552 __func__);
1553 mask = dwc2_readl(hsotg->regs + epmsk_reg);
1554 mask |= DOEPMSK_OUTTKNEPDISMSK;
1555 dwc2_writel(mask, hsotg->regs + epmsk_reg);
1556 }
1557}
1558
5b7d70c6 1559/**
1f91b4cc 1560 * dwc2_hsotg_process_req_feature - process request {SET,CLEAR}_FEATURE
5b7d70c6
BD		 1561 * @hsotg: The device state
1562 * @ctrl: USB control request
1563 */
1f91b4cc 1564static int dwc2_hsotg_process_req_feature(struct dwc2_hsotg *hsotg,
9da51974 1565 struct usb_ctrlrequest *ctrl)
5b7d70c6 1566{
1f91b4cc
FB		 1567 struct dwc2_hsotg_ep *ep0 = hsotg->eps_out[0];
1568 struct dwc2_hsotg_req *hs_req;
5b7d70c6 1569 bool set = (ctrl->bRequest == USB_REQ_SET_FEATURE);
1f91b4cc 1570 struct dwc2_hsotg_ep *ep;
26ab3d0c 1571 int ret;
bd9ef7bf 1572 bool halted;
9e14d0a5
GH		 1573 u32 recip;
1574 u32 wValue;
1575 u32 wIndex;
5b7d70c6
BD		 1576
1577 dev_dbg(hsotg->dev, "%s: %s_FEATURE\n",
1578 __func__, set ? "SET" : "CLEAR");
1579
9e14d0a5
GH		 1580 wValue = le16_to_cpu(ctrl->wValue);
1581 wIndex = le16_to_cpu(ctrl->wIndex);
1582 recip = ctrl->bRequestType & USB_RECIP_MASK;
1583
1584 switch (recip) {
1585 case USB_RECIP_DEVICE:
1586 switch (wValue) {
1587 case USB_DEVICE_TEST_MODE:
1588 if ((wIndex & 0xff) != 0)
1589 return -EINVAL;
1590 if (!set)
1591 return -EINVAL;
1592
1593 hsotg->test_mode = wIndex >> 8;
1f91b4cc 1594 ret = dwc2_hsotg_send_reply(hsotg, ep0, NULL, 0);
9e14d0a5
GH		 1595 if (ret) {
1596 dev_err(hsotg->dev,
1597 "%s: failed to send reply\n", __func__);
1598 return ret;
1599 }
1600 break;
1601 default:
1602 return -ENOENT;
1603 }
1604 break;
1605
1606 case USB_RECIP_ENDPOINT:
1607 ep = ep_from_windex(hsotg, wIndex);
5b7d70c6
BD		 1608 if (!ep) {
1609 dev_dbg(hsotg->dev, "%s: no endpoint for 0x%04x\n",
9e14d0a5 1610 __func__, wIndex);
5b7d70c6
BD		 1611 return -ENOENT;
1612 }
1613
9e14d0a5 1614 switch (wValue) {
5b7d70c6 1615 case USB_ENDPOINT_HALT:
bd9ef7bf
RB		 1616 halted = ep->halted;
1617
51da43b5 1618 dwc2_hsotg_ep_sethalt(&ep->ep, set, true);
26ab3d0c 1619
1f91b4cc 1620 ret = dwc2_hsotg_send_reply(hsotg, ep0, NULL, 0);
26ab3d0c
AT		 1621 if (ret) {
1622 dev_err(hsotg->dev,
1623 "%s: failed to send reply\n", __func__);
1624 return ret;
1625 }
9c39ddc6 1626
bd9ef7bf
RB		 1627 /*
1628 * we have to complete all requests for ep if it was
1629 * halted, and the halt was cleared by CLEAR_FEATURE
1630 */
1631
1632 if (!set && halted) {
9c39ddc6
AT		 1633 /*
1634 * If we have request in progress,
1635 * then complete it
1636 */
1637 if (ep->req) {
1638 hs_req = ep->req;
1639 ep->req = NULL;
1640 list_del_init(&hs_req->queue);
c00dd4a6
GH		 1641 if (hs_req->req.complete) {
1642 spin_unlock(&hsotg->lock);
1643 usb_gadget_giveback_request(
1644 &ep->ep, &hs_req->req);
1645 spin_lock(&hsotg->lock);
1646 }
9c39ddc6
AT		 1647 }
1648
1649 /* If we have pending request, then start it */
34c0887f 1650 if (!ep->req)
41cc4cd2 1651 dwc2_gadget_start_next_request(ep);
9c39ddc6
AT		 1652 }
1653
5b7d70c6
BD		 1654 break;
1655
1656 default:
1657 return -ENOENT;
1658 }
9e14d0a5
GH		 1659 break;
1660 default:
1661 return -ENOENT;
1662 }
5b7d70c6
BD		 1663 return 1;
1664}
1665
1f91b4cc 1666static void dwc2_hsotg_enqueue_setup(struct dwc2_hsotg *hsotg);
ab93e014 1667
c9f721b2 1668/**
1f91b4cc 1669 * dwc2_hsotg_stall_ep0 - stall ep0
c9f721b2
RB		 1670 * @hsotg: The device state
1671 *
1672 * Set stall for ep0 as response for setup request.
1673 */
1f91b4cc 1674static void dwc2_hsotg_stall_ep0(struct dwc2_hsotg *hsotg)
e9ebe7c3 1675{
1f91b4cc 1676 struct dwc2_hsotg_ep *ep0 = hsotg->eps_out[0];
c9f721b2
RB		 1677 u32 reg;
1678 u32 ctrl;
1679
1680 dev_dbg(hsotg->dev, "ep0 stall (dir=%d)\n", ep0->dir_in);
1681 reg = (ep0->dir_in) ? DIEPCTL0 : DOEPCTL0;
1682
1683 /*
1684 * DxEPCTL_Stall will be cleared by EP once it has
1685 * taken effect, so no need to clear later.
1686 */
1687
95c8bc36 1688 ctrl = dwc2_readl(hsotg->regs + reg);
47a1685f
DN		 1689 ctrl |= DXEPCTL_STALL;
1690 ctrl |= DXEPCTL_CNAK;
95c8bc36 1691 dwc2_writel(ctrl, hsotg->regs + reg);
c9f721b2
RB		 1692
1693 dev_dbg(hsotg->dev,
47a1685f 1694 "written DXEPCTL=0x%08x to %08x (DXEPCTL=0x%08x)\n",
95c8bc36 1695 ctrl, reg, dwc2_readl(hsotg->regs + reg));
c9f721b2
RB		 1696
1697 /*
1698 * complete won't be called, so we enqueue
1699 * setup request here
1700 */
1f91b4cc 1701 dwc2_hsotg_enqueue_setup(hsotg);
c9f721b2
RB		 1702}
1703
5b7d70c6 1704/**
1f91b4cc 1705 * dwc2_hsotg_process_control - process a control request
5b7d70c6
BD		 1706 * @hsotg: The device state
1707 * @ctrl: The control request received
1708 *
1709 * The controller has received the SETUP phase of a control request, and
1710 * needs to work out what to do next (and whether to pass it on to the
1711 * gadget driver).
1712 */
1f91b4cc 1713static void dwc2_hsotg_process_control(struct dwc2_hsotg *hsotg,
9da51974 1714 struct usb_ctrlrequest *ctrl)
5b7d70c6 1715{
1f91b4cc 1716 struct dwc2_hsotg_ep *ep0 = hsotg->eps_out[0];
5b7d70c6
BD		 1717 int ret = 0;
1718 u32 dcfg;
1719
e525e743
MYK		 1720 dev_dbg(hsotg->dev,
1721 "ctrl Type=%02x, Req=%02x, V=%04x, I=%04x, L=%04x\n",
1722 ctrl->bRequestType, ctrl->bRequest, ctrl->wValue,
1723 ctrl->wIndex, ctrl->wLength);
5b7d70c6 1724
fe0b94ab
MYK		 1725 if (ctrl->wLength == 0) {
1726 ep0->dir_in = 1;
1727 hsotg->ep0_state = DWC2_EP0_STATUS_IN;
1728 } else if (ctrl->bRequestType & USB_DIR_IN) {
5b7d70c6 1729 ep0->dir_in = 1;
fe0b94ab
MYK		 1730 hsotg->ep0_state = DWC2_EP0_DATA_IN;
1731 } else {
1732 ep0->dir_in = 0;
1733 hsotg->ep0_state = DWC2_EP0_DATA_OUT;
1734 }
5b7d70c6
BD		 1735
1736 if ((ctrl->bRequestType & USB_TYPE_MASK) == USB_TYPE_STANDARD) {
1737 switch (ctrl->bRequest) {
1738 case USB_REQ_SET_ADDRESS:
6d713c15 1739 hsotg->connected = 1;
95c8bc36 1740 dcfg = dwc2_readl(hsotg->regs + DCFG);
47a1685f 1741 dcfg &= ~DCFG_DEVADDR_MASK;
d5dbd3f7
PZ		 1742 dcfg |= (le16_to_cpu(ctrl->wValue) <<
1743 DCFG_DEVADDR_SHIFT) & DCFG_DEVADDR_MASK;
95c8bc36 1744 dwc2_writel(dcfg, hsotg->regs + DCFG);
5b7d70c6
BD		 1745
1746 dev_info(hsotg->dev, "new address %d\n", ctrl->wValue);
1747
1f91b4cc 1748 ret = dwc2_hsotg_send_reply(hsotg, ep0, NULL, 0);
5b7d70c6
BD		 1749 return;
1750
1751 case USB_REQ_GET_STATUS:
1f91b4cc 1752 ret = dwc2_hsotg_process_req_status(hsotg, ctrl);
5b7d70c6
BD		 1753 break;
1754
1755 case USB_REQ_CLEAR_FEATURE:
1756 case USB_REQ_SET_FEATURE:
1f91b4cc 1757 ret = dwc2_hsotg_process_req_feature(hsotg, ctrl);
5b7d70c6
BD		 1758 break;
1759 }
1760 }
1761
1762 /* as a fallback, try delivering it to the driver to deal with */
1763
1764 if (ret == 0 && hsotg->driver) {
93f599f2 1765 spin_unlock(&hsotg->lock);
5b7d70c6 1766 ret = hsotg->driver->setup(&hsotg->gadget, ctrl);
93f599f2 1767 spin_lock(&hsotg->lock);
5b7d70c6
BD		 1768 if (ret < 0)
1769 dev_dbg(hsotg->dev, "driver->setup() ret %d\n", ret);
1770 }
1771
8b9bc460
LM		 1772 /*
1773 * the request is either unhandlable, or is not formatted correctly
5b7d70c6
BD		 1774 * so respond with a STALL for the status stage to indicate failure.
1775 */
1776
c9f721b2 1777 if (ret < 0)
1f91b4cc 1778 dwc2_hsotg_stall_ep0(hsotg);
5b7d70c6
BD		 1779}
1780
5b7d70c6 1781/**
1f91b4cc 1782 * dwc2_hsotg_complete_setup - completion of a setup transfer
5b7d70c6
BD		 1783 * @ep: The endpoint the request was on.
1784 * @req: The request completed.
1785 *
1786 * Called on completion of any requests the driver itself submitted for
1787 * EP0 setup packets
1788 */
1f91b4cc 1789static void dwc2_hsotg_complete_setup(struct usb_ep *ep,
9da51974 1790 struct usb_request *req)
5b7d70c6 1791{
1f91b4cc 1792 struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
941fcce4 1793 struct dwc2_hsotg *hsotg = hs_ep->parent;
5b7d70c6
BD		 1794
1795 if (req->status < 0) {
1796 dev_dbg(hsotg->dev, "%s: failed %d\n", __func__, req->status);
1797 return;
1798 }
1799
93f599f2 1800 spin_lock(&hsotg->lock);
5b7d70c6 1801 if (req->actual == 0)
1f91b4cc 1802 dwc2_hsotg_enqueue_setup(hsotg);
5b7d70c6 1803 else
1f91b4cc 1804 dwc2_hsotg_process_control(hsotg, req->buf);
93f599f2 1805 spin_unlock(&hsotg->lock);
5b7d70c6
BD		 1806}
1807
1808/**
1f91b4cc 1809 * dwc2_hsotg_enqueue_setup - start a request for EP0 packets
5b7d70c6
BD		 1810 * @hsotg: The device state.
1811 *
1812 * Enqueue a request on EP0 if necessary to received any SETUP packets
1813 * received from the host.
1814 */
1f91b4cc 1815static void dwc2_hsotg_enqueue_setup(struct dwc2_hsotg *hsotg)
5b7d70c6
BD		 1816{
1817 struct usb_request *req = hsotg->ctrl_req;
1f91b4cc 1818 struct dwc2_hsotg_req *hs_req = our_req(req);
5b7d70c6
BD		 1819 int ret;
1820
1821 dev_dbg(hsotg->dev, "%s: queueing setup request\n", __func__);
1822
1823 req->zero = 0;
1824 req->length = 8;
1825 req->buf = hsotg->ctrl_buff;
1f91b4cc 1826 req->complete = dwc2_hsotg_complete_setup;
5b7d70c6
BD		 1827
1828 if (!list_empty(&hs_req->queue)) {
1829 dev_dbg(hsotg->dev, "%s already queued???\n", __func__);
1830 return;
1831 }
1832
c6f5c050 1833 hsotg->eps_out[0]->dir_in = 0;
8a20fa45 1834 hsotg->eps_out[0]->send_zlp = 0;
fe0b94ab 1835 hsotg->ep0_state = DWC2_EP0_SETUP;
5b7d70c6 1836
1f91b4cc 1837 ret = dwc2_hsotg_ep_queue(&hsotg->eps_out[0]->ep, req, GFP_ATOMIC);
5b7d70c6
BD		 1838 if (ret < 0) {
1839 dev_err(hsotg->dev, "%s: failed queue (%d)\n", __func__, ret);
8b9bc460
LM		 1840 /*
1841 * Don't think there's much we can do other than watch the
1842 * driver fail.
1843 */
5b7d70c6
BD		 1844 }
1845}
1846
1f91b4cc 1847static void dwc2_hsotg_program_zlp(struct dwc2_hsotg *hsotg,
9da51974 1848 struct dwc2_hsotg_ep *hs_ep)
fe0b94ab
MYK		 1849{
1850 u32 ctrl;
1851 u8 index = hs_ep->index;
1852 u32 epctl_reg = hs_ep->dir_in ? DIEPCTL(index) : DOEPCTL(index);
1853 u32 epsiz_reg = hs_ep->dir_in ? DIEPTSIZ(index) : DOEPTSIZ(index);
1854
ccb34a91
MYK		 1855 if (hs_ep->dir_in)
1856 dev_dbg(hsotg->dev, "Sending zero-length packet on ep%d\n",
e02f9aa6 1857 index);
ccb34a91
MYK		 1858 else
1859 dev_dbg(hsotg->dev, "Receiving zero-length packet on ep%d\n",
e02f9aa6
VA		 1860 index);
1861 if (using_desc_dma(hsotg)) {
1862 /* Not specific buffer needed for ep0 ZLP */
1863 dma_addr_t dma = hs_ep->desc_list_dma;
fe0b94ab 1864
e02f9aa6
VA		 1865 dwc2_gadget_set_ep0_desc_chain(hsotg, hs_ep);
1866 dwc2_gadget_config_nonisoc_xfer_ddma(hs_ep, dma, 0);
1867 } else {
1868 dwc2_writel(DXEPTSIZ_MC(1) | DXEPTSIZ_PKTCNT(1) |
1869 DXEPTSIZ_XFERSIZE(0), hsotg->regs +
1870 epsiz_reg);
1871 }
fe0b94ab 1872
95c8bc36 1873 ctrl = dwc2_readl(hsotg->regs + epctl_reg);
fe0b94ab
MYK		 1874 ctrl |= DXEPCTL_CNAK; /* clear NAK set by core */
1875 ctrl |= DXEPCTL_EPENA; /* ensure ep enabled */
1876 ctrl |= DXEPCTL_USBACTEP;
95c8bc36 1877 dwc2_writel(ctrl, hsotg->regs + epctl_reg);
fe0b94ab
MYK		 1878}
1879
5b7d70c6 1880/**
1f91b4cc 1881 * dwc2_hsotg_complete_request - complete a request given to us
5b7d70c6
BD		 1882 * @hsotg: The device state.
1883 * @hs_ep: The endpoint the request was on.
1884 * @hs_req: The request to complete.
1885 * @result: The result code (0 => Ok, otherwise errno)
1886 *
1887 * The given request has finished, so call the necessary completion
1888 * if it has one and then look to see if we can start a new request
1889 * on the endpoint.
1890 *
1891 * Note, expects the ep to already be locked as appropriate.
8b9bc460 1892 */
1f91b4cc 1893static void dwc2_hsotg_complete_request(struct dwc2_hsotg *hsotg,
9da51974 1894 struct dwc2_hsotg_ep *hs_ep,
1f91b4cc 1895 struct dwc2_hsotg_req *hs_req,
5b7d70c6
BD		 1896 int result)
1897{
5b7d70c6
BD		 1898 if (!hs_req) {
1899 dev_dbg(hsotg->dev, "%s: nothing to complete?\n", __func__);
1900 return;
1901 }
1902
1903 dev_dbg(hsotg->dev, "complete: ep %p %s, req %p, %d => %p\n",
1904 hs_ep, hs_ep->ep.name, hs_req, result, hs_req->req.complete);
1905
8b9bc460
LM		 1906 /*
1907 * only replace the status if we've not already set an error
1908 * from a previous transaction
1909 */
5b7d70c6
BD		 1910
1911 if (hs_req->req.status == -EINPROGRESS)
1912 hs_req->req.status = result;
1913
44583fec
YL		 1914 if (using_dma(hsotg))
1915 dwc2_hsotg_unmap_dma(hsotg, hs_ep, hs_req);
1916
1f91b4cc 1917 dwc2_hsotg_handle_unaligned_buf_complete(hsotg, hs_ep, hs_req);
7d24c1b5 1918
5b7d70c6
BD		 1919 hs_ep->req = NULL;
1920 list_del_init(&hs_req->queue);
1921
8b9bc460
LM		 1922 /*
1923 * call the complete request with the locks off, just in case the
1924 * request tries to queue more work for this endpoint.
1925 */
5b7d70c6
BD		 1926
1927 if (hs_req->req.complete) {
22258f49 1928 spin_unlock(&hsotg->lock);
304f7e5e 1929 usb_gadget_giveback_request(&hs_ep->ep, &hs_req->req);
22258f49 1930 spin_lock(&hsotg->lock);
5b7d70c6
BD		 1931 }
1932
540ccba0
VA		 1933 /* In DDMA don't need to proceed to starting of next ISOC request */
1934 if (using_desc_dma(hsotg) && hs_ep->isochronous)
1935 return;
1936
8b9bc460
LM		 1937 /*
1938 * Look to see if there is anything else to do. Note, the completion
5b7d70c6 1939 * of the previous request may have caused a new request to be started
8b9bc460
LM		 1940 * so be careful when doing this.
1941 */
5b7d70c6 1942
34c0887f 1943 if (!hs_ep->req && result >= 0)
41cc4cd2 1944 dwc2_gadget_start_next_request(hs_ep);
5b7d70c6
BD		 1945}
1946
540ccba0
VA		 1947/*
1948 * dwc2_gadget_complete_isoc_request_ddma - complete an isoc request in DDMA
1949 * @hs_ep: The endpoint the request was on.
1950 *
1951 * Get first request from the ep queue, determine descriptor on which complete
1952 * happened. SW based on isoc_chain_num discovers which half of the descriptor
1953 * chain is currently in use by HW, adjusts dma_address and calculates index
1954 * of completed descriptor based on the value of DEPDMA register. Update actual
1955 * length of request, giveback to gadget.
1956 */
1957static void dwc2_gadget_complete_isoc_request_ddma(struct dwc2_hsotg_ep *hs_ep)
1958{
1959 struct dwc2_hsotg *hsotg = hs_ep->parent;
1960 struct dwc2_hsotg_req *hs_req;
1961 struct usb_request *ureq;
1962 int index;
1963 dma_addr_t dma_addr;
1964 u32 dma_reg;
1965 u32 depdma;
1966 u32 desc_sts;
1967 u32 mask;
1968
1969 hs_req = get_ep_head(hs_ep);
1970 if (!hs_req) {
1971 dev_warn(hsotg->dev, "%s: ISOC EP queue empty\n", __func__);
1972 return;
1973 }
1974 ureq = &hs_req->req;
1975
1976 dma_addr = hs_ep->desc_list_dma;
1977
1978 /*
1979 * If lower half of descriptor chain is currently use by SW,
1980 * that means higher half is being processed by HW, so shift
1981 * DMA address to higher half of descriptor chain.
1982 */
1983 if (!hs_ep->isoc_chain_num)
1984 dma_addr += sizeof(struct dwc2_dma_desc) *
1985 (MAX_DMA_DESC_NUM_GENERIC / 2);
1986
1987 dma_reg = hs_ep->dir_in ? DIEPDMA(hs_ep->index) : DOEPDMA(hs_ep->index);
1988 depdma = dwc2_readl(hsotg->regs + dma_reg);
1989
1990 index = (depdma - dma_addr) / sizeof(struct dwc2_dma_desc) - 1;
1991 desc_sts = hs_ep->desc_list[index].status;
1992
1993 mask = hs_ep->dir_in ? DEV_DMA_ISOC_TX_NBYTES_MASK :
1994 DEV_DMA_ISOC_RX_NBYTES_MASK;
1995 ureq->actual = ureq->length -
1996 ((desc_sts & mask) >> DEV_DMA_ISOC_NBYTES_SHIFT);
1997
95d2b037
VA		 1998 /* Adjust actual length for ISOC Out if length is not align of 4 */
1999 if (!hs_ep->dir_in && ureq->length & 0x3)
2000 ureq->actual += 4 - (ureq->length & 0x3);
2001
540ccba0
VA		 2002 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0);
2003}
2004
2005/*
2006 * dwc2_gadget_start_next_isoc_ddma - start next isoc request, if any.
2007 * @hs_ep: The isochronous endpoint to be re-enabled.
2008 *
2009 * If ep has been disabled due to last descriptor servicing (IN endpoint) or
2010 * BNA (OUT endpoint) check the status of other half of descriptor chain that
2011 * was under SW control till HW was busy and restart the endpoint if needed.
2012 */
2013static void dwc2_gadget_start_next_isoc_ddma(struct dwc2_hsotg_ep *hs_ep)
2014{
2015 struct dwc2_hsotg *hsotg = hs_ep->parent;
2016 u32 depctl;
2017 u32 dma_reg;
2018 u32 ctrl;
2019 u32 dma_addr = hs_ep->desc_list_dma;
2020 unsigned char index = hs_ep->index;
2021
2022 dma_reg = hs_ep->dir_in ? DIEPDMA(index) : DOEPDMA(index);
2023 depctl = hs_ep->dir_in ? DIEPCTL(index) : DOEPCTL(index);
2024
2025 ctrl = dwc2_readl(hsotg->regs + depctl);
2026
2027 /*
2028 * EP was disabled if HW has processed last descriptor or BNA was set.
2029 * So restart ep if SW has prepared new descriptor chain in ep_queue
2030 * routine while HW was busy.
2031 */
2032 if (!(ctrl & DXEPCTL_EPENA)) {
2033 if (!hs_ep->next_desc) {
2034 dev_dbg(hsotg->dev, "%s: No more ISOC requests\n",
2035 __func__);
2036 return;
2037 }
2038
2039 dma_addr += sizeof(struct dwc2_dma_desc) *
2040 (MAX_DMA_DESC_NUM_GENERIC / 2) *
2041 hs_ep->isoc_chain_num;
2042 dwc2_writel(dma_addr, hsotg->regs + dma_reg);
2043
2044 ctrl |= DXEPCTL_EPENA | DXEPCTL_CNAK;
2045 dwc2_writel(ctrl, hsotg->regs + depctl);
2046
2047 /* Switch ISOC descriptor chain number being processed by SW*/
2048 hs_ep->isoc_chain_num = (hs_ep->isoc_chain_num ^ 1) & 0x1;
2049 hs_ep->next_desc = 0;
2050
2051 dev_dbg(hsotg->dev, "%s: Restarted isochronous endpoint\n",
2052 __func__);
2053 }
2054}
2055
5b7d70c6 2056/**
1f91b4cc 2057 * dwc2_hsotg_rx_data - receive data from the FIFO for an endpoint
5b7d70c6
BD		 2058 * @hsotg: The device state.
2059 * @ep_idx: The endpoint index for the data
2060 * @size: The size of data in the fifo, in bytes
2061 *
2062 * The FIFO status shows there is data to read from the FIFO for a given
2063 * endpoint, so sort out whether we need to read the data into a request
2064 * that has been made for that endpoint.
2065 */
1f91b4cc 2066static void dwc2_hsotg_rx_data(struct dwc2_hsotg *hsotg, int ep_idx, int size)
5b7d70c6 2067{
1f91b4cc
FB		 2068 struct dwc2_hsotg_ep *hs_ep = hsotg->eps_out[ep_idx];
2069 struct dwc2_hsotg_req *hs_req = hs_ep->req;
94cb8fd6 2070 void __iomem *fifo = hsotg->regs + EPFIFO(ep_idx);
5b7d70c6
BD		 2071 int to_read;
2072 int max_req;
2073 int read_ptr;
2074
2075 if (!hs_req) {
95c8bc36 2076 u32 epctl = dwc2_readl(hsotg->regs + DOEPCTL(ep_idx));
5b7d70c6
BD		 2077 int ptr;
2078
6b448af4 2079 dev_dbg(hsotg->dev,
9da51974 2080 "%s: FIFO %d bytes on ep%d but no req (DXEPCTl=0x%08x)\n",
5b7d70c6
BD		 2081 __func__, size, ep_idx, epctl);
2082
2083 /* dump the data from the FIFO, we've nothing we can do */
2084 for (ptr = 0; ptr < size; ptr += 4)
95c8bc36 2085 (void)dwc2_readl(fifo);
5b7d70c6
BD		 2086
2087 return;
2088 }
2089
5b7d70c6
BD		 2090 to_read = size;
2091 read_ptr = hs_req->req.actual;
2092 max_req = hs_req->req.length - read_ptr;
2093
a33e7136
BD		 2094 dev_dbg(hsotg->dev, "%s: read %d/%d, done %d/%d\n",
2095 __func__, to_read, max_req, read_ptr, hs_req->req.length);
2096
5b7d70c6 2097 if (to_read > max_req) {
8b9bc460
LM		 2098 /*
2099 * more data appeared than we where willing
5b7d70c6
BD		 2100 * to deal with in this request.
2101 */
2102
2103 /* currently we don't deal this */
2104 WARN_ON_ONCE(1);
2105 }
2106
5b7d70c6
BD		 2107 hs_ep->total_data += to_read;
2108 hs_req->req.actual += to_read;
2109 to_read = DIV_ROUND_UP(to_read, 4);
2110
8b9bc460
LM		 2111 /*
2112 * note, we might over-write the buffer end by 3 bytes depending on
2113 * alignment of the data.
2114 */
1a7ed5be 2115 ioread32_rep(fifo, hs_req->req.buf + read_ptr, to_read);
5b7d70c6
BD		 2116}
2117
2118/**
1f91b4cc 2119 * dwc2_hsotg_ep0_zlp - send/receive zero-length packet on control endpoint
5b7d70c6 2120 * @hsotg: The device instance
fe0b94ab 2121 * @dir_in: If IN zlp
5b7d70c6
BD		 2122 *
2123 * Generate a zero-length IN packet request for terminating a SETUP
2124 * transaction.
2125 *
2126 * Note, since we don't write any data to the TxFIFO, then it is
25985edc 2127 * currently believed that we do not need to wait for any space in
5b7d70c6
BD		 2128 * the TxFIFO.
2129 */
1f91b4cc 2130static void dwc2_hsotg_ep0_zlp(struct dwc2_hsotg *hsotg, bool dir_in)
5b7d70c6 2131{
c6f5c050 2132 /* eps_out[0] is used in both directions */
fe0b94ab
MYK		 2133 hsotg->eps_out[0]->dir_in = dir_in;
2134 hsotg->ep0_state = dir_in ? DWC2_EP0_STATUS_IN : DWC2_EP0_STATUS_OUT;
5b7d70c6 2135
1f91b4cc 2136 dwc2_hsotg_program_zlp(hsotg, hsotg->eps_out[0]);
5b7d70c6
BD		 2137}
2138
ec1f9d9f 2139static void dwc2_hsotg_change_ep_iso_parity(struct dwc2_hsotg *hsotg,
9da51974 2140 u32 epctl_reg)
ec1f9d9f
RB		 2141{
2142 u32 ctrl;
2143
2144 ctrl = dwc2_readl(hsotg->regs + epctl_reg);
2145 if (ctrl & DXEPCTL_EOFRNUM)
2146 ctrl |= DXEPCTL_SETEVENFR;
2147 else
2148 ctrl |= DXEPCTL_SETODDFR;
2149 dwc2_writel(ctrl, hsotg->regs + epctl_reg);
2150}
2151
aa3e8bc8
VA		 2152/*
2153 * dwc2_gadget_get_xfersize_ddma - get transferred bytes amount from desc
2154 * @hs_ep - The endpoint on which transfer went
2155 *
2156 * Iterate over endpoints descriptor chain and get info on bytes remained
2157 * in DMA descriptors after transfer has completed. Used for non isoc EPs.
2158 */
2159static unsigned int dwc2_gadget_get_xfersize_ddma(struct dwc2_hsotg_ep *hs_ep)
2160{
2161 struct dwc2_hsotg *hsotg = hs_ep->parent;
2162 unsigned int bytes_rem = 0;
2163 struct dwc2_dma_desc *desc = hs_ep->desc_list;
2164 int i;
2165 u32 status;
2166
2167 if (!desc)
2168 return -EINVAL;
2169
2170 for (i = 0; i < hs_ep->desc_count; ++i) {
2171 status = desc->status;
2172 bytes_rem += status & DEV_DMA_NBYTES_MASK;
2173
2174 if (status & DEV_DMA_STS_MASK)
2175 dev_err(hsotg->dev, "descriptor %d closed with %x\n",
2176 i, status & DEV_DMA_STS_MASK);
2177 }
2178
2179 return bytes_rem;
2180}
2181
5b7d70c6 2182/**
1f91b4cc 2183 * dwc2_hsotg_handle_outdone - handle receiving OutDone/SetupDone from RXFIFO
5b7d70c6
BD		 2184 * @hsotg: The device instance
2185 * @epnum: The endpoint received from
5b7d70c6
BD		 2186 *
2187 * The RXFIFO has delivered an OutDone event, which means that the data
2188 * transfer for an OUT endpoint has been completed, either by a short
2189 * packet or by the finish of a transfer.
8b9bc460 2190 */
1f91b4cc 2191static void dwc2_hsotg_handle_outdone(struct dwc2_hsotg *hsotg, int epnum)
5b7d70c6 2192{
95c8bc36 2193 u32 epsize = dwc2_readl(hsotg->regs + DOEPTSIZ(epnum));
1f91b4cc
FB		 2194 struct dwc2_hsotg_ep *hs_ep = hsotg->eps_out[epnum];
2195 struct dwc2_hsotg_req *hs_req = hs_ep->req;
5b7d70c6 2196 struct usb_request *req = &hs_req->req;
9da51974 2197 unsigned int size_left = DXEPTSIZ_XFERSIZE_GET(epsize);
5b7d70c6
BD		 2198 int result = 0;
2199
2200 if (!hs_req) {
2201 dev_dbg(hsotg->dev, "%s: no request active\n", __func__);
2202 return;
2203 }
2204
fe0b94ab
MYK		 2205 if (epnum == 0 && hsotg->ep0_state == DWC2_EP0_STATUS_OUT) {
2206 dev_dbg(hsotg->dev, "zlp packet received\n");
1f91b4cc
FB		 2207 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0);
2208 dwc2_hsotg_enqueue_setup(hsotg);
fe0b94ab
MYK		 2209 return;
2210 }
2211
aa3e8bc8
VA		 2212 if (using_desc_dma(hsotg))
2213 size_left = dwc2_gadget_get_xfersize_ddma(hs_ep);
2214
5b7d70c6 2215 if (using_dma(hsotg)) {
9da51974 2216 unsigned int size_done;
5b7d70c6 2217
8b9bc460
LM		 2218 /*
2219 * Calculate the size of the transfer by checking how much
5b7d70c6
BD		 2220 * is left in the endpoint size register and then working it
2221 * out from the amount we loaded for the transfer.
2222 *
2223 * We need to do this as DMA pointers are always 32bit aligned
2224 * so may overshoot/undershoot the transfer.
2225 */
2226
5b7d70c6
BD		 2227 size_done = hs_ep->size_loaded - size_left;
2228 size_done += hs_ep->last_load;
2229
2230 req->actual = size_done;
2231 }
2232
a33e7136
BD		 2233 /* if there is more request to do, schedule new transfer */
2234 if (req->actual < req->length && size_left == 0) {
1f91b4cc 2235 dwc2_hsotg_start_req(hsotg, hs_ep, hs_req, true);
a33e7136
BD		 2236 return;
2237 }
2238
5b7d70c6
BD		 2239 if (req->actual < req->length && req->short_not_ok) {
2240 dev_dbg(hsotg->dev, "%s: got %d/%d (short not ok) => error\n",
2241 __func__, req->actual, req->length);
2242
8b9bc460
LM		 2243 /*
2244 * todo - what should we return here? there's no one else
2245 * even bothering to check the status.
2246 */
5b7d70c6
BD		 2247 }
2248
ef750c71
VA		 2249 /* DDMA IN status phase will start from StsPhseRcvd interrupt */
2250 if (!using_desc_dma(hsotg) && epnum == 0 &&
2251 hsotg->ep0_state == DWC2_EP0_DATA_OUT) {
fe0b94ab 2252 /* Move to STATUS IN */
1f91b4cc 2253 dwc2_hsotg_ep0_zlp(hsotg, true);
fe0b94ab 2254 return;
5b7d70c6
BD		 2255 }
2256
ec1f9d9f
RB		 2257 /*
2258 * Slave mode OUT transfers do not go through XferComplete so
2259 * adjust the ISOC parity here.
2260 */
2261 if (!using_dma(hsotg)) {
ec1f9d9f
RB		 2262 if (hs_ep->isochronous && hs_ep->interval == 1)
2263 dwc2_hsotg_change_ep_iso_parity(hsotg, DOEPCTL(epnum));
837e9f00
VM		 2264 else if (hs_ep->isochronous && hs_ep->interval > 1)
2265 dwc2_gadget_incr_frame_num(hs_ep);
ec1f9d9f
RB		 2266 }
2267
1f91b4cc 2268 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, result);
5b7d70c6
BD		 2269}
2270
5b7d70c6 2271/**
1f91b4cc 2272 * dwc2_hsotg_handle_rx - RX FIFO has data
5b7d70c6
BD		 2273 * @hsotg: The device instance
2274 *
2275 * The IRQ handler has detected that the RX FIFO has some data in it
2276 * that requires processing, so find out what is in there and do the
2277 * appropriate read.
2278 *
25985edc 2279 * The RXFIFO is a true FIFO, the packets coming out are still in packet
5b7d70c6
BD		 2280 * chunks, so if you have x packets received on an endpoint you'll get x
2281 * FIFO events delivered, each with a packet's worth of data in it.
2282 *
2283 * When using DMA, we should not be processing events from the RXFIFO
2284 * as the actual data should be sent to the memory directly and we turn
2285 * on the completion interrupts to get notifications of transfer completion.
2286 */
1f91b4cc 2287static void dwc2_hsotg_handle_rx(struct dwc2_hsotg *hsotg)
5b7d70c6 2288{
95c8bc36 2289 u32 grxstsr = dwc2_readl(hsotg->regs + GRXSTSP);
5b7d70c6
BD		 2290 u32 epnum, status, size;
2291
2292 WARN_ON(using_dma(hsotg));
2293
47a1685f
DN		 2294 epnum = grxstsr & GRXSTS_EPNUM_MASK;
2295 status = grxstsr & GRXSTS_PKTSTS_MASK;
5b7d70c6 2296
47a1685f
DN		 2297 size = grxstsr & GRXSTS_BYTECNT_MASK;
2298 size >>= GRXSTS_BYTECNT_SHIFT;
5b7d70c6 2299
d7c747c5 2300 dev_dbg(hsotg->dev, "%s: GRXSTSP=0x%08x (%d@%d)\n",
9da51974 2301 __func__, grxstsr, size, epnum);
5b7d70c6 2302
47a1685f
DN		 2303 switch ((status & GRXSTS_PKTSTS_MASK) >> GRXSTS_PKTSTS_SHIFT) {
2304 case GRXSTS_PKTSTS_GLOBALOUTNAK:
2305 dev_dbg(hsotg->dev, "GLOBALOUTNAK\n");
5b7d70c6
BD		 2306 break;
2307
47a1685f 2308 case GRXSTS_PKTSTS_OUTDONE:
5b7d70c6 2309 dev_dbg(hsotg->dev, "OutDone (Frame=0x%08x)\n",
1f91b4cc 2310 dwc2_hsotg_read_frameno(hsotg));
5b7d70c6
BD		 2311
2312 if (!using_dma(hsotg))
1f91b4cc 2313 dwc2_hsotg_handle_outdone(hsotg, epnum);
5b7d70c6
BD		 2314 break;
2315
47a1685f 2316 case GRXSTS_PKTSTS_SETUPDONE:
5b7d70c6
BD		 2317 dev_dbg(hsotg->dev,
2318 "SetupDone (Frame=0x%08x, DOPEPCTL=0x%08x)\n",
1f91b4cc 2319 dwc2_hsotg_read_frameno(hsotg),
95c8bc36 2320 dwc2_readl(hsotg->regs + DOEPCTL(0)));
fe0b94ab 2321 /*
1f91b4cc 2322 * Call dwc2_hsotg_handle_outdone here if it was not called from
fe0b94ab
MYK		 2323 * GRXSTS_PKTSTS_OUTDONE. That is, if the core didn't
2324 * generate GRXSTS_PKTSTS_OUTDONE for setup packet.
2325 */
2326 if (hsotg->ep0_state == DWC2_EP0_SETUP)
1f91b4cc 2327 dwc2_hsotg_handle_outdone(hsotg, epnum);
5b7d70c6
BD		 2328 break;
2329
47a1685f 2330 case GRXSTS_PKTSTS_OUTRX:
1f91b4cc 2331 dwc2_hsotg_rx_data(hsotg, epnum, size);
5b7d70c6
BD		 2332 break;
2333
47a1685f 2334 case GRXSTS_PKTSTS_SETUPRX:
5b7d70c6
BD		 2335 dev_dbg(hsotg->dev,
2336 "SetupRX (Frame=0x%08x, DOPEPCTL=0x%08x)\n",
1f91b4cc 2337 dwc2_hsotg_read_frameno(hsotg),
95c8bc36 2338 dwc2_readl(hsotg->regs + DOEPCTL(0)));
5b7d70c6 2339
fe0b94ab
MYK		 2340 WARN_ON(hsotg->ep0_state != DWC2_EP0_SETUP);
2341
1f91b4cc 2342 dwc2_hsotg_rx_data(hsotg, epnum, size);
5b7d70c6
BD		 2343 break;
2344
2345 default:
2346 dev_warn(hsotg->dev, "%s: unknown status %08x\n",
2347 __func__, grxstsr);
2348
1f91b4cc 2349 dwc2_hsotg_dump(hsotg);
5b7d70c6
BD		 2350 break;
2351 }
2352}
2353
2354/**
1f91b4cc 2355 * dwc2_hsotg_ep0_mps - turn max packet size into register setting
5b7d70c6 2356 * @mps: The maximum packet size in bytes.
8b9bc460 2357 */
1f91b4cc 2358static u32 dwc2_hsotg_ep0_mps(unsigned int mps)
5b7d70c6
BD		 2359{
2360 switch (mps) {
2361 case 64:
94cb8fd6 2362 return D0EPCTL_MPS_64;
5b7d70c6 2363 case 32:
94cb8fd6 2364 return D0EPCTL_MPS_32;
5b7d70c6 2365 case 16:
94cb8fd6 2366 return D0EPCTL_MPS_16;
5b7d70c6 2367 case 8:
94cb8fd6 2368 return D0EPCTL_MPS_8;
5b7d70c6
BD		 2369 }
2370
2371 /* bad max packet size, warn and return invalid result */
2372 WARN_ON(1);
2373 return (u32)-1;
2374}
2375
2376/**
1f91b4cc 2377 * dwc2_hsotg_set_ep_maxpacket - set endpoint's max-packet field
5b7d70c6
BD		 2378 * @hsotg: The driver state.
2379 * @ep: The index number of the endpoint
2380 * @mps: The maximum packet size in bytes
ee2c40de 2381 * @mc: The multicount value
5b7d70c6
BD		 2382 *
2383 * Configure the maximum packet size for the given endpoint, updating
2384 * the hardware control registers to reflect this.
2385 */
1f91b4cc 2386static void dwc2_hsotg_set_ep_maxpacket(struct dwc2_hsotg *hsotg,
ee2c40de
VM		 2387 unsigned int ep, unsigned int mps,
2388 unsigned int mc, unsigned int dir_in)
5b7d70c6 2389{
1f91b4cc 2390 struct dwc2_hsotg_ep *hs_ep;
5b7d70c6 2391 void __iomem *regs = hsotg->regs;
5b7d70c6
BD		 2392 u32 reg;
2393
c6f5c050
MYK		 2394 hs_ep = index_to_ep(hsotg, ep, dir_in);
2395 if (!hs_ep)
2396 return;
2397
5b7d70c6 2398 if (ep == 0) {
ee2c40de
VM		 2399 u32 mps_bytes = mps;
2400
5b7d70c6 2401 /* EP0 is a special case */
ee2c40de
VM		 2402 mps = dwc2_hsotg_ep0_mps(mps_bytes);
2403 if (mps > 3)
5b7d70c6 2404 goto bad_mps;
ee2c40de 2405 hs_ep->ep.maxpacket = mps_bytes;
4fca54aa 2406 hs_ep->mc = 1;
5b7d70c6 2407 } else {
ee2c40de 2408 if (mps > 1024)
5b7d70c6 2409 goto bad_mps;
ee2c40de
VM		 2410 hs_ep->mc = mc;
2411 if (mc > 3)
4fca54aa 2412 goto bad_mps;
ee2c40de 2413 hs_ep->ep.maxpacket = mps;
5b7d70c6
BD		 2414 }
2415
c6f5c050 2416 if (dir_in) {
95c8bc36 2417 reg = dwc2_readl(regs + DIEPCTL(ep));
c6f5c050 2418 reg &= ~DXEPCTL_MPS_MASK;
ee2c40de 2419 reg |= mps;
95c8bc36 2420 dwc2_writel(reg, regs + DIEPCTL(ep));
c6f5c050 2421 } else {
95c8bc36 2422 reg = dwc2_readl(regs + DOEPCTL(ep));
47a1685f 2423 reg &= ~DXEPCTL_MPS_MASK;
ee2c40de 2424 reg |= mps;
95c8bc36 2425 dwc2_writel(reg, regs + DOEPCTL(ep));
659ad60c 2426 }
5b7d70c6
BD		 2427
2428 return;
2429
2430bad_mps:
2431 dev_err(hsotg->dev, "ep%d: bad mps of %d\n", ep, mps);
2432}
2433
9c39ddc6 2434/**
1f91b4cc 2435 * dwc2_hsotg_txfifo_flush - flush Tx FIFO
9c39ddc6
AT		 2436 * @hsotg: The driver state
2437 * @idx: The index for the endpoint (0..15)
2438 */
1f91b4cc 2439static void dwc2_hsotg_txfifo_flush(struct dwc2_hsotg *hsotg, unsigned int idx)
9c39ddc6
AT		 2440{
2441 int timeout;
2442 int val;
2443
95c8bc36
AS		 2444 dwc2_writel(GRSTCTL_TXFNUM(idx) | GRSTCTL_TXFFLSH,
2445 hsotg->regs + GRSTCTL);
9c39ddc6
AT		 2446
2447 /* wait until the fifo is flushed */
2448 timeout = 100;
2449
2450 while (1) {
95c8bc36 2451 val = dwc2_readl(hsotg->regs + GRSTCTL);
9c39ddc6 2452
47a1685f 2453 if ((val & (GRSTCTL_TXFFLSH)) == 0)
9c39ddc6
AT		 2454 break;
2455
2456 if (--timeout == 0) {
2457 dev_err(hsotg->dev,
2458 "%s: timeout flushing fifo (GRSTCTL=%08x)\n",
2459 __func__, val);
e0cbe595 2460 break;
9c39ddc6
AT		 2461 }
2462
2463 udelay(1);
2464 }
2465}
5b7d70c6
BD		 2466
2467/**
1f91b4cc 2468 * dwc2_hsotg_trytx - check to see if anything needs transmitting
5b7d70c6
BD		 2469 * @hsotg: The driver state
2470 * @hs_ep: The driver endpoint to check.
2471 *
2472 * Check to see if there is a request that has data to send, and if so
2473 * make an attempt to write data into the FIFO.
2474 */
1f91b4cc 2475static int dwc2_hsotg_trytx(struct dwc2_hsotg *hsotg,
9da51974 2476 struct dwc2_hsotg_ep *hs_ep)
5b7d70c6 2477{
1f91b4cc 2478 struct dwc2_hsotg_req *hs_req = hs_ep->req;
5b7d70c6 2479
afcf4169
RB		 2480 if (!hs_ep->dir_in || !hs_req) {
2481 /**
2482 * if request is not enqueued, we disable interrupts
2483 * for endpoints, excepting ep0
2484 */
2485 if (hs_ep->index != 0)
1f91b4cc 2486 dwc2_hsotg_ctrl_epint(hsotg, hs_ep->index,
9da51974 2487 hs_ep->dir_in, 0);
5b7d70c6 2488 return 0;
afcf4169 2489 }
5b7d70c6
BD		 2490
2491 if (hs_req->req.actual < hs_req->req.length) {
2492 dev_dbg(hsotg->dev, "trying to write more for ep%d\n",
2493 hs_ep->index);
1f91b4cc 2494 return dwc2_hsotg_write_fifo(hsotg, hs_ep, hs_req);
5b7d70c6
BD		 2495 }
2496
2497 return 0;
2498}
2499
2500/**
1f91b4cc 2501 * dwc2_hsotg_complete_in - complete IN transfer
5b7d70c6
BD		 2502 * @hsotg: The device state.
2503 * @hs_ep: The endpoint that has just completed.
2504 *
2505 * An IN transfer has been completed, update the transfer's state and then
2506 * call the relevant completion routines.
2507 */
1f91b4cc 2508static void dwc2_hsotg_complete_in(struct dwc2_hsotg *hsotg,
9da51974 2509 struct dwc2_hsotg_ep *hs_ep)
5b7d70c6 2510{
1f91b4cc 2511 struct dwc2_hsotg_req *hs_req = hs_ep->req;
95c8bc36 2512 u32 epsize = dwc2_readl(hsotg->regs + DIEPTSIZ(hs_ep->index));
5b7d70c6
BD		 2513 int size_left, size_done;
2514
2515 if (!hs_req) {
2516 dev_dbg(hsotg->dev, "XferCompl but no req\n");
2517 return;
2518 }
2519
d3ca0259 2520 /* Finish ZLP handling for IN EP0 transactions */
fe0b94ab
MYK		 2521 if (hs_ep->index == 0 && hsotg->ep0_state == DWC2_EP0_STATUS_IN) {
2522 dev_dbg(hsotg->dev, "zlp packet sent\n");
c3b22fe2
RK		 2523
2524 /*
2525 * While send zlp for DWC2_EP0_STATUS_IN EP direction was
2526 * changed to IN. Change back to complete OUT transfer request
2527 */
2528 hs_ep->dir_in = 0;
2529
1f91b4cc 2530 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0);
9e14d0a5
GH		 2531 if (hsotg->test_mode) {
2532 int ret;
2533
1f91b4cc 2534 ret = dwc2_hsotg_set_test_mode(hsotg, hsotg->test_mode);
9e14d0a5
GH		 2535 if (ret < 0) {
2536 dev_dbg(hsotg->dev, "Invalid Test #%d\n",
9da51974 2537 hsotg->test_mode);
1f91b4cc 2538 dwc2_hsotg_stall_ep0(hsotg);
9e14d0a5
GH		 2539 return;
2540 }
2541 }
1f91b4cc 2542 dwc2_hsotg_enqueue_setup(hsotg);
d3ca0259
LM		 2543 return;
2544 }
2545
8b9bc460
LM		 2546 /*
2547 * Calculate the size of the transfer by checking how much is left
5b7d70c6
BD		 2548 * in the endpoint size register and then working it out from
2549 * the amount we loaded for the transfer.
2550 *
2551 * We do this even for DMA, as the transfer may have incremented
2552 * past the end of the buffer (DMA transfers are always 32bit
2553 * aligned).
2554 */
aa3e8bc8
VA		 2555 if (using_desc_dma(hsotg)) {
2556 size_left = dwc2_gadget_get_xfersize_ddma(hs_ep);
2557 if (size_left < 0)
2558 dev_err(hsotg->dev, "error parsing DDMA results %d\n",
2559 size_left);
2560 } else {
2561 size_left = DXEPTSIZ_XFERSIZE_GET(epsize);
2562 }
5b7d70c6
BD		 2563
2564 size_done = hs_ep->size_loaded - size_left;
2565 size_done += hs_ep->last_load;
2566
2567 if (hs_req->req.actual != size_done)
2568 dev_dbg(hsotg->dev, "%s: adjusting size done %d => %d\n",
2569 __func__, hs_req->req.actual, size_done);
2570
2571 hs_req->req.actual = size_done;
d3ca0259
LM		 2572 dev_dbg(hsotg->dev, "req->length:%d req->actual:%d req->zero:%d\n",
2573 hs_req->req.length, hs_req->req.actual, hs_req->req.zero);
2574
5b7d70c6
BD		 2575 if (!size_left && hs_req->req.actual < hs_req->req.length) {
2576 dev_dbg(hsotg->dev, "%s trying more for req...\n", __func__);
1f91b4cc 2577 dwc2_hsotg_start_req(hsotg, hs_ep, hs_req, true);
fe0b94ab
MYK		 2578 return;
2579 }
2580
f71b5e25 2581 /* Zlp for all endpoints, for ep0 only in DATA IN stage */
8a20fa45 2582 if (hs_ep->send_zlp) {
1f91b4cc 2583 dwc2_hsotg_program_zlp(hsotg, hs_ep);
8a20fa45 2584 hs_ep->send_zlp = 0;
f71b5e25
MYK		 2585 /* transfer will be completed on next complete interrupt */
2586 return;
2587 }
2588
fe0b94ab
MYK		 2589 if (hs_ep->index == 0 && hsotg->ep0_state == DWC2_EP0_DATA_IN) {
2590 /* Move to STATUS OUT */
1f91b4cc 2591 dwc2_hsotg_ep0_zlp(hsotg, false);
fe0b94ab
MYK		 2592 return;
2593 }
2594
1f91b4cc 2595 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0);
5b7d70c6
BD		 2596}
2597
32601588
VM		 2598/**
2599 * dwc2_gadget_read_ep_interrupts - reads interrupts for given ep
2600 * @hsotg: The device state.
2601 * @idx: Index of ep.
2602 * @dir_in: Endpoint direction 1-in 0-out.
2603 *
2604 * Reads for endpoint with given index and direction, by masking
2605 * epint_reg with coresponding mask.
2606 */
2607static u32 dwc2_gadget_read_ep_interrupts(struct dwc2_hsotg *hsotg,
2608 unsigned int idx, int dir_in)
2609{
2610 u32 epmsk_reg = dir_in ? DIEPMSK : DOEPMSK;
2611 u32 epint_reg = dir_in ? DIEPINT(idx) : DOEPINT(idx);
2612 u32 ints;
2613 u32 mask;
2614 u32 diepempmsk;
2615
2616 mask = dwc2_readl(hsotg->regs + epmsk_reg);
2617 diepempmsk = dwc2_readl(hsotg->regs + DIEPEMPMSK);
2618 mask |= ((diepempmsk >> idx) & 0x1) ? DIEPMSK_TXFIFOEMPTY : 0;
2619 mask |= DXEPINT_SETUP_RCVD;
2620
2621 ints = dwc2_readl(hsotg->regs + epint_reg);
2622 ints &= mask;
2623 return ints;
2624}
2625
bd9971f0
VM		 2626/**
2627 * dwc2_gadget_handle_ep_disabled - handle DXEPINT_EPDISBLD
2628 * @hs_ep: The endpoint on which interrupt is asserted.
2629 *
2630 * This interrupt indicates that the endpoint has been disabled per the
2631 * application's request.
2632 *
2633 * For IN endpoints flushes txfifo, in case of BULK clears DCTL_CGNPINNAK,
2634 * in case of ISOC completes current request.
2635 *
2636 * For ISOC-OUT endpoints completes expired requests. If there is remaining
2637 * request starts it.
2638 */
2639static void dwc2_gadget_handle_ep_disabled(struct dwc2_hsotg_ep *hs_ep)
2640{
2641 struct dwc2_hsotg *hsotg = hs_ep->parent;
2642 struct dwc2_hsotg_req *hs_req;
2643 unsigned char idx = hs_ep->index;
2644 int dir_in = hs_ep->dir_in;
2645 u32 epctl_reg = dir_in ? DIEPCTL(idx) : DOEPCTL(idx);
2646 int dctl = dwc2_readl(hsotg->regs + DCTL);
2647
2648 dev_dbg(hsotg->dev, "%s: EPDisbld\n", __func__);
2649
2650 if (dir_in) {
2651 int epctl = dwc2_readl(hsotg->regs + epctl_reg);
2652
2653 dwc2_hsotg_txfifo_flush(hsotg, hs_ep->fifo_index);
2654
2655 if (hs_ep->isochronous) {
2656 dwc2_hsotg_complete_in(hsotg, hs_ep);
2657 return;
2658 }
2659
2660 if ((epctl & DXEPCTL_STALL) && (epctl & DXEPCTL_EPTYPE_BULK)) {
2661 int dctl = dwc2_readl(hsotg->regs + DCTL);
2662
2663 dctl |= DCTL_CGNPINNAK;
2664 dwc2_writel(dctl, hsotg->regs + DCTL);
2665 }
2666 return;
2667 }
2668
2669 if (dctl & DCTL_GOUTNAKSTS) {
2670 dctl |= DCTL_CGOUTNAK;
2671 dwc2_writel(dctl, hsotg->regs + DCTL);
2672 }
2673
2674 if (!hs_ep->isochronous)
2675 return;
2676
2677 if (list_empty(&hs_ep->queue)) {
2678 dev_dbg(hsotg->dev, "%s: complete_ep 0x%p, ep->queue empty!\n",
2679 __func__, hs_ep);
2680 return;
2681 }
2682
2683 do {
2684 hs_req = get_ep_head(hs_ep);
2685 if (hs_req)
2686 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req,
2687 -ENODATA);
2688 dwc2_gadget_incr_frame_num(hs_ep);
2689 } while (dwc2_gadget_target_frame_elapsed(hs_ep));
2690
2691 dwc2_gadget_start_next_request(hs_ep);
2692}
2693
5321922c
VM		 2694/**
2695 * dwc2_gadget_handle_out_token_ep_disabled - handle DXEPINT_OUTTKNEPDIS
2696 * @hs_ep: The endpoint on which interrupt is asserted.
2697 *
2698 * This is starting point for ISOC-OUT transfer, synchronization done with
2699 * first out token received from host while corresponding EP is disabled.
2700 *
2701 * Device does not know initial frame in which out token will come. For this
2702 * HW generates OUTTKNEPDIS - out token is received while EP is disabled. Upon
2703 * getting this interrupt SW starts calculation for next transfer frame.
2704 */
2705static void dwc2_gadget_handle_out_token_ep_disabled(struct dwc2_hsotg_ep *ep)
2706{
2707 struct dwc2_hsotg *hsotg = ep->parent;
2708 int dir_in = ep->dir_in;
2709 u32 doepmsk;
540ccba0 2710 u32 tmp;
5321922c
VM		 2711
2712 if (dir_in || !ep->isochronous)
2713 return;
2714
540ccba0
VA		 2715 /*
2716 * Store frame in which irq was asserted here, as
2717 * it can change while completing request below.
2718 */
2719 tmp = dwc2_hsotg_read_frameno(hsotg);
2720
5321922c
VM		 2721 dwc2_hsotg_complete_request(hsotg, ep, get_ep_head(ep), -ENODATA);
2722
540ccba0
VA		 2723 if (using_desc_dma(hsotg)) {
2724 if (ep->target_frame == TARGET_FRAME_INITIAL) {
2725 /* Start first ISO Out */
2726 ep->target_frame = tmp;
2727 dwc2_gadget_start_isoc_ddma(ep);
2728 }
2729 return;
2730 }
2731
5321922c
VM		 2732 if (ep->interval > 1 &&
2733 ep->target_frame == TARGET_FRAME_INITIAL) {
2734 u32 dsts;
2735 u32 ctrl;
2736
2737 dsts = dwc2_readl(hsotg->regs + DSTS);
2738 ep->target_frame = dwc2_hsotg_read_frameno(hsotg);
2739 dwc2_gadget_incr_frame_num(ep);
2740
2741 ctrl = dwc2_readl(hsotg->regs + DOEPCTL(ep->index));
2742 if (ep->target_frame & 0x1)
2743 ctrl |= DXEPCTL_SETODDFR;
2744 else
2745 ctrl |= DXEPCTL_SETEVENFR;
2746
2747 dwc2_writel(ctrl, hsotg->regs + DOEPCTL(ep->index));
2748 }
2749
2750 dwc2_gadget_start_next_request(ep);
2751 doepmsk = dwc2_readl(hsotg->regs + DOEPMSK);
2752 doepmsk &= ~DOEPMSK_OUTTKNEPDISMSK;
2753 dwc2_writel(doepmsk, hsotg->regs + DOEPMSK);
2754}
2755
2756/**
38beaec6
JY		 2757 * dwc2_gadget_handle_nak - handle NAK interrupt
2758 * @hs_ep: The endpoint on which interrupt is asserted.
2759 *
2760 * This is starting point for ISOC-IN transfer, synchronization done with
2761 * first IN token received from host while corresponding EP is disabled.
2762 *
2763 * Device does not know when first one token will arrive from host. On first
2764 * token arrival HW generates 2 interrupts: 'in token received while FIFO empty'
2765 * and 'NAK'. NAK interrupt for ISOC-IN means that token has arrived and ZLP was
2766 * sent in response to that as there was no data in FIFO. SW is basing on this
2767 * interrupt to obtain frame in which token has come and then based on the
2768 * interval calculates next frame for transfer.
2769 */
5321922c
VM		 2770static void dwc2_gadget_handle_nak(struct dwc2_hsotg_ep *hs_ep)
2771{
2772 struct dwc2_hsotg *hsotg = hs_ep->parent;
2773 int dir_in = hs_ep->dir_in;
2774
2775 if (!dir_in || !hs_ep->isochronous)
2776 return;
2777
2778 if (hs_ep->target_frame == TARGET_FRAME_INITIAL) {
2779 hs_ep->target_frame = dwc2_hsotg_read_frameno(hsotg);
540ccba0
VA		 2780
2781 if (using_desc_dma(hsotg)) {
2782 dwc2_gadget_start_isoc_ddma(hs_ep);
2783 return;
2784 }
2785
5321922c
VM		 2786 if (hs_ep->interval > 1) {
2787 u32 ctrl = dwc2_readl(hsotg->regs +
2788 DIEPCTL(hs_ep->index));
2789 if (hs_ep->target_frame & 0x1)
2790 ctrl |= DXEPCTL_SETODDFR;
2791 else
2792 ctrl |= DXEPCTL_SETEVENFR;
2793
2794 dwc2_writel(ctrl, hsotg->regs + DIEPCTL(hs_ep->index));
2795 }
2796
2797 dwc2_hsotg_complete_request(hsotg, hs_ep,
2798 get_ep_head(hs_ep), 0);
2799 }
2800
2801 dwc2_gadget_incr_frame_num(hs_ep);
2802}
2803
5b7d70c6 2804/**
1f91b4cc 2805 * dwc2_hsotg_epint - handle an in/out endpoint interrupt
5b7d70c6
BD		 2806 * @hsotg: The driver state
2807 * @idx: The index for the endpoint (0..15)
2808 * @dir_in: Set if this is an IN endpoint
2809 *
2810 * Process and clear any interrupt pending for an individual endpoint
8b9bc460 2811 */
1f91b4cc 2812static void dwc2_hsotg_epint(struct dwc2_hsotg *hsotg, unsigned int idx,
9da51974 2813 int dir_in)
5b7d70c6 2814{
1f91b4cc 2815 struct dwc2_hsotg_ep *hs_ep = index_to_ep(hsotg, idx, dir_in);
94cb8fd6
LM		 2816 u32 epint_reg = dir_in ? DIEPINT(idx) : DOEPINT(idx);
2817 u32 epctl_reg = dir_in ? DIEPCTL(idx) : DOEPCTL(idx);
2818 u32 epsiz_reg = dir_in ? DIEPTSIZ(idx) : DOEPTSIZ(idx);
5b7d70c6 2819 u32 ints;
1479e841 2820 u32 ctrl;
5b7d70c6 2821
32601588 2822 ints = dwc2_gadget_read_ep_interrupts(hsotg, idx, dir_in);
95c8bc36 2823 ctrl = dwc2_readl(hsotg->regs + epctl_reg);
5b7d70c6 2824
a3395f0d 2825 /* Clear endpoint interrupts */
95c8bc36 2826 dwc2_writel(ints, hsotg->regs + epint_reg);
a3395f0d 2827
c6f5c050
MYK		 2828 if (!hs_ep) {
2829 dev_err(hsotg->dev, "%s:Interrupt for unconfigured ep%d(%s)\n",
9da51974 2830 __func__, idx, dir_in ? "in" : "out");
c6f5c050
MYK		 2831 return;
2832 }
2833
5b7d70c6
BD		 2834 dev_dbg(hsotg->dev, "%s: ep%d(%s) DxEPINT=0x%08x\n",
2835 __func__, idx, dir_in ? "in" : "out", ints);
2836
b787d755
MYK		 2837 /* Don't process XferCompl interrupt if it is a setup packet */
2838 if (idx == 0 && (ints & (DXEPINT_SETUP | DXEPINT_SETUP_RCVD)))
2839 ints &= ~DXEPINT_XFERCOMPL;
2840
f0afdb42
VA		 2841 /*
2842 * Don't process XferCompl interrupt in DDMA if EP0 is still in SETUP
2843 * stage and xfercomplete was generated without SETUP phase done
2844 * interrupt. SW should parse received setup packet only after host's
2845 * exit from setup phase of control transfer.
2846 */
2847 if (using_desc_dma(hsotg) && idx == 0 && !hs_ep->dir_in &&
2848 hsotg->ep0_state == DWC2_EP0_SETUP && !(ints & DXEPINT_SETUP))
2849 ints &= ~DXEPINT_XFERCOMPL;
2850
837e9f00 2851 if (ints & DXEPINT_XFERCOMPL) {
5b7d70c6 2852 dev_dbg(hsotg->dev,
47a1685f 2853 "%s: XferCompl: DxEPCTL=0x%08x, DXEPTSIZ=%08x\n",
95c8bc36
AS		 2854 __func__, dwc2_readl(hsotg->regs + epctl_reg),
2855 dwc2_readl(hsotg->regs + epsiz_reg));
5b7d70c6 2856
540ccba0
VA		 2857 /* In DDMA handle isochronous requests separately */
2858 if (using_desc_dma(hsotg) && hs_ep->isochronous) {
2859 dwc2_gadget_complete_isoc_request_ddma(hs_ep);
2860 /* Try to start next isoc request */
2861 dwc2_gadget_start_next_isoc_ddma(hs_ep);
2862 } else if (dir_in) {
2863 /*
2864 * We get OutDone from the FIFO, so we only
2865 * need to look at completing IN requests here
2866 * if operating slave mode
2867 */
837e9f00
VM		 2868 if (hs_ep->isochronous && hs_ep->interval > 1)
2869 dwc2_gadget_incr_frame_num(hs_ep);
2870
1f91b4cc 2871 dwc2_hsotg_complete_in(hsotg, hs_ep);
837e9f00
VM		 2872 if (ints & DXEPINT_NAKINTRPT)
2873 ints &= ~DXEPINT_NAKINTRPT;
5b7d70c6 2874
c9a64ea8 2875 if (idx == 0 && !hs_ep->req)
1f91b4cc 2876 dwc2_hsotg_enqueue_setup(hsotg);
5b7d70c6 2877 } else if (using_dma(hsotg)) {
8b9bc460
LM		 2878 /*
2879 * We're using DMA, we need to fire an OutDone here
2880 * as we ignore the RXFIFO.
2881 */
837e9f00
VM		 2882 if (hs_ep->isochronous && hs_ep->interval > 1)
2883 dwc2_gadget_incr_frame_num(hs_ep);
5b7d70c6 2884
1f91b4cc 2885 dwc2_hsotg_handle_outdone(hsotg, idx);
5b7d70c6 2886 }
5b7d70c6
BD		 2887 }
2888
bd9971f0
VM		 2889 if (ints & DXEPINT_EPDISBLD)
2890 dwc2_gadget_handle_ep_disabled(hs_ep);
9c39ddc6 2891
5321922c
VM		 2892 if (ints & DXEPINT_OUTTKNEPDIS)
2893 dwc2_gadget_handle_out_token_ep_disabled(hs_ep);
2894
2895 if (ints & DXEPINT_NAKINTRPT)
2896 dwc2_gadget_handle_nak(hs_ep);
2897
47a1685f 2898 if (ints & DXEPINT_AHBERR)
5b7d70c6 2899 dev_dbg(hsotg->dev, "%s: AHBErr\n", __func__);
5b7d70c6 2900
47a1685f 2901 if (ints & DXEPINT_SETUP) { /* Setup or Timeout */
5b7d70c6
BD		 2902 dev_dbg(hsotg->dev, "%s: Setup/Timeout\n", __func__);
2903
2904 if (using_dma(hsotg) && idx == 0) {
8b9bc460
LM		 2905 /*
2906 * this is the notification we've received a
5b7d70c6
BD		 2907 * setup packet. In non-DMA mode we'd get this
2908 * from the RXFIFO, instead we need to process
8b9bc460
LM		 2909 * the setup here.
2910 */
5b7d70c6
BD		 2911
2912 if (dir_in)
2913 WARN_ON_ONCE(1);
2914 else
1f91b4cc 2915 dwc2_hsotg_handle_outdone(hsotg, 0);
5b7d70c6 2916 }
5b7d70c6
BD		 2917 }
2918
ef750c71 2919 if (ints & DXEPINT_STSPHSERCVD) {
9d9a6b07
VA		 2920 dev_dbg(hsotg->dev, "%s: StsPhseRcvd\n", __func__);
2921
ef750c71
VA		 2922 /* Move to STATUS IN for DDMA */
2923 if (using_desc_dma(hsotg))
2924 dwc2_hsotg_ep0_zlp(hsotg, true);
2925 }
2926
47a1685f 2927 if (ints & DXEPINT_BACK2BACKSETUP)
5b7d70c6 2928 dev_dbg(hsotg->dev, "%s: B2BSetup/INEPNakEff\n", __func__);
5b7d70c6 2929
540ccba0
VA		 2930 if (ints & DXEPINT_BNAINTR) {
2931 dev_dbg(hsotg->dev, "%s: BNA interrupt\n", __func__);
2932
2933 /*
2934 * Try to start next isoc request, if any.
2935 * Sometimes the endpoint remains enabled after BNA interrupt
2936 * assertion, which is not expected, hence we can enter here
2937 * couple of times.
2938 */
2939 if (hs_ep->isochronous)
2940 dwc2_gadget_start_next_isoc_ddma(hs_ep);
2941 }
2942
1479e841 2943 if (dir_in && !hs_ep->isochronous) {
8b9bc460 2944 /* not sure if this is important, but we'll clear it anyway */
26ddef5d 2945 if (ints & DXEPINT_INTKNTXFEMP) {
5b7d70c6
BD		 2946 dev_dbg(hsotg->dev, "%s: ep%d: INTknTXFEmpMsk\n",
2947 __func__, idx);
5b7d70c6
BD		 2948 }
2949
2950 /* this probably means something bad is happening */
26ddef5d 2951 if (ints & DXEPINT_INTKNEPMIS) {
5b7d70c6
BD		 2952 dev_warn(hsotg->dev, "%s: ep%d: INTknEP\n",
2953 __func__, idx);
5b7d70c6 2954 }
10aebc77
BD		 2955
2956 /* FIFO has space or is empty (see GAHBCFG) */
2957 if (hsotg->dedicated_fifos &&
26ddef5d 2958 ints & DXEPINT_TXFEMP) {
10aebc77
BD		 2959 dev_dbg(hsotg->dev, "%s: ep%d: TxFIFOEmpty\n",
2960 __func__, idx);
70fa030f 2961 if (!using_dma(hsotg))
1f91b4cc 2962 dwc2_hsotg_trytx(hsotg, hs_ep);
10aebc77 2963 }
5b7d70c6 2964 }
5b7d70c6
BD		 2965}
2966
2967/**
1f91b4cc 2968 * dwc2_hsotg_irq_enumdone - Handle EnumDone interrupt (enumeration done)
5b7d70c6
BD		 2969 * @hsotg: The device state.
2970 *
2971 * Handle updating the device settings after the enumeration phase has
2972 * been completed.
8b9bc460 2973 */
1f91b4cc 2974static void dwc2_hsotg_irq_enumdone(struct dwc2_hsotg *hsotg)
5b7d70c6 2975{
95c8bc36 2976 u32 dsts = dwc2_readl(hsotg->regs + DSTS);
9b2667f1 2977 int ep0_mps = 0, ep_mps = 8;
5b7d70c6 2978
8b9bc460
LM		 2979 /*
2980 * This should signal the finish of the enumeration phase
5b7d70c6 2981 * of the USB handshaking, so we should now know what rate
8b9bc460
LM		 2982 * we connected at.
2983 */
5b7d70c6
BD		 2984
2985 dev_dbg(hsotg->dev, "EnumDone (DSTS=0x%08x)\n", dsts);
2986
8b9bc460
LM		 2987 /*
2988 * note, since we're limited by the size of transfer on EP0, and
5b7d70c6 2989 * it seems IN transfers must be a even number of packets we do
8b9bc460
LM		 2990 * not advertise a 64byte MPS on EP0.
2991 */
5b7d70c6
BD		 2992
2993 /* catch both EnumSpd_FS and EnumSpd_FS48 */
6d76c92c 2994 switch ((dsts & DSTS_ENUMSPD_MASK) >> DSTS_ENUMSPD_SHIFT) {
47a1685f
DN		 2995 case DSTS_ENUMSPD_FS:
2996 case DSTS_ENUMSPD_FS48:
5b7d70c6 2997 hsotg->gadget.speed = USB_SPEED_FULL;
5b7d70c6 2998 ep0_mps = EP0_MPS_LIMIT;
295538ff 2999 ep_mps = 1023;
5b7d70c6
BD		 3000 break;
3001
47a1685f 3002 case DSTS_ENUMSPD_HS:
5b7d70c6 3003 hsotg->gadget.speed = USB_SPEED_HIGH;
5b7d70c6 3004 ep0_mps = EP0_MPS_LIMIT;
295538ff 3005 ep_mps = 1024;
5b7d70c6
BD		 3006 break;
3007
47a1685f 3008 case DSTS_ENUMSPD_LS:
5b7d70c6 3009 hsotg->gadget.speed = USB_SPEED_LOW;
552d940f
VM		 3010 ep0_mps = 8;
3011 ep_mps = 8;
8b9bc460
LM		 3012 /*
3013 * note, we don't actually support LS in this driver at the
5b7d70c6
BD		 3014 * moment, and the documentation seems to imply that it isn't
3015 * supported by the PHYs on some of the devices.
3016 */
3017 break;
3018 }
e538dfda
MN		 3019 dev_info(hsotg->dev, "new device is %s\n",
3020 usb_speed_string(hsotg->gadget.speed));
5b7d70c6 3021
8b9bc460
LM		 3022 /*
3023 * we should now know the maximum packet size for an
3024 * endpoint, so set the endpoints to a default value.
3025 */
5b7d70c6
BD		 3026
3027 if (ep0_mps) {
3028 int i;
c6f5c050 3029 /* Initialize ep0 for both in and out directions */
ee2c40de
VM		 3030 dwc2_hsotg_set_ep_maxpacket(hsotg, 0, ep0_mps, 0, 1);
3031 dwc2_hsotg_set_ep_maxpacket(hsotg, 0, ep0_mps, 0, 0);
c6f5c050
MYK		 3032 for (i = 1; i < hsotg->num_of_eps; i++) {
3033 if (hsotg->eps_in[i])
ee2c40de
VM		 3034 dwc2_hsotg_set_ep_maxpacket(hsotg, i, ep_mps,
3035 0, 1);
c6f5c050 3036 if (hsotg->eps_out[i])
ee2c40de
VM		 3037 dwc2_hsotg_set_ep_maxpacket(hsotg, i, ep_mps,
3038 0, 0);
c6f5c050 3039 }
5b7d70c6
BD		 3040 }
3041
3042 /* ensure after enumeration our EP0 is active */
3043
1f91b4cc 3044 dwc2_hsotg_enqueue_setup(hsotg);
5b7d70c6
BD		 3045
3046 dev_dbg(hsotg->dev, "EP0: DIEPCTL0=0x%08x, DOEPCTL0=0x%08x\n",
95c8bc36
AS		 3047 dwc2_readl(hsotg->regs + DIEPCTL0),
3048 dwc2_readl(hsotg->regs + DOEPCTL0));
5b7d70c6
BD		 3049}
3050
3051/**
3052 * kill_all_requests - remove all requests from the endpoint's queue
3053 * @hsotg: The device state.
3054 * @ep: The endpoint the requests may be on.
3055 * @result: The result code to use.
5b7d70c6
BD		 3056 *
3057 * Go through the requests on the given endpoint and mark them
3058 * completed with the given result code.
3059 */
941fcce4 3060static void kill_all_requests(struct dwc2_hsotg *hsotg,
1f91b4cc 3061 struct dwc2_hsotg_ep *ep,
6b448af4 3062 int result)
5b7d70c6 3063{
1f91b4cc 3064 struct dwc2_hsotg_req *req, *treq;
9da51974 3065 unsigned int size;
5b7d70c6 3066
6b448af4 3067 ep->req = NULL;
5b7d70c6 3068
6b448af4 3069 list_for_each_entry_safe(req, treq, &ep->queue, queue)
1f91b4cc 3070 dwc2_hsotg_complete_request(hsotg, ep, req,
9da51974 3071 result);
6b448af4 3072
b203d0a2
RB		 3073 if (!hsotg->dedicated_fifos)
3074 return;
ad674a15 3075 size = (dwc2_readl(hsotg->regs + DTXFSTS(ep->fifo_index)) & 0xffff) * 4;
b203d0a2 3076 if (size < ep->fifo_size)
1f91b4cc 3077 dwc2_hsotg_txfifo_flush(hsotg, ep->fifo_index);
5b7d70c6
BD		 3078}
3079
5b7d70c6 3080/**
1f91b4cc 3081 * dwc2_hsotg_disconnect - disconnect service
5b7d70c6
BD		 3082 * @hsotg: The device state.
3083 *
5e891342
LM		 3084 * The device has been disconnected. Remove all current
3085 * transactions and signal the gadget driver that this
3086 * has happened.
8b9bc460 3087 */
1f91b4cc 3088void dwc2_hsotg_disconnect(struct dwc2_hsotg *hsotg)
5b7d70c6 3089{
9da51974 3090 unsigned int ep;
5b7d70c6 3091
4ace06e8
MS		 3092 if (!hsotg->connected)
3093 return;
3094
3095 hsotg->connected = 0;
9e14d0a5 3096 hsotg->test_mode = 0;
c6f5c050
MYK		 3097
3098 for (ep = 0; ep < hsotg->num_of_eps; ep++) {
3099 if (hsotg->eps_in[ep])
3100 kill_all_requests(hsotg, hsotg->eps_in[ep],
9da51974 3101 -ESHUTDOWN);
c6f5c050
MYK		 3102 if (hsotg->eps_out[ep])
3103 kill_all_requests(hsotg, hsotg->eps_out[ep],
9da51974 3104 -ESHUTDOWN);
c6f5c050 3105 }
5b7d70c6
BD		 3106
3107 call_gadget(hsotg, disconnect);
065d3931 3108 hsotg->lx_state = DWC2_L3;
5b7d70c6
BD		 3109}
3110
3111/**
1f91b4cc 3112 * dwc2_hsotg_irq_fifoempty - TX FIFO empty interrupt handler
5b7d70c6
BD		 3113 * @hsotg: The device state:
3114 * @periodic: True if this is a periodic FIFO interrupt
3115 */
1f91b4cc 3116static void dwc2_hsotg_irq_fifoempty(struct dwc2_hsotg *hsotg, bool periodic)
5b7d70c6 3117{
1f91b4cc 3118 struct dwc2_hsotg_ep *ep;
5b7d70c6
BD		 3119 int epno, ret;
3120
3121 /* look through for any more data to transmit */
b3f489b2 3122 for (epno = 0; epno < hsotg->num_of_eps; epno++) {
c6f5c050
MYK		 3123 ep = index_to_ep(hsotg, epno, 1);
3124
3125 if (!ep)
3126 continue;
5b7d70c6
BD		 3127
3128 if (!ep->dir_in)
3129 continue;
3130
3131 if ((periodic && !ep->periodic) ||
3132 (!periodic && ep->periodic))
3133 continue;
3134
1f91b4cc 3135 ret = dwc2_hsotg_trytx(hsotg, ep);
5b7d70c6
BD		 3136 if (ret < 0)
3137 break;
3138 }
3139}
3140
5b7d70c6 3141/* IRQ flags which will trigger a retry around the IRQ loop */
47a1685f
DN		 3142#define IRQ_RETRY_MASK (GINTSTS_NPTXFEMP | \
3143 GINTSTS_PTXFEMP | \
3144 GINTSTS_RXFLVL)
5b7d70c6 3145
8b9bc460 3146/**
1f91b4cc 3147 * dwc2_hsotg_core_init - issue softreset to the core
8b9bc460
LM		 3148 * @hsotg: The device state
3149 *
3150 * Issue a soft reset to the core, and await the core finishing it.
3151 */
1f91b4cc 3152void dwc2_hsotg_core_init_disconnected(struct dwc2_hsotg *hsotg,
9da51974 3153 bool is_usb_reset)
308d734e 3154{
1ee6903b 3155 u32 intmsk;
643cc4de 3156 u32 val;
ecd9a7ad 3157 u32 usbcfg;
79c3b5bb 3158 u32 dcfg = 0;
643cc4de 3159
5390d438
MYK		 3160 /* Kill any ep0 requests as controller will be reinitialized */
3161 kill_all_requests(hsotg, hsotg->eps_out[0], -ECONNRESET);
3162
643cc4de 3163 if (!is_usb_reset)
241729ba 3164 if (dwc2_core_reset(hsotg))
86de4895 3165 return;
308d734e
LM		 3166
3167 /*
3168 * we must now enable ep0 ready for host detection and then
3169 * set configuration.
3170 */
3171
ecd9a7ad
PR		 3172 /* keep other bits untouched (so e.g. forced modes are not lost) */
3173 usbcfg = dwc2_readl(hsotg->regs + GUSBCFG);
3174 usbcfg &= ~(GUSBCFG_TOUTCAL_MASK | GUSBCFG_PHYIF16 | GUSBCFG_SRPCAP |
3175 GUSBCFG_HNPCAP);
3176
79c3b5bb 3177 if (hsotg->params.phy_type == DWC2_PHY_TYPE_PARAM_FS &&
38e9002b
VM		 3178 (hsotg->params.speed == DWC2_SPEED_PARAM_FULL ||
3179 hsotg->params.speed == DWC2_SPEED_PARAM_LOW)) {
79c3b5bb
VA		 3180 /* FS/LS Dedicated Transceiver Interface */
3181 usbcfg |= GUSBCFG_PHYSEL;
3182 } else {
3183 /* set the PLL on, remove the HNP/SRP and set the PHY */
3184 val = (hsotg->phyif == GUSBCFG_PHYIF8) ? 9 : 5;
3185 usbcfg |= hsotg->phyif | GUSBCFG_TOUTCAL(7) |
3186 (val << GUSBCFG_USBTRDTIM_SHIFT);
3187 }
ecd9a7ad 3188 dwc2_writel(usbcfg, hsotg->regs + GUSBCFG);
308d734e 3189
1f91b4cc 3190 dwc2_hsotg_init_fifo(hsotg);
308d734e 3191
643cc4de
GH		 3192 if (!is_usb_reset)
3193 __orr32(hsotg->regs + DCTL, DCTL_SFTDISCON);
308d734e 3194
79c3b5bb 3195 dcfg |= DCFG_EPMISCNT(1);
38e9002b
VM		 3196
3197 switch (hsotg->params.speed) {
3198 case DWC2_SPEED_PARAM_LOW:
3199 dcfg |= DCFG_DEVSPD_LS;
3200 break;
3201 case DWC2_SPEED_PARAM_FULL:
79c3b5bb
VA		 3202 if (hsotg->params.phy_type == DWC2_PHY_TYPE_PARAM_FS)
3203 dcfg |= DCFG_DEVSPD_FS48;
3204 else
3205 dcfg |= DCFG_DEVSPD_FS;
38e9002b
VM		 3206 break;
3207 default:
79c3b5bb
VA		 3208 dcfg |= DCFG_DEVSPD_HS;
3209 }
38e9002b 3210
79c3b5bb 3211 dwc2_writel(dcfg, hsotg->regs + DCFG);
308d734e
LM		 3212
3213 /* Clear any pending OTG interrupts */
95c8bc36 3214 dwc2_writel(0xffffffff, hsotg->regs + GOTGINT);
308d734e
LM		 3215
3216 /* Clear any pending interrupts */
95c8bc36 3217 dwc2_writel(0xffffffff, hsotg->regs + GINTSTS);
1ee6903b 3218 intmsk = GINTSTS_ERLYSUSP | GINTSTS_SESSREQINT |
47a1685f 3219 GINTSTS_GOUTNAKEFF | GINTSTS_GINNAKEFF |
1ee6903b
GH		 3220 GINTSTS_USBRST | GINTSTS_RESETDET |
3221 GINTSTS_ENUMDONE | GINTSTS_OTGINT |
f4736701
VA		 3222 GINTSTS_USBSUSP | GINTSTS_WKUPINT;
3223
3224 if (!using_desc_dma(hsotg))
3225 intmsk |= GINTSTS_INCOMPL_SOIN | GINTSTS_INCOMPL_SOOUT;
1ee6903b 3226
bea8e86c 3227 if (hsotg->params.external_id_pin_ctl <= 0)
1ee6903b
GH		 3228 intmsk |= GINTSTS_CONIDSTSCHNG;
3229
3230 dwc2_writel(intmsk, hsotg->regs + GINTMSK);
308d734e 3231
a5c18f11 3232 if (using_dma(hsotg)) {
95c8bc36
AS		 3233 dwc2_writel(GAHBCFG_GLBL_INTR_EN | GAHBCFG_DMA_EN |
3234 (GAHBCFG_HBSTLEN_INCR4 << GAHBCFG_HBSTLEN_SHIFT),
3235 hsotg->regs + GAHBCFG);
a5c18f11
VA		 3236
3237 /* Set DDMA mode support in the core if needed */
3238 if (using_desc_dma(hsotg))
3239 __orr32(hsotg->regs + DCFG, DCFG_DESCDMA_EN);
3240
3241 } else {
95c8bc36
AS		 3242 dwc2_writel(((hsotg->dedicated_fifos) ?
3243 (GAHBCFG_NP_TXF_EMP_LVL |
3244 GAHBCFG_P_TXF_EMP_LVL) : 0) |
3245 GAHBCFG_GLBL_INTR_EN, hsotg->regs + GAHBCFG);
a5c18f11 3246 }
308d734e
LM		 3247
3248 /*
8acc8296
RB		 3249 * If INTknTXFEmpMsk is enabled, it's important to disable ep interrupts
3250 * when we have no data to transfer. Otherwise we get being flooded by
3251 * interrupts.
308d734e
LM		 3252 */
3253
95c8bc36 3254 dwc2_writel(((hsotg->dedicated_fifos && !using_dma(hsotg)) ?
6ff2e832 3255 DIEPMSK_TXFIFOEMPTY | DIEPMSK_INTKNTXFEMPMSK : 0) |
47a1685f 3256 DIEPMSK_EPDISBLDMSK | DIEPMSK_XFERCOMPLMSK |
837e9f00 3257 DIEPMSK_TIMEOUTMSK | DIEPMSK_AHBERRMSK,
47a1685f 3258 hsotg->regs + DIEPMSK);
308d734e
LM		 3259
3260 /*
3261 * don't need XferCompl, we get that from RXFIFO in slave mode. In
9d9a6b07 3262 * DMA mode we may need this and StsPhseRcvd.
308d734e 3263 */
9d9a6b07
VA		 3264 dwc2_writel((using_dma(hsotg) ? (DIEPMSK_XFERCOMPLMSK |
3265 DOEPMSK_STSPHSERCVDMSK) : 0) |
47a1685f 3266 DOEPMSK_EPDISBLDMSK | DOEPMSK_AHBERRMSK |
9d9a6b07 3267 DOEPMSK_SETUPMSK,
47a1685f 3268 hsotg->regs + DOEPMSK);
308d734e 3269
ec01f0b2
VA		 3270 /* Enable BNA interrupt for DDMA */
3271 if (using_desc_dma(hsotg))
3272 __orr32(hsotg->regs + DOEPMSK, DOEPMSK_BNAMSK);
3273
95c8bc36 3274 dwc2_writel(0, hsotg->regs + DAINTMSK);
308d734e
LM		 3275
3276 dev_dbg(hsotg->dev, "EP0: DIEPCTL0=0x%08x, DOEPCTL0=0x%08x\n",
95c8bc36
AS		 3277 dwc2_readl(hsotg->regs + DIEPCTL0),
3278 dwc2_readl(hsotg->regs + DOEPCTL0));
308d734e
LM		 3279
3280 /* enable in and out endpoint interrupts */
1f91b4cc 3281 dwc2_hsotg_en_gsint(hsotg, GINTSTS_OEPINT | GINTSTS_IEPINT);
308d734e
LM		 3282
3283 /*
3284 * Enable the RXFIFO when in slave mode, as this is how we collect
3285 * the data. In DMA mode, we get events from the FIFO but also
3286 * things we cannot process, so do not use it.
3287 */
3288 if (!using_dma(hsotg))
1f91b4cc 3289 dwc2_hsotg_en_gsint(hsotg, GINTSTS_RXFLVL);
308d734e
LM		 3290
3291 /* Enable interrupts for EP0 in and out */
1f91b4cc
FB		 3292 dwc2_hsotg_ctrl_epint(hsotg, 0, 0, 1);
3293 dwc2_hsotg_ctrl_epint(hsotg, 0, 1, 1);
308d734e 3294
643cc4de
GH		 3295 if (!is_usb_reset) {
3296 __orr32(hsotg->regs + DCTL, DCTL_PWRONPRGDONE);
3297 udelay(10); /* see openiboot */
3298 __bic32(hsotg->regs + DCTL, DCTL_PWRONPRGDONE);
3299 }
308d734e 3300
95c8bc36 3301 dev_dbg(hsotg->dev, "DCTL=0x%08x\n", dwc2_readl(hsotg->regs + DCTL));
308d734e
LM		 3302
3303 /*
94cb8fd6 3304 * DxEPCTL_USBActEp says RO in manual, but seems to be set by
308d734e
LM		 3305 * writing to the EPCTL register..
3306 */
3307
3308 /* set to read 1 8byte packet */
95c8bc36 3309 dwc2_writel(DXEPTSIZ_MC(1) | DXEPTSIZ_PKTCNT(1) |
47a1685f 3310 DXEPTSIZ_XFERSIZE(8), hsotg->regs + DOEPTSIZ0);
308d734e 3311
95c8bc36 3312 dwc2_writel(dwc2_hsotg_ep0_mps(hsotg->eps_out[0]->ep.maxpacket) |
47a1685f
DN		 3313 DXEPCTL_CNAK | DXEPCTL_EPENA |
3314 DXEPCTL_USBACTEP,
94cb8fd6 3315 hsotg->regs + DOEPCTL0);
308d734e
LM		 3316
3317 /* enable, but don't activate EP0in */
95c8bc36 3318 dwc2_writel(dwc2_hsotg_ep0_mps(hsotg->eps_out[0]->ep.maxpacket) |
47a1685f 3319 DXEPCTL_USBACTEP, hsotg->regs + DIEPCTL0);
308d734e 3320
1f91b4cc 3321 dwc2_hsotg_enqueue_setup(hsotg);
308d734e
LM		 3322
3323 dev_dbg(hsotg->dev, "EP0: DIEPCTL0=0x%08x, DOEPCTL0=0x%08x\n",
95c8bc36
AS		 3324 dwc2_readl(hsotg->regs + DIEPCTL0),
3325 dwc2_readl(hsotg->regs + DOEPCTL0));
308d734e
LM		 3326
3327 /* clear global NAKs */
643cc4de
GH		 3328 val = DCTL_CGOUTNAK | DCTL_CGNPINNAK;
3329 if (!is_usb_reset)
3330 val |= DCTL_SFTDISCON;
3331 __orr32(hsotg->regs + DCTL, val);
308d734e
LM		 3332
3333 /* must be at-least 3ms to allow bus to see disconnect */
3334 mdelay(3);
3335
065d3931 3336 hsotg->lx_state = DWC2_L0;
ad38dc5d
MS		 3337}
3338
1f91b4cc 3339static void dwc2_hsotg_core_disconnect(struct dwc2_hsotg *hsotg)
ad38dc5d
MS		 3340{
3341 /* set the soft-disconnect bit */
3342 __orr32(hsotg->regs + DCTL, DCTL_SFTDISCON);
3343}
ac3c81f3 3344
1f91b4cc 3345void dwc2_hsotg_core_connect(struct dwc2_hsotg *hsotg)
ad38dc5d 3346{
308d734e 3347 /* remove the soft-disconnect and let's go */
47a1685f 3348 __bic32(hsotg->regs + DCTL, DCTL_SFTDISCON);
308d734e
LM		 3349}
3350
381fc8f8
VM		 3351/**
3352 * dwc2_gadget_handle_incomplete_isoc_in - handle incomplete ISO IN Interrupt.
3353 * @hsotg: The device state:
3354 *
3355 * This interrupt indicates one of the following conditions occurred while
3356 * transmitting an ISOC transaction.
3357 * - Corrupted IN Token for ISOC EP.
3358 * - Packet not complete in FIFO.
3359 *
3360 * The following actions will be taken:
3361 * - Determine the EP
3362 * - Disable EP; when 'Endpoint Disabled' interrupt is received Flush FIFO
3363 */
3364static void dwc2_gadget_handle_incomplete_isoc_in(struct dwc2_hsotg *hsotg)
3365{
3366 struct dwc2_hsotg_ep *hs_ep;
3367 u32 epctrl;
3368 u32 idx;
3369
3370 dev_dbg(hsotg->dev, "Incomplete isoc in interrupt received:\n");
3371
3372 for (idx = 1; idx <= hsotg->num_of_eps; idx++) {
3373 hs_ep = hsotg->eps_in[idx];
3374 epctrl = dwc2_readl(hsotg->regs + DIEPCTL(idx));
3375 if ((epctrl & DXEPCTL_EPENA) && hs_ep->isochronous &&
3376 dwc2_gadget_target_frame_elapsed(hs_ep)) {
3377 epctrl |= DXEPCTL_SNAK;
3378 epctrl |= DXEPCTL_EPDIS;
3379 dwc2_writel(epctrl, hsotg->regs + DIEPCTL(idx));
3380 }
3381 }
3382
3383 /* Clear interrupt */
3384 dwc2_writel(GINTSTS_INCOMPL_SOIN, hsotg->regs + GINTSTS);
3385}
3386
3387/**
3388 * dwc2_gadget_handle_incomplete_isoc_out - handle incomplete ISO OUT Interrupt
3389 * @hsotg: The device state:
3390 *
3391 * This interrupt indicates one of the following conditions occurred while
3392 * transmitting an ISOC transaction.
3393 * - Corrupted OUT Token for ISOC EP.
3394 * - Packet not complete in FIFO.
3395 *
3396 * The following actions will be taken:
3397 * - Determine the EP
3398 * - Set DCTL_SGOUTNAK and unmask GOUTNAKEFF if target frame elapsed.
3399 */
3400static void dwc2_gadget_handle_incomplete_isoc_out(struct dwc2_hsotg *hsotg)
3401{
3402 u32 gintsts;
3403 u32 gintmsk;
3404 u32 epctrl;
3405 struct dwc2_hsotg_ep *hs_ep;
3406 int idx;
3407
3408 dev_dbg(hsotg->dev, "%s: GINTSTS_INCOMPL_SOOUT\n", __func__);
3409
3410 for (idx = 1; idx <= hsotg->num_of_eps; idx++) {
3411 hs_ep = hsotg->eps_out[idx];
3412 epctrl = dwc2_readl(hsotg->regs + DOEPCTL(idx));
3413 if ((epctrl & DXEPCTL_EPENA) && hs_ep->isochronous &&
3414 dwc2_gadget_target_frame_elapsed(hs_ep)) {
3415 /* Unmask GOUTNAKEFF interrupt */
3416 gintmsk = dwc2_readl(hsotg->regs + GINTMSK);
3417 gintmsk |= GINTSTS_GOUTNAKEFF;
3418 dwc2_writel(gintmsk, hsotg->regs + GINTMSK);
3419
3420 gintsts = dwc2_readl(hsotg->regs + GINTSTS);
3421 if (!(gintsts & GINTSTS_GOUTNAKEFF))
3422 __orr32(hsotg->regs + DCTL, DCTL_SGOUTNAK);
3423 }
3424 }
3425
3426 /* Clear interrupt */
3427 dwc2_writel(GINTSTS_INCOMPL_SOOUT, hsotg->regs + GINTSTS);
3428}
3429
5b7d70c6 3430/**
1f91b4cc 3431 * dwc2_hsotg_irq - handle device interrupt
5b7d70c6
BD		 3432 * @irq: The IRQ number triggered
3433 * @pw: The pw value when registered the handler.
3434 */
1f91b4cc 3435static irqreturn_t dwc2_hsotg_irq(int irq, void *pw)
5b7d70c6 3436{
941fcce4 3437 struct dwc2_hsotg *hsotg = pw;
5b7d70c6
BD		 3438 int retry_count = 8;
3439 u32 gintsts;
3440 u32 gintmsk;
3441
ee3de8d7
VM		 3442 if (!dwc2_is_device_mode(hsotg))
3443 return IRQ_NONE;
3444
5ad1d316 3445 spin_lock(&hsotg->lock);
5b7d70c6 3446irq_retry:
95c8bc36
AS		 3447 gintsts = dwc2_readl(hsotg->regs + GINTSTS);
3448 gintmsk = dwc2_readl(hsotg->regs + GINTMSK);
5b7d70c6
BD		 3449
3450 dev_dbg(hsotg->dev, "%s: %08x %08x (%08x) retry %d\n",
3451 __func__, gintsts, gintsts & gintmsk, gintmsk, retry_count);
3452
3453 gintsts &= gintmsk;
3454
8fc37b82
MYK		 3455 if (gintsts & GINTSTS_RESETDET) {
3456 dev_dbg(hsotg->dev, "%s: USBRstDet\n", __func__);
3457
3458 dwc2_writel(GINTSTS_RESETDET, hsotg->regs + GINTSTS);
3459
3460 /* This event must be used only if controller is suspended */
3461 if (hsotg->lx_state == DWC2_L2) {
3462 dwc2_exit_hibernation(hsotg, true);
3463 hsotg->lx_state = DWC2_L0;
3464 }
3465 }
3466
3467 if (gintsts & (GINTSTS_USBRST | GINTSTS_RESETDET)) {
8fc37b82
MYK		 3468 u32 usb_status = dwc2_readl(hsotg->regs + GOTGCTL);
3469 u32 connected = hsotg->connected;
3470
3471 dev_dbg(hsotg->dev, "%s: USBRst\n", __func__);
3472 dev_dbg(hsotg->dev, "GNPTXSTS=%08x\n",
3473 dwc2_readl(hsotg->regs + GNPTXSTS));
3474
3475 dwc2_writel(GINTSTS_USBRST, hsotg->regs + GINTSTS);
3476
3477 /* Report disconnection if it is not already done. */
3478 dwc2_hsotg_disconnect(hsotg);
3479
3480 if (usb_status & GOTGCTL_BSESVLD && connected)
3481 dwc2_hsotg_core_init_disconnected(hsotg, true);
3482 }
3483
47a1685f 3484 if (gintsts & GINTSTS_ENUMDONE) {
95c8bc36 3485 dwc2_writel(GINTSTS_ENUMDONE, hsotg->regs + GINTSTS);
a3395f0d 3486
1f91b4cc 3487 dwc2_hsotg_irq_enumdone(hsotg);
5b7d70c6
BD		 3488 }
3489
47a1685f 3490 if (gintsts & (GINTSTS_OEPINT | GINTSTS_IEPINT)) {
95c8bc36
AS		 3491 u32 daint = dwc2_readl(hsotg->regs + DAINT);
3492 u32 daintmsk = dwc2_readl(hsotg->regs + DAINTMSK);
7e804650 3493 u32 daint_out, daint_in;
5b7d70c6
BD		 3494 int ep;
3495
7e804650 3496 daint &= daintmsk;
47a1685f
DN		 3497 daint_out = daint >> DAINT_OUTEP_SHIFT;
3498 daint_in = daint & ~(daint_out << DAINT_OUTEP_SHIFT);
7e804650 3499
5b7d70c6
BD		 3500 dev_dbg(hsotg->dev, "%s: daint=%08x\n", __func__, daint);
3501
cec87f1d
MYK		 3502 for (ep = 0; ep < hsotg->num_of_eps && daint_out;
3503 ep++, daint_out >>= 1) {
5b7d70c6 3504 if (daint_out & 1)
1f91b4cc 3505 dwc2_hsotg_epint(hsotg, ep, 0);
5b7d70c6
BD		 3506 }
3507
cec87f1d
MYK		 3508 for (ep = 0; ep < hsotg->num_of_eps && daint_in;
3509 ep++, daint_in >>= 1) {
5b7d70c6 3510 if (daint_in & 1)
1f91b4cc 3511 dwc2_hsotg_epint(hsotg, ep, 1);
5b7d70c6 3512 }
5b7d70c6
BD		 3513 }
3514
5b7d70c6
BD		 3515 /* check both FIFOs */
3516
47a1685f 3517 if (gintsts & GINTSTS_NPTXFEMP) {
5b7d70c6
BD		 3518 dev_dbg(hsotg->dev, "NPTxFEmp\n");
3519
8b9bc460
LM		 3520 /*
3521 * Disable the interrupt to stop it happening again
5b7d70c6 3522 * unless one of these endpoint routines decides that
8b9bc460
LM		 3523 * it needs re-enabling
3524 */
5b7d70c6 3525
1f91b4cc
FB		 3526 dwc2_hsotg_disable_gsint(hsotg, GINTSTS_NPTXFEMP);
3527 dwc2_hsotg_irq_fifoempty(hsotg, false);
5b7d70c6
BD		 3528 }
3529
47a1685f 3530 if (gintsts & GINTSTS_PTXFEMP) {
5b7d70c6
BD		 3531 dev_dbg(hsotg->dev, "PTxFEmp\n");
3532
94cb8fd6 3533 /* See note in GINTSTS_NPTxFEmp */
5b7d70c6 3534
1f91b4cc
FB		 3535 dwc2_hsotg_disable_gsint(hsotg, GINTSTS_PTXFEMP);
3536 dwc2_hsotg_irq_fifoempty(hsotg, true);
5b7d70c6
BD		 3537 }
3538
47a1685f 3539 if (gintsts & GINTSTS_RXFLVL) {
8b9bc460
LM		 3540 /*
3541 * note, since GINTSTS_RxFLvl doubles as FIFO-not-empty,
1f91b4cc 3542 * we need to retry dwc2_hsotg_handle_rx if this is still
8b9bc460
LM		 3543 * set.
3544 */
5b7d70c6 3545
1f91b4cc 3546 dwc2_hsotg_handle_rx(hsotg);
5b7d70c6
BD		 3547 }
3548
47a1685f 3549 if (gintsts & GINTSTS_ERLYSUSP) {
94cb8fd6 3550 dev_dbg(hsotg->dev, "GINTSTS_ErlySusp\n");
95c8bc36 3551 dwc2_writel(GINTSTS_ERLYSUSP, hsotg->regs + GINTSTS);
5b7d70c6
BD		 3552 }
3553
8b9bc460
LM		 3554 /*
3555 * these next two seem to crop-up occasionally causing the core
5b7d70c6 3556 * to shutdown the USB transfer, so try clearing them and logging
8b9bc460
LM		 3557 * the occurrence.
3558 */
5b7d70c6 3559
47a1685f 3560 if (gintsts & GINTSTS_GOUTNAKEFF) {
837e9f00
VM		 3561 u8 idx;
3562 u32 epctrl;
3563 u32 gintmsk;
3564 struct dwc2_hsotg_ep *hs_ep;
3565
3566 /* Mask this interrupt */
3567 gintmsk = dwc2_readl(hsotg->regs + GINTMSK);
3568 gintmsk &= ~GINTSTS_GOUTNAKEFF;
3569 dwc2_writel(gintmsk, hsotg->regs + GINTMSK);
3570
3571 dev_dbg(hsotg->dev, "GOUTNakEff triggered\n");
3572 for (idx = 1; idx <= hsotg->num_of_eps; idx++) {
3573 hs_ep = hsotg->eps_out[idx];
3574 epctrl = dwc2_readl(hsotg->regs + DOEPCTL(idx));
3575
3576 if ((epctrl & DXEPCTL_EPENA) && hs_ep->isochronous) {
3577 epctrl |= DXEPCTL_SNAK;
3578 epctrl |= DXEPCTL_EPDIS;
3579 dwc2_writel(epctrl, hsotg->regs + DOEPCTL(idx));
3580 }
3581 }
a3395f0d 3582
837e9f00 3583 /* This interrupt bit is cleared in DXEPINT_EPDISBLD handler */
5b7d70c6
BD		 3584 }
3585
47a1685f 3586 if (gintsts & GINTSTS_GINNAKEFF) {
5b7d70c6
BD		 3587 dev_info(hsotg->dev, "GINNakEff triggered\n");
3588
3be99cd0 3589 __orr32(hsotg->regs + DCTL, DCTL_CGNPINNAK);
a3395f0d 3590
1f91b4cc 3591 dwc2_hsotg_dump(hsotg);
5b7d70c6
BD		 3592 }
3593
381fc8f8
VM		 3594 if (gintsts & GINTSTS_INCOMPL_SOIN)
3595 dwc2_gadget_handle_incomplete_isoc_in(hsotg);
ec1f9d9f 3596
381fc8f8
VM		 3597 if (gintsts & GINTSTS_INCOMPL_SOOUT)
3598 dwc2_gadget_handle_incomplete_isoc_out(hsotg);
ec1f9d9f 3599
8b9bc460
LM		 3600 /*
3601 * if we've had fifo events, we should try and go around the
3602 * loop again to see if there's any point in returning yet.
3603 */
5b7d70c6
BD		 3604
3605 if (gintsts & IRQ_RETRY_MASK && --retry_count > 0)
3606 goto irq_retry;
3607
5ad1d316
LM		 3608 spin_unlock(&hsotg->lock);
3609
5b7d70c6
BD		 3610 return IRQ_HANDLED;
3611}
3612
a4f82771
VA		 3613static int dwc2_hsotg_wait_bit_set(struct dwc2_hsotg *hs_otg, u32 reg,
3614 u32 bit, u32 timeout)
3615{
3616 u32 i;
3617
3618 for (i = 0; i < timeout; i++) {
3619 if (dwc2_readl(hs_otg->regs + reg) & bit)
3620 return 0;
3621 udelay(1);
3622 }
3623
3624 return -ETIMEDOUT;
3625}
3626
3627static void dwc2_hsotg_ep_stop_xfr(struct dwc2_hsotg *hsotg,
3628 struct dwc2_hsotg_ep *hs_ep)
3629{
3630 u32 epctrl_reg;
3631 u32 epint_reg;
3632
3633 epctrl_reg = hs_ep->dir_in ? DIEPCTL(hs_ep->index) :
3634 DOEPCTL(hs_ep->index);
3635 epint_reg = hs_ep->dir_in ? DIEPINT(hs_ep->index) :
3636 DOEPINT(hs_ep->index);
3637
3638 dev_dbg(hsotg->dev, "%s: stopping transfer on %s\n", __func__,
3639 hs_ep->name);
3640
3641 if (hs_ep->dir_in) {
3642 if (hsotg->dedicated_fifos || hs_ep->periodic) {
3643 __orr32(hsotg->regs + epctrl_reg, DXEPCTL_SNAK);
3644 /* Wait for Nak effect */
3645 if (dwc2_hsotg_wait_bit_set(hsotg, epint_reg,
3646 DXEPINT_INEPNAKEFF, 100))
3647 dev_warn(hsotg->dev,
3648 "%s: timeout DIEPINT.NAKEFF\n",
3649 __func__);
3650 } else {
3651 __orr32(hsotg->regs + DCTL, DCTL_SGNPINNAK);
3652 /* Wait for Nak effect */
3653 if (dwc2_hsotg_wait_bit_set(hsotg, GINTSTS,
3654 GINTSTS_GINNAKEFF, 100))
3655 dev_warn(hsotg->dev,
3656 "%s: timeout GINTSTS.GINNAKEFF\n",
3657 __func__);
3658 }
3659 } else {
3660 if (!(dwc2_readl(hsotg->regs + GINTSTS) & GINTSTS_GOUTNAKEFF))
3661 __orr32(hsotg->regs + DCTL, DCTL_SGOUTNAK);
3662
3663 /* Wait for global nak to take effect */
3664 if (dwc2_hsotg_wait_bit_set(hsotg, GINTSTS,
3665 GINTSTS_GOUTNAKEFF, 100))
3666 dev_warn(hsotg->dev, "%s: timeout GINTSTS.GOUTNAKEFF\n",
3667 __func__);
3668 }
3669
3670 /* Disable ep */
3671 __orr32(hsotg->regs + epctrl_reg, DXEPCTL_EPDIS | DXEPCTL_SNAK);
3672
3673 /* Wait for ep to be disabled */
3674 if (dwc2_hsotg_wait_bit_set(hsotg, epint_reg, DXEPINT_EPDISBLD, 100))
3675 dev_warn(hsotg->dev,
3676 "%s: timeout DOEPCTL.EPDisable\n", __func__);
3677
3678 /* Clear EPDISBLD interrupt */
3679 __orr32(hsotg->regs + epint_reg, DXEPINT_EPDISBLD);
3680
3681 if (hs_ep->dir_in) {
3682 unsigned short fifo_index;
3683
3684 if (hsotg->dedicated_fifos || hs_ep->periodic)
3685 fifo_index = hs_ep->fifo_index;
3686 else
3687 fifo_index = 0;
3688
3689 /* Flush TX FIFO */
3690 dwc2_flush_tx_fifo(hsotg, fifo_index);
3691
3692 /* Clear Global In NP NAK in Shared FIFO for non periodic ep */
3693 if (!hsotg->dedicated_fifos && !hs_ep->periodic)
3694 __orr32(hsotg->regs + DCTL, DCTL_CGNPINNAK);
3695
3696 } else {
3697 /* Remove global NAKs */
3698 __orr32(hsotg->regs + DCTL, DCTL_CGOUTNAK);
3699 }
3700}
3701
5b7d70c6 3702/**
1f91b4cc 3703 * dwc2_hsotg_ep_enable - enable the given endpoint
5b7d70c6
BD		 3704 * @ep: The USB endpint to configure
3705 * @desc: The USB endpoint descriptor to configure with.
3706 *
3707 * This is called from the USB gadget code's usb_ep_enable().
8b9bc460 3708 */
1f91b4cc 3709static int dwc2_hsotg_ep_enable(struct usb_ep *ep,
9da51974 3710 const struct usb_endpoint_descriptor *desc)
5b7d70c6 3711{
1f91b4cc 3712 struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
941fcce4 3713 struct dwc2_hsotg *hsotg = hs_ep->parent;
5b7d70c6 3714 unsigned long flags;
ca4c55ad 3715 unsigned int index = hs_ep->index;
5b7d70c6
BD		 3716 u32 epctrl_reg;
3717 u32 epctrl;
3718 u32 mps;
ee2c40de 3719 u32 mc;
837e9f00 3720 u32 mask;
ca4c55ad
MYK		 3721 unsigned int dir_in;
3722 unsigned int i, val, size;
19c190f9 3723 int ret = 0;
5b7d70c6
BD		 3724
3725 dev_dbg(hsotg->dev,
3726 "%s: ep %s: a 0x%02x, attr 0x%02x, mps 0x%04x, intr %d\n",
3727 __func__, ep->name, desc->bEndpointAddress, desc->bmAttributes,
3728 desc->wMaxPacketSize, desc->bInterval);
3729
3730 /* not to be called for EP0 */
8c3d6092
VA		 3731 if (index == 0) {
3732 dev_err(hsotg->dev, "%s: called for EP 0\n", __func__);
3733 return -EINVAL;
3734 }
5b7d70c6
BD		 3735
3736 dir_in = (desc->bEndpointAddress & USB_ENDPOINT_DIR_MASK) ? 1 : 0;
3737 if (dir_in != hs_ep->dir_in) {
3738 dev_err(hsotg->dev, "%s: direction mismatch!\n", __func__);
3739 return -EINVAL;
3740 }
3741
29cc8897 3742 mps = usb_endpoint_maxp(desc);
ee2c40de 3743 mc = usb_endpoint_maxp_mult(desc);
5b7d70c6 3744
1f91b4cc 3745 /* note, we handle this here instead of dwc2_hsotg_set_ep_maxpacket */
5b7d70c6 3746
94cb8fd6 3747 epctrl_reg = dir_in ? DIEPCTL(index) : DOEPCTL(index);
95c8bc36 3748 epctrl = dwc2_readl(hsotg->regs + epctrl_reg);
5b7d70c6
BD		 3749
3750 dev_dbg(hsotg->dev, "%s: read DxEPCTL=0x%08x from 0x%08x\n",
3751 __func__, epctrl, epctrl_reg);
3752
5f54c54b
VA		 3753 /* Allocate DMA descriptor chain for non-ctrl endpoints */
3754 if (using_desc_dma(hsotg)) {
3755 hs_ep->desc_list = dma_alloc_coherent(hsotg->dev,
3756 MAX_DMA_DESC_NUM_GENERIC *
3757 sizeof(struct dwc2_dma_desc),
86e881e7 3758 &hs_ep->desc_list_dma, GFP_ATOMIC);
5f54c54b
VA		 3759 if (!hs_ep->desc_list) {
3760 ret = -ENOMEM;
3761 goto error2;
3762 }
3763 }
3764
22258f49 3765 spin_lock_irqsave(&hsotg->lock, flags);
5b7d70c6 3766
47a1685f
DN		 3767 epctrl &= ~(DXEPCTL_EPTYPE_MASK | DXEPCTL_MPS_MASK);
3768 epctrl |= DXEPCTL_MPS(mps);
5b7d70c6 3769
8b9bc460
LM		 3770 /*
3771 * mark the endpoint as active, otherwise the core may ignore
3772 * transactions entirely for this endpoint
3773 */
47a1685f 3774 epctrl |= DXEPCTL_USBACTEP;
5b7d70c6 3775
5b7d70c6 3776 /* update the endpoint state */
ee2c40de 3777 dwc2_hsotg_set_ep_maxpacket(hsotg, hs_ep->index, mps, mc, dir_in);
5b7d70c6
BD		 3778
3779 /* default, set to non-periodic */
1479e841 3780 hs_ep->isochronous = 0;
5b7d70c6 3781 hs_ep->periodic = 0;
a18ed7b0 3782 hs_ep->halted = 0;
1479e841 3783 hs_ep->interval = desc->bInterval;
4fca54aa 3784
5b7d70c6
BD		 3785 switch (desc->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK) {
3786 case USB_ENDPOINT_XFER_ISOC:
47a1685f
DN		 3787 epctrl |= DXEPCTL_EPTYPE_ISO;
3788 epctrl |= DXEPCTL_SETEVENFR;
1479e841 3789 hs_ep->isochronous = 1;
142bd33f 3790 hs_ep->interval = 1 << (desc->bInterval - 1);
837e9f00 3791 hs_ep->target_frame = TARGET_FRAME_INITIAL;
ab7d2192
VA		 3792 hs_ep->isoc_chain_num = 0;
3793 hs_ep->next_desc = 0;
837e9f00 3794 if (dir_in) {
1479e841 3795 hs_ep->periodic = 1;
837e9f00
VM		 3796 mask = dwc2_readl(hsotg->regs + DIEPMSK);
3797 mask |= DIEPMSK_NAKMSK;
3798 dwc2_writel(mask, hsotg->regs + DIEPMSK);
3799 } else {
3800 mask = dwc2_readl(hsotg->regs + DOEPMSK);
3801 mask |= DOEPMSK_OUTTKNEPDISMSK;
3802 dwc2_writel(mask, hsotg->regs + DOEPMSK);
3803 }
1479e841 3804 break;
5b7d70c6
BD		 3805
3806 case USB_ENDPOINT_XFER_BULK:
47a1685f 3807 epctrl |= DXEPCTL_EPTYPE_BULK;
5b7d70c6
BD		 3808 break;
3809
3810 case USB_ENDPOINT_XFER_INT:
b203d0a2 3811 if (dir_in)
5b7d70c6 3812 hs_ep->periodic = 1;
5b7d70c6 3813
142bd33f
VM		 3814 if (hsotg->gadget.speed == USB_SPEED_HIGH)
3815 hs_ep->interval = 1 << (desc->bInterval - 1);
3816
47a1685f 3817 epctrl |= DXEPCTL_EPTYPE_INTERRUPT;
5b7d70c6
BD		 3818 break;
3819
3820 case USB_ENDPOINT_XFER_CONTROL:
47a1685f 3821 epctrl |= DXEPCTL_EPTYPE_CONTROL;
5b7d70c6
BD		 3822 break;
3823 }
3824
8b9bc460
LM		 3825 /*
3826 * if the hardware has dedicated fifos, we must give each IN EP
10aebc77
BD		 3827 * a unique tx-fifo even if it is non-periodic.
3828 */
21f3bb52 3829 if (dir_in && hsotg->dedicated_fifos) {
ca4c55ad
MYK		 3830 u32 fifo_index = 0;
3831 u32 fifo_size = UINT_MAX;
9da51974
JY		 3832
3833 size = hs_ep->ep.maxpacket * hs_ep->mc;
5f2196bd 3834 for (i = 1; i < hsotg->num_of_eps; ++i) {
9da51974 3835 if (hsotg->fifo_map & (1 << i))
b203d0a2 3836 continue;
95c8bc36 3837 val = dwc2_readl(hsotg->regs + DPTXFSIZN(i));
9da51974 3838 val = (val >> FIFOSIZE_DEPTH_SHIFT) * 4;
b203d0a2
RB		 3839 if (val < size)
3840 continue;
ca4c55ad
MYK		 3841 /* Search for smallest acceptable fifo */
3842 if (val < fifo_size) {
3843 fifo_size = val;
3844 fifo_index = i;
3845 }
b203d0a2 3846 }
ca4c55ad 3847 if (!fifo_index) {
5f2196bd
MYK		 3848 dev_err(hsotg->dev,
3849 "%s: No suitable fifo found\n", __func__);
b585a48b 3850 ret = -ENOMEM;
5f54c54b 3851 goto error1;
b585a48b 3852 }
ca4c55ad
MYK		 3853 hsotg->fifo_map |= 1 << fifo_index;
3854 epctrl |= DXEPCTL_TXFNUM(fifo_index);
3855 hs_ep->fifo_index = fifo_index;
3856 hs_ep->fifo_size = fifo_size;
b203d0a2 3857 }
10aebc77 3858
5b7d70c6 3859 /* for non control endpoints, set PID to D0 */
837e9f00 3860 if (index && !hs_ep->isochronous)
47a1685f 3861 epctrl |= DXEPCTL_SETD0PID;
5b7d70c6
BD		 3862
3863 dev_dbg(hsotg->dev, "%s: write DxEPCTL=0x%08x\n",
3864 __func__, epctrl);
3865
95c8bc36 3866 dwc2_writel(epctrl, hsotg->regs + epctrl_reg);
5b7d70c6 3867 dev_dbg(hsotg->dev, "%s: read DxEPCTL=0x%08x\n",
95c8bc36 3868 __func__, dwc2_readl(hsotg->regs + epctrl_reg));
5b7d70c6
BD		 3869
3870 /* enable the endpoint interrupt */
1f91b4cc 3871 dwc2_hsotg_ctrl_epint(hsotg, index, dir_in, 1);
5b7d70c6 3872
5f54c54b 3873error1:
22258f49 3874 spin_unlock_irqrestore(&hsotg->lock, flags);
5f54c54b
VA		 3875
3876error2:
3877 if (ret && using_desc_dma(hsotg) && hs_ep->desc_list) {
3878 dma_free_coherent(hsotg->dev, MAX_DMA_DESC_NUM_GENERIC *
3879 sizeof(struct dwc2_dma_desc),
3880 hs_ep->desc_list, hs_ep->desc_list_dma);
3881 hs_ep->desc_list = NULL;
3882 }
3883
19c190f9 3884 return ret;
5b7d70c6
BD		 3885}
3886
8b9bc460 3887/**
1f91b4cc 3888 * dwc2_hsotg_ep_disable - disable given endpoint
8b9bc460
LM		 3889 * @ep: The endpoint to disable.
3890 */
1f91b4cc 3891static int dwc2_hsotg_ep_disable(struct usb_ep *ep)
5b7d70c6 3892{
1f91b4cc 3893 struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
941fcce4 3894 struct dwc2_hsotg *hsotg = hs_ep->parent;
5b7d70c6
BD		 3895 int dir_in = hs_ep->dir_in;
3896 int index = hs_ep->index;
3897 unsigned long flags;
3898 u32 epctrl_reg;
3899 u32 ctrl;
3900
1e011293 3901 dev_dbg(hsotg->dev, "%s(ep %p)\n", __func__, ep);
5b7d70c6 3902
c6f5c050 3903 if (ep == &hsotg->eps_out[0]->ep) {
5b7d70c6
BD		 3904 dev_err(hsotg->dev, "%s: called for ep0\n", __func__);
3905 return -EINVAL;
3906 }
3907
5f54c54b
VA		 3908 /* Remove DMA memory allocated for non-control Endpoints */
3909 if (using_desc_dma(hsotg)) {
3910 dma_free_coherent(hsotg->dev, MAX_DMA_DESC_NUM_GENERIC *
3911 sizeof(struct dwc2_dma_desc),
3912 hs_ep->desc_list, hs_ep->desc_list_dma);
3913 hs_ep->desc_list = NULL;
3914 }
3915
94cb8fd6 3916 epctrl_reg = dir_in ? DIEPCTL(index) : DOEPCTL(index);
5b7d70c6 3917
5ad1d316 3918 spin_lock_irqsave(&hsotg->lock, flags);
5b7d70c6 3919
95c8bc36 3920 ctrl = dwc2_readl(hsotg->regs + epctrl_reg);
a4f82771
VA		 3921
3922 if (ctrl & DXEPCTL_EPENA)
3923 dwc2_hsotg_ep_stop_xfr(hsotg, hs_ep);
3924
47a1685f
DN		 3925 ctrl &= ~DXEPCTL_EPENA;
3926 ctrl &= ~DXEPCTL_USBACTEP;
3927 ctrl |= DXEPCTL_SNAK;
5b7d70c6
BD		 3928
3929 dev_dbg(hsotg->dev, "%s: DxEPCTL=0x%08x\n", __func__, ctrl);
95c8bc36 3930 dwc2_writel(ctrl, hsotg->regs + epctrl_reg);
5b7d70c6
BD		 3931
3932 /* disable endpoint interrupts */
1f91b4cc 3933 dwc2_hsotg_ctrl_epint(hsotg, hs_ep->index, hs_ep->dir_in, 0);
5b7d70c6 3934
1141ea01
MYK		 3935 /* terminate all requests with shutdown */
3936 kill_all_requests(hsotg, hs_ep, -ESHUTDOWN);
3937
1c07b20e
RB		 3938 hsotg->fifo_map &= ~(1 << hs_ep->fifo_index);
3939 hs_ep->fifo_index = 0;
3940 hs_ep->fifo_size = 0;
3941
22258f49 3942 spin_unlock_irqrestore(&hsotg->lock, flags);
5b7d70c6
BD		 3943 return 0;
3944}
3945
3946/**
3947 * on_list - check request is on the given endpoint
3948 * @ep: The endpoint to check.
3949 * @test: The request to test if it is on the endpoint.
8b9bc460 3950 */
1f91b4cc 3951static bool on_list(struct dwc2_hsotg_ep *ep, struct dwc2_hsotg_req *test)
5b7d70c6 3952{
1f91b4cc 3953 struct dwc2_hsotg_req *req, *treq;
5b7d70c6
BD		 3954
3955 list_for_each_entry_safe(req, treq, &ep->queue, queue) {
3956 if (req == test)
3957 return true;
3958 }
3959
3960 return false;
3961}
3962
8b9bc460 3963/**
1f91b4cc 3964 * dwc2_hsotg_ep_dequeue - dequeue given endpoint
8b9bc460
LM		 3965 * @ep: The endpoint to dequeue.
3966 * @req: The request to be removed from a queue.
3967 */
1f91b4cc 3968static int dwc2_hsotg_ep_dequeue(struct usb_ep *ep, struct usb_request *req)
5b7d70c6 3969{
1f91b4cc
FB		 3970 struct dwc2_hsotg_req *hs_req = our_req(req);
3971 struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
941fcce4 3972 struct dwc2_hsotg *hs = hs_ep->parent;
5b7d70c6
BD		 3973 unsigned long flags;
3974
1e011293 3975 dev_dbg(hs->dev, "ep_dequeue(%p,%p)\n", ep, req);
5b7d70c6 3976
22258f49 3977 spin_lock_irqsave(&hs->lock, flags);
5b7d70c6
BD		 3978
3979 if (!on_list(hs_ep, hs_req)) {
22258f49 3980 spin_unlock_irqrestore(&hs->lock, flags);
5b7d70c6
BD		 3981 return -EINVAL;
3982 }
3983
c524dd5f
MYK		 3984 /* Dequeue already started request */
3985 if (req == &hs_ep->req->req)
3986 dwc2_hsotg_ep_stop_xfr(hs, hs_ep);
3987
1f91b4cc 3988 dwc2_hsotg_complete_request(hs, hs_ep, hs_req, -ECONNRESET);
22258f49 3989 spin_unlock_irqrestore(&hs->lock, flags);
5b7d70c6
BD		 3990
3991 return 0;
3992}
3993
8b9bc460 3994/**
1f91b4cc 3995 * dwc2_hsotg_ep_sethalt - set halt on a given endpoint
8b9bc460
LM		 3996 * @ep: The endpoint to set halt.
3997 * @value: Set or unset the halt.
51da43b5
VA		 3998 * @now: If true, stall the endpoint now. Otherwise return -EAGAIN if
3999 * the endpoint is busy processing requests.
4000 *
4001 * We need to stall the endpoint immediately if request comes from set_feature
4002 * protocol command handler.
8b9bc460 4003 */
51da43b5 4004static int dwc2_hsotg_ep_sethalt(struct usb_ep *ep, int value, bool now)
5b7d70c6 4005{
1f91b4cc 4006 struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
941fcce4 4007 struct dwc2_hsotg *hs = hs_ep->parent;
5b7d70c6 4008 int index = hs_ep->index;
5b7d70c6
BD		 4009 u32 epreg;
4010 u32 epctl;
9c39ddc6 4011 u32 xfertype;
5b7d70c6
BD		 4012
4013 dev_info(hs->dev, "%s(ep %p %s, %d)\n", __func__, ep, ep->name, value);
4014
c9f721b2
RB		 4015 if (index == 0) {
4016 if (value)
1f91b4cc 4017 dwc2_hsotg_stall_ep0(hs);
c9f721b2
RB		 4018 else
4019 dev_warn(hs->dev,
4020 "%s: can't clear halt on ep0\n", __func__);
4021 return 0;
4022 }
4023
15186f10
VA		 4024 if (hs_ep->isochronous) {
4025 dev_err(hs->dev, "%s is Isochronous Endpoint\n", ep->name);
4026 return -EINVAL;
4027 }
4028
51da43b5
VA		 4029 if (!now && value && !list_empty(&hs_ep->queue)) {
4030 dev_dbg(hs->dev, "%s request is pending, cannot halt\n",
4031 ep->name);
4032 return -EAGAIN;
4033 }
4034
c6f5c050
MYK		 4035 if (hs_ep->dir_in) {
4036 epreg = DIEPCTL(index);
95c8bc36 4037 epctl = dwc2_readl(hs->regs + epreg);
c6f5c050
MYK		 4038
4039 if (value) {
5a350d53 4040 epctl |= DXEPCTL_STALL | DXEPCTL_SNAK;
c6f5c050
MYK		 4041 if (epctl & DXEPCTL_EPENA)
4042 epctl |= DXEPCTL_EPDIS;
4043 } else {
4044 epctl &= ~DXEPCTL_STALL;
4045 xfertype = epctl & DXEPCTL_EPTYPE_MASK;
4046 if (xfertype == DXEPCTL_EPTYPE_BULK ||
9da51974 4047 xfertype == DXEPCTL_EPTYPE_INTERRUPT)
c6f5c050
MYK		 4048 epctl |= DXEPCTL_SETD0PID;
4049 }
95c8bc36 4050 dwc2_writel(epctl, hs->regs + epreg);
9c39ddc6 4051 } else {
c6f5c050 4052 epreg = DOEPCTL(index);
95c8bc36 4053 epctl = dwc2_readl(hs->regs + epreg);
5b7d70c6 4054
34c0887f 4055 if (value) {
c6f5c050 4056 epctl |= DXEPCTL_STALL;
34c0887f 4057 } else {
c6f5c050
MYK		 4058 epctl &= ~DXEPCTL_STALL;
4059 xfertype = epctl & DXEPCTL_EPTYPE_MASK;
4060 if (xfertype == DXEPCTL_EPTYPE_BULK ||
9da51974 4061 xfertype == DXEPCTL_EPTYPE_INTERRUPT)
c6f5c050
MYK		 4062 epctl |= DXEPCTL_SETD0PID;
4063 }
95c8bc36 4064 dwc2_writel(epctl, hs->regs + epreg);
9c39ddc6 4065 }
5b7d70c6 4066
a18ed7b0
RB		 4067 hs_ep->halted = value;
4068
5b7d70c6
BD		 4069 return 0;
4070}
4071
5ad1d316 4072/**
1f91b4cc 4073 * dwc2_hsotg_ep_sethalt_lock - set halt on a given endpoint with lock held
5ad1d316
LM		 4074 * @ep: The endpoint to set halt.
4075 * @value: Set or unset the halt.
4076 */
1f91b4cc 4077static int dwc2_hsotg_ep_sethalt_lock(struct usb_ep *ep, int value)
5ad1d316 4078{
1f91b4cc 4079 struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
941fcce4 4080 struct dwc2_hsotg *hs = hs_ep->parent;
5ad1d316
LM		 4081 unsigned long flags = 0;
4082 int ret = 0;
4083
4084 spin_lock_irqsave(&hs->lock, flags);
51da43b5 4085 ret = dwc2_hsotg_ep_sethalt(ep, value, false);
5ad1d316
LM		 4086 spin_unlock_irqrestore(&hs->lock, flags);
4087
4088 return ret;
4089}
4090
1f91b4cc
FB		 4091static struct usb_ep_ops dwc2_hsotg_ep_ops = {
4092 .enable = dwc2_hsotg_ep_enable,
4093 .disable = dwc2_hsotg_ep_disable,
4094 .alloc_request = dwc2_hsotg_ep_alloc_request,
4095 .free_request = dwc2_hsotg_ep_free_request,
4096 .queue = dwc2_hsotg_ep_queue_lock,
4097 .dequeue = dwc2_hsotg_ep_dequeue,
4098 .set_halt = dwc2_hsotg_ep_sethalt_lock,
25985edc 4099 /* note, don't believe we have any call for the fifo routines */
5b7d70c6
BD		 4100};
4101
8b9bc460 4102/**
9da51974 4103 * dwc2_hsotg_init - initialize the usb core
8b9bc460
LM		 4104 * @hsotg: The driver state
4105 */
1f91b4cc 4106static void dwc2_hsotg_init(struct dwc2_hsotg *hsotg)
b3f489b2 4107{
fa4a8d72 4108 u32 trdtim;
ecd9a7ad 4109 u32 usbcfg;
b3f489b2
LM		 4110 /* unmask subset of endpoint interrupts */
4111
95c8bc36
AS		 4112 dwc2_writel(DIEPMSK_TIMEOUTMSK | DIEPMSK_AHBERRMSK |
4113 DIEPMSK_EPDISBLDMSK | DIEPMSK_XFERCOMPLMSK,
4114 hsotg->regs + DIEPMSK);
b3f489b2 4115
95c8bc36
AS		 4116 dwc2_writel(DOEPMSK_SETUPMSK | DOEPMSK_AHBERRMSK |
4117 DOEPMSK_EPDISBLDMSK | DOEPMSK_XFERCOMPLMSK,
4118 hsotg->regs + DOEPMSK);
b3f489b2 4119
95c8bc36 4120 dwc2_writel(0, hsotg->regs + DAINTMSK);
b3f489b2
LM		 4121
4122 /* Be in disconnected state until gadget is registered */
47a1685f 4123 __orr32(hsotg->regs + DCTL, DCTL_SFTDISCON);
b3f489b2 4124
b3f489b2
LM		 4125 /* setup fifos */
4126
4127 dev_dbg(hsotg->dev, "GRXFSIZ=0x%08x, GNPTXFSIZ=0x%08x\n",
95c8bc36
AS		 4128 dwc2_readl(hsotg->regs + GRXFSIZ),
4129 dwc2_readl(hsotg->regs + GNPTXFSIZ));
b3f489b2 4130
1f91b4cc 4131 dwc2_hsotg_init_fifo(hsotg);
b3f489b2 4132
ecd9a7ad
PR		 4133 /* keep other bits untouched (so e.g. forced modes are not lost) */
4134 usbcfg = dwc2_readl(hsotg->regs + GUSBCFG);
4135 usbcfg &= ~(GUSBCFG_TOUTCAL_MASK | GUSBCFG_PHYIF16 | GUSBCFG_SRPCAP |
4136 GUSBCFG_HNPCAP);
4137
b3f489b2 4138 /* set the PLL on, remove the HNP/SRP and set the PHY */
fa4a8d72 4139 trdtim = (hsotg->phyif == GUSBCFG_PHYIF8) ? 9 : 5;
ecd9a7ad
PR		 4140 usbcfg |= hsotg->phyif | GUSBCFG_TOUTCAL(7) |
4141 (trdtim << GUSBCFG_USBTRDTIM_SHIFT);
4142 dwc2_writel(usbcfg, hsotg->regs + GUSBCFG);
b3f489b2 4143
f5090044
GH		 4144 if (using_dma(hsotg))
4145 __orr32(hsotg->regs + GAHBCFG, GAHBCFG_DMA_EN);
b3f489b2
LM		 4146}
4147
8b9bc460 4148/**
1f91b4cc 4149 * dwc2_hsotg_udc_start - prepare the udc for work
8b9bc460
LM		 4150 * @gadget: The usb gadget state
4151 * @driver: The usb gadget driver
4152 *
4153 * Perform initialization to prepare udc device and driver
4154 * to work.
4155 */
1f91b4cc 4156static int dwc2_hsotg_udc_start(struct usb_gadget *gadget,
9da51974 4157 struct usb_gadget_driver *driver)
5b7d70c6 4158{
941fcce4 4159 struct dwc2_hsotg *hsotg = to_hsotg(gadget);
5b9451f8 4160 unsigned long flags;
5b7d70c6
BD		 4161 int ret;
4162
4163 if (!hsotg) {
a023da33 4164 pr_err("%s: called with no device\n", __func__);
5b7d70c6
BD		 4165 return -ENODEV;
4166 }
4167
4168 if (!driver) {
4169 dev_err(hsotg->dev, "%s: no driver\n", __func__);
4170 return -EINVAL;
4171 }
4172
7177aed4 4173 if (driver->max_speed < USB_SPEED_FULL)
5b7d70c6 4174 dev_err(hsotg->dev, "%s: bad speed\n", __func__);
5b7d70c6 4175
f65f0f10 4176 if (!driver->setup) {
5b7d70c6
BD		 4177 dev_err(hsotg->dev, "%s: missing entry points\n", __func__);
4178 return -EINVAL;
4179 }
4180
4181 WARN_ON(hsotg->driver);
4182
4183 driver->driver.bus = NULL;
4184 hsotg->driver = driver;
7d7b2292 4185 hsotg->gadget.dev.of_node = hsotg->dev->of_node;
5b7d70c6
BD		 4186 hsotg->gadget.speed = USB_SPEED_UNKNOWN;
4187
09a75e85
MS		 4188 if (hsotg->dr_mode == USB_DR_MODE_PERIPHERAL) {
4189 ret = dwc2_lowlevel_hw_enable(hsotg);
4190 if (ret)
4191 goto err;
5b7d70c6
BD		 4192 }
4193
f6c01592
GH		 4194 if (!IS_ERR_OR_NULL(hsotg->uphy))
4195 otg_set_peripheral(hsotg->uphy->otg, &hsotg->gadget);
c816c47f 4196
5b9451f8 4197 spin_lock_irqsave(&hsotg->lock, flags);
d0f0ac56
JY		 4198 if (dwc2_hw_is_device(hsotg)) {
4199 dwc2_hsotg_init(hsotg);
4200 dwc2_hsotg_core_init_disconnected(hsotg, false);
4201 }
4202
dc6e69e6 4203 hsotg->enabled = 0;
5b9451f8
MS		 4204 spin_unlock_irqrestore(&hsotg->lock, flags);
4205
5b7d70c6 4206 dev_info(hsotg->dev, "bound driver %s\n", driver->driver.name);
5b9451f8 4207
5b7d70c6
BD		 4208 return 0;
4209
4210err:
4211 hsotg->driver = NULL;
5b7d70c6
BD		 4212 return ret;
4213}
4214
8b9bc460 4215/**
1f91b4cc 4216 * dwc2_hsotg_udc_stop - stop the udc
8b9bc460
LM		 4217 * @gadget: The usb gadget state
4218 * @driver: The usb gadget driver
4219 *
4220 * Stop udc hw block and stay tunned for future transmissions
4221 */
1f91b4cc 4222static int dwc2_hsotg_udc_stop(struct usb_gadget *gadget)
5b7d70c6 4223{
941fcce4 4224 struct dwc2_hsotg *hsotg = to_hsotg(gadget);
2b19a52c 4225 unsigned long flags = 0;
5b7d70c6
BD		 4226 int ep;
4227
4228 if (!hsotg)
4229 return -ENODEV;
4230
5b7d70c6 4231 /* all endpoints should be shutdown */
c6f5c050
MYK		 4232 for (ep = 1; ep < hsotg->num_of_eps; ep++) {
4233 if (hsotg->eps_in[ep])
1f91b4cc 4234 dwc2_hsotg_ep_disable(&hsotg->eps_in[ep]->ep);
c6f5c050 4235 if (hsotg->eps_out[ep])
1f91b4cc 4236 dwc2_hsotg_ep_disable(&hsotg->eps_out[ep]->ep);
c6f5c050 4237 }
5b7d70c6 4238
2b19a52c
LM		 4239 spin_lock_irqsave(&hsotg->lock, flags);
4240
32805c35 4241 hsotg->driver = NULL;
5b7d70c6 4242 hsotg->gadget.speed = USB_SPEED_UNKNOWN;
dc6e69e6 4243 hsotg->enabled = 0;
5b7d70c6 4244
2b19a52c
LM		 4245 spin_unlock_irqrestore(&hsotg->lock, flags);
4246
f6c01592
GH		 4247 if (!IS_ERR_OR_NULL(hsotg->uphy))
4248 otg_set_peripheral(hsotg->uphy->otg, NULL);
c816c47f 4249
09a75e85
MS		 4250 if (hsotg->dr_mode == USB_DR_MODE_PERIPHERAL)
4251 dwc2_lowlevel_hw_disable(hsotg);
5b7d70c6
BD		 4252
4253 return 0;
4254}
5b7d70c6 4255
8b9bc460 4256/**
1f91b4cc 4257 * dwc2_hsotg_gadget_getframe - read the frame number
8b9bc460
LM		 4258 * @gadget: The usb gadget state
4259 *
4260 * Read the {micro} frame number
4261 */
1f91b4cc 4262static int dwc2_hsotg_gadget_getframe(struct usb_gadget *gadget)
5b7d70c6 4263{
1f91b4cc 4264 return dwc2_hsotg_read_frameno(to_hsotg(gadget));
5b7d70c6
BD		 4265}
4266
a188b689 4267/**
1f91b4cc 4268 * dwc2_hsotg_pullup - connect/disconnect the USB PHY
a188b689
LM		 4269 * @gadget: The usb gadget state
4270 * @is_on: Current state of the USB PHY
4271 *
4272 * Connect/Disconnect the USB PHY pullup
4273 */
1f91b4cc 4274static int dwc2_hsotg_pullup(struct usb_gadget *gadget, int is_on)
a188b689 4275{
941fcce4 4276 struct dwc2_hsotg *hsotg = to_hsotg(gadget);
a188b689
LM		 4277 unsigned long flags = 0;
4278
77ba9119 4279 dev_dbg(hsotg->dev, "%s: is_on: %d op_state: %d\n", __func__, is_on,
9da51974 4280 hsotg->op_state);
77ba9119
GH		 4281
4282 /* Don't modify pullup state while in host mode */
4283 if (hsotg->op_state != OTG_STATE_B_PERIPHERAL) {
4284 hsotg->enabled = is_on;
4285 return 0;
4286 }
a188b689
LM		 4287
4288 spin_lock_irqsave(&hsotg->lock, flags);
4289 if (is_on) {
dc6e69e6 4290 hsotg->enabled = 1;
1f91b4cc
FB		 4291 dwc2_hsotg_core_init_disconnected(hsotg, false);
4292 dwc2_hsotg_core_connect(hsotg);
a188b689 4293 } else {
1f91b4cc
FB		 4294 dwc2_hsotg_core_disconnect(hsotg);
4295 dwc2_hsotg_disconnect(hsotg);
dc6e69e6 4296 hsotg->enabled = 0;
a188b689
LM		 4297 }
4298
4299 hsotg->gadget.speed = USB_SPEED_UNKNOWN;
4300 spin_unlock_irqrestore(&hsotg->lock, flags);
4301
4302 return 0;
4303}
4304
1f91b4cc 4305static int dwc2_hsotg_vbus_session(struct usb_gadget *gadget, int is_active)
83d98223
GH		 4306{
4307 struct dwc2_hsotg *hsotg = to_hsotg(gadget);
4308 unsigned long flags;
4309
4310 dev_dbg(hsotg->dev, "%s: is_active: %d\n", __func__, is_active);
4311 spin_lock_irqsave(&hsotg->lock, flags);
4312
61f7223b
GH		 4313 /*
4314 * If controller is hibernated, it must exit from hibernation
4315 * before being initialized / de-initialized
4316 */
4317 if (hsotg->lx_state == DWC2_L2)
4318 dwc2_exit_hibernation(hsotg, false);
4319
83d98223 4320 if (is_active) {
cd0e641c 4321 hsotg->op_state = OTG_STATE_B_PERIPHERAL;
065d3931 4322
1f91b4cc 4323 dwc2_hsotg_core_init_disconnected(hsotg, false);
83d98223 4324 if (hsotg->enabled)
1f91b4cc 4325 dwc2_hsotg_core_connect(hsotg);
83d98223 4326 } else {
1f91b4cc
FB		 4327 dwc2_hsotg_core_disconnect(hsotg);
4328 dwc2_hsotg_disconnect(hsotg);
83d98223
GH		 4329 }
4330
4331 spin_unlock_irqrestore(&hsotg->lock, flags);
4332 return 0;
4333}
4334
596d696a 4335/**
1f91b4cc 4336 * dwc2_hsotg_vbus_draw - report bMaxPower field
596d696a
GH		 4337 * @gadget: The usb gadget state
4338 * @mA: Amount of current
4339 *
4340 * Report how much power the device may consume to the phy.
4341 */
9da51974 4342static int dwc2_hsotg_vbus_draw(struct usb_gadget *gadget, unsigned int mA)
596d696a
GH		 4343{
4344 struct dwc2_hsotg *hsotg = to_hsotg(gadget);
4345
4346 if (IS_ERR_OR_NULL(hsotg->uphy))
4347 return -ENOTSUPP;
4348 return usb_phy_set_power(hsotg->uphy, mA);
4349}
4350
1f91b4cc
FB		 4351static const struct usb_gadget_ops dwc2_hsotg_gadget_ops = {
4352 .get_frame = dwc2_hsotg_gadget_getframe,
4353 .udc_start = dwc2_hsotg_udc_start,
4354 .udc_stop = dwc2_hsotg_udc_stop,
4355 .pullup = dwc2_hsotg_pullup,
4356 .vbus_session = dwc2_hsotg_vbus_session,
4357 .vbus_draw = dwc2_hsotg_vbus_draw,
5b7d70c6
BD		 4358};
4359
4360/**
1f91b4cc 4361 * dwc2_hsotg_initep - initialise a single endpoint
5b7d70c6
BD		 4362 * @hsotg: The device state.
4363 * @hs_ep: The endpoint to be initialised.
4364 * @epnum: The endpoint number
4365 *
4366 * Initialise the given endpoint (as part of the probe and device state
4367 * creation) to give to the gadget driver. Setup the endpoint name, any
4368 * direction information and other state that may be required.
4369 */
1f91b4cc 4370static void dwc2_hsotg_initep(struct dwc2_hsotg *hsotg,
9da51974 4371 struct dwc2_hsotg_ep *hs_ep,
c6f5c050
MYK		 4372 int epnum,
4373 bool dir_in)
5b7d70c6 4374{
5b7d70c6
BD		 4375 char *dir;
4376
4377 if (epnum == 0)
4378 dir = "";
c6f5c050 4379 else if (dir_in)
5b7d70c6 4380 dir = "in";
c6f5c050
MYK		 4381 else
4382 dir = "out";
5b7d70c6 4383
c6f5c050 4384 hs_ep->dir_in = dir_in;
5b7d70c6
BD		 4385 hs_ep->index = epnum;
4386
4387 snprintf(hs_ep->name, sizeof(hs_ep->name), "ep%d%s", epnum, dir);
4388
4389 INIT_LIST_HEAD(&hs_ep->queue);
4390 INIT_LIST_HEAD(&hs_ep->ep.ep_list);
4391
5b7d70c6
BD		 4392 /* add to the list of endpoints known by the gadget driver */
4393 if (epnum)
4394 list_add_tail(&hs_ep->ep.ep_list, &hsotg->gadget.ep_list);
4395
4396 hs_ep->parent = hsotg;
4397 hs_ep->ep.name = hs_ep->name;
38e9002b
VM		 4398
4399 if (hsotg->params.speed == DWC2_SPEED_PARAM_LOW)
4400 usb_ep_set_maxpacket_limit(&hs_ep->ep, 8);
4401 else
4402 usb_ep_set_maxpacket_limit(&hs_ep->ep,
4403 epnum ? 1024 : EP0_MPS_LIMIT);
1f91b4cc 4404 hs_ep->ep.ops = &dwc2_hsotg_ep_ops;
5b7d70c6 4405
2954522f
RB		 4406 if (epnum == 0) {
4407 hs_ep->ep.caps.type_control = true;
4408 } else {
38e9002b
VM		 4409 if (hsotg->params.speed != DWC2_SPEED_PARAM_LOW) {
4410 hs_ep->ep.caps.type_iso = true;
4411 hs_ep->ep.caps.type_bulk = true;
4412 }
2954522f
RB		 4413 hs_ep->ep.caps.type_int = true;
4414 }
4415
4416 if (dir_in)
4417 hs_ep->ep.caps.dir_in = true;
4418 else
4419 hs_ep->ep.caps.dir_out = true;
4420
8b9bc460
LM		 4421 /*
4422 * if we're using dma, we need to set the next-endpoint pointer
5b7d70c6
BD		 4423 * to be something valid.
4424 */
4425
4426 if (using_dma(hsotg)) {
47a1685f 4427 u32 next = DXEPCTL_NEXTEP((epnum + 1) % 15);
9da51974 4428
c6f5c050 4429 if (dir_in)
95c8bc36 4430 dwc2_writel(next, hsotg->regs + DIEPCTL(epnum));
c6f5c050 4431 else
95c8bc36 4432 dwc2_writel(next, hsotg->regs + DOEPCTL(epnum));
5b7d70c6
BD		 4433 }
4434}
4435
b3f489b2 4436/**
1f91b4cc 4437 * dwc2_hsotg_hw_cfg - read HW configuration registers
b3f489b2
LM		 4438 * @param: The device state
4439 *
4440 * Read the USB core HW configuration registers
4441 */
1f91b4cc 4442static int dwc2_hsotg_hw_cfg(struct dwc2_hsotg *hsotg)
5b7d70c6 4443{
c6f5c050
MYK		 4444 u32 cfg;
4445 u32 ep_type;
4446 u32 i;
4447
b3f489b2 4448 /* check hardware configuration */
5b7d70c6 4449
43e90349
JY		 4450 hsotg->num_of_eps = hsotg->hw_params.num_dev_ep;
4451
c6f5c050
MYK		 4452 /* Add ep0 */
4453 hsotg->num_of_eps++;
10aebc77 4454
b98866c2
JY		 4455 hsotg->eps_in[0] = devm_kzalloc(hsotg->dev,
4456 sizeof(struct dwc2_hsotg_ep),
4457 GFP_KERNEL);
c6f5c050
MYK		 4458 if (!hsotg->eps_in[0])
4459 return -ENOMEM;
1f91b4cc 4460 /* Same dwc2_hsotg_ep is used in both directions for ep0 */
c6f5c050
MYK		 4461 hsotg->eps_out[0] = hsotg->eps_in[0];
4462
43e90349 4463 cfg = hsotg->hw_params.dev_ep_dirs;
251a17f5 4464 for (i = 1, cfg >>= 2; i < hsotg->num_of_eps; i++, cfg >>= 2) {
c6f5c050
MYK		 4465 ep_type = cfg & 3;
4466 /* Direction in or both */
4467 if (!(ep_type & 2)) {
4468 hsotg->eps_in[i] = devm_kzalloc(hsotg->dev,
1f91b4cc 4469 sizeof(struct dwc2_hsotg_ep), GFP_KERNEL);
c6f5c050
MYK		 4470 if (!hsotg->eps_in[i])
4471 return -ENOMEM;
4472 }
4473 /* Direction out or both */
4474 if (!(ep_type & 1)) {
4475 hsotg->eps_out[i] = devm_kzalloc(hsotg->dev,
1f91b4cc 4476 sizeof(struct dwc2_hsotg_ep), GFP_KERNEL);
c6f5c050
MYK		 4477 if (!hsotg->eps_out[i])
4478 return -ENOMEM;
4479 }
4480 }
4481
43e90349
JY		 4482 hsotg->fifo_mem = hsotg->hw_params.total_fifo_size;
4483 hsotg->dedicated_fifos = hsotg->hw_params.en_multiple_tx_fifo;
10aebc77 4484
cff9eb75
MS		 4485 dev_info(hsotg->dev, "EPs: %d, %s fifos, %d entries in SPRAM\n",
4486 hsotg->num_of_eps,
4487 hsotg->dedicated_fifos ? "dedicated" : "shared",
4488 hsotg->fifo_mem);
c6f5c050 4489 return 0;
5b7d70c6
BD		 4490}
4491
8b9bc460 4492/**
1f91b4cc 4493 * dwc2_hsotg_dump - dump state of the udc
8b9bc460
LM		 4494 * @param: The device state
4495 */
1f91b4cc 4496static void dwc2_hsotg_dump(struct dwc2_hsotg *hsotg)
5b7d70c6 4497{
83a01804 4498#ifdef DEBUG
5b7d70c6
BD		 4499 struct device *dev = hsotg->dev;
4500 void __iomem *regs = hsotg->regs;
4501 u32 val;
4502 int idx;
4503
4504 dev_info(dev, "DCFG=0x%08x, DCTL=0x%08x, DIEPMSK=%08x\n",
95c8bc36
AS		 4505 dwc2_readl(regs + DCFG), dwc2_readl(regs + DCTL),
4506 dwc2_readl(regs + DIEPMSK));
5b7d70c6 4507
f889f23d 4508 dev_info(dev, "GAHBCFG=0x%08x, GHWCFG1=0x%08x\n",
95c8bc36 4509 dwc2_readl(regs + GAHBCFG), dwc2_readl(regs + GHWCFG1));
5b7d70c6
BD		 4510
4511 dev_info(dev, "GRXFSIZ=0x%08x, GNPTXFSIZ=0x%08x\n",
95c8bc36 4512 dwc2_readl(regs + GRXFSIZ), dwc2_readl(regs + GNPTXFSIZ));
5b7d70c6
BD		 4513
4514 /* show periodic fifo settings */
4515
364f8e93 4516 for (idx = 1; idx < hsotg->num_of_eps; idx++) {
95c8bc36 4517 val = dwc2_readl(regs + DPTXFSIZN(idx));
5b7d70c6 4518 dev_info(dev, "DPTx[%d] FSize=%d, StAddr=0x%08x\n", idx,
47a1685f
DN		 4519 val >> FIFOSIZE_DEPTH_SHIFT,
4520 val & FIFOSIZE_STARTADDR_MASK);
5b7d70c6
BD		 4521 }
4522
364f8e93 4523 for (idx = 0; idx < hsotg->num_of_eps; idx++) {
5b7d70c6
BD		 4524 dev_info(dev,
4525 "ep%d-in: EPCTL=0x%08x, SIZ=0x%08x, DMA=0x%08x\n", idx,
95c8bc36
AS		 4526 dwc2_readl(regs + DIEPCTL(idx)),
4527 dwc2_readl(regs + DIEPTSIZ(idx)),
4528 dwc2_readl(regs + DIEPDMA(idx)));
5b7d70c6 4529
95c8bc36 4530 val = dwc2_readl(regs + DOEPCTL(idx));
5b7d70c6
BD		 4531 dev_info(dev,
4532 "ep%d-out: EPCTL=0x%08x, SIZ=0x%08x, DMA=0x%08x\n",
95c8bc36
AS		 4533 idx, dwc2_readl(regs + DOEPCTL(idx)),
4534 dwc2_readl(regs + DOEPTSIZ(idx)),
4535 dwc2_readl(regs + DOEPDMA(idx)));
5b7d70c6
BD		 4536 }
4537
4538 dev_info(dev, "DVBUSDIS=0x%08x, DVBUSPULSE=%08x\n",
95c8bc36 4539 dwc2_readl(regs + DVBUSDIS), dwc2_readl(regs + DVBUSPULSE));
83a01804 4540#endif
5b7d70c6
BD		 4541}
4542
8b9bc460 4543/**
117777b2
DN		 4544 * dwc2_gadget_init - init function for gadget
4545 * @dwc2: The data structure for the DWC2 driver.
4546 * @irq: The IRQ number for the controller.
8b9bc460 4547 */
117777b2 4548int dwc2_gadget_init(struct dwc2_hsotg *hsotg, int irq)
5b7d70c6 4549{
117777b2 4550 struct device *dev = hsotg->dev;
5b7d70c6
BD		 4551 int epnum;
4552 int ret;
43e90349 4553
0a176279
GH		 4554 /* Dump fifo information */
4555 dev_dbg(dev, "NonPeriodic TXFIFO size: %d\n",
05ee799f
JY		 4556 hsotg->params.g_np_tx_fifo_size);
4557 dev_dbg(dev, "RXFIFO size: %d\n", hsotg->params.g_rx_fifo_size);
5b7d70c6 4558
d327ab5b 4559 hsotg->gadget.max_speed = USB_SPEED_HIGH;
1f91b4cc 4560 hsotg->gadget.ops = &dwc2_hsotg_gadget_ops;
5b7d70c6 4561 hsotg->gadget.name = dev_name(dev);
097ee662
GH		 4562 if (hsotg->dr_mode == USB_DR_MODE_OTG)
4563 hsotg->gadget.is_otg = 1;
ec4cc657
MYK		 4564 else if (hsotg->dr_mode == USB_DR_MODE_PERIPHERAL)
4565 hsotg->op_state = OTG_STATE_B_PERIPHERAL;
5b7d70c6 4566
1f91b4cc 4567 ret = dwc2_hsotg_hw_cfg(hsotg);
c6f5c050
MYK		 4568 if (ret) {
4569 dev_err(hsotg->dev, "Hardware configuration failed: %d\n", ret);
09a75e85 4570 return ret;
c6f5c050
MYK		 4571 }
4572
3f95001d
MYK		 4573 hsotg->ctrl_buff = devm_kzalloc(hsotg->dev,
4574 DWC2_CTRL_BUFF_SIZE, GFP_KERNEL);
8bae0f8c 4575 if (!hsotg->ctrl_buff)
09a75e85 4576 return -ENOMEM;
3f95001d
MYK		 4577
4578 hsotg->ep0_buff = devm_kzalloc(hsotg->dev,
4579 DWC2_CTRL_BUFF_SIZE, GFP_KERNEL);
8bae0f8c 4580 if (!hsotg->ep0_buff)
09a75e85 4581 return -ENOMEM;
3f95001d 4582
0f6b80c0
VA		 4583 if (using_desc_dma(hsotg)) {
4584 ret = dwc2_gadget_alloc_ctrl_desc_chains(hsotg);
4585 if (ret < 0)
4586 return ret;
4587 }
4588
1f91b4cc 4589 ret = devm_request_irq(hsotg->dev, irq, dwc2_hsotg_irq, IRQF_SHARED,
9da51974 4590 dev_name(hsotg->dev), hsotg);
eb3c56c5 4591 if (ret < 0) {
db8178c3 4592 dev_err(dev, "cannot claim IRQ for gadget\n");
09a75e85 4593 return ret;
eb3c56c5
MS		 4594 }
4595
b3f489b2
LM		 4596 /* hsotg->num_of_eps holds number of EPs other than ep0 */
4597
4598 if (hsotg->num_of_eps == 0) {
4599 dev_err(dev, "wrong number of EPs (zero)\n");
09a75e85 4600 return -EINVAL;
b3f489b2
LM		 4601 }
4602
b3f489b2
LM		 4603 /* setup endpoint information */
4604
4605 INIT_LIST_HEAD(&hsotg->gadget.ep_list);
c6f5c050 4606 hsotg->gadget.ep0 = &hsotg->eps_out[0]->ep;
b3f489b2
LM		 4607
4608 /* allocate EP0 request */
4609
1f91b4cc 4610 hsotg->ctrl_req = dwc2_hsotg_ep_alloc_request(&hsotg->eps_out[0]->ep,
b3f489b2
LM		 4611 GFP_KERNEL);
4612 if (!hsotg->ctrl_req) {
4613 dev_err(dev, "failed to allocate ctrl req\n");
09a75e85 4614 return -ENOMEM;
b3f489b2 4615 }
5b7d70c6
BD		 4616
4617 /* initialise the endpoints now the core has been initialised */
c6f5c050
MYK		 4618 for (epnum = 0; epnum < hsotg->num_of_eps; epnum++) {
4619 if (hsotg->eps_in[epnum])
1f91b4cc 4620 dwc2_hsotg_initep(hsotg, hsotg->eps_in[epnum],
9da51974 4621 epnum, 1);
c6f5c050 4622 if (hsotg->eps_out[epnum])
1f91b4cc 4623 dwc2_hsotg_initep(hsotg, hsotg->eps_out[epnum],
9da51974 4624 epnum, 0);
c6f5c050 4625 }
5b7d70c6 4626
117777b2 4627 ret = usb_add_gadget_udc(dev, &hsotg->gadget);
0f91349b 4628 if (ret)
09a75e85 4629 return ret;
0f91349b 4630
1f91b4cc 4631 dwc2_hsotg_dump(hsotg);
5b7d70c6 4632
5b7d70c6 4633 return 0;
5b7d70c6
BD		 4634}
4635
8b9bc460 4636/**
1f91b4cc 4637 * dwc2_hsotg_remove - remove function for hsotg driver
8b9bc460
LM		 4638 * @pdev: The platform information for the driver
4639 */
1f91b4cc 4640int dwc2_hsotg_remove(struct dwc2_hsotg *hsotg)
5b7d70c6 4641{
0f91349b 4642 usb_del_gadget_udc(&hsotg->gadget);
31ee04de 4643
5b7d70c6
BD		 4644 return 0;
4645}
4646
1f91b4cc 4647int dwc2_hsotg_suspend(struct dwc2_hsotg *hsotg)
b83e333a 4648{
b83e333a 4649 unsigned long flags;
b83e333a 4650
9e779778 4651 if (hsotg->lx_state != DWC2_L0)
09a75e85 4652 return 0;
9e779778 4653
dc6e69e6
MS		 4654 if (hsotg->driver) {
4655 int ep;
4656
b83e333a
MS		 4657 dev_info(hsotg->dev, "suspending usb gadget %s\n",
4658 hsotg->driver->driver.name);
4659
dc6e69e6
MS		 4660 spin_lock_irqsave(&hsotg->lock, flags);
4661 if (hsotg->enabled)
1f91b4cc
FB		 4662 dwc2_hsotg_core_disconnect(hsotg);
4663 dwc2_hsotg_disconnect(hsotg);
dc6e69e6
MS		 4664 hsotg->gadget.speed = USB_SPEED_UNKNOWN;
4665 spin_unlock_irqrestore(&hsotg->lock, flags);
b83e333a 4666
c6f5c050
MYK		 4667 for (ep = 0; ep < hsotg->num_of_eps; ep++) {
4668 if (hsotg->eps_in[ep])
1f91b4cc 4669 dwc2_hsotg_ep_disable(&hsotg->eps_in[ep]->ep);
c6f5c050 4670 if (hsotg->eps_out[ep])
1f91b4cc 4671 dwc2_hsotg_ep_disable(&hsotg->eps_out[ep]->ep);
c6f5c050 4672 }
b83e333a
MS		 4673 }
4674
09a75e85 4675 return 0;
b83e333a
MS		 4676}
4677
1f91b4cc 4678int dwc2_hsotg_resume(struct dwc2_hsotg *hsotg)
b83e333a 4679{
b83e333a 4680 unsigned long flags;
b83e333a 4681
9e779778 4682 if (hsotg->lx_state == DWC2_L2)
09a75e85 4683 return 0;
9e779778 4684
b83e333a
MS		 4685 if (hsotg->driver) {
4686 dev_info(hsotg->dev, "resuming usb gadget %s\n",
4687 hsotg->driver->driver.name);
d00b4142 4688
dc6e69e6 4689 spin_lock_irqsave(&hsotg->lock, flags);
1f91b4cc 4690 dwc2_hsotg_core_init_disconnected(hsotg, false);
dc6e69e6 4691 if (hsotg->enabled)
1f91b4cc 4692 dwc2_hsotg_core_connect(hsotg);
dc6e69e6
MS		 4693 spin_unlock_irqrestore(&hsotg->lock, flags);
4694 }
b83e333a 4695
09a75e85 4696 return 0;
b83e333a 4697}
58e52ff6
JY		 4698
4699/**
4700 * dwc2_backup_device_registers() - Backup controller device registers.
4701 * When suspending usb bus, registers needs to be backuped
4702 * if controller power is disabled once suspended.
4703 *
4704 * @hsotg: Programming view of the DWC_otg controller
4705 */
4706int dwc2_backup_device_registers(struct dwc2_hsotg *hsotg)
4707{
4708 struct dwc2_dregs_backup *dr;
4709 int i;
4710
4711 dev_dbg(hsotg->dev, "%s\n", __func__);
4712
4713 /* Backup dev regs */
4714 dr = &hsotg->dr_backup;
4715
4716 dr->dcfg = dwc2_readl(hsotg->regs + DCFG);
4717 dr->dctl = dwc2_readl(hsotg->regs + DCTL);
4718 dr->daintmsk = dwc2_readl(hsotg->regs + DAINTMSK);
4719 dr->diepmsk = dwc2_readl(hsotg->regs + DIEPMSK);
4720 dr->doepmsk = dwc2_readl(hsotg->regs + DOEPMSK);
4721
4722 for (i = 0; i < hsotg->num_of_eps; i++) {
4723 /* Backup IN EPs */
4724 dr->diepctl[i] = dwc2_readl(hsotg->regs + DIEPCTL(i));
4725
4726 /* Ensure DATA PID is correctly configured */
4727 if (dr->diepctl[i] & DXEPCTL_DPID)
4728 dr->diepctl[i] |= DXEPCTL_SETD1PID;
4729 else
4730 dr->diepctl[i] |= DXEPCTL_SETD0PID;
4731
4732 dr->dieptsiz[i] = dwc2_readl(hsotg->regs + DIEPTSIZ(i));
4733 dr->diepdma[i] = dwc2_readl(hsotg->regs + DIEPDMA(i));
4734
4735 /* Backup OUT EPs */
4736 dr->doepctl[i] = dwc2_readl(hsotg->regs + DOEPCTL(i));
4737
4738 /* Ensure DATA PID is correctly configured */
4739 if (dr->doepctl[i] & DXEPCTL_DPID)
4740 dr->doepctl[i] |= DXEPCTL_SETD1PID;
4741 else
4742 dr->doepctl[i] |= DXEPCTL_SETD0PID;
4743
4744 dr->doeptsiz[i] = dwc2_readl(hsotg->regs + DOEPTSIZ(i));
4745 dr->doepdma[i] = dwc2_readl(hsotg->regs + DOEPDMA(i));
4746 }
4747 dr->valid = true;
4748 return 0;
4749}
4750
4751/**
4752 * dwc2_restore_device_registers() - Restore controller device registers.
4753 * When resuming usb bus, device registers needs to be restored
4754 * if controller power were disabled.
4755 *
4756 * @hsotg: Programming view of the DWC_otg controller
4757 */
4758int dwc2_restore_device_registers(struct dwc2_hsotg *hsotg)
4759{
4760 struct dwc2_dregs_backup *dr;
4761 u32 dctl;
4762 int i;
4763
4764 dev_dbg(hsotg->dev, "%s\n", __func__);
4765
4766 /* Restore dev regs */
4767 dr = &hsotg->dr_backup;
4768 if (!dr->valid) {
4769 dev_err(hsotg->dev, "%s: no device registers to restore\n",
4770 __func__);
4771 return -EINVAL;
4772 }
4773 dr->valid = false;
4774
4775 dwc2_writel(dr->dcfg, hsotg->regs + DCFG);
4776 dwc2_writel(dr->dctl, hsotg->regs + DCTL);
4777 dwc2_writel(dr->daintmsk, hsotg->regs + DAINTMSK);
4778 dwc2_writel(dr->diepmsk, hsotg->regs + DIEPMSK);
4779 dwc2_writel(dr->doepmsk, hsotg->regs + DOEPMSK);
4780
4781 for (i = 0; i < hsotg->num_of_eps; i++) {
4782 /* Restore IN EPs */
4783 dwc2_writel(dr->diepctl[i], hsotg->regs + DIEPCTL(i));
4784 dwc2_writel(dr->dieptsiz[i], hsotg->regs + DIEPTSIZ(i));
4785 dwc2_writel(dr->diepdma[i], hsotg->regs + DIEPDMA(i));
4786
4787 /* Restore OUT EPs */
4788 dwc2_writel(dr->doepctl[i], hsotg->regs + DOEPCTL(i));
4789 dwc2_writel(dr->doeptsiz[i], hsotg->regs + DOEPTSIZ(i));
4790 dwc2_writel(dr->doepdma[i], hsotg->regs + DOEPDMA(i));
4791 }
4792
4793 /* Set the Power-On Programming done bit */
4794 dctl = dwc2_readl(hsotg->regs + DCTL);
4795 dctl |= DCTL_PWRONPRGDONE;
4796 dwc2_writel(dctl, hsotg->regs + DCTL);
4797
4798 return 0;
4799}
Linux 6.x block layer and io_uring tree(s)