projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
usb: dwc3: Add DWC_usb31 GRXTHRCFG bit fields
[linux-2.6-block.git] / drivers / usb / dwc3 / gadget.c
CommitLineData
5fd54ace 1// SPDX-License-Identifier: GPL-2.0
bfad65ee 2/*
72246da4
FB		 3 * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
4 *
5 * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com
72246da4
FB		 6 *
7 * Authors: Felipe Balbi <balbi@ti.com>,
8 * Sebastian Andrzej Siewior <bigeasy@linutronix.de>
72246da4
FB		 9 */
10
11#include <linux/kernel.h>
12#include <linux/delay.h>
13#include <linux/slab.h>
14#include <linux/spinlock.h>
15#include <linux/platform_device.h>
16#include <linux/pm_runtime.h>
17#include <linux/interrupt.h>
18#include <linux/io.h>
19#include <linux/list.h>
20#include <linux/dma-mapping.h>
21
22#include <linux/usb/ch9.h>
23#include <linux/usb/gadget.h>
24
80977dc9 25#include "debug.h"
72246da4
FB		 26#include "core.h"
27#include "gadget.h"
28#include "io.h"
29
04a9bfcd 30/**
bfad65ee 31 * dwc3_gadget_set_test_mode - enables usb2 test modes
04a9bfcd
FB		 32 * @dwc: pointer to our context structure
33 * @mode: the mode to set (J, K SE0 NAK, Force Enable)
34 *
bfad65ee
FB		 35 * Caller should take care of locking. This function will return 0 on
36 * success or -EINVAL if wrong Test Selector is passed.
04a9bfcd
FB		 37 */
38int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
39{
40 u32 reg;
41
42 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
43 reg &= ~DWC3_DCTL_TSTCTRL_MASK;
44
45 switch (mode) {
46 case TEST_J:
47 case TEST_K:
48 case TEST_SE0_NAK:
49 case TEST_PACKET:
50 case TEST_FORCE_EN:
51 reg |= mode << 1;
52 break;
53 default:
54 return -EINVAL;
55 }
56
57 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
58
59 return 0;
60}
61
911f1f88 62/**
bfad65ee 63 * dwc3_gadget_get_link_state - gets current state of usb link
911f1f88
PZ		 64 * @dwc: pointer to our context structure
65 *
66 * Caller should take care of locking. This function will
67 * return the link state on success (>= 0) or -ETIMEDOUT.
68 */
69int dwc3_gadget_get_link_state(struct dwc3 *dwc)
70{
71 u32 reg;
72
73 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
74
75 return DWC3_DSTS_USBLNKST(reg);
76}
77
8598bde7 78/**
bfad65ee 79 * dwc3_gadget_set_link_state - sets usb link to a particular state
8598bde7
FB		 80 * @dwc: pointer to our context structure
81 * @state: the state to put link into
82 *
83 * Caller should take care of locking. This function will
aee63e3c 84 * return 0 on success or -ETIMEDOUT.
8598bde7
FB		 85 */
86int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
87{
aee63e3c 88 int retries = 10000;
8598bde7
FB		 89 u32 reg;
90
802fde98
PZ		 91 /*
92 * Wait until device controller is ready. Only applies to 1.94a and
93 * later RTL.
94 */
95 if (dwc->revision >= DWC3_REVISION_194A) {
96 while (--retries) {
97 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
98 if (reg & DWC3_DSTS_DCNRD)
99 udelay(5);
100 else
101 break;
102 }
103
104 if (retries <= 0)
105 return -ETIMEDOUT;
106 }
107
8598bde7
FB		 108 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
109 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
110
111 /* set requested state */
112 reg |= DWC3_DCTL_ULSTCHNGREQ(state);
113 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
114
802fde98
PZ		 115 /*
116 * The following code is racy when called from dwc3_gadget_wakeup,
117 * and is not needed, at least on newer versions
118 */
119 if (dwc->revision >= DWC3_REVISION_194A)
120 return 0;
121
8598bde7 122 /* wait for a change in DSTS */
aed430e5 123 retries = 10000;
8598bde7
FB		 124 while (--retries) {
125 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
126
8598bde7
FB		 127 if (DWC3_DSTS_USBLNKST(reg) == state)
128 return 0;
129
aee63e3c 130 udelay(5);
8598bde7
FB		 131 }
132
8598bde7
FB		 133 return -ETIMEDOUT;
134}
135
dca0119c 136/**
bfad65ee
FB		 137 * dwc3_ep_inc_trb - increment a trb index.
138 * @index: Pointer to the TRB index to increment.
dca0119c
JY		 139 *
140 * The index should never point to the link TRB. After incrementing,
141 * if it is point to the link TRB, wrap around to the beginning. The
142 * link TRB is always at the last TRB entry.
143 */
144static void dwc3_ep_inc_trb(u8 *index)
457e84b6 145{
dca0119c
JY		 146 (*index)++;
147 if (*index == (DWC3_TRB_NUM - 1))
148 *index = 0;
ef966b9d 149}
457e84b6 150
bfad65ee
FB		 151/**
152 * dwc3_ep_inc_enq - increment endpoint's enqueue pointer
153 * @dep: The endpoint whose enqueue pointer we're incrementing
154 */
dca0119c 155static void dwc3_ep_inc_enq(struct dwc3_ep *dep)
ef966b9d 156{
dca0119c 157 dwc3_ep_inc_trb(&dep->trb_enqueue);
ef966b9d 158}
457e84b6 159
bfad65ee
FB		 160/**
161 * dwc3_ep_inc_deq - increment endpoint's dequeue pointer
162 * @dep: The endpoint whose enqueue pointer we're incrementing
163 */
dca0119c 164static void dwc3_ep_inc_deq(struct dwc3_ep *dep)
ef966b9d 165{
dca0119c 166 dwc3_ep_inc_trb(&dep->trb_dequeue);
457e84b6
FB		 167}
168
bfad65ee
FB		 169/**
170 * dwc3_gadget_giveback - call struct usb_request's ->complete callback
171 * @dep: The endpoint to whom the request belongs to
172 * @req: The request we're giving back
173 * @status: completion code for the request
174 *
175 * Must be called with controller's lock held and interrupts disabled. This
176 * function will unmap @req and call its ->complete() callback to notify upper
177 * layers that it has completed.
178 */
72246da4
FB		 179void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
180 int status)
181{
182 struct dwc3 *dwc = dep->dwc;
183
737f1ae2 184 req->started = false;
72246da4 185 list_del(&req->list);
e62c5bc5 186 req->remaining = 0;
72246da4
FB		 187
188 if (req->request.status == -EINPROGRESS)
189 req->request.status = status;
190
4a71fcb8
JP		 191 if (req->trb)
192 usb_gadget_unmap_request_by_dev(dwc->sysdev,
193 &req->request, req->direction);
194
195 req->trb = NULL;
72246da4 196
2c4cbe6e 197 trace_dwc3_gadget_giveback(req);
72246da4
FB		 198
199 spin_unlock(&dwc->lock);
304f7e5e 200 usb_gadget_giveback_request(&dep->endpoint, &req->request);
72246da4 201 spin_lock(&dwc->lock);
fc8bb91b
FB		 202
203 if (dep->number > 1)
204 pm_runtime_put(dwc->dev);
72246da4
FB		 205}
206
bfad65ee
FB		 207/**
208 * dwc3_send_gadget_generic_command - issue a generic command for the controller
209 * @dwc: pointer to the controller context
210 * @cmd: the command to be issued
211 * @param: command parameter
212 *
213 * Caller should take care of locking. Issue @cmd with a given @param to @dwc
214 * and wait for its completion.
215 */
3ece0ec4 216int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned cmd, u32 param)
b09bb642
FB		 217{
218 u32 timeout = 500;
71f7e702 219 int status = 0;
0fe886cd 220 int ret = 0;
b09bb642
FB		 221 u32 reg;
222
223 dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
224 dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);
225
226 do {
227 reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
228 if (!(reg & DWC3_DGCMD_CMDACT)) {
71f7e702
FB		 229 status = DWC3_DGCMD_STATUS(reg);
230 if (status)
0fe886cd
FB		 231 ret = -EINVAL;
232 break;
b09bb642 233 }
e3aee486 234 } while (--timeout);
0fe886cd
FB		 235
236 if (!timeout) {
0fe886cd 237 ret = -ETIMEDOUT;
71f7e702 238 status = -ETIMEDOUT;
0fe886cd
FB		 239 }
240
71f7e702
FB		 241 trace_dwc3_gadget_generic_cmd(cmd, param, status);
242
0fe886cd 243 return ret;
b09bb642
FB		 244}
245
c36d8e94
FB		 246static int __dwc3_gadget_wakeup(struct dwc3 *dwc);
247
bfad65ee
FB		 248/**
249 * dwc3_send_gadget_ep_cmd - issue an endpoint command
250 * @dep: the endpoint to which the command is going to be issued
251 * @cmd: the command to be issued
252 * @params: parameters to the command
253 *
254 * Caller should handle locking. This function will issue @cmd with given
255 * @params to @dep and wait for its completion.
256 */
2cd4718d
FB		 257int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned cmd,
258 struct dwc3_gadget_ep_cmd_params *params)
72246da4 259{
8897a761 260 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
2cd4718d 261 struct dwc3 *dwc = dep->dwc;
8722e095 262 u32 timeout = 1000;
72246da4
FB		 263 u32 reg;
264
0933df15 265 int cmd_status = 0;
2b0f11df 266 int susphy = false;
c0ca324d 267 int ret = -EINVAL;
72246da4 268
2b0f11df
FB		 269 /*
270 * Synopsys Databook 2.60a states, on section 6.3.2.5.[1-8], that if
271 * we're issuing an endpoint command, we must check if
272 * GUSB2PHYCFG.SUSPHY bit is set. If it is, then we need to clear it.
273 *
274 * We will also set SUSPHY bit to what it was before returning as stated
275 * by the same section on Synopsys databook.
276 */
ab2a92e7
FB		 277 if (dwc->gadget.speed <= USB_SPEED_HIGH) {
278 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
279 if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) {
280 susphy = true;
281 reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
282 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
283 }
2b0f11df
FB		 284 }
285
5999914f 286 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
c36d8e94
FB		 287 int needs_wakeup;
288
289 needs_wakeup = (dwc->link_state == DWC3_LINK_STATE_U1 ||
290 dwc->link_state == DWC3_LINK_STATE_U2 ||
291 dwc->link_state == DWC3_LINK_STATE_U3);
292
293 if (unlikely(needs_wakeup)) {
294 ret = __dwc3_gadget_wakeup(dwc);
295 dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n",
296 ret);
297 }
298 }
299
2eb88016
FB		 300 dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0);
301 dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);
302 dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);
72246da4 303
8897a761
FB		 304 /*
305 * Synopsys Databook 2.60a states in section 6.3.2.5.6 of that if we're
306 * not relying on XferNotReady, we can make use of a special "No
307 * Response Update Transfer" command where we should clear both CmdAct
308 * and CmdIOC bits.
309 *
310 * With this, we don't need to wait for command completion and can
311 * straight away issue further commands to the endpoint.
312 *
313 * NOTICE: We're making an assumption that control endpoints will never
314 * make use of Update Transfer command. This is a safe assumption
315 * because we can never have more than one request at a time with
316 * Control Endpoints. If anybody changes that assumption, this chunk
317 * needs to be updated accordingly.
318 */
319 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_UPDATETRANSFER &&
320 !usb_endpoint_xfer_isoc(desc))
321 cmd &= ~(DWC3_DEPCMD_CMDIOC | DWC3_DEPCMD_CMDACT);
322 else
323 cmd |= DWC3_DEPCMD_CMDACT;
324
325 dwc3_writel(dep->regs, DWC3_DEPCMD, cmd);
72246da4 326 do {
2eb88016 327 reg = dwc3_readl(dep->regs, DWC3_DEPCMD);
72246da4 328 if (!(reg & DWC3_DEPCMD_CMDACT)) {
0933df15 329 cmd_status = DWC3_DEPCMD_STATUS(reg);
7b9cc7a2 330
7b9cc7a2
KL		 331 switch (cmd_status) {
332 case 0:
333 ret = 0;
334 break;
335 case DEPEVT_TRANSFER_NO_RESOURCE:
7b9cc7a2 336 ret = -EINVAL;
c0ca324d 337 break;
7b9cc7a2
KL		 338 case DEPEVT_TRANSFER_BUS_EXPIRY:
339 /*
340 * SW issues START TRANSFER command to
341 * isochronous ep with future frame interval. If
342 * future interval time has already passed when
343 * core receives the command, it will respond
344 * with an error status of 'Bus Expiry'.
345 *
346 * Instead of always returning -EINVAL, let's
347 * give a hint to the gadget driver that this is
348 * the case by returning -EAGAIN.
349 */
7b9cc7a2
KL		 350 ret = -EAGAIN;
351 break;
352 default:
353 dev_WARN(dwc->dev, "UNKNOWN cmd status\n");
354 }
355
c0ca324d 356 break;
72246da4 357 }
f6bb225b 358 } while (--timeout);
72246da4 359
f6bb225b 360 if (timeout == 0) {
f6bb225b 361 ret = -ETIMEDOUT;
0933df15 362 cmd_status = -ETIMEDOUT;
f6bb225b 363 }
c0ca324d 364
0933df15
FB		 365 trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);
366
6cb2e4e3
FB		 367 if (ret == 0) {
368 switch (DWC3_DEPCMD_CMD(cmd)) {
369 case DWC3_DEPCMD_STARTTRANSFER:
370 dep->flags |= DWC3_EP_TRANSFER_STARTED;
371 break;
372 case DWC3_DEPCMD_ENDTRANSFER:
373 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
374 break;
375 default:
376 /* nothing */
377 break;
378 }
379 }
380
2b0f11df
FB		 381 if (unlikely(susphy)) {
382 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
383 reg |= DWC3_GUSB2PHYCFG_SUSPHY;
384 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
385 }
386
c0ca324d 387 return ret;
72246da4
FB		 388}
389
50c763f8
JY		 390static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
391{
392 struct dwc3 *dwc = dep->dwc;
393 struct dwc3_gadget_ep_cmd_params params;
394 u32 cmd = DWC3_DEPCMD_CLEARSTALL;
395
396 /*
397 * As of core revision 2.60a the recommended programming model
398 * is to set the ClearPendIN bit when issuing a Clear Stall EP
399 * command for IN endpoints. This is to prevent an issue where
400 * some (non-compliant) hosts may not send ACK TPs for pending
401 * IN transfers due to a mishandled error condition. Synopsys
402 * STAR 9000614252.
403 */
5e6c88d2
LB		 404 if (dep->direction && (dwc->revision >= DWC3_REVISION_260A) &&
405 (dwc->gadget.speed >= USB_SPEED_SUPER))
50c763f8
JY		 406 cmd |= DWC3_DEPCMD_CLEARPENDIN;
407
408 memset(&params, 0, sizeof(params));
409
2cd4718d 410 return dwc3_send_gadget_ep_cmd(dep, cmd, &params);
50c763f8
JY		 411}
412
72246da4 413static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
f6bafc6a 414 struct dwc3_trb *trb)
72246da4 415{
c439ef87 416 u32 offset = (char *) trb - (char *) dep->trb_pool;
72246da4
FB		 417
418 return dep->trb_pool_dma + offset;
419}
420
421static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
422{
423 struct dwc3 *dwc = dep->dwc;
424
425 if (dep->trb_pool)
426 return 0;
427
d64ff406 428 dep->trb_pool = dma_alloc_coherent(dwc->sysdev,
72246da4
FB		 429 sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
430 &dep->trb_pool_dma, GFP_KERNEL);
431 if (!dep->trb_pool) {
432 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
433 dep->name);
434 return -ENOMEM;
435 }
436
437 return 0;
438}
439
440static void dwc3_free_trb_pool(struct dwc3_ep *dep)
441{
442 struct dwc3 *dwc = dep->dwc;
443
d64ff406 444 dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
72246da4
FB		 445 dep->trb_pool, dep->trb_pool_dma);
446
447 dep->trb_pool = NULL;
448 dep->trb_pool_dma = 0;
449}
450
c4509601
JY		 451static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep);
452
453/**
bfad65ee 454 * dwc3_gadget_start_config - configure ep resources
c4509601
JY		 455 * @dwc: pointer to our controller context structure
456 * @dep: endpoint that is being enabled
457 *
bfad65ee
FB		 458 * Issue a %DWC3_DEPCMD_DEPSTARTCFG command to @dep. After the command's
459 * completion, it will set Transfer Resource for all available endpoints.
c4509601 460 *
bfad65ee
FB		 461 * The assignment of transfer resources cannot perfectly follow the data book
462 * due to the fact that the controller driver does not have all knowledge of the
463 * configuration in advance. It is given this information piecemeal by the
464 * composite gadget framework after every SET_CONFIGURATION and
465 * SET_INTERFACE. Trying to follow the databook programming model in this
466 * scenario can cause errors. For two reasons:
c4509601 467 *
bfad65ee
FB		 468 * 1) The databook says to do %DWC3_DEPCMD_DEPSTARTCFG for every
469 * %USB_REQ_SET_CONFIGURATION and %USB_REQ_SET_INTERFACE (8.1.5). This is
470 * incorrect in the scenario of multiple interfaces.
471 *
472 * 2) The databook does not mention doing more %DWC3_DEPCMD_DEPXFERCFG for new
c4509601
JY		 473 * endpoint on alt setting (8.1.6).
474 *
475 * The following simplified method is used instead:
476 *
bfad65ee
FB		 477 * All hardware endpoints can be assigned a transfer resource and this setting
478 * will stay persistent until either a core reset or hibernation. So whenever we
479 * do a %DWC3_DEPCMD_DEPSTARTCFG(0) we can go ahead and do
480 * %DWC3_DEPCMD_DEPXFERCFG for every hardware endpoint as well. We are
c4509601
JY		 481 * guaranteed that there are as many transfer resources as endpoints.
482 *
bfad65ee
FB		 483 * This function is called for each endpoint when it is being enabled but is
484 * triggered only when called for EP0-out, which always happens first, and which
485 * should only happen in one of the above conditions.
c4509601 486 */
72246da4
FB		 487static int dwc3_gadget_start_config(struct dwc3 *dwc, struct dwc3_ep *dep)
488{
489 struct dwc3_gadget_ep_cmd_params params;
490 u32 cmd;
c4509601
JY		 491 int i;
492 int ret;
493
494 if (dep->number)
495 return 0;
72246da4
FB		 496
497 memset(&params, 0x00, sizeof(params));
c4509601 498 cmd = DWC3_DEPCMD_DEPSTARTCFG;
72246da4 499
2cd4718d 500 ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
c4509601
JY		 501 if (ret)
502 return ret;
503
504 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
505 struct dwc3_ep *dep = dwc->eps[i];
72246da4 506
c4509601
JY		 507 if (!dep)
508 continue;
509
510 ret = dwc3_gadget_set_xfer_resource(dwc, dep);
511 if (ret)
512 return ret;
72246da4
FB		 513 }
514
515 return 0;
516}
517
518static int dwc3_gadget_set_ep_config(struct dwc3 *dwc, struct dwc3_ep *dep,
21e64bf2 519 bool modify, bool restore)
72246da4 520{
39ebb05c
JY		 521 const struct usb_ss_ep_comp_descriptor *comp_desc;
522 const struct usb_endpoint_descriptor *desc;
72246da4
FB		 523 struct dwc3_gadget_ep_cmd_params params;
524
21e64bf2
FB		 525 if (dev_WARN_ONCE(dwc->dev, modify && restore,
526 "Can't modify and restore\n"))
527 return -EINVAL;
528
39ebb05c
JY		 529 comp_desc = dep->endpoint.comp_desc;
530 desc = dep->endpoint.desc;
531
72246da4
FB		 532 memset(&params, 0x00, sizeof(params));
533
dc1c70a7 534 params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
d2e9a13a
CP		 535 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
536
537 /* Burst size is only needed in SuperSpeed mode */
ee5cd41c 538 if (dwc->gadget.speed >= USB_SPEED_SUPER) {
676e3497 539 u32 burst = dep->endpoint.maxburst;
676e3497 540 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1);
d2e9a13a 541 }
72246da4 542
21e64bf2
FB		 543 if (modify) {
544 params.param0 |= DWC3_DEPCFG_ACTION_MODIFY;
545 } else if (restore) {
265b70a7
PZ		 546 params.param0 |= DWC3_DEPCFG_ACTION_RESTORE;
547 params.param2 |= dep->saved_state;
21e64bf2
FB		 548 } else {
549 params.param0 |= DWC3_DEPCFG_ACTION_INIT;
265b70a7
PZ		 550 }
551
4bc48c97
FB		 552 if (usb_endpoint_xfer_control(desc))
553 params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN;
13fa2e69
FB		 554
555 if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc))
556 params.param1 |= DWC3_DEPCFG_XFER_NOT_READY_EN;
72246da4 557
18b7ede5 558 if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
dc1c70a7
FB		 559 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
560 | DWC3_DEPCFG_STREAM_EVENT_EN;
879631aa
FB		 561 dep->stream_capable = true;
562 }
563
0b93a4c8 564 if (!usb_endpoint_xfer_control(desc))
dc1c70a7 565 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
72246da4
FB		 566
567 /*
568 * We are doing 1:1 mapping for endpoints, meaning
569 * Physical Endpoints 2 maps to Logical Endpoint 2 and
570 * so on. We consider the direction bit as part of the physical
571 * endpoint number. So USB endpoint 0x81 is 0x03.
572 */
dc1c70a7 573 params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
72246da4
FB		 574
575 /*
576 * We must use the lower 16 TX FIFOs even though
577 * HW might have more
578 */
579 if (dep->direction)
dc1c70a7 580 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
72246da4
FB		 581
582 if (desc->bInterval) {
dc1c70a7 583 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(desc->bInterval - 1);
72246da4
FB		 584 dep->interval = 1 << (desc->bInterval - 1);
585 }
586
2cd4718d 587 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, &params);
72246da4
FB		 588}
589
590static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep)
591{
592 struct dwc3_gadget_ep_cmd_params params;
593
594 memset(&params, 0x00, sizeof(params));
595
dc1c70a7 596 params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
72246da4 597
2cd4718d
FB		 598 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE,
599 &params);
72246da4
FB		 600}
601
602/**
bfad65ee 603 * __dwc3_gadget_ep_enable - initializes a hw endpoint
72246da4 604 * @dep: endpoint to be initialized
bfad65ee
FB		 605 * @modify: if true, modify existing endpoint configuration
606 * @restore: if true, restore endpoint configuration from scratch buffer
72246da4 607 *
bfad65ee
FB		 608 * Caller should take care of locking. Execute all necessary commands to
609 * initialize a HW endpoint so it can be used by a gadget driver.
72246da4
FB		 610 */
611static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep,
21e64bf2 612 bool modify, bool restore)
72246da4 613{
39ebb05c 614 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
72246da4 615 struct dwc3 *dwc = dep->dwc;
39ebb05c 616
72246da4 617 u32 reg;
b09e99ee 618 int ret;
72246da4
FB		 619
620 if (!(dep->flags & DWC3_EP_ENABLED)) {
621 ret = dwc3_gadget_start_config(dwc, dep);
622 if (ret)
623 return ret;
624 }
625
39ebb05c 626 ret = dwc3_gadget_set_ep_config(dwc, dep, modify, restore);
72246da4
FB		 627 if (ret)
628 return ret;
629
630 if (!(dep->flags & DWC3_EP_ENABLED)) {
f6bafc6a
FB		 631 struct dwc3_trb *trb_st_hw;
632 struct dwc3_trb *trb_link;
72246da4 633
72246da4
FB		 634 dep->type = usb_endpoint_type(desc);
635 dep->flags |= DWC3_EP_ENABLED;
76a638f8 636 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
72246da4
FB		 637
638 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
639 reg |= DWC3_DALEPENA_EP(dep->number);
640 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
641
76a638f8
BW		 642 init_waitqueue_head(&dep->wait_end_transfer);
643
36b68aae 644 if (usb_endpoint_xfer_control(desc))
2870e501 645 goto out;
72246da4 646
0d25744a
JY		 647 /* Initialize the TRB ring */
648 dep->trb_dequeue = 0;
649 dep->trb_enqueue = 0;
650 memset(dep->trb_pool, 0,
651 sizeof(struct dwc3_trb) * DWC3_TRB_NUM);
652
36b68aae 653 /* Link TRB. The HWO bit is never reset */
72246da4
FB		 654 trb_st_hw = &dep->trb_pool[0];
655
f6bafc6a 656 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
f6bafc6a
FB		 657 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
658 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
659 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
660 trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
72246da4
FB		 661 }
662
a97ea994
FB		 663 /*
664 * Issue StartTransfer here with no-op TRB so we can always rely on No
665 * Response Update Transfer command.
666 */
667 if (usb_endpoint_xfer_bulk(desc)) {
668 struct dwc3_gadget_ep_cmd_params params;
669 struct dwc3_trb *trb;
670 dma_addr_t trb_dma;
671 u32 cmd;
672
673 memset(&params, 0, sizeof(params));
674 trb = &dep->trb_pool[0];
675 trb_dma = dwc3_trb_dma_offset(dep, trb);
676
677 params.param0 = upper_32_bits(trb_dma);
678 params.param1 = lower_32_bits(trb_dma);
679
680 cmd = DWC3_DEPCMD_STARTTRANSFER;
681
682 ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
683 if (ret < 0)
684 return ret;
685
686 dep->flags |= DWC3_EP_BUSY;
687
688 dep->resource_index = dwc3_gadget_ep_get_transfer_index(dep);
689 WARN_ON_ONCE(!dep->resource_index);
690 }
691
2870e501
FB		 692
693out:
694 trace_dwc3_gadget_ep_enable(dep);
695
72246da4
FB		 696 return 0;
697}
698
b992e681 699static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force);
624407f9 700static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)
72246da4
FB		 701{
702 struct dwc3_request *req;
703
0e146028 704 dwc3_stop_active_transfer(dwc, dep->number, true);
624407f9 705
0e146028
FB		 706 /* - giveback all requests to gadget driver */
707 while (!list_empty(&dep->started_list)) {
708 req = next_request(&dep->started_list);
1591633e 709
0e146028 710 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
ea53b882
FB		 711 }
712
aa3342c8
FB		 713 while (!list_empty(&dep->pending_list)) {
714 req = next_request(&dep->pending_list);
72246da4 715
624407f9 716 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
72246da4 717 }
72246da4
FB		 718}
719
720/**
bfad65ee 721 * __dwc3_gadget_ep_disable - disables a hw endpoint
72246da4
FB		 722 * @dep: the endpoint to disable
723 *
bfad65ee
FB		 724 * This function undoes what __dwc3_gadget_ep_enable did and also removes
725 * requests which are currently being processed by the hardware and those which
726 * are not yet scheduled.
727 *
624407f9 728 * Caller should take care of locking.
72246da4 729 */
72246da4
FB		 730static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
731{
732 struct dwc3 *dwc = dep->dwc;
733 u32 reg;
734
2870e501 735 trace_dwc3_gadget_ep_disable(dep);
7eaeac5c 736
624407f9 737 dwc3_remove_requests(dwc, dep);
72246da4 738
687ef981
FB		 739 /* make sure HW endpoint isn't stalled */
740 if (dep->flags & DWC3_EP_STALL)
7a608559 741 __dwc3_gadget_ep_set_halt(dep, 0, false);
687ef981 742
72246da4
FB		 743 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
744 reg &= ~DWC3_DALEPENA_EP(dep->number);
745 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
746
879631aa 747 dep->stream_capable = false;
72246da4 748 dep->type = 0;
76a638f8 749 dep->flags &= DWC3_EP_END_TRANSFER_PENDING;
72246da4 750
39ebb05c
JY		 751 /* Clear out the ep descriptors for non-ep0 */
752 if (dep->number > 1) {
753 dep->endpoint.comp_desc = NULL;
754 dep->endpoint.desc = NULL;
755 }
756
72246da4
FB		 757 return 0;
758}
759
760/* -------------------------------------------------------------------------- */
761
762static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
763 const struct usb_endpoint_descriptor *desc)
764{
765 return -EINVAL;
766}
767
768static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
769{
770 return -EINVAL;
771}
772
773/* -------------------------------------------------------------------------- */
774
775static int dwc3_gadget_ep_enable(struct usb_ep *ep,
776 const struct usb_endpoint_descriptor *desc)
777{
778 struct dwc3_ep *dep;
779 struct dwc3 *dwc;
780 unsigned long flags;
781 int ret;
782
783 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
784 pr_debug("dwc3: invalid parameters\n");
785 return -EINVAL;
786 }
787
788 if (!desc->wMaxPacketSize) {
789 pr_debug("dwc3: missing wMaxPacketSize\n");
790 return -EINVAL;
791 }
792
793 dep = to_dwc3_ep(ep);
794 dwc = dep->dwc;
795
95ca961c
FB		 796 if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
797 "%s is already enabled\n",
798 dep->name))
c6f83f38 799 return 0;
c6f83f38 800
72246da4 801 spin_lock_irqsave(&dwc->lock, flags);
39ebb05c 802 ret = __dwc3_gadget_ep_enable(dep, false, false);
72246da4
FB		 803 spin_unlock_irqrestore(&dwc->lock, flags);
804
805 return ret;
806}
807
808static int dwc3_gadget_ep_disable(struct usb_ep *ep)
809{
810 struct dwc3_ep *dep;
811 struct dwc3 *dwc;
812 unsigned long flags;
813 int ret;
814
815 if (!ep) {
816 pr_debug("dwc3: invalid parameters\n");
817 return -EINVAL;
818 }
819
820 dep = to_dwc3_ep(ep);
821 dwc = dep->dwc;
822
95ca961c
FB		 823 if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
824 "%s is already disabled\n",
825 dep->name))
72246da4 826 return 0;
72246da4 827
72246da4
FB		 828 spin_lock_irqsave(&dwc->lock, flags);
829 ret = __dwc3_gadget_ep_disable(dep);
830 spin_unlock_irqrestore(&dwc->lock, flags);
831
832 return ret;
833}
834
835static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
836 gfp_t gfp_flags)
837{
838 struct dwc3_request *req;
839 struct dwc3_ep *dep = to_dwc3_ep(ep);
72246da4
FB		 840
841 req = kzalloc(sizeof(*req), gfp_flags);
734d5a53 842 if (!req)
72246da4 843 return NULL;
72246da4
FB		 844
845 req->epnum = dep->number;
846 req->dep = dep;
72246da4 847
68d34c8a
FB		 848 dep->allocated_requests++;
849
2c4cbe6e
FB		 850 trace_dwc3_alloc_request(req);
851
72246da4
FB		 852 return &req->request;
853}
854
855static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
856 struct usb_request *request)
857{
858 struct dwc3_request *req = to_dwc3_request(request);
68d34c8a 859 struct dwc3_ep *dep = to_dwc3_ep(ep);
72246da4 860
68d34c8a 861 dep->allocated_requests--;
2c4cbe6e 862 trace_dwc3_free_request(req);
72246da4
FB		 863 kfree(req);
864}
865
2c78c029
FB		 866static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep);
867
e49d3cf4
FB		 868static void __dwc3_prepare_one_trb(struct dwc3_ep *dep, struct dwc3_trb *trb,
869 dma_addr_t dma, unsigned length, unsigned chain, unsigned node,
870 unsigned stream_id, unsigned short_not_ok, unsigned no_interrupt)
c71fc37c 871{
6b9018d4
FB		 872 struct dwc3 *dwc = dep->dwc;
873 struct usb_gadget *gadget = &dwc->gadget;
874 enum usb_device_speed speed = gadget->speed;
c71fc37c 875
ef966b9d 876 dwc3_ep_inc_enq(dep);
e5ba5ec8 877
f6bafc6a
FB		 878 trb->size = DWC3_TRB_SIZE_LENGTH(length);
879 trb->bpl = lower_32_bits(dma);
880 trb->bph = upper_32_bits(dma);
c71fc37c 881
16e78db7 882 switch (usb_endpoint_type(dep->endpoint.desc)) {
c71fc37c 883 case USB_ENDPOINT_XFER_CONTROL:
f6bafc6a 884 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
c71fc37c
FB		 885 break;
886
887 case USB_ENDPOINT_XFER_ISOC:
6b9018d4 888 if (!node) {
e5ba5ec8 889 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
6b9018d4 890
40d829fb
MG		 891 /*
892 * USB Specification 2.0 Section 5.9.2 states that: "If
893 * there is only a single transaction in the microframe,
894 * only a DATA0 data packet PID is used. If there are
895 * two transactions per microframe, DATA1 is used for
896 * the first transaction data packet and DATA0 is used
897 * for the second transaction data packet. If there are
898 * three transactions per microframe, DATA2 is used for
899 * the first transaction data packet, DATA1 is used for
900 * the second, and DATA0 is used for the third."
901 *
902 * IOW, we should satisfy the following cases:
903 *
904 * 1) length <= maxpacket
905 * - DATA0
906 *
907 * 2) maxpacket < length <= (2 * maxpacket)
908 * - DATA1, DATA0
909 *
910 * 3) (2 * maxpacket) < length <= (3 * maxpacket)
911 * - DATA2, DATA1, DATA0
912 */
6b9018d4
FB		 913 if (speed == USB_SPEED_HIGH) {
914 struct usb_ep *ep = &dep->endpoint;
ec5bb87e 915 unsigned int mult = 2;
40d829fb
MG		 916 unsigned int maxp = usb_endpoint_maxp(ep->desc);
917
918 if (length <= (2 * maxp))
919 mult--;
920
921 if (length <= maxp)
922 mult--;
923
924 trb->size |= DWC3_TRB_SIZE_PCM1(mult);
6b9018d4
FB		 925 }
926 } else {
e5ba5ec8 927 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
6b9018d4 928 }
ca4d44ea
FB		 929
930 /* always enable Interrupt on Missed ISOC */
931 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
c71fc37c
FB		 932 break;
933
934 case USB_ENDPOINT_XFER_BULK:
935 case USB_ENDPOINT_XFER_INT:
f6bafc6a 936 trb->ctrl = DWC3_TRBCTL_NORMAL;
c71fc37c
FB		 937 break;
938 default:
939 /*
940 * This is only possible with faulty memory because we
941 * checked it already :)
942 */
0a695d4c
FB		 943 dev_WARN(dwc->dev, "Unknown endpoint type %d\n",
944 usb_endpoint_type(dep->endpoint.desc));
c71fc37c
FB		 945 }
946
ca4d44ea 947 /* always enable Continue on Short Packet */
c9508c8c 948 if (usb_endpoint_dir_out(dep->endpoint.desc)) {
58f29034 949 trb->ctrl |= DWC3_TRB_CTRL_CSP;
f3af3651 950
e49d3cf4 951 if (short_not_ok)
c9508c8c
FB		 952 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
953 }
954
e49d3cf4 955 if ((!no_interrupt && !chain) ||
2c78c029 956 (dwc3_calc_trbs_left(dep) == 0))
c9508c8c 957 trb->ctrl |= DWC3_TRB_CTRL_IOC;
f3af3651 958
e5ba5ec8
PA		 959 if (chain)
960 trb->ctrl |= DWC3_TRB_CTRL_CHN;
961
16e78db7 962 if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
e49d3cf4 963 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id);
c71fc37c 964
f6bafc6a 965 trb->ctrl |= DWC3_TRB_CTRL_HWO;
2c4cbe6e
FB		 966
967 trace_dwc3_prepare_trb(dep, trb);
c71fc37c
FB		 968}
969
e49d3cf4
FB		 970/**
971 * dwc3_prepare_one_trb - setup one TRB from one request
972 * @dep: endpoint for which this request is prepared
973 * @req: dwc3_request pointer
974 * @chain: should this TRB be chained to the next?
975 * @node: only for isochronous endpoints. First TRB needs different type.
976 */
977static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
978 struct dwc3_request *req, unsigned chain, unsigned node)
979{
980 struct dwc3_trb *trb;
981 unsigned length = req->request.length;
982 unsigned stream_id = req->request.stream_id;
983 unsigned short_not_ok = req->request.short_not_ok;
984 unsigned no_interrupt = req->request.no_interrupt;
985 dma_addr_t dma = req->request.dma;
986
987 trb = &dep->trb_pool[dep->trb_enqueue];
988
989 if (!req->trb) {
990 dwc3_gadget_move_started_request(req);
991 req->trb = trb;
992 req->trb_dma = dwc3_trb_dma_offset(dep, trb);
993 dep->queued_requests++;
994 }
995
996 __dwc3_prepare_one_trb(dep, trb, dma, length, chain, node,
997 stream_id, short_not_ok, no_interrupt);
998}
999
361572b5 1000/**
bfad65ee 1001 * dwc3_ep_prev_trb - returns the previous TRB in the ring
361572b5
JY		 1002 * @dep: The endpoint with the TRB ring
1003 * @index: The index of the current TRB in the ring
1004 *
1005 * Returns the TRB prior to the one pointed to by the index. If the
1006 * index is 0, we will wrap backwards, skip the link TRB, and return
1007 * the one just before that.
1008 */
1009static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index)
1010{
45438a0c 1011 u8 tmp = index;
361572b5 1012
45438a0c
FB		 1013 if (!tmp)
1014 tmp = DWC3_TRB_NUM - 1;
361572b5 1015
45438a0c 1016 return &dep->trb_pool[tmp - 1];
361572b5
JY		 1017}
1018
c4233573
FB		 1019static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
1020{
1021 struct dwc3_trb *tmp;
32db3d94 1022 u8 trbs_left;
c4233573
FB		 1023
1024 /*
1025 * If enqueue & dequeue are equal than it is either full or empty.
1026 *
1027 * One way to know for sure is if the TRB right before us has HWO bit
1028 * set or not. If it has, then we're definitely full and can't fit any
1029 * more transfers in our ring.
1030 */
1031 if (dep->trb_enqueue == dep->trb_dequeue) {
361572b5 1032 tmp = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
202adafe 1033 if (tmp->ctrl & DWC3_TRB_CTRL_HWO)
361572b5 1034 return 0;
c4233573
FB		 1035
1036 return DWC3_TRB_NUM - 1;
1037 }
1038
9d7aba77 1039 trbs_left = dep->trb_dequeue - dep->trb_enqueue;
3de2685f 1040 trbs_left &= (DWC3_TRB_NUM - 1);
32db3d94 1041
9d7aba77
JY		 1042 if (dep->trb_dequeue < dep->trb_enqueue)
1043 trbs_left--;
1044
32db3d94 1045 return trbs_left;
c4233573
FB		 1046}
1047
5ee85d89 1048static void dwc3_prepare_one_trb_sg(struct dwc3_ep *dep,
7ae7df49 1049 struct dwc3_request *req)
5ee85d89 1050{
1f512119 1051 struct scatterlist *sg = req->sg;
5ee85d89 1052 struct scatterlist *s;
5ee85d89
FB		 1053 int i;
1054
1f512119 1055 for_each_sg(sg, s, req->num_pending_sgs, i) {
c6267a51
FB		 1056 unsigned int length = req->request.length;
1057 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1058 unsigned int rem = length % maxp;
5ee85d89
FB		 1059 unsigned chain = true;
1060
4bc48c97 1061 if (sg_is_last(s))
5ee85d89
FB		 1062 chain = false;
1063
c6267a51
FB		 1064 if (rem && usb_endpoint_dir_out(dep->endpoint.desc) && !chain) {
1065 struct dwc3 *dwc = dep->dwc;
1066 struct dwc3_trb *trb;
1067
1068 req->unaligned = true;
1069
1070 /* prepare normal TRB */
1071 dwc3_prepare_one_trb(dep, req, true, i);
1072
1073 /* Now prepare one extra TRB to align transfer size */
1074 trb = &dep->trb_pool[dep->trb_enqueue];
1075 __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr,
1076 maxp - rem, false, 0,
1077 req->request.stream_id,
1078 req->request.short_not_ok,
1079 req->request.no_interrupt);
1080 } else {
1081 dwc3_prepare_one_trb(dep, req, chain, i);
1082 }
5ee85d89 1083
7ae7df49 1084 if (!dwc3_calc_trbs_left(dep))
5ee85d89
FB		 1085 break;
1086 }
1087}
1088
1089static void dwc3_prepare_one_trb_linear(struct dwc3_ep *dep,
7ae7df49 1090 struct dwc3_request *req)
5ee85d89 1091{
c6267a51
FB		 1092 unsigned int length = req->request.length;
1093 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1094 unsigned int rem = length % maxp;
1095
1096 if (rem && usb_endpoint_dir_out(dep->endpoint.desc)) {
1097 struct dwc3 *dwc = dep->dwc;
1098 struct dwc3_trb *trb;
1099
1100 req->unaligned = true;
1101
1102 /* prepare normal TRB */
1103 dwc3_prepare_one_trb(dep, req, true, 0);
1104
1105 /* Now prepare one extra TRB to align transfer size */
1106 trb = &dep->trb_pool[dep->trb_enqueue];
1107 __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, maxp - rem,
1108 false, 0, req->request.stream_id,
1109 req->request.short_not_ok,
1110 req->request.no_interrupt);
d6e5a549
FB		 1111 } else if (req->request.zero && req->request.length &&
1112 (IS_ALIGNED(req->request.length,dep->endpoint.maxpacket))) {
1113 struct dwc3 *dwc = dep->dwc;
1114 struct dwc3_trb *trb;
1115
1116 req->zero = true;
1117
1118 /* prepare normal TRB */
1119 dwc3_prepare_one_trb(dep, req, true, 0);
1120
1121 /* Now prepare one extra TRB to handle ZLP */
1122 trb = &dep->trb_pool[dep->trb_enqueue];
1123 __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, 0,
1124 false, 0, req->request.stream_id,
1125 req->request.short_not_ok,
1126 req->request.no_interrupt);
c6267a51
FB		 1127 } else {
1128 dwc3_prepare_one_trb(dep, req, false, 0);
1129 }
5ee85d89
FB		 1130}
1131
72246da4
FB		 1132/*
1133 * dwc3_prepare_trbs - setup TRBs from requests
1134 * @dep: endpoint for which requests are being prepared
72246da4 1135 *
1d046793
PZ		 1136 * The function goes through the requests list and sets up TRBs for the
1137 * transfers. The function returns once there are no more TRBs available or
1138 * it runs out of requests.
72246da4 1139 */
c4233573 1140static void dwc3_prepare_trbs(struct dwc3_ep *dep)
72246da4 1141{
68e823e2 1142 struct dwc3_request *req, *n;
72246da4
FB		 1143
1144 BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
1145
d86c5a67
FB		 1146 /*
1147 * We can get in a situation where there's a request in the started list
1148 * but there weren't enough TRBs to fully kick it in the first time
1149 * around, so it has been waiting for more TRBs to be freed up.
1150 *
1151 * In that case, we should check if we have a request with pending_sgs
1152 * in the started list and prepare TRBs for that request first,
1153 * otherwise we will prepare TRBs completely out of order and that will
1154 * break things.
1155 */
1156 list_for_each_entry(req, &dep->started_list, list) {
1157 if (req->num_pending_sgs > 0)
1158 dwc3_prepare_one_trb_sg(dep, req);
1159
1160 if (!dwc3_calc_trbs_left(dep))
1161 return;
1162 }
1163
aa3342c8 1164 list_for_each_entry_safe(req, n, &dep->pending_list, list) {
cdb55b39
FB		 1165 struct dwc3 *dwc = dep->dwc;
1166 int ret;
1167
1168 ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request,
1169 dep->direction);
1170 if (ret)
1171 return;
1172
1173 req->sg = req->request.sg;
1174 req->num_pending_sgs = req->request.num_mapped_sgs;
1175
1f512119 1176 if (req->num_pending_sgs > 0)
7ae7df49 1177 dwc3_prepare_one_trb_sg(dep, req);
5ee85d89 1178 else
7ae7df49 1179 dwc3_prepare_one_trb_linear(dep, req);
72246da4 1180
7ae7df49 1181 if (!dwc3_calc_trbs_left(dep))
5ee85d89 1182 return;
72246da4 1183 }
72246da4
FB		 1184}
1185
7fdca766 1186static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep)
72246da4
FB		 1187{
1188 struct dwc3_gadget_ep_cmd_params params;
1189 struct dwc3_request *req;
4fae2e3e 1190 int starting;
72246da4
FB		 1191 int ret;
1192 u32 cmd;
1193
ccb94ebf
FB		 1194 if (!dwc3_calc_trbs_left(dep))
1195 return 0;
1196
4fae2e3e 1197 starting = !(dep->flags & DWC3_EP_BUSY);
72246da4 1198
4fae2e3e
FB		 1199 dwc3_prepare_trbs(dep);
1200 req = next_request(&dep->started_list);
72246da4
FB		 1201 if (!req) {
1202 dep->flags |= DWC3_EP_PENDING_REQUEST;
1203 return 0;
1204 }
1205
1206 memset(&params, 0, sizeof(params));
72246da4 1207
4fae2e3e 1208 if (starting) {
1877d6c9
PA		 1209 params.param0 = upper_32_bits(req->trb_dma);
1210 params.param1 = lower_32_bits(req->trb_dma);
7fdca766
FB		 1211 cmd = DWC3_DEPCMD_STARTTRANSFER;
1212
1213 if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
1214 cmd |= DWC3_DEPCMD_PARAM(dep->frame_number);
1877d6c9 1215 } else {
b6b1c6db
FB		 1216 cmd = DWC3_DEPCMD_UPDATETRANSFER |
1217 DWC3_DEPCMD_PARAM(dep->resource_index);
1877d6c9 1218 }
72246da4 1219
2cd4718d 1220 ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
72246da4 1221 if (ret < 0) {
72246da4
FB		 1222 /*
1223 * FIXME we need to iterate over the list of requests
1224 * here and stop, unmap, free and del each of the linked
1d046793 1225 * requests instead of what we do now.
72246da4 1226 */
ce3fc8b3
JD		 1227 if (req->trb)
1228 memset(req->trb, 0, sizeof(struct dwc3_trb));
8ab89da4 1229 dep->queued_requests--;
15b8d933 1230 dwc3_gadget_giveback(dep, req, ret);
72246da4
FB		 1231 return ret;
1232 }
1233
1234 dep->flags |= DWC3_EP_BUSY;
25b8ff68 1235
4fae2e3e 1236 if (starting) {
2eb88016 1237 dep->resource_index = dwc3_gadget_ep_get_transfer_index(dep);
b4996a86 1238 WARN_ON_ONCE(!dep->resource_index);
f898ae09 1239 }
25b8ff68 1240
72246da4
FB		 1241 return 0;
1242}
1243
6cb2e4e3
FB		 1244static int __dwc3_gadget_get_frame(struct dwc3 *dwc)
1245{
1246 u32 reg;
1247
1248 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1249 return DWC3_DSTS_SOFFN(reg);
1250}
1251
d6d6ec7b
PA		 1252static void __dwc3_gadget_start_isoc(struct dwc3 *dwc,
1253 struct dwc3_ep *dep, u32 cur_uf)
1254{
aa3342c8 1255 if (list_empty(&dep->pending_list)) {
5eb30ced 1256 dev_info(dwc->dev, "%s: ran out of requests\n",
73815280 1257 dep->name);
f4a53c55 1258 dep->flags |= DWC3_EP_PENDING_REQUEST;
d6d6ec7b
PA		 1259 return;
1260 }
1261
af771d73
JY		 1262 /*
1263 * Schedule the first trb for one interval in the future or at
1264 * least 4 microframes.
1265 */
502a37b9 1266 dep->frame_number = cur_uf + max_t(u32, 4, dep->interval);
7fdca766 1267 __dwc3_gadget_kick_transfer(dep);
d6d6ec7b
PA		 1268}
1269
1270static void dwc3_gadget_start_isoc(struct dwc3 *dwc,
1271 struct dwc3_ep *dep, const struct dwc3_event_depevt *event)
1272{
1273 u32 cur_uf, mask;
1274
1275 mask = ~(dep->interval - 1);
1276 cur_uf = event->parameters & mask;
1277
1278 __dwc3_gadget_start_isoc(dwc, dep, cur_uf);
1279}
1280
72246da4
FB		 1281static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1282{
0fc9a1be 1283 struct dwc3 *dwc = dep->dwc;
0fc9a1be 1284
bb423984 1285 if (!dep->endpoint.desc) {
5eb30ced
FB		 1286 dev_err(dwc->dev, "%s: can't queue to disabled endpoint\n",
1287 dep->name);
bb423984
FB		 1288 return -ESHUTDOWN;
1289 }
1290
04fb365c
FB		 1291 if (WARN(req->dep != dep, "request %pK belongs to '%s'\n",
1292 &req->request, req->dep->name))
bb423984 1293 return -EINVAL;
bb423984 1294
fc8bb91b
FB		 1295 pm_runtime_get(dwc->dev);
1296
72246da4
FB		 1297 req->request.actual = 0;
1298 req->request.status = -EINPROGRESS;
1299 req->direction = dep->direction;
1300 req->epnum = dep->number;
1301
fe84f522
FB		 1302 trace_dwc3_ep_queue(req);
1303
aa3342c8 1304 list_add_tail(&req->list, &dep->pending_list);
72246da4 1305
d889c23c
FB		 1306 /*
1307 * NOTICE: Isochronous endpoints should NEVER be prestarted. We must
1308 * wait for a XferNotReady event so we will know what's the current
1309 * (micro-)frame number.
1310 *
1311 * Without this trick, we are very, very likely gonna get Bus Expiry
1312 * errors which will force us issue EndTransfer command.
1313 */
1314 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
6cb2e4e3
FB		 1315 if ((dep->flags & DWC3_EP_PENDING_REQUEST)) {
1316 if (dep->flags & DWC3_EP_TRANSFER_STARTED) {
1317 dwc3_stop_active_transfer(dwc, dep->number, true);
1318 dep->flags = DWC3_EP_ENABLED;
1319 } else {
1320 u32 cur_uf;
1321
1322 cur_uf = __dwc3_gadget_get_frame(dwc);
1323 __dwc3_gadget_start_isoc(dwc, dep, cur_uf);
87aba106 1324 dep->flags &= ~DWC3_EP_PENDING_REQUEST;
6cb2e4e3 1325 }
f1d6826c 1326 return 0;
08a36b54 1327 }
f1d6826c
RQ		 1328
1329 if ((dep->flags & DWC3_EP_BUSY) &&
64e01080
FB		 1330 !(dep->flags & DWC3_EP_MISSED_ISOC))
1331 goto out;
72246da4 1332
594e121f 1333 return 0;
64e01080 1334 }
b997ada5 1335
f1d6826c 1336out:
7fdca766 1337 return __dwc3_gadget_kick_transfer(dep);
72246da4
FB		 1338}
1339
1340static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1341 gfp_t gfp_flags)
1342{
1343 struct dwc3_request *req = to_dwc3_request(request);
1344 struct dwc3_ep *dep = to_dwc3_ep(ep);
1345 struct dwc3 *dwc = dep->dwc;
1346
1347 unsigned long flags;
1348
1349 int ret;
1350
fdee4eba 1351 spin_lock_irqsave(&dwc->lock, flags);
72246da4
FB		 1352 ret = __dwc3_gadget_ep_queue(dep, req);
1353 spin_unlock_irqrestore(&dwc->lock, flags);
1354
1355 return ret;
1356}
1357
1358static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
1359 struct usb_request *request)
1360{
1361 struct dwc3_request *req = to_dwc3_request(request);
1362 struct dwc3_request *r = NULL;
1363
1364 struct dwc3_ep *dep = to_dwc3_ep(ep);
1365 struct dwc3 *dwc = dep->dwc;
1366
1367 unsigned long flags;
1368 int ret = 0;
1369
2c4cbe6e
FB		 1370 trace_dwc3_ep_dequeue(req);
1371
72246da4
FB		 1372 spin_lock_irqsave(&dwc->lock, flags);
1373
aa3342c8 1374 list_for_each_entry(r, &dep->pending_list, list) {
72246da4
FB		 1375 if (r == req)
1376 break;
1377 }
1378
1379 if (r != req) {
aa3342c8 1380 list_for_each_entry(r, &dep->started_list, list) {
72246da4
FB		 1381 if (r == req)
1382 break;
1383 }
1384 if (r == req) {
1385 /* wait until it is processed */
b992e681 1386 dwc3_stop_active_transfer(dwc, dep->number, true);
cf3113d8
FB		 1387
1388 /*
1389 * If request was already started, this means we had to
1390 * stop the transfer. With that we also need to ignore
1391 * all TRBs used by the request, however TRBs can only
1392 * be modified after completion of END_TRANSFER
1393 * command. So what we do here is that we wait for
1394 * END_TRANSFER completion and only after that, we jump
1395 * over TRBs by clearing HWO and incrementing dequeue
1396 * pointer.
1397 *
1398 * Note that we have 2 possible types of transfers here:
1399 *
1400 * i) Linear buffer request
1401 * ii) SG-list based request
1402 *
1403 * SG-list based requests will have r->num_pending_sgs
1404 * set to a valid number (> 0). Linear requests,
1405 * normally use a single TRB.
1406 *
1407 * For each of these two cases, if r->unaligned flag is
1408 * set, one extra TRB has been used to align transfer
1409 * size to wMaxPacketSize.
1410 *
1411 * All of these cases need to be taken into
1412 * consideration so we don't mess up our TRB ring
1413 * pointers.
1414 */
1415 wait_event_lock_irq(dep->wait_end_transfer,
1416 !(dep->flags & DWC3_EP_END_TRANSFER_PENDING),
1417 dwc->lock);
1418
1419 if (!r->trb)
1420 goto out1;
1421
1422 if (r->num_pending_sgs) {
1423 struct dwc3_trb *trb;
1424 int i = 0;
1425
1426 for (i = 0; i < r->num_pending_sgs; i++) {
1427 trb = r->trb + i;
1428 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1429 dwc3_ep_inc_deq(dep);
1430 }
1431
d6e5a549 1432 if (r->unaligned || r->zero) {
cf3113d8
FB		 1433 trb = r->trb + r->num_pending_sgs + 1;
1434 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1435 dwc3_ep_inc_deq(dep);
1436 }
1437 } else {
1438 struct dwc3_trb *trb = r->trb;
1439
1440 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1441 dwc3_ep_inc_deq(dep);
1442
d6e5a549 1443 if (r->unaligned || r->zero) {
cf3113d8
FB		 1444 trb = r->trb + 1;
1445 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1446 dwc3_ep_inc_deq(dep);
1447 }
1448 }
e8d4e8be 1449 goto out1;
72246da4 1450 }
04fb365c 1451 dev_err(dwc->dev, "request %pK was not queued to %s\n",
72246da4
FB		 1452 request, ep->name);
1453 ret = -EINVAL;
1454 goto out0;
1455 }
1456
e8d4e8be 1457out1:
72246da4 1458 /* giveback the request */
cf3113d8 1459 dep->queued_requests--;
72246da4
FB		 1460 dwc3_gadget_giveback(dep, req, -ECONNRESET);
1461
1462out0:
1463 spin_unlock_irqrestore(&dwc->lock, flags);
1464
1465 return ret;
1466}
1467
7a608559 1468int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
72246da4
FB		 1469{
1470 struct dwc3_gadget_ep_cmd_params params;
1471 struct dwc3 *dwc = dep->dwc;
1472 int ret;
1473
5ad02fb8
FB		 1474 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1475 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
1476 return -EINVAL;
1477 }
1478
72246da4
FB		 1479 memset(&params, 0x00, sizeof(params));
1480
1481 if (value) {
69450c4d
FB		 1482 struct dwc3_trb *trb;
1483
1484 unsigned transfer_in_flight;
1485 unsigned started;
1486
ffb80fc6
FB		 1487 if (dep->flags & DWC3_EP_STALL)
1488 return 0;
1489
69450c4d
FB		 1490 if (dep->number > 1)
1491 trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
1492 else
1493 trb = &dwc->ep0_trb[dep->trb_enqueue];
1494
1495 transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO;
1496 started = !list_empty(&dep->started_list);
1497
1498 if (!protocol && ((dep->direction && transfer_in_flight) ||
1499 (!dep->direction && started))) {
7a608559
FB		 1500 return -EAGAIN;
1501 }
1502
2cd4718d
FB		 1503 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL,
1504 &params);
72246da4 1505 if (ret)
3f89204b 1506 dev_err(dwc->dev, "failed to set STALL on %s\n",
72246da4
FB		 1507 dep->name);
1508 else
1509 dep->flags |= DWC3_EP_STALL;
1510 } else {
ffb80fc6
FB		 1511 if (!(dep->flags & DWC3_EP_STALL))
1512 return 0;
2cd4718d 1513
50c763f8 1514 ret = dwc3_send_clear_stall_ep_cmd(dep);
72246da4 1515 if (ret)
3f89204b 1516 dev_err(dwc->dev, "failed to clear STALL on %s\n",
72246da4
FB		 1517 dep->name);
1518 else
a535d81c 1519 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
72246da4 1520 }
5275455a 1521
72246da4
FB		 1522 return ret;
1523}
1524
1525static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
1526{
1527 struct dwc3_ep *dep = to_dwc3_ep(ep);
1528 struct dwc3 *dwc = dep->dwc;
1529
1530 unsigned long flags;
1531
1532 int ret;
1533
1534 spin_lock_irqsave(&dwc->lock, flags);
7a608559 1535 ret = __dwc3_gadget_ep_set_halt(dep, value, false);
72246da4
FB		 1536 spin_unlock_irqrestore(&dwc->lock, flags);
1537
1538 return ret;
1539}
1540
1541static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
1542{
1543 struct dwc3_ep *dep = to_dwc3_ep(ep);
249a4569
PZ		 1544 struct dwc3 *dwc = dep->dwc;
1545 unsigned long flags;
95aa4e8d 1546 int ret;
72246da4 1547
249a4569 1548 spin_lock_irqsave(&dwc->lock, flags);
72246da4
FB		 1549 dep->flags |= DWC3_EP_WEDGE;
1550
08f0d966 1551 if (dep->number == 0 || dep->number == 1)
95aa4e8d 1552 ret = __dwc3_gadget_ep0_set_halt(ep, 1);
08f0d966 1553 else
7a608559 1554 ret = __dwc3_gadget_ep_set_halt(dep, 1, false);
95aa4e8d
FB		 1555 spin_unlock_irqrestore(&dwc->lock, flags);
1556
1557 return ret;
72246da4
FB		 1558}
1559
1560/* -------------------------------------------------------------------------- */
1561
1562static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
1563 .bLength = USB_DT_ENDPOINT_SIZE,
1564 .bDescriptorType = USB_DT_ENDPOINT,
1565 .bmAttributes = USB_ENDPOINT_XFER_CONTROL,
1566};
1567
1568static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
1569 .enable = dwc3_gadget_ep0_enable,
1570 .disable = dwc3_gadget_ep0_disable,
1571 .alloc_request = dwc3_gadget_ep_alloc_request,
1572 .free_request = dwc3_gadget_ep_free_request,
1573 .queue = dwc3_gadget_ep0_queue,
1574 .dequeue = dwc3_gadget_ep_dequeue,
08f0d966 1575 .set_halt = dwc3_gadget_ep0_set_halt,
72246da4
FB		 1576 .set_wedge = dwc3_gadget_ep_set_wedge,
1577};
1578
1579static const struct usb_ep_ops dwc3_gadget_ep_ops = {
1580 .enable = dwc3_gadget_ep_enable,
1581 .disable = dwc3_gadget_ep_disable,
1582 .alloc_request = dwc3_gadget_ep_alloc_request,
1583 .free_request = dwc3_gadget_ep_free_request,
1584 .queue = dwc3_gadget_ep_queue,
1585 .dequeue = dwc3_gadget_ep_dequeue,
1586 .set_halt = dwc3_gadget_ep_set_halt,
1587 .set_wedge = dwc3_gadget_ep_set_wedge,
1588};
1589
1590/* -------------------------------------------------------------------------- */
1591
1592static int dwc3_gadget_get_frame(struct usb_gadget *g)
1593{
1594 struct dwc3 *dwc = gadget_to_dwc(g);
72246da4 1595
6cb2e4e3 1596 return __dwc3_gadget_get_frame(dwc);
72246da4
FB		 1597}
1598
218ef7b6 1599static int __dwc3_gadget_wakeup(struct dwc3 *dwc)
72246da4 1600{
d6011f6f 1601 int retries;
72246da4 1602
218ef7b6 1603 int ret;
72246da4
FB		 1604 u32 reg;
1605
72246da4
FB		 1606 u8 link_state;
1607 u8 speed;
1608
72246da4
FB		 1609 /*
1610 * According to the Databook Remote wakeup request should
1611 * be issued only when the device is in early suspend state.
1612 *
1613 * We can check that via USB Link State bits in DSTS register.
1614 */
1615 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1616
1617 speed = reg & DWC3_DSTS_CONNECTSPD;
ee5cd41c 1618 if ((speed == DWC3_DSTS_SUPERSPEED) ||
5eb30ced 1619 (speed == DWC3_DSTS_SUPERSPEED_PLUS))
6b742899 1620 return 0;
72246da4
FB		 1621
1622 link_state = DWC3_DSTS_USBLNKST(reg);
1623
1624 switch (link_state) {
1625 case DWC3_LINK_STATE_RX_DET: /* in HS, means Early Suspend */
1626 case DWC3_LINK_STATE_U3: /* in HS, means SUSPEND */
1627 break;
1628 default:
218ef7b6 1629 return -EINVAL;
72246da4
FB		 1630 }
1631
8598bde7
FB		 1632 ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
1633 if (ret < 0) {
1634 dev_err(dwc->dev, "failed to put link in Recovery\n");
218ef7b6 1635 return ret;
8598bde7 1636 }
72246da4 1637
802fde98
PZ		 1638 /* Recent versions do this automatically */
1639 if (dwc->revision < DWC3_REVISION_194A) {
1640 /* write zeroes to Link Change Request */
fcc023c7 1641 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
802fde98
PZ		 1642 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
1643 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1644 }
72246da4 1645
1d046793 1646 /* poll until Link State changes to ON */
d6011f6f 1647 retries = 20000;
72246da4 1648
d6011f6f 1649 while (retries--) {
72246da4
FB		 1650 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1651
1652 /* in HS, means ON */
1653 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
1654 break;
1655 }
1656
1657 if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
1658 dev_err(dwc->dev, "failed to send remote wakeup\n");
218ef7b6 1659 return -EINVAL;
72246da4
FB		 1660 }
1661
218ef7b6
FB		 1662 return 0;
1663}
1664
1665static int dwc3_gadget_wakeup(struct usb_gadget *g)
1666{
1667 struct dwc3 *dwc = gadget_to_dwc(g);
1668 unsigned long flags;
1669 int ret;
1670
1671 spin_lock_irqsave(&dwc->lock, flags);
1672 ret = __dwc3_gadget_wakeup(dwc);
72246da4
FB		 1673 spin_unlock_irqrestore(&dwc->lock, flags);
1674
1675 return ret;
1676}
1677
1678static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
1679 int is_selfpowered)
1680{
1681 struct dwc3 *dwc = gadget_to_dwc(g);
249a4569 1682 unsigned long flags;
72246da4 1683
249a4569 1684 spin_lock_irqsave(&dwc->lock, flags);
bcdea503 1685 g->is_selfpowered = !!is_selfpowered;
249a4569 1686 spin_unlock_irqrestore(&dwc->lock, flags);
72246da4
FB		 1687
1688 return 0;
1689}
1690
7b2a0368 1691static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend)
72246da4
FB		 1692{
1693 u32 reg;
61d58242 1694 u32 timeout = 500;
72246da4 1695
fc8bb91b
FB		 1696 if (pm_runtime_suspended(dwc->dev))
1697 return 0;
1698
72246da4 1699 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
8db7ed15 1700 if (is_on) {
802fde98
PZ		 1701 if (dwc->revision <= DWC3_REVISION_187A) {
1702 reg &= ~DWC3_DCTL_TRGTULST_MASK;
1703 reg |= DWC3_DCTL_TRGTULST_RX_DET;
1704 }
1705
1706 if (dwc->revision >= DWC3_REVISION_194A)
1707 reg &= ~DWC3_DCTL_KEEP_CONNECT;
1708 reg |= DWC3_DCTL_RUN_STOP;
7b2a0368
FB		 1709
1710 if (dwc->has_hibernation)
1711 reg |= DWC3_DCTL_KEEP_CONNECT;
1712
9fcb3bd8 1713 dwc->pullups_connected = true;
8db7ed15 1714 } else {
72246da4 1715 reg &= ~DWC3_DCTL_RUN_STOP;
7b2a0368
FB		 1716
1717 if (dwc->has_hibernation && !suspend)
1718 reg &= ~DWC3_DCTL_KEEP_CONNECT;
1719
9fcb3bd8 1720 dwc->pullups_connected = false;
8db7ed15 1721 }
72246da4
FB		 1722
1723 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1724
1725 do {
1726 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
b6d4e16e
FB		 1727 reg &= DWC3_DSTS_DEVCTRLHLT;
1728 } while (--timeout && !(!is_on ^ !reg));
f2df679b
FB		 1729
1730 if (!timeout)
1731 return -ETIMEDOUT;
72246da4 1732
6f17f74b 1733 return 0;
72246da4
FB		 1734}
1735
1736static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
1737{
1738 struct dwc3 *dwc = gadget_to_dwc(g);
1739 unsigned long flags;
6f17f74b 1740 int ret;
72246da4
FB		 1741
1742 is_on = !!is_on;
1743
bb014736
BW		 1744 /*
1745 * Per databook, when we want to stop the gadget, if a control transfer
1746 * is still in process, complete it and get the core into setup phase.
1747 */
1748 if (!is_on && dwc->ep0state != EP0_SETUP_PHASE) {
1749 reinit_completion(&dwc->ep0_in_setup);
1750
1751 ret = wait_for_completion_timeout(&dwc->ep0_in_setup,
1752 msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT));
1753 if (ret == 0) {
1754 dev_err(dwc->dev, "timed out waiting for SETUP phase\n");
1755 return -ETIMEDOUT;
1756 }
1757 }
1758
72246da4 1759 spin_lock_irqsave(&dwc->lock, flags);
7b2a0368 1760 ret = dwc3_gadget_run_stop(dwc, is_on, false);
72246da4
FB		 1761 spin_unlock_irqrestore(&dwc->lock, flags);
1762
6f17f74b 1763 return ret;
72246da4
FB		 1764}
1765
8698e2ac
FB		 1766static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
1767{
1768 u32 reg;
1769
1770 /* Enable all but Start and End of Frame IRQs */
1771 reg = (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN |
1772 DWC3_DEVTEN_EVNTOVERFLOWEN |
1773 DWC3_DEVTEN_CMDCMPLTEN |
1774 DWC3_DEVTEN_ERRTICERREN |
1775 DWC3_DEVTEN_WKUPEVTEN |
8698e2ac
FB		 1776 DWC3_DEVTEN_CONNECTDONEEN |
1777 DWC3_DEVTEN_USBRSTEN |
1778 DWC3_DEVTEN_DISCONNEVTEN);
1779
799e9dc8
FB		 1780 if (dwc->revision < DWC3_REVISION_250A)
1781 reg |= DWC3_DEVTEN_ULSTCNGEN;
1782
8698e2ac
FB		 1783 dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
1784}
1785
1786static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
1787{
1788 /* mask all interrupts */
1789 dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
1790}
1791
1792static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
b15a762f 1793static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
8698e2ac 1794
4e99472b 1795/**
bfad65ee
FB		 1796 * dwc3_gadget_setup_nump - calculate and initialize NUMP field of %DWC3_DCFG
1797 * @dwc: pointer to our context structure
4e99472b
FB		 1798 *
1799 * The following looks like complex but it's actually very simple. In order to
1800 * calculate the number of packets we can burst at once on OUT transfers, we're
1801 * gonna use RxFIFO size.
1802 *
1803 * To calculate RxFIFO size we need two numbers:
1804 * MDWIDTH = size, in bits, of the internal memory bus
1805 * RAM2_DEPTH = depth, in MDWIDTH, of internal RAM2 (where RxFIFO sits)
1806 *
1807 * Given these two numbers, the formula is simple:
1808 *
1809 * RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16;
1810 *
1811 * 24 bytes is for 3x SETUP packets
1812 * 16 bytes is a clock domain crossing tolerance
1813 *
1814 * Given RxFIFO Size, NUMP = RxFIFOSize / 1024;
1815 */
1816static void dwc3_gadget_setup_nump(struct dwc3 *dwc)
1817{
1818 u32 ram2_depth;
1819 u32 mdwidth;
1820 u32 nump;
1821 u32 reg;
1822
1823 ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7);
1824 mdwidth = DWC3_GHWPARAMS0_MDWIDTH(dwc->hwparams.hwparams0);
1825
1826 nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024;
1827 nump = min_t(u32, nump, 16);
1828
1829 /* update NumP */
1830 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
1831 reg &= ~DWC3_DCFG_NUMP_MASK;
1832 reg |= nump << DWC3_DCFG_NUMP_SHIFT;
1833 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
1834}
1835
d7be2952 1836static int __dwc3_gadget_start(struct dwc3 *dwc)
72246da4 1837{
72246da4 1838 struct dwc3_ep *dep;
72246da4
FB		 1839 int ret = 0;
1840 u32 reg;
1841
cf40b86b
JY		 1842 /*
1843 * Use IMOD if enabled via dwc->imod_interval. Otherwise, if
1844 * the core supports IMOD, disable it.
1845 */
1846 if (dwc->imod_interval) {
1847 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
1848 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
1849 } else if (dwc3_has_imod(dwc)) {
1850 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0);
1851 }
1852
2a58f9c1
FB		 1853 /*
1854 * We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP
1855 * field instead of letting dwc3 itself calculate that automatically.
1856 *
1857 * This way, we maximize the chances that we'll be able to get several
1858 * bursts of data without going through any sort of endpoint throttling.
1859 */
1860 reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG);
1861 reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
1862 dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg);
1863
4e99472b
FB		 1864 dwc3_gadget_setup_nump(dwc);
1865
72246da4
FB		 1866 /* Start with SuperSpeed Default */
1867 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1868
1869 dep = dwc->eps[0];
39ebb05c 1870 ret = __dwc3_gadget_ep_enable(dep, false, false);
72246da4
FB		 1871 if (ret) {
1872 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
d7be2952 1873 goto err0;
72246da4
FB		 1874 }
1875
1876 dep = dwc->eps[1];
39ebb05c 1877 ret = __dwc3_gadget_ep_enable(dep, false, false);
72246da4
FB		 1878 if (ret) {
1879 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
d7be2952 1880 goto err1;
72246da4
FB		 1881 }
1882
1883 /* begin to receive SETUP packets */
c7fcdeb2 1884 dwc->ep0state = EP0_SETUP_PHASE;
72246da4
FB		 1885 dwc3_ep0_out_start(dwc);
1886
8698e2ac
FB		 1887 dwc3_gadget_enable_irq(dwc);
1888
72246da4
FB		 1889 return 0;
1890
b0d7ffd4 1891err1:
d7be2952 1892 __dwc3_gadget_ep_disable(dwc->eps[0]);
b0d7ffd4
FB		 1893
1894err0:
72246da4
FB		 1895 return ret;
1896}
1897
d7be2952
FB		 1898static int dwc3_gadget_start(struct usb_gadget *g,
1899 struct usb_gadget_driver *driver)
72246da4
FB		 1900{
1901 struct dwc3 *dwc = gadget_to_dwc(g);
1902 unsigned long flags;
d7be2952 1903 int ret = 0;
8698e2ac 1904 int irq;
72246da4 1905
9522def4 1906 irq = dwc->irq_gadget;
d7be2952
FB		 1907 ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt,
1908 IRQF_SHARED, "dwc3", dwc->ev_buf);
1909 if (ret) {
1910 dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
1911 irq, ret);
1912 goto err0;
1913 }
1914
72246da4 1915 spin_lock_irqsave(&dwc->lock, flags);
d7be2952
FB		 1916 if (dwc->gadget_driver) {
1917 dev_err(dwc->dev, "%s is already bound to %s\n",
1918 dwc->gadget.name,
1919 dwc->gadget_driver->driver.name);
1920 ret = -EBUSY;
1921 goto err1;
1922 }
1923
1924 dwc->gadget_driver = driver;
1925
fc8bb91b
FB		 1926 if (pm_runtime_active(dwc->dev))
1927 __dwc3_gadget_start(dwc);
1928
d7be2952
FB		 1929 spin_unlock_irqrestore(&dwc->lock, flags);
1930
1931 return 0;
1932
1933err1:
1934 spin_unlock_irqrestore(&dwc->lock, flags);
1935 free_irq(irq, dwc);
1936
1937err0:
1938 return ret;
1939}
72246da4 1940
d7be2952
FB		 1941static void __dwc3_gadget_stop(struct dwc3 *dwc)
1942{
8698e2ac 1943 dwc3_gadget_disable_irq(dwc);
72246da4
FB		 1944 __dwc3_gadget_ep_disable(dwc->eps[0]);
1945 __dwc3_gadget_ep_disable(dwc->eps[1]);
d7be2952 1946}
72246da4 1947
d7be2952
FB		 1948static int dwc3_gadget_stop(struct usb_gadget *g)
1949{
1950 struct dwc3 *dwc = gadget_to_dwc(g);
1951 unsigned long flags;
76a638f8 1952 int epnum;
498f0478 1953 u32 tmo_eps = 0;
72246da4 1954
d7be2952 1955 spin_lock_irqsave(&dwc->lock, flags);
76a638f8
BW		 1956
1957 if (pm_runtime_suspended(dwc->dev))
1958 goto out;
1959
d7be2952 1960 __dwc3_gadget_stop(dwc);
76a638f8
BW		 1961
1962 for (epnum = 2; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
1963 struct dwc3_ep *dep = dwc->eps[epnum];
498f0478 1964 int ret;
76a638f8
BW		 1965
1966 if (!dep)
1967 continue;
1968
1969 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
1970 continue;
1971
498f0478
RQ		 1972 ret = wait_event_interruptible_lock_irq_timeout(dep->wait_end_transfer,
1973 !(dep->flags & DWC3_EP_END_TRANSFER_PENDING),
1974 dwc->lock, msecs_to_jiffies(5));
1975
1976 if (ret <= 0) {
1977 /* Timed out or interrupted! There's nothing much
1978 * we can do so we just log here and print which
1979 * endpoints timed out at the end.
1980 */
1981 tmo_eps |= 1 << epnum;
1982 dep->flags &= DWC3_EP_END_TRANSFER_PENDING;
1983 }
1984 }
1985
1986 if (tmo_eps) {
1987 dev_err(dwc->dev,
1988 "end transfer timed out on endpoints 0x%x [bitmap]\n",
1989 tmo_eps);
76a638f8
BW		 1990 }
1991
1992out:
d7be2952 1993 dwc->gadget_driver = NULL;
72246da4
FB		 1994 spin_unlock_irqrestore(&dwc->lock, flags);
1995
3f308d17 1996 free_irq(dwc->irq_gadget, dwc->ev_buf);
b0d7ffd4 1997
72246da4
FB		 1998 return 0;
1999}
802fde98 2000
7d8d0639
FB		 2001static void dwc3_gadget_set_speed(struct usb_gadget *g,
2002 enum usb_device_speed speed)
2003{
2004 struct dwc3 *dwc = gadget_to_dwc(g);
2005 unsigned long flags;
2006 u32 reg;
2007
2008 spin_lock_irqsave(&dwc->lock, flags);
2009 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2010 reg &= ~(DWC3_DCFG_SPEED_MASK);
2011
2012 /*
2013 * WORKAROUND: DWC3 revision < 2.20a have an issue
2014 * which would cause metastability state on Run/Stop
2015 * bit if we try to force the IP to USB2-only mode.
2016 *
2017 * Because of that, we cannot configure the IP to any
2018 * speed other than the SuperSpeed
2019 *
2020 * Refers to:
2021 *
2022 * STAR#9000525659: Clock Domain Crossing on DCTL in
2023 * USB 2.0 Mode
2024 */
42bf02ec
RQ		 2025 if (dwc->revision < DWC3_REVISION_220A &&
2026 !dwc->dis_metastability_quirk) {
7d8d0639
FB		 2027 reg |= DWC3_DCFG_SUPERSPEED;
2028 } else {
2029 switch (speed) {
2030 case USB_SPEED_LOW:
2031 reg |= DWC3_DCFG_LOWSPEED;
2032 break;
2033 case USB_SPEED_FULL:
2034 reg |= DWC3_DCFG_FULLSPEED;
2035 break;
2036 case USB_SPEED_HIGH:
2037 reg |= DWC3_DCFG_HIGHSPEED;
2038 break;
2039 case USB_SPEED_SUPER:
2040 reg |= DWC3_DCFG_SUPERSPEED;
2041 break;
2042 case USB_SPEED_SUPER_PLUS:
2043 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2044 break;
2045 default:
2046 dev_err(dwc->dev, "invalid speed (%d)\n", speed);
2047
2048 if (dwc->revision & DWC3_REVISION_IS_DWC31)
2049 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2050 else
2051 reg |= DWC3_DCFG_SUPERSPEED;
2052 }
2053 }
2054 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2055
2056 spin_unlock_irqrestore(&dwc->lock, flags);
2057}
2058
72246da4
FB		 2059static const struct usb_gadget_ops dwc3_gadget_ops = {
2060 .get_frame = dwc3_gadget_get_frame,
2061 .wakeup = dwc3_gadget_wakeup,
2062 .set_selfpowered = dwc3_gadget_set_selfpowered,
2063 .pullup = dwc3_gadget_pullup,
2064 .udc_start = dwc3_gadget_start,
2065 .udc_stop = dwc3_gadget_stop,
7d8d0639 2066 .udc_set_speed = dwc3_gadget_set_speed,
72246da4
FB		 2067};
2068
2069/* -------------------------------------------------------------------------- */
2070
46b780d4 2071static int dwc3_gadget_init_endpoints(struct dwc3 *dwc, u8 total)
72246da4
FB		 2072{
2073 struct dwc3_ep *dep;
47d3946e 2074 u8 epnum;
72246da4 2075
f3bcfc7e
BD		 2076 INIT_LIST_HEAD(&dwc->gadget.ep_list);
2077
46b780d4 2078 for (epnum = 0; epnum < total; epnum++) {
47d3946e 2079 bool direction = epnum & 1;
46b780d4 2080 u8 num = epnum >> 1;
72246da4 2081
72246da4 2082 dep = kzalloc(sizeof(*dep), GFP_KERNEL);
734d5a53 2083 if (!dep)
72246da4 2084 return -ENOMEM;
72246da4
FB		 2085
2086 dep->dwc = dwc;
2087 dep->number = epnum;
47d3946e 2088 dep->direction = direction;
2eb88016 2089 dep->regs = dwc->regs + DWC3_DEP_BASE(epnum);
72246da4
FB		 2090 dwc->eps[epnum] = dep;
2091
46b780d4 2092 snprintf(dep->name, sizeof(dep->name), "ep%u%s", num,
47d3946e 2093 direction ? "in" : "out");
6a1e3ef4 2094
72246da4 2095 dep->endpoint.name = dep->name;
39ebb05c
JY		 2096
2097 if (!(dep->number > 1)) {
2098 dep->endpoint.desc = &dwc3_gadget_ep0_desc;
2099 dep->endpoint.comp_desc = NULL;
2100 }
2101
74674cbf 2102 spin_lock_init(&dep->lock);
72246da4 2103
46b780d4 2104 if (num == 0) {
e117e742 2105 usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
6048e4c6 2106 dep->endpoint.maxburst = 1;
72246da4 2107 dep->endpoint.ops = &dwc3_gadget_ep0_ops;
46b780d4 2108 if (!direction)
72246da4 2109 dwc->gadget.ep0 = &dep->endpoint;
28781789
FB		 2110 } else if (direction) {
2111 int mdwidth;
46b780d4 2112 int kbytes;
28781789
FB		 2113 int size;
2114 int ret;
28781789
FB		 2115
2116 mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);
2117 /* MDWIDTH is represented in bits, we need it in bytes */
2118 mdwidth /= 8;
2119
46b780d4 2120 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(num));
d548a617
TN		 2121 if (dwc3_is_usb31(dwc))
2122 size = DWC31_GTXFIFOSIZ_TXFDEF(size);
2123 else
2124 size = DWC3_GTXFIFOSIZ_TXFDEF(size);
28781789
FB		 2125
2126 /* FIFO Depth is in MDWDITH bytes. Multiply */
2127 size *= mdwidth;
2128
46b780d4
AS		 2129 kbytes = size / 1024;
2130 if (kbytes == 0)
2131 kbytes = 1;
28781789
FB		 2132
2133 /*
46b780d4 2134 * FIFO sizes account an extra MDWIDTH * (kbytes + 1) bytes for
28781789
FB		 2135 * internal overhead. We don't really know how these are used,
2136 * but documentation say it exists.
2137 */
46b780d4
AS		 2138 size -= mdwidth * (kbytes + 1);
2139 size /= kbytes;
28781789
FB		 2140
2141 usb_ep_set_maxpacket_limit(&dep->endpoint, size);
2142
2143 dep->endpoint.max_streams = 15;
2144 dep->endpoint.ops = &dwc3_gadget_ep_ops;
2145 list_add_tail(&dep->endpoint.ep_list,
2146 &dwc->gadget.ep_list);
2147
2148 ret = dwc3_alloc_trb_pool(dep);
2149 if (ret)
2150 return ret;
72246da4
FB		 2151 } else {
2152 int ret;
2153
e117e742 2154 usb_ep_set_maxpacket_limit(&dep->endpoint, 1024);
12d36c16 2155 dep->endpoint.max_streams = 15;
72246da4
FB		 2156 dep->endpoint.ops = &dwc3_gadget_ep_ops;
2157 list_add_tail(&dep->endpoint.ep_list,
2158 &dwc->gadget.ep_list);
2159
2160 ret = dwc3_alloc_trb_pool(dep);
25b8ff68 2161 if (ret)
72246da4 2162 return ret;
72246da4 2163 }
25b8ff68 2164
46b780d4 2165 if (num == 0) {
a474d3b7
RB		 2166 dep->endpoint.caps.type_control = true;
2167 } else {
2168 dep->endpoint.caps.type_iso = true;
2169 dep->endpoint.caps.type_bulk = true;
2170 dep->endpoint.caps.type_int = true;
2171 }
2172
47d3946e 2173 dep->endpoint.caps.dir_in = direction;
a474d3b7
RB		 2174 dep->endpoint.caps.dir_out = !direction;
2175
aa3342c8
FB		 2176 INIT_LIST_HEAD(&dep->pending_list);
2177 INIT_LIST_HEAD(&dep->started_list);
72246da4
FB		 2178 }
2179
2180 return 0;
2181}
2182
2183static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
2184{
2185 struct dwc3_ep *dep;
2186 u8 epnum;
2187
2188 for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2189 dep = dwc->eps[epnum];
6a1e3ef4
FB		 2190 if (!dep)
2191 continue;
5bf8fae3
GC		 2192 /*
2193 * Physical endpoints 0 and 1 are special; they form the
2194 * bi-directional USB endpoint 0.
2195 *
2196 * For those two physical endpoints, we don't allocate a TRB
2197 * pool nor do we add them the endpoints list. Due to that, we
2198 * shouldn't do these two operations otherwise we would end up
2199 * with all sorts of bugs when removing dwc3.ko.
2200 */
2201 if (epnum != 0 && epnum != 1) {
2202 dwc3_free_trb_pool(dep);
72246da4 2203 list_del(&dep->endpoint.ep_list);
5bf8fae3 2204 }
72246da4
FB		 2205
2206 kfree(dep);
2207 }
2208}
2209
72246da4 2210/* -------------------------------------------------------------------------- */
e5caff68 2211
e5ba5ec8
PA		 2212static int __dwc3_cleanup_done_trbs(struct dwc3 *dwc, struct dwc3_ep *dep,
2213 struct dwc3_request *req, struct dwc3_trb *trb,
e5b36ae2
FB		 2214 const struct dwc3_event_depevt *event, int status,
2215 int chain)
72246da4 2216{
72246da4
FB		 2217 unsigned int count;
2218 unsigned int s_pkt = 0;
d6d6ec7b 2219 unsigned int trb_status;
72246da4 2220
dc55c67e 2221 dwc3_ep_inc_deq(dep);
a9c3ca5f
FB		 2222
2223 if (req->trb == trb)
2224 dep->queued_requests--;
2225
2c4cbe6e
FB		 2226 trace_dwc3_complete_trb(dep, trb);
2227
e5b36ae2
FB		 2228 /*
2229 * If we're in the middle of series of chained TRBs and we
2230 * receive a short transfer along the way, DWC3 will skip
2231 * through all TRBs including the last TRB in the chain (the
2232 * where CHN bit is zero. DWC3 will also avoid clearing HWO
2233 * bit and SW has to do it manually.
2234 *
2235 * We're going to do that here to avoid problems of HW trying
2236 * to use bogus TRBs for transfers.
2237 */
2238 if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO))
2239 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2240
c6267a51
FB		 2241 /*
2242 * If we're dealing with unaligned size OUT transfer, we will be left
2243 * with one TRB pending in the ring. We need to manually clear HWO bit
2244 * from that TRB.
2245 */
d6e5a549 2246 if ((req->zero || req->unaligned) && (trb->ctrl & DWC3_TRB_CTRL_HWO)) {
c6267a51
FB		 2247 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2248 return 1;
2249 }
2250
e5ba5ec8 2251 count = trb->size & DWC3_TRB_SIZE_MASK;
e62c5bc5 2252 req->remaining += count;
e5ba5ec8 2253
35b2719e
FB		 2254 if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
2255 return 1;
2256
e5ba5ec8
PA		 2257 if (dep->direction) {
2258 if (count) {
2259 trb_status = DWC3_TRB_SIZE_TRBSTS(trb->size);
2260 if (trb_status == DWC3_TRBSTS_MISSED_ISOC) {
e5ba5ec8
PA		 2261 /*
2262 * If missed isoc occurred and there is
2263 * no request queued then issue END
2264 * TRANSFER, so that core generates
2265 * next xfernotready and we will issue
2266 * a fresh START TRANSFER.
2267 * If there are still queued request
2268 * then wait, do not issue either END
2269 * or UPDATE TRANSFER, just attach next
aa3342c8 2270 * request in pending_list during
e5ba5ec8
PA		 2271 * giveback.If any future queued request
2272 * is successfully transferred then we
2273 * will issue UPDATE TRANSFER for all
aa3342c8 2274 * request in the pending_list.
e5ba5ec8
PA		 2275 */
2276 dep->flags |= DWC3_EP_MISSED_ISOC;
2277 } else {
2278 dev_err(dwc->dev, "incomplete IN transfer %s\n",
2279 dep->name);
2280 status = -ECONNRESET;
2281 }
2282 } else {
2283 dep->flags &= ~DWC3_EP_MISSED_ISOC;
2284 }
2285 } else {
2286 if (count && (event->status & DEPEVT_STATUS_SHORT))
2287 s_pkt = 1;
2288 }
2289
7c705dfe 2290 if (s_pkt && !chain)
e5ba5ec8 2291 return 1;
f99f53f2 2292
e5ba5ec8
PA		 2293 if ((event->status & DEPEVT_STATUS_IOC) &&
2294 (trb->ctrl & DWC3_TRB_CTRL_IOC))
2295 return 1;
f99f53f2 2296
e5ba5ec8
PA		 2297 return 0;
2298}
2299
2300static int dwc3_cleanup_done_reqs(struct dwc3 *dwc, struct dwc3_ep *dep,
2301 const struct dwc3_event_depevt *event, int status)
2302{
31162af4 2303 struct dwc3_request *req, *n;
e5ba5ec8 2304 struct dwc3_trb *trb;
d6e10bf2 2305 bool ioc = false;
e62c5bc5 2306 int ret = 0;
e5ba5ec8 2307
31162af4 2308 list_for_each_entry_safe(req, n, &dep->started_list, list) {
1f512119 2309 unsigned length;
e5b36ae2
FB		 2310 int chain;
2311
1f512119
FB		 2312 length = req->request.length;
2313 chain = req->num_pending_sgs > 0;
31162af4 2314 if (chain) {
1f512119 2315 struct scatterlist *sg = req->sg;
31162af4 2316 struct scatterlist *s;
1f512119 2317 unsigned int pending = req->num_pending_sgs;
31162af4 2318 unsigned int i;
c7de5734 2319
1f512119 2320 for_each_sg(sg, s, pending, i) {
31162af4 2321 trb = &dep->trb_pool[dep->trb_dequeue];
31162af4 2322
7282c4ef
FB		 2323 if (trb->ctrl & DWC3_TRB_CTRL_HWO)
2324 break;
2325
1f512119
FB		 2326 req->sg = sg_next(s);
2327 req->num_pending_sgs--;
2328
31162af4
FB		 2329 ret = __dwc3_cleanup_done_trbs(dwc, dep, req, trb,
2330 event, status, chain);
1f512119
FB		 2331 if (ret)
2332 break;
31162af4
FB		 2333 }
2334 } else {
737f1ae2 2335 trb = &dep->trb_pool[dep->trb_dequeue];
d115d705 2336 ret = __dwc3_cleanup_done_trbs(dwc, dep, req, trb,
e5b36ae2 2337 event, status, chain);
31162af4 2338 }
d115d705 2339
d6e5a549 2340 if (req->unaligned || req->zero) {
c6267a51
FB		 2341 trb = &dep->trb_pool[dep->trb_dequeue];
2342 ret = __dwc3_cleanup_done_trbs(dwc, dep, req, trb,
2343 event, status, false);
2344 req->unaligned = false;
d6e5a549 2345 req->zero = false;
c6267a51
FB		 2346 }
2347
e62c5bc5 2348 req->request.actual = length - req->remaining;
1f512119 2349
ff377ae4 2350 if ((req->request.actual < length) && req->num_pending_sgs)
7fdca766 2351 return __dwc3_gadget_kick_transfer(dep);
1f512119 2352
d115d705 2353 dwc3_gadget_giveback(dep, req, status);
e5ba5ec8 2354
d6e10bf2
AB		 2355 if (ret) {
2356 if ((event->status & DEPEVT_STATUS_IOC) &&
2357 (trb->ctrl & DWC3_TRB_CTRL_IOC))
2358 ioc = true;
72246da4 2359 break;
d6e10bf2 2360 }
31162af4 2361 }
72246da4 2362
4cb42217
FB		 2363 /*
2364 * Our endpoint might get disabled by another thread during
2365 * dwc3_gadget_giveback(). If that happens, we're just gonna return 1
2366 * early on so DWC3_EP_BUSY flag gets cleared
2367 */
2368 if (!dep->endpoint.desc)
2369 return 1;
2370
cdc359dd 2371 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
aa3342c8
FB		 2372 list_empty(&dep->started_list)) {
2373 if (list_empty(&dep->pending_list)) {
cdc359dd
PA		 2374 /*
2375 * If there is no entry in request list then do
2376 * not issue END TRANSFER now. Just set PENDING
2377 * flag, so that END TRANSFER is issued when an
2378 * entry is added into request list.
2379 */
2380 dep->flags = DWC3_EP_PENDING_REQUEST;
2381 } else {
b992e681 2382 dwc3_stop_active_transfer(dwc, dep->number, true);
cdc359dd
PA		 2383 dep->flags = DWC3_EP_ENABLED;
2384 }
7efea86c
PA		 2385 return 1;
2386 }
2387
d6e10bf2
AB		 2388 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && ioc)
2389 return 0;
2390
72246da4
FB		 2391 return 1;
2392}
2393
2394static void dwc3_endpoint_transfer_complete(struct dwc3 *dwc,
029d97ff 2395 struct dwc3_ep *dep, const struct dwc3_event_depevt *event)
72246da4
FB		 2396{
2397 unsigned status = 0;
2398 int clean_busy;
e18b7975
FB		 2399 u32 is_xfer_complete;
2400
2401 is_xfer_complete = (event->endpoint_event == DWC3_DEPEVT_XFERCOMPLETE);
72246da4
FB		 2402
2403 if (event->status & DEPEVT_STATUS_BUSERR)
2404 status = -ECONNRESET;
2405
1d046793 2406 clean_busy = dwc3_cleanup_done_reqs(dwc, dep, event, status);
4cb42217 2407 if (clean_busy && (!dep->endpoint.desc || is_xfer_complete ||
e18b7975 2408 usb_endpoint_xfer_isoc(dep->endpoint.desc)))
72246da4 2409 dep->flags &= ~DWC3_EP_BUSY;
fae2b904
FB		 2410
2411 /*
2412 * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
2413 * See dwc3_gadget_linksts_change_interrupt() for 1st half.
2414 */
2415 if (dwc->revision < DWC3_REVISION_183A) {
2416 u32 reg;
2417 int i;
2418
2419 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
348e026f 2420 dep = dwc->eps[i];
fae2b904
FB		 2421
2422 if (!(dep->flags & DWC3_EP_ENABLED))
2423 continue;
2424
aa3342c8 2425 if (!list_empty(&dep->started_list))
fae2b904
FB		 2426 return;
2427 }
2428
2429 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2430 reg |= dwc->u1u2;
2431 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2432
2433 dwc->u1u2 = 0;
2434 }
8a1a9c9e 2435
4cb42217
FB		 2436 /*
2437 * Our endpoint might get disabled by another thread during
2438 * dwc3_gadget_giveback(). If that happens, we're just gonna return 1
2439 * early on so DWC3_EP_BUSY flag gets cleared
2440 */
2441 if (!dep->endpoint.desc)
2442 return;
2443
7fdca766
FB		 2444 if (!usb_endpoint_xfer_isoc(dep->endpoint.desc))
2445 __dwc3_gadget_kick_transfer(dep);
72246da4
FB		 2446}
2447
72246da4
FB		 2448static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
2449 const struct dwc3_event_depevt *event)
2450{
2451 struct dwc3_ep *dep;
2452 u8 epnum = event->endpoint_number;
76a638f8 2453 u8 cmd;
72246da4
FB		 2454
2455 dep = dwc->eps[epnum];
2456
d7fd41c6
JD		 2457 if (!(dep->flags & DWC3_EP_ENABLED)) {
2458 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
2459 return;
2460
2461 /* Handle only EPCMDCMPLT when EP disabled */
2462 if (event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT)
2463 return;
2464 }
3336abb5 2465
72246da4
FB		 2466 if (epnum == 0 || epnum == 1) {
2467 dwc3_ep0_interrupt(dwc, event);
2468 return;
2469 }
2470
2471 switch (event->endpoint_event) {
2472 case DWC3_DEPEVT_XFERCOMPLETE:
b4996a86 2473 dep->resource_index = 0;
c2df85ca 2474
16e78db7 2475 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
8566cd1a 2476 dev_err(dwc->dev, "XferComplete for Isochronous endpoint\n");
72246da4
FB		 2477 return;
2478 }
2479
029d97ff 2480 dwc3_endpoint_transfer_complete(dwc, dep, event);
72246da4
FB		 2481 break;
2482 case DWC3_DEPEVT_XFERINPROGRESS:
029d97ff 2483 dwc3_endpoint_transfer_complete(dwc, dep, event);
72246da4
FB		 2484 break;
2485 case DWC3_DEPEVT_XFERNOTREADY:
7fdca766 2486 if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
72246da4 2487 dwc3_gadget_start_isoc(dwc, dep, event);
7fdca766
FB		 2488 else
2489 __dwc3_gadget_kick_transfer(dep);
72246da4 2490
879631aa
FB		 2491 break;
2492 case DWC3_DEPEVT_STREAMEVT:
16e78db7 2493 if (!usb_endpoint_xfer_bulk(dep->endpoint.desc)) {
879631aa
FB		 2494 dev_err(dwc->dev, "Stream event for non-Bulk %s\n",
2495 dep->name);
2496 return;
2497 }
72246da4 2498 break;
72246da4 2499 case DWC3_DEPEVT_EPCMDCMPLT:
76a638f8
BW		 2500 cmd = DEPEVT_PARAMETER_CMD(event->parameters);
2501
2502 if (cmd == DWC3_DEPCMD_ENDTRANSFER) {
2503 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
2504 wake_up(&dep->wait_end_transfer);
2505 }
2506 break;
2507 case DWC3_DEPEVT_RXTXFIFOEVT:
72246da4
FB		 2508 break;
2509 }
2510}
2511
2512static void dwc3_disconnect_gadget(struct dwc3 *dwc)
2513{
2514 if (dwc->gadget_driver && dwc->gadget_driver->disconnect) {
2515 spin_unlock(&dwc->lock);
2516 dwc->gadget_driver->disconnect(&dwc->gadget);
2517 spin_lock(&dwc->lock);
2518 }
2519}
2520
bc5ba2e0
FB		 2521static void dwc3_suspend_gadget(struct dwc3 *dwc)
2522{
73a30bfc 2523 if (dwc->gadget_driver && dwc->gadget_driver->suspend) {
bc5ba2e0
FB		 2524 spin_unlock(&dwc->lock);
2525 dwc->gadget_driver->suspend(&dwc->gadget);
2526 spin_lock(&dwc->lock);
2527 }
2528}
2529
2530static void dwc3_resume_gadget(struct dwc3 *dwc)
2531{
73a30bfc 2532 if (dwc->gadget_driver && dwc->gadget_driver->resume) {
bc5ba2e0
FB		 2533 spin_unlock(&dwc->lock);
2534 dwc->gadget_driver->resume(&dwc->gadget);
5c7b3b02 2535 spin_lock(&dwc->lock);
8e74475b
FB		 2536 }
2537}
2538
2539static void dwc3_reset_gadget(struct dwc3 *dwc)
2540{
2541 if (!dwc->gadget_driver)
2542 return;
2543
2544 if (dwc->gadget.speed != USB_SPEED_UNKNOWN) {
2545 spin_unlock(&dwc->lock);
2546 usb_gadget_udc_reset(&dwc->gadget, dwc->gadget_driver);
bc5ba2e0
FB		 2547 spin_lock(&dwc->lock);
2548 }
2549}
2550
b992e681 2551static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force)
72246da4
FB		 2552{
2553 struct dwc3_ep *dep;
2554 struct dwc3_gadget_ep_cmd_params params;
2555 u32 cmd;
2556 int ret;
2557
2558 dep = dwc->eps[epnum];
2559
76a638f8
BW		 2560 if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) ||
2561 !dep->resource_index)
3daf74d7
PA		 2562 return;
2563
57911504
PA		 2564 /*
2565 * NOTICE: We are violating what the Databook says about the
2566 * EndTransfer command. Ideally we would _always_ wait for the
2567 * EndTransfer Command Completion IRQ, but that's causing too
2568 * much trouble synchronizing between us and gadget driver.
2569 *
2570 * We have discussed this with the IP Provider and it was
2571 * suggested to giveback all requests here, but give HW some
2572 * extra time to synchronize with the interconnect. We're using
dc93b41a 2573 * an arbitrary 100us delay for that.
57911504
PA		 2574 *
2575 * Note also that a similar handling was tested by Synopsys
2576 * (thanks a lot Paul) and nothing bad has come out of it.
2577 * In short, what we're doing is:
2578 *
2579 * - Issue EndTransfer WITH CMDIOC bit set
2580 * - Wait 100us
06281d46
JY		 2581 *
2582 * As of IP version 3.10a of the DWC_usb3 IP, the controller
2583 * supports a mode to work around the above limitation. The
2584 * software can poll the CMDACT bit in the DEPCMD register
2585 * after issuing a EndTransfer command. This mode is enabled
2586 * by writing GUCTL2[14]. This polling is already done in the
2587 * dwc3_send_gadget_ep_cmd() function so if the mode is
2588 * enabled, the EndTransfer command will have completed upon
2589 * returning from this function and we don't need to delay for
2590 * 100us.
2591 *
2592 * This mode is NOT available on the DWC_usb31 IP.
57911504
PA		 2593 */
2594
3daf74d7 2595 cmd = DWC3_DEPCMD_ENDTRANSFER;
b992e681
PZ		 2596 cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
2597 cmd |= DWC3_DEPCMD_CMDIOC;
b4996a86 2598 cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
3daf74d7 2599 memset(&params, 0, sizeof(params));
2cd4718d 2600 ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
3daf74d7 2601 WARN_ON_ONCE(ret);
b4996a86 2602 dep->resource_index = 0;
041d81f4 2603 dep->flags &= ~DWC3_EP_BUSY;
06281d46 2604
76a638f8
BW		 2605 if (dwc3_is_usb31(dwc) || dwc->revision < DWC3_REVISION_310A) {
2606 dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
06281d46 2607 udelay(100);
76a638f8 2608 }
72246da4
FB		 2609}
2610
72246da4
FB		 2611static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
2612{
2613 u32 epnum;
2614
2615 for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2616 struct dwc3_ep *dep;
72246da4
FB		 2617 int ret;
2618
2619 dep = dwc->eps[epnum];
6a1e3ef4
FB		 2620 if (!dep)
2621 continue;
72246da4
FB		 2622
2623 if (!(dep->flags & DWC3_EP_STALL))
2624 continue;
2625
2626 dep->flags &= ~DWC3_EP_STALL;
2627
50c763f8 2628 ret = dwc3_send_clear_stall_ep_cmd(dep);
72246da4
FB		 2629 WARN_ON_ONCE(ret);
2630 }
2631}
2632
2633static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
2634{
c4430a26
FB		 2635 int reg;
2636
72246da4
FB		 2637 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2638 reg &= ~DWC3_DCTL_INITU1ENA;
2639 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2640
2641 reg &= ~DWC3_DCTL_INITU2ENA;
2642 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
72246da4 2643
72246da4
FB		 2644 dwc3_disconnect_gadget(dwc);
2645
2646 dwc->gadget.speed = USB_SPEED_UNKNOWN;
df62df56 2647 dwc->setup_packet_pending = false;
06a374ed 2648 usb_gadget_set_state(&dwc->gadget, USB_STATE_NOTATTACHED);
fc8bb91b
FB		 2649
2650 dwc->connected = false;
72246da4
FB		 2651}
2652
72246da4
FB		 2653static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
2654{
2655 u32 reg;
2656
fc8bb91b
FB		 2657 dwc->connected = true;
2658
df62df56
FB		 2659 /*
2660 * WORKAROUND: DWC3 revisions <1.88a have an issue which
2661 * would cause a missing Disconnect Event if there's a
2662 * pending Setup Packet in the FIFO.
2663 *
2664 * There's no suggested workaround on the official Bug
2665 * report, which states that "unless the driver/application
2666 * is doing any special handling of a disconnect event,
2667 * there is no functional issue".
2668 *
2669 * Unfortunately, it turns out that we _do_ some special
2670 * handling of a disconnect event, namely complete all
2671 * pending transfers, notify gadget driver of the
2672 * disconnection, and so on.
2673 *
2674 * Our suggested workaround is to follow the Disconnect
2675 * Event steps here, instead, based on a setup_packet_pending
b5d335e5
FB		 2676 * flag. Such flag gets set whenever we have a SETUP_PENDING
2677 * status for EP0 TRBs and gets cleared on XferComplete for the
df62df56
FB		 2678 * same endpoint.
2679 *
2680 * Refers to:
2681 *
2682 * STAR#9000466709: RTL: Device : Disconnect event not
2683 * generated if setup packet pending in FIFO
2684 */
2685 if (dwc->revision < DWC3_REVISION_188A) {
2686 if (dwc->setup_packet_pending)
2687 dwc3_gadget_disconnect_interrupt(dwc);
2688 }
2689
8e74475b 2690 dwc3_reset_gadget(dwc);
72246da4
FB		 2691
2692 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2693 reg &= ~DWC3_DCTL_TSTCTRL_MASK;
2694 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
3b637367 2695 dwc->test_mode = false;
72246da4
FB		 2696 dwc3_clear_stall_all_ep(dwc);
2697
2698 /* Reset device address to zero */
2699 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2700 reg &= ~(DWC3_DCFG_DEVADDR_MASK);
2701 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
72246da4
FB		 2702}
2703
72246da4
FB		 2704static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
2705{
72246da4
FB		 2706 struct dwc3_ep *dep;
2707 int ret;
2708 u32 reg;
2709 u8 speed;
2710
72246da4
FB		 2711 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2712 speed = reg & DWC3_DSTS_CONNECTSPD;
2713 dwc->speed = speed;
2714
5fb6fdaf
JY		 2715 /*
2716 * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
2717 * each time on Connect Done.
2718 *
2719 * Currently we always use the reset value. If any platform
2720 * wants to set this to a different value, we need to add a
2721 * setting and update GCTL.RAMCLKSEL here.
2722 */
72246da4
FB		 2723
2724 switch (speed) {
2da9ad76 2725 case DWC3_DSTS_SUPERSPEED_PLUS:
7580862b
JY		 2726 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2727 dwc->gadget.ep0->maxpacket = 512;
2728 dwc->gadget.speed = USB_SPEED_SUPER_PLUS;
2729 break;
2da9ad76 2730 case DWC3_DSTS_SUPERSPEED:
05870c5b
FB		 2731 /*
2732 * WORKAROUND: DWC3 revisions <1.90a have an issue which
2733 * would cause a missing USB3 Reset event.
2734 *
2735 * In such situations, we should force a USB3 Reset
2736 * event by calling our dwc3_gadget_reset_interrupt()
2737 * routine.
2738 *
2739 * Refers to:
2740 *
2741 * STAR#9000483510: RTL: SS : USB3 reset event may
2742 * not be generated always when the link enters poll
2743 */
2744 if (dwc->revision < DWC3_REVISION_190A)
2745 dwc3_gadget_reset_interrupt(dwc);
2746
72246da4
FB		 2747 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2748 dwc->gadget.ep0->maxpacket = 512;
2749 dwc->gadget.speed = USB_SPEED_SUPER;
2750 break;
2da9ad76 2751 case DWC3_DSTS_HIGHSPEED:
72246da4
FB		 2752 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2753 dwc->gadget.ep0->maxpacket = 64;
2754 dwc->gadget.speed = USB_SPEED_HIGH;
2755 break;
9418ee15 2756 case DWC3_DSTS_FULLSPEED:
72246da4
FB		 2757 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2758 dwc->gadget.ep0->maxpacket = 64;
2759 dwc->gadget.speed = USB_SPEED_FULL;
2760 break;
2da9ad76 2761 case DWC3_DSTS_LOWSPEED:
72246da4
FB		 2762 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(8);
2763 dwc->gadget.ep0->maxpacket = 8;
2764 dwc->gadget.speed = USB_SPEED_LOW;
2765 break;
2766 }
2767
61800263
TN		 2768 dwc->eps[1]->endpoint.maxpacket = dwc->gadget.ep0->maxpacket;
2769
2b758350
PA		 2770 /* Enable USB2 LPM Capability */
2771
ee5cd41c 2772 if ((dwc->revision > DWC3_REVISION_194A) &&
2da9ad76
JY		 2773 (speed != DWC3_DSTS_SUPERSPEED) &&
2774 (speed != DWC3_DSTS_SUPERSPEED_PLUS)) {
2b758350
PA		 2775 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2776 reg |= DWC3_DCFG_LPM_CAP;
2777 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2778
2779 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2780 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
2781
460d098c 2782 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold);
2b758350 2783
80caf7d2
HR		 2784 /*
2785 * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and
2786 * DCFG.LPMCap is set, core responses with an ACK and the
2787 * BESL value in the LPM token is less than or equal to LPM
2788 * NYET threshold.
2789 */
2790 WARN_ONCE(dwc->revision < DWC3_REVISION_240A
2791 && dwc->has_lpm_erratum,
9165dabb 2792 "LPM Erratum not available on dwc3 revisions < 2.40a\n");
80caf7d2
HR		 2793
2794 if (dwc->has_lpm_erratum && dwc->revision >= DWC3_REVISION_240A)
2795 reg |= DWC3_DCTL_LPM_ERRATA(dwc->lpm_nyet_threshold);
2796
356363bf
FB		 2797 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2798 } else {
2799 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2800 reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
2b758350
PA		 2801 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2802 }
2803
72246da4 2804 dep = dwc->eps[0];
39ebb05c 2805 ret = __dwc3_gadget_ep_enable(dep, true, false);
72246da4
FB		 2806 if (ret) {
2807 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2808 return;
2809 }
2810
2811 dep = dwc->eps[1];
39ebb05c 2812 ret = __dwc3_gadget_ep_enable(dep, true, false);
72246da4
FB		 2813 if (ret) {
2814 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2815 return;
2816 }
2817
2818 /*
2819 * Configure PHY via GUSB3PIPECTLn if required.
2820 *
2821 * Update GTXFIFOSIZn
2822 *
2823 * In both cases reset values should be sufficient.
2824 */
2825}
2826
2827static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
2828{
72246da4
FB		 2829 /*
2830 * TODO take core out of low power mode when that's
2831 * implemented.
2832 */
2833
ad14d4e0
JL		 2834 if (dwc->gadget_driver && dwc->gadget_driver->resume) {
2835 spin_unlock(&dwc->lock);
2836 dwc->gadget_driver->resume(&dwc->gadget);
2837 spin_lock(&dwc->lock);
2838 }
72246da4
FB		 2839}
2840
2841static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
2842 unsigned int evtinfo)
2843{
fae2b904 2844 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
0b0cc1cd
FB		 2845 unsigned int pwropt;
2846
2847 /*
2848 * WORKAROUND: DWC3 < 2.50a have an issue when configured without
2849 * Hibernation mode enabled which would show up when device detects
2850 * host-initiated U3 exit.
2851 *
2852 * In that case, device will generate a Link State Change Interrupt
2853 * from U3 to RESUME which is only necessary if Hibernation is
2854 * configured in.
2855 *
2856 * There are no functional changes due to such spurious event and we
2857 * just need to ignore it.
2858 *
2859 * Refers to:
2860 *
2861 * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation
2862 * operational mode
2863 */
2864 pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
2865 if ((dwc->revision < DWC3_REVISION_250A) &&
2866 (pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
2867 if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
2868 (next == DWC3_LINK_STATE_RESUME)) {
0b0cc1cd
FB		 2869 return;
2870 }
2871 }
fae2b904
FB		 2872
2873 /*
2874 * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
2875 * on the link partner, the USB session might do multiple entry/exit
2876 * of low power states before a transfer takes place.
2877 *
2878 * Due to this problem, we might experience lower throughput. The
2879 * suggested workaround is to disable DCTL[12:9] bits if we're
2880 * transitioning from U1/U2 to U0 and enable those bits again
2881 * after a transfer completes and there are no pending transfers
2882 * on any of the enabled endpoints.
2883 *
2884 * This is the first half of that workaround.
2885 *
2886 * Refers to:
2887 *
2888 * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
2889 * core send LGO_Ux entering U0
2890 */
2891 if (dwc->revision < DWC3_REVISION_183A) {
2892 if (next == DWC3_LINK_STATE_U0) {
2893 u32 u1u2;
2894 u32 reg;
2895
2896 switch (dwc->link_state) {
2897 case DWC3_LINK_STATE_U1:
2898 case DWC3_LINK_STATE_U2:
2899 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2900 u1u2 = reg & (DWC3_DCTL_INITU2ENA
2901 | DWC3_DCTL_ACCEPTU2ENA
2902 | DWC3_DCTL_INITU1ENA
2903 | DWC3_DCTL_ACCEPTU1ENA);
2904
2905 if (!dwc->u1u2)
2906 dwc->u1u2 = reg & u1u2;
2907
2908 reg &= ~u1u2;
2909
2910 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2911 break;
2912 default:
2913 /* do nothing */
2914 break;
2915 }
2916 }
2917 }
2918
bc5ba2e0
FB		 2919 switch (next) {
2920 case DWC3_LINK_STATE_U1:
2921 if (dwc->speed == USB_SPEED_SUPER)
2922 dwc3_suspend_gadget(dwc);
2923 break;
2924 case DWC3_LINK_STATE_U2:
2925 case DWC3_LINK_STATE_U3:
2926 dwc3_suspend_gadget(dwc);
2927 break;
2928 case DWC3_LINK_STATE_RESUME:
2929 dwc3_resume_gadget(dwc);
2930 break;
2931 default:
2932 /* do nothing */
2933 break;
2934 }
2935
e57ebc1d 2936 dwc->link_state = next;
72246da4
FB		 2937}
2938
72704f87
BW		 2939static void dwc3_gadget_suspend_interrupt(struct dwc3 *dwc,
2940 unsigned int evtinfo)
2941{
2942 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
2943
2944 if (dwc->link_state != next && next == DWC3_LINK_STATE_U3)
2945 dwc3_suspend_gadget(dwc);
2946
2947 dwc->link_state = next;
2948}
2949
e1dadd3b
FB		 2950static void dwc3_gadget_hibernation_interrupt(struct dwc3 *dwc,
2951 unsigned int evtinfo)
2952{
2953 unsigned int is_ss = evtinfo & BIT(4);
2954
bfad65ee 2955 /*
e1dadd3b
FB		 2956 * WORKAROUND: DWC3 revison 2.20a with hibernation support
2957 * have a known issue which can cause USB CV TD.9.23 to fail
2958 * randomly.
2959 *
2960 * Because of this issue, core could generate bogus hibernation
2961 * events which SW needs to ignore.
2962 *
2963 * Refers to:
2964 *
2965 * STAR#9000546576: Device Mode Hibernation: Issue in USB 2.0
2966 * Device Fallback from SuperSpeed
2967 */
2968 if (is_ss ^ (dwc->speed == USB_SPEED_SUPER))
2969 return;
2970
2971 /* enter hibernation here */
2972}
2973
72246da4
FB		 2974static void dwc3_gadget_interrupt(struct dwc3 *dwc,
2975 const struct dwc3_event_devt *event)
2976{
2977 switch (event->type) {
2978 case DWC3_DEVICE_EVENT_DISCONNECT:
2979 dwc3_gadget_disconnect_interrupt(dwc);
2980 break;
2981 case DWC3_DEVICE_EVENT_RESET:
2982 dwc3_gadget_reset_interrupt(dwc);
2983 break;
2984 case DWC3_DEVICE_EVENT_CONNECT_DONE:
2985 dwc3_gadget_conndone_interrupt(dwc);
2986 break;
2987 case DWC3_DEVICE_EVENT_WAKEUP:
2988 dwc3_gadget_wakeup_interrupt(dwc);
2989 break;
e1dadd3b
FB		 2990 case DWC3_DEVICE_EVENT_HIBER_REQ:
2991 if (dev_WARN_ONCE(dwc->dev, !dwc->has_hibernation,
2992 "unexpected hibernation event\n"))
2993 break;
2994
2995 dwc3_gadget_hibernation_interrupt(dwc, event->event_info);
2996 break;
72246da4
FB		 2997 case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
2998 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
2999 break;
3000 case DWC3_DEVICE_EVENT_EOPF:
72704f87 3001 /* It changed to be suspend event for version 2.30a and above */
5eb30ced 3002 if (dwc->revision >= DWC3_REVISION_230A) {
72704f87
BW		 3003 /*
3004 * Ignore suspend event until the gadget enters into
3005 * USB_STATE_CONFIGURED state.
3006 */
3007 if (dwc->gadget.state >= USB_STATE_CONFIGURED)
3008 dwc3_gadget_suspend_interrupt(dwc,
3009 event->event_info);
3010 }
72246da4
FB		 3011 break;
3012 case DWC3_DEVICE_EVENT_SOF:
72246da4 3013 case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
72246da4 3014 case DWC3_DEVICE_EVENT_CMD_CMPL:
72246da4 3015 case DWC3_DEVICE_EVENT_OVERFLOW:
72246da4
FB		 3016 break;
3017 default:
e9f2aa87 3018 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
72246da4
FB		 3019 }
3020}
3021
3022static void dwc3_process_event_entry(struct dwc3 *dwc,
3023 const union dwc3_event *event)
3024{
43c96be1 3025 trace_dwc3_event(event->raw, dwc);
2c4cbe6e 3026
dfc5e805
FB		 3027 if (!event->type.is_devspec)
3028 dwc3_endpoint_interrupt(dwc, &event->depevt);
3029 else if (event->type.type == DWC3_EVENT_TYPE_DEV)
72246da4 3030 dwc3_gadget_interrupt(dwc, &event->devt);
dfc5e805 3031 else
72246da4 3032 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
72246da4
FB		 3033}
3034
dea520a4 3035static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
b15a762f 3036{
dea520a4 3037 struct dwc3 *dwc = evt->dwc;
b15a762f 3038 irqreturn_t ret = IRQ_NONE;
f42f2447 3039 int left;
e8adfc30 3040 u32 reg;
b15a762f 3041
f42f2447 3042 left = evt->count;
b15a762f 3043
f42f2447
FB		 3044 if (!(evt->flags & DWC3_EVENT_PENDING))
3045 return IRQ_NONE;
b15a762f 3046
f42f2447
FB		 3047 while (left > 0) {
3048 union dwc3_event event;
b15a762f 3049
ebbb2d59 3050 event.raw = *(u32 *) (evt->cache + evt->lpos);
b15a762f 3051
f42f2447 3052 dwc3_process_event_entry(dwc, &event);
b15a762f 3053
f42f2447
FB		 3054 /*
3055 * FIXME we wrap around correctly to the next entry as
3056 * almost all entries are 4 bytes in size. There is one
3057 * entry which has 12 bytes which is a regular entry
3058 * followed by 8 bytes data. ATM I don't know how
3059 * things are organized if we get next to the a
3060 * boundary so I worry about that once we try to handle
3061 * that.
3062 */
caefe6c7 3063 evt->lpos = (evt->lpos + 4) % evt->length;
f42f2447 3064 left -= 4;
f42f2447 3065 }
b15a762f 3066
f42f2447
FB		 3067 evt->count = 0;
3068 evt->flags &= ~DWC3_EVENT_PENDING;
3069 ret = IRQ_HANDLED;
b15a762f 3070
f42f2447 3071 /* Unmask interrupt */
660e9bde 3072 reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
f42f2447 3073 reg &= ~DWC3_GEVNTSIZ_INTMASK;
660e9bde 3074 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
b15a762f 3075
cf40b86b
JY		 3076 if (dwc->imod_interval) {
3077 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
3078 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
3079 }
3080
f42f2447
FB		 3081 return ret;
3082}
e8adfc30 3083
dea520a4 3084static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt)
f42f2447 3085{
dea520a4
FB		 3086 struct dwc3_event_buffer *evt = _evt;
3087 struct dwc3 *dwc = evt->dwc;
e5f68b4a 3088 unsigned long flags;
f42f2447 3089 irqreturn_t ret = IRQ_NONE;
f42f2447 3090
e5f68b4a 3091 spin_lock_irqsave(&dwc->lock, flags);
dea520a4 3092 ret = dwc3_process_event_buf(evt);
e5f68b4a 3093 spin_unlock_irqrestore(&dwc->lock, flags);
b15a762f
FB		 3094
3095 return ret;
3096}
3097
dea520a4 3098static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt)
72246da4 3099{
dea520a4 3100 struct dwc3 *dwc = evt->dwc;
ebbb2d59 3101 u32 amount;
72246da4 3102 u32 count;
e8adfc30 3103 u32 reg;
72246da4 3104
fc8bb91b
FB		 3105 if (pm_runtime_suspended(dwc->dev)) {
3106 pm_runtime_get(dwc->dev);
3107 disable_irq_nosync(dwc->irq_gadget);
3108 dwc->pending_events = true;
3109 return IRQ_HANDLED;
3110 }
3111
d325a1de
TN		 3112 /*
3113 * With PCIe legacy interrupt, test shows that top-half irq handler can
3114 * be called again after HW interrupt deassertion. Check if bottom-half
3115 * irq event handler completes before caching new event to prevent
3116 * losing events.
3117 */
3118 if (evt->flags & DWC3_EVENT_PENDING)
3119 return IRQ_HANDLED;
3120
660e9bde 3121 count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0));
72246da4
FB		 3122 count &= DWC3_GEVNTCOUNT_MASK;
3123 if (!count)
3124 return IRQ_NONE;
3125
b15a762f
FB		 3126 evt->count = count;
3127 evt->flags |= DWC3_EVENT_PENDING;
72246da4 3128
e8adfc30 3129 /* Mask interrupt */
660e9bde 3130 reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
e8adfc30 3131 reg |= DWC3_GEVNTSIZ_INTMASK;
660e9bde 3132 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
e8adfc30 3133
ebbb2d59
JY		 3134 amount = min(count, evt->length - evt->lpos);
3135 memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount);
3136
3137 if (amount < count)
3138 memcpy(evt->cache, evt->buf, count - amount);
3139
65aca320
JY		 3140 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count);
3141
b15a762f 3142 return IRQ_WAKE_THREAD;
72246da4
FB		 3143}
3144
dea520a4 3145static irqreturn_t dwc3_interrupt(int irq, void *_evt)
72246da4 3146{
dea520a4 3147 struct dwc3_event_buffer *evt = _evt;
72246da4 3148
dea520a4 3149 return dwc3_check_event_buf(evt);
72246da4
FB		 3150}
3151
6db3812e
FB		 3152static int dwc3_gadget_get_irq(struct dwc3 *dwc)
3153{
3154 struct platform_device *dwc3_pdev = to_platform_device(dwc->dev);
3155 int irq;
3156
3157 irq = platform_get_irq_byname(dwc3_pdev, "peripheral");
3158 if (irq > 0)
3159 goto out;
3160
3161 if (irq == -EPROBE_DEFER)
3162 goto out;
3163
3164 irq = platform_get_irq_byname(dwc3_pdev, "dwc_usb3");
3165 if (irq > 0)
3166 goto out;
3167
3168 if (irq == -EPROBE_DEFER)
3169 goto out;
3170
3171 irq = platform_get_irq(dwc3_pdev, 0);
3172 if (irq > 0)
3173 goto out;
3174
3175 if (irq != -EPROBE_DEFER)
3176 dev_err(dwc->dev, "missing peripheral IRQ\n");
3177
3178 if (!irq)
3179 irq = -EINVAL;
3180
3181out:
3182 return irq;
3183}
3184
72246da4 3185/**
bfad65ee 3186 * dwc3_gadget_init - initializes gadget related registers
1d046793 3187 * @dwc: pointer to our controller context structure
72246da4
FB		 3188 *
3189 * Returns 0 on success otherwise negative errno.
3190 */
41ac7b3a 3191int dwc3_gadget_init(struct dwc3 *dwc)
72246da4 3192{
6db3812e
FB		 3193 int ret;
3194 int irq;
9522def4 3195
6db3812e
FB		 3196 irq = dwc3_gadget_get_irq(dwc);
3197 if (irq < 0) {
3198 ret = irq;
3199 goto err0;
9522def4
RQ		 3200 }
3201
3202 dwc->irq_gadget = irq;
72246da4 3203
d64ff406
AB		 3204 dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev,
3205 sizeof(*dwc->ep0_trb) * 2,
3206 &dwc->ep0_trb_addr, GFP_KERNEL);
72246da4
FB		 3207 if (!dwc->ep0_trb) {
3208 dev_err(dwc->dev, "failed to allocate ep0 trb\n");
3209 ret = -ENOMEM;
7d5e650a 3210 goto err0;
72246da4
FB		 3211 }
3212
4199c5f8 3213 dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL);
72246da4 3214 if (!dwc->setup_buf) {
72246da4 3215 ret = -ENOMEM;
7d5e650a 3216 goto err1;
72246da4
FB		 3217 }
3218
905dc04e
FB		 3219 dwc->bounce = dma_alloc_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE,
3220 &dwc->bounce_addr, GFP_KERNEL);
3221 if (!dwc->bounce) {
3222 ret = -ENOMEM;
d6e5a549 3223 goto err2;
905dc04e
FB		 3224 }
3225
bb014736
BW		 3226 init_completion(&dwc->ep0_in_setup);
3227
72246da4 3228 dwc->gadget.ops = &dwc3_gadget_ops;
72246da4 3229 dwc->gadget.speed = USB_SPEED_UNKNOWN;
eeb720fb 3230 dwc->gadget.sg_supported = true;
72246da4 3231 dwc->gadget.name = "dwc3-gadget";
6a4290cc 3232 dwc->gadget.is_otg = dwc->dr_mode == USB_DR_MODE_OTG;
72246da4 3233
b9e51b2b
BM		 3234 /*
3235 * FIXME We might be setting max_speed to <SUPER, however versions
3236 * <2.20a of dwc3 have an issue with metastability (documented
3237 * elsewhere in this driver) which tells us we can't set max speed to
3238 * anything lower than SUPER.
3239 *
3240 * Because gadget.max_speed is only used by composite.c and function
3241 * drivers (i.e. it won't go into dwc3's registers) we are allowing this
3242 * to happen so we avoid sending SuperSpeed Capability descriptor
3243 * together with our BOS descriptor as that could confuse host into
3244 * thinking we can handle super speed.
3245 *
3246 * Note that, in fact, we won't even support GetBOS requests when speed
3247 * is less than super speed because we don't have means, yet, to tell
3248 * composite.c that we are USB 2.0 + LPM ECN.
3249 */
42bf02ec
RQ		 3250 if (dwc->revision < DWC3_REVISION_220A &&
3251 !dwc->dis_metastability_quirk)
5eb30ced 3252 dev_info(dwc->dev, "changing max_speed on rev %08x\n",
b9e51b2b
BM		 3253 dwc->revision);
3254
3255 dwc->gadget.max_speed = dwc->maximum_speed;
3256
72246da4
FB		 3257 /*
3258 * REVISIT: Here we should clear all pending IRQs to be
3259 * sure we're starting from a well known location.
3260 */
3261
f3bcfc7e 3262 ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps);
72246da4 3263 if (ret)
d6e5a549 3264 goto err3;
72246da4 3265
72246da4
FB		 3266 ret = usb_add_gadget_udc(dwc->dev, &dwc->gadget);
3267 if (ret) {
3268 dev_err(dwc->dev, "failed to register udc\n");
d6e5a549 3269 goto err4;
72246da4
FB		 3270 }
3271
3272 return 0;
3273
7d5e650a 3274err4:
d6e5a549 3275 dwc3_gadget_free_endpoints(dwc);
04c03d10 3276
7d5e650a 3277err3:
d6e5a549
FB		 3278 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
3279 dwc->bounce_addr);
5812b1c2 3280
7d5e650a 3281err2:
0fc9a1be 3282 kfree(dwc->setup_buf);
72246da4 3283
7d5e650a 3284err1:
d64ff406 3285 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
72246da4
FB		 3286 dwc->ep0_trb, dwc->ep0_trb_addr);
3287
72246da4
FB		 3288err0:
3289 return ret;
3290}
3291
7415f17c
FB		 3292/* -------------------------------------------------------------------------- */
3293
72246da4
FB		 3294void dwc3_gadget_exit(struct dwc3 *dwc)
3295{
72246da4 3296 usb_del_gadget_udc(&dwc->gadget);
72246da4 3297 dwc3_gadget_free_endpoints(dwc);
905dc04e 3298 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
d6e5a549 3299 dwc->bounce_addr);
0fc9a1be 3300 kfree(dwc->setup_buf);
d64ff406 3301 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
d6e5a549 3302 dwc->ep0_trb, dwc->ep0_trb_addr);
72246da4 3303}
7415f17c 3304
0b0231aa 3305int dwc3_gadget_suspend(struct dwc3 *dwc)
7415f17c 3306{
9772b47a
RQ		 3307 if (!dwc->gadget_driver)
3308 return 0;
3309
1551e35e 3310 dwc3_gadget_run_stop(dwc, false, false);
9f8a67b6
FB		 3311 dwc3_disconnect_gadget(dwc);
3312 __dwc3_gadget_stop(dwc);
7415f17c
FB		 3313
3314 return 0;
3315}
3316
3317int dwc3_gadget_resume(struct dwc3 *dwc)
3318{
7415f17c
FB		 3319 int ret;
3320
9772b47a
RQ		 3321 if (!dwc->gadget_driver)
3322 return 0;
3323
9f8a67b6
FB		 3324 ret = __dwc3_gadget_start(dwc);
3325 if (ret < 0)
7415f17c
FB		 3326 goto err0;
3327
9f8a67b6
FB		 3328 ret = dwc3_gadget_run_stop(dwc, true, false);
3329 if (ret < 0)
7415f17c
FB		 3330 goto err1;
3331
7415f17c
FB		 3332 return 0;
3333
3334err1:
9f8a67b6 3335 __dwc3_gadget_stop(dwc);
7415f17c
FB		 3336
3337err0:
3338 return ret;
3339}
fc8bb91b
FB		 3340
3341void dwc3_gadget_process_pending_events(struct dwc3 *dwc)
3342{
3343 if (dwc->pending_events) {
3344 dwc3_interrupt(dwc->irq_gadget, dwc->ev_buf);
3345 dwc->pending_events = false;
3346 enable_irq(dwc->irq_gadget);
3347 }
3348}
Linux 6.x block layer and io_uring tree(s)