projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'selinux-pr-20190801' of git://git.kernel.org/pub/scm/linux/kernel/git...
[linux-2.6-block.git] / drivers / usb / dwc2 / hcd_ddma.c
CommitLineData
5fd54ace 1// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
dc4c76e7
PZ		 2/*
3 * hcd_ddma.c - DesignWare HS OTG Controller descriptor DMA routines
4 *
5 * Copyright (C) 2004-2013 Synopsys, Inc.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions, and the following disclaimer,
12 * without modification.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. The names of the above-listed copyright holders may not be used
17 * to endorse or promote products derived from this software without
18 * specific prior written permission.
19 *
20 * ALTERNATIVELY, this software may be distributed under the terms of the
21 * GNU General Public License ("GPL") as published by the Free Software
22 * Foundation; either version 2 of the License, or (at your option) any
23 * later version.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
26 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
27 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
29 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 */
37
38/*
39 * This file contains the Descriptor DMA implementation for Host mode
40 */
41#include <linux/kernel.h>
42#include <linux/module.h>
43#include <linux/spinlock.h>
44#include <linux/interrupt.h>
45#include <linux/dma-mapping.h>
46#include <linux/io.h>
47#include <linux/slab.h>
48#include <linux/usb.h>
49
50#include <linux/usb/hcd.h>
51#include <linux/usb/ch11.h>
52
53#include "core.h"
54#include "hcd.h"
55
56static u16 dwc2_frame_list_idx(u16 frame)
57{
58 return frame & (FRLISTEN_64_SIZE - 1);
59}
60
61static u16 dwc2_desclist_idx_inc(u16 idx, u16 inc, u8 speed)
62{
63 return (idx + inc) &
64 ((speed == USB_SPEED_HIGH ? MAX_DMA_DESC_NUM_HS_ISOC :
65 MAX_DMA_DESC_NUM_GENERIC) - 1);
66}
67
68static u16 dwc2_desclist_idx_dec(u16 idx, u16 inc, u8 speed)
69{
70 return (idx - inc) &
71 ((speed == USB_SPEED_HIGH ? MAX_DMA_DESC_NUM_HS_ISOC :
72 MAX_DMA_DESC_NUM_GENERIC) - 1);
73}
74
75static u16 dwc2_max_desc_num(struct dwc2_qh *qh)
76{
77 return (qh->ep_type == USB_ENDPOINT_XFER_ISOC &&
78 qh->dev_speed == USB_SPEED_HIGH) ?
79 MAX_DMA_DESC_NUM_HS_ISOC : MAX_DMA_DESC_NUM_GENERIC;
80}
81
82static u16 dwc2_frame_incr_val(struct dwc2_qh *qh)
83{
84 return qh->dev_speed == USB_SPEED_HIGH ?
ced9eee1 85 (qh->host_interval + 8 - 1) / 8 : qh->host_interval;
dc4c76e7
PZ		 86}
87
88static int dwc2_desc_list_alloc(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,
89 gfp_t flags)
90{
3b5fcc9a
GH		 91 struct kmem_cache *desc_cache;
92
ab283202
JY		 93 if (qh->ep_type == USB_ENDPOINT_XFER_ISOC &&
94 qh->dev_speed == USB_SPEED_HIGH)
3b5fcc9a
GH		 95 desc_cache = hsotg->desc_hsisoc_cache;
96 else
97 desc_cache = hsotg->desc_gen_cache;
98
ec703251 99 qh->desc_list_sz = sizeof(struct dwc2_dma_desc) *
95105a99 100 dwc2_max_desc_num(qh);
dc4c76e7 101
3b5fcc9a 102 qh->desc_list = kmem_cache_zalloc(desc_cache, flags | GFP_DMA);
dc4c76e7
PZ		 103 if (!qh->desc_list)
104 return -ENOMEM;
105
95105a99
GH		 106 qh->desc_list_dma = dma_map_single(hsotg->dev, qh->desc_list,
107 qh->desc_list_sz,
108 DMA_TO_DEVICE);
dc4c76e7 109
9da51974 110 qh->n_bytes = kcalloc(dwc2_max_desc_num(qh), sizeof(u32), flags);
dc4c76e7 111 if (!qh->n_bytes) {
95105a99
GH		 112 dma_unmap_single(hsotg->dev, qh->desc_list_dma,
113 qh->desc_list_sz,
114 DMA_FROM_DEVICE);
9bbe91a1 115 kmem_cache_free(desc_cache, qh->desc_list);
dc4c76e7
PZ		 116 qh->desc_list = NULL;
117 return -ENOMEM;
118 }
119
120 return 0;
121}
122
123static void dwc2_desc_list_free(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)
124{
3b5fcc9a
GH		 125 struct kmem_cache *desc_cache;
126
ab283202
JY		 127 if (qh->ep_type == USB_ENDPOINT_XFER_ISOC &&
128 qh->dev_speed == USB_SPEED_HIGH)
3b5fcc9a
GH		 129 desc_cache = hsotg->desc_hsisoc_cache;
130 else
131 desc_cache = hsotg->desc_gen_cache;
132
dc4c76e7 133 if (qh->desc_list) {
95105a99
GH		 134 dma_unmap_single(hsotg->dev, qh->desc_list_dma,
135 qh->desc_list_sz, DMA_FROM_DEVICE);
3b5fcc9a 136 kmem_cache_free(desc_cache, qh->desc_list);
dc4c76e7
PZ		 137 qh->desc_list = NULL;
138 }
139
140 kfree(qh->n_bytes);
141 qh->n_bytes = NULL;
142}
143
144static int dwc2_frame_list_alloc(struct dwc2_hsotg *hsotg, gfp_t mem_flags)
145{
146 if (hsotg->frame_list)
147 return 0;
148
95105a99
GH		 149 hsotg->frame_list_sz = 4 * FRLISTEN_64_SIZE;
150 hsotg->frame_list = kzalloc(hsotg->frame_list_sz, GFP_ATOMIC | GFP_DMA);
dc4c76e7
PZ		 151 if (!hsotg->frame_list)
152 return -ENOMEM;
153
95105a99
GH		 154 hsotg->frame_list_dma = dma_map_single(hsotg->dev, hsotg->frame_list,
155 hsotg->frame_list_sz,
156 DMA_TO_DEVICE);
157
dc4c76e7
PZ		 158 return 0;
159}
160
161static void dwc2_frame_list_free(struct dwc2_hsotg *hsotg)
162{
dc4c76e7
PZ		 163 unsigned long flags;
164
165 spin_lock_irqsave(&hsotg->lock, flags);
166
167 if (!hsotg->frame_list) {
168 spin_unlock_irqrestore(&hsotg->lock, flags);
169 return;
170 }
171
95105a99
GH		 172 dma_unmap_single(hsotg->dev, hsotg->frame_list_dma,
173 hsotg->frame_list_sz, DMA_FROM_DEVICE);
174
175 kfree(hsotg->frame_list);
dc4c76e7
PZ		 176 hsotg->frame_list = NULL;
177
178 spin_unlock_irqrestore(&hsotg->lock, flags);
dc4c76e7
PZ		 179}
180
181static void dwc2_per_sched_enable(struct dwc2_hsotg *hsotg, u32 fr_list_en)
182{
183 u32 hcfg;
184 unsigned long flags;
185
186 spin_lock_irqsave(&hsotg->lock, flags);
187
f25c42b8 188 hcfg = dwc2_readl(hsotg, HCFG);
dc4c76e7
PZ		 189 if (hcfg & HCFG_PERSCHEDENA) {
190 /* already enabled */
191 spin_unlock_irqrestore(&hsotg->lock, flags);
192 return;
193 }
194
f25c42b8 195 dwc2_writel(hsotg, hsotg->frame_list_dma, HFLBADDR);
dc4c76e7
PZ		 196
197 hcfg &= ~HCFG_FRLISTEN_MASK;
198 hcfg |= fr_list_en | HCFG_PERSCHEDENA;
199 dev_vdbg(hsotg->dev, "Enabling Periodic schedule\n");
f25c42b8 200 dwc2_writel(hsotg, hcfg, HCFG);
dc4c76e7
PZ		 201
202 spin_unlock_irqrestore(&hsotg->lock, flags);
203}
204
205static void dwc2_per_sched_disable(struct dwc2_hsotg *hsotg)
206{
207 u32 hcfg;
208 unsigned long flags;
209
210 spin_lock_irqsave(&hsotg->lock, flags);
211
f25c42b8 212 hcfg = dwc2_readl(hsotg, HCFG);
dc4c76e7
PZ		 213 if (!(hcfg & HCFG_PERSCHEDENA)) {
214 /* already disabled */
215 spin_unlock_irqrestore(&hsotg->lock, flags);
216 return;
217 }
218
219 hcfg &= ~HCFG_PERSCHEDENA;
220 dev_vdbg(hsotg->dev, "Disabling Periodic schedule\n");
f25c42b8 221 dwc2_writel(hsotg, hcfg, HCFG);
dc4c76e7
PZ		 222
223 spin_unlock_irqrestore(&hsotg->lock, flags);
224}
225
226/*
227 * Activates/Deactivates FrameList entries for the channel based on endpoint
228 * servicing period
229 */
230static void dwc2_update_frame_list(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,
231 int enable)
232{
233 struct dwc2_host_chan *chan;
234 u16 i, j, inc;
235
ddf58468 236 if (!hsotg) {
de9169a1 237 pr_err("hsotg = %p\n", hsotg);
dc4c76e7
PZ		 238 return;
239 }
240
ddf58468 241 if (!qh->channel) {
de9169a1 242 dev_err(hsotg->dev, "qh->channel = %p\n", qh->channel);
dc4c76e7
PZ		 243 return;
244 }
245
246 if (!hsotg->frame_list) {
de9169a1 247 dev_err(hsotg->dev, "hsotg->frame_list = %p\n",
dc4c76e7
PZ		 248 hsotg->frame_list);
249 return;
250 }
251
252 chan = qh->channel;
253 inc = dwc2_frame_incr_val(qh);
254 if (qh->ep_type == USB_ENDPOINT_XFER_ISOC)
ced9eee1 255 i = dwc2_frame_list_idx(qh->next_active_frame);
dc4c76e7
PZ		 256 else
257 i = 0;
258
259 j = i;
260 do {
261 if (enable)
262 hsotg->frame_list[j] |= 1 << chan->hc_num;
263 else
264 hsotg->frame_list[j] &= ~(1 << chan->hc_num);
265 j = (j + inc) & (FRLISTEN_64_SIZE - 1);
266 } while (j != i);
267
95105a99
GH		 268 /*
269 * Sync frame list since controller will access it if periodic
270 * channel is currently enabled.
271 */
272 dma_sync_single_for_device(hsotg->dev,
273 hsotg->frame_list_dma,
274 hsotg->frame_list_sz,
275 DMA_TO_DEVICE);
276
dc4c76e7
PZ		 277 if (!enable)
278 return;
279
280 chan->schinfo = 0;
ced9eee1 281 if (chan->speed == USB_SPEED_HIGH && qh->host_interval) {
dc4c76e7
PZ		 282 j = 1;
283 /* TODO - check this */
ced9eee1 284 inc = (8 + qh->host_interval - 1) / qh->host_interval;
dc4c76e7
PZ		 285 for (i = 0; i < inc; i++) {
286 chan->schinfo |= j;
ced9eee1 287 j = j << qh->host_interval;
dc4c76e7
PZ		 288 }
289 } else {
290 chan->schinfo = 0xff;
291 }
292}
293
294static void dwc2_release_channel_ddma(struct dwc2_hsotg *hsotg,
295 struct dwc2_qh *qh)
296{
297 struct dwc2_host_chan *chan = qh->channel;
298
20f2eb9c 299 if (dwc2_qh_is_non_per(qh)) {
95832c00 300 if (hsotg->params.uframe_sched)
20f2eb9c
DC		 301 hsotg->available_host_channels++;
302 else
303 hsotg->non_periodic_channels--;
304 } else {
dc4c76e7 305 dwc2_update_frame_list(hsotg, qh, 0);
3f808bda 306 hsotg->available_host_channels++;
20f2eb9c 307 }
dc4c76e7
PZ		 308
309 /*
310 * The condition is added to prevent double cleanup try in case of
311 * device disconnect. See channel cleanup in dwc2_hcd_disconnect().
312 */
313 if (chan->qh) {
314 if (!list_empty(&chan->hc_list_entry))
315 list_del(&chan->hc_list_entry);
316 dwc2_hc_cleanup(hsotg, chan);
317 list_add_tail(&chan->hc_list_entry, &hsotg->free_hc_list);
318 chan->qh = NULL;
319 }
320
321 qh->channel = NULL;
322 qh->ntd = 0;
323
324 if (qh->desc_list)
ec703251 325 memset(qh->desc_list, 0, sizeof(struct dwc2_dma_desc) *
dc4c76e7
PZ		 326 dwc2_max_desc_num(qh));
327}
328
329/**
330 * dwc2_hcd_qh_init_ddma() - Initializes a QH structure's Descriptor DMA
331 * related members
332 *
333 * @hsotg: The HCD state structure for the DWC OTG controller
334 * @qh: The QH to init
6fb914d7 335 * @mem_flags: Indicates the type of memory allocation
dc4c76e7
PZ		 336 *
337 * Return: 0 if successful, negative error code otherwise
338 *
339 * Allocates memory for the descriptor list. For the first periodic QH,
340 * allocates memory for the FrameList and enables periodic scheduling.
341 */
342int dwc2_hcd_qh_init_ddma(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,
343 gfp_t mem_flags)
344{
345 int retval;
346
347 if (qh->do_split) {
348 dev_err(hsotg->dev,
349 "SPLIT Transfers are not supported in Descriptor DMA mode.\n");
350 retval = -EINVAL;
351 goto err0;
352 }
353
354 retval = dwc2_desc_list_alloc(hsotg, qh, mem_flags);
355 if (retval)
356 goto err0;
357
358 if (qh->ep_type == USB_ENDPOINT_XFER_ISOC ||
359 qh->ep_type == USB_ENDPOINT_XFER_INT) {
360 if (!hsotg->frame_list) {
361 retval = dwc2_frame_list_alloc(hsotg, mem_flags);
362 if (retval)
363 goto err1;
364 /* Enable periodic schedule on first periodic QH */
365 dwc2_per_sched_enable(hsotg, HCFG_FRLISTEN_64);
366 }
367 }
368
369 qh->ntd = 0;
370 return 0;
371
372err1:
373 dwc2_desc_list_free(hsotg, qh);
374err0:
375 return retval;
376}
377
378/**
379 * dwc2_hcd_qh_free_ddma() - Frees a QH structure's Descriptor DMA related
380 * members
381 *
382 * @hsotg: The HCD state structure for the DWC OTG controller
383 * @qh: The QH to free
384 *
385 * Frees descriptor list memory associated with the QH. If QH is periodic and
386 * the last, frees FrameList memory and disables periodic scheduling.
387 */
388void dwc2_hcd_qh_free_ddma(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)
389{
2b046bc5
GH		 390 unsigned long flags;
391
dc4c76e7
PZ		 392 dwc2_desc_list_free(hsotg, qh);
393
394 /*
395 * Channel still assigned due to some reasons.
396 * Seen on Isoc URB dequeue. Channel halted but no subsequent
397 * ChHalted interrupt to release the channel. Afterwards
398 * when it comes here from endpoint disable routine
399 * channel remains assigned.
400 */
2b046bc5 401 spin_lock_irqsave(&hsotg->lock, flags);
dc4c76e7
PZ		 402 if (qh->channel)
403 dwc2_release_channel_ddma(hsotg, qh);
2b046bc5 404 spin_unlock_irqrestore(&hsotg->lock, flags);
dc4c76e7
PZ		 405
406 if ((qh->ep_type == USB_ENDPOINT_XFER_ISOC ||
407 qh->ep_type == USB_ENDPOINT_XFER_INT) &&
95832c00 408 (hsotg->params.uframe_sched ||
20f2eb9c 409 !hsotg->periodic_channels) && hsotg->frame_list) {
dc4c76e7
PZ		 410 dwc2_per_sched_disable(hsotg);
411 dwc2_frame_list_free(hsotg);
412 }
413}
414
415static u8 dwc2_frame_to_desc_idx(struct dwc2_qh *qh, u16 frame_idx)
416{
417 if (qh->dev_speed == USB_SPEED_HIGH)
418 /* Descriptor set (8 descriptors) index which is 8-aligned */
419 return (frame_idx & ((MAX_DMA_DESC_NUM_HS_ISOC / 8) - 1)) * 8;
420 else
421 return frame_idx & (MAX_DMA_DESC_NUM_GENERIC - 1);
422}
423
424/*
425 * Determine starting frame for Isochronous transfer.
426 * Few frames skipped to prevent race condition with HC.
427 */
428static u16 dwc2_calc_starting_frame(struct dwc2_hsotg *hsotg,
429 struct dwc2_qh *qh, u16 *skip_frames)
430{
431 u16 frame;
432
433 hsotg->frame_number = dwc2_hcd_get_frame_number(hsotg);
434
ced9eee1
DA		 435 /*
436 * next_active_frame is always frame number (not uFrame) both in FS
437 * and HS!
438 */
dc4c76e7
PZ		 439
440 /*
441 * skip_frames is used to limit activated descriptors number
442 * to avoid the situation when HC services the last activated
443 * descriptor firstly.
444 * Example for FS:
445 * Current frame is 1, scheduled frame is 3. Since HC always fetches
446 * the descriptor corresponding to curr_frame+1, the descriptor
447 * corresponding to frame 2 will be fetched. If the number of
448 * descriptors is max=64 (or greather) the list will be fully programmed
449 * with Active descriptors and it is possible case (rare) that the
450 * latest descriptor(considering rollback) corresponding to frame 2 will
451 * be serviced first. HS case is more probable because, in fact, up to
452 * 11 uframes (16 in the code) may be skipped.
453 */
454 if (qh->dev_speed == USB_SPEED_HIGH) {
455 /*
456 * Consider uframe counter also, to start xfer asap. If half of
457 * the frame elapsed skip 2 frames otherwise just 1 frame.
458 * Starting descriptor index must be 8-aligned, so if the
459 * current frame is near to complete the next one is skipped as
460 * well.
461 */
462 if (dwc2_micro_frame_num(hsotg->frame_number) >= 5) {
463 *skip_frames = 2 * 8;
464 frame = dwc2_frame_num_inc(hsotg->frame_number,
465 *skip_frames);
466 } else {
467 *skip_frames = 1 * 8;
468 frame = dwc2_frame_num_inc(hsotg->frame_number,
469 *skip_frames);
470 }
471
472 frame = dwc2_full_frame_num(frame);
473 } else {
474 /*
475 * Two frames are skipped for FS - the current and the next.
476 * But for descriptor programming, 1 frame (descriptor) is
477 * enough, see example above.
478 */
479 *skip_frames = 1;
480 frame = dwc2_frame_num_inc(hsotg->frame_number, 2);
481 }
482
483 return frame;
484}
485
486/*
487 * Calculate initial descriptor index for isochronous transfer based on
488 * scheduled frame
489 */
490static u16 dwc2_recalc_initial_desc_idx(struct dwc2_hsotg *hsotg,
491 struct dwc2_qh *qh)
492{
493 u16 frame, fr_idx, fr_idx_tmp, skip_frames;
494
495 /*
496 * With current ISOC processing algorithm the channel is being released
497 * when no more QTDs in the list (qh->ntd == 0). Thus this function is
498 * called only when qh->ntd == 0 and qh->channel == 0.
499 *
500 * So qh->channel != NULL branch is not used and just not removed from
501 * the source file. It is required for another possible approach which
502 * is, do not disable and release the channel when ISOC session
503 * completed, just move QH to inactive schedule until new QTD arrives.
504 * On new QTD, the QH moved back to 'ready' schedule, starting frame and
505 * therefore starting desc_index are recalculated. In this case channel
506 * is released only on ep_disable.
507 */
508
509 /*
510 * Calculate starting descriptor index. For INTERRUPT endpoint it is
511 * always 0.
512 */
513 if (qh->channel) {
514 frame = dwc2_calc_starting_frame(hsotg, qh, &skip_frames);
515 /*
516 * Calculate initial descriptor index based on FrameList current
517 * bitmap and servicing period
518 */
519 fr_idx_tmp = dwc2_frame_list_idx(frame);
520 fr_idx = (FRLISTEN_64_SIZE +
ced9eee1
DA		 521 dwc2_frame_list_idx(qh->next_active_frame) -
522 fr_idx_tmp) % dwc2_frame_incr_val(qh);
dc4c76e7
PZ		 523 fr_idx = (fr_idx + fr_idx_tmp) % FRLISTEN_64_SIZE;
524 } else {
ced9eee1 525 qh->next_active_frame = dwc2_calc_starting_frame(hsotg, qh,
dc4c76e7 526 &skip_frames);
ced9eee1 527 fr_idx = dwc2_frame_list_idx(qh->next_active_frame);
dc4c76e7
PZ		 528 }
529
530 qh->td_first = qh->td_last = dwc2_frame_to_desc_idx(qh, fr_idx);
531
532 return skip_frames;
533}
534
535#define ISOC_URB_GIVEBACK_ASAP
536
537#define MAX_ISOC_XFER_SIZE_FS 1023
538#define MAX_ISOC_XFER_SIZE_HS 3072
539#define DESCNUM_THRESHOLD 4
540
541static void dwc2_fill_host_isoc_dma_desc(struct dwc2_hsotg *hsotg,
542 struct dwc2_qtd *qtd,
543 struct dwc2_qh *qh, u32 max_xfer_size,
544 u16 idx)
545{
ec703251 546 struct dwc2_dma_desc *dma_desc = &qh->desc_list[idx];
dc4c76e7
PZ		 547 struct dwc2_hcd_iso_packet_desc *frame_desc;
548
549 memset(dma_desc, 0, sizeof(*dma_desc));
550 frame_desc = &qtd->urb->iso_descs[qtd->isoc_frame_index_last];
551
552 if (frame_desc->length > max_xfer_size)
553 qh->n_bytes[idx] = max_xfer_size;
554 else
555 qh->n_bytes[idx] = frame_desc->length;
556
557 dma_desc->buf = (u32)(qtd->urb->dma + frame_desc->offset);
558 dma_desc->status = qh->n_bytes[idx] << HOST_DMA_ISOC_NBYTES_SHIFT &
559 HOST_DMA_ISOC_NBYTES_MASK;
560
dde4c1bf
GH		 561 /* Set active bit */
562 dma_desc->status |= HOST_DMA_A;
563
3ac38d26
GH		 564 qh->ntd++;
565 qtd->isoc_frame_index_last++;
566
dc4c76e7
PZ		 567#ifdef ISOC_URB_GIVEBACK_ASAP
568 /* Set IOC for each descriptor corresponding to last frame of URB */
569 if (qtd->isoc_frame_index_last == qtd->urb->packet_count)
570 dma_desc->status |= HOST_DMA_IOC;
571#endif
572
95105a99 573 dma_sync_single_for_device(hsotg->dev,
9da51974 574 qh->desc_list_dma +
ec703251
VA		 575 (idx * sizeof(struct dwc2_dma_desc)),
576 sizeof(struct dwc2_dma_desc),
95105a99 577 DMA_TO_DEVICE);
dc4c76e7
PZ		 578}
579
580static void dwc2_init_isoc_dma_desc(struct dwc2_hsotg *hsotg,
581 struct dwc2_qh *qh, u16 skip_frames)
582{
583 struct dwc2_qtd *qtd;
584 u32 max_xfer_size;
c17b337c
GH		 585 u16 idx, inc, n_desc = 0, ntd_max = 0;
586 u16 cur_idx;
587 u16 next_idx;
dc4c76e7
PZ		 588
589 idx = qh->td_last;
ced9eee1 590 inc = qh->host_interval;
c17b337c
GH		 591 hsotg->frame_number = dwc2_hcd_get_frame_number(hsotg);
592 cur_idx = dwc2_frame_list_idx(hsotg->frame_number);
593 next_idx = dwc2_desclist_idx_inc(qh->td_last, inc, qh->dev_speed);
594
595 /*
596 * Ensure current frame number didn't overstep last scheduled
597 * descriptor. If it happens, the only way to recover is to move
598 * qh->td_last to current frame number + 1.
599 * So that next isoc descriptor will be scheduled on frame number + 1
600 * and not on a past frame.
601 */
602 if (dwc2_frame_idx_num_gt(cur_idx, next_idx) || (cur_idx == next_idx)) {
603 if (inc < 32) {
604 dev_vdbg(hsotg->dev,
605 "current frame number overstep last descriptor\n");
606 qh->td_last = dwc2_desclist_idx_inc(cur_idx, inc,
607 qh->dev_speed);
608 idx = qh->td_last;
609 }
610 }
dc4c76e7 611
ced9eee1
DA		 612 if (qh->host_interval) {
613 ntd_max = (dwc2_max_desc_num(qh) + qh->host_interval - 1) /
614 qh->host_interval;
dc4c76e7 615 if (skip_frames && !qh->channel)
ced9eee1 616 ntd_max -= skip_frames / qh->host_interval;
dc4c76e7
PZ		 617 }
618
619 max_xfer_size = qh->dev_speed == USB_SPEED_HIGH ?
620 MAX_ISOC_XFER_SIZE_HS : MAX_ISOC_XFER_SIZE_FS;
621
622 list_for_each_entry(qtd, &qh->qtd_list, qtd_list_entry) {
c17b337c
GH		 623 if (qtd->in_process &&
624 qtd->isoc_frame_index_last ==
625 qtd->urb->packet_count)
626 continue;
627
628 qtd->isoc_td_first = idx;
dc4c76e7
PZ		 629 while (qh->ntd < ntd_max && qtd->isoc_frame_index_last <
630 qtd->urb->packet_count) {
dc4c76e7
PZ		 631 dwc2_fill_host_isoc_dma_desc(hsotg, qtd, qh,
632 max_xfer_size, idx);
633 idx = dwc2_desclist_idx_inc(idx, inc, qh->dev_speed);
634 n_desc++;
635 }
c17b337c 636 qtd->isoc_td_last = idx;
dc4c76e7
PZ		 637 qtd->in_process = 1;
638 }
639
640 qh->td_last = idx;
641
642#ifdef ISOC_URB_GIVEBACK_ASAP
643 /* Set IOC for last descriptor if descriptor list is full */
644 if (qh->ntd == ntd_max) {
645 idx = dwc2_desclist_idx_dec(qh->td_last, inc, qh->dev_speed);
646 qh->desc_list[idx].status |= HOST_DMA_IOC;
95105a99
GH		 647 dma_sync_single_for_device(hsotg->dev,
648 qh->desc_list_dma + (idx *
ec703251
VA		 649 sizeof(struct dwc2_dma_desc)),
650 sizeof(struct dwc2_dma_desc),
95105a99 651 DMA_TO_DEVICE);
dc4c76e7
PZ		 652 }
653#else
654 /*
655 * Set IOC bit only for one descriptor. Always try to be ahead of HW
656 * processing, i.e. on IOC generation driver activates next descriptor
657 * but core continues to process descriptors following the one with IOC
658 * set.
659 */
660
661 if (n_desc > DESCNUM_THRESHOLD)
662 /*
663 * Move IOC "up". Required even if there is only one QTD
664 * in the list, because QTDs might continue to be queued,
665 * but during the activation it was only one queued.
666 * Actually more than one QTD might be in the list if this
667 * function called from XferCompletion - QTDs was queued during
668 * HW processing of the previous descriptor chunk.
669 */
670 idx = dwc2_desclist_idx_dec(idx, inc * ((qh->ntd + 1) / 2),
671 qh->dev_speed);
672 else
673 /*
674 * Set the IOC for the latest descriptor if either number of
675 * descriptors is not greater than threshold or no more new
676 * descriptors activated
677 */
678 idx = dwc2_desclist_idx_dec(qh->td_last, inc, qh->dev_speed);
679
680 qh->desc_list[idx].status |= HOST_DMA_IOC;
95105a99
GH		 681 dma_sync_single_for_device(hsotg->dev,
682 qh->desc_list_dma +
ec703251
VA		 683 (idx * sizeof(struct dwc2_dma_desc)),
684 sizeof(struct dwc2_dma_desc),
95105a99 685 DMA_TO_DEVICE);
dc4c76e7 686#endif
dc4c76e7
PZ		 687}
688
689static void dwc2_fill_host_dma_desc(struct dwc2_hsotg *hsotg,
690 struct dwc2_host_chan *chan,
691 struct dwc2_qtd *qtd, struct dwc2_qh *qh,
692 int n_desc)
693{
ec703251 694 struct dwc2_dma_desc *dma_desc = &qh->desc_list[n_desc];
dc4c76e7
PZ		 695 int len = chan->xfer_len;
696
3a1ec351
VA		 697 if (len > HOST_DMA_NBYTES_LIMIT - (chan->max_packet - 1))
698 len = HOST_DMA_NBYTES_LIMIT - (chan->max_packet - 1);
dc4c76e7
PZ		 699
700 if (chan->ep_is_in) {
701 int num_packets;
702
703 if (len > 0 && chan->max_packet)
704 num_packets = (len + chan->max_packet - 1)
705 / chan->max_packet;
706 else
707 /* Need 1 packet for transfer length of 0 */
708 num_packets = 1;
709
710 /* Always program an integral # of packets for IN transfers */
711 len = num_packets * chan->max_packet;
712 }
713
714 dma_desc->status = len << HOST_DMA_NBYTES_SHIFT & HOST_DMA_NBYTES_MASK;
715 qh->n_bytes[n_desc] = len;
716
717 if (qh->ep_type == USB_ENDPOINT_XFER_CONTROL &&
718 qtd->control_phase == DWC2_CONTROL_SETUP)
719 dma_desc->status |= HOST_DMA_SUP;
720
721 dma_desc->buf = (u32)chan->xfer_dma;
722
95105a99
GH		 723 dma_sync_single_for_device(hsotg->dev,
724 qh->desc_list_dma +
ec703251
VA		 725 (n_desc * sizeof(struct dwc2_dma_desc)),
726 sizeof(struct dwc2_dma_desc),
95105a99
GH		 727 DMA_TO_DEVICE);
728
dc4c76e7
PZ		 729 /*
730 * Last (or only) descriptor of IN transfer with actual size less
731 * than MaxPacket
732 */
733 if (len > chan->xfer_len) {
734 chan->xfer_len = 0;
735 } else {
736 chan->xfer_dma += len;
737 chan->xfer_len -= len;
738 }
739}
740
741static void dwc2_init_non_isoc_dma_desc(struct dwc2_hsotg *hsotg,
742 struct dwc2_qh *qh)
743{
744 struct dwc2_qtd *qtd;
745 struct dwc2_host_chan *chan = qh->channel;
746 int n_desc = 0;
747
748 dev_vdbg(hsotg->dev, "%s(): qh=%p dma=%08lx len=%d\n", __func__, qh,
749 (unsigned long)chan->xfer_dma, chan->xfer_len);
750
751 /*
752 * Start with chan->xfer_dma initialized in assign_and_init_hc(), then
753 * if SG transfer consists of multiple URBs, this pointer is re-assigned
754 * to the buffer of the currently processed QTD. For non-SG request
755 * there is always one QTD active.
756 */
757
758 list_for_each_entry(qtd, &qh->qtd_list, qtd_list_entry) {
759 dev_vdbg(hsotg->dev, "qtd=%p\n", qtd);
760
761 if (n_desc) {
762 /* SG request - more than 1 QTD */
763 chan->xfer_dma = qtd->urb->dma +
764 qtd->urb->actual_length;
765 chan->xfer_len = qtd->urb->length -
766 qtd->urb->actual_length;
767 dev_vdbg(hsotg->dev, "buf=%08lx len=%d\n",
768 (unsigned long)chan->xfer_dma, chan->xfer_len);
769 }
770
771 qtd->n_desc = 0;
772 do {
773 if (n_desc > 1) {
774 qh->desc_list[n_desc - 1].status |= HOST_DMA_A;
775 dev_vdbg(hsotg->dev,
776 "set A bit in desc %d (%p)\n",
777 n_desc - 1,
778 &qh->desc_list[n_desc - 1]);
95105a99 779 dma_sync_single_for_device(hsotg->dev,
9da51974 780 qh->desc_list_dma +
95105a99 781 ((n_desc - 1) *
ec703251
VA		 782 sizeof(struct dwc2_dma_desc)),
783 sizeof(struct dwc2_dma_desc),
95105a99 784 DMA_TO_DEVICE);
dc4c76e7
PZ		 785 }
786 dwc2_fill_host_dma_desc(hsotg, chan, qtd, qh, n_desc);
787 dev_vdbg(hsotg->dev,
788 "desc %d (%p) buf=%08x status=%08x\n",
789 n_desc, &qh->desc_list[n_desc],
790 qh->desc_list[n_desc].buf,
791 qh->desc_list[n_desc].status);
792 qtd->n_desc++;
793 n_desc++;
794 } while (chan->xfer_len > 0 &&
795 n_desc != MAX_DMA_DESC_NUM_GENERIC);
796
797 dev_vdbg(hsotg->dev, "n_desc=%d\n", n_desc);
798 qtd->in_process = 1;
799 if (qh->ep_type == USB_ENDPOINT_XFER_CONTROL)
800 break;
801 if (n_desc == MAX_DMA_DESC_NUM_GENERIC)
802 break;
803 }
804
805 if (n_desc) {
806 qh->desc_list[n_desc - 1].status |=
807 HOST_DMA_IOC | HOST_DMA_EOL | HOST_DMA_A;
808 dev_vdbg(hsotg->dev, "set IOC/EOL/A bits in desc %d (%p)\n",
809 n_desc - 1, &qh->desc_list[n_desc - 1]);
95105a99
GH		 810 dma_sync_single_for_device(hsotg->dev,
811 qh->desc_list_dma + (n_desc - 1) *
ec703251
VA		 812 sizeof(struct dwc2_dma_desc),
813 sizeof(struct dwc2_dma_desc),
95105a99 814 DMA_TO_DEVICE);
dc4c76e7
PZ		 815 if (n_desc > 1) {
816 qh->desc_list[0].status |= HOST_DMA_A;
817 dev_vdbg(hsotg->dev, "set A bit in desc 0 (%p)\n",
818 &qh->desc_list[0]);
95105a99 819 dma_sync_single_for_device(hsotg->dev,
9da51974 820 qh->desc_list_dma,
ec703251 821 sizeof(struct dwc2_dma_desc),
95105a99 822 DMA_TO_DEVICE);
dc4c76e7
PZ		 823 }
824 chan->ntd = n_desc;
825 }
826}
827
828/**
829 * dwc2_hcd_start_xfer_ddma() - Starts a transfer in Descriptor DMA mode
830 *
831 * @hsotg: The HCD state structure for the DWC OTG controller
832 * @qh: The QH to init
833 *
834 * Return: 0 if successful, negative error code otherwise
835 *
836 * For Control and Bulk endpoints, initializes descriptor list and starts the
837 * transfer. For Interrupt and Isochronous endpoints, initializes descriptor
838 * list then updates FrameList, marking appropriate entries as active.
839 *
840 * For Isochronous endpoints the starting descriptor index is calculated based
841 * on the scheduled frame, but only on the first transfer descriptor within a
842 * session. Then the transfer is started via enabling the channel.
843 *
844 * For Isochronous endpoints the channel is not halted on XferComplete
845 * interrupt so remains assigned to the endpoint(QH) until session is done.
846 */
847void dwc2_hcd_start_xfer_ddma(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)
848{
849 /* Channel is already assigned */
850 struct dwc2_host_chan *chan = qh->channel;
851 u16 skip_frames = 0;
852
853 switch (chan->ep_type) {
854 case USB_ENDPOINT_XFER_CONTROL:
855 case USB_ENDPOINT_XFER_BULK:
856 dwc2_init_non_isoc_dma_desc(hsotg, qh);
857 dwc2_hc_start_transfer_ddma(hsotg, chan);
858 break;
859 case USB_ENDPOINT_XFER_INT:
860 dwc2_init_non_isoc_dma_desc(hsotg, qh);
861 dwc2_update_frame_list(hsotg, qh, 1);
862 dwc2_hc_start_transfer_ddma(hsotg, chan);
863 break;
864 case USB_ENDPOINT_XFER_ISOC:
865 if (!qh->ntd)
866 skip_frames = dwc2_recalc_initial_desc_idx(hsotg, qh);
867 dwc2_init_isoc_dma_desc(hsotg, qh, skip_frames);
868
869 if (!chan->xfer_started) {
870 dwc2_update_frame_list(hsotg, qh, 1);
871
872 /*
873 * Always set to max, instead of actual size. Otherwise
874 * ntd will be changed with channel being enabled. Not
875 * recommended.
876 */
877 chan->ntd = dwc2_max_desc_num(qh);
878
879 /* Enable channel only once for ISOC */
880 dwc2_hc_start_transfer_ddma(hsotg, chan);
881 }
882
883 break;
884 default:
885 break;
886 }
887}
888
889#define DWC2_CMPL_DONE 1
890#define DWC2_CMPL_STOP 2
891
892static int dwc2_cmpl_host_isoc_dma_desc(struct dwc2_hsotg *hsotg,
893 struct dwc2_host_chan *chan,
894 struct dwc2_qtd *qtd,
895 struct dwc2_qh *qh, u16 idx)
896{
ec703251 897 struct dwc2_dma_desc *dma_desc;
dc4c76e7
PZ		 898 struct dwc2_hcd_iso_packet_desc *frame_desc;
899 u16 remain = 0;
900 int rc = 0;
901
0d012b98
PZ		 902 if (!qtd->urb)
903 return -EINVAL;
904
95105a99 905 dma_sync_single_for_cpu(hsotg->dev, qh->desc_list_dma + (idx *
ec703251
VA		 906 sizeof(struct dwc2_dma_desc)),
907 sizeof(struct dwc2_dma_desc),
95105a99
GH		 908 DMA_FROM_DEVICE);
909
910 dma_desc = &qh->desc_list[idx];
911
dc4c76e7
PZ		 912 frame_desc = &qtd->urb->iso_descs[qtd->isoc_frame_index_last];
913 dma_desc->buf = (u32)(qtd->urb->dma + frame_desc->offset);
914 if (chan->ep_is_in)
d6ec53e0
MK		 915 remain = (dma_desc->status & HOST_DMA_ISOC_NBYTES_MASK) >>
916 HOST_DMA_ISOC_NBYTES_SHIFT;
dc4c76e7
PZ		 917
918 if ((dma_desc->status & HOST_DMA_STS_MASK) == HOST_DMA_STS_PKTERR) {
919 /*
920 * XactError, or unable to complete all the transactions
921 * in the scheduled micro-frame/frame, both indicated by
922 * HOST_DMA_STS_PKTERR
923 */
924 qtd->urb->error_count++;
925 frame_desc->actual_length = qh->n_bytes[idx] - remain;
926 frame_desc->status = -EPROTO;
927 } else {
928 /* Success */
929 frame_desc->actual_length = qh->n_bytes[idx] - remain;
930 frame_desc->status = 0;
931 }
932
933 if (++qtd->isoc_frame_index == qtd->urb->packet_count) {
934 /*
935 * urb->status is not used for isoc transfers here. The
936 * individual frame_desc status are used instead.
937 */
0d012b98 938 dwc2_host_complete(hsotg, qtd, 0);
dc4c76e7
PZ		 939 dwc2_hcd_qtd_unlink_and_free(hsotg, qtd, qh);
940
941 /*
942 * This check is necessary because urb_dequeue can be called
943 * from urb complete callback (sound driver for example). All
944 * pending URBs are dequeued there, so no need for further
945 * processing.
946 */
947 if (chan->halt_status == DWC2_HC_XFER_URB_DEQUEUE)
948 return -1;
949 rc = DWC2_CMPL_DONE;
950 }
951
952 qh->ntd--;
953
954 /* Stop if IOC requested descriptor reached */
955 if (dma_desc->status & HOST_DMA_IOC)
956 rc = DWC2_CMPL_STOP;
957
958 return rc;
959}
960
961static void dwc2_complete_isoc_xfer_ddma(struct dwc2_hsotg *hsotg,
962 struct dwc2_host_chan *chan,
963 enum dwc2_halt_status halt_status)
964{
965 struct dwc2_hcd_iso_packet_desc *frame_desc;
966 struct dwc2_qtd *qtd, *qtd_tmp;
967 struct dwc2_qh *qh;
968 u16 idx;
969 int rc;
970
971 qh = chan->qh;
972 idx = qh->td_first;
973
974 if (chan->halt_status == DWC2_HC_XFER_URB_DEQUEUE) {
975 list_for_each_entry(qtd, &qh->qtd_list, qtd_list_entry)
976 qtd->in_process = 0;
977 return;
978 }
979
980 if (halt_status == DWC2_HC_XFER_AHB_ERR ||
981 halt_status == DWC2_HC_XFER_BABBLE_ERR) {
982 /*
983 * Channel is halted in these error cases, considered as serious
984 * issues.
985 * Complete all URBs marking all frames as failed, irrespective
986 * whether some of the descriptors (frames) succeeded or not.
987 * Pass error code to completion routine as well, to update
988 * urb->status, some of class drivers might use it to stop
989 * queing transfer requests.
990 */
991 int err = halt_status == DWC2_HC_XFER_AHB_ERR ?
992 -EIO : -EOVERFLOW;
993
994 list_for_each_entry_safe(qtd, qtd_tmp, &qh->qtd_list,
995 qtd_list_entry) {
0d012b98
PZ		 996 if (qtd->urb) {
997 for (idx = 0; idx < qtd->urb->packet_count;
998 idx++) {
999 frame_desc = &qtd->urb->iso_descs[idx];
1000 frame_desc->status = err;
1001 }
1002
1003 dwc2_host_complete(hsotg, qtd, err);
dc4c76e7
PZ		 1004 }
1005
dc4c76e7
PZ		 1006 dwc2_hcd_qtd_unlink_and_free(hsotg, qtd, qh);
1007 }
1008
1009 return;
1010 }
1011
1012 list_for_each_entry_safe(qtd, qtd_tmp, &qh->qtd_list, qtd_list_entry) {
1013 if (!qtd->in_process)
1014 break;
762d3a1a
GH		 1015
1016 /*
1017 * Ensure idx corresponds to descriptor where first urb of this
1018 * qtd was added. In fact, during isoc desc init, dwc2 may skip
1019 * an index if current frame number is already over this index.
1020 */
1021 if (idx != qtd->isoc_td_first) {
1022 dev_vdbg(hsotg->dev,
1023 "try to complete %d instead of %d\n",
1024 idx, qtd->isoc_td_first);
1025 idx = qtd->isoc_td_first;
1026 }
1027
dc4c76e7 1028 do {
762d3a1a
GH		 1029 struct dwc2_qtd *qtd_next;
1030 u16 cur_idx;
1031
dc4c76e7
PZ		 1032 rc = dwc2_cmpl_host_isoc_dma_desc(hsotg, chan, qtd, qh,
1033 idx);
1034 if (rc < 0)
1035 return;
ced9eee1 1036 idx = dwc2_desclist_idx_inc(idx, qh->host_interval,
dc4c76e7 1037 chan->speed);
762d3a1a
GH		 1038 if (!rc)
1039 continue;
1040
dc4c76e7
PZ		 1041 if (rc == DWC2_CMPL_DONE)
1042 break;
762d3a1a
GH		 1043
1044 /* rc == DWC2_CMPL_STOP */
1045
ced9eee1 1046 if (qh->host_interval >= 32)
762d3a1a
GH		 1047 goto stop_scan;
1048
1049 qh->td_first = idx;
1050 cur_idx = dwc2_frame_list_idx(hsotg->frame_number);
1051 qtd_next = list_first_entry(&qh->qtd_list,
1052 struct dwc2_qtd,
1053 qtd_list_entry);
1054 if (dwc2_frame_idx_num_gt(cur_idx,
1055 qtd_next->isoc_td_last))
1056 break;
1057
1058 goto stop_scan;
1059
dc4c76e7
PZ		 1060 } while (idx != qh->td_first);
1061 }
1062
1063stop_scan:
1064 qh->td_first = idx;
1065}
1066
1067static int dwc2_update_non_isoc_urb_state_ddma(struct dwc2_hsotg *hsotg,
9da51974 1068 struct dwc2_host_chan *chan,
dc4c76e7 1069 struct dwc2_qtd *qtd,
ec703251 1070 struct dwc2_dma_desc *dma_desc,
dc4c76e7
PZ		 1071 enum dwc2_halt_status halt_status,
1072 u32 n_bytes, int *xfer_done)
1073{
1074 struct dwc2_hcd_urb *urb = qtd->urb;
1075 u16 remain = 0;
1076
1077 if (chan->ep_is_in)
d6ec53e0
MK		 1078 remain = (dma_desc->status & HOST_DMA_NBYTES_MASK) >>
1079 HOST_DMA_NBYTES_SHIFT;
dc4c76e7
PZ		 1080
1081 dev_vdbg(hsotg->dev, "remain=%d dwc2_urb=%p\n", remain, urb);
1082
1083 if (halt_status == DWC2_HC_XFER_AHB_ERR) {
1084 dev_err(hsotg->dev, "EIO\n");
1085 urb->status = -EIO;
1086 return 1;
1087 }
1088
1089 if ((dma_desc->status & HOST_DMA_STS_MASK) == HOST_DMA_STS_PKTERR) {
1090 switch (halt_status) {
1091 case DWC2_HC_XFER_STALL:
1092 dev_vdbg(hsotg->dev, "Stall\n");
1093 urb->status = -EPIPE;
1094 break;
1095 case DWC2_HC_XFER_BABBLE_ERR:
1096 dev_err(hsotg->dev, "Babble\n");
1097 urb->status = -EOVERFLOW;
1098 break;
1099 case DWC2_HC_XFER_XACT_ERR:
1100 dev_err(hsotg->dev, "XactErr\n");
1101 urb->status = -EPROTO;
1102 break;
1103 default:
1104 dev_err(hsotg->dev,
1105 "%s: Unhandled descriptor error status (%d)\n",
1106 __func__, halt_status);
1107 break;
1108 }
1109 return 1;
1110 }
1111
1112 if (dma_desc->status & HOST_DMA_A) {
1113 dev_vdbg(hsotg->dev,
1114 "Active descriptor encountered on channel %d\n",
1115 chan->hc_num);
1116 return 0;
1117 }
1118
1119 if (chan->ep_type == USB_ENDPOINT_XFER_CONTROL) {
1120 if (qtd->control_phase == DWC2_CONTROL_DATA) {
1121 urb->actual_length += n_bytes - remain;
1122 if (remain || urb->actual_length >= urb->length) {
1123 /*
1124 * For Control Data stage do not set urb->status
1125 * to 0, to prevent URB callback. Set it when
1126 * Status phase is done. See below.
1127 */
1128 *xfer_done = 1;
1129 }
1130 } else if (qtd->control_phase == DWC2_CONTROL_STATUS) {
1131 urb->status = 0;
1132 *xfer_done = 1;
1133 }
1134 /* No handling for SETUP stage */
1135 } else {
1136 /* BULK and INTR */
1137 urb->actual_length += n_bytes - remain;
1138 dev_vdbg(hsotg->dev, "length=%d actual=%d\n", urb->length,
1139 urb->actual_length);
1140 if (remain || urb->actual_length >= urb->length) {
1141 urb->status = 0;
1142 *xfer_done = 1;
1143 }
1144 }
1145
1146 return 0;
1147}
1148
1149static int dwc2_process_non_isoc_desc(struct dwc2_hsotg *hsotg,
1150 struct dwc2_host_chan *chan,
1151 int chnum, struct dwc2_qtd *qtd,
1152 int desc_num,
1153 enum dwc2_halt_status halt_status,
1154 int *xfer_done)
1155{
1156 struct dwc2_qh *qh = chan->qh;
1157 struct dwc2_hcd_urb *urb = qtd->urb;
ec703251 1158 struct dwc2_dma_desc *dma_desc;
dc4c76e7
PZ		 1159 u32 n_bytes;
1160 int failed;
1161
1162 dev_vdbg(hsotg->dev, "%s()\n", __func__);
1163
0d012b98
PZ		 1164 if (!urb)
1165 return -EINVAL;
1166
95105a99
GH		 1167 dma_sync_single_for_cpu(hsotg->dev,
1168 qh->desc_list_dma + (desc_num *
ec703251
VA		 1169 sizeof(struct dwc2_dma_desc)),
1170 sizeof(struct dwc2_dma_desc),
95105a99
GH		 1171 DMA_FROM_DEVICE);
1172
dc4c76e7
PZ		 1173 dma_desc = &qh->desc_list[desc_num];
1174 n_bytes = qh->n_bytes[desc_num];
1175 dev_vdbg(hsotg->dev,
1176 "qtd=%p dwc2_urb=%p desc_num=%d desc=%p n_bytes=%d\n",
1177 qtd, urb, desc_num, dma_desc, n_bytes);
1178 failed = dwc2_update_non_isoc_urb_state_ddma(hsotg, chan, qtd, dma_desc,
1179 halt_status, n_bytes,
1180 xfer_done);
3142a16b 1181 if (failed || (*xfer_done && urb->status != -EINPROGRESS)) {
0d012b98 1182 dwc2_host_complete(hsotg, qtd, urb->status);
dc4c76e7 1183 dwc2_hcd_qtd_unlink_and_free(hsotg, qtd, qh);
3142a16b
VM		 1184 dev_vdbg(hsotg->dev, "failed=%1x xfer_done=%1x\n",
1185 failed, *xfer_done);
dc4c76e7
PZ		 1186 return failed;
1187 }
1188
1189 if (qh->ep_type == USB_ENDPOINT_XFER_CONTROL) {
1190 switch (qtd->control_phase) {
1191 case DWC2_CONTROL_SETUP:
1192 if (urb->length > 0)
1193 qtd->control_phase = DWC2_CONTROL_DATA;
1194 else
1195 qtd->control_phase = DWC2_CONTROL_STATUS;
1196 dev_vdbg(hsotg->dev,
1197 " Control setup transaction done\n");
1198 break;
1199 case DWC2_CONTROL_DATA:
1200 if (*xfer_done) {
1201 qtd->control_phase = DWC2_CONTROL_STATUS;
1202 dev_vdbg(hsotg->dev,
1203 " Control data transfer done\n");
1204 } else if (desc_num + 1 == qtd->n_desc) {
1205 /*
1206 * Last descriptor for Control data stage which
1207 * is not completed yet
1208 */
1209 dwc2_hcd_save_data_toggle(hsotg, chan, chnum,
1210 qtd);
1211 }
1212 break;
1213 default:
1214 break;
1215 }
1216 }
1217
1218 return 0;
1219}
1220
1221static void dwc2_complete_non_isoc_xfer_ddma(struct dwc2_hsotg *hsotg,
1222 struct dwc2_host_chan *chan,
1223 int chnum,
1224 enum dwc2_halt_status halt_status)
1225{
1226 struct list_head *qtd_item, *qtd_tmp;
1227 struct dwc2_qh *qh = chan->qh;
1228 struct dwc2_qtd *qtd = NULL;
1229 int xfer_done;
1230 int desc_num = 0;
1231
1232 if (chan->halt_status == DWC2_HC_XFER_URB_DEQUEUE) {
1233 list_for_each_entry(qtd, &qh->qtd_list, qtd_list_entry)
1234 qtd->in_process = 0;
1235 return;
1236 }
1237
1238 list_for_each_safe(qtd_item, qtd_tmp, &qh->qtd_list) {
1239 int i;
3142a16b 1240 int qtd_desc_count;
dc4c76e7
PZ		 1241
1242 qtd = list_entry(qtd_item, struct dwc2_qtd, qtd_list_entry);
1243 xfer_done = 0;
3142a16b 1244 qtd_desc_count = qtd->n_desc;
dc4c76e7 1245
3142a16b 1246 for (i = 0; i < qtd_desc_count; i++) {
dc4c76e7
PZ		 1247 if (dwc2_process_non_isoc_desc(hsotg, chan, chnum, qtd,
1248 desc_num, halt_status,
1fc65989
JY		 1249 &xfer_done)) {
1250 qtd = NULL;
3142a16b 1251 goto stop_scan;
1fc65989 1252 }
3142a16b 1253
dc4c76e7
PZ		 1254 desc_num++;
1255 }
1256 }
1257
3142a16b 1258stop_scan:
dc4c76e7
PZ		 1259 if (qh->ep_type != USB_ENDPOINT_XFER_CONTROL) {
1260 /*
1261 * Resetting the data toggle for bulk and interrupt endpoints
1262 * in case of stall. See handle_hc_stall_intr().
1263 */
1264 if (halt_status == DWC2_HC_XFER_STALL)
1265 qh->data_toggle = DWC2_HC_PID_DATA0;
62943b7d 1266 else
1fc65989 1267 dwc2_hcd_save_data_toggle(hsotg, chan, chnum, NULL);
dc4c76e7
PZ		 1268 }
1269
1270 if (halt_status == DWC2_HC_XFER_COMPLETE) {
1271 if (chan->hcint & HCINTMSK_NYET) {
1272 /*
1273 * Got a NYET on the last transaction of the transfer.
1274 * It means that the endpoint should be in the PING
1275 * state at the beginning of the next transfer.
1276 */
1277 qh->ping_state = 1;
1278 }
1279 }
1280}
1281
1282/**
1283 * dwc2_hcd_complete_xfer_ddma() - Scans the descriptor list, updates URB's
1284 * status and calls completion routine for the URB if it's done. Called from
1285 * interrupt handlers.
1286 *
1287 * @hsotg: The HCD state structure for the DWC OTG controller
1288 * @chan: Host channel the transfer is completed on
1289 * @chnum: Index of Host channel registers
1290 * @halt_status: Reason the channel is being halted or just XferComplete
1291 * for isochronous transfers
1292 *
1293 * Releases the channel to be used by other transfers.
1294 * In case of Isochronous endpoint the channel is not halted until the end of
1295 * the session, i.e. QTD list is empty.
1296 * If periodic channel released the FrameList is updated accordingly.
1297 * Calls transaction selection routines to activate pending transfers.
1298 */
1299void dwc2_hcd_complete_xfer_ddma(struct dwc2_hsotg *hsotg,
1300 struct dwc2_host_chan *chan, int chnum,
1301 enum dwc2_halt_status halt_status)
1302{
1303 struct dwc2_qh *qh = chan->qh;
1304 int continue_isoc_xfer = 0;
1305 enum dwc2_transaction_type tr_type;
1306
1307 if (chan->ep_type == USB_ENDPOINT_XFER_ISOC) {
1308 dwc2_complete_isoc_xfer_ddma(hsotg, chan, halt_status);
1309
1310 /* Release the channel if halted or session completed */
1311 if (halt_status != DWC2_HC_XFER_COMPLETE ||
1312 list_empty(&qh->qtd_list)) {
c503b381
GH		 1313 struct dwc2_qtd *qtd, *qtd_tmp;
1314
1315 /*
1316 * Kill all remainings QTDs since channel has been
1317 * halted.
1318 */
1319 list_for_each_entry_safe(qtd, qtd_tmp,
1320 &qh->qtd_list,
1321 qtd_list_entry) {
1322 dwc2_host_complete(hsotg, qtd,
1323 -ECONNRESET);
1324 dwc2_hcd_qtd_unlink_and_free(hsotg,
1325 qtd, qh);
1326 }
1327
dc4c76e7
PZ		 1328 /* Halt the channel if session completed */
1329 if (halt_status == DWC2_HC_XFER_COMPLETE)
1330 dwc2_hc_halt(hsotg, chan, halt_status);
1331 dwc2_release_channel_ddma(hsotg, qh);
1332 dwc2_hcd_qh_unlink(hsotg, qh);
1333 } else {
1334 /* Keep in assigned schedule to continue transfer */
94ef7aee
DA		 1335 list_move_tail(&qh->qh_list_entry,
1336 &hsotg->periodic_sched_assigned);
c503b381
GH		 1337 /*
1338 * If channel has been halted during giveback of urb
1339 * then prevent any new scheduling.
1340 */
1341 if (!chan->halt_status)
1342 continue_isoc_xfer = 1;
dc4c76e7
PZ		 1343 }
1344 /*
1345 * Todo: Consider the case when period exceeds FrameList size.
1346 * Frame Rollover interrupt should be used.
1347 */
1348 } else {
1349 /*
1350 * Scan descriptor list to complete the URB(s), then release
1351 * the channel
1352 */
1353 dwc2_complete_non_isoc_xfer_ddma(hsotg, chan, chnum,
1354 halt_status);
1355 dwc2_release_channel_ddma(hsotg, qh);
1356 dwc2_hcd_qh_unlink(hsotg, qh);
1357
1358 if (!list_empty(&qh->qtd_list)) {
1359 /*
1360 * Add back to inactive non-periodic schedule on normal
1361 * completion
1362 */
1363 dwc2_hcd_qh_add(hsotg, qh);
1364 }
1365 }
1366
1367 tr_type = dwc2_hcd_select_transactions(hsotg);
1368 if (tr_type != DWC2_TRANSACTION_NONE || continue_isoc_xfer) {
1369 if (continue_isoc_xfer) {
1370 if (tr_type == DWC2_TRANSACTION_NONE)
1371 tr_type = DWC2_TRANSACTION_PERIODIC;
1372 else if (tr_type == DWC2_TRANSACTION_NON_PERIODIC)
1373 tr_type = DWC2_TRANSACTION_ALL;
1374 }
1375 dwc2_hcd_queue_transactions(hsotg, tr_type);
1376 }
1377}
Linux 6.x block layer and io_uring tree(s)