drivers/usb/dwc2/hcd_intr.c

   1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
   2 /*
   3  * hcd_intr.c - DesignWare HS OTG Controller host-mode interrupt handling
   4  *
   5  * Copyright (C) 2004-2013 Synopsys, Inc.
   6  */
   7
   8 /*
   9  * This file contains the interrupt handlers for Host mode
  10  */
  11 #include <linux/kernel.h>
  12 #include <linux/module.h>
  13 #include <linux/spinlock.h>
  14 #include <linux/interrupt.h>
  15 #include <linux/dma-mapping.h>
  16 #include <linux/io.h>
  17 #include <linux/slab.h>
  18 #include <linux/usb.h>
  19
  20 #include <linux/usb/hcd.h>
  21 #include <linux/usb/ch11.h>
  22
  23 #include "core.h"
  24 #include "hcd.h"
  25
  26 /*
  27  * If we get this many NAKs on a split transaction we'll slow down
  28  * retransmission.  A 1 here means delay after the first NAK.
  29  */
  30 #define DWC2_NAKS_BEFORE_DELAY          3
  31
  32 /* This function is for debug only */
  33 static void dwc2_track_missed_sofs(struct dwc2_hsotg *hsotg)
  34 {
  35         u16 curr_frame_number = hsotg->frame_number;
  36         u16 expected = dwc2_frame_num_inc(hsotg->last_frame_num, 1);
  37
  38         if (expected != curr_frame_number)
  39                 dwc2_sch_vdbg(hsotg, "MISSED SOF %04x != %04x\n",
  40                               expected, curr_frame_number);
  41
  42 #ifdef CONFIG_USB_DWC2_TRACK_MISSED_SOFS
  43         if (hsotg->frame_num_idx < FRAME_NUM_ARRAY_SIZE) {
  44                 if (expected != curr_frame_number) {
  45                         hsotg->frame_num_array[hsotg->frame_num_idx] =
  46                                         curr_frame_number;
  47                         hsotg->last_frame_num_array[hsotg->frame_num_idx] =
  48                                         hsotg->last_frame_num;
  49                         hsotg->frame_num_idx++;
  50                 }
  51         } else if (!hsotg->dumped_frame_num_array) {
  52                 int i;
  53
  54                 dev_info(hsotg->dev, "Frame     Last Frame\n");
  55                 dev_info(hsotg->dev, "-----     ----------\n");
  56                 for (i = 0; i < FRAME_NUM_ARRAY_SIZE; i++) {
  57                         dev_info(hsotg->dev, "0x%04x    0x%04x\n",
  58                                  hsotg->frame_num_array[i],
  59                                  hsotg->last_frame_num_array[i]);
  60                 }
  61                 hsotg->dumped_frame_num_array = 1;
  62         }
  63 #endif
  64         hsotg->last_frame_num = curr_frame_number;
  65 }
  66
  67 static void dwc2_hc_handle_tt_clear(struct dwc2_hsotg *hsotg,
  68                                     struct dwc2_host_chan *chan,
  69                                     struct dwc2_qtd *qtd)
  70 {
  71         struct usb_device *root_hub = dwc2_hsotg_to_hcd(hsotg)->self.root_hub;
  72         struct urb *usb_urb;
  73
  74         if (!chan->qh)
  75                 return;
  76
  77         if (chan->qh->dev_speed == USB_SPEED_HIGH)
  78                 return;
  79
  80         if (!qtd->urb)
  81                 return;
  82
  83         usb_urb = qtd->urb->priv;
  84         if (!usb_urb || !usb_urb->dev || !usb_urb->dev->tt)
  85                 return;
  86
  87         /*
  88          * The root hub doesn't really have a TT, but Linux thinks it
  89          * does because how could you have a "high speed hub" that
  90          * directly talks directly to low speed devices without a TT?
  91          * It's all lies.  Lies, I tell you.
  92          */
  93         if (usb_urb->dev->tt->hub == root_hub)
  94                 return;
  95
  96         if (qtd->urb->status != -EPIPE && qtd->urb->status != -EREMOTEIO) {
  97                 chan->qh->tt_buffer_dirty = 1;
  98                 if (usb_hub_clear_tt_buffer(usb_urb))
  99                         /* Clear failed; let's hope things work anyway */
 100                         chan->qh->tt_buffer_dirty = 0;
 101         }
 102 }
 103
 104 /*
 105  * Handles the start-of-frame interrupt in host mode. Non-periodic
 106  * transactions may be queued to the DWC_otg controller for the current
 107  * (micro)frame. Periodic transactions may be queued to the controller
 108  * for the next (micro)frame.
 109  */
 110 static void dwc2_sof_intr(struct dwc2_hsotg *hsotg)
 111 {
 112         struct list_head *qh_entry;
 113         struct dwc2_qh *qh;
 114         enum dwc2_transaction_type tr_type;
 115
 116         /* Clear interrupt */
 117         dwc2_writel(hsotg, GINTSTS_SOF, GINTSTS);
 118
 119 #ifdef DEBUG_SOF
 120         dev_vdbg(hsotg->dev, "--Start of Frame Interrupt--\n");
 121 #endif
 122
 123         hsotg->frame_number = dwc2_hcd_get_frame_number(hsotg);
 124
 125         dwc2_track_missed_sofs(hsotg);
 126
 127         /* Determine whether any periodic QHs should be executed */
 128         qh_entry = hsotg->periodic_sched_inactive.next;
 129         while (qh_entry != &hsotg->periodic_sched_inactive) {
 130                 qh = list_entry(qh_entry, struct dwc2_qh, qh_list_entry);
 131                 qh_entry = qh_entry->next;
 132                 if (dwc2_frame_num_le(qh->next_active_frame,
 133                                       hsotg->frame_number)) {
 134                         dwc2_sch_vdbg(hsotg, "QH=%p ready fn=%04x, nxt=%04x\n",
 135                                       qh, hsotg->frame_number,
 136                                       qh->next_active_frame);
 137
 138                         /*
 139                          * Move QH to the ready list to be executed next
 140                          * (micro)frame
 141                          */
 142                         list_move_tail(&qh->qh_list_entry,
 143                                        &hsotg->periodic_sched_ready);
 144                 }
 145         }
 146         tr_type = dwc2_hcd_select_transactions(hsotg);
 147         if (tr_type != DWC2_TRANSACTION_NONE)
 148                 dwc2_hcd_queue_transactions(hsotg, tr_type);
 149 }
 150
 151 /*
 152  * Handles the Rx FIFO Level Interrupt, which indicates that there is
 153  * at least one packet in the Rx FIFO. The packets are moved from the FIFO to
 154  * memory if the DWC_otg controller is operating in Slave mode.
 155  */
 156 static void dwc2_rx_fifo_level_intr(struct dwc2_hsotg *hsotg)
 157 {
 158         u32 grxsts, chnum, bcnt, dpid, pktsts;
 159         struct dwc2_host_chan *chan;
 160
 161         if (dbg_perio())
 162                 dev_vdbg(hsotg->dev, "--RxFIFO Level Interrupt--\n");
 163
 164         grxsts = dwc2_readl(hsotg, GRXSTSP);
 165         chnum = (grxsts & GRXSTS_HCHNUM_MASK) >> GRXSTS_HCHNUM_SHIFT;
 166         chan = hsotg->hc_ptr_array[chnum];
 167         if (!chan) {
 168                 dev_err(hsotg->dev, "Unable to get corresponding channel\n");
 169                 return;
 170         }
 171
 172         bcnt = (grxsts & GRXSTS_BYTECNT_MASK) >> GRXSTS_BYTECNT_SHIFT;
 173         dpid = (grxsts & GRXSTS_DPID_MASK) >> GRXSTS_DPID_SHIFT;
 174         pktsts = (grxsts & GRXSTS_PKTSTS_MASK) >> GRXSTS_PKTSTS_SHIFT;
 175
 176         /* Packet Status */
 177         if (dbg_perio()) {
 178                 dev_vdbg(hsotg->dev, "    Ch num = %d\n", chnum);
 179                 dev_vdbg(hsotg->dev, "    Count = %d\n", bcnt);
 180                 dev_vdbg(hsotg->dev, "    DPID = %d, chan.dpid = %d\n", dpid,
 181                          chan->data_pid_start);
 182                 dev_vdbg(hsotg->dev, "    PStatus = %d\n", pktsts);
 183         }
 184
 185         switch (pktsts) {
 186         case GRXSTS_PKTSTS_HCHIN:
 187                 /* Read the data into the host buffer */
 188                 if (bcnt > 0) {
 189                         dwc2_read_packet(hsotg, chan->xfer_buf, bcnt);
 190
 191                         /* Update the HC fields for the next packet received */
 192                         chan->xfer_count += bcnt;
 193                         chan->xfer_buf += bcnt;
 194                 }
 195                 break;
 196         case GRXSTS_PKTSTS_HCHIN_XFER_COMP:
 197         case GRXSTS_PKTSTS_DATATOGGLEERR:
 198         case GRXSTS_PKTSTS_HCHHALTED:
 199                 /* Handled in interrupt, just ignore data */
 200                 break;
 201         default:
 202                 dev_err(hsotg->dev,
 203                         "RxFIFO Level Interrupt: Unknown status %d\n", pktsts);
 204                 break;
 205         }
 206 }
 207
 208 /*
 209  * This interrupt occurs when the non-periodic Tx FIFO is half-empty. More
 210  * data packets may be written to the FIFO for OUT transfers. More requests
 211  * may be written to the non-periodic request queue for IN transfers. This
 212  * interrupt is enabled only in Slave mode.
 213  */
 214 static void dwc2_np_tx_fifo_empty_intr(struct dwc2_hsotg *hsotg)
 215 {
 216         dev_vdbg(hsotg->dev, "--Non-Periodic TxFIFO Empty Interrupt--\n");
 217         dwc2_hcd_queue_transactions(hsotg, DWC2_TRANSACTION_NON_PERIODIC);
 218 }
 219
 220 /*
 221  * This interrupt occurs when the periodic Tx FIFO is half-empty. More data
 222  * packets may be written to the FIFO for OUT transfers. More requests may be
 223  * written to the periodic request queue for IN transfers. This interrupt is
 224  * enabled only in Slave mode.
 225  */
 226 static void dwc2_perio_tx_fifo_empty_intr(struct dwc2_hsotg *hsotg)
 227 {
 228         if (dbg_perio())
 229                 dev_vdbg(hsotg->dev, "--Periodic TxFIFO Empty Interrupt--\n");
 230         dwc2_hcd_queue_transactions(hsotg, DWC2_TRANSACTION_PERIODIC);
 231 }
 232
 233 static void dwc2_hprt0_enable(struct dwc2_hsotg *hsotg, u32 hprt0,
 234                               u32 *hprt0_modify)
 235 {
 236         struct dwc2_core_params *params = &hsotg->params;
 237         int do_reset = 0;
 238         u32 usbcfg;
 239         u32 prtspd;
 240         u32 hcfg;
 241         u32 fslspclksel;
 242         u32 hfir;
 243
 244         dev_vdbg(hsotg->dev, "%s(%p)\n", __func__, hsotg);
 245
 246         /* Every time when port enables calculate HFIR.FrInterval */
 247         hfir = dwc2_readl(hsotg, HFIR);
 248         hfir &= ~HFIR_FRINT_MASK;
 249         hfir |= dwc2_calc_frame_interval(hsotg) << HFIR_FRINT_SHIFT &
 250                 HFIR_FRINT_MASK;
 251         dwc2_writel(hsotg, hfir, HFIR);
 252
 253         /* Check if we need to adjust the PHY clock speed for low power */
 254         if (!params->host_support_fs_ls_low_power) {
 255                 /* Port has been enabled, set the reset change flag */
 256                 hsotg->flags.b.port_reset_change = 1;
 257                 return;
 258         }
 259
 260         usbcfg = dwc2_readl(hsotg, GUSBCFG);
 261         prtspd = (hprt0 & HPRT0_SPD_MASK) >> HPRT0_SPD_SHIFT;
 262
 263         if (prtspd == HPRT0_SPD_LOW_SPEED || prtspd == HPRT0_SPD_FULL_SPEED) {
 264                 /* Low power */
 265                 if (!(usbcfg & GUSBCFG_PHY_LP_CLK_SEL)) {
 266                         /* Set PHY low power clock select for FS/LS devices */
 267                         usbcfg |= GUSBCFG_PHY_LP_CLK_SEL;
 268                         dwc2_writel(hsotg, usbcfg, GUSBCFG);
 269                         do_reset = 1;
 270                 }
 271
 272                 hcfg = dwc2_readl(hsotg, HCFG);
 273                 fslspclksel = (hcfg & HCFG_FSLSPCLKSEL_MASK) >>
 274                               HCFG_FSLSPCLKSEL_SHIFT;
 275
 276                 if (prtspd == HPRT0_SPD_LOW_SPEED &&
 277                     params->host_ls_low_power_phy_clk) {
 278                         /* 6 MHZ */
 279                         dev_vdbg(hsotg->dev,
 280                                  "FS_PHY programming HCFG to 6 MHz\n");
 281                         if (fslspclksel != HCFG_FSLSPCLKSEL_6_MHZ) {
 282                                 fslspclksel = HCFG_FSLSPCLKSEL_6_MHZ;
 283                                 hcfg &= ~HCFG_FSLSPCLKSEL_MASK;
 284                                 hcfg |= fslspclksel << HCFG_FSLSPCLKSEL_SHIFT;
 285                                 dwc2_writel(hsotg, hcfg, HCFG);
 286                                 do_reset = 1;
 287                         }
 288                 } else {
 289                         /* 48 MHZ */
 290                         dev_vdbg(hsotg->dev,
 291                                  "FS_PHY programming HCFG to 48 MHz\n");
 292                         if (fslspclksel != HCFG_FSLSPCLKSEL_48_MHZ) {
 293                                 fslspclksel = HCFG_FSLSPCLKSEL_48_MHZ;
 294                                 hcfg &= ~HCFG_FSLSPCLKSEL_MASK;
 295                                 hcfg |= fslspclksel << HCFG_FSLSPCLKSEL_SHIFT;
 296                                 dwc2_writel(hsotg, hcfg, HCFG);
 297                                 do_reset = 1;
 298                         }
 299                 }
 300         } else {
 301                 /* Not low power */
 302                 if (usbcfg & GUSBCFG_PHY_LP_CLK_SEL) {
 303                         usbcfg &= ~GUSBCFG_PHY_LP_CLK_SEL;
 304                         dwc2_writel(hsotg, usbcfg, GUSBCFG);
 305                         do_reset = 1;
 306                 }
 307         }
 308
 309         if (do_reset) {
 310                 *hprt0_modify |= HPRT0_RST;
 311                 dwc2_writel(hsotg, *hprt0_modify, HPRT0);
 312                 queue_delayed_work(hsotg->wq_otg, &hsotg->reset_work,
 313                                    msecs_to_jiffies(60));
 314         } else {
 315                 /* Port has been enabled, set the reset change flag */
 316                 hsotg->flags.b.port_reset_change = 1;
 317         }
 318 }
 319
 320 /*
 321  * There are multiple conditions that can cause a port interrupt. This function
 322  * determines which interrupt conditions have occurred and handles them
 323  * appropriately.
 324  */
 325 static void dwc2_port_intr(struct dwc2_hsotg *hsotg)
 326 {
 327         u32 hprt0;
 328         u32 hprt0_modify;
 329
 330         dev_vdbg(hsotg->dev, "--Port Interrupt--\n");
 331
 332         hprt0 = dwc2_readl(hsotg, HPRT0);
 333         hprt0_modify = hprt0;
 334
 335         /*
 336          * Clear appropriate bits in HPRT0 to clear the interrupt bit in
 337          * GINTSTS
 338          */
 339         hprt0_modify &= ~(HPRT0_ENA | HPRT0_CONNDET | HPRT0_ENACHG |
 340                           HPRT0_OVRCURRCHG);
 341
 342         /*
 343          * Port Connect Detected
 344          * Set flag and clear if detected
 345          */
 346         if (hprt0 & HPRT0_CONNDET) {
 347                 dwc2_writel(hsotg, hprt0_modify | HPRT0_CONNDET, HPRT0);
 348
 349                 dev_vdbg(hsotg->dev,
 350                          "--Port Interrupt HPRT0=0x%08x Port Connect Detected--\n",
 351                          hprt0);
 352                 dwc2_hcd_connect(hsotg);
 353
 354                 /*
 355                  * The Hub driver asserts a reset when it sees port connect
 356                  * status change flag
 357                  */
 358         }
 359
 360         /*
 361          * Port Enable Changed
 362          * Clear if detected - Set internal flag if disabled
 363          */
 364         if (hprt0 & HPRT0_ENACHG) {
 365                 dwc2_writel(hsotg, hprt0_modify | HPRT0_ENACHG, HPRT0);
 366                 dev_vdbg(hsotg->dev,
 367                          "  --Port Interrupt HPRT0=0x%08x Port Enable Changed (now %d)--\n",
 368                          hprt0, !!(hprt0 & HPRT0_ENA));
 369                 if (hprt0 & HPRT0_ENA) {
 370                         hsotg->new_connection = true;
 371                         dwc2_hprt0_enable(hsotg, hprt0, &hprt0_modify);
 372                 } else {
 373                         hsotg->flags.b.port_enable_change = 1;
 374                         if (hsotg->params.dma_desc_fs_enable) {
 375                                 u32 hcfg;
 376
 377                                 hsotg->params.dma_desc_enable = false;
 378                                 hsotg->new_connection = false;
 379                                 hcfg = dwc2_readl(hsotg, HCFG);
 380                                 hcfg &= ~HCFG_DESCDMA;
 381                                 dwc2_writel(hsotg, hcfg, HCFG);
 382                         }
 383                 }
 384         }
 385
 386         /* Overcurrent Change Interrupt */
 387         if (hprt0 & HPRT0_OVRCURRCHG) {
 388                 dwc2_writel(hsotg, hprt0_modify | HPRT0_OVRCURRCHG,
 389                             HPRT0);
 390                 dev_vdbg(hsotg->dev,
 391                          "  --Port Interrupt HPRT0=0x%08x Port Overcurrent Changed--\n",
 392                          hprt0);
 393                 hsotg->flags.b.port_over_current_change = 1;
 394         }
 395 }
 396
 397 /*
 398  * Gets the actual length of a transfer after the transfer halts. halt_status
 399  * holds the reason for the halt.
 400  *
 401  * For IN transfers where halt_status is DWC2_HC_XFER_COMPLETE, *short_read
 402  * is set to 1 upon return if less than the requested number of bytes were
 403  * transferred. short_read may also be NULL on entry, in which case it remains
 404  * unchanged.
 405  */
 406 static u32 dwc2_get_actual_xfer_length(struct dwc2_hsotg *hsotg,
 407                                        struct dwc2_host_chan *chan, int chnum,
 408                                        struct dwc2_qtd *qtd,
 409                                        enum dwc2_halt_status halt_status,
 410                                        int *short_read)
 411 {
 412         u32 hctsiz, count, length;
 413
 414         hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
 415
 416         if (halt_status == DWC2_HC_XFER_COMPLETE) {
 417                 if (chan->ep_is_in) {
 418                         count = (hctsiz & TSIZ_XFERSIZE_MASK) >>
 419                                 TSIZ_XFERSIZE_SHIFT;
 420                         length = chan->xfer_len - count;
 421                         if (short_read)
 422                                 *short_read = (count != 0);
 423                 } else if (chan->qh->do_split) {
 424                         length = qtd->ssplit_out_xfer_count;
 425                 } else {
 426                         length = chan->xfer_len;
 427                 }
 428         } else {
 429                 /*
 430                  * Must use the hctsiz.pktcnt field to determine how much data
 431                  * has been transferred. This field reflects the number of
 432                  * packets that have been transferred via the USB. This is
 433                  * always an integral number of packets if the transfer was
 434                  * halted before its normal completion. (Can't use the
 435                  * hctsiz.xfersize field because that reflects the number of
 436                  * bytes transferred via the AHB, not the USB).
 437                  */
 438                 count = (hctsiz & TSIZ_PKTCNT_MASK) >> TSIZ_PKTCNT_SHIFT;
 439                 length = (chan->start_pkt_count - count) * chan->max_packet;
 440         }
 441
 442         return length;
 443 }
 444
 445 /**
 446  * dwc2_update_urb_state() - Updates the state of the URB after a Transfer
 447  * Complete interrupt on the host channel. Updates the actual_length field
 448  * of the URB based on the number of bytes transferred via the host channel.
 449  * Sets the URB status if the data transfer is finished.
 450  *
 451  * @hsotg: Programming view of the DWC_otg controller
 452  * @chan: Programming view of host channel
 453  * @chnum: Channel number
 454  * @urb: Processing URB
 455  * @qtd: Queue transfer descriptor
 456  *
 457  * Return: 1 if the data transfer specified by the URB is completely finished,
 458  * 0 otherwise
 459  */
 460 static int dwc2_update_urb_state(struct dwc2_hsotg *hsotg,
 461                                  struct dwc2_host_chan *chan, int chnum,
 462                                  struct dwc2_hcd_urb *urb,
 463                                  struct dwc2_qtd *qtd)
 464 {
 465         u32 hctsiz;
 466         int xfer_done = 0;
 467         int short_read = 0;
 468         int xfer_length = dwc2_get_actual_xfer_length(hsotg, chan, chnum, qtd,
 469                                                       DWC2_HC_XFER_COMPLETE,
 470                                                       &short_read);
 471
 472         if (urb->actual_length + xfer_length > urb->length) {
 473                 dev_dbg(hsotg->dev, "%s(): trimming xfer length\n", __func__);
 474                 xfer_length = urb->length - urb->actual_length;
 475         }
 476
 477         dev_vdbg(hsotg->dev, "urb->actual_length=%d xfer_length=%d\n",
 478                  urb->actual_length, xfer_length);
 479         urb->actual_length += xfer_length;
 480
 481         if (xfer_length && chan->ep_type == USB_ENDPOINT_XFER_BULK &&
 482             (urb->flags & URB_SEND_ZERO_PACKET) &&
 483             urb->actual_length >= urb->length &&
 484             !(urb->length % chan->max_packet)) {
 485                 xfer_done = 0;
 486         } else if (short_read || urb->actual_length >= urb->length) {
 487                 xfer_done = 1;
 488                 urb->status = 0;
 489         }
 490
 491         hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
 492         dev_vdbg(hsotg->dev, "DWC_otg: %s: %s, channel %d\n",
 493                  __func__, (chan->ep_is_in ? "IN" : "OUT"), chnum);
 494         dev_vdbg(hsotg->dev, "  chan->xfer_len %d\n", chan->xfer_len);
 495         dev_vdbg(hsotg->dev, "  hctsiz.xfersize %d\n",
 496                  (hctsiz & TSIZ_XFERSIZE_MASK) >> TSIZ_XFERSIZE_SHIFT);
 497         dev_vdbg(hsotg->dev, "  urb->transfer_buffer_length %d\n", urb->length);
 498         dev_vdbg(hsotg->dev, "  urb->actual_length %d\n", urb->actual_length);
 499         dev_vdbg(hsotg->dev, "  short_read %d, xfer_done %d\n", short_read,
 500                  xfer_done);
 501
 502         return xfer_done;
 503 }
 504
 505 /*
 506  * Save the starting data toggle for the next transfer. The data toggle is
 507  * saved in the QH for non-control transfers and it's saved in the QTD for
 508  * control transfers.
 509  */
 510 void dwc2_hcd_save_data_toggle(struct dwc2_hsotg *hsotg,
 511                                struct dwc2_host_chan *chan, int chnum,
 512                                struct dwc2_qtd *qtd)
 513 {
 514         u32 hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
 515         u32 pid = (hctsiz & TSIZ_SC_MC_PID_MASK) >> TSIZ_SC_MC_PID_SHIFT;
 516
 517         if (chan->ep_type != USB_ENDPOINT_XFER_CONTROL) {
 518                 if (WARN(!chan || !chan->qh,
 519                          "chan->qh must be specified for non-control eps\n"))
 520                         return;
 521
 522                 if (pid == TSIZ_SC_MC_PID_DATA0)
 523                         chan->qh->data_toggle = DWC2_HC_PID_DATA0;
 524                 else
 525                         chan->qh->data_toggle = DWC2_HC_PID_DATA1;
 526         } else {
 527                 if (WARN(!qtd,
 528                          "qtd must be specified for control eps\n"))
 529                         return;
 530
 531                 if (pid == TSIZ_SC_MC_PID_DATA0)
 532                         qtd->data_toggle = DWC2_HC_PID_DATA0;
 533                 else
 534                         qtd->data_toggle = DWC2_HC_PID_DATA1;
 535         }
 536 }
 537
 538 /**
 539  * dwc2_update_isoc_urb_state() - Updates the state of an Isochronous URB when
 540  * the transfer is stopped for any reason. The fields of the current entry in
 541  * the frame descriptor array are set based on the transfer state and the input
 542  * halt_status. Completes the Isochronous URB if all the URB frames have been
 543  * completed.
 544  *
 545  * @hsotg: Programming view of the DWC_otg controller
 546  * @chan: Programming view of host channel
 547  * @chnum: Channel number
 548  * @halt_status: Reason for halting a host channel
 549  * @qtd: Queue transfer descriptor
 550  *
 551  * Return: DWC2_HC_XFER_COMPLETE if there are more frames remaining to be
 552  * transferred in the URB. Otherwise return DWC2_HC_XFER_URB_COMPLETE.
 553  */
 554 static enum dwc2_halt_status dwc2_update_isoc_urb_state(
 555                 struct dwc2_hsotg *hsotg, struct dwc2_host_chan *chan,
 556                 int chnum, struct dwc2_qtd *qtd,
 557                 enum dwc2_halt_status halt_status)
 558 {
 559         struct dwc2_hcd_iso_packet_desc *frame_desc;
 560         struct dwc2_hcd_urb *urb = qtd->urb;
 561
 562         if (!urb)
 563                 return DWC2_HC_XFER_NO_HALT_STATUS;
 564
 565         frame_desc = &urb->iso_descs[qtd->isoc_frame_index];
 566
 567         switch (halt_status) {
 568         case DWC2_HC_XFER_COMPLETE:
 569                 frame_desc->status = 0;
 570                 frame_desc->actual_length = dwc2_get_actual_xfer_length(hsotg,
 571                                         chan, chnum, qtd, halt_status, NULL);
 572                 break;
 573         case DWC2_HC_XFER_FRAME_OVERRUN:
 574                 urb->error_count++;
 575                 if (chan->ep_is_in)
 576                         frame_desc->status = -ENOSR;
 577                 else
 578                         frame_desc->status = -ECOMM;
 579                 frame_desc->actual_length = 0;
 580                 break;
 581         case DWC2_HC_XFER_BABBLE_ERR:
 582                 urb->error_count++;
 583                 frame_desc->status = -EOVERFLOW;
 584                 /* Don't need to update actual_length in this case */
 585                 break;
 586         case DWC2_HC_XFER_XACT_ERR:
 587                 urb->error_count++;
 588                 frame_desc->status = -EPROTO;
 589                 frame_desc->actual_length = dwc2_get_actual_xfer_length(hsotg,
 590                                         chan, chnum, qtd, halt_status, NULL);
 591
 592                 /* Skip whole frame */
 593                 if (chan->qh->do_split &&
 594                     chan->ep_type == USB_ENDPOINT_XFER_ISOC && chan->ep_is_in &&
 595                     hsotg->params.host_dma) {
 596                         qtd->complete_split = 0;
 597                         qtd->isoc_split_offset = 0;
 598                 }
 599
 600                 break;
 601         default:
 602                 dev_err(hsotg->dev, "Unhandled halt_status (%d)\n",
 603                         halt_status);
 604                 break;
 605         }
 606
 607         if (++qtd->isoc_frame_index == urb->packet_count) {
 608                 /*
 609                  * urb->status is not used for isoc transfers. The individual
 610                  * frame_desc statuses are used instead.
 611                  */
 612                 dwc2_host_complete(hsotg, qtd, 0);
 613                 halt_status = DWC2_HC_XFER_URB_COMPLETE;
 614         } else {
 615                 halt_status = DWC2_HC_XFER_COMPLETE;
 616         }
 617
 618         return halt_status;
 619 }
 620
 621 /*
 622  * Frees the first QTD in the QH's list if free_qtd is 1. For non-periodic
 623  * QHs, removes the QH from the active non-periodic schedule. If any QTDs are
 624  * still linked to the QH, the QH is added to the end of the inactive
 625  * non-periodic schedule. For periodic QHs, removes the QH from the periodic
 626  * schedule if no more QTDs are linked to the QH.
 627  */
 628 static void dwc2_deactivate_qh(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,
 629                                int free_qtd)
 630 {
 631         int continue_split = 0;
 632         struct dwc2_qtd *qtd;
 633
 634         if (dbg_qh(qh))
 635                 dev_vdbg(hsotg->dev, "  %s(%p,%p,%d)\n", __func__,
 636                          hsotg, qh, free_qtd);
 637
 638         if (list_empty(&qh->qtd_list)) {
 639                 dev_dbg(hsotg->dev, "## QTD list empty ##\n");
 640                 goto no_qtd;
 641         }
 642
 643         qtd = list_first_entry(&qh->qtd_list, struct dwc2_qtd, qtd_list_entry);
 644
 645         if (qtd->complete_split)
 646                 continue_split = 1;
 647         else if (qtd->isoc_split_pos == DWC2_HCSPLT_XACTPOS_MID ||
 648                  qtd->isoc_split_pos == DWC2_HCSPLT_XACTPOS_END)
 649                 continue_split = 1;
 650
 651         if (free_qtd) {
 652                 dwc2_hcd_qtd_unlink_and_free(hsotg, qtd, qh);
 653                 continue_split = 0;
 654         }
 655
 656 no_qtd:
 657         qh->channel = NULL;
 658         dwc2_hcd_qh_deactivate(hsotg, qh, continue_split);
 659 }
 660
 661 /**
 662  * dwc2_release_channel() - Releases a host channel for use by other transfers
 663  *
 664  * @hsotg:       The HCD state structure
 665  * @chan:        The host channel to release
 666  * @qtd:         The QTD associated with the host channel. This QTD may be
 667  *               freed if the transfer is complete or an error has occurred.
 668  * @halt_status: Reason the channel is being released. This status
 669  *               determines the actions taken by this function.
 670  *
 671  * Also attempts to select and queue more transactions since at least one host
 672  * channel is available.
 673  */
 674 static void dwc2_release_channel(struct dwc2_hsotg *hsotg,
 675                                  struct dwc2_host_chan *chan,
 676                                  struct dwc2_qtd *qtd,
 677                                  enum dwc2_halt_status halt_status)
 678 {
 679         enum dwc2_transaction_type tr_type;
 680         u32 haintmsk;
 681         int free_qtd = 0;
 682
 683         if (dbg_hc(chan))
 684                 dev_vdbg(hsotg->dev, "  %s: channel %d, halt_status %d\n",
 685                          __func__, chan->hc_num, halt_status);
 686
 687         switch (halt_status) {
 688         case DWC2_HC_XFER_URB_COMPLETE:
 689                 free_qtd = 1;
 690                 break;
 691         case DWC2_HC_XFER_AHB_ERR:
 692         case DWC2_HC_XFER_STALL:
 693         case DWC2_HC_XFER_BABBLE_ERR:
 694                 free_qtd = 1;
 695                 break;
 696         case DWC2_HC_XFER_XACT_ERR:
 697                 if (qtd && qtd->error_count >= 3) {
 698                         dev_vdbg(hsotg->dev,
 699                                  "  Complete URB with transaction error\n");
 700                         free_qtd = 1;
 701                         dwc2_host_complete(hsotg, qtd, -EPROTO);
 702                 }
 703                 break;
 704         case DWC2_HC_XFER_URB_DEQUEUE:
 705                 /*
 706                  * The QTD has already been removed and the QH has been
 707                  * deactivated. Don't want to do anything except release the
 708                  * host channel and try to queue more transfers.
 709                  */
 710                 goto cleanup;
 711         case DWC2_HC_XFER_PERIODIC_INCOMPLETE:
 712                 dev_vdbg(hsotg->dev, "  Complete URB with I/O error\n");
 713                 free_qtd = 1;
 714                 dwc2_host_complete(hsotg, qtd, -EIO);
 715                 break;
 716         case DWC2_HC_XFER_NO_HALT_STATUS:
 717         default:
 718                 break;
 719         }
 720
 721         dwc2_deactivate_qh(hsotg, chan->qh, free_qtd);
 722
 723 cleanup:
 724         /*
 725          * Release the host channel for use by other transfers. The cleanup
 726          * function clears the channel interrupt enables and conditions, so
 727          * there's no need to clear the Channel Halted interrupt separately.
 728          */
 729         if (!list_empty(&chan->hc_list_entry))
 730                 list_del(&chan->hc_list_entry);
 731         dwc2_hc_cleanup(hsotg, chan);
 732         list_add_tail(&chan->hc_list_entry, &hsotg->free_hc_list);
 733
 734         if (hsotg->params.uframe_sched) {
 735                 hsotg->available_host_channels++;
 736         } else {
 737                 switch (chan->ep_type) {
 738                 case USB_ENDPOINT_XFER_CONTROL:
 739                 case USB_ENDPOINT_XFER_BULK:
 740                         hsotg->non_periodic_channels--;
 741                         break;
 742                 default:
 743                         /*
 744                          * Don't release reservations for periodic channels
 745                          * here. That's done when a periodic transfer is
 746                          * descheduled (i.e. when the QH is removed from the
 747                          * periodic schedule).
 748                          */
 749                         break;
 750                 }
 751         }
 752
 753         haintmsk = dwc2_readl(hsotg, HAINTMSK);
 754         haintmsk &= ~(1 << chan->hc_num);
 755         dwc2_writel(hsotg, haintmsk, HAINTMSK);
 756
 757         /* Try to queue more transfers now that there's a free channel */
 758         tr_type = dwc2_hcd_select_transactions(hsotg);
 759         if (tr_type != DWC2_TRANSACTION_NONE)
 760                 dwc2_hcd_queue_transactions(hsotg, tr_type);
 761 }
 762
 763 /*
 764  * Halts a host channel. If the channel cannot be halted immediately because
 765  * the request queue is full, this function ensures that the FIFO empty
 766  * interrupt for the appropriate queue is enabled so that the halt request can
 767  * be queued when there is space in the request queue.
 768  *
 769  * This function may also be called in DMA mode. In that case, the channel is
 770  * simply released since the core always halts the channel automatically in
 771  * DMA mode.
 772  */
 773 static void dwc2_halt_channel(struct dwc2_hsotg *hsotg,
 774                               struct dwc2_host_chan *chan, struct dwc2_qtd *qtd,
 775                               enum dwc2_halt_status halt_status)
 776 {
 777         if (dbg_hc(chan))
 778                 dev_vdbg(hsotg->dev, "%s()\n", __func__);
 779
 780         if (hsotg->params.host_dma) {
 781                 if (dbg_hc(chan))
 782                         dev_vdbg(hsotg->dev, "DMA enabled\n");
 783                 dwc2_release_channel(hsotg, chan, qtd, halt_status);
 784                 return;
 785         }
 786
 787         /* Slave mode processing */
 788         dwc2_hc_halt(hsotg, chan, halt_status);
 789
 790         if (chan->halt_on_queue) {
 791                 u32 gintmsk;
 792
 793                 dev_vdbg(hsotg->dev, "Halt on queue\n");
 794                 if (chan->ep_type == USB_ENDPOINT_XFER_CONTROL ||
 795                     chan->ep_type == USB_ENDPOINT_XFER_BULK) {
 796                         dev_vdbg(hsotg->dev, "control/bulk\n");
 797                         /*
 798                          * Make sure the Non-periodic Tx FIFO empty interrupt
 799                          * is enabled so that the non-periodic schedule will
 800                          * be processed
 801                          */
 802                         gintmsk = dwc2_readl(hsotg, GINTMSK);
 803                         gintmsk |= GINTSTS_NPTXFEMP;
 804                         dwc2_writel(hsotg, gintmsk, GINTMSK);
 805                 } else {
 806                         dev_vdbg(hsotg->dev, "isoc/intr\n");
 807                         /*
 808                          * Move the QH from the periodic queued schedule to
 809                          * the periodic assigned schedule. This allows the
 810                          * halt to be queued when the periodic schedule is
 811                          * processed.
 812                          */
 813                         list_move_tail(&chan->qh->qh_list_entry,
 814                                        &hsotg->periodic_sched_assigned);
 815
 816                         /*
 817                          * Make sure the Periodic Tx FIFO Empty interrupt is
 818                          * enabled so that the periodic schedule will be
 819                          * processed
 820                          */
 821                         gintmsk = dwc2_readl(hsotg, GINTMSK);
 822                         gintmsk |= GINTSTS_PTXFEMP;
 823                         dwc2_writel(hsotg, gintmsk, GINTMSK);
 824                 }
 825         }
 826 }
 827
 828 /*
 829  * Performs common cleanup for non-periodic transfers after a Transfer
 830  * Complete interrupt. This function should be called after any endpoint type
 831  * specific handling is finished to release the host channel.
 832  */
 833 static void dwc2_complete_non_periodic_xfer(struct dwc2_hsotg *hsotg,
 834                                             struct dwc2_host_chan *chan,
 835                                             int chnum, struct dwc2_qtd *qtd,
 836                                             enum dwc2_halt_status halt_status)
 837 {
 838         dev_vdbg(hsotg->dev, "%s()\n", __func__);
 839
 840         qtd->error_count = 0;
 841
 842         if (chan->hcint & HCINTMSK_NYET) {
 843                 /*
 844                  * Got a NYET on the last transaction of the transfer. This
 845                  * means that the endpoint should be in the PING state at the
 846                  * beginning of the next transfer.
 847                  */
 848                 dev_vdbg(hsotg->dev, "got NYET\n");
 849                 chan->qh->ping_state = 1;
 850         }
 851
 852         /*
 853          * Always halt and release the host channel to make it available for
 854          * more transfers. There may still be more phases for a control
 855          * transfer or more data packets for a bulk transfer at this point,
 856          * but the host channel is still halted. A channel will be reassigned
 857          * to the transfer when the non-periodic schedule is processed after
 858          * the channel is released. This allows transactions to be queued
 859          * properly via dwc2_hcd_queue_transactions, which also enables the
 860          * Tx FIFO Empty interrupt if necessary.
 861          */
 862         if (chan->ep_is_in) {
 863                 /*
 864                  * IN transfers in Slave mode require an explicit disable to
 865                  * halt the channel. (In DMA mode, this call simply releases
 866                  * the channel.)
 867                  */
 868                 dwc2_halt_channel(hsotg, chan, qtd, halt_status);
 869         } else {
 870                 /*
 871                  * The channel is automatically disabled by the core for OUT
 872                  * transfers in Slave mode
 873                  */
 874                 dwc2_release_channel(hsotg, chan, qtd, halt_status);
 875         }
 876 }
 877
 878 /*
 879  * Performs common cleanup for periodic transfers after a Transfer Complete
 880  * interrupt. This function should be called after any endpoint type specific
 881  * handling is finished to release the host channel.
 882  */
 883 static void dwc2_complete_periodic_xfer(struct dwc2_hsotg *hsotg,
 884                                         struct dwc2_host_chan *chan, int chnum,
 885                                         struct dwc2_qtd *qtd,
 886                                         enum dwc2_halt_status halt_status)
 887 {
 888         u32 hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
 889
 890         qtd->error_count = 0;
 891
 892         if (!chan->ep_is_in || (hctsiz & TSIZ_PKTCNT_MASK) == 0)
 893                 /* Core halts channel in these cases */
 894                 dwc2_release_channel(hsotg, chan, qtd, halt_status);
 895         else
 896                 /* Flush any outstanding requests from the Tx queue */
 897                 dwc2_halt_channel(hsotg, chan, qtd, halt_status);
 898 }
 899
 900 static int dwc2_xfercomp_isoc_split_in(struct dwc2_hsotg *hsotg,
 901                                        struct dwc2_host_chan *chan, int chnum,
 902                                        struct dwc2_qtd *qtd)
 903 {
 904         struct dwc2_hcd_iso_packet_desc *frame_desc;
 905         u32 len;
 906         u32 hctsiz;
 907         u32 pid;
 908
 909         if (!qtd->urb)
 910                 return 0;
 911
 912         frame_desc = &qtd->urb->iso_descs[qtd->isoc_frame_index];
 913         len = dwc2_get_actual_xfer_length(hsotg, chan, chnum, qtd,
 914                                           DWC2_HC_XFER_COMPLETE, NULL);
 915         if (!len && !qtd->isoc_split_offset) {
 916                 qtd->complete_split = 0;
 917                 return 0;
 918         }
 919
 920         frame_desc->actual_length += len;
 921
 922         if (chan->align_buf) {
 923                 dev_vdbg(hsotg->dev, "non-aligned buffer\n");
 924                 dma_unmap_single(hsotg->dev, chan->qh->dw_align_buf_dma,
 925                                  DWC2_KMEM_UNALIGNED_BUF_SIZE, DMA_FROM_DEVICE);
 926                 memcpy(qtd->urb->buf + (chan->xfer_dma - qtd->urb->dma),
 927                        chan->qh->dw_align_buf, len);
 928         }
 929
 930         qtd->isoc_split_offset += len;
 931
 932         hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
 933         pid = (hctsiz & TSIZ_SC_MC_PID_MASK) >> TSIZ_SC_MC_PID_SHIFT;
 934
 935         if (frame_desc->actual_length >= frame_desc->length || pid == 0) {
 936                 frame_desc->status = 0;
 937                 qtd->isoc_frame_index++;
 938                 qtd->complete_split = 0;
 939                 qtd->isoc_split_offset = 0;
 940         }
 941
 942         if (qtd->isoc_frame_index == qtd->urb->packet_count) {
 943                 dwc2_host_complete(hsotg, qtd, 0);
 944                 dwc2_release_channel(hsotg, chan, qtd,
 945                                      DWC2_HC_XFER_URB_COMPLETE);
 946         } else {
 947                 dwc2_release_channel(hsotg, chan, qtd,
 948                                      DWC2_HC_XFER_NO_HALT_STATUS);
 949         }
 950
 951         return 1;       /* Indicates that channel released */
 952 }
 953
 954 /*
 955  * Handles a host channel Transfer Complete interrupt. This handler may be
 956  * called in either DMA mode or Slave mode.
 957  */
 958 static void dwc2_hc_xfercomp_intr(struct dwc2_hsotg *hsotg,
 959                                   struct dwc2_host_chan *chan, int chnum,
 960                                   struct dwc2_qtd *qtd)
 961 {
 962         struct dwc2_hcd_urb *urb = qtd->urb;
 963         enum dwc2_halt_status halt_status = DWC2_HC_XFER_COMPLETE;
 964         int pipe_type;
 965         int urb_xfer_done;
 966
 967         if (dbg_hc(chan))
 968                 dev_vdbg(hsotg->dev,
 969                          "--Host Channel %d Interrupt: Transfer Complete--\n",
 970                          chnum);
 971
 972         if (!urb)
 973                 goto handle_xfercomp_done;
 974
 975         pipe_type = dwc2_hcd_get_pipe_type(&urb->pipe_info);
 976
 977         if (hsotg->params.dma_desc_enable) {
 978                 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum, halt_status);
 979                 if (pipe_type == USB_ENDPOINT_XFER_ISOC)
 980                         /* Do not disable the interrupt, just clear it */
 981                         return;
 982                 goto handle_xfercomp_done;
 983         }
 984
 985         /* Handle xfer complete on CSPLIT */
 986         if (chan->qh->do_split) {
 987                 if (chan->ep_type == USB_ENDPOINT_XFER_ISOC && chan->ep_is_in &&
 988                     hsotg->params.host_dma) {
 989                         if (qtd->complete_split &&
 990                             dwc2_xfercomp_isoc_split_in(hsotg, chan, chnum,
 991                                                         qtd))
 992                                 goto handle_xfercomp_done;
 993                 } else {
 994                         qtd->complete_split = 0;
 995                 }
 996         }
 997
 998         /* Update the QTD and URB states */
 999         switch (pipe_type) {
1000         case USB_ENDPOINT_XFER_CONTROL:
1001                 switch (qtd->control_phase) {
1002                 case DWC2_CONTROL_SETUP:
1003                         if (urb->length > 0)
1004                                 qtd->control_phase = DWC2_CONTROL_DATA;
1005                         else
1006                                 qtd->control_phase = DWC2_CONTROL_STATUS;
1007                         dev_vdbg(hsotg->dev,
1008                                  "  Control setup transaction done\n");
1009                         halt_status = DWC2_HC_XFER_COMPLETE;
1010                         break;
1011                 case DWC2_CONTROL_DATA:
1012                         urb_xfer_done = dwc2_update_urb_state(hsotg, chan,
1013                                                               chnum, urb, qtd);
1014                         if (urb_xfer_done) {
1015                                 qtd->control_phase = DWC2_CONTROL_STATUS;
1016                                 dev_vdbg(hsotg->dev,
1017                                          "  Control data transfer done\n");
1018                         } else {
1019                                 dwc2_hcd_save_data_toggle(hsotg, chan, chnum,
1020                                                           qtd);
1021                         }
1022                         halt_status = DWC2_HC_XFER_COMPLETE;
1023                         break;
1024                 case DWC2_CONTROL_STATUS:
1025                         dev_vdbg(hsotg->dev, "  Control transfer complete\n");
1026                         if (urb->status == -EINPROGRESS)
1027                                 urb->status = 0;
1028                         dwc2_host_complete(hsotg, qtd, urb->status);
1029                         halt_status = DWC2_HC_XFER_URB_COMPLETE;
1030                         break;
1031                 }
1032
1033                 dwc2_complete_non_periodic_xfer(hsotg, chan, chnum, qtd,
1034                                                 halt_status);
1035                 break;
1036         case USB_ENDPOINT_XFER_BULK:
1037                 dev_vdbg(hsotg->dev, "  Bulk transfer complete\n");
1038                 urb_xfer_done = dwc2_update_urb_state(hsotg, chan, chnum, urb,
1039                                                       qtd);
1040                 if (urb_xfer_done) {
1041                         dwc2_host_complete(hsotg, qtd, urb->status);
1042                         halt_status = DWC2_HC_XFER_URB_COMPLETE;
1043                 } else {
1044                         halt_status = DWC2_HC_XFER_COMPLETE;
1045                 }
1046
1047                 dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1048                 dwc2_complete_non_periodic_xfer(hsotg, chan, chnum, qtd,
1049                                                 halt_status);
1050                 break;
1051         case USB_ENDPOINT_XFER_INT:
1052                 dev_vdbg(hsotg->dev, "  Interrupt transfer complete\n");
1053                 urb_xfer_done = dwc2_update_urb_state(hsotg, chan, chnum, urb,
1054                                                       qtd);
1055
1056                 /*
1057                  * Interrupt URB is done on the first transfer complete
1058                  * interrupt
1059                  */
1060                 if (urb_xfer_done) {
1061                         dwc2_host_complete(hsotg, qtd, urb->status);
1062                         halt_status = DWC2_HC_XFER_URB_COMPLETE;
1063                 } else {
1064                         halt_status = DWC2_HC_XFER_COMPLETE;
1065                 }
1066
1067                 dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1068                 dwc2_complete_periodic_xfer(hsotg, chan, chnum, qtd,
1069                                             halt_status);
1070                 break;
1071         case USB_ENDPOINT_XFER_ISOC:
1072                 if (dbg_perio())
1073                         dev_vdbg(hsotg->dev, "  Isochronous transfer complete\n");
1074                 if (qtd->isoc_split_pos == DWC2_HCSPLT_XACTPOS_ALL)
1075                         halt_status = dwc2_update_isoc_urb_state(hsotg, chan,
1076                                                         chnum, qtd,
1077                                                         DWC2_HC_XFER_COMPLETE);
1078                 dwc2_complete_periodic_xfer(hsotg, chan, chnum, qtd,
1079                                             halt_status);
1080                 break;
1081         }
1082
1083 handle_xfercomp_done:
1084         disable_hc_int(hsotg, chnum, HCINTMSK_XFERCOMPL);
1085 }
1086
1087 /*
1088  * Handles a host channel STALL interrupt. This handler may be called in
1089  * either DMA mode or Slave mode.
1090  */
1091 static void dwc2_hc_stall_intr(struct dwc2_hsotg *hsotg,
1092                                struct dwc2_host_chan *chan, int chnum,
1093                                struct dwc2_qtd *qtd)
1094 {
1095         struct dwc2_hcd_urb *urb = qtd->urb;
1096         int pipe_type;
1097
1098         dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: STALL Received--\n",
1099                 chnum);
1100
1101         if (hsotg->params.dma_desc_enable) {
1102                 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1103                                             DWC2_HC_XFER_STALL);
1104                 goto handle_stall_done;
1105         }
1106
1107         if (!urb)
1108                 goto handle_stall_halt;
1109
1110         pipe_type = dwc2_hcd_get_pipe_type(&urb->pipe_info);
1111
1112         if (pipe_type == USB_ENDPOINT_XFER_CONTROL)
1113                 dwc2_host_complete(hsotg, qtd, -EPIPE);
1114
1115         if (pipe_type == USB_ENDPOINT_XFER_BULK ||
1116             pipe_type == USB_ENDPOINT_XFER_INT) {
1117                 dwc2_host_complete(hsotg, qtd, -EPIPE);
1118                 /*
1119                  * USB protocol requires resetting the data toggle for bulk
1120                  * and interrupt endpoints when a CLEAR_FEATURE(ENDPOINT_HALT)
1121                  * setup command is issued to the endpoint. Anticipate the
1122                  * CLEAR_FEATURE command since a STALL has occurred and reset
1123                  * the data toggle now.
1124                  */
1125                 chan->qh->data_toggle = 0;
1126         }
1127
1128 handle_stall_halt:
1129         dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_STALL);
1130
1131 handle_stall_done:
1132         disable_hc_int(hsotg, chnum, HCINTMSK_STALL);
1133 }
1134
1135 /*
1136  * Updates the state of the URB when a transfer has been stopped due to an
1137  * abnormal condition before the transfer completes. Modifies the
1138  * actual_length field of the URB to reflect the number of bytes that have
1139  * actually been transferred via the host channel.
1140  */
1141 static void dwc2_update_urb_state_abn(struct dwc2_hsotg *hsotg,
1142                                       struct dwc2_host_chan *chan, int chnum,
1143                                       struct dwc2_hcd_urb *urb,
1144                                       struct dwc2_qtd *qtd,
1145                                       enum dwc2_halt_status halt_status)
1146 {
1147         u32 xfer_length = dwc2_get_actual_xfer_length(hsotg, chan, chnum,
1148                                                       qtd, halt_status, NULL);
1149         u32 hctsiz;
1150
1151         if (urb->actual_length + xfer_length > urb->length) {
1152                 dev_warn(hsotg->dev, "%s(): trimming xfer length\n", __func__);
1153                 xfer_length = urb->length - urb->actual_length;
1154         }
1155
1156         urb->actual_length += xfer_length;
1157
1158         hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
1159         dev_vdbg(hsotg->dev, "DWC_otg: %s: %s, channel %d\n",
1160                  __func__, (chan->ep_is_in ? "IN" : "OUT"), chnum);
1161         dev_vdbg(hsotg->dev, "  chan->start_pkt_count %d\n",
1162                  chan->start_pkt_count);
1163         dev_vdbg(hsotg->dev, "  hctsiz.pktcnt %d\n",
1164                  (hctsiz & TSIZ_PKTCNT_MASK) >> TSIZ_PKTCNT_SHIFT);
1165         dev_vdbg(hsotg->dev, "  chan->max_packet %d\n", chan->max_packet);
1166         dev_vdbg(hsotg->dev, "  bytes_transferred %d\n",
1167                  xfer_length);
1168         dev_vdbg(hsotg->dev, "  urb->actual_length %d\n",
1169                  urb->actual_length);
1170         dev_vdbg(hsotg->dev, "  urb->transfer_buffer_length %d\n",
1171                  urb->length);
1172 }
1173
1174 /*
1175  * Handles a host channel NAK interrupt. This handler may be called in either
1176  * DMA mode or Slave mode.
1177  */
1178 static void dwc2_hc_nak_intr(struct dwc2_hsotg *hsotg,
1179                              struct dwc2_host_chan *chan, int chnum,
1180                              struct dwc2_qtd *qtd)
1181 {
1182         if (!qtd) {
1183                 dev_dbg(hsotg->dev, "%s: qtd is NULL\n", __func__);
1184                 return;
1185         }
1186
1187         if (!qtd->urb) {
1188                 dev_dbg(hsotg->dev, "%s: qtd->urb is NULL\n", __func__);
1189                 return;
1190         }
1191
1192         if (dbg_hc(chan))
1193                 dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: NAK Received--\n",
1194                          chnum);
1195
1196         /*
1197          * Handle NAK for IN/OUT SSPLIT/CSPLIT transfers, bulk, control, and
1198          * interrupt. Re-start the SSPLIT transfer.
1199          *
1200          * Normally for non-periodic transfers we'll retry right away, but to
1201          * avoid interrupt storms we'll wait before retrying if we've got
1202          * several NAKs. If we didn't do this we'd retry directly from the
1203          * interrupt handler and could end up quickly getting another
1204          * interrupt (another NAK), which we'd retry. Note that we do not
1205          * delay retries for IN parts of control requests, as those are expected
1206          * to complete fairly quickly, and if we delay them we risk confusing
1207          * the device and cause it issue STALL.
1208          *
1209          * Note that in DMA mode software only gets involved to re-send NAKed
1210          * transfers for split transactions, so we only need to apply this
1211          * delaying logic when handling splits. In non-DMA mode presumably we
1212          * might want a similar delay if someone can demonstrate this problem
1213          * affects that code path too.
1214          */
1215         if (chan->do_split) {
1216                 if (chan->complete_split)
1217                         qtd->error_count = 0;
1218                 qtd->complete_split = 0;
1219                 qtd->num_naks++;
1220                 qtd->qh->want_wait = qtd->num_naks >= DWC2_NAKS_BEFORE_DELAY &&
1221                                 !(chan->ep_type == USB_ENDPOINT_XFER_CONTROL &&
1222                                   chan->ep_is_in);
1223                 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NAK);
1224                 goto handle_nak_done;
1225         }
1226
1227         switch (dwc2_hcd_get_pipe_type(&qtd->urb->pipe_info)) {
1228         case USB_ENDPOINT_XFER_CONTROL:
1229         case USB_ENDPOINT_XFER_BULK:
1230                 if (hsotg->params.host_dma && chan->ep_is_in) {
1231                         /*
1232                          * NAK interrupts are enabled on bulk/control IN
1233                          * transfers in DMA mode for the sole purpose of
1234                          * resetting the error count after a transaction error
1235                          * occurs. The core will continue transferring data.
1236                          */
1237                         qtd->error_count = 0;
1238                         break;
1239                 }
1240
1241                 /*
1242                  * NAK interrupts normally occur during OUT transfers in DMA
1243                  * or Slave mode. For IN transfers, more requests will be
1244                  * queued as request queue space is available.
1245                  */
1246                 qtd->error_count = 0;
1247
1248                 if (!chan->qh->ping_state) {
1249                         dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb,
1250                                                   qtd, DWC2_HC_XFER_NAK);
1251                         dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1252
1253                         if (chan->speed == USB_SPEED_HIGH)
1254                                 chan->qh->ping_state = 1;
1255                 }
1256
1257                 /*
1258                  * Halt the channel so the transfer can be re-started from
1259                  * the appropriate point or the PING protocol will
1260                  * start/continue
1261                  */
1262                 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NAK);
1263                 break;
1264         case USB_ENDPOINT_XFER_INT:
1265                 qtd->error_count = 0;
1266                 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NAK);
1267                 break;
1268         case USB_ENDPOINT_XFER_ISOC:
1269                 /* Should never get called for isochronous transfers */
1270                 dev_err(hsotg->dev, "NACK interrupt for ISOC transfer\n");
1271                 break;
1272         }
1273
1274 handle_nak_done:
1275         disable_hc_int(hsotg, chnum, HCINTMSK_NAK);
1276 }
1277
1278 /*
1279  * Handles a host channel ACK interrupt. This interrupt is enabled when
1280  * performing the PING protocol in Slave mode, when errors occur during
1281  * either Slave mode or DMA mode, and during Start Split transactions.
1282  */
1283 static void dwc2_hc_ack_intr(struct dwc2_hsotg *hsotg,
1284                              struct dwc2_host_chan *chan, int chnum,
1285                              struct dwc2_qtd *qtd)
1286 {
1287         struct dwc2_hcd_iso_packet_desc *frame_desc;
1288
1289         if (dbg_hc(chan))
1290                 dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: ACK Received--\n",
1291                          chnum);
1292
1293         if (chan->do_split) {
1294                 /* Handle ACK on SSPLIT. ACK should not occur in CSPLIT. */
1295                 if (!chan->ep_is_in &&
1296                     chan->data_pid_start != DWC2_HC_PID_SETUP)
1297                         qtd->ssplit_out_xfer_count = chan->xfer_len;
1298
1299                 if (chan->ep_type != USB_ENDPOINT_XFER_ISOC || chan->ep_is_in) {
1300                         qtd->complete_split = 1;
1301                         dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_ACK);
1302                 } else {
1303                         /* ISOC OUT */
1304                         switch (chan->xact_pos) {
1305                         case DWC2_HCSPLT_XACTPOS_ALL:
1306                                 break;
1307                         case DWC2_HCSPLT_XACTPOS_END:
1308                                 qtd->isoc_split_pos = DWC2_HCSPLT_XACTPOS_ALL;
1309                                 qtd->isoc_split_offset = 0;
1310                                 break;
1311                         case DWC2_HCSPLT_XACTPOS_BEGIN:
1312                         case DWC2_HCSPLT_XACTPOS_MID:
1313                                 /*
1314                                  * For BEGIN or MID, calculate the length for
1315                                  * the next microframe to determine the correct
1316                                  * SSPLIT token, either MID or END
1317                                  */
1318                                 frame_desc = &qtd->urb->iso_descs[
1319                                                 qtd->isoc_frame_index];
1320                                 qtd->isoc_split_offset += 188;
1321
1322                                 if (frame_desc->length - qtd->isoc_split_offset
1323                                                         <= 188)
1324                                         qtd->isoc_split_pos =
1325                                                         DWC2_HCSPLT_XACTPOS_END;
1326                                 else
1327                                         qtd->isoc_split_pos =
1328                                                         DWC2_HCSPLT_XACTPOS_MID;
1329                                 break;
1330                         }
1331                 }
1332         } else {
1333                 qtd->error_count = 0;
1334
1335                 if (chan->qh->ping_state) {
1336                         chan->qh->ping_state = 0;
1337                         /*
1338                          * Halt the channel so the transfer can be re-started
1339                          * from the appropriate point. This only happens in
1340                          * Slave mode. In DMA mode, the ping_state is cleared
1341                          * when the transfer is started because the core
1342                          * automatically executes the PING, then the transfer.
1343                          */
1344                         dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_ACK);
1345                 }
1346         }
1347
1348         /*
1349          * If the ACK occurred when _not_ in the PING state, let the channel
1350          * continue transferring data after clearing the error count
1351          */
1352         disable_hc_int(hsotg, chnum, HCINTMSK_ACK);
1353 }
1354
1355 /*
1356  * Handles a host channel NYET interrupt. This interrupt should only occur on
1357  * Bulk and Control OUT endpoints and for complete split transactions. If a
1358  * NYET occurs at the same time as a Transfer Complete interrupt, it is
1359  * handled in the xfercomp interrupt handler, not here. This handler may be
1360  * called in either DMA mode or Slave mode.
1361  */
1362 static void dwc2_hc_nyet_intr(struct dwc2_hsotg *hsotg,
1363                               struct dwc2_host_chan *chan, int chnum,
1364                               struct dwc2_qtd *qtd)
1365 {
1366         if (dbg_hc(chan))
1367                 dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: NYET Received--\n",
1368                          chnum);
1369
1370         /*
1371          * NYET on CSPLIT
1372          * re-do the CSPLIT immediately on non-periodic
1373          */
1374         if (chan->do_split && chan->complete_split) {
1375                 if (chan->ep_is_in && chan->ep_type == USB_ENDPOINT_XFER_ISOC &&
1376                     hsotg->params.host_dma) {
1377                         qtd->complete_split = 0;
1378                         qtd->isoc_split_offset = 0;
1379                         qtd->isoc_frame_index++;
1380                         if (qtd->urb &&
1381                             qtd->isoc_frame_index == qtd->urb->packet_count) {
1382                                 dwc2_host_complete(hsotg, qtd, 0);
1383                                 dwc2_release_channel(hsotg, chan, qtd,
1384                                                      DWC2_HC_XFER_URB_COMPLETE);
1385                         } else {
1386                                 dwc2_release_channel(hsotg, chan, qtd,
1387                                                 DWC2_HC_XFER_NO_HALT_STATUS);
1388                         }
1389                         goto handle_nyet_done;
1390                 }
1391
1392                 if (chan->ep_type == USB_ENDPOINT_XFER_INT ||
1393                     chan->ep_type == USB_ENDPOINT_XFER_ISOC) {
1394                         struct dwc2_qh *qh = chan->qh;
1395                         bool past_end;
1396
1397                         if (!hsotg->params.uframe_sched) {
1398                                 int frnum = dwc2_hcd_get_frame_number(hsotg);
1399
1400                                 /* Don't have num_hs_transfers; simple logic */
1401                                 past_end = dwc2_full_frame_num(frnum) !=
1402                                      dwc2_full_frame_num(qh->next_active_frame);
1403                         } else {
1404                                 int end_frnum;
1405
1406                                 /*
1407                                  * Figure out the end frame based on
1408                                  * schedule.
1409                                  *
1410                                  * We don't want to go on trying again
1411                                  * and again forever. Let's stop when
1412                                  * we've done all the transfers that
1413                                  * were scheduled.
1414                                  *
1415                                  * We're going to be comparing
1416                                  * start_active_frame and
1417                                  * next_active_frame, both of which
1418                                  * are 1 before the time the packet
1419                                  * goes on the wire, so that cancels
1420                                  * out. Basically if had 1 transfer
1421                                  * and we saw 1 NYET then we're done.
1422                                  * We're getting a NYET here so if
1423                                  * next >= (start + num_transfers)
1424                                  * we're done. The complexity is that
1425                                  * for all but ISOC_OUT we skip one
1426                                  * slot.
1427                                  */
1428                                 end_frnum = dwc2_frame_num_inc(
1429                                         qh->start_active_frame,
1430                                         qh->num_hs_transfers);
1431
1432                                 if (qh->ep_type != USB_ENDPOINT_XFER_ISOC ||
1433                                     qh->ep_is_in)
1434                                         end_frnum =
1435                                                dwc2_frame_num_inc(end_frnum, 1);
1436
1437                                 past_end = dwc2_frame_num_le(
1438                                         end_frnum, qh->next_active_frame);
1439                         }
1440
1441                         if (past_end) {
1442                                 /* Treat this as a transaction error. */
1443 #if 0
1444                                 /*
1445                                  * Todo: Fix system performance so this can
1446                                  * be treated as an error. Right now complete
1447                                  * splits cannot be scheduled precisely enough
1448                                  * due to other system activity, so this error
1449                                  * occurs regularly in Slave mode.
1450                                  */
1451                                 qtd->error_count++;
1452 #endif
1453                                 qtd->complete_split = 0;
1454                                 dwc2_halt_channel(hsotg, chan, qtd,
1455                                                   DWC2_HC_XFER_XACT_ERR);
1456                                 /* Todo: add support for isoc release */
1457                                 goto handle_nyet_done;
1458                         }
1459                 }
1460
1461                 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NYET);
1462                 goto handle_nyet_done;
1463         }
1464
1465         chan->qh->ping_state = 1;
1466         qtd->error_count = 0;
1467
1468         dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb, qtd,
1469                                   DWC2_HC_XFER_NYET);
1470         dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1471
1472         /*
1473          * Halt the channel and re-start the transfer so the PING protocol
1474          * will start
1475          */
1476         dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NYET);
1477
1478 handle_nyet_done:
1479         disable_hc_int(hsotg, chnum, HCINTMSK_NYET);
1480 }
1481
1482 /*
1483  * Handles a host channel babble interrupt. This handler may be called in
1484  * either DMA mode or Slave mode.
1485  */
1486 static void dwc2_hc_babble_intr(struct dwc2_hsotg *hsotg,
1487                                 struct dwc2_host_chan *chan, int chnum,
1488                                 struct dwc2_qtd *qtd)
1489 {
1490         dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: Babble Error--\n",
1491                 chnum);
1492
1493         dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1494
1495         if (hsotg->params.dma_desc_enable) {
1496                 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1497                                             DWC2_HC_XFER_BABBLE_ERR);
1498                 goto disable_int;
1499         }
1500
1501         if (chan->ep_type != USB_ENDPOINT_XFER_ISOC) {
1502                 dwc2_host_complete(hsotg, qtd, -EOVERFLOW);
1503                 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_BABBLE_ERR);
1504         } else {
1505                 enum dwc2_halt_status halt_status;
1506
1507                 halt_status = dwc2_update_isoc_urb_state(hsotg, chan, chnum,
1508                                                 qtd, DWC2_HC_XFER_BABBLE_ERR);
1509                 dwc2_halt_channel(hsotg, chan, qtd, halt_status);
1510         }
1511
1512 disable_int:
1513         disable_hc_int(hsotg, chnum, HCINTMSK_BBLERR);
1514 }
1515
1516 /*
1517  * Handles a host channel AHB error interrupt. This handler is only called in
1518  * DMA mode.
1519  */
1520 static void dwc2_hc_ahberr_intr(struct dwc2_hsotg *hsotg,
1521                                 struct dwc2_host_chan *chan, int chnum,
1522                                 struct dwc2_qtd *qtd)
1523 {
1524         struct dwc2_hcd_urb *urb = qtd->urb;
1525         char *pipetype, *speed;
1526         u32 hcchar;
1527         u32 hcsplt;
1528         u32 hctsiz;
1529         u32 hc_dma;
1530
1531         dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: AHB Error--\n",
1532                 chnum);
1533
1534         if (!urb)
1535                 goto handle_ahberr_halt;
1536
1537         dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1538
1539         hcchar = dwc2_readl(hsotg, HCCHAR(chnum));
1540         hcsplt = dwc2_readl(hsotg, HCSPLT(chnum));
1541         hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
1542         hc_dma = dwc2_readl(hsotg, HCDMA(chnum));
1543
1544         dev_err(hsotg->dev, "AHB ERROR, Channel %d\n", chnum);
1545         dev_err(hsotg->dev, "  hcchar 0x%08x, hcsplt 0x%08x\n", hcchar, hcsplt);
1546         dev_err(hsotg->dev, "  hctsiz 0x%08x, hc_dma 0x%08x\n", hctsiz, hc_dma);
1547         dev_err(hsotg->dev, "  Device address: %d\n",
1548                 dwc2_hcd_get_dev_addr(&urb->pipe_info));
1549         dev_err(hsotg->dev, "  Endpoint: %d, %s\n",
1550                 dwc2_hcd_get_ep_num(&urb->pipe_info),
1551                 dwc2_hcd_is_pipe_in(&urb->pipe_info) ? "IN" : "OUT");
1552
1553         switch (dwc2_hcd_get_pipe_type(&urb->pipe_info)) {
1554         case USB_ENDPOINT_XFER_CONTROL:
1555                 pipetype = "CONTROL";
1556                 break;
1557         case USB_ENDPOINT_XFER_BULK:
1558                 pipetype = "BULK";
1559                 break;
1560         case USB_ENDPOINT_XFER_INT:
1561                 pipetype = "INTERRUPT";
1562                 break;
1563         case USB_ENDPOINT_XFER_ISOC:
1564                 pipetype = "ISOCHRONOUS";
1565                 break;
1566         default:
1567                 pipetype = "UNKNOWN";
1568                 break;
1569         }
1570
1571         dev_err(hsotg->dev, "  Endpoint type: %s\n", pipetype);
1572
1573         switch (chan->speed) {
1574         case USB_SPEED_HIGH:
1575                 speed = "HIGH";
1576                 break;
1577         case USB_SPEED_FULL:
1578                 speed = "FULL";
1579                 break;
1580         case USB_SPEED_LOW:
1581                 speed = "LOW";
1582                 break;
1583         default:
1584                 speed = "UNKNOWN";
1585                 break;
1586         }
1587
1588         dev_err(hsotg->dev, "  Speed: %s\n", speed);
1589
1590         dev_err(hsotg->dev, "  Max packet size: %d (mult %d)\n",
1591                 dwc2_hcd_get_maxp(&urb->pipe_info),
1592                 dwc2_hcd_get_maxp_mult(&urb->pipe_info));
1593         dev_err(hsotg->dev, "  Data buffer length: %d\n", urb->length);
1594         dev_err(hsotg->dev, "  Transfer buffer: %p, Transfer DMA: %08lx\n",
1595                 urb->buf, (unsigned long)urb->dma);
1596         dev_err(hsotg->dev, "  Setup buffer: %p, Setup DMA: %08lx\n",
1597                 urb->setup_packet, (unsigned long)urb->setup_dma);
1598         dev_err(hsotg->dev, "  Interval: %d\n", urb->interval);
1599
1600         /* Core halts the channel for Descriptor DMA mode */
1601         if (hsotg->params.dma_desc_enable) {
1602                 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1603                                             DWC2_HC_XFER_AHB_ERR);
1604                 goto handle_ahberr_done;
1605         }
1606
1607         dwc2_host_complete(hsotg, qtd, -EIO);
1608
1609 handle_ahberr_halt:
1610         /*
1611          * Force a channel halt. Don't call dwc2_halt_channel because that won't
1612          * write to the HCCHARn register in DMA mode to force the halt.
1613          */
1614         dwc2_hc_halt(hsotg, chan, DWC2_HC_XFER_AHB_ERR);
1615
1616 handle_ahberr_done:
1617         disable_hc_int(hsotg, chnum, HCINTMSK_AHBERR);
1618 }
1619
1620 /*
1621  * Handles a host channel transaction error interrupt. This handler may be
1622  * called in either DMA mode or Slave mode.
1623  */
1624 static void dwc2_hc_xacterr_intr(struct dwc2_hsotg *hsotg,
1625                                  struct dwc2_host_chan *chan, int chnum,
1626                                  struct dwc2_qtd *qtd)
1627 {
1628         dev_dbg(hsotg->dev,
1629                 "--Host Channel %d Interrupt: Transaction Error--\n", chnum);
1630
1631         dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1632
1633         if (hsotg->params.dma_desc_enable) {
1634                 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1635                                             DWC2_HC_XFER_XACT_ERR);
1636                 goto handle_xacterr_done;
1637         }
1638
1639         switch (dwc2_hcd_get_pipe_type(&qtd->urb->pipe_info)) {
1640         case USB_ENDPOINT_XFER_CONTROL:
1641         case USB_ENDPOINT_XFER_BULK:
1642                 qtd->error_count++;
1643                 if (!chan->qh->ping_state) {
1644                         dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb,
1645                                                   qtd, DWC2_HC_XFER_XACT_ERR);
1646                         dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1647                         if (!chan->ep_is_in && chan->speed == USB_SPEED_HIGH)
1648                                 chan->qh->ping_state = 1;
1649                 }
1650
1651                 /*
1652                  * Halt the channel so the transfer can be re-started from
1653                  * the appropriate point or the PING protocol will start
1654                  */
1655                 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_XACT_ERR);
1656                 break;
1657         case USB_ENDPOINT_XFER_INT:
1658                 qtd->error_count++;
1659                 if (chan->do_split && chan->complete_split)
1660                         qtd->complete_split = 0;
1661                 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_XACT_ERR);
1662                 break;
1663         case USB_ENDPOINT_XFER_ISOC:
1664                 {
1665                         enum dwc2_halt_status halt_status;
1666
1667                         halt_status = dwc2_update_isoc_urb_state(hsotg, chan,
1668                                          chnum, qtd, DWC2_HC_XFER_XACT_ERR);
1669                         dwc2_halt_channel(hsotg, chan, qtd, halt_status);
1670                 }
1671                 break;
1672         }
1673
1674 handle_xacterr_done:
1675         disable_hc_int(hsotg, chnum, HCINTMSK_XACTERR);
1676 }
1677
1678 /*
1679  * Handles a host channel frame overrun interrupt. This handler may be called
1680  * in either DMA mode or Slave mode.
1681  */
1682 static void dwc2_hc_frmovrun_intr(struct dwc2_hsotg *hsotg,
1683                                   struct dwc2_host_chan *chan, int chnum,
1684                                   struct dwc2_qtd *qtd)
1685 {
1686         enum dwc2_halt_status halt_status;
1687
1688         if (dbg_hc(chan))
1689                 dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: Frame Overrun--\n",
1690                         chnum);
1691
1692         dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1693
1694         switch (dwc2_hcd_get_pipe_type(&qtd->urb->pipe_info)) {
1695         case USB_ENDPOINT_XFER_CONTROL:
1696         case USB_ENDPOINT_XFER_BULK:
1697                 break;
1698         case USB_ENDPOINT_XFER_INT:
1699                 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_FRAME_OVERRUN);
1700                 break;
1701         case USB_ENDPOINT_XFER_ISOC:
1702                 halt_status = dwc2_update_isoc_urb_state(hsotg, chan, chnum,
1703                                         qtd, DWC2_HC_XFER_FRAME_OVERRUN);
1704                 dwc2_halt_channel(hsotg, chan, qtd, halt_status);
1705                 break;
1706         }
1707
1708         disable_hc_int(hsotg, chnum, HCINTMSK_FRMOVRUN);
1709 }
1710
1711 /*
1712  * Handles a host channel data toggle error interrupt. This handler may be
1713  * called in either DMA mode or Slave mode.
1714  */
1715 static void dwc2_hc_datatglerr_intr(struct dwc2_hsotg *hsotg,
1716                                     struct dwc2_host_chan *chan, int chnum,
1717                                     struct dwc2_qtd *qtd)
1718 {
1719         dev_dbg(hsotg->dev,
1720                 "--Host Channel %d Interrupt: Data Toggle Error--\n", chnum);
1721
1722         if (chan->ep_is_in)
1723                 qtd->error_count = 0;
1724         else
1725                 dev_err(hsotg->dev,
1726                         "Data Toggle Error on OUT transfer, channel %d\n",
1727                         chnum);
1728
1729         dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1730         disable_hc_int(hsotg, chnum, HCINTMSK_DATATGLERR);
1731 }
1732
1733 /*
1734  * For debug only. It checks that a valid halt status is set and that
1735  * HCCHARn.chdis is clear. If there's a problem, corrective action is
1736  * taken and a warning is issued.
1737  *
1738  * Return: true if halt status is ok, false otherwise
1739  */
1740 static bool dwc2_halt_status_ok(struct dwc2_hsotg *hsotg,
1741                                 struct dwc2_host_chan *chan, int chnum,
1742                                 struct dwc2_qtd *qtd)
1743 {
1744 #ifdef DEBUG
1745         u32 hcchar;
1746         u32 hctsiz;
1747         u32 hcintmsk;
1748         u32 hcsplt;
1749
1750         if (chan->halt_status == DWC2_HC_XFER_NO_HALT_STATUS) {
1751                 /*
1752                  * This code is here only as a check. This condition should
1753                  * never happen. Ignore the halt if it does occur.
1754                  */
1755                 hcchar = dwc2_readl(hsotg, HCCHAR(chnum));
1756                 hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
1757                 hcintmsk = dwc2_readl(hsotg, HCINTMSK(chnum));
1758                 hcsplt = dwc2_readl(hsotg, HCSPLT(chnum));
1759                 dev_dbg(hsotg->dev,
1760                         "%s: chan->halt_status DWC2_HC_XFER_NO_HALT_STATUS,\n",
1761                          __func__);
1762                 dev_dbg(hsotg->dev,
1763                         "channel %d, hcchar 0x%08x, hctsiz 0x%08x,\n",
1764                         chnum, hcchar, hctsiz);
1765                 dev_dbg(hsotg->dev,
1766                         "hcint 0x%08x, hcintmsk 0x%08x, hcsplt 0x%08x,\n",
1767                         chan->hcint, hcintmsk, hcsplt);
1768                 if (qtd)
1769                         dev_dbg(hsotg->dev, "qtd->complete_split %d\n",
1770                                 qtd->complete_split);
1771                 dev_warn(hsotg->dev,
1772                          "%s: no halt status, channel %d, ignoring interrupt\n",
1773                          __func__, chnum);
1774                 return false;
1775         }
1776
1777         /*
1778          * This code is here only as a check. hcchar.chdis should never be set
1779          * when the halt interrupt occurs. Halt the channel again if it does
1780          * occur.
1781          */
1782         hcchar = dwc2_readl(hsotg, HCCHAR(chnum));
1783         if (hcchar & HCCHAR_CHDIS) {
1784                 dev_warn(hsotg->dev,
1785                          "%s: hcchar.chdis set unexpectedly, hcchar 0x%08x, trying to halt again\n",
1786                          __func__, hcchar);
1787                 chan->halt_pending = 0;
1788                 dwc2_halt_channel(hsotg, chan, qtd, chan->halt_status);
1789                 return false;
1790         }
1791 #endif
1792
1793         return true;
1794 }
1795
1796 /*
1797  * Handles a host Channel Halted interrupt in DMA mode. This handler
1798  * determines the reason the channel halted and proceeds accordingly.
1799  */
1800 static void dwc2_hc_chhltd_intr_dma(struct dwc2_hsotg *hsotg,
1801                                     struct dwc2_host_chan *chan, int chnum,
1802                                     struct dwc2_qtd *qtd)
1803 {
1804         u32 hcintmsk;
1805         int out_nak_enh = 0;
1806
1807         if (dbg_hc(chan))
1808                 dev_vdbg(hsotg->dev,
1809                          "--Host Channel %d Interrupt: DMA Channel Halted--\n",
1810                          chnum);
1811
1812         /*
1813          * For core with OUT NAK enhancement, the flow for high-speed
1814          * CONTROL/BULK OUT is handled a little differently
1815          */
1816         if (hsotg->hw_params.snpsid >= DWC2_CORE_REV_2_71a) {
1817                 if (chan->speed == USB_SPEED_HIGH && !chan->ep_is_in &&
1818                     (chan->ep_type == USB_ENDPOINT_XFER_CONTROL ||
1819                      chan->ep_type == USB_ENDPOINT_XFER_BULK)) {
1820                         out_nak_enh = 1;
1821                 }
1822         }
1823
1824         if (chan->halt_status == DWC2_HC_XFER_URB_DEQUEUE ||
1825             (chan->halt_status == DWC2_HC_XFER_AHB_ERR &&
1826              !hsotg->params.dma_desc_enable)) {
1827                 if (hsotg->params.dma_desc_enable)
1828                         dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1829                                                     chan->halt_status);
1830                 else
1831                         /*
1832                          * Just release the channel. A dequeue can happen on a
1833                          * transfer timeout. In the case of an AHB Error, the
1834                          * channel was forced to halt because there's no way to
1835                          * gracefully recover.
1836                          */
1837                         dwc2_release_channel(hsotg, chan, qtd,
1838                                              chan->halt_status);
1839                 return;
1840         }
1841
1842         hcintmsk = dwc2_readl(hsotg, HCINTMSK(chnum));
1843
1844         if (chan->hcint & HCINTMSK_XFERCOMPL) {
1845                 /*
1846                  * Todo: This is here because of a possible hardware bug. Spec
1847                  * says that on SPLIT-ISOC OUT transfers in DMA mode that a HALT
1848                  * interrupt w/ACK bit set should occur, but I only see the
1849                  * XFERCOMP bit, even with it masked out. This is a workaround
1850                  * for that behavior. Should fix this when hardware is fixed.
1851                  */
1852                 if (chan->ep_type == USB_ENDPOINT_XFER_ISOC && !chan->ep_is_in)
1853                         dwc2_hc_ack_intr(hsotg, chan, chnum, qtd);
1854                 dwc2_hc_xfercomp_intr(hsotg, chan, chnum, qtd);
1855         } else if (chan->hcint & HCINTMSK_STALL) {
1856                 dwc2_hc_stall_intr(hsotg, chan, chnum, qtd);
1857         } else if ((chan->hcint & HCINTMSK_XACTERR) &&
1858                    !hsotg->params.dma_desc_enable) {
1859                 if (out_nak_enh) {
1860                         if (chan->hcint &
1861                             (HCINTMSK_NYET | HCINTMSK_NAK | HCINTMSK_ACK)) {
1862                                 dev_vdbg(hsotg->dev,
1863                                          "XactErr with NYET/NAK/ACK\n");
1864                                 qtd->error_count = 0;
1865                         } else {
1866                                 dev_vdbg(hsotg->dev,
1867                                          "XactErr without NYET/NAK/ACK\n");
1868                         }
1869                 }
1870
1871                 /*
1872                  * Must handle xacterr before nak or ack. Could get a xacterr
1873                  * at the same time as either of these on a BULK/CONTROL OUT
1874                  * that started with a PING. The xacterr takes precedence.
1875                  */
1876                 dwc2_hc_xacterr_intr(hsotg, chan, chnum, qtd);
1877         } else if ((chan->hcint & HCINTMSK_XCS_XACT) &&
1878                    hsotg->params.dma_desc_enable) {
1879                 dwc2_hc_xacterr_intr(hsotg, chan, chnum, qtd);
1880         } else if ((chan->hcint & HCINTMSK_AHBERR) &&
1881                    hsotg->params.dma_desc_enable) {
1882                 dwc2_hc_ahberr_intr(hsotg, chan, chnum, qtd);
1883         } else if (chan->hcint & HCINTMSK_BBLERR) {
1884                 dwc2_hc_babble_intr(hsotg, chan, chnum, qtd);
1885         } else if (chan->hcint & HCINTMSK_FRMOVRUN) {
1886                 dwc2_hc_frmovrun_intr(hsotg, chan, chnum, qtd);
1887         } else if (!out_nak_enh) {
1888                 if (chan->hcint & HCINTMSK_NYET) {
1889                         /*
1890                          * Must handle nyet before nak or ack. Could get a nyet
1891                          * at the same time as either of those on a BULK/CONTROL
1892                          * OUT that started with a PING. The nyet takes
1893                          * precedence.
1894                          */
1895                         dwc2_hc_nyet_intr(hsotg, chan, chnum, qtd);
1896                 } else if ((chan->hcint & HCINTMSK_NAK) &&
1897                            !(hcintmsk & HCINTMSK_NAK)) {
1898                         /*
1899                          * If nak is not masked, it's because a non-split IN
1900                          * transfer is in an error state. In that case, the nak
1901                          * is handled by the nak interrupt handler, not here.
1902                          * Handle nak here for BULK/CONTROL OUT transfers, which
1903                          * halt on a NAK to allow rewinding the buffer pointer.
1904                          */
1905                         dwc2_hc_nak_intr(hsotg, chan, chnum, qtd);
1906                 } else if ((chan->hcint & HCINTMSK_ACK) &&
1907                            !(hcintmsk & HCINTMSK_ACK)) {
1908                         /*
1909                          * If ack is not masked, it's because a non-split IN
1910                          * transfer is in an error state. In that case, the ack
1911                          * is handled by the ack interrupt handler, not here.
1912                          * Handle ack here for split transfers. Start splits
1913                          * halt on ACK.
1914                          */
1915                         dwc2_hc_ack_intr(hsotg, chan, chnum, qtd);
1916                 } else {
1917                         if (chan->ep_type == USB_ENDPOINT_XFER_INT ||
1918                             chan->ep_type == USB_ENDPOINT_XFER_ISOC) {
1919                                 /*
1920                                  * A periodic transfer halted with no other
1921                                  * channel interrupts set. Assume it was halted
1922                                  * by the core because it could not be completed
1923                                  * in its scheduled (micro)frame.
1924                                  */
1925                                 dev_dbg(hsotg->dev,
1926                                         "%s: Halt channel %d (assume incomplete periodic transfer)\n",
1927                                         __func__, chnum);
1928                                 dwc2_halt_channel(hsotg, chan, qtd,
1929                                         DWC2_HC_XFER_PERIODIC_INCOMPLETE);
1930                         } else {
1931                                 dev_err(hsotg->dev,
1932                                         "%s: Channel %d - ChHltd set, but reason is unknown\n",
1933                                         __func__, chnum);
1934                                 dev_err(hsotg->dev,
1935                                         "hcint 0x%08x, intsts 0x%08x\n",
1936                                         chan->hcint,
1937                                         dwc2_readl(hsotg, GINTSTS));
1938                                 goto error;
1939                         }
1940                 }
1941         } else {
1942                 dev_info(hsotg->dev,
1943                          "NYET/NAK/ACK/other in non-error case, 0x%08x\n",
1944                          chan->hcint);
1945 error:
1946                 /* Failthrough: use 3-strikes rule */
1947                 qtd->error_count++;
1948                 dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb,
1949                                           qtd, DWC2_HC_XFER_XACT_ERR);
1950                 /*
1951                  * We can get here after a completed transaction
1952                  * (urb->actual_length >= urb->length) which was not reported
1953                  * as completed. If that is the case, and we do not abort
1954                  * the transfer, a transfer of size 0 will be enqueued
1955                  * subsequently. If urb->actual_length is not DMA-aligned,
1956                  * the buffer will then point to an unaligned address, and
1957                  * the resulting behavior is undefined. Bail out in that
1958                  * situation.
1959                  */
1960                 if (qtd->urb->actual_length >= qtd->urb->length)
1961                         qtd->error_count = 3;
1962                 dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1963                 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_XACT_ERR);
1964         }
1965 }
1966
1967 /*
1968  * Handles a host channel Channel Halted interrupt
1969  *
1970  * In slave mode, this handler is called only when the driver specifically
1971  * requests a halt. This occurs during handling other host channel interrupts
1972  * (e.g. nak, xacterr, stall, nyet, etc.).
1973  *
1974  * In DMA mode, this is the interrupt that occurs when the core has finished
1975  * processing a transfer on a channel. Other host channel interrupts (except
1976  * ahberr) are disabled in DMA mode.
1977  */
1978 static void dwc2_hc_chhltd_intr(struct dwc2_hsotg *hsotg,
1979                                 struct dwc2_host_chan *chan, int chnum,
1980                                 struct dwc2_qtd *qtd)
1981 {
1982         if (dbg_hc(chan))
1983                 dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: Channel Halted--\n",
1984                          chnum);
1985
1986         if (hsotg->params.host_dma) {
1987                 dwc2_hc_chhltd_intr_dma(hsotg, chan, chnum, qtd);
1988         } else {
1989                 if (!dwc2_halt_status_ok(hsotg, chan, chnum, qtd))
1990                         return;
1991                 dwc2_release_channel(hsotg, chan, qtd, chan->halt_status);
1992         }
1993 }
1994
1995 /*
1996  * Check if the given qtd is still the top of the list (and thus valid).
1997  *
1998  * If dwc2_hcd_qtd_unlink_and_free() has been called since we grabbed
1999  * the qtd from the top of the list, this will return false (otherwise true).
2000  */
2001 static bool dwc2_check_qtd_still_ok(struct dwc2_qtd *qtd, struct dwc2_qh *qh)
2002 {
2003         struct dwc2_qtd *cur_head;
2004
2005         if (!qh)
2006                 return false;
2007
2008         cur_head = list_first_entry(&qh->qtd_list, struct dwc2_qtd,
2009                                     qtd_list_entry);
2010         return (cur_head == qtd);
2011 }
2012
2013 /* Handles interrupt for a specific Host Channel */
2014 static void dwc2_hc_n_intr(struct dwc2_hsotg *hsotg, int chnum)
2015 {
2016         struct dwc2_qtd *qtd;
2017         struct dwc2_host_chan *chan;
2018         u32 hcint, hcintmsk;
2019
2020         chan = hsotg->hc_ptr_array[chnum];
2021
2022         hcint = dwc2_readl(hsotg, HCINT(chnum));
2023         hcintmsk = dwc2_readl(hsotg, HCINTMSK(chnum));
2024         if (!chan) {
2025                 dev_err(hsotg->dev, "## hc_ptr_array for channel is NULL ##\n");
2026                 dwc2_writel(hsotg, hcint, HCINT(chnum));
2027                 return;
2028         }
2029
2030         if (dbg_hc(chan)) {
2031                 dev_vdbg(hsotg->dev, "--Host Channel Interrupt--, Channel %d\n",
2032                          chnum);
2033                 dev_vdbg(hsotg->dev,
2034                          "  hcint 0x%08x, hcintmsk 0x%08x, hcint&hcintmsk 0x%08x\n",
2035                          hcint, hcintmsk, hcint & hcintmsk);
2036         }
2037
2038         dwc2_writel(hsotg, hcint, HCINT(chnum));
2039
2040         /*
2041          * If we got an interrupt after someone called
2042          * dwc2_hcd_endpoint_disable() we don't want to crash below
2043          */
2044         if (!chan->qh) {
2045                 dev_warn(hsotg->dev, "Interrupt on disabled channel\n");
2046                 return;
2047         }
2048
2049         chan->hcint = hcint;
2050         hcint &= hcintmsk;
2051
2052         /*
2053          * If the channel was halted due to a dequeue, the qtd list might
2054          * be empty or at least the first entry will not be the active qtd.
2055          * In this case, take a shortcut and just release the channel.
2056          */
2057         if (chan->halt_status == DWC2_HC_XFER_URB_DEQUEUE) {
2058                 /*
2059                  * If the channel was halted, this should be the only
2060                  * interrupt unmasked
2061                  */
2062                 WARN_ON(hcint != HCINTMSK_CHHLTD);
2063                 if (hsotg->params.dma_desc_enable)
2064                         dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
2065                                                     chan->halt_status);
2066                 else
2067                         dwc2_release_channel(hsotg, chan, NULL,
2068                                              chan->halt_status);
2069                 return;
2070         }
2071
2072         if (list_empty(&chan->qh->qtd_list)) {
2073                 /*
2074                  * TODO: Will this ever happen with the
2075                  * DWC2_HC_XFER_URB_DEQUEUE handling above?
2076                  */
2077                 dev_dbg(hsotg->dev, "## no QTD queued for channel %d ##\n",
2078                         chnum);
2079                 dev_dbg(hsotg->dev,
2080                         "  hcint 0x%08x, hcintmsk 0x%08x, hcint&hcintmsk 0x%08x\n",
2081                         chan->hcint, hcintmsk, hcint);
2082                 chan->halt_status = DWC2_HC_XFER_NO_HALT_STATUS;
2083                 disable_hc_int(hsotg, chnum, HCINTMSK_CHHLTD);
2084                 chan->hcint = 0;
2085                 return;
2086         }
2087
2088         qtd = list_first_entry(&chan->qh->qtd_list, struct dwc2_qtd,
2089                                qtd_list_entry);
2090
2091         if (!hsotg->params.host_dma) {
2092                 if ((hcint & HCINTMSK_CHHLTD) && hcint != HCINTMSK_CHHLTD)
2093                         hcint &= ~HCINTMSK_CHHLTD;
2094         }
2095
2096         if (hcint & HCINTMSK_XFERCOMPL) {
2097                 dwc2_hc_xfercomp_intr(hsotg, chan, chnum, qtd);
2098                 /*
2099                  * If NYET occurred at same time as Xfer Complete, the NYET is
2100                  * handled by the Xfer Complete interrupt handler. Don't want
2101                  * to call the NYET interrupt handler in this case.
2102                  */
2103                 hcint &= ~HCINTMSK_NYET;
2104         }
2105
2106         if (hcint & HCINTMSK_CHHLTD) {
2107                 dwc2_hc_chhltd_intr(hsotg, chan, chnum, qtd);
2108                 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2109                         goto exit;
2110         }
2111         if (hcint & HCINTMSK_AHBERR) {
2112                 dwc2_hc_ahberr_intr(hsotg, chan, chnum, qtd);
2113                 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2114                         goto exit;
2115         }
2116         if (hcint & HCINTMSK_STALL) {
2117                 dwc2_hc_stall_intr(hsotg, chan, chnum, qtd);
2118                 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2119                         goto exit;
2120         }
2121         if (hcint & HCINTMSK_NAK) {
2122                 dwc2_hc_nak_intr(hsotg, chan, chnum, qtd);
2123                 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2124                         goto exit;
2125         }
2126         if (hcint & HCINTMSK_ACK) {
2127                 dwc2_hc_ack_intr(hsotg, chan, chnum, qtd);
2128                 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2129                         goto exit;
2130         }
2131         if (hcint & HCINTMSK_NYET) {
2132                 dwc2_hc_nyet_intr(hsotg, chan, chnum, qtd);
2133                 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2134                         goto exit;
2135         }
2136         if (hcint & HCINTMSK_XACTERR) {
2137                 dwc2_hc_xacterr_intr(hsotg, chan, chnum, qtd);
2138                 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2139                         goto exit;
2140         }
2141         if (hcint & HCINTMSK_BBLERR) {
2142                 dwc2_hc_babble_intr(hsotg, chan, chnum, qtd);
2143                 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2144                         goto exit;
2145         }
2146         if (hcint & HCINTMSK_FRMOVRUN) {
2147                 dwc2_hc_frmovrun_intr(hsotg, chan, chnum, qtd);
2148                 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2149                         goto exit;
2150         }
2151         if (hcint & HCINTMSK_DATATGLERR) {
2152                 dwc2_hc_datatglerr_intr(hsotg, chan, chnum, qtd);
2153                 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2154                         goto exit;
2155         }
2156
2157 exit:
2158         chan->hcint = 0;
2159 }
2160
2161 /*
2162  * This interrupt indicates that one or more host channels has a pending
2163  * interrupt. There are multiple conditions that can cause each host channel
2164  * interrupt. This function determines which conditions have occurred for each
2165  * host channel interrupt and handles them appropriately.
2166  */
2167 static void dwc2_hc_intr(struct dwc2_hsotg *hsotg)
2168 {
2169         u32 haint;
2170         int i;
2171         struct dwc2_host_chan *chan, *chan_tmp;
2172
2173         haint = dwc2_readl(hsotg, HAINT);
2174         if (dbg_perio()) {
2175                 dev_vdbg(hsotg->dev, "%s()\n", __func__);
2176
2177                 dev_vdbg(hsotg->dev, "HAINT=%08x\n", haint);
2178         }
2179
2180         /*
2181          * According to USB 2.0 spec section 11.18.8, a host must
2182          * issue complete-split transactions in a microframe for a
2183          * set of full-/low-speed endpoints in the same relative
2184          * order as the start-splits were issued in a microframe for.
2185          */
2186         list_for_each_entry_safe(chan, chan_tmp, &hsotg->split_order,
2187                                  split_order_list_entry) {
2188                 int hc_num = chan->hc_num;
2189
2190                 if (haint & (1 << hc_num)) {
2191                         dwc2_hc_n_intr(hsotg, hc_num);
2192                         haint &= ~(1 << hc_num);
2193                 }
2194         }
2195
2196         for (i = 0; i < hsotg->params.host_channels; i++) {
2197                 if (haint & (1 << i))
2198                         dwc2_hc_n_intr(hsotg, i);
2199         }
2200 }
2201
2202 /* This function handles interrupts for the HCD */
2203 irqreturn_t dwc2_handle_hcd_intr(struct dwc2_hsotg *hsotg)
2204 {
2205         u32 gintsts, dbg_gintsts;
2206         irqreturn_t retval = IRQ_NONE;
2207
2208         if (!dwc2_is_controller_alive(hsotg)) {
2209                 dev_warn(hsotg->dev, "Controller is dead\n");
2210                 return retval;
2211         }
2212
2213         spin_lock(&hsotg->lock);
2214
2215         /* Check if HOST Mode */
2216         if (dwc2_is_host_mode(hsotg)) {
2217                 gintsts = dwc2_read_core_intr(hsotg);
2218                 if (!gintsts) {
2219                         spin_unlock(&hsotg->lock);
2220                         return retval;
2221                 }
2222
2223                 retval = IRQ_HANDLED;
2224
2225                 dbg_gintsts = gintsts;
2226 #ifndef DEBUG_SOF
2227                 dbg_gintsts &= ~GINTSTS_SOF;
2228 #endif
2229                 if (!dbg_perio())
2230                         dbg_gintsts &= ~(GINTSTS_HCHINT | GINTSTS_RXFLVL |
2231                                          GINTSTS_PTXFEMP);
2232
2233                 /* Only print if there are any non-suppressed interrupts left */
2234                 if (dbg_gintsts)
2235                         dev_vdbg(hsotg->dev,
2236                                  "DWC OTG HCD Interrupt Detected gintsts&gintmsk=0x%08x\n",
2237                                  gintsts);
2238
2239                 if (gintsts & GINTSTS_SOF)
2240                         dwc2_sof_intr(hsotg);
2241                 if (gintsts & GINTSTS_RXFLVL)
2242                         dwc2_rx_fifo_level_intr(hsotg);
2243                 if (gintsts & GINTSTS_NPTXFEMP)
2244                         dwc2_np_tx_fifo_empty_intr(hsotg);
2245                 if (gintsts & GINTSTS_PRTINT)
2246                         dwc2_port_intr(hsotg);
2247                 if (gintsts & GINTSTS_HCHINT)
2248                         dwc2_hc_intr(hsotg);
2249                 if (gintsts & GINTSTS_PTXFEMP)
2250                         dwc2_perio_tx_fifo_empty_intr(hsotg);
2251
2252                 if (dbg_gintsts) {
2253                         dev_vdbg(hsotg->dev,
2254                                  "DWC OTG HCD Finished Servicing Interrupts\n");
2255                         dev_vdbg(hsotg->dev,
2256                                  "DWC OTG HCD gintsts=0x%08x gintmsk=0x%08x\n",
2257                                  dwc2_readl(hsotg, GINTSTS),
2258                                  dwc2_readl(hsotg, GINTMSK));
2259                 }
2260         }
2261
2262         spin_unlock(&hsotg->lock);
2263
2264         return retval;
2265 }