drivers/usb/cdns3/cdns3-gadget.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Cadence USBSS DRD Driver - gadget side.
   4  *
   5  * Copyright (C) 2018-2019 Cadence Design Systems.
   6  * Copyright (C) 2017-2018 NXP
   7  *
   8  * Authors: Pawel Jez <pjez@cadence.com>,
   9  *          Pawel Laszczak <pawell@cadence.com>
  10  *          Peter Chen <peter.chen@nxp.com>
  11  */
  12
  13 /*
  14  * Work around 1:
  15  * At some situations, the controller may get stale data address in TRB
  16  * at below sequences:
  17  * 1. Controller read TRB includes data address
  18  * 2. Software updates TRBs includes data address and Cycle bit
  19  * 3. Controller read TRB which includes Cycle bit
  20  * 4. DMA run with stale data address
  21  *
  22  * To fix this problem, driver needs to make the first TRB in TD as invalid.
  23  * After preparing all TRBs driver needs to check the position of DMA and
  24  * if the DMA point to the first just added TRB and doorbell is 1,
  25  * then driver must defer making this TRB as valid. This TRB will be make
  26  * as valid during adding next TRB only if DMA is stopped or at TRBERR
  27  * interrupt.
  28  *
  29  * Issue has been fixed in DEV_VER_V3 version of controller.
  30  *
  31  * Work around 2:
  32  * Controller for OUT endpoints has shared on-chip buffers for all incoming
  33  * packets, including ep0out. It's FIFO buffer, so packets must be handle by DMA
  34  * in correct order. If the first packet in the buffer will not be handled,
  35  * then the following packets directed for other endpoints and  functions
  36  * will be blocked.
  37  * Additionally the packets directed to one endpoint can block entire on-chip
  38  * buffers. In this case transfer to other endpoints also will blocked.
  39  *
  40  * To resolve this issue after raising the descriptor missing interrupt
  41  * driver prepares internal usb_request object and use it to arm DMA transfer.
  42  *
  43  * The problematic situation was observed in case when endpoint has been enabled
  44  * but no usb_request were queued. Driver try detects such endpoints and will
  45  * use this workaround only for these endpoint.
  46  *
  47  * Driver use limited number of buffer. This number can be set by macro
  48  * CDNS3_WA2_NUM_BUFFERS.
  49  *
  50  * Such blocking situation was observed on ACM gadget. For this function
  51  * host send OUT data packet but ACM function is not prepared for this packet.
  52  * It's cause that buffer placed in on chip memory block transfer to other
  53  * endpoints.
  54  *
  55  * Issue has been fixed in DEV_VER_V2 version of controller.
  56  *
  57  */
  58
  59 #include <linux/dma-mapping.h>
  60 #include <linux/usb/gadget.h>
  61 #include <linux/module.h>
  62 #include <linux/dmapool.h>
  63 #include <linux/iopoll.h>
  64
  65 #include "core.h"
  66 #include "gadget-export.h"
  67 #include "cdns3-gadget.h"
  68 #include "cdns3-trace.h"
  69 #include "drd.h"
  70
  71 static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
  72                                    struct usb_request *request,
  73                                    gfp_t gfp_flags);
  74
  75 static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
  76                                  struct usb_request *request);
  77
  78 static int cdns3_ep_run_stream_transfer(struct cdns3_endpoint *priv_ep,
  79                                         struct usb_request *request);
  80
  81 /**
  82  * cdns3_clear_register_bit - clear bit in given register.
  83  * @ptr: address of device controller register to be read and changed
  84  * @mask: bits requested to clar
  85  */
  86 static void cdns3_clear_register_bit(void __iomem *ptr, u32 mask)
  87 {
  88         mask = readl(ptr) & ~mask;
  89         writel(mask, ptr);
  90 }
  91
  92 /**
  93  * cdns3_set_register_bit - set bit in given register.
  94  * @ptr: address of device controller register to be read and changed
  95  * @mask: bits requested to set
  96  */
  97 void cdns3_set_register_bit(void __iomem *ptr, u32 mask)
  98 {
  99         mask = readl(ptr) | mask;
 100         writel(mask, ptr);
 101 }
 102
 103 /**
 104  * cdns3_ep_addr_to_index - Macro converts endpoint address to
 105  * index of endpoint object in cdns3_device.eps[] container
 106  * @ep_addr: endpoint address for which endpoint object is required
 107  *
 108  */
 109 u8 cdns3_ep_addr_to_index(u8 ep_addr)
 110 {
 111         return (((ep_addr & 0x7F)) + ((ep_addr & USB_DIR_IN) ? 16 : 0));
 112 }
 113
 114 static int cdns3_get_dma_pos(struct cdns3_device *priv_dev,
 115                              struct cdns3_endpoint *priv_ep)
 116 {
 117         int dma_index;
 118
 119         dma_index = readl(&priv_dev->regs->ep_traddr) - priv_ep->trb_pool_dma;
 120
 121         return dma_index / TRB_SIZE;
 122 }
 123
 124 /**
 125  * cdns3_next_request - returns next request from list
 126  * @list: list containing requests
 127  *
 128  * Returns request or NULL if no requests in list
 129  */
 130 struct usb_request *cdns3_next_request(struct list_head *list)
 131 {
 132         return list_first_entry_or_null(list, struct usb_request, list);
 133 }
 134
 135 /**
 136  * cdns3_next_align_buf - returns next buffer from list
 137  * @list: list containing buffers
 138  *
 139  * Returns buffer or NULL if no buffers in list
 140  */
 141 static struct cdns3_aligned_buf *cdns3_next_align_buf(struct list_head *list)
 142 {
 143         return list_first_entry_or_null(list, struct cdns3_aligned_buf, list);
 144 }
 145
 146 /**
 147  * cdns3_next_priv_request - returns next request from list
 148  * @list: list containing requests
 149  *
 150  * Returns request or NULL if no requests in list
 151  */
 152 static struct cdns3_request *cdns3_next_priv_request(struct list_head *list)
 153 {
 154         return list_first_entry_or_null(list, struct cdns3_request, list);
 155 }
 156
 157 /**
 158  * cdns3_select_ep - selects endpoint
 159  * @priv_dev:  extended gadget object
 160  * @ep: endpoint address
 161  */
 162 void cdns3_select_ep(struct cdns3_device *priv_dev, u32 ep)
 163 {
 164         if (priv_dev->selected_ep == ep)
 165                 return;
 166
 167         priv_dev->selected_ep = ep;
 168         writel(ep, &priv_dev->regs->ep_sel);
 169 }
 170
 171 /**
 172  * cdns3_get_tdl - gets current tdl for selected endpoint.
 173  * @priv_dev:  extended gadget object
 174  *
 175  * Before calling this function the appropriate endpoint must
 176  * be selected by means of cdns3_select_ep function.
 177  */
 178 static int cdns3_get_tdl(struct cdns3_device *priv_dev)
 179 {
 180         if (priv_dev->dev_ver < DEV_VER_V3)
 181                 return EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));
 182         else
 183                 return readl(&priv_dev->regs->ep_tdl);
 184 }
 185
 186 dma_addr_t cdns3_trb_virt_to_dma(struct cdns3_endpoint *priv_ep,
 187                                  struct cdns3_trb *trb)
 188 {
 189         u32 offset = (char *)trb - (char *)priv_ep->trb_pool;
 190
 191         return priv_ep->trb_pool_dma + offset;
 192 }
 193
 194 static void cdns3_free_trb_pool(struct cdns3_endpoint *priv_ep)
 195 {
 196         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 197
 198         if (priv_ep->trb_pool) {
 199                 dma_pool_free(priv_dev->eps_dma_pool,
 200                               priv_ep->trb_pool, priv_ep->trb_pool_dma);
 201                 priv_ep->trb_pool = NULL;
 202         }
 203 }
 204
 205 /**
 206  * cdns3_allocate_trb_pool - Allocates TRB's pool for selected endpoint
 207  * @priv_ep:  endpoint object
 208  *
 209  * Function will return 0 on success or -ENOMEM on allocation error
 210  */
 211 int cdns3_allocate_trb_pool(struct cdns3_endpoint *priv_ep)
 212 {
 213         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 214         int ring_size = TRB_RING_SIZE;
 215         int num_trbs = ring_size / TRB_SIZE;
 216         struct cdns3_trb *link_trb;
 217
 218         if (priv_ep->trb_pool && priv_ep->alloc_ring_size < ring_size)
 219                 cdns3_free_trb_pool(priv_ep);
 220
 221         if (!priv_ep->trb_pool) {
 222                 priv_ep->trb_pool = dma_pool_alloc(priv_dev->eps_dma_pool,
 223                                                    GFP_ATOMIC,
 224                                                    &priv_ep->trb_pool_dma);
 225
 226                 if (!priv_ep->trb_pool)
 227                         return -ENOMEM;
 228
 229                 priv_ep->alloc_ring_size = ring_size;
 230         }
 231
 232         memset(priv_ep->trb_pool, 0, ring_size);
 233
 234         priv_ep->num_trbs = num_trbs;
 235
 236         if (!priv_ep->num)
 237                 return 0;
 238
 239         /* Initialize the last TRB as Link TRB */
 240         link_trb = (priv_ep->trb_pool + (priv_ep->num_trbs - 1));
 241
 242         if (priv_ep->use_streams) {
 243                 /*
 244                  * For stream capable endpoints driver use single correct TRB.
 245                  * The last trb has zeroed cycle bit
 246                  */
 247                 link_trb->control = 0;
 248         } else {
 249                 link_trb->buffer = cpu_to_le32(TRB_BUFFER(priv_ep->trb_pool_dma));
 250                 link_trb->control = cpu_to_le32(TRB_CYCLE | TRB_TYPE(TRB_LINK) | TRB_TOGGLE);
 251         }
 252         return 0;
 253 }
 254
 255 /**
 256  * cdns3_ep_stall_flush - Stalls and flushes selected endpoint
 257  * @priv_ep: endpoint object
 258  *
 259  * Endpoint must be selected before call to this function
 260  */
 261 static void cdns3_ep_stall_flush(struct cdns3_endpoint *priv_ep)
 262 {
 263         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 264         int val;
 265
 266         trace_cdns3_halt(priv_ep, 1, 1);
 267
 268         writel(EP_CMD_DFLUSH | EP_CMD_ERDY | EP_CMD_SSTALL,
 269                &priv_dev->regs->ep_cmd);
 270
 271         /* wait for DFLUSH cleared */
 272         readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
 273                                   !(val & EP_CMD_DFLUSH), 1, 1000);
 274         priv_ep->flags |= EP_STALLED;
 275         priv_ep->flags &= ~EP_STALL_PENDING;
 276 }
 277
 278 /**
 279  * cdns3_hw_reset_eps_config - reset endpoints configuration kept by controller.
 280  * @priv_dev: extended gadget object
 281  */
 282 void cdns3_hw_reset_eps_config(struct cdns3_device *priv_dev)
 283 {
 284         int i;
 285
 286         writel(USB_CONF_CFGRST, &priv_dev->regs->usb_conf);
 287
 288         cdns3_allow_enable_l1(priv_dev, 0);
 289         priv_dev->hw_configured_flag = 0;
 290         priv_dev->onchip_used_size = 0;
 291         priv_dev->out_mem_is_allocated = 0;
 292         priv_dev->wait_for_setup = 0;
 293         priv_dev->using_streams = 0;
 294
 295         for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++)
 296                 if (priv_dev->eps[i])
 297                         priv_dev->eps[i]->flags &= ~EP_CONFIGURED;
 298 }
 299
 300 /**
 301  * cdns3_ep_inc_trb - increment a trb index.
 302  * @index: Pointer to the TRB index to increment.
 303  * @cs: Cycle state
 304  * @trb_in_seg: number of TRBs in segment
 305  *
 306  * The index should never point to the link TRB. After incrementing,
 307  * if it is point to the link TRB, wrap around to the beginning and revert
 308  * cycle state bit The
 309  * link TRB is always at the last TRB entry.
 310  */
 311 static void cdns3_ep_inc_trb(int *index, u8 *cs, int trb_in_seg)
 312 {
 313         (*index)++;
 314         if (*index == (trb_in_seg - 1)) {
 315                 *index = 0;
 316                 *cs ^=  1;
 317         }
 318 }
 319
 320 /**
 321  * cdns3_ep_inc_enq - increment endpoint's enqueue pointer
 322  * @priv_ep: The endpoint whose enqueue pointer we're incrementing
 323  */
 324 static void cdns3_ep_inc_enq(struct cdns3_endpoint *priv_ep)
 325 {
 326         priv_ep->free_trbs--;
 327         cdns3_ep_inc_trb(&priv_ep->enqueue, &priv_ep->pcs, priv_ep->num_trbs);
 328 }
 329
 330 /**
 331  * cdns3_ep_inc_deq - increment endpoint's dequeue pointer
 332  * @priv_ep: The endpoint whose dequeue pointer we're incrementing
 333  */
 334 static void cdns3_ep_inc_deq(struct cdns3_endpoint *priv_ep)
 335 {
 336         priv_ep->free_trbs++;
 337         cdns3_ep_inc_trb(&priv_ep->dequeue, &priv_ep->ccs, priv_ep->num_trbs);
 338 }
 339
 340 /**
 341  * cdns3_allow_enable_l1 - enable/disable permits to transition to L1.
 342  * @priv_dev: Extended gadget object
 343  * @enable: Enable/disable permit to transition to L1.
 344  *
 345  * If bit USB_CONF_L1EN is set and device receive Extended Token packet,
 346  * then controller answer with ACK handshake.
 347  * If bit USB_CONF_L1DS is set and device receive Extended Token packet,
 348  * then controller answer with NYET handshake.
 349  */
 350 void cdns3_allow_enable_l1(struct cdns3_device *priv_dev, int enable)
 351 {
 352         if (enable)
 353                 writel(USB_CONF_L1EN, &priv_dev->regs->usb_conf);
 354         else
 355                 writel(USB_CONF_L1DS, &priv_dev->regs->usb_conf);
 356 }
 357
 358 enum usb_device_speed cdns3_get_speed(struct cdns3_device *priv_dev)
 359 {
 360         u32 reg;
 361
 362         reg = readl(&priv_dev->regs->usb_sts);
 363
 364         if (DEV_SUPERSPEED(reg))
 365                 return USB_SPEED_SUPER;
 366         else if (DEV_HIGHSPEED(reg))
 367                 return USB_SPEED_HIGH;
 368         else if (DEV_FULLSPEED(reg))
 369                 return USB_SPEED_FULL;
 370         else if (DEV_LOWSPEED(reg))
 371                 return USB_SPEED_LOW;
 372         return USB_SPEED_UNKNOWN;
 373 }
 374
 375 /**
 376  * cdns3_start_all_request - add to ring all request not started
 377  * @priv_dev: Extended gadget object
 378  * @priv_ep: The endpoint for whom request will be started.
 379  *
 380  * Returns return ENOMEM if transfer ring i not enough TRBs to start
 381  *         all requests.
 382  */
 383 static int cdns3_start_all_request(struct cdns3_device *priv_dev,
 384                                    struct cdns3_endpoint *priv_ep)
 385 {
 386         struct usb_request *request;
 387         int ret = 0;
 388         u8 pending_empty = list_empty(&priv_ep->pending_req_list);
 389
 390         /*
 391          * If the last pending transfer is INTERNAL
 392          * OR streams are enabled for this endpoint
 393          * do NOT start new transfer till the last one is pending
 394          */
 395         if (!pending_empty) {
 396                 struct cdns3_request *priv_req;
 397
 398                 request = cdns3_next_request(&priv_ep->pending_req_list);
 399                 priv_req = to_cdns3_request(request);
 400                 if ((priv_req->flags & REQUEST_INTERNAL) ||
 401                     (priv_ep->flags & EP_TDLCHK_EN) ||
 402                         priv_ep->use_streams) {
 403                         dev_dbg(priv_dev->dev, "Blocking external request\n");
 404                         return ret;
 405                 }
 406         }
 407
 408         while (!list_empty(&priv_ep->deferred_req_list)) {
 409                 request = cdns3_next_request(&priv_ep->deferred_req_list);
 410
 411                 if (!priv_ep->use_streams) {
 412                         ret = cdns3_ep_run_transfer(priv_ep, request);
 413                 } else {
 414                         priv_ep->stream_sg_idx = 0;
 415                         ret = cdns3_ep_run_stream_transfer(priv_ep, request);
 416                 }
 417                 if (ret)
 418                         return ret;
 419
 420                 list_move_tail(&request->list, &priv_ep->pending_req_list);
 421                 if (request->stream_id != 0 || (priv_ep->flags & EP_TDLCHK_EN))
 422                         break;
 423         }
 424
 425         priv_ep->flags &= ~EP_RING_FULL;
 426         return ret;
 427 }
 428
 429 /*
 430  * WA2: Set flag for all not ISOC OUT endpoints. If this flag is set
 431  * driver try to detect whether endpoint need additional internal
 432  * buffer for unblocking on-chip FIFO buffer. This flag will be cleared
 433  * if before first DESCMISS interrupt the DMA will be armed.
 434  */
 435 #define cdns3_wa2_enable_detection(priv_dev, priv_ep, reg) do { \
 436         if (!priv_ep->dir && priv_ep->type != USB_ENDPOINT_XFER_ISOC) { \
 437                 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_DET; \
 438                 (reg) |= EP_STS_EN_DESCMISEN; \
 439         } } while (0)
 440
 441 static void __cdns3_descmiss_copy_data(struct usb_request *request,
 442         struct usb_request *descmiss_req)
 443 {
 444         int length = request->actual + descmiss_req->actual;
 445         struct scatterlist *s = request->sg;
 446
 447         if (!s) {
 448                 if (length <= request->length) {
 449                         memcpy(&((u8 *)request->buf)[request->actual],
 450                                descmiss_req->buf,
 451                                descmiss_req->actual);
 452                         request->actual = length;
 453                 } else {
 454                         /* It should never occures */
 455                         request->status = -ENOMEM;
 456                 }
 457         } else {
 458                 if (length <= sg_dma_len(s)) {
 459                         void *p = phys_to_virt(sg_dma_address(s));
 460
 461                         memcpy(&((u8 *)p)[request->actual],
 462                                 descmiss_req->buf,
 463                                 descmiss_req->actual);
 464                         request->actual = length;
 465                 } else {
 466                         request->status = -ENOMEM;
 467                 }
 468         }
 469 }
 470
 471 /**
 472  * cdns3_wa2_descmiss_copy_data - copy data from internal requests to
 473  * request queued by class driver.
 474  * @priv_ep: extended endpoint object
 475  * @request: request object
 476  */
 477 static void cdns3_wa2_descmiss_copy_data(struct cdns3_endpoint *priv_ep,
 478                                          struct usb_request *request)
 479 {
 480         struct usb_request *descmiss_req;
 481         struct cdns3_request *descmiss_priv_req;
 482
 483         while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
 484                 int chunk_end;
 485
 486                 descmiss_priv_req =
 487                         cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
 488                 descmiss_req = &descmiss_priv_req->request;
 489
 490                 /* driver can't touch pending request */
 491                 if (descmiss_priv_req->flags & REQUEST_PENDING)
 492                         break;
 493
 494                 chunk_end = descmiss_priv_req->flags & REQUEST_INTERNAL_CH;
 495                 request->status = descmiss_req->status;
 496                 __cdns3_descmiss_copy_data(request, descmiss_req);
 497                 list_del_init(&descmiss_priv_req->list);
 498                 kfree(descmiss_req->buf);
 499                 cdns3_gadget_ep_free_request(&priv_ep->endpoint, descmiss_req);
 500                 --priv_ep->wa2_counter;
 501
 502                 if (!chunk_end)
 503                         break;
 504         }
 505 }
 506
 507 static struct usb_request *cdns3_wa2_gadget_giveback(struct cdns3_device *priv_dev,
 508                                                      struct cdns3_endpoint *priv_ep,
 509                                                      struct cdns3_request *priv_req)
 510 {
 511         if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN &&
 512             priv_req->flags & REQUEST_INTERNAL) {
 513                 struct usb_request *req;
 514
 515                 req = cdns3_next_request(&priv_ep->deferred_req_list);
 516
 517                 priv_ep->descmis_req = NULL;
 518
 519                 if (!req)
 520                         return NULL;
 521
 522                 /* unmap the gadget request before copying data */
 523                 usb_gadget_unmap_request_by_dev(priv_dev->sysdev, req,
 524                                                 priv_ep->dir);
 525
 526                 cdns3_wa2_descmiss_copy_data(priv_ep, req);
 527                 if (!(priv_ep->flags & EP_QUIRK_END_TRANSFER) &&
 528                     req->length != req->actual) {
 529                         /* wait for next part of transfer */
 530                         /* re-map the gadget request buffer*/
 531                         usb_gadget_map_request_by_dev(priv_dev->sysdev, req,
 532                                 usb_endpoint_dir_in(priv_ep->endpoint.desc));
 533                         return NULL;
 534                 }
 535
 536                 if (req->status == -EINPROGRESS)
 537                         req->status = 0;
 538
 539                 list_del_init(&req->list);
 540                 cdns3_start_all_request(priv_dev, priv_ep);
 541                 return req;
 542         }
 543
 544         return &priv_req->request;
 545 }
 546
 547 static int cdns3_wa2_gadget_ep_queue(struct cdns3_device *priv_dev,
 548                                      struct cdns3_endpoint *priv_ep,
 549                                      struct cdns3_request *priv_req)
 550 {
 551         int deferred = 0;
 552
 553         /*
 554          * If transfer was queued before DESCMISS appear than we
 555          * can disable handling of DESCMISS interrupt. Driver assumes that it
 556          * can disable special treatment for this endpoint.
 557          */
 558         if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
 559                 u32 reg;
 560
 561                 cdns3_select_ep(priv_dev, priv_ep->num | priv_ep->dir);
 562                 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
 563                 reg = readl(&priv_dev->regs->ep_sts_en);
 564                 reg &= ~EP_STS_EN_DESCMISEN;
 565                 trace_cdns3_wa2(priv_ep, "workaround disabled\n");
 566                 writel(reg, &priv_dev->regs->ep_sts_en);
 567         }
 568
 569         if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
 570                 u8 pending_empty = list_empty(&priv_ep->pending_req_list);
 571                 u8 descmiss_empty = list_empty(&priv_ep->wa2_descmiss_req_list);
 572
 573                 /*
 574                  *  DESCMISS transfer has been finished, so data will be
 575                  *  directly copied from internal allocated usb_request
 576                  *  objects.
 577                  */
 578                 if (pending_empty && !descmiss_empty &&
 579                     !(priv_req->flags & REQUEST_INTERNAL)) {
 580                         cdns3_wa2_descmiss_copy_data(priv_ep,
 581                                                      &priv_req->request);
 582
 583                         trace_cdns3_wa2(priv_ep, "get internal stored data");
 584
 585                         list_add_tail(&priv_req->request.list,
 586                                       &priv_ep->pending_req_list);
 587                         cdns3_gadget_giveback(priv_ep, priv_req,
 588                                               priv_req->request.status);
 589
 590                         /*
 591                          * Intentionally driver returns positive value as
 592                          * correct value. It informs that transfer has
 593                          * been finished.
 594                          */
 595                         return EINPROGRESS;
 596                 }
 597
 598                 /*
 599                  * Driver will wait for completion DESCMISS transfer,
 600                  * before starts new, not DESCMISS transfer.
 601                  */
 602                 if (!pending_empty && !descmiss_empty) {
 603                         trace_cdns3_wa2(priv_ep, "wait for pending transfer\n");
 604                         deferred = 1;
 605                 }
 606
 607                 if (priv_req->flags & REQUEST_INTERNAL)
 608                         list_add_tail(&priv_req->list,
 609                                       &priv_ep->wa2_descmiss_req_list);
 610         }
 611
 612         return deferred;
 613 }
 614
 615 static void cdns3_wa2_remove_old_request(struct cdns3_endpoint *priv_ep)
 616 {
 617         struct cdns3_request *priv_req;
 618
 619         while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
 620                 u8 chain;
 621
 622                 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
 623                 chain = !!(priv_req->flags & REQUEST_INTERNAL_CH);
 624
 625                 trace_cdns3_wa2(priv_ep, "removes eldest request");
 626
 627                 kfree(priv_req->request.buf);
 628                 list_del_init(&priv_req->list);
 629                 cdns3_gadget_ep_free_request(&priv_ep->endpoint,
 630                                              &priv_req->request);
 631                 --priv_ep->wa2_counter;
 632
 633                 if (!chain)
 634                         break;
 635         }
 636 }
 637
 638 /**
 639  * cdns3_wa2_descmissing_packet - handles descriptor missing event.
 640  * @priv_ep: extended gadget object
 641  *
 642  * This function is used only for WA2. For more information see Work around 2
 643  * description.
 644  */
 645 static void cdns3_wa2_descmissing_packet(struct cdns3_endpoint *priv_ep)
 646 {
 647         struct cdns3_request *priv_req;
 648         struct usb_request *request;
 649         u8 pending_empty = list_empty(&priv_ep->pending_req_list);
 650
 651         /* check for pending transfer */
 652         if (!pending_empty) {
 653                 trace_cdns3_wa2(priv_ep, "Ignoring Descriptor missing IRQ\n");
 654                 return;
 655         }
 656
 657         if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
 658                 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
 659                 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_EN;
 660         }
 661
 662         trace_cdns3_wa2(priv_ep, "Description Missing detected\n");
 663
 664         if (priv_ep->wa2_counter >= CDNS3_WA2_NUM_BUFFERS) {
 665                 trace_cdns3_wa2(priv_ep, "WA2 overflow\n");
 666                 cdns3_wa2_remove_old_request(priv_ep);
 667         }
 668
 669         request = cdns3_gadget_ep_alloc_request(&priv_ep->endpoint,
 670                                                 GFP_ATOMIC);
 671         if (!request)
 672                 goto err;
 673
 674         priv_req = to_cdns3_request(request);
 675         priv_req->flags |= REQUEST_INTERNAL;
 676
 677         /* if this field is still assigned it indicate that transfer related
 678          * with this request has not been finished yet. Driver in this
 679          * case simply allocate next request and assign flag REQUEST_INTERNAL_CH
 680          * flag to previous one. It will indicate that current request is
 681          * part of the previous one.
 682          */
 683         if (priv_ep->descmis_req)
 684                 priv_ep->descmis_req->flags |= REQUEST_INTERNAL_CH;
 685
 686         priv_req->request.buf = kzalloc(CDNS3_DESCMIS_BUF_SIZE,
 687                                         GFP_ATOMIC);
 688         priv_ep->wa2_counter++;
 689
 690         if (!priv_req->request.buf) {
 691                 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
 692                 goto err;
 693         }
 694
 695         priv_req->request.length = CDNS3_DESCMIS_BUF_SIZE;
 696         priv_ep->descmis_req = priv_req;
 697
 698         __cdns3_gadget_ep_queue(&priv_ep->endpoint,
 699                                 &priv_ep->descmis_req->request,
 700                                 GFP_ATOMIC);
 701
 702         return;
 703
 704 err:
 705         dev_err(priv_ep->cdns3_dev->dev,
 706                 "Failed: No sufficient memory for DESCMIS\n");
 707 }
 708
 709 static void cdns3_wa2_reset_tdl(struct cdns3_device *priv_dev)
 710 {
 711         u16 tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));
 712
 713         if (tdl) {
 714                 u16 reset_val = EP_CMD_TDL_MAX + 1 - tdl;
 715
 716                 writel(EP_CMD_TDL_SET(reset_val) | EP_CMD_STDL,
 717                        &priv_dev->regs->ep_cmd);
 718         }
 719 }
 720
 721 static void cdns3_wa2_check_outq_status(struct cdns3_device *priv_dev)
 722 {
 723         u32 ep_sts_reg;
 724
 725         /* select EP0-out */
 726         cdns3_select_ep(priv_dev, 0);
 727
 728         ep_sts_reg = readl(&priv_dev->regs->ep_sts);
 729
 730         if (EP_STS_OUTQ_VAL(ep_sts_reg)) {
 731                 u32 outq_ep_num = EP_STS_OUTQ_NO(ep_sts_reg);
 732                 struct cdns3_endpoint *outq_ep = priv_dev->eps[outq_ep_num];
 733
 734                 if ((outq_ep->flags & EP_ENABLED) && !(outq_ep->use_streams) &&
 735                     outq_ep->type != USB_ENDPOINT_XFER_ISOC && outq_ep_num) {
 736                         u8 pending_empty = list_empty(&outq_ep->pending_req_list);
 737
 738                         if ((outq_ep->flags & EP_QUIRK_EXTRA_BUF_DET) ||
 739                             (outq_ep->flags & EP_QUIRK_EXTRA_BUF_EN) ||
 740                             !pending_empty) {
 741                         } else {
 742                                 u32 ep_sts_en_reg;
 743                                 u32 ep_cmd_reg;
 744
 745                                 cdns3_select_ep(priv_dev, outq_ep->num |
 746                                                 outq_ep->dir);
 747                                 ep_sts_en_reg = readl(&priv_dev->regs->ep_sts_en);
 748                                 ep_cmd_reg = readl(&priv_dev->regs->ep_cmd);
 749
 750                                 outq_ep->flags |= EP_TDLCHK_EN;
 751                                 cdns3_set_register_bit(&priv_dev->regs->ep_cfg,
 752                                                        EP_CFG_TDL_CHK);
 753
 754                                 cdns3_wa2_enable_detection(priv_dev, outq_ep,
 755                                                            ep_sts_en_reg);
 756                                 writel(ep_sts_en_reg,
 757                                        &priv_dev->regs->ep_sts_en);
 758                                 /* reset tdl value to zero */
 759                                 cdns3_wa2_reset_tdl(priv_dev);
 760                                 /*
 761                                  * Memory barrier - Reset tdl before ringing the
 762                                  * doorbell.
 763                                  */
 764                                 wmb();
 765                                 if (EP_CMD_DRDY & ep_cmd_reg) {
 766                                         trace_cdns3_wa2(outq_ep, "Enabling WA2 skipping doorbell\n");
 767
 768                                 } else {
 769                                         trace_cdns3_wa2(outq_ep, "Enabling WA2 ringing doorbell\n");
 770                                         /*
 771                                          * ring doorbell to generate DESCMIS irq
 772                                          */
 773                                         writel(EP_CMD_DRDY,
 774                                                &priv_dev->regs->ep_cmd);
 775                                 }
 776                         }
 777                 }
 778         }
 779 }
 780
 781 /**
 782  * cdns3_gadget_giveback - call struct usb_request's ->complete callback
 783  * @priv_ep: The endpoint to whom the request belongs to
 784  * @priv_req: The request we're giving back
 785  * @status: completion code for the request
 786  *
 787  * Must be called with controller's lock held and interrupts disabled. This
 788  * function will unmap @req and call its ->complete() callback to notify upper
 789  * layers that it has completed.
 790  */
 791 void cdns3_gadget_giveback(struct cdns3_endpoint *priv_ep,
 792                            struct cdns3_request *priv_req,
 793                            int status)
 794 {
 795         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 796         struct usb_request *request = &priv_req->request;
 797
 798         list_del_init(&request->list);
 799
 800         if (request->status == -EINPROGRESS)
 801                 request->status = status;
 802
 803         usb_gadget_unmap_request_by_dev(priv_dev->sysdev, request,
 804                                         priv_ep->dir);
 805
 806         if ((priv_req->flags & REQUEST_UNALIGNED) &&
 807             priv_ep->dir == USB_DIR_OUT && !request->status) {
 808                 /* Make DMA buffer CPU accessible */
 809                 dma_sync_single_for_cpu(priv_dev->sysdev,
 810                         priv_req->aligned_buf->dma,
 811                         priv_req->aligned_buf->size,
 812                         priv_req->aligned_buf->dir);
 813                 memcpy(request->buf, priv_req->aligned_buf->buf,
 814                        request->length);
 815         }
 816
 817         priv_req->flags &= ~(REQUEST_PENDING | REQUEST_UNALIGNED);
 818         /* All TRBs have finished, clear the counter */
 819         priv_req->finished_trb = 0;
 820         trace_cdns3_gadget_giveback(priv_req);
 821
 822         if (priv_dev->dev_ver < DEV_VER_V2) {
 823                 request = cdns3_wa2_gadget_giveback(priv_dev, priv_ep,
 824                                                     priv_req);
 825                 if (!request)
 826                         return;
 827         }
 828
 829         if (request->complete) {
 830                 spin_unlock(&priv_dev->lock);
 831                 usb_gadget_giveback_request(&priv_ep->endpoint,
 832                                             request);
 833                 spin_lock(&priv_dev->lock);
 834         }
 835
 836         if (request->buf == priv_dev->zlp_buf)
 837                 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
 838 }
 839
 840 static void cdns3_wa1_restore_cycle_bit(struct cdns3_endpoint *priv_ep)
 841 {
 842         /* Work around for stale data address in TRB*/
 843         if (priv_ep->wa1_set) {
 844                 trace_cdns3_wa1(priv_ep, "restore cycle bit");
 845
 846                 priv_ep->wa1_set = 0;
 847                 priv_ep->wa1_trb_index = 0xFFFF;
 848                 if (priv_ep->wa1_cycle_bit) {
 849                         priv_ep->wa1_trb->control =
 850                                 priv_ep->wa1_trb->control | cpu_to_le32(0x1);
 851                 } else {
 852                         priv_ep->wa1_trb->control =
 853                                 priv_ep->wa1_trb->control & cpu_to_le32(~0x1);
 854                 }
 855         }
 856 }
 857
 858 static void cdns3_free_aligned_request_buf(struct work_struct *work)
 859 {
 860         struct cdns3_device *priv_dev = container_of(work, struct cdns3_device,
 861                                         aligned_buf_wq);
 862         struct cdns3_aligned_buf *buf, *tmp;
 863         unsigned long flags;
 864
 865         spin_lock_irqsave(&priv_dev->lock, flags);
 866
 867         list_for_each_entry_safe(buf, tmp, &priv_dev->aligned_buf_list, list) {
 868                 if (!buf->in_use) {
 869                         list_del(&buf->list);
 870
 871                         /*
 872                          * Re-enable interrupts to free DMA capable memory.
 873                          * Driver can't free this memory with disabled
 874                          * interrupts.
 875                          */
 876                         spin_unlock_irqrestore(&priv_dev->lock, flags);
 877                         dma_free_noncoherent(priv_dev->sysdev, buf->size,
 878                                           buf->buf, buf->dma, buf->dir);
 879                         kfree(buf);
 880                         spin_lock_irqsave(&priv_dev->lock, flags);
 881                 }
 882         }
 883
 884         spin_unlock_irqrestore(&priv_dev->lock, flags);
 885 }
 886
 887 static int cdns3_prepare_aligned_request_buf(struct cdns3_request *priv_req)
 888 {
 889         struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
 890         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 891         struct cdns3_aligned_buf *buf;
 892
 893         /* check if buffer is aligned to 8. */
 894         if (!((uintptr_t)priv_req->request.buf & 0x7))
 895                 return 0;
 896
 897         buf = priv_req->aligned_buf;
 898
 899         if (!buf || priv_req->request.length > buf->size) {
 900                 buf = kzalloc(sizeof(*buf), GFP_ATOMIC);
 901                 if (!buf)
 902                         return -ENOMEM;
 903
 904                 buf->size = priv_req->request.length;
 905                 buf->dir = usb_endpoint_dir_in(priv_ep->endpoint.desc) ?
 906                         DMA_TO_DEVICE : DMA_FROM_DEVICE;
 907
 908                 buf->buf = dma_alloc_noncoherent(priv_dev->sysdev,
 909                                               buf->size,
 910                                               &buf->dma,
 911                                               buf->dir,
 912                                               GFP_ATOMIC);
 913                 if (!buf->buf) {
 914                         kfree(buf);
 915                         return -ENOMEM;
 916                 }
 917
 918                 if (priv_req->aligned_buf) {
 919                         trace_cdns3_free_aligned_request(priv_req);
 920                         priv_req->aligned_buf->in_use = 0;
 921                         queue_work(system_freezable_wq,
 922                                    &priv_dev->aligned_buf_wq);
 923                 }
 924
 925                 buf->in_use = 1;
 926                 priv_req->aligned_buf = buf;
 927
 928                 list_add_tail(&buf->list,
 929                               &priv_dev->aligned_buf_list);
 930         }
 931
 932         if (priv_ep->dir == USB_DIR_IN) {
 933                 /* Make DMA buffer CPU accessible */
 934                 dma_sync_single_for_cpu(priv_dev->sysdev,
 935                         buf->dma, buf->size, buf->dir);
 936                 memcpy(buf->buf, priv_req->request.buf,
 937                        priv_req->request.length);
 938         }
 939
 940         /* Transfer DMA buffer ownership back to device */
 941         dma_sync_single_for_device(priv_dev->sysdev,
 942                         buf->dma, buf->size, buf->dir);
 943
 944         priv_req->flags |= REQUEST_UNALIGNED;
 945         trace_cdns3_prepare_aligned_request(priv_req);
 946
 947         return 0;
 948 }
 949
 950 static int cdns3_wa1_update_guard(struct cdns3_endpoint *priv_ep,
 951                                   struct cdns3_trb *trb)
 952 {
 953         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 954
 955         if (!priv_ep->wa1_set) {
 956                 u32 doorbell;
 957
 958                 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
 959
 960                 if (doorbell) {
 961                         priv_ep->wa1_cycle_bit = priv_ep->pcs ? TRB_CYCLE : 0;
 962                         priv_ep->wa1_set = 1;
 963                         priv_ep->wa1_trb = trb;
 964                         priv_ep->wa1_trb_index = priv_ep->enqueue;
 965                         trace_cdns3_wa1(priv_ep, "set guard");
 966                         return 0;
 967                 }
 968         }
 969         return 1;
 970 }
 971
 972 static void cdns3_wa1_tray_restore_cycle_bit(struct cdns3_device *priv_dev,
 973                                              struct cdns3_endpoint *priv_ep)
 974 {
 975         int dma_index;
 976         u32 doorbell;
 977
 978         doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
 979         dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
 980
 981         if (!doorbell || dma_index != priv_ep->wa1_trb_index)
 982                 cdns3_wa1_restore_cycle_bit(priv_ep);
 983 }
 984
 985 static int cdns3_ep_run_stream_transfer(struct cdns3_endpoint *priv_ep,
 986                                         struct usb_request *request)
 987 {
 988         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 989         struct cdns3_request *priv_req;
 990         struct cdns3_trb *trb;
 991         dma_addr_t trb_dma;
 992         int address;
 993         u32 control;
 994         u32 length;
 995         u32 tdl;
 996         unsigned int sg_idx = priv_ep->stream_sg_idx;
 997
 998         priv_req = to_cdns3_request(request);
 999         address = priv_ep->endpoint.desc->bEndpointAddress;
1000
1001         priv_ep->flags |= EP_PENDING_REQUEST;
1002
1003         /* must allocate buffer aligned to 8 */
1004         if (priv_req->flags & REQUEST_UNALIGNED)
1005                 trb_dma = priv_req->aligned_buf->dma;
1006         else
1007                 trb_dma = request->dma;
1008
1009         /*  For stream capable endpoints driver use only single TD. */
1010         trb = priv_ep->trb_pool + priv_ep->enqueue;
1011         priv_req->start_trb = priv_ep->enqueue;
1012         priv_req->end_trb = priv_req->start_trb;
1013         priv_req->trb = trb;
1014
1015         cdns3_select_ep(priv_ep->cdns3_dev, address);
1016
1017         control = TRB_TYPE(TRB_NORMAL) | TRB_CYCLE |
1018                   TRB_STREAM_ID(priv_req->request.stream_id) | TRB_ISP;
1019
1020         if (!request->num_sgs) {
1021                 trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma));
1022                 length = request->length;
1023         } else {
1024                 trb->buffer = cpu_to_le32(TRB_BUFFER(request->sg[sg_idx].dma_address));
1025                 length = request->sg[sg_idx].length;
1026         }
1027
1028         tdl = DIV_ROUND_UP(length, priv_ep->endpoint.maxpacket);
1029
1030         trb->length = cpu_to_le32(TRB_BURST_LEN(16) | TRB_LEN(length));
1031
1032         /*
1033          * For DEV_VER_V2 controller version we have enabled
1034          * USB_CONF2_EN_TDL_TRB in DMULT configuration.
1035          * This enables TDL calculation based on TRB, hence setting TDL in TRB.
1036          */
1037         if (priv_dev->dev_ver >= DEV_VER_V2) {
1038                 if (priv_dev->gadget.speed == USB_SPEED_SUPER)
1039                         trb->length |= cpu_to_le32(TRB_TDL_SS_SIZE(tdl));
1040         }
1041         priv_req->flags |= REQUEST_PENDING;
1042
1043         trb->control = cpu_to_le32(control);
1044
1045         trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
1046
1047         /*
1048          * Memory barrier - Cycle Bit must be set before trb->length  and
1049          * trb->buffer fields.
1050          */
1051         wmb();
1052
1053         /* always first element */
1054         writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma),
1055                &priv_dev->regs->ep_traddr);
1056
1057         if (!(priv_ep->flags & EP_STALLED)) {
1058                 trace_cdns3_ring(priv_ep);
1059                 /*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/
1060                 writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);
1061
1062                 priv_ep->prime_flag = false;
1063
1064                 /*
1065                  * Controller version DEV_VER_V2 tdl calculation
1066                  * is based on TRB
1067                  */
1068
1069                 if (priv_dev->dev_ver < DEV_VER_V2)
1070                         writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL,
1071                                &priv_dev->regs->ep_cmd);
1072                 else if (priv_dev->dev_ver > DEV_VER_V2)
1073                         writel(tdl, &priv_dev->regs->ep_tdl);
1074
1075                 priv_ep->last_stream_id = priv_req->request.stream_id;
1076                 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1077                 writel(EP_CMD_ERDY_SID(priv_req->request.stream_id) |
1078                        EP_CMD_ERDY, &priv_dev->regs->ep_cmd);
1079
1080                 trace_cdns3_doorbell_epx(priv_ep->name,
1081                                          readl(&priv_dev->regs->ep_traddr));
1082         }
1083
1084         /* WORKAROUND for transition to L0 */
1085         __cdns3_gadget_wakeup(priv_dev);
1086
1087         return 0;
1088 }
1089
1090 static void cdns3_rearm_drdy_if_needed(struct cdns3_endpoint *priv_ep)
1091 {
1092         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1093
1094         if (priv_dev->dev_ver < DEV_VER_V3)
1095                 return;
1096
1097         if (readl(&priv_dev->regs->ep_sts) & EP_STS_TRBERR) {
1098                 writel(EP_STS_TRBERR, &priv_dev->regs->ep_sts);
1099                 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1100         }
1101 }
1102
1103 /**
1104  * cdns3_ep_run_transfer - start transfer on no-default endpoint hardware
1105  * @priv_ep: endpoint object
1106  * @request: request object
1107  *
1108  * Returns zero on success or negative value on failure
1109  */
1110 static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
1111                                  struct usb_request *request)
1112 {
1113         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1114         struct cdns3_request *priv_req;
1115         struct cdns3_trb *trb;
1116         struct cdns3_trb *link_trb = NULL;
1117         dma_addr_t trb_dma;
1118         u32 togle_pcs = 1;
1119         int sg_iter = 0;
1120         int num_trb;
1121         int address;
1122         u32 control;
1123         int pcs;
1124         u16 total_tdl = 0;
1125         struct scatterlist *s = NULL;
1126         bool sg_supported = !!(request->num_mapped_sgs);
1127
1128         if (priv_ep->type == USB_ENDPOINT_XFER_ISOC)
1129                 num_trb = priv_ep->interval;
1130         else
1131                 num_trb = sg_supported ? request->num_mapped_sgs : 1;
1132
1133         if (num_trb > priv_ep->free_trbs) {
1134                 priv_ep->flags |= EP_RING_FULL;
1135                 return -ENOBUFS;
1136         }
1137
1138         priv_req = to_cdns3_request(request);
1139         address = priv_ep->endpoint.desc->bEndpointAddress;
1140
1141         priv_ep->flags |= EP_PENDING_REQUEST;
1142
1143         /* must allocate buffer aligned to 8 */
1144         if (priv_req->flags & REQUEST_UNALIGNED)
1145                 trb_dma = priv_req->aligned_buf->dma;
1146         else
1147                 trb_dma = request->dma;
1148
1149         trb = priv_ep->trb_pool + priv_ep->enqueue;
1150         priv_req->start_trb = priv_ep->enqueue;
1151         priv_req->trb = trb;
1152
1153         cdns3_select_ep(priv_ep->cdns3_dev, address);
1154
1155         /* prepare ring */
1156         if ((priv_ep->enqueue + num_trb)  >= (priv_ep->num_trbs - 1)) {
1157                 int doorbell, dma_index;
1158                 u32 ch_bit = 0;
1159
1160                 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
1161                 dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
1162
1163                 /* Driver can't update LINK TRB if it is current processed. */
1164                 if (doorbell && dma_index == priv_ep->num_trbs - 1) {
1165                         priv_ep->flags |= EP_DEFERRED_DRDY;
1166                         return -ENOBUFS;
1167                 }
1168
1169                 /*updating C bt in  Link TRB before starting DMA*/
1170                 link_trb = priv_ep->trb_pool + (priv_ep->num_trbs - 1);
1171                 /*
1172                  * For TRs size equal 2 enabling TRB_CHAIN for epXin causes
1173                  * that DMA stuck at the LINK TRB.
1174                  * On the other hand, removing TRB_CHAIN for longer TRs for
1175                  * epXout cause that DMA stuck after handling LINK TRB.
1176                  * To eliminate this strange behavioral driver set TRB_CHAIN
1177                  * bit only for TR size > 2.
1178                  */
1179                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC ||
1180                     TRBS_PER_SEGMENT > 2)
1181                         ch_bit = TRB_CHAIN;
1182
1183                 link_trb->control = cpu_to_le32(((priv_ep->pcs) ? TRB_CYCLE : 0) |
1184                                     TRB_TYPE(TRB_LINK) | TRB_TOGGLE | ch_bit);
1185         }
1186
1187         if (priv_dev->dev_ver <= DEV_VER_V2)
1188                 togle_pcs = cdns3_wa1_update_guard(priv_ep, trb);
1189
1190         if (sg_supported)
1191                 s = request->sg;
1192
1193         /* set incorrect Cycle Bit for first trb*/
1194         control = priv_ep->pcs ? 0 : TRB_CYCLE;
1195         trb->length = 0;
1196         if (priv_dev->dev_ver >= DEV_VER_V2) {
1197                 u16 td_size;
1198
1199                 td_size = DIV_ROUND_UP(request->length,
1200                                        priv_ep->endpoint.maxpacket);
1201                 if (priv_dev->gadget.speed == USB_SPEED_SUPER)
1202                         trb->length = cpu_to_le32(TRB_TDL_SS_SIZE(td_size));
1203                 else
1204                         control |= TRB_TDL_HS_SIZE(td_size);
1205         }
1206
1207         do {
1208                 u32 length;
1209
1210                 /* fill TRB */
1211                 control |= TRB_TYPE(TRB_NORMAL);
1212                 if (sg_supported) {
1213                         trb->buffer = cpu_to_le32(TRB_BUFFER(sg_dma_address(s)));
1214                         length = sg_dma_len(s);
1215                 } else {
1216                         trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma));
1217                         length = request->length;
1218                 }
1219
1220                 if (priv_ep->flags & EP_TDLCHK_EN)
1221                         total_tdl += DIV_ROUND_UP(length,
1222                                                priv_ep->endpoint.maxpacket);
1223
1224                 trb->length |= cpu_to_le32(TRB_BURST_LEN(priv_ep->trb_burst_size) |
1225                                         TRB_LEN(length));
1226                 pcs = priv_ep->pcs ? TRB_CYCLE : 0;
1227
1228                 /*
1229                  * first trb should be prepared as last to avoid processing
1230                  *  transfer to early
1231                  */
1232                 if (sg_iter != 0)
1233                         control |= pcs;
1234
1235                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir) {
1236                         control |= TRB_IOC | TRB_ISP;
1237                 } else {
1238                         /* for last element in TD or in SG list */
1239                         if (sg_iter == (num_trb - 1) && sg_iter != 0)
1240                                 control |= pcs | TRB_IOC | TRB_ISP;
1241                 }
1242
1243                 if (sg_iter)
1244                         trb->control = cpu_to_le32(control);
1245                 else
1246                         priv_req->trb->control = cpu_to_le32(control);
1247
1248                 if (sg_supported) {
1249                         trb->control |= cpu_to_le32(TRB_ISP);
1250                         /* Don't set chain bit for last TRB */
1251                         if (sg_iter < num_trb - 1)
1252                                 trb->control |= cpu_to_le32(TRB_CHAIN);
1253
1254                         s = sg_next(s);
1255                 }
1256
1257                 control = 0;
1258                 ++sg_iter;
1259                 priv_req->end_trb = priv_ep->enqueue;
1260                 cdns3_ep_inc_enq(priv_ep);
1261                 trb = priv_ep->trb_pool + priv_ep->enqueue;
1262                 trb->length = 0;
1263         } while (sg_iter < num_trb);
1264
1265         trb = priv_req->trb;
1266
1267         priv_req->flags |= REQUEST_PENDING;
1268         priv_req->num_of_trb = num_trb;
1269
1270         if (sg_iter == 1)
1271                 trb->control |= cpu_to_le32(TRB_IOC | TRB_ISP);
1272
1273         if (priv_dev->dev_ver < DEV_VER_V2 &&
1274             (priv_ep->flags & EP_TDLCHK_EN)) {
1275                 u16 tdl = total_tdl;
1276                 u16 old_tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));
1277
1278                 if (tdl > EP_CMD_TDL_MAX) {
1279                         tdl = EP_CMD_TDL_MAX;
1280                         priv_ep->pending_tdl = total_tdl - EP_CMD_TDL_MAX;
1281                 }
1282
1283                 if (old_tdl < tdl) {
1284                         tdl -= old_tdl;
1285                         writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL,
1286                                &priv_dev->regs->ep_cmd);
1287                 }
1288         }
1289
1290         /*
1291          * Memory barrier - cycle bit must be set before other filds in trb.
1292          */
1293         wmb();
1294
1295         /* give the TD to the consumer*/
1296         if (togle_pcs)
1297                 trb->control = trb->control ^ cpu_to_le32(1);
1298
1299         if (priv_dev->dev_ver <= DEV_VER_V2)
1300                 cdns3_wa1_tray_restore_cycle_bit(priv_dev, priv_ep);
1301
1302         if (num_trb > 1) {
1303                 int i = 0;
1304
1305                 while (i < num_trb) {
1306                         trace_cdns3_prepare_trb(priv_ep, trb + i);
1307                         if (trb + i == link_trb) {
1308                                 trb = priv_ep->trb_pool;
1309                                 num_trb = num_trb - i;
1310                                 i = 0;
1311                         } else {
1312                                 i++;
1313                         }
1314                 }
1315         } else {
1316                 trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
1317         }
1318
1319         /*
1320          * Memory barrier - Cycle Bit must be set before trb->length  and
1321          * trb->buffer fields.
1322          */
1323         wmb();
1324
1325         /*
1326          * For DMULT mode we can set address to transfer ring only once after
1327          * enabling endpoint.
1328          */
1329         if (priv_ep->flags & EP_UPDATE_EP_TRBADDR) {
1330                 /*
1331                  * Until SW is not ready to handle the OUT transfer the ISO OUT
1332                  * Endpoint should be disabled (EP_CFG.ENABLE = 0).
1333                  * EP_CFG_ENABLE must be set before updating ep_traddr.
1334                  */
1335                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir &&
1336                     !(priv_ep->flags & EP_QUIRK_ISO_OUT_EN)) {
1337                         priv_ep->flags |= EP_QUIRK_ISO_OUT_EN;
1338                         cdns3_set_register_bit(&priv_dev->regs->ep_cfg,
1339                                                EP_CFG_ENABLE);
1340                 }
1341
1342                 writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma +
1343                                         priv_req->start_trb * TRB_SIZE),
1344                                         &priv_dev->regs->ep_traddr);
1345
1346                 priv_ep->flags &= ~EP_UPDATE_EP_TRBADDR;
1347         }
1348
1349         if (!priv_ep->wa1_set && !(priv_ep->flags & EP_STALLED)) {
1350                 trace_cdns3_ring(priv_ep);
1351                 /*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/
1352                 writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);
1353                 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1354                 cdns3_rearm_drdy_if_needed(priv_ep);
1355                 trace_cdns3_doorbell_epx(priv_ep->name,
1356                                          readl(&priv_dev->regs->ep_traddr));
1357         }
1358
1359         /* WORKAROUND for transition to L0 */
1360         __cdns3_gadget_wakeup(priv_dev);
1361
1362         return 0;
1363 }
1364
1365 void cdns3_set_hw_configuration(struct cdns3_device *priv_dev)
1366 {
1367         struct cdns3_endpoint *priv_ep;
1368         struct usb_ep *ep;
1369
1370         if (priv_dev->hw_configured_flag)
1371                 return;
1372
1373         writel(USB_CONF_CFGSET, &priv_dev->regs->usb_conf);
1374
1375         cdns3_set_register_bit(&priv_dev->regs->usb_conf,
1376                                USB_CONF_U1EN | USB_CONF_U2EN);
1377
1378         priv_dev->hw_configured_flag = 1;
1379
1380         list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
1381                 if (ep->enabled) {
1382                         priv_ep = ep_to_cdns3_ep(ep);
1383                         cdns3_start_all_request(priv_dev, priv_ep);
1384                 }
1385         }
1386
1387         cdns3_allow_enable_l1(priv_dev, 1);
1388 }
1389
1390 /**
1391  * cdns3_trb_handled - check whether trb has been handled by DMA
1392  *
1393  * @priv_ep: extended endpoint object.
1394  * @priv_req: request object for checking
1395  *
1396  * Endpoint must be selected before invoking this function.
1397  *
1398  * Returns false if request has not been handled by DMA, else returns true.
1399  *
1400  * SR - start ring
1401  * ER -  end ring
1402  * DQ = priv_ep->dequeue - dequeue position
1403  * EQ = priv_ep->enqueue -  enqueue position
1404  * ST = priv_req->start_trb - index of first TRB in transfer ring
1405  * ET = priv_req->end_trb - index of last TRB in transfer ring
1406  * CI = current_index - index of processed TRB by DMA.
1407  *
1408  * As first step, we check if the TRB between the ST and ET.
1409  * Then, we check if cycle bit for index priv_ep->dequeue
1410  * is correct.
1411  *
1412  * some rules:
1413  * 1. priv_ep->dequeue never equals to current_index.
1414  * 2  priv_ep->enqueue never exceed priv_ep->dequeue
1415  * 3. exception: priv_ep->enqueue == priv_ep->dequeue
1416  *    and priv_ep->free_trbs is zero.
1417  *    This case indicate that TR is full.
1418  *
1419  * At below two cases, the request have been handled.
1420  * Case 1 - priv_ep->dequeue < current_index
1421  *      SR ... EQ ... DQ ... CI ... ER
1422  *      SR ... DQ ... CI ... EQ ... ER
1423  *
1424  * Case 2 - priv_ep->dequeue > current_index
1425  * This situation takes place when CI go through the LINK TRB at the end of
1426  * transfer ring.
1427  *      SR ... CI ... EQ ... DQ ... ER
1428  */
1429 static bool cdns3_trb_handled(struct cdns3_endpoint *priv_ep,
1430                                   struct cdns3_request *priv_req)
1431 {
1432         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1433         struct cdns3_trb *trb;
1434         int current_index = 0;
1435         int handled = 0;
1436         int doorbell;
1437
1438         current_index = cdns3_get_dma_pos(priv_dev, priv_ep);
1439         doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
1440
1441         /* current trb doesn't belong to this request */
1442         if (priv_req->start_trb < priv_req->end_trb) {
1443                 if (priv_ep->dequeue > priv_req->end_trb)
1444                         goto finish;
1445
1446                 if (priv_ep->dequeue < priv_req->start_trb)
1447                         goto finish;
1448         }
1449
1450         if ((priv_req->start_trb > priv_req->end_trb) &&
1451                 (priv_ep->dequeue > priv_req->end_trb) &&
1452                 (priv_ep->dequeue < priv_req->start_trb))
1453                 goto finish;
1454
1455         if ((priv_req->start_trb == priv_req->end_trb) &&
1456                 (priv_ep->dequeue != priv_req->end_trb))
1457                 goto finish;
1458
1459         trb = &priv_ep->trb_pool[priv_ep->dequeue];
1460
1461         if ((le32_to_cpu(trb->control) & TRB_CYCLE) != priv_ep->ccs)
1462                 goto finish;
1463
1464         if (doorbell == 1 && current_index == priv_ep->dequeue)
1465                 goto finish;
1466
1467         /* The corner case for TRBS_PER_SEGMENT equal 2). */
1468         if (TRBS_PER_SEGMENT == 2 && priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
1469                 handled = 1;
1470                 goto finish;
1471         }
1472
1473         if (priv_ep->enqueue == priv_ep->dequeue &&
1474             priv_ep->free_trbs == 0) {
1475                 handled = 1;
1476         } else if (priv_ep->dequeue < current_index) {
1477                 if ((current_index == (priv_ep->num_trbs - 1)) &&
1478                     !priv_ep->dequeue)
1479                         goto finish;
1480
1481                 handled = 1;
1482         } else if (priv_ep->dequeue  > current_index) {
1483                         handled = 1;
1484         }
1485
1486 finish:
1487         trace_cdns3_request_handled(priv_req, current_index, handled);
1488
1489         return handled;
1490 }
1491
1492 static void cdns3_transfer_completed(struct cdns3_device *priv_dev,
1493                                      struct cdns3_endpoint *priv_ep)
1494 {
1495         struct cdns3_request *priv_req;
1496         struct usb_request *request;
1497         struct cdns3_trb *trb;
1498         bool request_handled = false;
1499         bool transfer_end = false;
1500
1501         while (!list_empty(&priv_ep->pending_req_list)) {
1502                 request = cdns3_next_request(&priv_ep->pending_req_list);
1503                 priv_req = to_cdns3_request(request);
1504
1505                 trb = priv_ep->trb_pool + priv_ep->dequeue;
1506
1507                 /* The TRB was changed as link TRB, and the request was handled at ep_dequeue */
1508                 while (TRB_FIELD_TO_TYPE(le32_to_cpu(trb->control)) == TRB_LINK) {
1509                         trace_cdns3_complete_trb(priv_ep, trb);
1510                         cdns3_ep_inc_deq(priv_ep);
1511                         trb = priv_ep->trb_pool + priv_ep->dequeue;
1512                 }
1513
1514                 if (!request->stream_id) {
1515                         /* Re-select endpoint. It could be changed by other CPU
1516                          * during handling usb_gadget_giveback_request.
1517                          */
1518                         cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1519
1520                         while (cdns3_trb_handled(priv_ep, priv_req)) {
1521                                 priv_req->finished_trb++;
1522                                 if (priv_req->finished_trb >= priv_req->num_of_trb)
1523                                         request_handled = true;
1524
1525                                 trb = priv_ep->trb_pool + priv_ep->dequeue;
1526                                 trace_cdns3_complete_trb(priv_ep, trb);
1527
1528                                 if (!transfer_end)
1529                                         request->actual +=
1530                                                 TRB_LEN(le32_to_cpu(trb->length));
1531
1532                                 if (priv_req->num_of_trb > 1 &&
1533                                         le32_to_cpu(trb->control) & TRB_SMM &&
1534                                         le32_to_cpu(trb->control) & TRB_CHAIN)
1535                                         transfer_end = true;
1536
1537                                 cdns3_ep_inc_deq(priv_ep);
1538                         }
1539
1540                         if (request_handled) {
1541                                 cdns3_gadget_giveback(priv_ep, priv_req, 0);
1542                                 request_handled = false;
1543                                 transfer_end = false;
1544                         } else {
1545                                 goto prepare_next_td;
1546                         }
1547
1548                         if (priv_ep->type != USB_ENDPOINT_XFER_ISOC &&
1549                             TRBS_PER_SEGMENT == 2)
1550                                 break;
1551                 } else {
1552                         /* Re-select endpoint. It could be changed by other CPU
1553                          * during handling usb_gadget_giveback_request.
1554                          */
1555                         cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1556
1557                         trb = priv_ep->trb_pool;
1558                         trace_cdns3_complete_trb(priv_ep, trb);
1559
1560                         if (trb != priv_req->trb)
1561                                 dev_warn(priv_dev->dev,
1562                                          "request_trb=0x%p, queue_trb=0x%p\n",
1563                                          priv_req->trb, trb);
1564
1565                         request->actual += TRB_LEN(le32_to_cpu(trb->length));
1566
1567                         if (!request->num_sgs ||
1568                             (request->num_sgs == (priv_ep->stream_sg_idx + 1))) {
1569                                 priv_ep->stream_sg_idx = 0;
1570                                 cdns3_gadget_giveback(priv_ep, priv_req, 0);
1571                         } else {
1572                                 priv_ep->stream_sg_idx++;
1573                                 cdns3_ep_run_stream_transfer(priv_ep, request);
1574                         }
1575                         break;
1576                 }
1577         }
1578         priv_ep->flags &= ~EP_PENDING_REQUEST;
1579
1580 prepare_next_td:
1581         if (!(priv_ep->flags & EP_STALLED) &&
1582             !(priv_ep->flags & EP_STALL_PENDING))
1583                 cdns3_start_all_request(priv_dev, priv_ep);
1584 }
1585
1586 void cdns3_rearm_transfer(struct cdns3_endpoint *priv_ep, u8 rearm)
1587 {
1588         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1589
1590         cdns3_wa1_restore_cycle_bit(priv_ep);
1591
1592         if (rearm) {
1593                 trace_cdns3_ring(priv_ep);
1594
1595                 /* Cycle Bit must be updated before arming DMA. */
1596                 wmb();
1597                 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1598
1599                 __cdns3_gadget_wakeup(priv_dev);
1600
1601                 trace_cdns3_doorbell_epx(priv_ep->name,
1602                                          readl(&priv_dev->regs->ep_traddr));
1603         }
1604 }
1605
1606 static void cdns3_reprogram_tdl(struct cdns3_endpoint *priv_ep)
1607 {
1608         u16 tdl = priv_ep->pending_tdl;
1609         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1610
1611         if (tdl > EP_CMD_TDL_MAX) {
1612                 tdl = EP_CMD_TDL_MAX;
1613                 priv_ep->pending_tdl -= EP_CMD_TDL_MAX;
1614         } else {
1615                 priv_ep->pending_tdl = 0;
1616         }
1617
1618         writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL, &priv_dev->regs->ep_cmd);
1619 }
1620
1621 /**
1622  * cdns3_check_ep_interrupt_proceed - Processes interrupt related to endpoint
1623  * @priv_ep: endpoint object
1624  *
1625  * Returns 0
1626  */
1627 static int cdns3_check_ep_interrupt_proceed(struct cdns3_endpoint *priv_ep)
1628 {
1629         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1630         u32 ep_sts_reg;
1631         struct usb_request *deferred_request;
1632         struct usb_request *pending_request;
1633         u32 tdl = 0;
1634
1635         cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1636
1637         trace_cdns3_epx_irq(priv_dev, priv_ep);
1638
1639         ep_sts_reg = readl(&priv_dev->regs->ep_sts);
1640         writel(ep_sts_reg, &priv_dev->regs->ep_sts);
1641
1642         if ((ep_sts_reg & EP_STS_PRIME) && priv_ep->use_streams) {
1643                 bool dbusy = !!(ep_sts_reg & EP_STS_DBUSY);
1644
1645                 tdl = cdns3_get_tdl(priv_dev);
1646
1647                 /*
1648                  * Continue the previous transfer:
1649                  * There is some racing between ERDY and PRIME. The device send
1650                  * ERDY and almost in the same time Host send PRIME. It cause
1651                  * that host ignore the ERDY packet and driver has to send it
1652                  * again.
1653                  */
1654                 if (tdl && (dbusy || !EP_STS_BUFFEMPTY(ep_sts_reg) ||
1655                     EP_STS_HOSTPP(ep_sts_reg))) {
1656                         writel(EP_CMD_ERDY |
1657                                EP_CMD_ERDY_SID(priv_ep->last_stream_id),
1658                                &priv_dev->regs->ep_cmd);
1659                         ep_sts_reg &= ~(EP_STS_MD_EXIT | EP_STS_IOC);
1660                 } else {
1661                         priv_ep->prime_flag = true;
1662
1663                         pending_request = cdns3_next_request(&priv_ep->pending_req_list);
1664                         deferred_request = cdns3_next_request(&priv_ep->deferred_req_list);
1665
1666                         if (deferred_request && !pending_request) {
1667                                 cdns3_start_all_request(priv_dev, priv_ep);
1668                         }
1669                 }
1670         }
1671
1672         if (ep_sts_reg & EP_STS_TRBERR) {
1673                 if (priv_ep->flags & EP_STALL_PENDING &&
1674                     !(ep_sts_reg & EP_STS_DESCMIS &&
1675                     priv_dev->dev_ver < DEV_VER_V2)) {
1676                         cdns3_ep_stall_flush(priv_ep);
1677                 }
1678
1679                 /*
1680                  * For isochronous transfer driver completes request on
1681                  * IOC or on TRBERR. IOC appears only when device receive
1682                  * OUT data packet. If host disable stream or lost some packet
1683                  * then the only way to finish all queued transfer is to do it
1684                  * on TRBERR event.
1685                  */
1686                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC &&
1687                     !priv_ep->wa1_set) {
1688                         if (!priv_ep->dir) {
1689                                 u32 ep_cfg = readl(&priv_dev->regs->ep_cfg);
1690
1691                                 ep_cfg &= ~EP_CFG_ENABLE;
1692                                 writel(ep_cfg, &priv_dev->regs->ep_cfg);
1693                                 priv_ep->flags &= ~EP_QUIRK_ISO_OUT_EN;
1694                                 priv_ep->flags |= EP_UPDATE_EP_TRBADDR;
1695                         }
1696                         cdns3_transfer_completed(priv_dev, priv_ep);
1697                 } else if (!(priv_ep->flags & EP_STALLED) &&
1698                           !(priv_ep->flags & EP_STALL_PENDING)) {
1699                         if (priv_ep->flags & EP_DEFERRED_DRDY) {
1700                                 priv_ep->flags &= ~EP_DEFERRED_DRDY;
1701                                 cdns3_start_all_request(priv_dev, priv_ep);
1702                         } else {
1703                                 cdns3_rearm_transfer(priv_ep,
1704                                                      priv_ep->wa1_set);
1705                         }
1706                 }
1707         }
1708
1709         if ((ep_sts_reg & EP_STS_IOC) || (ep_sts_reg & EP_STS_ISP) ||
1710             (ep_sts_reg & EP_STS_IOT)) {
1711                 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
1712                         if (ep_sts_reg & EP_STS_ISP)
1713                                 priv_ep->flags |= EP_QUIRK_END_TRANSFER;
1714                         else
1715                                 priv_ep->flags &= ~EP_QUIRK_END_TRANSFER;
1716                 }
1717
1718                 if (!priv_ep->use_streams) {
1719                         if ((ep_sts_reg & EP_STS_IOC) ||
1720                             (ep_sts_reg & EP_STS_ISP)) {
1721                                 cdns3_transfer_completed(priv_dev, priv_ep);
1722                         } else if ((priv_ep->flags & EP_TDLCHK_EN) &
1723                                    priv_ep->pending_tdl) {
1724                                 /* handle IOT with pending tdl */
1725                                 cdns3_reprogram_tdl(priv_ep);
1726                         }
1727                 } else if (priv_ep->dir == USB_DIR_OUT) {
1728                         priv_ep->ep_sts_pending |= ep_sts_reg;
1729                 } else if (ep_sts_reg & EP_STS_IOT) {
1730                         cdns3_transfer_completed(priv_dev, priv_ep);
1731                 }
1732         }
1733
1734         /*
1735          * MD_EXIT interrupt sets when stream capable endpoint exits
1736          * from MOVE DATA state of Bulk IN/OUT stream protocol state machine
1737          */
1738         if (priv_ep->dir == USB_DIR_OUT && (ep_sts_reg & EP_STS_MD_EXIT) &&
1739             (priv_ep->ep_sts_pending & EP_STS_IOT) && priv_ep->use_streams) {
1740                 priv_ep->ep_sts_pending = 0;
1741                 cdns3_transfer_completed(priv_dev, priv_ep);
1742         }
1743
1744         /*
1745          * WA2: this condition should only be meet when
1746          * priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET or
1747          * priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN.
1748          * In other cases this interrupt will be disabled.
1749          */
1750         if (ep_sts_reg & EP_STS_DESCMIS && priv_dev->dev_ver < DEV_VER_V2 &&
1751             !(priv_ep->flags & EP_STALLED))
1752                 cdns3_wa2_descmissing_packet(priv_ep);
1753
1754         return 0;
1755 }
1756
1757 static void cdns3_disconnect_gadget(struct cdns3_device *priv_dev)
1758 {
1759         if (priv_dev->gadget_driver && priv_dev->gadget_driver->disconnect)
1760                 priv_dev->gadget_driver->disconnect(&priv_dev->gadget);
1761 }
1762
1763 /**
1764  * cdns3_check_usb_interrupt_proceed - Processes interrupt related to device
1765  * @priv_dev: extended gadget object
1766  * @usb_ists: bitmap representation of device's reported interrupts
1767  * (usb_ists register value)
1768  */
1769 static void cdns3_check_usb_interrupt_proceed(struct cdns3_device *priv_dev,
1770                                               u32 usb_ists)
1771 __must_hold(&priv_dev->lock)
1772 {
1773         int speed = 0;
1774
1775         trace_cdns3_usb_irq(priv_dev, usb_ists);
1776         if (usb_ists & USB_ISTS_L1ENTI) {
1777                 /*
1778                  * WORKAROUND: CDNS3 controller has issue with hardware resuming
1779                  * from L1. To fix it, if any DMA transfer is pending driver
1780                  * must starts driving resume signal immediately.
1781                  */
1782                 if (readl(&priv_dev->regs->drbl))
1783                         __cdns3_gadget_wakeup(priv_dev);
1784         }
1785
1786         /* Connection detected */
1787         if (usb_ists & (USB_ISTS_CON2I | USB_ISTS_CONI)) {
1788                 speed = cdns3_get_speed(priv_dev);
1789                 priv_dev->gadget.speed = speed;
1790                 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_POWERED);
1791                 cdns3_ep0_config(priv_dev);
1792         }
1793
1794         /* Disconnection detected */
1795         if (usb_ists & (USB_ISTS_DIS2I | USB_ISTS_DISI)) {
1796                 spin_unlock(&priv_dev->lock);
1797                 cdns3_disconnect_gadget(priv_dev);
1798                 spin_lock(&priv_dev->lock);
1799                 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
1800                 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
1801                 cdns3_hw_reset_eps_config(priv_dev);
1802         }
1803
1804         if (usb_ists & (USB_ISTS_L2ENTI | USB_ISTS_U3ENTI)) {
1805                 if (priv_dev->gadget_driver &&
1806                     priv_dev->gadget_driver->suspend) {
1807                         spin_unlock(&priv_dev->lock);
1808                         priv_dev->gadget_driver->suspend(&priv_dev->gadget);
1809                         spin_lock(&priv_dev->lock);
1810                 }
1811         }
1812
1813         if (usb_ists & (USB_ISTS_L2EXTI | USB_ISTS_U3EXTI)) {
1814                 if (priv_dev->gadget_driver &&
1815                     priv_dev->gadget_driver->resume) {
1816                         spin_unlock(&priv_dev->lock);
1817                         priv_dev->gadget_driver->resume(&priv_dev->gadget);
1818                         spin_lock(&priv_dev->lock);
1819                 }
1820         }
1821
1822         /* reset*/
1823         if (usb_ists & (USB_ISTS_UWRESI | USB_ISTS_UHRESI | USB_ISTS_U2RESI)) {
1824                 if (priv_dev->gadget_driver) {
1825                         spin_unlock(&priv_dev->lock);
1826                         usb_gadget_udc_reset(&priv_dev->gadget,
1827                                              priv_dev->gadget_driver);
1828                         spin_lock(&priv_dev->lock);
1829
1830                         /*read again to check the actual speed*/
1831                         speed = cdns3_get_speed(priv_dev);
1832                         priv_dev->gadget.speed = speed;
1833                         cdns3_hw_reset_eps_config(priv_dev);
1834                         cdns3_ep0_config(priv_dev);
1835                 }
1836         }
1837 }
1838
1839 /**
1840  * cdns3_device_irq_handler - interrupt handler for device part of controller
1841  *
1842  * @irq: irq number for cdns3 core device
1843  * @data: structure of cdns3
1844  *
1845  * Returns IRQ_HANDLED or IRQ_NONE
1846  */
1847 static irqreturn_t cdns3_device_irq_handler(int irq, void *data)
1848 {
1849         struct cdns3_device *priv_dev = data;
1850         struct cdns *cdns = dev_get_drvdata(priv_dev->dev);
1851         irqreturn_t ret = IRQ_NONE;
1852         u32 reg;
1853
1854         if (cdns->in_lpm)
1855                 return ret;
1856
1857         /* check USB device interrupt */
1858         reg = readl(&priv_dev->regs->usb_ists);
1859         if (reg) {
1860                 /* After masking interrupts the new interrupts won't be
1861                  * reported in usb_ists/ep_ists. In order to not lose some
1862                  * of them driver disables only detected interrupts.
1863                  * They will be enabled ASAP after clearing source of
1864                  * interrupt. This an unusual behavior only applies to
1865                  * usb_ists register.
1866                  */
1867                 reg = ~reg & readl(&priv_dev->regs->usb_ien);
1868                 /* mask deferred interrupt. */
1869                 writel(reg, &priv_dev->regs->usb_ien);
1870                 ret = IRQ_WAKE_THREAD;
1871         }
1872
1873         /* check endpoint interrupt */
1874         reg = readl(&priv_dev->regs->ep_ists);
1875         if (reg) {
1876                 writel(0, &priv_dev->regs->ep_ien);
1877                 ret = IRQ_WAKE_THREAD;
1878         }
1879
1880         return ret;
1881 }
1882
1883 /**
1884  * cdns3_device_thread_irq_handler - interrupt handler for device part
1885  * of controller
1886  *
1887  * @irq: irq number for cdns3 core device
1888  * @data: structure of cdns3
1889  *
1890  * Returns IRQ_HANDLED or IRQ_NONE
1891  */
1892 static irqreturn_t cdns3_device_thread_irq_handler(int irq, void *data)
1893 {
1894         struct cdns3_device *priv_dev = data;
1895         irqreturn_t ret = IRQ_NONE;
1896         unsigned long flags;
1897         unsigned int bit;
1898         unsigned long reg;
1899
1900         spin_lock_irqsave(&priv_dev->lock, flags);
1901
1902         reg = readl(&priv_dev->regs->usb_ists);
1903         if (reg) {
1904                 writel(reg, &priv_dev->regs->usb_ists);
1905                 writel(USB_IEN_INIT, &priv_dev->regs->usb_ien);
1906                 cdns3_check_usb_interrupt_proceed(priv_dev, reg);
1907                 ret = IRQ_HANDLED;
1908         }
1909
1910         reg = readl(&priv_dev->regs->ep_ists);
1911
1912         /* handle default endpoint OUT */
1913         if (reg & EP_ISTS_EP_OUT0) {
1914                 cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_OUT);
1915                 ret = IRQ_HANDLED;
1916         }
1917
1918         /* handle default endpoint IN */
1919         if (reg & EP_ISTS_EP_IN0) {
1920                 cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_IN);
1921                 ret = IRQ_HANDLED;
1922         }
1923
1924         /* check if interrupt from non default endpoint, if no exit */
1925         reg &= ~(EP_ISTS_EP_OUT0 | EP_ISTS_EP_IN0);
1926         if (!reg)
1927                 goto irqend;
1928
1929         for_each_set_bit(bit, &reg,
1930                          sizeof(u32) * BITS_PER_BYTE) {
1931                 cdns3_check_ep_interrupt_proceed(priv_dev->eps[bit]);
1932                 ret = IRQ_HANDLED;
1933         }
1934
1935         if (priv_dev->dev_ver < DEV_VER_V2 && priv_dev->using_streams)
1936                 cdns3_wa2_check_outq_status(priv_dev);
1937
1938 irqend:
1939         writel(~0, &priv_dev->regs->ep_ien);
1940         spin_unlock_irqrestore(&priv_dev->lock, flags);
1941
1942         return ret;
1943 }
1944
1945 /**
1946  * cdns3_ep_onchip_buffer_reserve - Try to reserve onchip buf for EP
1947  *
1948  * The real reservation will occur during write to EP_CFG register,
1949  * this function is used to check if the 'size' reservation is allowed.
1950  *
1951  * @priv_dev: extended gadget object
1952  * @size: the size (KB) for EP would like to allocate
1953  * @is_in: endpoint direction
1954  *
1955  * Return 0 if the required size can met or negative value on failure
1956  */
1957 static int cdns3_ep_onchip_buffer_reserve(struct cdns3_device *priv_dev,
1958                                           int size, int is_in)
1959 {
1960         int remained;
1961
1962         /* 2KB are reserved for EP0*/
1963         remained = priv_dev->onchip_buffers - priv_dev->onchip_used_size - 2;
1964
1965         if (is_in) {
1966                 if (remained < size)
1967                         return -EPERM;
1968
1969                 priv_dev->onchip_used_size += size;
1970         } else {
1971                 int required;
1972
1973                 /**
1974                  *  ALL OUT EPs are shared the same chunk onchip memory, so
1975                  * driver checks if it already has assigned enough buffers
1976                  */
1977                 if (priv_dev->out_mem_is_allocated >= size)
1978                         return 0;
1979
1980                 required = size - priv_dev->out_mem_is_allocated;
1981
1982                 if (required > remained)
1983                         return -EPERM;
1984
1985                 priv_dev->out_mem_is_allocated += required;
1986                 priv_dev->onchip_used_size += required;
1987         }
1988
1989         return 0;
1990 }
1991
1992 static void cdns3_configure_dmult(struct cdns3_device *priv_dev,
1993                                   struct cdns3_endpoint *priv_ep)
1994 {
1995         struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
1996
1997         /* For dev_ver > DEV_VER_V2 DMULT is configured per endpoint */
1998         if (priv_dev->dev_ver <= DEV_VER_V2)
1999                 writel(USB_CONF_DMULT, &regs->usb_conf);
2000
2001         if (priv_dev->dev_ver == DEV_VER_V2)
2002                 writel(USB_CONF2_EN_TDL_TRB, &regs->usb_conf2);
2003
2004         if (priv_dev->dev_ver >= DEV_VER_V3 && priv_ep) {
2005                 u32 mask;
2006
2007                 if (priv_ep->dir)
2008                         mask = BIT(priv_ep->num + 16);
2009                 else
2010                         mask = BIT(priv_ep->num);
2011
2012                 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir) {
2013                         cdns3_set_register_bit(&regs->tdl_from_trb, mask);
2014                         cdns3_set_register_bit(&regs->tdl_beh, mask);
2015                         cdns3_set_register_bit(&regs->tdl_beh2, mask);
2016                         cdns3_set_register_bit(&regs->dma_adv_td, mask);
2017                 }
2018
2019                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir)
2020                         cdns3_set_register_bit(&regs->tdl_from_trb, mask);
2021
2022                 cdns3_set_register_bit(&regs->dtrans, mask);
2023         }
2024 }
2025
2026 /**
2027  * cdns3_ep_config - Configure hardware endpoint
2028  * @priv_ep: extended endpoint object
2029  * @enable: set EP_CFG_ENABLE bit in ep_cfg register.
2030  */
2031 int cdns3_ep_config(struct cdns3_endpoint *priv_ep, bool enable)
2032 {
2033         bool is_iso_ep = (priv_ep->type == USB_ENDPOINT_XFER_ISOC);
2034         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2035         u32 bEndpointAddress = priv_ep->num | priv_ep->dir;
2036         u32 max_packet_size = 0;
2037         u8 maxburst = 0;
2038         u32 ep_cfg = 0;
2039         u8 buffering;
2040         u8 mult = 0;
2041         int ret;
2042
2043         buffering = priv_dev->ep_buf_size - 1;
2044
2045         cdns3_configure_dmult(priv_dev, priv_ep);
2046
2047         switch (priv_ep->type) {
2048         case USB_ENDPOINT_XFER_INT:
2049                 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_INT);
2050
2051                 if (priv_dev->dev_ver >= DEV_VER_V2 && !priv_ep->dir)
2052                         ep_cfg |= EP_CFG_TDL_CHK;
2053                 break;
2054         case USB_ENDPOINT_XFER_BULK:
2055                 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_BULK);
2056
2057                 if (priv_dev->dev_ver >= DEV_VER_V2 && !priv_ep->dir)
2058                         ep_cfg |= EP_CFG_TDL_CHK;
2059                 break;
2060         default:
2061                 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_ISOC);
2062                 mult = priv_dev->ep_iso_burst - 1;
2063                 buffering = mult + 1;
2064         }
2065
2066         switch (priv_dev->gadget.speed) {
2067         case USB_SPEED_FULL:
2068                 max_packet_size = is_iso_ep ? 1023 : 64;
2069                 break;
2070         case USB_SPEED_HIGH:
2071                 max_packet_size = is_iso_ep ? 1024 : 512;
2072                 break;
2073         case USB_SPEED_SUPER:
2074                 /* It's limitation that driver assumes in driver. */
2075                 mult = 0;
2076                 max_packet_size = 1024;
2077                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
2078                         maxburst = priv_dev->ep_iso_burst - 1;
2079                         buffering = (mult + 1) *
2080                                     (maxburst + 1);
2081
2082                         if (priv_ep->interval > 1)
2083                                 buffering++;
2084                 } else {
2085                         maxburst = priv_dev->ep_buf_size - 1;
2086                 }
2087                 break;
2088         default:
2089                 /* all other speed are not supported */
2090                 return -EINVAL;
2091         }
2092
2093         if (max_packet_size == 1024)
2094                 priv_ep->trb_burst_size = 128;
2095         else if (max_packet_size >= 512)
2096                 priv_ep->trb_burst_size = 64;
2097         else
2098                 priv_ep->trb_burst_size = 16;
2099
2100         mult = min_t(u8, mult, EP_CFG_MULT_MAX);
2101         buffering = min_t(u8, buffering, EP_CFG_BUFFERING_MAX);
2102         maxburst = min_t(u8, maxburst, EP_CFG_MAXBURST_MAX);
2103
2104         /* onchip buffer is only allocated before configuration */
2105         if (!priv_dev->hw_configured_flag) {
2106                 ret = cdns3_ep_onchip_buffer_reserve(priv_dev, buffering + 1,
2107                                                      !!priv_ep->dir);
2108                 if (ret) {
2109                         dev_err(priv_dev->dev, "onchip mem is full, ep is invalid\n");
2110                         return ret;
2111                 }
2112         }
2113
2114         if (enable)
2115                 ep_cfg |= EP_CFG_ENABLE;
2116
2117         if (priv_ep->use_streams && priv_dev->gadget.speed >= USB_SPEED_SUPER) {
2118                 if (priv_dev->dev_ver >= DEV_VER_V3) {
2119                         u32 mask = BIT(priv_ep->num + (priv_ep->dir ? 16 : 0));
2120
2121                         /*
2122                          * Stream capable endpoints are handled by using ep_tdl
2123                          * register. Other endpoints use TDL from TRB feature.
2124                          */
2125                         cdns3_clear_register_bit(&priv_dev->regs->tdl_from_trb,
2126                                                  mask);
2127                 }
2128
2129                 /*  Enable Stream Bit TDL chk and SID chk */
2130                 ep_cfg |=  EP_CFG_STREAM_EN | EP_CFG_TDL_CHK | EP_CFG_SID_CHK;
2131         }
2132
2133         ep_cfg |= EP_CFG_MAXPKTSIZE(max_packet_size) |
2134                   EP_CFG_MULT(mult) |
2135                   EP_CFG_BUFFERING(buffering) |
2136                   EP_CFG_MAXBURST(maxburst);
2137
2138         cdns3_select_ep(priv_dev, bEndpointAddress);
2139         writel(ep_cfg, &priv_dev->regs->ep_cfg);
2140         priv_ep->flags |= EP_CONFIGURED;
2141
2142         dev_dbg(priv_dev->dev, "Configure %s: with val %08x\n",
2143                 priv_ep->name, ep_cfg);
2144
2145         return 0;
2146 }
2147
2148 /* Find correct direction for HW endpoint according to description */
2149 static int cdns3_ep_dir_is_correct(struct usb_endpoint_descriptor *desc,
2150                                    struct cdns3_endpoint *priv_ep)
2151 {
2152         return (priv_ep->endpoint.caps.dir_in && usb_endpoint_dir_in(desc)) ||
2153                (priv_ep->endpoint.caps.dir_out && usb_endpoint_dir_out(desc));
2154 }
2155
2156 static struct
2157 cdns3_endpoint *cdns3_find_available_ep(struct cdns3_device *priv_dev,
2158                                         struct usb_endpoint_descriptor *desc)
2159 {
2160         struct usb_ep *ep;
2161         struct cdns3_endpoint *priv_ep;
2162
2163         list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
2164                 unsigned long num;
2165                 int ret;
2166                 /* ep name pattern likes epXin or epXout */
2167                 char c[2] = {ep->name[2], '\0'};
2168
2169                 ret = kstrtoul(c, 10, &num);
2170                 if (ret)
2171                         return ERR_PTR(ret);
2172
2173                 priv_ep = ep_to_cdns3_ep(ep);
2174                 if (cdns3_ep_dir_is_correct(desc, priv_ep)) {
2175                         if (!(priv_ep->flags & EP_CLAIMED)) {
2176                                 priv_ep->num  = num;
2177                                 return priv_ep;
2178                         }
2179                 }
2180         }
2181
2182         return ERR_PTR(-ENOENT);
2183 }
2184
2185 /*
2186  *  Cadence IP has one limitation that all endpoints must be configured
2187  * (Type & MaxPacketSize) before setting configuration through hardware
2188  * register, it means we can't change endpoints configuration after
2189  * set_configuration.
2190  *
2191  * This function set EP_CLAIMED flag which is added when the gadget driver
2192  * uses usb_ep_autoconfig to configure specific endpoint;
2193  * When the udc driver receives set_configurion request,
2194  * it goes through all claimed endpoints, and configure all endpoints
2195  * accordingly.
2196  *
2197  * At usb_ep_ops.enable/disable, we only enable and disable endpoint through
2198  * ep_cfg register which can be changed after set_configuration, and do
2199  * some software operation accordingly.
2200  */
2201 static struct
2202 usb_ep *cdns3_gadget_match_ep(struct usb_gadget *gadget,
2203                               struct usb_endpoint_descriptor *desc,
2204                               struct usb_ss_ep_comp_descriptor *comp_desc)
2205 {
2206         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2207         struct cdns3_endpoint *priv_ep;
2208         unsigned long flags;
2209
2210         priv_ep = cdns3_find_available_ep(priv_dev, desc);
2211         if (IS_ERR(priv_ep)) {
2212                 dev_err(priv_dev->dev, "no available ep\n");
2213                 return NULL;
2214         }
2215
2216         dev_dbg(priv_dev->dev, "match endpoint: %s\n", priv_ep->name);
2217
2218         spin_lock_irqsave(&priv_dev->lock, flags);
2219         priv_ep->endpoint.desc = desc;
2220         priv_ep->dir  = usb_endpoint_dir_in(desc) ? USB_DIR_IN : USB_DIR_OUT;
2221         priv_ep->type = usb_endpoint_type(desc);
2222         priv_ep->flags |= EP_CLAIMED;
2223         priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
2224
2225         spin_unlock_irqrestore(&priv_dev->lock, flags);
2226         return &priv_ep->endpoint;
2227 }
2228
2229 /**
2230  * cdns3_gadget_ep_alloc_request - Allocates request
2231  * @ep: endpoint object associated with request
2232  * @gfp_flags: gfp flags
2233  *
2234  * Returns allocated request address, NULL on allocation error
2235  */
2236 struct usb_request *cdns3_gadget_ep_alloc_request(struct usb_ep *ep,
2237                                                   gfp_t gfp_flags)
2238 {
2239         struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2240         struct cdns3_request *priv_req;
2241
2242         priv_req = kzalloc(sizeof(*priv_req), gfp_flags);
2243         if (!priv_req)
2244                 return NULL;
2245
2246         priv_req->priv_ep = priv_ep;
2247
2248         trace_cdns3_alloc_request(priv_req);
2249         return &priv_req->request;
2250 }
2251
2252 /**
2253  * cdns3_gadget_ep_free_request - Free memory occupied by request
2254  * @ep: endpoint object associated with request
2255  * @request: request to free memory
2256  */
2257 void cdns3_gadget_ep_free_request(struct usb_ep *ep,
2258                                   struct usb_request *request)
2259 {
2260         struct cdns3_request *priv_req = to_cdns3_request(request);
2261
2262         if (priv_req->aligned_buf)
2263                 priv_req->aligned_buf->in_use = 0;
2264
2265         trace_cdns3_free_request(priv_req);
2266         kfree(priv_req);
2267 }
2268
2269 /**
2270  * cdns3_gadget_ep_enable - Enable endpoint
2271  * @ep: endpoint object
2272  * @desc: endpoint descriptor
2273  *
2274  * Returns 0 on success, error code elsewhere
2275  */
2276 static int cdns3_gadget_ep_enable(struct usb_ep *ep,
2277                                   const struct usb_endpoint_descriptor *desc)
2278 {
2279         struct cdns3_endpoint *priv_ep;
2280         struct cdns3_device *priv_dev;
2281         const struct usb_ss_ep_comp_descriptor *comp_desc;
2282         u32 reg = EP_STS_EN_TRBERREN;
2283         u32 bEndpointAddress;
2284         unsigned long flags;
2285         int enable = 1;
2286         int ret = 0;
2287         int val;
2288
2289         if (!ep) {
2290                 pr_debug("usbss: ep not configured?\n");
2291                 return -EINVAL;
2292         }
2293
2294         priv_ep = ep_to_cdns3_ep(ep);
2295         priv_dev = priv_ep->cdns3_dev;
2296         comp_desc = priv_ep->endpoint.comp_desc;
2297
2298         if (!desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
2299                 dev_dbg(priv_dev->dev, "usbss: invalid parameters\n");
2300                 return -EINVAL;
2301         }
2302
2303         if (!desc->wMaxPacketSize) {
2304                 dev_err(priv_dev->dev, "usbss: missing wMaxPacketSize\n");
2305                 return -EINVAL;
2306         }
2307
2308         if (dev_WARN_ONCE(priv_dev->dev, priv_ep->flags & EP_ENABLED,
2309                           "%s is already enabled\n", priv_ep->name))
2310                 return 0;
2311
2312         spin_lock_irqsave(&priv_dev->lock, flags);
2313
2314         priv_ep->endpoint.desc = desc;
2315         priv_ep->type = usb_endpoint_type(desc);
2316         priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
2317
2318         if (priv_ep->interval > ISO_MAX_INTERVAL &&
2319             priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
2320                 dev_err(priv_dev->dev, "Driver is limited to %d period\n",
2321                         ISO_MAX_INTERVAL);
2322
2323                 ret =  -EINVAL;
2324                 goto exit;
2325         }
2326
2327         bEndpointAddress = priv_ep->num | priv_ep->dir;
2328         cdns3_select_ep(priv_dev, bEndpointAddress);
2329
2330         /*
2331          * For some versions of controller at some point during ISO OUT traffic
2332          * DMA reads Transfer Ring for the EP which has never got doorbell.
2333          * This issue was detected only on simulation, but to avoid this issue
2334          * driver add protection against it. To fix it driver enable ISO OUT
2335          * endpoint before setting DRBL. This special treatment of ISO OUT
2336          * endpoints are recommended by controller specification.
2337          */
2338         if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir)
2339                 enable = 0;
2340
2341         if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
2342                 /*
2343                  * Enable stream support (SS mode) related interrupts
2344                  * in EP_STS_EN Register
2345                  */
2346                 if (priv_dev->gadget.speed >= USB_SPEED_SUPER) {
2347                         reg |= EP_STS_EN_IOTEN | EP_STS_EN_PRIMEEEN |
2348                                 EP_STS_EN_SIDERREN | EP_STS_EN_MD_EXITEN |
2349                                 EP_STS_EN_STREAMREN;
2350                         priv_ep->use_streams = true;
2351                         ret = cdns3_ep_config(priv_ep, enable);
2352                         priv_dev->using_streams |= true;
2353                 }
2354         } else {
2355                 ret = cdns3_ep_config(priv_ep, enable);
2356         }
2357
2358         if (ret)
2359                 goto exit;
2360
2361         ret = cdns3_allocate_trb_pool(priv_ep);
2362         if (ret)
2363                 goto exit;
2364
2365         bEndpointAddress = priv_ep->num | priv_ep->dir;
2366         cdns3_select_ep(priv_dev, bEndpointAddress);
2367
2368         trace_cdns3_gadget_ep_enable(priv_ep);
2369
2370         writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2371
2372         ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2373                                         !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
2374                                         1, 1000);
2375
2376         if (unlikely(ret)) {
2377                 cdns3_free_trb_pool(priv_ep);
2378                 ret =  -EINVAL;
2379                 goto exit;
2380         }
2381
2382         /* enable interrupt for selected endpoint */
2383         cdns3_set_register_bit(&priv_dev->regs->ep_ien,
2384                                BIT(cdns3_ep_addr_to_index(bEndpointAddress)));
2385
2386         if (priv_dev->dev_ver < DEV_VER_V2)
2387                 cdns3_wa2_enable_detection(priv_dev, priv_ep, reg);
2388
2389         writel(reg, &priv_dev->regs->ep_sts_en);
2390
2391         ep->desc = desc;
2392         priv_ep->flags &= ~(EP_PENDING_REQUEST | EP_STALLED | EP_STALL_PENDING |
2393                             EP_QUIRK_ISO_OUT_EN | EP_QUIRK_EXTRA_BUF_EN);
2394         priv_ep->flags |= EP_ENABLED | EP_UPDATE_EP_TRBADDR;
2395         priv_ep->wa1_set = 0;
2396         priv_ep->enqueue = 0;
2397         priv_ep->dequeue = 0;
2398         reg = readl(&priv_dev->regs->ep_sts);
2399         priv_ep->pcs = !!EP_STS_CCS(reg);
2400         priv_ep->ccs = !!EP_STS_CCS(reg);
2401         /* one TRB is reserved for link TRB used in DMULT mode*/
2402         priv_ep->free_trbs = priv_ep->num_trbs - 1;
2403 exit:
2404         spin_unlock_irqrestore(&priv_dev->lock, flags);
2405
2406         return ret;
2407 }
2408
2409 /**
2410  * cdns3_gadget_ep_disable - Disable endpoint
2411  * @ep: endpoint object
2412  *
2413  * Returns 0 on success, error code elsewhere
2414  */
2415 static int cdns3_gadget_ep_disable(struct usb_ep *ep)
2416 {
2417         struct cdns3_endpoint *priv_ep;
2418         struct cdns3_request *priv_req;
2419         struct cdns3_device *priv_dev;
2420         struct usb_request *request;
2421         unsigned long flags;
2422         int ret = 0;
2423         u32 ep_cfg;
2424         int val;
2425
2426         if (!ep) {
2427                 pr_err("usbss: invalid parameters\n");
2428                 return -EINVAL;
2429         }
2430
2431         priv_ep = ep_to_cdns3_ep(ep);
2432         priv_dev = priv_ep->cdns3_dev;
2433
2434         if (dev_WARN_ONCE(priv_dev->dev, !(priv_ep->flags & EP_ENABLED),
2435                           "%s is already disabled\n", priv_ep->name))
2436                 return 0;
2437
2438         spin_lock_irqsave(&priv_dev->lock, flags);
2439
2440         trace_cdns3_gadget_ep_disable(priv_ep);
2441
2442         cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2443
2444         ep_cfg = readl(&priv_dev->regs->ep_cfg);
2445         ep_cfg &= ~EP_CFG_ENABLE;
2446         writel(ep_cfg, &priv_dev->regs->ep_cfg);
2447
2448         /**
2449          * Driver needs some time before resetting endpoint.
2450          * It need waits for clearing DBUSY bit or for timeout expired.
2451          * 10us is enough time for controller to stop transfer.
2452          */
2453         readl_poll_timeout_atomic(&priv_dev->regs->ep_sts, val,
2454                                   !(val & EP_STS_DBUSY), 1, 10);
2455         writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2456
2457         readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2458                                   !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
2459                                   1, 1000);
2460         if (unlikely(ret))
2461                 dev_err(priv_dev->dev, "Timeout: %s resetting failed.\n",
2462                         priv_ep->name);
2463
2464         while (!list_empty(&priv_ep->pending_req_list)) {
2465                 request = cdns3_next_request(&priv_ep->pending_req_list);
2466
2467                 cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
2468                                       -ESHUTDOWN);
2469         }
2470
2471         while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
2472                 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
2473
2474                 kfree(priv_req->request.buf);
2475                 cdns3_gadget_ep_free_request(&priv_ep->endpoint,
2476                                              &priv_req->request);
2477                 list_del_init(&priv_req->list);
2478                 --priv_ep->wa2_counter;
2479         }
2480
2481         while (!list_empty(&priv_ep->deferred_req_list)) {
2482                 request = cdns3_next_request(&priv_ep->deferred_req_list);
2483
2484                 cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
2485                                       -ESHUTDOWN);
2486         }
2487
2488         priv_ep->descmis_req = NULL;
2489
2490         ep->desc = NULL;
2491         priv_ep->flags &= ~EP_ENABLED;
2492         priv_ep->use_streams = false;
2493
2494         spin_unlock_irqrestore(&priv_dev->lock, flags);
2495
2496         return ret;
2497 }
2498
2499 /**
2500  * __cdns3_gadget_ep_queue - Transfer data on endpoint
2501  * @ep: endpoint object
2502  * @request: request object
2503  * @gfp_flags: gfp flags
2504  *
2505  * Returns 0 on success, error code elsewhere
2506  */
2507 static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
2508                                    struct usb_request *request,
2509                                    gfp_t gfp_flags)
2510 {
2511         struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2512         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2513         struct cdns3_request *priv_req;
2514         int ret = 0;
2515
2516         request->actual = 0;
2517         request->status = -EINPROGRESS;
2518         priv_req = to_cdns3_request(request);
2519         trace_cdns3_ep_queue(priv_req);
2520
2521         if (priv_dev->dev_ver < DEV_VER_V2) {
2522                 ret = cdns3_wa2_gadget_ep_queue(priv_dev, priv_ep,
2523                                                 priv_req);
2524
2525                 if (ret == EINPROGRESS)
2526                         return 0;
2527         }
2528
2529         ret = cdns3_prepare_aligned_request_buf(priv_req);
2530         if (ret < 0)
2531                 return ret;
2532
2533         ret = usb_gadget_map_request_by_dev(priv_dev->sysdev, request,
2534                                             usb_endpoint_dir_in(ep->desc));
2535         if (ret)
2536                 return ret;
2537
2538         list_add_tail(&request->list, &priv_ep->deferred_req_list);
2539
2540         /*
2541          * For stream capable endpoint if prime irq flag is set then only start
2542          * request.
2543          * If hardware endpoint configuration has not been set yet then
2544          * just queue request in deferred list. Transfer will be started in
2545          * cdns3_set_hw_configuration.
2546          */
2547         if (!request->stream_id) {
2548                 if (priv_dev->hw_configured_flag &&
2549                     !(priv_ep->flags & EP_STALLED) &&
2550                     !(priv_ep->flags & EP_STALL_PENDING))
2551                         cdns3_start_all_request(priv_dev, priv_ep);
2552         } else {
2553                 if (priv_dev->hw_configured_flag && priv_ep->prime_flag)
2554                         cdns3_start_all_request(priv_dev, priv_ep);
2555         }
2556
2557         return 0;
2558 }
2559
2560 static int cdns3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
2561                                  gfp_t gfp_flags)
2562 {
2563         struct usb_request *zlp_request;
2564         struct cdns3_endpoint *priv_ep;
2565         struct cdns3_device *priv_dev;
2566         unsigned long flags;
2567         int ret;
2568
2569         if (!request || !ep)
2570                 return -EINVAL;
2571
2572         priv_ep = ep_to_cdns3_ep(ep);
2573         priv_dev = priv_ep->cdns3_dev;
2574
2575         spin_lock_irqsave(&priv_dev->lock, flags);
2576
2577         ret = __cdns3_gadget_ep_queue(ep, request, gfp_flags);
2578
2579         if (ret == 0 && request->zero && request->length &&
2580             (request->length % ep->maxpacket == 0)) {
2581                 struct cdns3_request *priv_req;
2582
2583                 zlp_request = cdns3_gadget_ep_alloc_request(ep, GFP_ATOMIC);
2584                 zlp_request->buf = priv_dev->zlp_buf;
2585                 zlp_request->length = 0;
2586
2587                 priv_req = to_cdns3_request(zlp_request);
2588                 priv_req->flags |= REQUEST_ZLP;
2589
2590                 dev_dbg(priv_dev->dev, "Queuing ZLP for endpoint: %s\n",
2591                         priv_ep->name);
2592                 ret = __cdns3_gadget_ep_queue(ep, zlp_request, gfp_flags);
2593         }
2594
2595         spin_unlock_irqrestore(&priv_dev->lock, flags);
2596         return ret;
2597 }
2598
2599 /**
2600  * cdns3_gadget_ep_dequeue - Remove request from transfer queue
2601  * @ep: endpoint object associated with request
2602  * @request: request object
2603  *
2604  * Returns 0 on success, error code elsewhere
2605  */
2606 int cdns3_gadget_ep_dequeue(struct usb_ep *ep,
2607                             struct usb_request *request)
2608 {
2609         struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2610         struct cdns3_device *priv_dev;
2611         struct usb_request *req, *req_temp;
2612         struct cdns3_request *priv_req;
2613         struct cdns3_trb *link_trb;
2614         u8 req_on_hw_ring = 0;
2615         unsigned long flags;
2616         int ret = 0;
2617         int val;
2618
2619         if (!ep || !request || !ep->desc)
2620                 return -EINVAL;
2621
2622         priv_dev = priv_ep->cdns3_dev;
2623
2624         spin_lock_irqsave(&priv_dev->lock, flags);
2625
2626         priv_req = to_cdns3_request(request);
2627
2628         trace_cdns3_ep_dequeue(priv_req);
2629
2630         cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2631
2632         list_for_each_entry_safe(req, req_temp, &priv_ep->pending_req_list,
2633                                  list) {
2634                 if (request == req) {
2635                         req_on_hw_ring = 1;
2636                         goto found;
2637                 }
2638         }
2639
2640         list_for_each_entry_safe(req, req_temp, &priv_ep->deferred_req_list,
2641                                  list) {
2642                 if (request == req)
2643                         goto found;
2644         }
2645
2646         goto not_found;
2647
2648 found:
2649         link_trb = priv_req->trb;
2650
2651         /* Update ring only if removed request is on pending_req_list list */
2652         if (req_on_hw_ring && link_trb) {
2653                 /* Stop DMA */
2654                 writel(EP_CMD_DFLUSH, &priv_dev->regs->ep_cmd);
2655
2656                 /* wait for DFLUSH cleared */
2657                 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2658                                           !(val & EP_CMD_DFLUSH), 1, 1000);
2659
2660                 link_trb->buffer = cpu_to_le32(TRB_BUFFER(priv_ep->trb_pool_dma +
2661                         ((priv_req->end_trb + 1) * TRB_SIZE)));
2662                 link_trb->control = cpu_to_le32((le32_to_cpu(link_trb->control) & TRB_CYCLE) |
2663                                     TRB_TYPE(TRB_LINK) | TRB_CHAIN);
2664
2665                 if (priv_ep->wa1_trb == priv_req->trb)
2666                         cdns3_wa1_restore_cycle_bit(priv_ep);
2667         }
2668
2669         cdns3_gadget_giveback(priv_ep, priv_req, -ECONNRESET);
2670
2671         req = cdns3_next_request(&priv_ep->pending_req_list);
2672         if (req)
2673                 cdns3_rearm_transfer(priv_ep, 1);
2674
2675 not_found:
2676         spin_unlock_irqrestore(&priv_dev->lock, flags);
2677         return ret;
2678 }
2679
2680 /**
2681  * __cdns3_gadget_ep_set_halt - Sets stall on selected endpoint
2682  * Should be called after acquiring spin_lock and selecting ep
2683  * @priv_ep: endpoint object to set stall on.
2684  */
2685 void __cdns3_gadget_ep_set_halt(struct cdns3_endpoint *priv_ep)
2686 {
2687         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2688
2689         trace_cdns3_halt(priv_ep, 1, 0);
2690
2691         if (!(priv_ep->flags & EP_STALLED)) {
2692                 u32 ep_sts_reg = readl(&priv_dev->regs->ep_sts);
2693
2694                 if (!(ep_sts_reg & EP_STS_DBUSY))
2695                         cdns3_ep_stall_flush(priv_ep);
2696                 else
2697                         priv_ep->flags |= EP_STALL_PENDING;
2698         }
2699 }
2700
2701 /**
2702  * __cdns3_gadget_ep_clear_halt - Clears stall on selected endpoint
2703  * Should be called after acquiring spin_lock and selecting ep
2704  * @priv_ep: endpoint object to clear stall on
2705  */
2706 int __cdns3_gadget_ep_clear_halt(struct cdns3_endpoint *priv_ep)
2707 {
2708         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2709         struct usb_request *request;
2710         struct cdns3_request *priv_req;
2711         struct cdns3_trb *trb = NULL;
2712         struct cdns3_trb trb_tmp;
2713         int ret;
2714         int val;
2715
2716         trace_cdns3_halt(priv_ep, 0, 0);
2717
2718         request = cdns3_next_request(&priv_ep->pending_req_list);
2719         if (request) {
2720                 priv_req = to_cdns3_request(request);
2721                 trb = priv_req->trb;
2722                 if (trb) {
2723                         trb_tmp = *trb;
2724                         trb->control = trb->control ^ cpu_to_le32(TRB_CYCLE);
2725                 }
2726         }
2727
2728         writel(EP_CMD_CSTALL | EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2729
2730         /* wait for EPRST cleared */
2731         ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2732                                         !(val & EP_CMD_EPRST), 1, 100);
2733         if (ret)
2734                 return -EINVAL;
2735
2736         priv_ep->flags &= ~(EP_STALLED | EP_STALL_PENDING);
2737
2738         if (request) {
2739                 if (trb)
2740                         *trb = trb_tmp;
2741
2742                 cdns3_rearm_transfer(priv_ep, 1);
2743         }
2744
2745         cdns3_start_all_request(priv_dev, priv_ep);
2746         return ret;
2747 }
2748
2749 /**
2750  * cdns3_gadget_ep_set_halt - Sets/clears stall on selected endpoint
2751  * @ep: endpoint object to set/clear stall on
2752  * @value: 1 for set stall, 0 for clear stall
2753  *
2754  * Returns 0 on success, error code elsewhere
2755  */
2756 int cdns3_gadget_ep_set_halt(struct usb_ep *ep, int value)
2757 {
2758         struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2759         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2760         unsigned long flags;
2761         int ret = 0;
2762
2763         if (!(priv_ep->flags & EP_ENABLED))
2764                 return -EPERM;
2765
2766         spin_lock_irqsave(&priv_dev->lock, flags);
2767
2768         cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2769
2770         if (!value) {
2771                 priv_ep->flags &= ~EP_WEDGE;
2772                 ret = __cdns3_gadget_ep_clear_halt(priv_ep);
2773         } else {
2774                 __cdns3_gadget_ep_set_halt(priv_ep);
2775         }
2776
2777         spin_unlock_irqrestore(&priv_dev->lock, flags);
2778
2779         return ret;
2780 }
2781
2782 extern const struct usb_ep_ops cdns3_gadget_ep0_ops;
2783
2784 static const struct usb_ep_ops cdns3_gadget_ep_ops = {
2785         .enable = cdns3_gadget_ep_enable,
2786         .disable = cdns3_gadget_ep_disable,
2787         .alloc_request = cdns3_gadget_ep_alloc_request,
2788         .free_request = cdns3_gadget_ep_free_request,
2789         .queue = cdns3_gadget_ep_queue,
2790         .dequeue = cdns3_gadget_ep_dequeue,
2791         .set_halt = cdns3_gadget_ep_set_halt,
2792         .set_wedge = cdns3_gadget_ep_set_wedge,
2793 };
2794
2795 /**
2796  * cdns3_gadget_get_frame - Returns number of actual ITP frame
2797  * @gadget: gadget object
2798  *
2799  * Returns number of actual ITP frame
2800  */
2801 static int cdns3_gadget_get_frame(struct usb_gadget *gadget)
2802 {
2803         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2804
2805         return readl(&priv_dev->regs->usb_itpn);
2806 }
2807
2808 int __cdns3_gadget_wakeup(struct cdns3_device *priv_dev)
2809 {
2810         enum usb_device_speed speed;
2811
2812         speed = cdns3_get_speed(priv_dev);
2813
2814         if (speed >= USB_SPEED_SUPER)
2815                 return 0;
2816
2817         /* Start driving resume signaling to indicate remote wakeup. */
2818         writel(USB_CONF_LGO_L0, &priv_dev->regs->usb_conf);
2819
2820         return 0;
2821 }
2822
2823 static int cdns3_gadget_wakeup(struct usb_gadget *gadget)
2824 {
2825         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2826         unsigned long flags;
2827         int ret = 0;
2828
2829         spin_lock_irqsave(&priv_dev->lock, flags);
2830         ret = __cdns3_gadget_wakeup(priv_dev);
2831         spin_unlock_irqrestore(&priv_dev->lock, flags);
2832         return ret;
2833 }
2834
2835 static int cdns3_gadget_set_selfpowered(struct usb_gadget *gadget,
2836                                         int is_selfpowered)
2837 {
2838         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2839         unsigned long flags;
2840
2841         spin_lock_irqsave(&priv_dev->lock, flags);
2842         priv_dev->is_selfpowered = !!is_selfpowered;
2843         spin_unlock_irqrestore(&priv_dev->lock, flags);
2844         return 0;
2845 }
2846
2847 static int cdns3_gadget_pullup(struct usb_gadget *gadget, int is_on)
2848 {
2849         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2850
2851         if (is_on) {
2852                 writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf);
2853         } else {
2854                 writel(~0, &priv_dev->regs->ep_ists);
2855                 writel(~0, &priv_dev->regs->usb_ists);
2856                 writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
2857         }
2858
2859         return 0;
2860 }
2861
2862 static void cdns3_gadget_config(struct cdns3_device *priv_dev)
2863 {
2864         struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
2865         u32 reg;
2866
2867         cdns3_ep0_config(priv_dev);
2868
2869         /* enable interrupts for endpoint 0 (in and out) */
2870         writel(EP_IEN_EP_OUT0 | EP_IEN_EP_IN0, &regs->ep_ien);
2871
2872         /*
2873          * Driver needs to modify LFPS minimal U1 Exit time for DEV_VER_TI_V1
2874          * revision of controller.
2875          */
2876         if (priv_dev->dev_ver == DEV_VER_TI_V1) {
2877                 reg = readl(&regs->dbg_link1);
2878
2879                 reg &= ~DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_MASK;
2880                 reg |= DBG_LINK1_LFPS_MIN_GEN_U1_EXIT(0x55) |
2881                        DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_SET;
2882                 writel(reg, &regs->dbg_link1);
2883         }
2884
2885         /*
2886          * By default some platforms has set protected access to memory.
2887          * This cause problem with cache, so driver restore non-secure
2888          * access to memory.
2889          */
2890         reg = readl(&regs->dma_axi_ctrl);
2891         reg |= DMA_AXI_CTRL_MARPROT(DMA_AXI_CTRL_NON_SECURE) |
2892                DMA_AXI_CTRL_MAWPROT(DMA_AXI_CTRL_NON_SECURE);
2893         writel(reg, &regs->dma_axi_ctrl);
2894
2895         /* enable generic interrupt*/
2896         writel(USB_IEN_INIT, &regs->usb_ien);
2897         writel(USB_CONF_CLK2OFFDS | USB_CONF_L1DS, &regs->usb_conf);
2898         /*  keep Fast Access bit */
2899         writel(PUSB_PWR_FST_REG_ACCESS, &priv_dev->regs->usb_pwr);
2900
2901         cdns3_configure_dmult(priv_dev, NULL);
2902 }
2903
2904 /**
2905  * cdns3_gadget_udc_start - Gadget start
2906  * @gadget: gadget object
2907  * @driver: driver which operates on this gadget
2908  *
2909  * Returns 0 on success, error code elsewhere
2910  */
2911 static int cdns3_gadget_udc_start(struct usb_gadget *gadget,
2912                                   struct usb_gadget_driver *driver)
2913 {
2914         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2915         unsigned long flags;
2916         enum usb_device_speed max_speed = driver->max_speed;
2917
2918         spin_lock_irqsave(&priv_dev->lock, flags);
2919         priv_dev->gadget_driver = driver;
2920
2921         /* limit speed if necessary */
2922         max_speed = min(driver->max_speed, gadget->max_speed);
2923
2924         switch (max_speed) {
2925         case USB_SPEED_FULL:
2926                 writel(USB_CONF_SFORCE_FS, &priv_dev->regs->usb_conf);
2927                 writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
2928                 break;
2929         case USB_SPEED_HIGH:
2930                 writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
2931                 break;
2932         case USB_SPEED_SUPER:
2933                 break;
2934         default:
2935                 dev_err(priv_dev->dev,
2936                         "invalid maximum_speed parameter %d\n",
2937                         max_speed);
2938                 fallthrough;
2939         case USB_SPEED_UNKNOWN:
2940                 /* default to superspeed */
2941                 max_speed = USB_SPEED_SUPER;
2942                 break;
2943         }
2944
2945         cdns3_gadget_config(priv_dev);
2946         spin_unlock_irqrestore(&priv_dev->lock, flags);
2947         return 0;
2948 }
2949
2950 /**
2951  * cdns3_gadget_udc_stop - Stops gadget
2952  * @gadget: gadget object
2953  *
2954  * Returns 0
2955  */
2956 static int cdns3_gadget_udc_stop(struct usb_gadget *gadget)
2957 {
2958         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2959         struct cdns3_endpoint *priv_ep;
2960         u32 bEndpointAddress;
2961         struct usb_ep *ep;
2962         int val;
2963
2964         priv_dev->gadget_driver = NULL;
2965
2966         priv_dev->onchip_used_size = 0;
2967         priv_dev->out_mem_is_allocated = 0;
2968         priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
2969
2970         list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
2971                 priv_ep = ep_to_cdns3_ep(ep);
2972                 bEndpointAddress = priv_ep->num | priv_ep->dir;
2973                 cdns3_select_ep(priv_dev, bEndpointAddress);
2974                 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2975                 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2976                                           !(val & EP_CMD_EPRST), 1, 100);
2977
2978                 priv_ep->flags &= ~EP_CLAIMED;
2979         }
2980
2981         /* disable interrupt for device */
2982         writel(0, &priv_dev->regs->usb_ien);
2983         writel(0, &priv_dev->regs->usb_pwr);
2984         writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
2985
2986         return 0;
2987 }
2988
2989 /**
2990  * cdns3_gadget_check_config - ensure cdns3 can support the USB configuration
2991  * @gadget: pointer to the USB gadget
2992  *
2993  * Used to record the maximum number of endpoints being used in a USB composite
2994  * device. (across all configurations)  This is to be used in the calculation
2995  * of the TXFIFO sizes when resizing internal memory for individual endpoints.
2996  * It will help ensured that the resizing logic reserves enough space for at
2997  * least one max packet.
2998  */
2999 static int cdns3_gadget_check_config(struct usb_gadget *gadget)
3000 {
3001         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
3002         struct usb_ep *ep;
3003         int n_in = 0;
3004         int total;
3005
3006         list_for_each_entry(ep, &gadget->ep_list, ep_list) {
3007                 if (ep->claimed && (ep->address & USB_DIR_IN))
3008                         n_in++;
3009         }
3010
3011         /* 2KB are reserved for EP0, 1KB for out*/
3012         total = 2 + n_in + 1;
3013
3014         if (total > priv_dev->onchip_buffers)
3015                 return -ENOMEM;
3016
3017         priv_dev->ep_buf_size = priv_dev->ep_iso_burst =
3018                         (priv_dev->onchip_buffers - 2) / (n_in + 1);
3019
3020         return 0;
3021 }
3022
3023 static const struct usb_gadget_ops cdns3_gadget_ops = {
3024         .get_frame = cdns3_gadget_get_frame,
3025         .wakeup = cdns3_gadget_wakeup,
3026         .set_selfpowered = cdns3_gadget_set_selfpowered,
3027         .pullup = cdns3_gadget_pullup,
3028         .udc_start = cdns3_gadget_udc_start,
3029         .udc_stop = cdns3_gadget_udc_stop,
3030         .match_ep = cdns3_gadget_match_ep,
3031         .check_config = cdns3_gadget_check_config,
3032 };
3033
3034 static void cdns3_free_all_eps(struct cdns3_device *priv_dev)
3035 {
3036         int i;
3037
3038         /* ep0 OUT point to ep0 IN. */
3039         priv_dev->eps[16] = NULL;
3040
3041         for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++)
3042                 if (priv_dev->eps[i]) {
3043                         cdns3_free_trb_pool(priv_dev->eps[i]);
3044                         devm_kfree(priv_dev->dev, priv_dev->eps[i]);
3045                 }
3046 }
3047
3048 /**
3049  * cdns3_init_eps - Initializes software endpoints of gadget
3050  * @priv_dev: extended gadget object
3051  *
3052  * Returns 0 on success, error code elsewhere
3053  */
3054 static int cdns3_init_eps(struct cdns3_device *priv_dev)
3055 {
3056         u32 ep_enabled_reg, iso_ep_reg;
3057         struct cdns3_endpoint *priv_ep;
3058         int ep_dir, ep_number;
3059         u32 ep_mask;
3060         int ret = 0;
3061         int i;
3062
3063         /* Read it from USB_CAP3 to USB_CAP5 */
3064         ep_enabled_reg = readl(&priv_dev->regs->usb_cap3);
3065         iso_ep_reg = readl(&priv_dev->regs->usb_cap4);
3066
3067         dev_dbg(priv_dev->dev, "Initializing non-zero endpoints\n");
3068
3069         for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++) {
3070                 ep_dir = i >> 4;        /* i div 16 */
3071                 ep_number = i & 0xF;    /* i % 16 */
3072                 ep_mask = BIT(i);
3073
3074                 if (!(ep_enabled_reg & ep_mask))
3075                         continue;
3076
3077                 if (ep_dir && !ep_number) {
3078                         priv_dev->eps[i] = priv_dev->eps[0];
3079                         continue;
3080                 }
3081
3082                 priv_ep = devm_kzalloc(priv_dev->dev, sizeof(*priv_ep),
3083                                        GFP_KERNEL);
3084                 if (!priv_ep)
3085                         goto err;
3086
3087                 /* set parent of endpoint object */
3088                 priv_ep->cdns3_dev = priv_dev;
3089                 priv_dev->eps[i] = priv_ep;
3090                 priv_ep->num = ep_number;
3091                 priv_ep->dir = ep_dir ? USB_DIR_IN : USB_DIR_OUT;
3092
3093                 if (!ep_number) {
3094                         ret = cdns3_init_ep0(priv_dev, priv_ep);
3095                         if (ret) {
3096                                 dev_err(priv_dev->dev, "Failed to init ep0\n");
3097                                 goto err;
3098                         }
3099                 } else {
3100                         snprintf(priv_ep->name, sizeof(priv_ep->name), "ep%d%s",
3101                                  ep_number, !!ep_dir ? "in" : "out");
3102                         priv_ep->endpoint.name = priv_ep->name;
3103
3104                         usb_ep_set_maxpacket_limit(&priv_ep->endpoint,
3105                                                    CDNS3_EP_MAX_PACKET_LIMIT);
3106                         priv_ep->endpoint.max_streams = CDNS3_EP_MAX_STREAMS;
3107                         priv_ep->endpoint.ops = &cdns3_gadget_ep_ops;
3108                         if (ep_dir)
3109                                 priv_ep->endpoint.caps.dir_in = 1;
3110                         else
3111                                 priv_ep->endpoint.caps.dir_out = 1;
3112
3113                         if (iso_ep_reg & ep_mask)
3114                                 priv_ep->endpoint.caps.type_iso = 1;
3115
3116                         priv_ep->endpoint.caps.type_bulk = 1;
3117                         priv_ep->endpoint.caps.type_int = 1;
3118
3119                         list_add_tail(&priv_ep->endpoint.ep_list,
3120                                       &priv_dev->gadget.ep_list);
3121                 }
3122
3123                 priv_ep->flags = 0;
3124
3125                 dev_dbg(priv_dev->dev, "Initialized  %s support: %s %s\n",
3126                          priv_ep->name,
3127                          priv_ep->endpoint.caps.type_bulk ? "BULK, INT" : "",
3128                          priv_ep->endpoint.caps.type_iso ? "ISO" : "");
3129
3130                 INIT_LIST_HEAD(&priv_ep->pending_req_list);
3131                 INIT_LIST_HEAD(&priv_ep->deferred_req_list);
3132                 INIT_LIST_HEAD(&priv_ep->wa2_descmiss_req_list);
3133         }
3134
3135         return 0;
3136 err:
3137         cdns3_free_all_eps(priv_dev);
3138         return -ENOMEM;
3139 }
3140
3141 static void cdns3_gadget_release(struct device *dev)
3142 {
3143         struct cdns3_device *priv_dev = container_of(dev,
3144                         struct cdns3_device, gadget.dev);
3145
3146         kfree(priv_dev);
3147 }
3148
3149 static void cdns3_gadget_exit(struct cdns *cdns)
3150 {
3151         struct cdns3_device *priv_dev;
3152
3153         priv_dev = cdns->gadget_dev;
3154
3155
3156         pm_runtime_mark_last_busy(cdns->dev);
3157         pm_runtime_put_autosuspend(cdns->dev);
3158
3159         usb_del_gadget(&priv_dev->gadget);
3160         devm_free_irq(cdns->dev, cdns->dev_irq, priv_dev);
3161
3162         cdns3_free_all_eps(priv_dev);
3163
3164         while (!list_empty(&priv_dev->aligned_buf_list)) {
3165                 struct cdns3_aligned_buf *buf;
3166
3167                 buf = cdns3_next_align_buf(&priv_dev->aligned_buf_list);
3168                 dma_free_noncoherent(priv_dev->sysdev, buf->size,
3169                                   buf->buf,
3170                                   buf->dma,
3171                                   buf->dir);
3172
3173                 list_del(&buf->list);
3174                 kfree(buf);
3175         }
3176
3177         dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,
3178                           priv_dev->setup_dma);
3179         dma_pool_destroy(priv_dev->eps_dma_pool);
3180
3181         kfree(priv_dev->zlp_buf);
3182         usb_put_gadget(&priv_dev->gadget);
3183         cdns->gadget_dev = NULL;
3184         cdns_drd_gadget_off(cdns);
3185 }
3186
3187 static int cdns3_gadget_start(struct cdns *cdns)
3188 {
3189         struct cdns3_device *priv_dev;
3190         u32 max_speed;
3191         int ret;
3192
3193         priv_dev = kzalloc(sizeof(*priv_dev), GFP_KERNEL);
3194         if (!priv_dev)
3195                 return -ENOMEM;
3196
3197         usb_initialize_gadget(cdns->dev, &priv_dev->gadget,
3198                         cdns3_gadget_release);
3199         cdns->gadget_dev = priv_dev;
3200         priv_dev->sysdev = cdns->dev;
3201         priv_dev->dev = cdns->dev;
3202         priv_dev->regs = cdns->dev_regs;
3203
3204         device_property_read_u16(priv_dev->dev, "cdns,on-chip-buff-size",
3205                                  &priv_dev->onchip_buffers);
3206
3207         if (priv_dev->onchip_buffers <=  0) {
3208                 u32 reg = readl(&priv_dev->regs->usb_cap2);
3209
3210                 priv_dev->onchip_buffers = USB_CAP2_ACTUAL_MEM_SIZE(reg);
3211         }
3212
3213         if (!priv_dev->onchip_buffers)
3214                 priv_dev->onchip_buffers = 256;
3215
3216         max_speed = usb_get_maximum_speed(cdns->dev);
3217
3218         /* Check the maximum_speed parameter */
3219         switch (max_speed) {
3220         case USB_SPEED_FULL:
3221         case USB_SPEED_HIGH:
3222         case USB_SPEED_SUPER:
3223                 break;
3224         default:
3225                 dev_err(cdns->dev, "invalid maximum_speed parameter %d\n",
3226                         max_speed);
3227                 fallthrough;
3228         case USB_SPEED_UNKNOWN:
3229                 /* default to superspeed */
3230                 max_speed = USB_SPEED_SUPER;
3231                 break;
3232         }
3233
3234         /* fill gadget fields */
3235         priv_dev->gadget.max_speed = max_speed;
3236         priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
3237         priv_dev->gadget.ops = &cdns3_gadget_ops;
3238         priv_dev->gadget.name = "usb-ss-gadget";
3239         priv_dev->gadget.quirk_avoids_skb_reserve = 1;
3240         priv_dev->gadget.irq = cdns->dev_irq;
3241
3242         spin_lock_init(&priv_dev->lock);
3243         INIT_WORK(&priv_dev->pending_status_wq,
3244                   cdns3_pending_setup_status_handler);
3245
3246         INIT_WORK(&priv_dev->aligned_buf_wq,
3247                   cdns3_free_aligned_request_buf);
3248
3249         /* initialize endpoint container */
3250         INIT_LIST_HEAD(&priv_dev->gadget.ep_list);
3251         INIT_LIST_HEAD(&priv_dev->aligned_buf_list);
3252         priv_dev->eps_dma_pool = dma_pool_create("cdns3_eps_dma_pool",
3253                                                  priv_dev->sysdev,
3254                                                  TRB_RING_SIZE, 8, 0);
3255         if (!priv_dev->eps_dma_pool) {
3256                 dev_err(priv_dev->dev, "Failed to create TRB dma pool\n");
3257                 ret = -ENOMEM;
3258                 goto err1;
3259         }
3260
3261         ret = cdns3_init_eps(priv_dev);
3262         if (ret) {
3263                 dev_err(priv_dev->dev, "Failed to create endpoints\n");
3264                 goto err1;
3265         }
3266
3267         /* allocate memory for setup packet buffer */
3268         priv_dev->setup_buf = dma_alloc_coherent(priv_dev->sysdev, 8,
3269                                                  &priv_dev->setup_dma, GFP_DMA);
3270         if (!priv_dev->setup_buf) {
3271                 ret = -ENOMEM;
3272                 goto err2;
3273         }
3274
3275         priv_dev->dev_ver = readl(&priv_dev->regs->usb_cap6);
3276
3277         dev_dbg(priv_dev->dev, "Device Controller version: %08x\n",
3278                 readl(&priv_dev->regs->usb_cap6));
3279         dev_dbg(priv_dev->dev, "USB Capabilities:: %08x\n",
3280                 readl(&priv_dev->regs->usb_cap1));
3281         dev_dbg(priv_dev->dev, "On-Chip memory configuration: %08x\n",
3282                 readl(&priv_dev->regs->usb_cap2));
3283
3284         priv_dev->dev_ver = GET_DEV_BASE_VERSION(priv_dev->dev_ver);
3285         if (priv_dev->dev_ver >= DEV_VER_V2)
3286                 priv_dev->gadget.sg_supported = 1;
3287
3288         priv_dev->zlp_buf = kzalloc(CDNS3_EP_ZLP_BUF_SIZE, GFP_KERNEL);
3289         if (!priv_dev->zlp_buf) {
3290                 ret = -ENOMEM;
3291                 goto err3;
3292         }
3293
3294         /* add USB gadget device */
3295         ret = usb_add_gadget(&priv_dev->gadget);
3296         if (ret < 0) {
3297                 dev_err(priv_dev->dev, "Failed to add gadget\n");
3298                 goto err4;
3299         }
3300
3301         return 0;
3302 err4:
3303         kfree(priv_dev->zlp_buf);
3304 err3:
3305         dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,
3306                           priv_dev->setup_dma);
3307 err2:
3308         cdns3_free_all_eps(priv_dev);
3309 err1:
3310         dma_pool_destroy(priv_dev->eps_dma_pool);
3311
3312         usb_put_gadget(&priv_dev->gadget);
3313         cdns->gadget_dev = NULL;
3314         return ret;
3315 }
3316
3317 static int __cdns3_gadget_init(struct cdns *cdns)
3318 {
3319         int ret = 0;
3320
3321         /* Ensure 32-bit DMA Mask in case we switched back from Host mode */
3322         ret = dma_set_mask_and_coherent(cdns->dev, DMA_BIT_MASK(32));
3323         if (ret) {
3324                 dev_err(cdns->dev, "Failed to set dma mask: %d\n", ret);
3325                 return ret;
3326         }
3327
3328         cdns_drd_gadget_on(cdns);
3329         pm_runtime_get_sync(cdns->dev);
3330
3331         ret = cdns3_gadget_start(cdns);
3332         if (ret) {
3333                 pm_runtime_put_sync(cdns->dev);
3334                 return ret;
3335         }
3336
3337         /*
3338          * Because interrupt line can be shared with other components in
3339          * driver it can't use IRQF_ONESHOT flag here.
3340          */
3341         ret = devm_request_threaded_irq(cdns->dev, cdns->dev_irq,
3342                                         cdns3_device_irq_handler,
3343                                         cdns3_device_thread_irq_handler,
3344                                         IRQF_SHARED, dev_name(cdns->dev),
3345                                         cdns->gadget_dev);
3346
3347         if (ret)
3348                 goto err0;
3349
3350         return 0;
3351 err0:
3352         cdns3_gadget_exit(cdns);
3353         return ret;
3354 }
3355
3356 static int cdns3_gadget_suspend(struct cdns *cdns, bool do_wakeup)
3357 __must_hold(&cdns->lock)
3358 {
3359         struct cdns3_device *priv_dev = cdns->gadget_dev;
3360
3361         spin_unlock(&cdns->lock);
3362         cdns3_disconnect_gadget(priv_dev);
3363         spin_lock(&cdns->lock);
3364
3365         priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
3366         usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
3367         cdns3_hw_reset_eps_config(priv_dev);
3368
3369         /* disable interrupt for device */
3370         writel(0, &priv_dev->regs->usb_ien);
3371
3372         return 0;
3373 }
3374
3375 static int cdns3_gadget_resume(struct cdns *cdns, bool hibernated)
3376 {
3377         struct cdns3_device *priv_dev = cdns->gadget_dev;
3378
3379         if (!priv_dev->gadget_driver)
3380                 return 0;
3381
3382         cdns3_gadget_config(priv_dev);
3383         if (hibernated)
3384                 writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf);
3385
3386         return 0;
3387 }
3388
3389 /**
3390  * cdns3_gadget_init - initialize device structure
3391  *
3392  * @cdns: cdns instance
3393  *
3394  * This function initializes the gadget.
3395  */
3396 int cdns3_gadget_init(struct cdns *cdns)
3397 {
3398         struct cdns_role_driver *rdrv;
3399
3400         rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
3401         if (!rdrv)
3402                 return -ENOMEM;
3403
3404         rdrv->start     = __cdns3_gadget_init;
3405         rdrv->stop      = cdns3_gadget_exit;
3406         rdrv->suspend   = cdns3_gadget_suspend;
3407         rdrv->resume    = cdns3_gadget_resume;
3408         rdrv->state     = CDNS_ROLE_STATE_INACTIVE;
3409         rdrv->name      = "gadget";
3410         cdns->roles[USB_ROLE_DEVICE] = rdrv;
3411
3412         return 0;
3413 }