drivers/usb/cdns3/cdnsp-gadget.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Cadence CDNSP DRD Driver.
   4  *
   5  * Copyright (C) 2020 Cadence.
   6  *
   7  * Author: Pawel Laszczak <pawell@cadence.com>
   8  *
   9  */
  10
  11 #include <linux/moduleparam.h>
  12 #include <linux/dma-mapping.h>
  13 #include <linux/module.h>
  14 #include <linux/iopoll.h>
  15 #include <linux/delay.h>
  16 #include <linux/log2.h>
  17 #include <linux/slab.h>
  18 #include <linux/string_choices.h>
  19 #include <linux/pci.h>
  20 #include <linux/irq.h>
  21 #include <linux/dmi.h>
  22
  23 #include "core.h"
  24 #include "gadget-export.h"
  25 #include "drd.h"
  26 #include "cdnsp-gadget.h"
  27 #include "cdnsp-trace.h"
  28
  29 unsigned int cdnsp_port_speed(unsigned int port_status)
  30 {
  31         /*Detect gadget speed based on PORTSC register*/
  32         if (DEV_SUPERSPEEDPLUS(port_status) ||
  33             DEV_SSP_GEN1x2(port_status) || DEV_SSP_GEN2x2(port_status))
  34                 return USB_SPEED_SUPER_PLUS;
  35         else if (DEV_SUPERSPEED(port_status))
  36                 return USB_SPEED_SUPER;
  37         else if (DEV_HIGHSPEED(port_status))
  38                 return USB_SPEED_HIGH;
  39         else if (DEV_FULLSPEED(port_status))
  40                 return USB_SPEED_FULL;
  41
  42         /* If device is detached then speed will be USB_SPEED_UNKNOWN.*/
  43         return USB_SPEED_UNKNOWN;
  44 }
  45
  46 /*
  47  * Given a port state, this function returns a value that would result in the
  48  * port being in the same state, if the value was written to the port status
  49  * control register.
  50  * Save Read Only (RO) bits and save read/write bits where
  51  * writing a 0 clears the bit and writing a 1 sets the bit (RWS).
  52  * For all other types (RW1S, RW1CS, RW, and RZ), writing a '0' has no effect.
  53  */
  54 u32 cdnsp_port_state_to_neutral(u32 state)
  55 {
  56         /* Save read-only status and port state. */
  57         return (state & CDNSP_PORT_RO) | (state & CDNSP_PORT_RWS);
  58 }
  59
  60 /**
  61  * cdnsp_find_next_ext_cap - Find the offset of the extended capabilities
  62  *                           with capability ID id.
  63  * @base: PCI MMIO registers base address.
  64  * @start: Address at which to start looking, (0 or HCC_PARAMS to start at
  65  *         beginning of list)
  66  * @id: Extended capability ID to search for.
  67  *
  68  * Returns the offset of the next matching extended capability structure.
  69  * Some capabilities can occur several times,
  70  * e.g., the EXT_CAPS_PROTOCOL, and this provides a way to find them all.
  71  */
  72 int cdnsp_find_next_ext_cap(void __iomem *base, u32 start, int id)
  73 {
  74         u32 offset = start;
  75         u32 next;
  76         u32 val;
  77
  78         if (!start || start == HCC_PARAMS_OFFSET) {
  79                 val = readl(base + HCC_PARAMS_OFFSET);
  80                 if (val == ~0)
  81                         return 0;
  82
  83                 offset = HCC_EXT_CAPS(val) << 2;
  84                 if (!offset)
  85                         return 0;
  86         }
  87
  88         do {
  89                 val = readl(base + offset);
  90                 if (val == ~0)
  91                         return 0;
  92
  93                 if (EXT_CAPS_ID(val) == id && offset != start)
  94                         return offset;
  95
  96                 next = EXT_CAPS_NEXT(val);
  97                 offset += next << 2;
  98         } while (next);
  99
 100         return 0;
 101 }
 102
 103 void cdnsp_set_link_state(struct cdnsp_device *pdev,
 104                           __le32 __iomem *port_regs,
 105                           u32 link_state)
 106 {
 107         int port_num = 0xFF;
 108         u32 temp;
 109
 110         temp = readl(port_regs);
 111         temp = cdnsp_port_state_to_neutral(temp);
 112         temp |= PORT_WKCONN_E | PORT_WKDISC_E;
 113         writel(temp, port_regs);
 114
 115         temp &= ~PORT_PLS_MASK;
 116         temp |= PORT_LINK_STROBE | link_state;
 117
 118         if (pdev->active_port)
 119                 port_num = pdev->active_port->port_num;
 120
 121         trace_cdnsp_handle_port_status(port_num, readl(port_regs));
 122         writel(temp, port_regs);
 123         trace_cdnsp_link_state_changed(port_num, readl(port_regs));
 124 }
 125
 126 static void cdnsp_disable_port(struct cdnsp_device *pdev,
 127                                __le32 __iomem *port_regs)
 128 {
 129         u32 temp = cdnsp_port_state_to_neutral(readl(port_regs));
 130
 131         writel(temp | PORT_PED, port_regs);
 132 }
 133
 134 static void cdnsp_clear_port_change_bit(struct cdnsp_device *pdev,
 135                                         __le32 __iomem *port_regs)
 136 {
 137         u32 portsc = readl(port_regs);
 138
 139         writel(cdnsp_port_state_to_neutral(portsc) |
 140                (portsc & PORT_CHANGE_BITS), port_regs);
 141 }
 142
 143 static void cdnsp_set_apb_timeout_value(struct cdnsp_device *pdev)
 144 {
 145         struct cdns *cdns = dev_get_drvdata(pdev->dev);
 146         __le32 __iomem *reg;
 147         void __iomem *base;
 148         u32 offset = 0;
 149         u32 val;
 150
 151         if (!cdns->override_apb_timeout)
 152                 return;
 153
 154         base = &pdev->cap_regs->hc_capbase;
 155         offset = cdnsp_find_next_ext_cap(base, offset, D_XEC_PRE_REGS_CAP);
 156         reg = base + offset + REG_CHICKEN_BITS_3_OFFSET;
 157
 158         val  = le32_to_cpu(readl(reg));
 159         val = CHICKEN_APB_TIMEOUT_SET(val, cdns->override_apb_timeout);
 160         writel(cpu_to_le32(val), reg);
 161 }
 162
 163 static void cdnsp_set_chicken_bits_2(struct cdnsp_device *pdev, u32 bit)
 164 {
 165         __le32 __iomem *reg;
 166         void __iomem *base;
 167         u32 offset = 0;
 168
 169         base = &pdev->cap_regs->hc_capbase;
 170         offset = cdnsp_find_next_ext_cap(base, offset, D_XEC_PRE_REGS_CAP);
 171         reg = base + offset + REG_CHICKEN_BITS_2_OFFSET;
 172
 173         bit = readl(reg) | bit;
 174         writel(bit, reg);
 175 }
 176
 177 static void cdnsp_clear_chicken_bits_2(struct cdnsp_device *pdev, u32 bit)
 178 {
 179         __le32 __iomem *reg;
 180         void __iomem *base;
 181         u32 offset = 0;
 182
 183         base = &pdev->cap_regs->hc_capbase;
 184         offset = cdnsp_find_next_ext_cap(base, offset, D_XEC_PRE_REGS_CAP);
 185         reg = base + offset + REG_CHICKEN_BITS_2_OFFSET;
 186
 187         bit = readl(reg) & ~bit;
 188         writel(bit, reg);
 189 }
 190
 191 /*
 192  * Disable interrupts and begin the controller halting process.
 193  */
 194 static void cdnsp_quiesce(struct cdnsp_device *pdev)
 195 {
 196         u32 halted;
 197         u32 mask;
 198         u32 cmd;
 199
 200         mask = ~(u32)(CDNSP_IRQS);
 201
 202         halted = readl(&pdev->op_regs->status) & STS_HALT;
 203         if (!halted)
 204                 mask &= ~(CMD_R_S | CMD_DEVEN);
 205
 206         cmd = readl(&pdev->op_regs->command);
 207         cmd &= mask;
 208         writel(cmd, &pdev->op_regs->command);
 209 }
 210
 211 /*
 212  * Force controller into halt state.
 213  *
 214  * Disable any IRQs and clear the run/stop bit.
 215  * Controller will complete any current and actively pipelined transactions, and
 216  * should halt within 16 ms of the run/stop bit being cleared.
 217  * Read controller Halted bit in the status register to see when the
 218  * controller is finished.
 219  */
 220 int cdnsp_halt(struct cdnsp_device *pdev)
 221 {
 222         int ret;
 223         u32 val;
 224
 225         cdnsp_quiesce(pdev);
 226
 227         ret = readl_poll_timeout_atomic(&pdev->op_regs->status, val,
 228                                         val & STS_HALT, 1,
 229                                         CDNSP_MAX_HALT_USEC);
 230         if (ret) {
 231                 dev_err(pdev->dev, "ERROR: Device halt failed\n");
 232                 return ret;
 233         }
 234
 235         pdev->cdnsp_state |= CDNSP_STATE_HALTED;
 236
 237         return 0;
 238 }
 239
 240 /*
 241  * device controller died, register read returns 0xffffffff, or command never
 242  * ends.
 243  */
 244 void cdnsp_died(struct cdnsp_device *pdev)
 245 {
 246         dev_err(pdev->dev, "ERROR: CDNSP controller not responding\n");
 247         pdev->cdnsp_state |= CDNSP_STATE_DYING;
 248         cdnsp_halt(pdev);
 249 }
 250
 251 /*
 252  * Set the run bit and wait for the device to be running.
 253  */
 254 static int cdnsp_start(struct cdnsp_device *pdev)
 255 {
 256         u32 temp;
 257         int ret;
 258
 259         temp = readl(&pdev->op_regs->command);
 260         temp |= (CMD_R_S | CMD_DEVEN);
 261         writel(temp, &pdev->op_regs->command);
 262
 263         pdev->cdnsp_state = 0;
 264
 265         /*
 266          * Wait for the STS_HALT Status bit to be 0 to indicate the device is
 267          * running.
 268          */
 269         ret = readl_poll_timeout_atomic(&pdev->op_regs->status, temp,
 270                                         !(temp & STS_HALT), 1,
 271                                         CDNSP_MAX_HALT_USEC);
 272         if (ret) {
 273                 pdev->cdnsp_state = CDNSP_STATE_DYING;
 274                 dev_err(pdev->dev, "ERROR: Controller run failed\n");
 275         }
 276
 277         return ret;
 278 }
 279
 280 /*
 281  * Reset a halted controller.
 282  *
 283  * This resets pipelines, timers, counters, state machines, etc.
 284  * Transactions will be terminated immediately, and operational registers
 285  * will be set to their defaults.
 286  */
 287 int cdnsp_reset(struct cdnsp_device *pdev)
 288 {
 289         u32 command;
 290         u32 temp;
 291         int ret;
 292
 293         temp = readl(&pdev->op_regs->status);
 294
 295         if (temp == ~(u32)0) {
 296                 dev_err(pdev->dev, "Device not accessible, reset failed.\n");
 297                 return -ENODEV;
 298         }
 299
 300         if ((temp & STS_HALT) == 0) {
 301                 dev_err(pdev->dev, "Controller not halted, aborting reset.\n");
 302                 return -EINVAL;
 303         }
 304
 305         command = readl(&pdev->op_regs->command);
 306         command |= CMD_RESET;
 307         writel(command, &pdev->op_regs->command);
 308
 309         ret = readl_poll_timeout_atomic(&pdev->op_regs->command, temp,
 310                                         !(temp & CMD_RESET), 1,
 311                                         10 * 1000);
 312         if (ret) {
 313                 dev_err(pdev->dev, "ERROR: Controller reset failed\n");
 314                 return ret;
 315         }
 316
 317         /*
 318          * CDNSP cannot write any doorbells or operational registers other
 319          * than status until the "Controller Not Ready" flag is cleared.
 320          */
 321         ret = readl_poll_timeout_atomic(&pdev->op_regs->status, temp,
 322                                         !(temp & STS_CNR), 1,
 323                                         10 * 1000);
 324
 325         if (ret) {
 326                 dev_err(pdev->dev, "ERROR: Controller not ready to work\n");
 327                 return ret;
 328         }
 329
 330         dev_dbg(pdev->dev, "Controller ready to work");
 331
 332         return ret;
 333 }
 334
 335 /*
 336  * cdnsp_get_endpoint_index - Find the index for an endpoint given its
 337  * descriptor.Use the return value to right shift 1 for the bitmask.
 338  *
 339  * Index = (epnum * 2) + direction - 1,
 340  * where direction = 0 for OUT, 1 for IN.
 341  * For control endpoints, the IN index is used (OUT index is unused), so
 342  * index = (epnum * 2) + direction - 1 = (epnum * 2) + 1 - 1 = (epnum * 2)
 343  */
 344 static unsigned int
 345         cdnsp_get_endpoint_index(const struct usb_endpoint_descriptor *desc)
 346 {
 347         unsigned int index = (unsigned int)usb_endpoint_num(desc);
 348
 349         if (usb_endpoint_xfer_control(desc))
 350                 return index * 2;
 351
 352         return (index * 2) + (usb_endpoint_dir_in(desc) ? 1 : 0) - 1;
 353 }
 354
 355 /*
 356  * Find the flag for this endpoint (for use in the control context). Use the
 357  * endpoint index to create a bitmask. The slot context is bit 0, endpoint 0 is
 358  * bit 1, etc.
 359  */
 360 static unsigned int
 361         cdnsp_get_endpoint_flag(const struct usb_endpoint_descriptor *desc)
 362 {
 363         return 1 << (cdnsp_get_endpoint_index(desc) + 1);
 364 }
 365
 366 int cdnsp_ep_enqueue(struct cdnsp_ep *pep, struct cdnsp_request *preq)
 367 {
 368         struct cdnsp_device *pdev = pep->pdev;
 369         struct usb_request *request;
 370         int ret;
 371
 372         if (preq->epnum == 0 && !list_empty(&pep->pending_list)) {
 373                 trace_cdnsp_request_enqueue_busy(preq);
 374                 return -EBUSY;
 375         }
 376
 377         request = &preq->request;
 378         request->actual = 0;
 379         request->status = -EINPROGRESS;
 380         preq->direction = pep->direction;
 381         preq->epnum = pep->number;
 382         preq->td.drbl = 0;
 383
 384         ret = usb_gadget_map_request_by_dev(pdev->dev, request, pep->direction);
 385         if (ret) {
 386                 trace_cdnsp_request_enqueue_error(preq);
 387                 return ret;
 388         }
 389
 390         list_add_tail(&preq->list, &pep->pending_list);
 391
 392         trace_cdnsp_request_enqueue(preq);
 393
 394         switch (usb_endpoint_type(pep->endpoint.desc)) {
 395         case USB_ENDPOINT_XFER_CONTROL:
 396                 ret = cdnsp_queue_ctrl_tx(pdev, preq);
 397                 break;
 398         case USB_ENDPOINT_XFER_BULK:
 399         case USB_ENDPOINT_XFER_INT:
 400                 ret = cdnsp_queue_bulk_tx(pdev, preq);
 401                 break;
 402         case USB_ENDPOINT_XFER_ISOC:
 403                 ret = cdnsp_queue_isoc_tx(pdev, preq);
 404         }
 405
 406         if (ret)
 407                 goto unmap;
 408
 409         return 0;
 410
 411 unmap:
 412         usb_gadget_unmap_request_by_dev(pdev->dev, &preq->request,
 413                                         pep->direction);
 414         list_del(&preq->list);
 415         trace_cdnsp_request_enqueue_error(preq);
 416
 417         return ret;
 418 }
 419
 420 /*
 421  * Remove the request's TD from the endpoint ring. This may cause the
 422  * controller to stop USB transfers, potentially stopping in the middle of a
 423  * TRB buffer. The controller should pick up where it left off in the TD,
 424  * unless a Set Transfer Ring Dequeue Pointer is issued.
 425  *
 426  * The TRBs that make up the buffers for the canceled request will be "removed"
 427  * from the ring. Since the ring is a contiguous structure, they can't be
 428  * physically removed. Instead, there are two options:
 429  *
 430  *  1) If the controller is in the middle of processing the request to be
 431  *     canceled, we simply move the ring's dequeue pointer past those TRBs
 432  *     using the Set Transfer Ring Dequeue Pointer command. This will be
 433  *     the common case, when drivers timeout on the last submitted request
 434  *     and attempt to cancel.
 435  *
 436  *  2) If the controller is in the middle of a different TD, we turn the TRBs
 437  *     into a series of 1-TRB transfer no-op TDs. No-ops shouldn't be chained.
 438  *     The controller will need to invalidate the any TRBs it has cached after
 439  *     the stop endpoint command.
 440  *
 441  *  3) The TD may have completed by the time the Stop Endpoint Command
 442  *     completes, so software needs to handle that case too.
 443  *
 444  */
 445 int cdnsp_ep_dequeue(struct cdnsp_ep *pep, struct cdnsp_request *preq)
 446 {
 447         struct cdnsp_device *pdev = pep->pdev;
 448         int ret_stop = 0;
 449         int ret_rem;
 450
 451         trace_cdnsp_request_dequeue(preq);
 452
 453         if (GET_EP_CTX_STATE(pep->out_ctx) == EP_STATE_RUNNING)
 454                 ret_stop = cdnsp_cmd_stop_ep(pdev, pep);
 455
 456         ret_rem = cdnsp_remove_request(pdev, preq, pep);
 457
 458         return ret_rem ? ret_rem : ret_stop;
 459 }
 460
 461 static void cdnsp_zero_in_ctx(struct cdnsp_device *pdev)
 462 {
 463         struct cdnsp_input_control_ctx *ctrl_ctx;
 464         struct cdnsp_slot_ctx *slot_ctx;
 465         struct cdnsp_ep_ctx *ep_ctx;
 466         int i;
 467
 468         ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
 469
 470         /*
 471          * When a device's add flag and drop flag are zero, any subsequent
 472          * configure endpoint command will leave that endpoint's state
 473          * untouched. Make sure we don't leave any old state in the input
 474          * endpoint contexts.
 475          */
 476         ctrl_ctx->drop_flags = 0;
 477         ctrl_ctx->add_flags = 0;
 478         slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
 479         slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
 480
 481         /* Endpoint 0 is always valid */
 482         slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1));
 483         for (i = 1; i < CDNSP_ENDPOINTS_NUM; ++i) {
 484                 ep_ctx = cdnsp_get_ep_ctx(&pdev->in_ctx, i);
 485                 ep_ctx->ep_info = 0;
 486                 ep_ctx->ep_info2 = 0;
 487                 ep_ctx->deq = 0;
 488                 ep_ctx->tx_info = 0;
 489         }
 490 }
 491
 492 /* Issue a configure endpoint command and wait for it to finish. */
 493 static int cdnsp_configure_endpoint(struct cdnsp_device *pdev)
 494 {
 495         int ret;
 496
 497         cdnsp_queue_configure_endpoint(pdev, pdev->cmd.in_ctx->dma);
 498         cdnsp_ring_cmd_db(pdev);
 499         ret = cdnsp_wait_for_cmd_compl(pdev);
 500         if (ret) {
 501                 dev_err(pdev->dev,
 502                         "ERR: unexpected command completion code 0x%x.\n", ret);
 503                 return -EINVAL;
 504         }
 505
 506         return ret;
 507 }
 508
 509 static void cdnsp_invalidate_ep_events(struct cdnsp_device *pdev,
 510                                        struct cdnsp_ep *pep)
 511 {
 512         struct cdnsp_segment *segment;
 513         union cdnsp_trb *event;
 514         u32 cycle_state;
 515         u32  data;
 516
 517         event = pdev->event_ring->dequeue;
 518         segment = pdev->event_ring->deq_seg;
 519         cycle_state = pdev->event_ring->cycle_state;
 520
 521         while (1) {
 522                 data = le32_to_cpu(event->trans_event.flags);
 523
 524                 /* Check the owner of the TRB. */
 525                 if ((data & TRB_CYCLE) != cycle_state)
 526                         break;
 527
 528                 if (TRB_FIELD_TO_TYPE(data) == TRB_TRANSFER &&
 529                     TRB_TO_EP_ID(data) == (pep->idx + 1)) {
 530                         data |= TRB_EVENT_INVALIDATE;
 531                         event->trans_event.flags = cpu_to_le32(data);
 532                 }
 533
 534                 if (cdnsp_last_trb_on_seg(segment, event)) {
 535                         cycle_state ^= 1;
 536                         segment = pdev->event_ring->deq_seg->next;
 537                         event = segment->trbs;
 538                 } else {
 539                         event++;
 540                 }
 541         }
 542 }
 543
 544 int cdnsp_wait_for_cmd_compl(struct cdnsp_device *pdev)
 545 {
 546         struct cdnsp_segment *event_deq_seg;
 547         union cdnsp_trb *cmd_trb;
 548         dma_addr_t cmd_deq_dma;
 549         union cdnsp_trb *event;
 550         u32 cycle_state;
 551         u32 retry = 10;
 552         int ret, val;
 553         u64 cmd_dma;
 554         u32  flags;
 555
 556         cmd_trb = pdev->cmd.command_trb;
 557         pdev->cmd.status = 0;
 558
 559         trace_cdnsp_cmd_wait_for_compl(pdev->cmd_ring, &cmd_trb->generic);
 560
 561         ret = readl_poll_timeout_atomic(&pdev->op_regs->cmd_ring, val,
 562                                         !CMD_RING_BUSY(val), 1,
 563                                         CDNSP_CMD_TIMEOUT);
 564         if (ret) {
 565                 dev_err(pdev->dev, "ERR: Timeout while waiting for command\n");
 566                 trace_cdnsp_cmd_timeout(pdev->cmd_ring, &cmd_trb->generic);
 567                 pdev->cdnsp_state = CDNSP_STATE_DYING;
 568                 return -ETIMEDOUT;
 569         }
 570
 571         event = pdev->event_ring->dequeue;
 572         event_deq_seg = pdev->event_ring->deq_seg;
 573         cycle_state = pdev->event_ring->cycle_state;
 574
 575         cmd_deq_dma = cdnsp_trb_virt_to_dma(pdev->cmd_ring->deq_seg, cmd_trb);
 576         if (!cmd_deq_dma)
 577                 return -EINVAL;
 578
 579         while (1) {
 580                 flags = le32_to_cpu(event->event_cmd.flags);
 581
 582                 /* Check the owner of the TRB. */
 583                 if ((flags & TRB_CYCLE) != cycle_state) {
 584                         /*
 585                          * Give some extra time to get chance controller
 586                          * to finish command before returning error code.
 587                          * Checking CMD_RING_BUSY is not sufficient because
 588                          * this bit is cleared to '0' when the Command
 589                          * Descriptor has been executed by controller
 590                          * and not when command completion event has
 591                          * be added to event ring.
 592                          */
 593                         if (retry--) {
 594                                 udelay(20);
 595                                 continue;
 596                         }
 597
 598                         return -EINVAL;
 599                 }
 600
 601                 cmd_dma = le64_to_cpu(event->event_cmd.cmd_trb);
 602
 603                 /*
 604                  * Check whether the completion event is for last queued
 605                  * command.
 606                  */
 607                 if (TRB_FIELD_TO_TYPE(flags) != TRB_COMPLETION ||
 608                     cmd_dma != (u64)cmd_deq_dma) {
 609                         if (!cdnsp_last_trb_on_seg(event_deq_seg, event)) {
 610                                 event++;
 611                                 continue;
 612                         }
 613
 614                         if (cdnsp_last_trb_on_ring(pdev->event_ring,
 615                                                    event_deq_seg, event))
 616                                 cycle_state ^= 1;
 617
 618                         event_deq_seg = event_deq_seg->next;
 619                         event = event_deq_seg->trbs;
 620                         continue;
 621                 }
 622
 623                 trace_cdnsp_handle_command(pdev->cmd_ring, &cmd_trb->generic);
 624
 625                 pdev->cmd.status = GET_COMP_CODE(le32_to_cpu(event->event_cmd.status));
 626                 if (pdev->cmd.status == COMP_SUCCESS)
 627                         return 0;
 628
 629                 return -pdev->cmd.status;
 630         }
 631 }
 632
 633 int cdnsp_halt_endpoint(struct cdnsp_device *pdev,
 634                         struct cdnsp_ep *pep,
 635                         int value)
 636 {
 637         int ret;
 638
 639         trace_cdnsp_ep_halt(value ? "Set" : "Clear");
 640
 641         ret = cdnsp_cmd_stop_ep(pdev, pep);
 642         if (ret)
 643                 return ret;
 644
 645         if (value) {
 646                 if (GET_EP_CTX_STATE(pep->out_ctx) == EP_STATE_STOPPED) {
 647                         cdnsp_queue_halt_endpoint(pdev, pep->idx);
 648                         cdnsp_ring_cmd_db(pdev);
 649                         ret = cdnsp_wait_for_cmd_compl(pdev);
 650                 }
 651
 652                 pep->ep_state |= EP_HALTED;
 653         } else {
 654                 cdnsp_queue_reset_ep(pdev, pep->idx);
 655                 cdnsp_ring_cmd_db(pdev);
 656                 ret = cdnsp_wait_for_cmd_compl(pdev);
 657                 trace_cdnsp_handle_cmd_reset_ep(pep->out_ctx);
 658
 659                 if (ret)
 660                         return ret;
 661
 662                 pep->ep_state &= ~EP_HALTED;
 663
 664                 if (pep->idx != 0 && !(pep->ep_state & EP_WEDGE))
 665                         cdnsp_ring_doorbell_for_active_rings(pdev, pep);
 666
 667                 pep->ep_state &= ~EP_WEDGE;
 668         }
 669
 670         return 0;
 671 }
 672
 673 static int cdnsp_update_eps_configuration(struct cdnsp_device *pdev,
 674                                           struct cdnsp_ep *pep)
 675 {
 676         struct cdnsp_input_control_ctx *ctrl_ctx;
 677         struct cdnsp_slot_ctx *slot_ctx;
 678         int ret = 0;
 679         u32 ep_sts;
 680         int i;
 681
 682         ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
 683
 684         /* Don't issue the command if there's no endpoints to update. */
 685         if (ctrl_ctx->add_flags == 0 && ctrl_ctx->drop_flags == 0)
 686                 return 0;
 687
 688         ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
 689         ctrl_ctx->add_flags &= cpu_to_le32(~EP0_FLAG);
 690         ctrl_ctx->drop_flags &= cpu_to_le32(~(SLOT_FLAG | EP0_FLAG));
 691
 692         /* Fix up Context Entries field. Minimum value is EP0 == BIT(1). */
 693         slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
 694         for (i = CDNSP_ENDPOINTS_NUM; i >= 1; i--) {
 695                 __le32 le32 = cpu_to_le32(BIT(i));
 696
 697                 if ((pdev->eps[i - 1].ring && !(ctrl_ctx->drop_flags & le32)) ||
 698                     (ctrl_ctx->add_flags & le32) || i == 1) {
 699                         slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
 700                         slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(i));
 701                         break;
 702                 }
 703         }
 704
 705         ep_sts = GET_EP_CTX_STATE(pep->out_ctx);
 706
 707         if ((ctrl_ctx->add_flags != cpu_to_le32(SLOT_FLAG) &&
 708              ep_sts == EP_STATE_DISABLED) ||
 709             (ep_sts != EP_STATE_DISABLED && ctrl_ctx->drop_flags))
 710                 ret = cdnsp_configure_endpoint(pdev);
 711
 712         trace_cdnsp_configure_endpoint(cdnsp_get_slot_ctx(&pdev->out_ctx));
 713         trace_cdnsp_handle_cmd_config_ep(pep->out_ctx);
 714
 715         cdnsp_zero_in_ctx(pdev);
 716
 717         return ret;
 718 }
 719
 720 /*
 721  * This submits a Reset Device Command, which will set the device state to 0,
 722  * set the device address to 0, and disable all the endpoints except the default
 723  * control endpoint. The USB core should come back and call
 724  * cdnsp_setup_device(), and then re-set up the configuration.
 725  */
 726 int cdnsp_reset_device(struct cdnsp_device *pdev)
 727 {
 728         struct cdnsp_slot_ctx *slot_ctx;
 729         int slot_state;
 730         int ret, i;
 731
 732         slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
 733         slot_ctx->dev_info = 0;
 734         pdev->device_address = 0;
 735
 736         /* If device is not setup, there is no point in resetting it. */
 737         slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx);
 738         slot_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state));
 739         trace_cdnsp_reset_device(slot_ctx);
 740
 741         if (slot_state <= SLOT_STATE_DEFAULT &&
 742             pdev->eps[0].ep_state & EP_HALTED) {
 743                 cdnsp_halt_endpoint(pdev, &pdev->eps[0], 0);
 744         }
 745
 746         /*
 747          * During Reset Device command controller shall transition the
 748          * endpoint ep0 to the Running State.
 749          */
 750         pdev->eps[0].ep_state &= ~(EP_STOPPED | EP_HALTED);
 751         pdev->eps[0].ep_state |= EP_ENABLED;
 752
 753         if (slot_state <= SLOT_STATE_DEFAULT)
 754                 return 0;
 755
 756         cdnsp_queue_reset_device(pdev);
 757         cdnsp_ring_cmd_db(pdev);
 758         ret = cdnsp_wait_for_cmd_compl(pdev);
 759
 760         /*
 761          * After Reset Device command all not default endpoints
 762          * are in Disabled state.
 763          */
 764         for (i = 1; i < CDNSP_ENDPOINTS_NUM; ++i)
 765                 pdev->eps[i].ep_state |= EP_STOPPED | EP_UNCONFIGURED;
 766
 767         trace_cdnsp_handle_cmd_reset_dev(slot_ctx);
 768
 769         if (ret)
 770                 dev_err(pdev->dev, "Reset device failed with error code %d",
 771                         ret);
 772
 773         return ret;
 774 }
 775
 776 /*
 777  * Sets the MaxPStreams field and the Linear Stream Array field.
 778  * Sets the dequeue pointer to the stream context array.
 779  */
 780 static void cdnsp_setup_streams_ep_input_ctx(struct cdnsp_device *pdev,
 781                                              struct cdnsp_ep_ctx *ep_ctx,
 782                                              struct cdnsp_stream_info *stream_info)
 783 {
 784         u32 max_primary_streams;
 785
 786         /* MaxPStreams is the number of stream context array entries, not the
 787          * number we're actually using. Must be in 2^(MaxPstreams + 1) format.
 788          * fls(0) = 0, fls(0x1) = 1, fls(0x10) = 2, fls(0x100) = 3, etc.
 789          */
 790         max_primary_streams = fls(stream_info->num_stream_ctxs) - 2;
 791         ep_ctx->ep_info &= cpu_to_le32(~EP_MAXPSTREAMS_MASK);
 792         ep_ctx->ep_info |= cpu_to_le32(EP_MAXPSTREAMS(max_primary_streams)
 793                                        | EP_HAS_LSA);
 794         ep_ctx->deq  = cpu_to_le64(stream_info->ctx_array_dma);
 795 }
 796
 797 /*
 798  * The drivers use this function to prepare a bulk endpoints to use streams.
 799  *
 800  * Don't allow the call to succeed if endpoint only supports one stream
 801  * (which means it doesn't support streams at all).
 802  */
 803 int cdnsp_alloc_streams(struct cdnsp_device *pdev, struct cdnsp_ep *pep)
 804 {
 805         unsigned int num_streams = usb_ss_max_streams(pep->endpoint.comp_desc);
 806         unsigned int num_stream_ctxs;
 807         int ret;
 808
 809         if (num_streams ==  0)
 810                 return 0;
 811
 812         if (num_streams > STREAM_NUM_STREAMS)
 813                 return -EINVAL;
 814
 815         /*
 816          * Add two to the number of streams requested to account for
 817          * stream 0 that is reserved for controller usage and one additional
 818          * for TASK SET FULL response.
 819          */
 820         num_streams += 2;
 821
 822         /* The stream context array size must be a power of two */
 823         num_stream_ctxs = roundup_pow_of_two(num_streams);
 824
 825         trace_cdnsp_stream_number(pep, num_stream_ctxs, num_streams);
 826
 827         ret = cdnsp_alloc_stream_info(pdev, pep, num_stream_ctxs, num_streams);
 828         if (ret)
 829                 return ret;
 830
 831         cdnsp_setup_streams_ep_input_ctx(pdev, pep->in_ctx, &pep->stream_info);
 832
 833         pep->ep_state |= EP_HAS_STREAMS;
 834         pep->stream_info.td_count = 0;
 835         pep->stream_info.first_prime_det = 0;
 836
 837         /* Subtract 1 for stream 0, which drivers can't use. */
 838         return num_streams - 1;
 839 }
 840
 841 int cdnsp_disable_slot(struct cdnsp_device *pdev)
 842 {
 843         int ret;
 844
 845         cdnsp_queue_slot_control(pdev, TRB_DISABLE_SLOT);
 846         cdnsp_ring_cmd_db(pdev);
 847         ret = cdnsp_wait_for_cmd_compl(pdev);
 848
 849         pdev->slot_id = 0;
 850         pdev->active_port = NULL;
 851
 852         trace_cdnsp_handle_cmd_disable_slot(cdnsp_get_slot_ctx(&pdev->out_ctx));
 853
 854         memset(pdev->in_ctx.bytes, 0, CDNSP_CTX_SIZE);
 855         memset(pdev->out_ctx.bytes, 0, CDNSP_CTX_SIZE);
 856
 857         return ret;
 858 }
 859
 860 int cdnsp_enable_slot(struct cdnsp_device *pdev)
 861 {
 862         struct cdnsp_slot_ctx *slot_ctx;
 863         int slot_state;
 864         int ret;
 865
 866         /* If device is not setup, there is no point in resetting it */
 867         slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx);
 868         slot_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state));
 869
 870         if (slot_state != SLOT_STATE_DISABLED)
 871                 return 0;
 872
 873         cdnsp_queue_slot_control(pdev, TRB_ENABLE_SLOT);
 874         cdnsp_ring_cmd_db(pdev);
 875         ret = cdnsp_wait_for_cmd_compl(pdev);
 876         if (ret)
 877                 goto show_trace;
 878
 879         pdev->slot_id = 1;
 880
 881 show_trace:
 882         trace_cdnsp_handle_cmd_enable_slot(cdnsp_get_slot_ctx(&pdev->out_ctx));
 883
 884         return ret;
 885 }
 886
 887 /*
 888  * Issue an Address Device command with BSR=0 if setup is SETUP_CONTEXT_ONLY
 889  * or with BSR = 1 if set_address is SETUP_CONTEXT_ADDRESS.
 890  */
 891 int cdnsp_setup_device(struct cdnsp_device *pdev, enum cdnsp_setup_dev setup)
 892 {
 893         struct cdnsp_input_control_ctx *ctrl_ctx;
 894         struct cdnsp_slot_ctx *slot_ctx;
 895         int dev_state = 0;
 896         int ret;
 897
 898         if (!pdev->slot_id) {
 899                 trace_cdnsp_slot_id("incorrect");
 900                 return -EINVAL;
 901         }
 902
 903         if (!pdev->active_port->port_num)
 904                 return -EINVAL;
 905
 906         slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx);
 907         dev_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state));
 908
 909         if (setup == SETUP_CONTEXT_ONLY && dev_state == SLOT_STATE_DEFAULT) {
 910                 trace_cdnsp_slot_already_in_default(slot_ctx);
 911                 return 0;
 912         }
 913
 914         slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
 915         ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
 916
 917         if (!slot_ctx->dev_info || dev_state == SLOT_STATE_DEFAULT) {
 918                 ret = cdnsp_setup_addressable_priv_dev(pdev);
 919                 if (ret)
 920                         return ret;
 921         }
 922
 923         cdnsp_copy_ep0_dequeue_into_input_ctx(pdev);
 924
 925         ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG | EP0_FLAG);
 926         ctrl_ctx->drop_flags = 0;
 927
 928         trace_cdnsp_setup_device_slot(slot_ctx);
 929
 930         cdnsp_queue_address_device(pdev, pdev->in_ctx.dma, setup);
 931         cdnsp_ring_cmd_db(pdev);
 932         ret = cdnsp_wait_for_cmd_compl(pdev);
 933
 934         trace_cdnsp_handle_cmd_addr_dev(cdnsp_get_slot_ctx(&pdev->out_ctx));
 935
 936         /* Zero the input context control for later use. */
 937         ctrl_ctx->add_flags = 0;
 938         ctrl_ctx->drop_flags = 0;
 939
 940         return ret;
 941 }
 942
 943 void cdnsp_set_usb2_hardware_lpm(struct cdnsp_device *pdev,
 944                                  struct usb_request *req,
 945                                  int enable)
 946 {
 947         if (pdev->active_port != &pdev->usb2_port || !pdev->gadget.lpm_capable)
 948                 return;
 949
 950         trace_cdnsp_lpm(enable);
 951
 952         if (enable)
 953                 writel(PORT_BESL(CDNSP_DEFAULT_BESL) | PORT_L1S_NYET | PORT_HLE,
 954                        &pdev->active_port->regs->portpmsc);
 955         else
 956                 writel(PORT_L1S_NYET, &pdev->active_port->regs->portpmsc);
 957 }
 958
 959 static int cdnsp_get_frame(struct cdnsp_device *pdev)
 960 {
 961         return readl(&pdev->run_regs->microframe_index) >> 3;
 962 }
 963
 964 static int cdnsp_gadget_ep_enable(struct usb_ep *ep,
 965                                   const struct usb_endpoint_descriptor *desc)
 966 {
 967         struct cdnsp_input_control_ctx *ctrl_ctx;
 968         struct cdnsp_device *pdev;
 969         struct cdnsp_ep *pep;
 970         unsigned long flags;
 971         u32 added_ctxs;
 972         int ret;
 973
 974         if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT ||
 975             !desc->wMaxPacketSize)
 976                 return -EINVAL;
 977
 978         pep = to_cdnsp_ep(ep);
 979         pdev = pep->pdev;
 980         pep->ep_state &= ~EP_UNCONFIGURED;
 981
 982         if (dev_WARN_ONCE(pdev->dev, pep->ep_state & EP_ENABLED,
 983                           "%s is already enabled\n", pep->name))
 984                 return 0;
 985
 986         spin_lock_irqsave(&pdev->lock, flags);
 987
 988         added_ctxs = cdnsp_get_endpoint_flag(desc);
 989         if (added_ctxs == SLOT_FLAG || added_ctxs == EP0_FLAG) {
 990                 dev_err(pdev->dev, "ERROR: Bad endpoint number\n");
 991                 ret = -EINVAL;
 992                 goto unlock;
 993         }
 994
 995         pep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
 996
 997         if (pdev->gadget.speed == USB_SPEED_FULL) {
 998                 if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_INT)
 999                         pep->interval = desc->bInterval << 3;
1000                 if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_ISOC)
1001                         pep->interval = BIT(desc->bInterval - 1) << 3;
1002         }
1003
1004         if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_ISOC) {
1005                 if (pep->interval > BIT(12)) {
1006                         dev_err(pdev->dev, "bInterval %d not supported\n",
1007                                 desc->bInterval);
1008                         ret = -EINVAL;
1009                         goto unlock;
1010                 }
1011                 cdnsp_set_chicken_bits_2(pdev, CHICKEN_XDMA_2_TP_CACHE_DIS);
1012         }
1013
1014         ret = cdnsp_endpoint_init(pdev, pep, GFP_ATOMIC);
1015         if (ret)
1016                 goto unlock;
1017
1018         ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
1019         ctrl_ctx->add_flags = cpu_to_le32(added_ctxs);
1020         ctrl_ctx->drop_flags = 0;
1021
1022         ret = cdnsp_update_eps_configuration(pdev, pep);
1023         if (ret) {
1024                 cdnsp_free_endpoint_rings(pdev, pep);
1025                 goto unlock;
1026         }
1027
1028         pep->ep_state |= EP_ENABLED;
1029         pep->ep_state &= ~EP_STOPPED;
1030
1031 unlock:
1032         trace_cdnsp_ep_enable_end(pep, 0);
1033         spin_unlock_irqrestore(&pdev->lock, flags);
1034
1035         return ret;
1036 }
1037
1038 static int cdnsp_gadget_ep_disable(struct usb_ep *ep)
1039 {
1040         struct cdnsp_input_control_ctx *ctrl_ctx;
1041         struct cdnsp_request *preq;
1042         struct cdnsp_device *pdev;
1043         struct cdnsp_ep *pep;
1044         unsigned long flags;
1045         u32 drop_flag;
1046         int ret = 0;
1047
1048         if (!ep)
1049                 return -EINVAL;
1050
1051         pep = to_cdnsp_ep(ep);
1052         pdev = pep->pdev;
1053
1054         spin_lock_irqsave(&pdev->lock, flags);
1055
1056         if (!(pep->ep_state & EP_ENABLED)) {
1057                 dev_err(pdev->dev, "%s is already disabled\n", pep->name);
1058                 ret = -EINVAL;
1059                 goto finish;
1060         }
1061
1062         pep->ep_state |= EP_DIS_IN_RROGRESS;
1063
1064         /* Endpoint was unconfigured by Reset Device command. */
1065         if (!(pep->ep_state & EP_UNCONFIGURED))
1066                 cdnsp_cmd_stop_ep(pdev, pep);
1067
1068         /* Remove all queued USB requests. */
1069         while (!list_empty(&pep->pending_list)) {
1070                 preq = next_request(&pep->pending_list);
1071                 cdnsp_ep_dequeue(pep, preq);
1072         }
1073
1074         cdnsp_invalidate_ep_events(pdev, pep);
1075
1076         pep->ep_state &= ~EP_DIS_IN_RROGRESS;
1077         drop_flag = cdnsp_get_endpoint_flag(pep->endpoint.desc);
1078         ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
1079         ctrl_ctx->drop_flags = cpu_to_le32(drop_flag);
1080         ctrl_ctx->add_flags = 0;
1081
1082         cdnsp_endpoint_zero(pdev, pep);
1083
1084         if (!(pep->ep_state & EP_UNCONFIGURED))
1085                 ret = cdnsp_update_eps_configuration(pdev, pep);
1086
1087         cdnsp_free_endpoint_rings(pdev, pep);
1088
1089         pep->ep_state &= ~(EP_ENABLED | EP_UNCONFIGURED);
1090         pep->ep_state |= EP_STOPPED;
1091
1092 finish:
1093         trace_cdnsp_ep_disable_end(pep, 0);
1094         spin_unlock_irqrestore(&pdev->lock, flags);
1095
1096         return ret;
1097 }
1098
1099 static struct usb_request *cdnsp_gadget_ep_alloc_request(struct usb_ep *ep,
1100                                                          gfp_t gfp_flags)
1101 {
1102         struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1103         struct cdnsp_request *preq;
1104
1105         preq = kzalloc(sizeof(*preq), gfp_flags);
1106         if (!preq)
1107                 return NULL;
1108
1109         preq->epnum = pep->number;
1110         preq->pep = pep;
1111
1112         trace_cdnsp_alloc_request(preq);
1113
1114         return &preq->request;
1115 }
1116
1117 static void cdnsp_gadget_ep_free_request(struct usb_ep *ep,
1118                                          struct usb_request *request)
1119 {
1120         struct cdnsp_request *preq = to_cdnsp_request(request);
1121
1122         trace_cdnsp_free_request(preq);
1123         kfree(preq);
1124 }
1125
1126 static int cdnsp_gadget_ep_queue(struct usb_ep *ep,
1127                                  struct usb_request *request,
1128                                  gfp_t gfp_flags)
1129 {
1130         struct cdnsp_request *preq;
1131         struct cdnsp_device *pdev;
1132         struct cdnsp_ep *pep;
1133         unsigned long flags;
1134         int ret;
1135
1136         if (!request || !ep)
1137                 return -EINVAL;
1138
1139         pep = to_cdnsp_ep(ep);
1140         pdev = pep->pdev;
1141
1142         if (!(pep->ep_state & EP_ENABLED)) {
1143                 dev_err(pdev->dev, "%s: can't queue to disabled endpoint\n",
1144                         pep->name);
1145                 return -EINVAL;
1146         }
1147
1148         preq = to_cdnsp_request(request);
1149         spin_lock_irqsave(&pdev->lock, flags);
1150         ret = cdnsp_ep_enqueue(pep, preq);
1151         spin_unlock_irqrestore(&pdev->lock, flags);
1152
1153         return ret;
1154 }
1155
1156 static int cdnsp_gadget_ep_dequeue(struct usb_ep *ep,
1157                                    struct usb_request *request)
1158 {
1159         struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1160         struct cdnsp_device *pdev = pep->pdev;
1161         unsigned long flags;
1162         int ret;
1163
1164         if (request->status != -EINPROGRESS)
1165                 return 0;
1166
1167         if (!pep->endpoint.desc) {
1168                 dev_err(pdev->dev,
1169                         "%s: can't dequeue to disabled endpoint\n",
1170                         pep->name);
1171                 return -ESHUTDOWN;
1172         }
1173
1174         /* Requests has been dequeued during disabling endpoint. */
1175         if (!(pep->ep_state & EP_ENABLED))
1176                 return 0;
1177
1178         spin_lock_irqsave(&pdev->lock, flags);
1179         ret = cdnsp_ep_dequeue(pep, to_cdnsp_request(request));
1180         spin_unlock_irqrestore(&pdev->lock, flags);
1181
1182         return ret;
1183 }
1184
1185 static int cdnsp_gadget_ep_set_halt(struct usb_ep *ep, int value)
1186 {
1187         struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1188         struct cdnsp_device *pdev = pep->pdev;
1189         struct cdnsp_request *preq;
1190         unsigned long flags;
1191         int ret;
1192
1193         spin_lock_irqsave(&pdev->lock, flags);
1194
1195         preq = next_request(&pep->pending_list);
1196         if (value) {
1197                 if (preq) {
1198                         trace_cdnsp_ep_busy_try_halt_again(pep, 0);
1199                         ret = -EAGAIN;
1200                         goto done;
1201                 }
1202         }
1203
1204         ret = cdnsp_halt_endpoint(pdev, pep, value);
1205
1206 done:
1207         spin_unlock_irqrestore(&pdev->lock, flags);
1208         return ret;
1209 }
1210
1211 static int cdnsp_gadget_ep_set_wedge(struct usb_ep *ep)
1212 {
1213         struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1214         struct cdnsp_device *pdev = pep->pdev;
1215         unsigned long flags;
1216         int ret;
1217
1218         spin_lock_irqsave(&pdev->lock, flags);
1219         pep->ep_state |= EP_WEDGE;
1220         ret = cdnsp_halt_endpoint(pdev, pep, 1);
1221         spin_unlock_irqrestore(&pdev->lock, flags);
1222
1223         return ret;
1224 }
1225
1226 static const struct usb_ep_ops cdnsp_gadget_ep0_ops = {
1227         .enable         = cdnsp_gadget_ep_enable,
1228         .disable        = cdnsp_gadget_ep_disable,
1229         .alloc_request  = cdnsp_gadget_ep_alloc_request,
1230         .free_request   = cdnsp_gadget_ep_free_request,
1231         .queue          = cdnsp_gadget_ep_queue,
1232         .dequeue        = cdnsp_gadget_ep_dequeue,
1233         .set_halt       = cdnsp_gadget_ep_set_halt,
1234         .set_wedge      = cdnsp_gadget_ep_set_wedge,
1235 };
1236
1237 static const struct usb_ep_ops cdnsp_gadget_ep_ops = {
1238         .enable         = cdnsp_gadget_ep_enable,
1239         .disable        = cdnsp_gadget_ep_disable,
1240         .alloc_request  = cdnsp_gadget_ep_alloc_request,
1241         .free_request   = cdnsp_gadget_ep_free_request,
1242         .queue          = cdnsp_gadget_ep_queue,
1243         .dequeue        = cdnsp_gadget_ep_dequeue,
1244         .set_halt       = cdnsp_gadget_ep_set_halt,
1245         .set_wedge      = cdnsp_gadget_ep_set_wedge,
1246 };
1247
1248 void cdnsp_gadget_giveback(struct cdnsp_ep *pep,
1249                            struct cdnsp_request *preq,
1250                            int status)
1251 {
1252         struct cdnsp_device *pdev = pep->pdev;
1253
1254         list_del(&preq->list);
1255
1256         if (preq->request.status == -EINPROGRESS)
1257                 preq->request.status = status;
1258
1259         usb_gadget_unmap_request_by_dev(pdev->dev, &preq->request,
1260                                         preq->direction);
1261
1262         trace_cdnsp_request_giveback(preq);
1263
1264         if (preq != &pdev->ep0_preq) {
1265                 spin_unlock(&pdev->lock);
1266                 usb_gadget_giveback_request(&pep->endpoint, &preq->request);
1267                 spin_lock(&pdev->lock);
1268         }
1269 }
1270
1271 static struct usb_endpoint_descriptor cdnsp_gadget_ep0_desc = {
1272         .bLength =              USB_DT_ENDPOINT_SIZE,
1273         .bDescriptorType =      USB_DT_ENDPOINT,
1274         .bmAttributes =         USB_ENDPOINT_XFER_CONTROL,
1275 };
1276
1277 static int cdnsp_run(struct cdnsp_device *pdev,
1278                      enum usb_device_speed speed)
1279 {
1280         u32 fs_speed = 0;
1281         u32 temp;
1282         int ret;
1283
1284         temp = readl(&pdev->ir_set->irq_control);
1285         temp &= ~IMOD_INTERVAL_MASK;
1286         temp |= ((IMOD_DEFAULT_INTERVAL / 250) & IMOD_INTERVAL_MASK);
1287         writel(temp, &pdev->ir_set->irq_control);
1288
1289         temp = readl(&pdev->port3x_regs->mode_addr);
1290
1291         switch (speed) {
1292         case USB_SPEED_SUPER_PLUS:
1293                 temp |= CFG_3XPORT_SSP_SUPPORT;
1294                 break;
1295         case USB_SPEED_SUPER:
1296                 temp &= ~CFG_3XPORT_SSP_SUPPORT;
1297                 break;
1298         case USB_SPEED_HIGH:
1299                 break;
1300         case USB_SPEED_FULL:
1301                 fs_speed = PORT_REG6_FORCE_FS;
1302                 break;
1303         default:
1304                 dev_err(pdev->dev, "invalid maximum_speed parameter %d\n",
1305                         speed);
1306                 fallthrough;
1307         case USB_SPEED_UNKNOWN:
1308                 /* Default to superspeed. */
1309                 speed = USB_SPEED_SUPER;
1310                 break;
1311         }
1312
1313         if (speed >= USB_SPEED_SUPER) {
1314                 writel(temp, &pdev->port3x_regs->mode_addr);
1315                 cdnsp_set_link_state(pdev, &pdev->usb3_port.regs->portsc,
1316                                      XDEV_RXDETECT);
1317         } else {
1318                 cdnsp_disable_port(pdev, &pdev->usb3_port.regs->portsc);
1319         }
1320
1321         cdnsp_set_link_state(pdev, &pdev->usb2_port.regs->portsc,
1322                              XDEV_RXDETECT);
1323
1324         cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1325
1326         writel(PORT_REG6_L1_L0_HW_EN | fs_speed, &pdev->port20_regs->port_reg6);
1327
1328         ret = cdnsp_start(pdev);
1329         if (ret) {
1330                 ret = -ENODEV;
1331                 goto err;
1332         }
1333
1334         temp = readl(&pdev->op_regs->command);
1335         temp |= (CMD_INTE);
1336         writel(temp, &pdev->op_regs->command);
1337
1338         temp = readl(&pdev->ir_set->irq_pending);
1339         writel(IMAN_IE_SET(temp), &pdev->ir_set->irq_pending);
1340
1341         trace_cdnsp_init("Controller ready to work");
1342         return 0;
1343 err:
1344         cdnsp_halt(pdev);
1345         return ret;
1346 }
1347
1348 static int cdnsp_gadget_udc_start(struct usb_gadget *g,
1349                                   struct usb_gadget_driver *driver)
1350 {
1351         enum usb_device_speed max_speed = driver->max_speed;
1352         struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1353         unsigned long flags;
1354         int ret;
1355
1356         spin_lock_irqsave(&pdev->lock, flags);
1357         pdev->gadget_driver = driver;
1358
1359         /* limit speed if necessary */
1360         max_speed = min(driver->max_speed, g->max_speed);
1361         ret = cdnsp_run(pdev, max_speed);
1362
1363         spin_unlock_irqrestore(&pdev->lock, flags);
1364
1365         return ret;
1366 }
1367
1368 /*
1369  * Update Event Ring Dequeue Pointer:
1370  * - When all events have finished
1371  * - To avoid "Event Ring Full Error" condition
1372  */
1373 void cdnsp_update_erst_dequeue(struct cdnsp_device *pdev,
1374                                union cdnsp_trb *event_ring_deq,
1375                                u8 clear_ehb)
1376 {
1377         u64 temp_64;
1378         dma_addr_t deq;
1379
1380         temp_64 = cdnsp_read_64(&pdev->ir_set->erst_dequeue);
1381
1382         /* If necessary, update the HW's version of the event ring deq ptr. */
1383         if (event_ring_deq != pdev->event_ring->dequeue) {
1384                 deq = cdnsp_trb_virt_to_dma(pdev->event_ring->deq_seg,
1385                                             pdev->event_ring->dequeue);
1386                 temp_64 &= ERST_PTR_MASK;
1387                 temp_64 |= ((u64)deq & (u64)~ERST_PTR_MASK);
1388         }
1389
1390         /* Clear the event handler busy flag (RW1C). */
1391         if (clear_ehb)
1392                 temp_64 |= ERST_EHB;
1393         else
1394                 temp_64 &= ~ERST_EHB;
1395
1396         cdnsp_write_64(temp_64, &pdev->ir_set->erst_dequeue);
1397 }
1398
1399 static void cdnsp_clear_cmd_ring(struct cdnsp_device *pdev)
1400 {
1401         struct cdnsp_segment *seg;
1402         u64 val_64;
1403         int i;
1404
1405         cdnsp_initialize_ring_info(pdev->cmd_ring);
1406
1407         seg = pdev->cmd_ring->first_seg;
1408         for (i = 0; i < pdev->cmd_ring->num_segs; i++) {
1409                 memset(seg->trbs, 0,
1410                        sizeof(union cdnsp_trb) * (TRBS_PER_SEGMENT - 1));
1411                 seg = seg->next;
1412         }
1413
1414         /* Set the address in the Command Ring Control register. */
1415         val_64 = cdnsp_read_64(&pdev->op_regs->cmd_ring);
1416         val_64 = (val_64 & (u64)CMD_RING_RSVD_BITS) |
1417                  (pdev->cmd_ring->first_seg->dma & (u64)~CMD_RING_RSVD_BITS) |
1418                  pdev->cmd_ring->cycle_state;
1419         cdnsp_write_64(val_64, &pdev->op_regs->cmd_ring);
1420 }
1421
1422 static void cdnsp_consume_all_events(struct cdnsp_device *pdev)
1423 {
1424         struct cdnsp_segment *event_deq_seg;
1425         union cdnsp_trb *event_ring_deq;
1426         union cdnsp_trb *event;
1427         u32 cycle_bit;
1428
1429         event_ring_deq = pdev->event_ring->dequeue;
1430         event_deq_seg = pdev->event_ring->deq_seg;
1431         event = pdev->event_ring->dequeue;
1432
1433         /* Update ring dequeue pointer. */
1434         while (1) {
1435                 cycle_bit = (le32_to_cpu(event->event_cmd.flags) & TRB_CYCLE);
1436
1437                 /* Does the controller or driver own the TRB? */
1438                 if (cycle_bit != pdev->event_ring->cycle_state)
1439                         break;
1440
1441                 cdnsp_inc_deq(pdev, pdev->event_ring);
1442
1443                 if (!cdnsp_last_trb_on_seg(event_deq_seg, event)) {
1444                         event++;
1445                         continue;
1446                 }
1447
1448                 if (cdnsp_last_trb_on_ring(pdev->event_ring, event_deq_seg,
1449                                            event))
1450                         cycle_bit ^= 1;
1451
1452                 event_deq_seg = event_deq_seg->next;
1453                 event = event_deq_seg->trbs;
1454         }
1455
1456         cdnsp_update_erst_dequeue(pdev,  event_ring_deq, 1);
1457 }
1458
1459 static void cdnsp_stop(struct cdnsp_device *pdev)
1460 {
1461         u32 temp;
1462
1463         /* Remove internally queued request for ep0. */
1464         if (!list_empty(&pdev->eps[0].pending_list)) {
1465                 struct cdnsp_request *req;
1466
1467                 req = next_request(&pdev->eps[0].pending_list);
1468                 if (req == &pdev->ep0_preq)
1469                         cdnsp_ep_dequeue(&pdev->eps[0], req);
1470         }
1471
1472         cdnsp_disable_port(pdev, &pdev->usb2_port.regs->portsc);
1473         cdnsp_disable_port(pdev, &pdev->usb3_port.regs->portsc);
1474         cdnsp_disable_slot(pdev);
1475         cdnsp_halt(pdev);
1476
1477         temp = readl(&pdev->op_regs->status);
1478         writel((temp & ~0x1fff) | STS_EINT, &pdev->op_regs->status);
1479         temp = readl(&pdev->ir_set->irq_pending);
1480         writel(IMAN_IE_CLEAR(temp), &pdev->ir_set->irq_pending);
1481
1482         cdnsp_clear_port_change_bit(pdev, &pdev->usb2_port.regs->portsc);
1483         cdnsp_clear_port_change_bit(pdev, &pdev->usb3_port.regs->portsc);
1484
1485         /* Clear interrupt line */
1486         temp = readl(&pdev->ir_set->irq_pending);
1487         temp |= IMAN_IP;
1488         writel(temp, &pdev->ir_set->irq_pending);
1489
1490         cdnsp_consume_all_events(pdev);
1491         cdnsp_clear_cmd_ring(pdev);
1492
1493         trace_cdnsp_exit("Controller stopped.");
1494 }
1495
1496 /*
1497  * Stop controller.
1498  * This function is called by the gadget core when the driver is removed.
1499  * Disable slot, disable IRQs, and quiesce the controller.
1500  */
1501 static int cdnsp_gadget_udc_stop(struct usb_gadget *g)
1502 {
1503         struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1504         unsigned long flags;
1505
1506         spin_lock_irqsave(&pdev->lock, flags);
1507         cdnsp_stop(pdev);
1508         pdev->gadget_driver = NULL;
1509         spin_unlock_irqrestore(&pdev->lock, flags);
1510
1511         return 0;
1512 }
1513
1514 static int cdnsp_gadget_get_frame(struct usb_gadget *g)
1515 {
1516         struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1517
1518         return cdnsp_get_frame(pdev);
1519 }
1520
1521 static void __cdnsp_gadget_wakeup(struct cdnsp_device *pdev)
1522 {
1523         struct cdnsp_port_regs __iomem *port_regs;
1524         u32 portpm, portsc;
1525
1526         port_regs = pdev->active_port->regs;
1527         portsc = readl(&port_regs->portsc) & PORT_PLS_MASK;
1528
1529         /* Remote wakeup feature is not enabled by host. */
1530         if (pdev->gadget.speed < USB_SPEED_SUPER && portsc == XDEV_U2) {
1531                 portpm = readl(&port_regs->portpmsc);
1532
1533                 if (!(portpm & PORT_RWE))
1534                         return;
1535         }
1536
1537         if (portsc == XDEV_U3 && !pdev->may_wakeup)
1538                 return;
1539
1540         cdnsp_set_link_state(pdev, &port_regs->portsc, XDEV_U0);
1541
1542         pdev->cdnsp_state |= CDNSP_WAKEUP_PENDING;
1543 }
1544
1545 static int cdnsp_gadget_wakeup(struct usb_gadget *g)
1546 {
1547         struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1548         unsigned long flags;
1549
1550         spin_lock_irqsave(&pdev->lock, flags);
1551         __cdnsp_gadget_wakeup(pdev);
1552         spin_unlock_irqrestore(&pdev->lock, flags);
1553
1554         return 0;
1555 }
1556
1557 static int cdnsp_gadget_set_selfpowered(struct usb_gadget *g,
1558                                         int is_selfpowered)
1559 {
1560         struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1561         unsigned long flags;
1562
1563         spin_lock_irqsave(&pdev->lock, flags);
1564         g->is_selfpowered = !!is_selfpowered;
1565         spin_unlock_irqrestore(&pdev->lock, flags);
1566
1567         return 0;
1568 }
1569
1570 static int cdnsp_gadget_pullup(struct usb_gadget *gadget, int is_on)
1571 {
1572         struct cdnsp_device *pdev = gadget_to_cdnsp(gadget);
1573         struct cdns *cdns = dev_get_drvdata(pdev->dev);
1574         unsigned long flags;
1575
1576         trace_cdnsp_pullup(is_on);
1577
1578         /*
1579          * Disable events handling while controller is being
1580          * enabled/disabled.
1581          */
1582         disable_irq(cdns->dev_irq);
1583         spin_lock_irqsave(&pdev->lock, flags);
1584
1585         if (!is_on) {
1586                 cdnsp_reset_device(pdev);
1587                 cdns_clear_vbus(cdns);
1588         } else {
1589                 cdns_set_vbus(cdns);
1590         }
1591
1592         spin_unlock_irqrestore(&pdev->lock, flags);
1593         enable_irq(cdns->dev_irq);
1594
1595         return 0;
1596 }
1597
1598 static const struct usb_gadget_ops cdnsp_gadget_ops = {
1599         .get_frame              = cdnsp_gadget_get_frame,
1600         .wakeup                 = cdnsp_gadget_wakeup,
1601         .set_selfpowered        = cdnsp_gadget_set_selfpowered,
1602         .pullup                 = cdnsp_gadget_pullup,
1603         .udc_start              = cdnsp_gadget_udc_start,
1604         .udc_stop               = cdnsp_gadget_udc_stop,
1605 };
1606
1607 static void cdnsp_get_ep_buffering(struct cdnsp_device *pdev,
1608                                    struct cdnsp_ep *pep)
1609 {
1610         void __iomem *reg = &pdev->cap_regs->hc_capbase;
1611         int endpoints;
1612
1613         reg += cdnsp_find_next_ext_cap(reg, 0, XBUF_CAP_ID);
1614
1615         if (!pep->direction) {
1616                 pep->buffering = readl(reg + XBUF_RX_TAG_MASK_0_OFFSET);
1617                 pep->buffering_period = readl(reg + XBUF_RX_TAG_MASK_1_OFFSET);
1618                 pep->buffering = (pep->buffering + 1) / 2;
1619                 pep->buffering_period = (pep->buffering_period + 1) / 2;
1620                 return;
1621         }
1622
1623         endpoints = HCS_ENDPOINTS(pdev->hcs_params1) / 2;
1624
1625         /* Set to XBUF_TX_TAG_MASK_0 register. */
1626         reg += XBUF_TX_CMD_OFFSET + (endpoints * 2 + 2) * sizeof(u32);
1627         /* Set reg to XBUF_TX_TAG_MASK_N related with this endpoint. */
1628         reg += pep->number * sizeof(u32) * 2;
1629
1630         pep->buffering = (readl(reg) + 1) / 2;
1631         pep->buffering_period = pep->buffering;
1632 }
1633
1634 static int cdnsp_gadget_init_endpoints(struct cdnsp_device *pdev)
1635 {
1636         int max_streams = HCC_MAX_PSA(pdev->hcc_params);
1637         struct cdnsp_ep *pep;
1638         int i;
1639
1640         INIT_LIST_HEAD(&pdev->gadget.ep_list);
1641
1642         if (max_streams < STREAM_LOG_STREAMS) {
1643                 dev_err(pdev->dev, "Stream size %d not supported\n",
1644                         max_streams);
1645                 return -EINVAL;
1646         }
1647
1648         max_streams = STREAM_LOG_STREAMS;
1649
1650         for (i = 0; i < CDNSP_ENDPOINTS_NUM; i++) {
1651                 bool direction = !(i & 1); /* Start from OUT endpoint. */
1652                 u8 epnum = ((i + 1) >> 1);
1653
1654                 if (!CDNSP_IF_EP_EXIST(pdev, epnum, direction))
1655                         continue;
1656
1657                 pep = &pdev->eps[i];
1658                 pep->pdev = pdev;
1659                 pep->number = epnum;
1660                 pep->direction = direction; /* 0 for OUT, 1 for IN. */
1661
1662                 /*
1663                  * Ep0 is bidirectional, so ep0in and ep0out are represented by
1664                  * pdev->eps[0]
1665                  */
1666                 if (epnum == 0) {
1667                         snprintf(pep->name, sizeof(pep->name), "ep%d%s",
1668                                  epnum, "BiDir");
1669
1670                         pep->idx = 0;
1671                         usb_ep_set_maxpacket_limit(&pep->endpoint, 512);
1672                         pep->endpoint.maxburst = 1;
1673                         pep->endpoint.ops = &cdnsp_gadget_ep0_ops;
1674                         pep->endpoint.desc = &cdnsp_gadget_ep0_desc;
1675                         pep->endpoint.comp_desc = NULL;
1676                         pep->endpoint.caps.type_control = true;
1677                         pep->endpoint.caps.dir_in = true;
1678                         pep->endpoint.caps.dir_out = true;
1679
1680                         pdev->ep0_preq.epnum = pep->number;
1681                         pdev->ep0_preq.pep = pep;
1682                         pdev->gadget.ep0 = &pep->endpoint;
1683                 } else {
1684                         snprintf(pep->name, sizeof(pep->name), "ep%d%s",
1685                                  epnum, (pep->direction) ? "in" : "out");
1686
1687                         pep->idx =  (epnum * 2 + (direction ? 1 : 0)) - 1;
1688                         usb_ep_set_maxpacket_limit(&pep->endpoint, 1024);
1689
1690                         pep->endpoint.max_streams = max_streams;
1691                         pep->endpoint.ops = &cdnsp_gadget_ep_ops;
1692                         list_add_tail(&pep->endpoint.ep_list,
1693                                       &pdev->gadget.ep_list);
1694
1695                         pep->endpoint.caps.type_iso = true;
1696                         pep->endpoint.caps.type_bulk = true;
1697                         pep->endpoint.caps.type_int = true;
1698
1699                         pep->endpoint.caps.dir_in = direction;
1700                         pep->endpoint.caps.dir_out = !direction;
1701                 }
1702
1703                 pep->endpoint.name = pep->name;
1704                 pep->in_ctx = cdnsp_get_ep_ctx(&pdev->in_ctx, pep->idx);
1705                 pep->out_ctx = cdnsp_get_ep_ctx(&pdev->out_ctx, pep->idx);
1706                 cdnsp_get_ep_buffering(pdev, pep);
1707
1708                 dev_dbg(pdev->dev, "Init %s, MPS: %04x SupType: "
1709                         "CTRL: %s, INT: %s, BULK: %s, ISOC %s, "
1710                         "SupDir IN: %s, OUT: %s\n",
1711                         pep->name, 1024,
1712                         str_yes_no(pep->endpoint.caps.type_control),
1713                         str_yes_no(pep->endpoint.caps.type_int),
1714                         str_yes_no(pep->endpoint.caps.type_bulk),
1715                         str_yes_no(pep->endpoint.caps.type_iso),
1716                         str_yes_no(pep->endpoint.caps.dir_in),
1717                         str_yes_no(pep->endpoint.caps.dir_out));
1718
1719                 INIT_LIST_HEAD(&pep->pending_list);
1720         }
1721
1722         return 0;
1723 }
1724
1725 static void cdnsp_gadget_free_endpoints(struct cdnsp_device *pdev)
1726 {
1727         struct cdnsp_ep *pep;
1728         int i;
1729
1730         for (i = 0; i < CDNSP_ENDPOINTS_NUM; i++) {
1731                 pep = &pdev->eps[i];
1732                 if (pep->number != 0 && pep->out_ctx)
1733                         list_del(&pep->endpoint.ep_list);
1734         }
1735 }
1736
1737 void cdnsp_disconnect_gadget(struct cdnsp_device *pdev)
1738 {
1739         pdev->cdnsp_state |= CDNSP_STATE_DISCONNECT_PENDING;
1740
1741         if (pdev->gadget_driver && pdev->gadget_driver->disconnect) {
1742                 spin_unlock(&pdev->lock);
1743                 pdev->gadget_driver->disconnect(&pdev->gadget);
1744                 spin_lock(&pdev->lock);
1745         }
1746
1747         pdev->gadget.speed = USB_SPEED_UNKNOWN;
1748         usb_gadget_set_state(&pdev->gadget, USB_STATE_NOTATTACHED);
1749
1750         pdev->cdnsp_state &= ~CDNSP_STATE_DISCONNECT_PENDING;
1751 }
1752
1753 void cdnsp_suspend_gadget(struct cdnsp_device *pdev)
1754 {
1755         if (pdev->gadget_driver && pdev->gadget_driver->suspend) {
1756                 spin_unlock(&pdev->lock);
1757                 pdev->gadget_driver->suspend(&pdev->gadget);
1758                 spin_lock(&pdev->lock);
1759         }
1760 }
1761
1762 void cdnsp_resume_gadget(struct cdnsp_device *pdev)
1763 {
1764         if (pdev->gadget_driver && pdev->gadget_driver->resume) {
1765                 spin_unlock(&pdev->lock);
1766                 pdev->gadget_driver->resume(&pdev->gadget);
1767                 spin_lock(&pdev->lock);
1768         }
1769 }
1770
1771 void cdnsp_irq_reset(struct cdnsp_device *pdev)
1772 {
1773         struct cdnsp_port_regs __iomem *port_regs;
1774
1775         cdnsp_reset_device(pdev);
1776
1777         port_regs = pdev->active_port->regs;
1778         pdev->gadget.speed = cdnsp_port_speed(readl(port_regs));
1779
1780         spin_unlock(&pdev->lock);
1781         usb_gadget_udc_reset(&pdev->gadget, pdev->gadget_driver);
1782         spin_lock(&pdev->lock);
1783
1784         switch (pdev->gadget.speed) {
1785         case USB_SPEED_SUPER_PLUS:
1786         case USB_SPEED_SUPER:
1787                 cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1788                 pdev->gadget.ep0->maxpacket = 512;
1789                 break;
1790         case USB_SPEED_HIGH:
1791         case USB_SPEED_FULL:
1792                 cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
1793                 pdev->gadget.ep0->maxpacket = 64;
1794                 break;
1795         default:
1796                 /* Low speed is not supported. */
1797                 dev_err(pdev->dev, "Unknown device speed\n");
1798                 break;
1799         }
1800
1801         cdnsp_clear_chicken_bits_2(pdev, CHICKEN_XDMA_2_TP_CACHE_DIS);
1802         cdnsp_setup_device(pdev, SETUP_CONTEXT_ONLY);
1803         usb_gadget_set_state(&pdev->gadget, USB_STATE_DEFAULT);
1804 }
1805
1806 static void cdnsp_get_rev_cap(struct cdnsp_device *pdev)
1807 {
1808         void __iomem *reg = &pdev->cap_regs->hc_capbase;
1809
1810         reg += cdnsp_find_next_ext_cap(reg, 0, RTL_REV_CAP);
1811         pdev->rev_cap  = reg;
1812
1813         pdev->rtl_revision = readl(&pdev->rev_cap->rtl_revision);
1814
1815         dev_info(pdev->dev, "Rev: %08x/%08x, eps: %08x, buff: %08x/%08x\n",
1816                  readl(&pdev->rev_cap->ctrl_revision),
1817                  readl(&pdev->rev_cap->rtl_revision),
1818                  readl(&pdev->rev_cap->ep_supported),
1819                  readl(&pdev->rev_cap->rx_buff_size),
1820                  readl(&pdev->rev_cap->tx_buff_size));
1821 }
1822
1823 static int cdnsp_gen_setup(struct cdnsp_device *pdev)
1824 {
1825         int ret;
1826         u32 reg;
1827
1828         pdev->cap_regs = pdev->regs;
1829         pdev->op_regs = pdev->regs +
1830                 HC_LENGTH(readl(&pdev->cap_regs->hc_capbase));
1831         pdev->run_regs = pdev->regs +
1832                 (readl(&pdev->cap_regs->run_regs_off) & RTSOFF_MASK);
1833
1834         /* Cache read-only capability registers */
1835         pdev->hcs_params1 = readl(&pdev->cap_regs->hcs_params1);
1836         pdev->hcc_params = readl(&pdev->cap_regs->hc_capbase);
1837         pdev->hci_version = HC_VERSION(pdev->hcc_params);
1838         pdev->hcc_params = readl(&pdev->cap_regs->hcc_params);
1839
1840         /*
1841          * Override the APB timeout value to give the controller more time for
1842          * enabling UTMI clock and synchronizing APB and UTMI clock domains.
1843          * This fix is platform specific and is required to fixes issue with
1844          * reading incorrect value from PORTSC register after resuming
1845          * from L1 state.
1846          */
1847         cdnsp_set_apb_timeout_value(pdev);
1848
1849         cdnsp_get_rev_cap(pdev);
1850
1851         /* Make sure the Device Controller is halted. */
1852         ret = cdnsp_halt(pdev);
1853         if (ret)
1854                 return ret;
1855
1856         /* Reset the internal controller memory state and registers. */
1857         ret = cdnsp_reset(pdev);
1858         if (ret)
1859                 return ret;
1860
1861         /*
1862          * Set dma_mask and coherent_dma_mask to 64-bits,
1863          * if controller supports 64-bit addressing.
1864          */
1865         if (HCC_64BIT_ADDR(pdev->hcc_params) &&
1866             !dma_set_mask(pdev->dev, DMA_BIT_MASK(64))) {
1867                 dev_dbg(pdev->dev, "Enabling 64-bit DMA addresses.\n");
1868                 dma_set_coherent_mask(pdev->dev, DMA_BIT_MASK(64));
1869         } else {
1870                 /*
1871                  * This is to avoid error in cases where a 32-bit USB
1872                  * controller is used on a 64-bit capable system.
1873                  */
1874                 ret = dma_set_mask(pdev->dev, DMA_BIT_MASK(32));
1875                 if (ret)
1876                         return ret;
1877
1878                 dev_dbg(pdev->dev, "Enabling 32-bit DMA addresses.\n");
1879                 dma_set_coherent_mask(pdev->dev, DMA_BIT_MASK(32));
1880         }
1881
1882         spin_lock_init(&pdev->lock);
1883
1884         ret = cdnsp_mem_init(pdev);
1885         if (ret)
1886                 return ret;
1887
1888         /*
1889          * Software workaround for U1: after transition
1890          * to U1 the controller starts gating clock, and in some cases,
1891          * it causes that controller stack.
1892          */
1893         reg = readl(&pdev->port3x_regs->mode_2);
1894         reg &= ~CFG_3XPORT_U1_PIPE_CLK_GATE_EN;
1895         writel(reg, &pdev->port3x_regs->mode_2);
1896
1897         return 0;
1898 }
1899
1900 static int __cdnsp_gadget_init(struct cdns *cdns)
1901 {
1902         struct cdnsp_device *pdev;
1903         u32 max_speed;
1904         int ret = -ENOMEM;
1905
1906         cdns_drd_gadget_on(cdns);
1907
1908         pdev = kzalloc(sizeof(*pdev), GFP_KERNEL);
1909         if (!pdev)
1910                 return -ENOMEM;
1911
1912         pm_runtime_get_sync(cdns->dev);
1913
1914         cdns->gadget_dev = pdev;
1915         pdev->dev = cdns->dev;
1916         pdev->regs = cdns->dev_regs;
1917         max_speed = usb_get_maximum_speed(cdns->dev);
1918
1919         switch (max_speed) {
1920         case USB_SPEED_FULL:
1921         case USB_SPEED_HIGH:
1922         case USB_SPEED_SUPER:
1923         case USB_SPEED_SUPER_PLUS:
1924                 break;
1925         default:
1926                 dev_err(cdns->dev, "invalid speed parameter %d\n", max_speed);
1927                 fallthrough;
1928         case USB_SPEED_UNKNOWN:
1929                 /* Default to SSP */
1930                 max_speed = USB_SPEED_SUPER_PLUS;
1931                 break;
1932         }
1933
1934         pdev->gadget.ops = &cdnsp_gadget_ops;
1935         pdev->gadget.name = "cdnsp-gadget";
1936         pdev->gadget.speed = USB_SPEED_UNKNOWN;
1937         pdev->gadget.sg_supported = 1;
1938         pdev->gadget.max_speed = max_speed;
1939         pdev->gadget.lpm_capable = 1;
1940
1941         pdev->setup_buf = kzalloc(CDNSP_EP0_SETUP_SIZE, GFP_KERNEL);
1942         if (!pdev->setup_buf)
1943                 goto free_pdev;
1944
1945         /*
1946          * Controller supports not aligned buffer but it should improve
1947          * performance.
1948          */
1949         pdev->gadget.quirk_ep_out_aligned_size = true;
1950
1951         ret = cdnsp_gen_setup(pdev);
1952         if (ret) {
1953                 dev_err(pdev->dev, "Generic initialization failed %d\n", ret);
1954                 goto free_setup;
1955         }
1956
1957         ret = cdnsp_gadget_init_endpoints(pdev);
1958         if (ret) {
1959                 dev_err(pdev->dev, "failed to initialize endpoints\n");
1960                 goto halt_pdev;
1961         }
1962
1963         ret = usb_add_gadget_udc(pdev->dev, &pdev->gadget);
1964         if (ret) {
1965                 dev_err(pdev->dev, "failed to register udc\n");
1966                 goto free_endpoints;
1967         }
1968
1969         ret = devm_request_threaded_irq(pdev->dev, cdns->dev_irq,
1970                                         cdnsp_irq_handler,
1971                                         cdnsp_thread_irq_handler, IRQF_SHARED,
1972                                         dev_name(pdev->dev), pdev);
1973         if (ret)
1974                 goto del_gadget;
1975
1976         return 0;
1977
1978 del_gadget:
1979         usb_del_gadget_udc(&pdev->gadget);
1980 free_endpoints:
1981         cdnsp_gadget_free_endpoints(pdev);
1982 halt_pdev:
1983         cdnsp_halt(pdev);
1984         cdnsp_reset(pdev);
1985         cdnsp_mem_cleanup(pdev);
1986 free_setup:
1987         kfree(pdev->setup_buf);
1988 free_pdev:
1989         kfree(pdev);
1990
1991         return ret;
1992 }
1993
1994 static void cdnsp_gadget_exit(struct cdns *cdns)
1995 {
1996         struct cdnsp_device *pdev = cdns->gadget_dev;
1997
1998         devm_free_irq(pdev->dev, cdns->dev_irq, pdev);
1999         pm_runtime_mark_last_busy(cdns->dev);
2000         pm_runtime_put_autosuspend(cdns->dev);
2001         usb_del_gadget_udc(&pdev->gadget);
2002         cdnsp_gadget_free_endpoints(pdev);
2003         cdnsp_mem_cleanup(pdev);
2004         kfree(pdev);
2005         cdns->gadget_dev = NULL;
2006         cdns_drd_gadget_off(cdns);
2007 }
2008
2009 static int cdnsp_gadget_suspend(struct cdns *cdns, bool do_wakeup)
2010 {
2011         struct cdnsp_device *pdev = cdns->gadget_dev;
2012         unsigned long flags;
2013
2014         if (pdev->link_state == XDEV_U3)
2015                 return 0;
2016
2017         spin_lock_irqsave(&pdev->lock, flags);
2018         cdnsp_disconnect_gadget(pdev);
2019         cdnsp_stop(pdev);
2020         spin_unlock_irqrestore(&pdev->lock, flags);
2021
2022         return 0;
2023 }
2024
2025 static int cdnsp_gadget_resume(struct cdns *cdns, bool lost_power)
2026 {
2027         struct cdnsp_device *pdev = cdns->gadget_dev;
2028         enum usb_device_speed max_speed;
2029         unsigned long flags;
2030         int ret;
2031
2032         if (!pdev->gadget_driver)
2033                 return 0;
2034
2035         spin_lock_irqsave(&pdev->lock, flags);
2036         max_speed = pdev->gadget_driver->max_speed;
2037
2038         /* Limit speed if necessary. */
2039         max_speed = min(max_speed, pdev->gadget.max_speed);
2040
2041         ret = cdnsp_run(pdev, max_speed);
2042
2043         if (pdev->link_state == XDEV_U3)
2044                 __cdnsp_gadget_wakeup(pdev);
2045
2046         spin_unlock_irqrestore(&pdev->lock, flags);
2047
2048         return ret;
2049 }
2050
2051 /**
2052  * cdnsp_gadget_init - initialize device structure
2053  * @cdns: cdnsp instance
2054  *
2055  * This function initializes the gadget.
2056  */
2057 int cdnsp_gadget_init(struct cdns *cdns)
2058 {
2059         struct cdns_role_driver *rdrv;
2060
2061         rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
2062         if (!rdrv)
2063                 return -ENOMEM;
2064
2065         rdrv->start     = __cdnsp_gadget_init;
2066         rdrv->stop      = cdnsp_gadget_exit;
2067         rdrv->suspend   = cdnsp_gadget_suspend;
2068         rdrv->resume    = cdnsp_gadget_resume;
2069         rdrv->state     = CDNS_ROLE_STATE_INACTIVE;
2070         rdrv->name      = "gadget";
2071         cdns->roles[USB_ROLE_DEVICE] = rdrv;
2072
2073         return 0;
2074 }