drivers/accel/qaic/qaic_control.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2
   3 /* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. */
   4 /* Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */
   5
   6 #include <asm/byteorder.h>
   7 #include <linux/completion.h>
   8 #include <linux/crc32.h>
   9 #include <linux/delay.h>
  10 #include <linux/dma-mapping.h>
  11 #include <linux/kref.h>
  12 #include <linux/list.h>
  13 #include <linux/mhi.h>
  14 #include <linux/mm.h>
  15 #include <linux/moduleparam.h>
  16 #include <linux/mutex.h>
  17 #include <linux/overflow.h>
  18 #include <linux/pci.h>
  19 #include <linux/scatterlist.h>
  20 #include <linux/types.h>
  21 #include <linux/uaccess.h>
  22 #include <linux/workqueue.h>
  23 #include <linux/wait.h>
  24 #include <drm/drm_device.h>
  25 #include <drm/drm_file.h>
  26 #include <uapi/drm/qaic_accel.h>
  27
  28 #include "qaic.h"
  29
  30 #define MANAGE_MAGIC_NUMBER             ((__force __le32)0x43494151) /* "QAIC" in little endian */
  31 #define QAIC_DBC_Q_GAP                  SZ_256
  32 #define QAIC_DBC_Q_BUF_ALIGN            SZ_4K
  33 #define QAIC_MANAGE_EXT_MSG_LENGTH      SZ_64K /* Max DMA message length */
  34 #define QAIC_WRAPPER_MAX_SIZE           SZ_4K
  35 #define QAIC_MHI_RETRY_WAIT_MS          100
  36 #define QAIC_MHI_RETRY_MAX              20
  37
  38 static unsigned int control_resp_timeout_s = 60; /* 60 sec default */
  39 module_param(control_resp_timeout_s, uint, 0600);
  40 MODULE_PARM_DESC(control_resp_timeout_s, "Timeout for NNC responses from QSM");
  41
  42 struct manage_msg {
  43         u32 len;
  44         u32 count;
  45         u8 data[];
  46 };
  47
  48 /*
  49  * wire encoding structures for the manage protocol.
  50  * All fields are little endian on the wire
  51  */
  52 struct wire_msg_hdr {
  53         __le32 crc32; /* crc of everything following this field in the message */
  54         __le32 magic_number;
  55         __le32 sequence_number;
  56         __le32 len; /* length of this message */
  57         __le32 count; /* number of transactions in this message */
  58         __le32 handle; /* unique id to track the resources consumed */
  59         __le32 partition_id; /* partition id for the request (signed) */
  60         __le32 padding; /* must be 0 */
  61 } __packed;
  62
  63 struct wire_msg {
  64         struct wire_msg_hdr hdr;
  65         u8 data[];
  66 } __packed;
  67
  68 struct wire_trans_hdr {
  69         __le32 type;
  70         __le32 len;
  71 } __packed;
  72
  73 /* Each message sent from driver to device are organized in a list of wrapper_msg */
  74 struct wrapper_msg {
  75         struct list_head list;
  76         struct kref ref_count;
  77         u32 len; /* length of data to transfer */
  78         struct wrapper_list *head;
  79         union {
  80                 struct wire_msg msg;
  81                 struct wire_trans_hdr trans;
  82         };
  83 };
  84
  85 struct wrapper_list {
  86         struct list_head list;
  87         spinlock_t lock; /* Protects the list state during additions and removals */
  88 };
  89
  90 struct wire_trans_passthrough {
  91         struct wire_trans_hdr hdr;
  92         u8 data[];
  93 } __packed;
  94
  95 struct wire_addr_size_pair {
  96         __le64 addr;
  97         __le64 size;
  98 } __packed;
  99
 100 struct wire_trans_dma_xfer {
 101         struct wire_trans_hdr hdr;
 102         __le32 tag;
 103         __le32 count;
 104         __le32 dma_chunk_id;
 105         __le32 padding;
 106         struct wire_addr_size_pair data[];
 107 } __packed;
 108
 109 /* Initiated by device to continue the DMA xfer of a large piece of data */
 110 struct wire_trans_dma_xfer_cont {
 111         struct wire_trans_hdr hdr;
 112         __le32 dma_chunk_id;
 113         __le32 padding;
 114         __le64 xferred_size;
 115 } __packed;
 116
 117 struct wire_trans_activate_to_dev {
 118         struct wire_trans_hdr hdr;
 119         __le64 req_q_addr;
 120         __le64 rsp_q_addr;
 121         __le32 req_q_size;
 122         __le32 rsp_q_size;
 123         __le32 buf_len;
 124         __le32 options; /* unused, but BIT(16) has meaning to the device */
 125 } __packed;
 126
 127 struct wire_trans_activate_from_dev {
 128         struct wire_trans_hdr hdr;
 129         __le32 status;
 130         __le32 dbc_id;
 131         __le64 options; /* unused */
 132 } __packed;
 133
 134 struct wire_trans_deactivate_from_dev {
 135         struct wire_trans_hdr hdr;
 136         __le32 status;
 137         __le32 dbc_id;
 138 } __packed;
 139
 140 struct wire_trans_terminate_to_dev {
 141         struct wire_trans_hdr hdr;
 142         __le32 handle;
 143         __le32 padding;
 144 } __packed;
 145
 146 struct wire_trans_terminate_from_dev {
 147         struct wire_trans_hdr hdr;
 148         __le32 status;
 149         __le32 padding;
 150 } __packed;
 151
 152 struct wire_trans_status_to_dev {
 153         struct wire_trans_hdr hdr;
 154 } __packed;
 155
 156 struct wire_trans_status_from_dev {
 157         struct wire_trans_hdr hdr;
 158         __le16 major;
 159         __le16 minor;
 160         __le32 status;
 161         __le64 status_flags;
 162 } __packed;
 163
 164 struct wire_trans_validate_part_to_dev {
 165         struct wire_trans_hdr hdr;
 166         __le32 part_id;
 167         __le32 padding;
 168 } __packed;
 169
 170 struct wire_trans_validate_part_from_dev {
 171         struct wire_trans_hdr hdr;
 172         __le32 status;
 173         __le32 padding;
 174 } __packed;
 175
 176 struct xfer_queue_elem {
 177         /*
 178          * Node in list of ongoing transfer request on control channel.
 179          * Maintained by root device struct.
 180          */
 181         struct list_head list;
 182         /* Sequence number of this transfer request */
 183         u32 seq_num;
 184         /* This is used to wait on until completion of transfer request */
 185         struct completion xfer_done;
 186         /* Received data from device */
 187         void *buf;
 188 };
 189
 190 struct dma_xfer {
 191         /* Node in list of DMA transfers which is used for cleanup */
 192         struct list_head list;
 193         /* SG table of memory used for DMA */
 194         struct sg_table *sgt;
 195         /* Array pages used for DMA */
 196         struct page **page_list;
 197         /* Number of pages used for DMA */
 198         unsigned long nr_pages;
 199 };
 200
 201 struct ioctl_resources {
 202         /* List of all DMA transfers which is used later for cleanup */
 203         struct list_head dma_xfers;
 204         /* Base address of request queue which belongs to a DBC */
 205         void *buf;
 206         /*
 207          * Base bus address of request queue which belongs to a DBC. Response
 208          * queue base bus address can be calculated by adding size of request
 209          * queue to base bus address of request queue.
 210          */
 211         dma_addr_t dma_addr;
 212         /* Total size of request queue and response queue in byte */
 213         u32 total_size;
 214         /* Total number of elements that can be queued in each of request and response queue */
 215         u32 nelem;
 216         /* Base address of response queue which belongs to a DBC */
 217         void *rsp_q_base;
 218         /* Status of the NNC message received */
 219         u32 status;
 220         /* DBC id of the DBC received from device */
 221         u32 dbc_id;
 222         /*
 223          * DMA transfer request messages can be big in size and it may not be
 224          * possible to send them in one shot. In such cases the messages are
 225          * broken into chunks, this field stores ID of such chunks.
 226          */
 227         u32 dma_chunk_id;
 228         /* Total number of bytes transferred for a DMA xfer request */
 229         u64 xferred_dma_size;
 230         /* Header of transaction message received from user. Used during DMA xfer request. */
 231         void *trans_hdr;
 232 };
 233
 234 struct resp_work {
 235         struct work_struct work;
 236         struct qaic_device *qdev;
 237         void *buf;
 238 };
 239
 240 /*
 241  * Since we're working with little endian messages, its useful to be able to
 242  * increment without filling a whole line with conversions back and forth just
 243  * to add one(1) to a message count.
 244  */
 245 static __le32 incr_le32(__le32 val)
 246 {
 247         return cpu_to_le32(le32_to_cpu(val) + 1);
 248 }
 249
 250 static u32 gen_crc(void *msg)
 251 {
 252         struct wrapper_list *wrappers = msg;
 253         struct wrapper_msg *w;
 254         u32 crc = ~0;
 255
 256         list_for_each_entry(w, &wrappers->list, list)
 257                 crc = crc32(crc, &w->msg, w->len);
 258
 259         return crc ^ ~0;
 260 }
 261
 262 static u32 gen_crc_stub(void *msg)
 263 {
 264         return 0;
 265 }
 266
 267 static bool valid_crc(void *msg)
 268 {
 269         struct wire_msg_hdr *hdr = msg;
 270         bool ret;
 271         u32 crc;
 272
 273         /*
 274          * The output of this algorithm is always converted to the native
 275          * endianness.
 276          */
 277         crc = le32_to_cpu(hdr->crc32);
 278         hdr->crc32 = 0;
 279         ret = (crc32(~0, msg, le32_to_cpu(hdr->len)) ^ ~0) == crc;
 280         hdr->crc32 = cpu_to_le32(crc);
 281         return ret;
 282 }
 283
 284 static bool valid_crc_stub(void *msg)
 285 {
 286         return true;
 287 }
 288
 289 static void free_wrapper(struct kref *ref)
 290 {
 291         struct wrapper_msg *wrapper = container_of(ref, struct wrapper_msg, ref_count);
 292
 293         list_del(&wrapper->list);
 294         kfree(wrapper);
 295 }
 296
 297 static void save_dbc_buf(struct qaic_device *qdev, struct ioctl_resources *resources,
 298                          struct qaic_user *usr)
 299 {
 300         u32 dbc_id = resources->dbc_id;
 301
 302         if (resources->buf) {
 303                 wait_event_interruptible(qdev->dbc[dbc_id].dbc_release, !qdev->dbc[dbc_id].in_use);
 304                 qdev->dbc[dbc_id].req_q_base = resources->buf;
 305                 qdev->dbc[dbc_id].rsp_q_base = resources->rsp_q_base;
 306                 qdev->dbc[dbc_id].dma_addr = resources->dma_addr;
 307                 qdev->dbc[dbc_id].total_size = resources->total_size;
 308                 qdev->dbc[dbc_id].nelem = resources->nelem;
 309                 enable_dbc(qdev, dbc_id, usr);
 310                 qdev->dbc[dbc_id].in_use = true;
 311                 resources->buf = NULL;
 312         }
 313 }
 314
 315 static void free_dbc_buf(struct qaic_device *qdev, struct ioctl_resources *resources)
 316 {
 317         if (resources->buf)
 318                 dma_free_coherent(&qdev->pdev->dev, resources->total_size, resources->buf,
 319                                   resources->dma_addr);
 320         resources->buf = NULL;
 321 }
 322
 323 static void free_dma_xfers(struct qaic_device *qdev, struct ioctl_resources *resources)
 324 {
 325         struct dma_xfer *xfer;
 326         struct dma_xfer *x;
 327         int i;
 328
 329         list_for_each_entry_safe(xfer, x, &resources->dma_xfers, list) {
 330                 dma_unmap_sgtable(&qdev->pdev->dev, xfer->sgt, DMA_TO_DEVICE, 0);
 331                 sg_free_table(xfer->sgt);
 332                 kfree(xfer->sgt);
 333                 for (i = 0; i < xfer->nr_pages; ++i)
 334                         put_page(xfer->page_list[i]);
 335                 kfree(xfer->page_list);
 336                 list_del(&xfer->list);
 337                 kfree(xfer);
 338         }
 339 }
 340
 341 static struct wrapper_msg *add_wrapper(struct wrapper_list *wrappers, u32 size)
 342 {
 343         struct wrapper_msg *w = kzalloc(size, GFP_KERNEL);
 344
 345         if (!w)
 346                 return NULL;
 347         list_add_tail(&w->list, &wrappers->list);
 348         kref_init(&w->ref_count);
 349         w->head = wrappers;
 350         return w;
 351 }
 352
 353 static int encode_passthrough(struct qaic_device *qdev, void *trans, struct wrapper_list *wrappers,
 354                               u32 *user_len)
 355 {
 356         struct qaic_manage_trans_passthrough *in_trans = trans;
 357         struct wire_trans_passthrough *out_trans;
 358         struct wrapper_msg *trans_wrapper;
 359         struct wrapper_msg *wrapper;
 360         struct wire_msg *msg;
 361         u32 msg_hdr_len;
 362
 363         wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
 364         msg = &wrapper->msg;
 365         msg_hdr_len = le32_to_cpu(msg->hdr.len);
 366
 367         if (in_trans->hdr.len % 8 != 0)
 368                 return -EINVAL;
 369
 370         if (size_add(msg_hdr_len, in_trans->hdr.len) > QAIC_MANAGE_EXT_MSG_LENGTH)
 371                 return -ENOSPC;
 372
 373         trans_wrapper = add_wrapper(wrappers,
 374                                     offsetof(struct wrapper_msg, trans) + in_trans->hdr.len);
 375         if (!trans_wrapper)
 376                 return -ENOMEM;
 377         trans_wrapper->len = in_trans->hdr.len;
 378         out_trans = (struct wire_trans_passthrough *)&trans_wrapper->trans;
 379
 380         memcpy(out_trans->data, in_trans->data, in_trans->hdr.len - sizeof(in_trans->hdr));
 381         msg->hdr.len = cpu_to_le32(msg_hdr_len + in_trans->hdr.len);
 382         msg->hdr.count = incr_le32(msg->hdr.count);
 383         *user_len += in_trans->hdr.len;
 384         out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_PASSTHROUGH_TO_DEV);
 385         out_trans->hdr.len = cpu_to_le32(in_trans->hdr.len);
 386
 387         return 0;
 388 }
 389
 390 /* returns error code for failure, 0 if enough pages alloc'd, 1 if dma_cont is needed */
 391 static int find_and_map_user_pages(struct qaic_device *qdev,
 392                                    struct qaic_manage_trans_dma_xfer *in_trans,
 393                                    struct ioctl_resources *resources, struct dma_xfer *xfer)
 394 {
 395         unsigned long need_pages;
 396         struct page **page_list;
 397         unsigned long nr_pages;
 398         struct sg_table *sgt;
 399         u64 xfer_start_addr;
 400         int ret;
 401         int i;
 402
 403         xfer_start_addr = in_trans->addr + resources->xferred_dma_size;
 404
 405         need_pages = DIV_ROUND_UP(in_trans->size + offset_in_page(xfer_start_addr) -
 406                                   resources->xferred_dma_size, PAGE_SIZE);
 407
 408         nr_pages = need_pages;
 409
 410         while (1) {
 411                 page_list = kmalloc_array(nr_pages, sizeof(*page_list), GFP_KERNEL | __GFP_NOWARN);
 412                 if (!page_list) {
 413                         nr_pages = nr_pages / 2;
 414                         if (!nr_pages)
 415                                 return -ENOMEM;
 416                 } else {
 417                         break;
 418                 }
 419         }
 420
 421         ret = get_user_pages_fast(xfer_start_addr, nr_pages, 0, page_list);
 422         if (ret < 0)
 423                 goto free_page_list;
 424         if (ret != nr_pages) {
 425                 nr_pages = ret;
 426                 ret = -EFAULT;
 427                 goto put_pages;
 428         }
 429
 430         sgt = kmalloc(sizeof(*sgt), GFP_KERNEL);
 431         if (!sgt) {
 432                 ret = -ENOMEM;
 433                 goto put_pages;
 434         }
 435
 436         ret = sg_alloc_table_from_pages(sgt, page_list, nr_pages,
 437                                         offset_in_page(xfer_start_addr),
 438                                         in_trans->size - resources->xferred_dma_size, GFP_KERNEL);
 439         if (ret) {
 440                 ret = -ENOMEM;
 441                 goto free_sgt;
 442         }
 443
 444         ret = dma_map_sgtable(&qdev->pdev->dev, sgt, DMA_TO_DEVICE, 0);
 445         if (ret)
 446                 goto free_table;
 447
 448         xfer->sgt = sgt;
 449         xfer->page_list = page_list;
 450         xfer->nr_pages = nr_pages;
 451
 452         return need_pages > nr_pages ? 1 : 0;
 453
 454 free_table:
 455         sg_free_table(sgt);
 456 free_sgt:
 457         kfree(sgt);
 458 put_pages:
 459         for (i = 0; i < nr_pages; ++i)
 460                 put_page(page_list[i]);
 461 free_page_list:
 462         kfree(page_list);
 463         return ret;
 464 }
 465
 466 /* returns error code for failure, 0 if everything was encoded, 1 if dma_cont is needed */
 467 static int encode_addr_size_pairs(struct dma_xfer *xfer, struct wrapper_list *wrappers,
 468                                   struct ioctl_resources *resources, u32 msg_hdr_len, u32 *size,
 469                                   struct wire_trans_dma_xfer **out_trans)
 470 {
 471         struct wrapper_msg *trans_wrapper;
 472         struct sg_table *sgt = xfer->sgt;
 473         struct wire_addr_size_pair *asp;
 474         struct scatterlist *sg;
 475         struct wrapper_msg *w;
 476         unsigned int dma_len;
 477         u64 dma_chunk_len;
 478         void *boundary;
 479         int nents_dma;
 480         int nents;
 481         int i;
 482
 483         nents = sgt->nents;
 484         nents_dma = nents;
 485         *size = QAIC_MANAGE_EXT_MSG_LENGTH - msg_hdr_len - sizeof(**out_trans);
 486         for_each_sgtable_sg(sgt, sg, i) {
 487                 *size -= sizeof(*asp);
 488                 /* Save 1K for possible follow-up transactions. */
 489                 if (*size < SZ_1K) {
 490                         nents_dma = i;
 491                         break;
 492                 }
 493         }
 494
 495         trans_wrapper = add_wrapper(wrappers, QAIC_WRAPPER_MAX_SIZE);
 496         if (!trans_wrapper)
 497                 return -ENOMEM;
 498         *out_trans = (struct wire_trans_dma_xfer *)&trans_wrapper->trans;
 499
 500         asp = (*out_trans)->data;
 501         boundary = (void *)trans_wrapper + QAIC_WRAPPER_MAX_SIZE;
 502         *size = 0;
 503
 504         dma_len = 0;
 505         w = trans_wrapper;
 506         dma_chunk_len = 0;
 507         for_each_sg(sgt->sgl, sg, nents_dma, i) {
 508                 asp->size = cpu_to_le64(dma_len);
 509                 dma_chunk_len += dma_len;
 510                 if (dma_len) {
 511                         asp++;
 512                         if ((void *)asp + sizeof(*asp) > boundary) {
 513                                 w->len = (void *)asp - (void *)&w->msg;
 514                                 *size += w->len;
 515                                 w = add_wrapper(wrappers, QAIC_WRAPPER_MAX_SIZE);
 516                                 if (!w)
 517                                         return -ENOMEM;
 518                                 boundary = (void *)w + QAIC_WRAPPER_MAX_SIZE;
 519                                 asp = (struct wire_addr_size_pair *)&w->msg;
 520                         }
 521                 }
 522                 asp->addr = cpu_to_le64(sg_dma_address(sg));
 523                 dma_len = sg_dma_len(sg);
 524         }
 525         /* finalize the last segment */
 526         asp->size = cpu_to_le64(dma_len);
 527         w->len = (void *)asp + sizeof(*asp) - (void *)&w->msg;
 528         *size += w->len;
 529         dma_chunk_len += dma_len;
 530         resources->xferred_dma_size += dma_chunk_len;
 531
 532         return nents_dma < nents ? 1 : 0;
 533 }
 534
 535 static void cleanup_xfer(struct qaic_device *qdev, struct dma_xfer *xfer)
 536 {
 537         int i;
 538
 539         dma_unmap_sgtable(&qdev->pdev->dev, xfer->sgt, DMA_TO_DEVICE, 0);
 540         sg_free_table(xfer->sgt);
 541         kfree(xfer->sgt);
 542         for (i = 0; i < xfer->nr_pages; ++i)
 543                 put_page(xfer->page_list[i]);
 544         kfree(xfer->page_list);
 545 }
 546
 547 static int encode_dma(struct qaic_device *qdev, void *trans, struct wrapper_list *wrappers,
 548                       u32 *user_len, struct ioctl_resources *resources, struct qaic_user *usr)
 549 {
 550         struct qaic_manage_trans_dma_xfer *in_trans = trans;
 551         struct wire_trans_dma_xfer *out_trans;
 552         struct wrapper_msg *wrapper;
 553         struct dma_xfer *xfer;
 554         struct wire_msg *msg;
 555         bool need_cont_dma;
 556         u32 msg_hdr_len;
 557         u32 size;
 558         int ret;
 559
 560         wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
 561         msg = &wrapper->msg;
 562         msg_hdr_len = le32_to_cpu(msg->hdr.len);
 563
 564         /* There should be enough space to hold at least one ASP entry. */
 565         if (size_add(msg_hdr_len, sizeof(*out_trans) + sizeof(struct wire_addr_size_pair)) >
 566             QAIC_MANAGE_EXT_MSG_LENGTH)
 567                 return -ENOMEM;
 568
 569         if (in_trans->addr + in_trans->size < in_trans->addr || !in_trans->size)
 570                 return -EINVAL;
 571
 572         xfer = kmalloc(sizeof(*xfer), GFP_KERNEL);
 573         if (!xfer)
 574                 return -ENOMEM;
 575
 576         ret = find_and_map_user_pages(qdev, in_trans, resources, xfer);
 577         if (ret < 0)
 578                 goto free_xfer;
 579
 580         need_cont_dma = (bool)ret;
 581
 582         ret = encode_addr_size_pairs(xfer, wrappers, resources, msg_hdr_len, &size, &out_trans);
 583         if (ret < 0)
 584                 goto cleanup_xfer;
 585
 586         need_cont_dma = need_cont_dma || (bool)ret;
 587
 588         msg->hdr.len = cpu_to_le32(msg_hdr_len + size);
 589         msg->hdr.count = incr_le32(msg->hdr.count);
 590
 591         out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_DMA_XFER_TO_DEV);
 592         out_trans->hdr.len = cpu_to_le32(size);
 593         out_trans->tag = cpu_to_le32(in_trans->tag);
 594         out_trans->count = cpu_to_le32((size - sizeof(*out_trans)) /
 595                                                                 sizeof(struct wire_addr_size_pair));
 596
 597         *user_len += in_trans->hdr.len;
 598
 599         if (resources->dma_chunk_id) {
 600                 out_trans->dma_chunk_id = cpu_to_le32(resources->dma_chunk_id);
 601         } else if (need_cont_dma) {
 602                 while (resources->dma_chunk_id == 0)
 603                         resources->dma_chunk_id = atomic_inc_return(&usr->chunk_id);
 604
 605                 out_trans->dma_chunk_id = cpu_to_le32(resources->dma_chunk_id);
 606         }
 607         resources->trans_hdr = trans;
 608
 609         list_add(&xfer->list, &resources->dma_xfers);
 610         return 0;
 611
 612 cleanup_xfer:
 613         cleanup_xfer(qdev, xfer);
 614 free_xfer:
 615         kfree(xfer);
 616         return ret;
 617 }
 618
 619 static int encode_activate(struct qaic_device *qdev, void *trans, struct wrapper_list *wrappers,
 620                            u32 *user_len, struct ioctl_resources *resources)
 621 {
 622         struct qaic_manage_trans_activate_to_dev *in_trans = trans;
 623         struct wire_trans_activate_to_dev *out_trans;
 624         struct wrapper_msg *trans_wrapper;
 625         struct wrapper_msg *wrapper;
 626         struct wire_msg *msg;
 627         dma_addr_t dma_addr;
 628         u32 msg_hdr_len;
 629         void *buf;
 630         u32 nelem;
 631         u32 size;
 632         int ret;
 633
 634         wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
 635         msg = &wrapper->msg;
 636         msg_hdr_len = le32_to_cpu(msg->hdr.len);
 637
 638         if (size_add(msg_hdr_len, sizeof(*out_trans)) > QAIC_MANAGE_MAX_MSG_LENGTH)
 639                 return -ENOSPC;
 640
 641         if (!in_trans->queue_size)
 642                 return -EINVAL;
 643
 644         if (in_trans->pad)
 645                 return -EINVAL;
 646
 647         nelem = in_trans->queue_size;
 648         size = (get_dbc_req_elem_size() + get_dbc_rsp_elem_size()) * nelem;
 649         if (size / nelem != get_dbc_req_elem_size() + get_dbc_rsp_elem_size())
 650                 return -EINVAL;
 651
 652         if (size + QAIC_DBC_Q_GAP + QAIC_DBC_Q_BUF_ALIGN < size)
 653                 return -EINVAL;
 654
 655         size = ALIGN((size + QAIC_DBC_Q_GAP), QAIC_DBC_Q_BUF_ALIGN);
 656
 657         buf = dma_alloc_coherent(&qdev->pdev->dev, size, &dma_addr, GFP_KERNEL);
 658         if (!buf)
 659                 return -ENOMEM;
 660
 661         trans_wrapper = add_wrapper(wrappers,
 662                                     offsetof(struct wrapper_msg, trans) + sizeof(*out_trans));
 663         if (!trans_wrapper) {
 664                 ret = -ENOMEM;
 665                 goto free_dma;
 666         }
 667         trans_wrapper->len = sizeof(*out_trans);
 668         out_trans = (struct wire_trans_activate_to_dev *)&trans_wrapper->trans;
 669
 670         out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_ACTIVATE_TO_DEV);
 671         out_trans->hdr.len = cpu_to_le32(sizeof(*out_trans));
 672         out_trans->buf_len = cpu_to_le32(size);
 673         out_trans->req_q_addr = cpu_to_le64(dma_addr);
 674         out_trans->req_q_size = cpu_to_le32(nelem);
 675         out_trans->rsp_q_addr = cpu_to_le64(dma_addr + size - nelem * get_dbc_rsp_elem_size());
 676         out_trans->rsp_q_size = cpu_to_le32(nelem);
 677         out_trans->options = cpu_to_le32(in_trans->options);
 678
 679         *user_len += in_trans->hdr.len;
 680         msg->hdr.len = cpu_to_le32(msg_hdr_len + sizeof(*out_trans));
 681         msg->hdr.count = incr_le32(msg->hdr.count);
 682
 683         resources->buf = buf;
 684         resources->dma_addr = dma_addr;
 685         resources->total_size = size;
 686         resources->nelem = nelem;
 687         resources->rsp_q_base = buf + size - nelem * get_dbc_rsp_elem_size();
 688         return 0;
 689
 690 free_dma:
 691         dma_free_coherent(&qdev->pdev->dev, size, buf, dma_addr);
 692         return ret;
 693 }
 694
 695 static int encode_deactivate(struct qaic_device *qdev, void *trans,
 696                              u32 *user_len, struct qaic_user *usr)
 697 {
 698         struct qaic_manage_trans_deactivate *in_trans = trans;
 699
 700         if (in_trans->dbc_id >= qdev->num_dbc || in_trans->pad)
 701                 return -EINVAL;
 702
 703         *user_len += in_trans->hdr.len;
 704
 705         return disable_dbc(qdev, in_trans->dbc_id, usr);
 706 }
 707
 708 static int encode_status(struct qaic_device *qdev, void *trans, struct wrapper_list *wrappers,
 709                          u32 *user_len)
 710 {
 711         struct qaic_manage_trans_status_to_dev *in_trans = trans;
 712         struct wire_trans_status_to_dev *out_trans;
 713         struct wrapper_msg *trans_wrapper;
 714         struct wrapper_msg *wrapper;
 715         struct wire_msg *msg;
 716         u32 msg_hdr_len;
 717
 718         wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
 719         msg = &wrapper->msg;
 720         msg_hdr_len = le32_to_cpu(msg->hdr.len);
 721
 722         if (size_add(msg_hdr_len, in_trans->hdr.len) > QAIC_MANAGE_MAX_MSG_LENGTH)
 723                 return -ENOSPC;
 724
 725         trans_wrapper = add_wrapper(wrappers, sizeof(*trans_wrapper));
 726         if (!trans_wrapper)
 727                 return -ENOMEM;
 728
 729         trans_wrapper->len = sizeof(*out_trans);
 730         out_trans = (struct wire_trans_status_to_dev *)&trans_wrapper->trans;
 731
 732         out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_STATUS_TO_DEV);
 733         out_trans->hdr.len = cpu_to_le32(in_trans->hdr.len);
 734         msg->hdr.len = cpu_to_le32(msg_hdr_len + in_trans->hdr.len);
 735         msg->hdr.count = incr_le32(msg->hdr.count);
 736         *user_len += in_trans->hdr.len;
 737
 738         return 0;
 739 }
 740
 741 static int encode_message(struct qaic_device *qdev, struct manage_msg *user_msg,
 742                           struct wrapper_list *wrappers, struct ioctl_resources *resources,
 743                           struct qaic_user *usr)
 744 {
 745         struct qaic_manage_trans_hdr *trans_hdr;
 746         struct wrapper_msg *wrapper;
 747         struct wire_msg *msg;
 748         u32 user_len = 0;
 749         int ret;
 750         int i;
 751
 752         if (!user_msg->count ||
 753             user_msg->len < sizeof(*trans_hdr)) {
 754                 ret = -EINVAL;
 755                 goto out;
 756         }
 757
 758         wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
 759         msg = &wrapper->msg;
 760
 761         msg->hdr.len = cpu_to_le32(sizeof(msg->hdr));
 762
 763         if (resources->dma_chunk_id) {
 764                 ret = encode_dma(qdev, resources->trans_hdr, wrappers, &user_len, resources, usr);
 765                 msg->hdr.count = cpu_to_le32(1);
 766                 goto out;
 767         }
 768
 769         for (i = 0; i < user_msg->count; ++i) {
 770                 if (user_len > user_msg->len - sizeof(*trans_hdr)) {
 771                         ret = -EINVAL;
 772                         break;
 773                 }
 774                 trans_hdr = (struct qaic_manage_trans_hdr *)(user_msg->data + user_len);
 775                 if (trans_hdr->len < sizeof(trans_hdr) ||
 776                     size_add(user_len, trans_hdr->len) > user_msg->len) {
 777                         ret = -EINVAL;
 778                         break;
 779                 }
 780
 781                 switch (trans_hdr->type) {
 782                 case QAIC_TRANS_PASSTHROUGH_FROM_USR:
 783                         ret = encode_passthrough(qdev, trans_hdr, wrappers, &user_len);
 784                         break;
 785                 case QAIC_TRANS_DMA_XFER_FROM_USR:
 786                         ret = encode_dma(qdev, trans_hdr, wrappers, &user_len, resources, usr);
 787                         break;
 788                 case QAIC_TRANS_ACTIVATE_FROM_USR:
 789                         ret = encode_activate(qdev, trans_hdr, wrappers, &user_len, resources);
 790                         break;
 791                 case QAIC_TRANS_DEACTIVATE_FROM_USR:
 792                         ret = encode_deactivate(qdev, trans_hdr, &user_len, usr);
 793                         break;
 794                 case QAIC_TRANS_STATUS_FROM_USR:
 795                         ret = encode_status(qdev, trans_hdr, wrappers, &user_len);
 796                         break;
 797                 default:
 798                         ret = -EINVAL;
 799                         break;
 800                 }
 801
 802                 if (ret)
 803                         break;
 804         }
 805
 806         if (user_len != user_msg->len)
 807                 ret = -EINVAL;
 808 out:
 809         if (ret) {
 810                 free_dma_xfers(qdev, resources);
 811                 free_dbc_buf(qdev, resources);
 812                 return ret;
 813         }
 814
 815         return 0;
 816 }
 817
 818 static int decode_passthrough(struct qaic_device *qdev, void *trans, struct manage_msg *user_msg,
 819                               u32 *msg_len)
 820 {
 821         struct qaic_manage_trans_passthrough *out_trans;
 822         struct wire_trans_passthrough *in_trans = trans;
 823         u32 len;
 824
 825         out_trans = (void *)user_msg->data + user_msg->len;
 826
 827         len = le32_to_cpu(in_trans->hdr.len);
 828         if (len % 8 != 0)
 829                 return -EINVAL;
 830
 831         if (user_msg->len + len > QAIC_MANAGE_MAX_MSG_LENGTH)
 832                 return -ENOSPC;
 833
 834         memcpy(out_trans->data, in_trans->data, len - sizeof(in_trans->hdr));
 835         user_msg->len += len;
 836         *msg_len += len;
 837         out_trans->hdr.type = le32_to_cpu(in_trans->hdr.type);
 838         out_trans->hdr.len = len;
 839
 840         return 0;
 841 }
 842
 843 static int decode_activate(struct qaic_device *qdev, void *trans, struct manage_msg *user_msg,
 844                            u32 *msg_len, struct ioctl_resources *resources, struct qaic_user *usr)
 845 {
 846         struct qaic_manage_trans_activate_from_dev *out_trans;
 847         struct wire_trans_activate_from_dev *in_trans = trans;
 848         u32 len;
 849
 850         out_trans = (void *)user_msg->data + user_msg->len;
 851
 852         len = le32_to_cpu(in_trans->hdr.len);
 853         if (user_msg->len + len > QAIC_MANAGE_MAX_MSG_LENGTH)
 854                 return -ENOSPC;
 855
 856         user_msg->len += len;
 857         *msg_len += len;
 858         out_trans->hdr.type = le32_to_cpu(in_trans->hdr.type);
 859         out_trans->hdr.len = len;
 860         out_trans->status = le32_to_cpu(in_trans->status);
 861         out_trans->dbc_id = le32_to_cpu(in_trans->dbc_id);
 862         out_trans->options = le64_to_cpu(in_trans->options);
 863
 864         if (!resources->buf)
 865                 /* how did we get an activate response without a request? */
 866                 return -EINVAL;
 867
 868         if (out_trans->dbc_id >= qdev->num_dbc)
 869                 /*
 870                  * The device assigned an invalid resource, which should never
 871                  * happen. Return an error so the user can try to recover.
 872                  */
 873                 return -ENODEV;
 874
 875         if (out_trans->status)
 876                 /*
 877                  * Allocating resources failed on device side. This is not an
 878                  * expected behaviour, user is expected to handle this situation.
 879                  */
 880                 return -ECANCELED;
 881
 882         resources->status = out_trans->status;
 883         resources->dbc_id = out_trans->dbc_id;
 884         save_dbc_buf(qdev, resources, usr);
 885
 886         return 0;
 887 }
 888
 889 static int decode_deactivate(struct qaic_device *qdev, void *trans, u32 *msg_len,
 890                              struct qaic_user *usr)
 891 {
 892         struct wire_trans_deactivate_from_dev *in_trans = trans;
 893         u32 dbc_id = le32_to_cpu(in_trans->dbc_id);
 894         u32 status = le32_to_cpu(in_trans->status);
 895
 896         if (dbc_id >= qdev->num_dbc)
 897                 /*
 898                  * The device assigned an invalid resource, which should never
 899                  * happen. Inject an error so the user can try to recover.
 900                  */
 901                 return -ENODEV;
 902
 903         if (status) {
 904                 /*
 905                  * Releasing resources failed on the device side, which puts
 906                  * us in a bind since they may still be in use, so enable the
 907                  * dbc. User is expected to retry deactivation.
 908                  */
 909                 enable_dbc(qdev, dbc_id, usr);
 910                 return -ECANCELED;
 911         }
 912
 913         release_dbc(qdev, dbc_id);
 914         *msg_len += sizeof(*in_trans);
 915
 916         return 0;
 917 }
 918
 919 static int decode_status(struct qaic_device *qdev, void *trans, struct manage_msg *user_msg,
 920                          u32 *user_len, struct wire_msg *msg)
 921 {
 922         struct qaic_manage_trans_status_from_dev *out_trans;
 923         struct wire_trans_status_from_dev *in_trans = trans;
 924         u32 len;
 925
 926         out_trans = (void *)user_msg->data + user_msg->len;
 927
 928         len = le32_to_cpu(in_trans->hdr.len);
 929         if (user_msg->len + len > QAIC_MANAGE_MAX_MSG_LENGTH)
 930                 return -ENOSPC;
 931
 932         out_trans->hdr.type = QAIC_TRANS_STATUS_FROM_DEV;
 933         out_trans->hdr.len = len;
 934         out_trans->major = le16_to_cpu(in_trans->major);
 935         out_trans->minor = le16_to_cpu(in_trans->minor);
 936         out_trans->status_flags = le64_to_cpu(in_trans->status_flags);
 937         out_trans->status = le32_to_cpu(in_trans->status);
 938         *user_len += le32_to_cpu(in_trans->hdr.len);
 939         user_msg->len += len;
 940
 941         if (out_trans->status)
 942                 return -ECANCELED;
 943         if (out_trans->status_flags & BIT(0) && !valid_crc(msg))
 944                 return -EPIPE;
 945
 946         return 0;
 947 }
 948
 949 static int decode_message(struct qaic_device *qdev, struct manage_msg *user_msg,
 950                           struct wire_msg *msg, struct ioctl_resources *resources,
 951                           struct qaic_user *usr)
 952 {
 953         u32 msg_hdr_len = le32_to_cpu(msg->hdr.len);
 954         struct wire_trans_hdr *trans_hdr;
 955         u32 msg_len = 0;
 956         int ret;
 957         int i;
 958
 959         if (msg_hdr_len < sizeof(*trans_hdr) ||
 960             msg_hdr_len > QAIC_MANAGE_MAX_MSG_LENGTH)
 961                 return -EINVAL;
 962
 963         user_msg->len = 0;
 964         user_msg->count = le32_to_cpu(msg->hdr.count);
 965
 966         for (i = 0; i < user_msg->count; ++i) {
 967                 u32 hdr_len;
 968
 969                 if (msg_len > msg_hdr_len - sizeof(*trans_hdr))
 970                         return -EINVAL;
 971
 972                 trans_hdr = (struct wire_trans_hdr *)(msg->data + msg_len);
 973                 hdr_len = le32_to_cpu(trans_hdr->len);
 974                 if (hdr_len < sizeof(*trans_hdr) ||
 975                     size_add(msg_len, hdr_len) > msg_hdr_len)
 976                         return -EINVAL;
 977
 978                 switch (le32_to_cpu(trans_hdr->type)) {
 979                 case QAIC_TRANS_PASSTHROUGH_FROM_DEV:
 980                         ret = decode_passthrough(qdev, trans_hdr, user_msg, &msg_len);
 981                         break;
 982                 case QAIC_TRANS_ACTIVATE_FROM_DEV:
 983                         ret = decode_activate(qdev, trans_hdr, user_msg, &msg_len, resources, usr);
 984                         break;
 985                 case QAIC_TRANS_DEACTIVATE_FROM_DEV:
 986                         ret = decode_deactivate(qdev, trans_hdr, &msg_len, usr);
 987                         break;
 988                 case QAIC_TRANS_STATUS_FROM_DEV:
 989                         ret = decode_status(qdev, trans_hdr, user_msg, &msg_len, msg);
 990                         break;
 991                 default:
 992                         return -EINVAL;
 993                 }
 994
 995                 if (ret)
 996                         return ret;
 997         }
 998
 999         if (msg_len != (msg_hdr_len - sizeof(msg->hdr)))
1000                 return -EINVAL;
1001
1002         return 0;
1003 }
1004
1005 static void *msg_xfer(struct qaic_device *qdev, struct wrapper_list *wrappers, u32 seq_num,
1006                       bool ignore_signal)
1007 {
1008         struct xfer_queue_elem elem;
1009         struct wire_msg *out_buf;
1010         struct wrapper_msg *w;
1011         long ret = -EAGAIN;
1012         int xfer_count = 0;
1013         int retry_count;
1014
1015         if (qdev->in_reset) {
1016                 mutex_unlock(&qdev->cntl_mutex);
1017                 return ERR_PTR(-ENODEV);
1018         }
1019
1020         /* Attempt to avoid a partial commit of a message */
1021         list_for_each_entry(w, &wrappers->list, list)
1022                 xfer_count++;
1023
1024         for (retry_count = 0; retry_count < QAIC_MHI_RETRY_MAX; retry_count++) {
1025                 if (xfer_count <= mhi_get_free_desc_count(qdev->cntl_ch, DMA_TO_DEVICE)) {
1026                         ret = 0;
1027                         break;
1028                 }
1029                 msleep_interruptible(QAIC_MHI_RETRY_WAIT_MS);
1030                 if (signal_pending(current))
1031                         break;
1032         }
1033
1034         if (ret) {
1035                 mutex_unlock(&qdev->cntl_mutex);
1036                 return ERR_PTR(ret);
1037         }
1038
1039         elem.seq_num = seq_num;
1040         elem.buf = NULL;
1041         init_completion(&elem.xfer_done);
1042         if (likely(!qdev->cntl_lost_buf)) {
1043                 /*
1044                  * The max size of request to device is QAIC_MANAGE_EXT_MSG_LENGTH.
1045                  * The max size of response from device is QAIC_MANAGE_MAX_MSG_LENGTH.
1046                  */
1047                 out_buf = kmalloc(QAIC_MANAGE_MAX_MSG_LENGTH, GFP_KERNEL);
1048                 if (!out_buf) {
1049                         mutex_unlock(&qdev->cntl_mutex);
1050                         return ERR_PTR(-ENOMEM);
1051                 }
1052
1053                 ret = mhi_queue_buf(qdev->cntl_ch, DMA_FROM_DEVICE, out_buf,
1054                                     QAIC_MANAGE_MAX_MSG_LENGTH, MHI_EOT);
1055                 if (ret) {
1056                         mutex_unlock(&qdev->cntl_mutex);
1057                         return ERR_PTR(ret);
1058                 }
1059         } else {
1060                 /*
1061                  * we lost a buffer because we queued a recv buf, but then
1062                  * queuing the corresponding tx buf failed. To try to avoid
1063                  * a memory leak, lets reclaim it and use it for this
1064                  * transaction.
1065                  */
1066                 qdev->cntl_lost_buf = false;
1067         }
1068
1069         list_for_each_entry(w, &wrappers->list, list) {
1070                 kref_get(&w->ref_count);
1071                 retry_count = 0;
1072                 ret = mhi_queue_buf(qdev->cntl_ch, DMA_TO_DEVICE, &w->msg, w->len,
1073                                     list_is_last(&w->list, &wrappers->list) ? MHI_EOT : MHI_CHAIN);
1074                 if (ret) {
1075                         qdev->cntl_lost_buf = true;
1076                         kref_put(&w->ref_count, free_wrapper);
1077                         mutex_unlock(&qdev->cntl_mutex);
1078                         return ERR_PTR(ret);
1079                 }
1080         }
1081
1082         list_add_tail(&elem.list, &qdev->cntl_xfer_list);
1083         mutex_unlock(&qdev->cntl_mutex);
1084
1085         if (ignore_signal)
1086                 ret = wait_for_completion_timeout(&elem.xfer_done, control_resp_timeout_s * HZ);
1087         else
1088                 ret = wait_for_completion_interruptible_timeout(&elem.xfer_done,
1089                                                                 control_resp_timeout_s * HZ);
1090         /*
1091          * not using _interruptable because we have to cleanup or we'll
1092          * likely cause memory corruption
1093          */
1094         mutex_lock(&qdev->cntl_mutex);
1095         if (!list_empty(&elem.list))
1096                 list_del(&elem.list);
1097         if (!ret && !elem.buf)
1098                 ret = -ETIMEDOUT;
1099         else if (ret > 0 && !elem.buf)
1100                 ret = -EIO;
1101         mutex_unlock(&qdev->cntl_mutex);
1102
1103         if (ret < 0) {
1104                 kfree(elem.buf);
1105                 return ERR_PTR(ret);
1106         } else if (!qdev->valid_crc(elem.buf)) {
1107                 kfree(elem.buf);
1108                 return ERR_PTR(-EPIPE);
1109         }
1110
1111         return elem.buf;
1112 }
1113
1114 /* Add a transaction to abort the outstanding DMA continuation */
1115 static int abort_dma_cont(struct qaic_device *qdev, struct wrapper_list *wrappers, u32 dma_chunk_id)
1116 {
1117         struct wire_trans_dma_xfer *out_trans;
1118         u32 size = sizeof(*out_trans);
1119         struct wrapper_msg *wrapper;
1120         struct wrapper_msg *w;
1121         struct wire_msg *msg;
1122
1123         wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
1124         msg = &wrapper->msg;
1125
1126         /* Remove all but the first wrapper which has the msg header */
1127         list_for_each_entry_safe(wrapper, w, &wrappers->list, list)
1128                 if (!list_is_first(&wrapper->list, &wrappers->list))
1129                         kref_put(&wrapper->ref_count, free_wrapper);
1130
1131         wrapper = add_wrapper(wrappers, offsetof(struct wrapper_msg, trans) + sizeof(*out_trans));
1132
1133         if (!wrapper)
1134                 return -ENOMEM;
1135
1136         out_trans = (struct wire_trans_dma_xfer *)&wrapper->trans;
1137         out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_DMA_XFER_TO_DEV);
1138         out_trans->hdr.len = cpu_to_le32(size);
1139         out_trans->tag = cpu_to_le32(0);
1140         out_trans->count = cpu_to_le32(0);
1141         out_trans->dma_chunk_id = cpu_to_le32(dma_chunk_id);
1142
1143         msg->hdr.len = cpu_to_le32(size + sizeof(*msg));
1144         msg->hdr.count = cpu_to_le32(1);
1145         wrapper->len = size;
1146
1147         return 0;
1148 }
1149
1150 static struct wrapper_list *alloc_wrapper_list(void)
1151 {
1152         struct wrapper_list *wrappers;
1153
1154         wrappers = kmalloc(sizeof(*wrappers), GFP_KERNEL);
1155         if (!wrappers)
1156                 return NULL;
1157         INIT_LIST_HEAD(&wrappers->list);
1158         spin_lock_init(&wrappers->lock);
1159
1160         return wrappers;
1161 }
1162
1163 static int qaic_manage_msg_xfer(struct qaic_device *qdev, struct qaic_user *usr,
1164                                 struct manage_msg *user_msg, struct ioctl_resources *resources,
1165                                 struct wire_msg **rsp)
1166 {
1167         struct wrapper_list *wrappers;
1168         struct wrapper_msg *wrapper;
1169         struct wrapper_msg *w;
1170         bool all_done = false;
1171         struct wire_msg *msg;
1172         int ret;
1173
1174         wrappers = alloc_wrapper_list();
1175         if (!wrappers)
1176                 return -ENOMEM;
1177
1178         wrapper = add_wrapper(wrappers, sizeof(*wrapper));
1179         if (!wrapper) {
1180                 kfree(wrappers);
1181                 return -ENOMEM;
1182         }
1183
1184         msg = &wrapper->msg;
1185         wrapper->len = sizeof(*msg);
1186
1187         ret = encode_message(qdev, user_msg, wrappers, resources, usr);
1188         if (ret && resources->dma_chunk_id)
1189                 ret = abort_dma_cont(qdev, wrappers, resources->dma_chunk_id);
1190         if (ret)
1191                 goto encode_failed;
1192
1193         ret = mutex_lock_interruptible(&qdev->cntl_mutex);
1194         if (ret)
1195                 goto lock_failed;
1196
1197         msg->hdr.magic_number = MANAGE_MAGIC_NUMBER;
1198         msg->hdr.sequence_number = cpu_to_le32(qdev->next_seq_num++);
1199
1200         if (usr) {
1201                 msg->hdr.handle = cpu_to_le32(usr->handle);
1202                 msg->hdr.partition_id = cpu_to_le32(usr->qddev->partition_id);
1203         } else {
1204                 msg->hdr.handle = 0;
1205                 msg->hdr.partition_id = cpu_to_le32(QAIC_NO_PARTITION);
1206         }
1207
1208         msg->hdr.padding = cpu_to_le32(0);
1209         msg->hdr.crc32 = cpu_to_le32(qdev->gen_crc(wrappers));
1210
1211         /* msg_xfer releases the mutex */
1212         *rsp = msg_xfer(qdev, wrappers, qdev->next_seq_num - 1, false);
1213         if (IS_ERR(*rsp))
1214                 ret = PTR_ERR(*rsp);
1215
1216 lock_failed:
1217         free_dma_xfers(qdev, resources);
1218 encode_failed:
1219         spin_lock(&wrappers->lock);
1220         list_for_each_entry_safe(wrapper, w, &wrappers->list, list)
1221                 kref_put(&wrapper->ref_count, free_wrapper);
1222         all_done = list_empty(&wrappers->list);
1223         spin_unlock(&wrappers->lock);
1224         if (all_done)
1225                 kfree(wrappers);
1226
1227         return ret;
1228 }
1229
1230 static int qaic_manage(struct qaic_device *qdev, struct qaic_user *usr, struct manage_msg *user_msg)
1231 {
1232         struct wire_trans_dma_xfer_cont *dma_cont = NULL;
1233         struct ioctl_resources resources;
1234         struct wire_msg *rsp = NULL;
1235         int ret;
1236
1237         memset(&resources, 0, sizeof(struct ioctl_resources));
1238
1239         INIT_LIST_HEAD(&resources.dma_xfers);
1240
1241         if (user_msg->len > QAIC_MANAGE_MAX_MSG_LENGTH ||
1242             user_msg->count > QAIC_MANAGE_MAX_MSG_LENGTH / sizeof(struct qaic_manage_trans_hdr))
1243                 return -EINVAL;
1244
1245 dma_xfer_continue:
1246         ret = qaic_manage_msg_xfer(qdev, usr, user_msg, &resources, &rsp);
1247         if (ret)
1248                 return ret;
1249         /* dma_cont should be the only transaction if present */
1250         if (le32_to_cpu(rsp->hdr.count) == 1) {
1251                 dma_cont = (struct wire_trans_dma_xfer_cont *)rsp->data;
1252                 if (le32_to_cpu(dma_cont->hdr.type) != QAIC_TRANS_DMA_XFER_CONT)
1253                         dma_cont = NULL;
1254         }
1255         if (dma_cont) {
1256                 if (le32_to_cpu(dma_cont->dma_chunk_id) == resources.dma_chunk_id &&
1257                     le64_to_cpu(dma_cont->xferred_size) == resources.xferred_dma_size) {
1258                         kfree(rsp);
1259                         goto dma_xfer_continue;
1260                 }
1261
1262                 ret = -EINVAL;
1263                 goto dma_cont_failed;
1264         }
1265
1266         ret = decode_message(qdev, user_msg, rsp, &resources, usr);
1267
1268 dma_cont_failed:
1269         free_dbc_buf(qdev, &resources);
1270         kfree(rsp);
1271         return ret;
1272 }
1273
1274 int qaic_manage_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
1275 {
1276         struct qaic_manage_msg *user_msg = data;
1277         struct qaic_device *qdev;
1278         struct manage_msg *msg;
1279         struct qaic_user *usr;
1280         u8 __user *user_data;
1281         int qdev_rcu_id;
1282         int usr_rcu_id;
1283         int ret;
1284
1285         if (user_msg->len > QAIC_MANAGE_MAX_MSG_LENGTH)
1286                 return -EINVAL;
1287
1288         usr = file_priv->driver_priv;
1289
1290         usr_rcu_id = srcu_read_lock(&usr->qddev_lock);
1291         if (!usr->qddev) {
1292                 srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
1293                 return -ENODEV;
1294         }
1295
1296         qdev = usr->qddev->qdev;
1297
1298         qdev_rcu_id = srcu_read_lock(&qdev->dev_lock);
1299         if (qdev->in_reset) {
1300                 srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
1301                 srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
1302                 return -ENODEV;
1303         }
1304
1305         msg = kzalloc(QAIC_MANAGE_MAX_MSG_LENGTH + sizeof(*msg), GFP_KERNEL);
1306         if (!msg) {
1307                 ret = -ENOMEM;
1308                 goto out;
1309         }
1310
1311         msg->len = user_msg->len;
1312         msg->count = user_msg->count;
1313
1314         user_data = u64_to_user_ptr(user_msg->data);
1315
1316         if (copy_from_user(msg->data, user_data, user_msg->len)) {
1317                 ret = -EFAULT;
1318                 goto free_msg;
1319         }
1320
1321         ret = qaic_manage(qdev, usr, msg);
1322
1323         /*
1324          * If the qaic_manage() is successful then we copy the message onto
1325          * userspace memory but we have an exception for -ECANCELED.
1326          * For -ECANCELED, it means that device has NACKed the message with a
1327          * status error code which userspace would like to know.
1328          */
1329         if (ret == -ECANCELED || !ret) {
1330                 if (copy_to_user(user_data, msg->data, msg->len)) {
1331                         ret = -EFAULT;
1332                 } else {
1333                         user_msg->len = msg->len;
1334                         user_msg->count = msg->count;
1335                 }
1336         }
1337
1338 free_msg:
1339         kfree(msg);
1340 out:
1341         srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
1342         srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
1343         return ret;
1344 }
1345
1346 int get_cntl_version(struct qaic_device *qdev, struct qaic_user *usr, u16 *major, u16 *minor)
1347 {
1348         struct qaic_manage_trans_status_from_dev *status_result;
1349         struct qaic_manage_trans_status_to_dev *status_query;
1350         struct manage_msg *user_msg;
1351         int ret;
1352
1353         user_msg = kmalloc(sizeof(*user_msg) + sizeof(*status_result), GFP_KERNEL);
1354         if (!user_msg) {
1355                 ret = -ENOMEM;
1356                 goto out;
1357         }
1358         user_msg->len = sizeof(*status_query);
1359         user_msg->count = 1;
1360
1361         status_query = (struct qaic_manage_trans_status_to_dev *)user_msg->data;
1362         status_query->hdr.type = QAIC_TRANS_STATUS_FROM_USR;
1363         status_query->hdr.len = sizeof(status_query->hdr);
1364
1365         ret = qaic_manage(qdev, usr, user_msg);
1366         if (ret)
1367                 goto kfree_user_msg;
1368         status_result = (struct qaic_manage_trans_status_from_dev *)user_msg->data;
1369         *major = status_result->major;
1370         *minor = status_result->minor;
1371
1372         if (status_result->status_flags & BIT(0)) { /* device is using CRC */
1373                 /* By default qdev->gen_crc is programmed to generate CRC */
1374                 qdev->valid_crc = valid_crc;
1375         } else {
1376                 /* By default qdev->valid_crc is programmed to bypass CRC */
1377                 qdev->gen_crc = gen_crc_stub;
1378         }
1379
1380 kfree_user_msg:
1381         kfree(user_msg);
1382 out:
1383         return ret;
1384 }
1385
1386 static void resp_worker(struct work_struct *work)
1387 {
1388         struct resp_work *resp = container_of(work, struct resp_work, work);
1389         struct qaic_device *qdev = resp->qdev;
1390         struct wire_msg *msg = resp->buf;
1391         struct xfer_queue_elem *elem;
1392         struct xfer_queue_elem *i;
1393         bool found = false;
1394
1395         mutex_lock(&qdev->cntl_mutex);
1396         list_for_each_entry_safe(elem, i, &qdev->cntl_xfer_list, list) {
1397                 if (elem->seq_num == le32_to_cpu(msg->hdr.sequence_number)) {
1398                         found = true;
1399                         list_del_init(&elem->list);
1400                         elem->buf = msg;
1401                         complete_all(&elem->xfer_done);
1402                         break;
1403                 }
1404         }
1405         mutex_unlock(&qdev->cntl_mutex);
1406
1407         if (!found)
1408                 /* request must have timed out, drop packet */
1409                 kfree(msg);
1410
1411         kfree(resp);
1412 }
1413
1414 static void free_wrapper_from_list(struct wrapper_list *wrappers, struct wrapper_msg *wrapper)
1415 {
1416         bool all_done = false;
1417
1418         spin_lock(&wrappers->lock);
1419         kref_put(&wrapper->ref_count, free_wrapper);
1420         all_done = list_empty(&wrappers->list);
1421         spin_unlock(&wrappers->lock);
1422
1423         if (all_done)
1424                 kfree(wrappers);
1425 }
1426
1427 void qaic_mhi_ul_xfer_cb(struct mhi_device *mhi_dev, struct mhi_result *mhi_result)
1428 {
1429         struct wire_msg *msg = mhi_result->buf_addr;
1430         struct wrapper_msg *wrapper = container_of(msg, struct wrapper_msg, msg);
1431
1432         free_wrapper_from_list(wrapper->head, wrapper);
1433 }
1434
1435 void qaic_mhi_dl_xfer_cb(struct mhi_device *mhi_dev, struct mhi_result *mhi_result)
1436 {
1437         struct qaic_device *qdev = dev_get_drvdata(&mhi_dev->dev);
1438         struct wire_msg *msg = mhi_result->buf_addr;
1439         struct resp_work *resp;
1440
1441         if (mhi_result->transaction_status || msg->hdr.magic_number != MANAGE_MAGIC_NUMBER) {
1442                 kfree(msg);
1443                 return;
1444         }
1445
1446         resp = kmalloc(sizeof(*resp), GFP_ATOMIC);
1447         if (!resp) {
1448                 kfree(msg);
1449                 return;
1450         }
1451
1452         INIT_WORK(&resp->work, resp_worker);
1453         resp->qdev = qdev;
1454         resp->buf = msg;
1455         queue_work(qdev->cntl_wq, &resp->work);
1456 }
1457
1458 int qaic_control_open(struct qaic_device *qdev)
1459 {
1460         if (!qdev->cntl_ch)
1461                 return -ENODEV;
1462
1463         qdev->cntl_lost_buf = false;
1464         /*
1465          * By default qaic should assume that device has CRC enabled.
1466          * Qaic comes to know if device has CRC enabled or disabled during the
1467          * device status transaction, which is the first transaction performed
1468          * on control channel.
1469          *
1470          * So CRC validation of first device status transaction response is
1471          * ignored (by calling valid_crc_stub) and is done later during decoding
1472          * if device has CRC enabled.
1473          * Now that qaic knows whether device has CRC enabled or not it acts
1474          * accordingly.
1475          */
1476         qdev->gen_crc = gen_crc;
1477         qdev->valid_crc = valid_crc_stub;
1478
1479         return mhi_prepare_for_transfer(qdev->cntl_ch);
1480 }
1481
1482 void qaic_control_close(struct qaic_device *qdev)
1483 {
1484         mhi_unprepare_from_transfer(qdev->cntl_ch);
1485 }
1486
1487 void qaic_release_usr(struct qaic_device *qdev, struct qaic_user *usr)
1488 {
1489         struct wire_trans_terminate_to_dev *trans;
1490         struct wrapper_list *wrappers;
1491         struct wrapper_msg *wrapper;
1492         struct wire_msg *msg;
1493         struct wire_msg *rsp;
1494
1495         wrappers = alloc_wrapper_list();
1496         if (!wrappers)
1497                 return;
1498
1499         wrapper = add_wrapper(wrappers, sizeof(*wrapper) + sizeof(*msg) + sizeof(*trans));
1500         if (!wrapper)
1501                 return;
1502
1503         msg = &wrapper->msg;
1504
1505         trans = (struct wire_trans_terminate_to_dev *)msg->data;
1506
1507         trans->hdr.type = cpu_to_le32(QAIC_TRANS_TERMINATE_TO_DEV);
1508         trans->hdr.len = cpu_to_le32(sizeof(*trans));
1509         trans->handle = cpu_to_le32(usr->handle);
1510
1511         mutex_lock(&qdev->cntl_mutex);
1512         wrapper->len = sizeof(msg->hdr) + sizeof(*trans);
1513         msg->hdr.magic_number = MANAGE_MAGIC_NUMBER;
1514         msg->hdr.sequence_number = cpu_to_le32(qdev->next_seq_num++);
1515         msg->hdr.len = cpu_to_le32(wrapper->len);
1516         msg->hdr.count = cpu_to_le32(1);
1517         msg->hdr.handle = cpu_to_le32(usr->handle);
1518         msg->hdr.padding = cpu_to_le32(0);
1519         msg->hdr.crc32 = cpu_to_le32(qdev->gen_crc(wrappers));
1520
1521         /*
1522          * msg_xfer releases the mutex
1523          * We don't care about the return of msg_xfer since we will not do
1524          * anything different based on what happens.
1525          * We ignore pending signals since one will be set if the user is
1526          * killed, and we need give the device a chance to cleanup, otherwise
1527          * DMA may still be in progress when we return.
1528          */
1529         rsp = msg_xfer(qdev, wrappers, qdev->next_seq_num - 1, true);
1530         if (!IS_ERR(rsp))
1531                 kfree(rsp);
1532         free_wrapper_from_list(wrappers, wrapper);
1533 }
1534
1535 void wake_all_cntl(struct qaic_device *qdev)
1536 {
1537         struct xfer_queue_elem *elem;
1538         struct xfer_queue_elem *i;
1539
1540         mutex_lock(&qdev->cntl_mutex);
1541         list_for_each_entry_safe(elem, i, &qdev->cntl_xfer_list, list) {
1542                 list_del_init(&elem->list);
1543                 complete_all(&elem->xfer_done);
1544         }
1545         mutex_unlock(&qdev->cntl_mutex);
1546 }