projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
RDMA/restrack: Resource-tracker should not use uobject pointers
[linux-2.6-block.git] / drivers / infiniband / core / uverbs_cmd.c
CommitLineData
bc38a6ab
RD		 1/*
2 * Copyright (c) 2005 Topspin Communications. All rights reserved.
f7c6a7b5 3 * Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved.
eb9d3cd5 4 * Copyright (c) 2005 PathScale, Inc. All rights reserved.
8bdb0e86 5 * Copyright (c) 2006 Mellanox Technologies. All rights reserved.
bc38a6ab
RD		 6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * OpenIB.org BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
15 * conditions are met:
16 *
17 * - Redistributions of source code must retain the above
18 * copyright notice, this list of conditions and the following
19 * disclaimer.
20 *
21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials
24 * provided with the distribution.
25 *
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 * SOFTWARE.
bc38a6ab
RD		 34 */
35
6b73597e 36#include <linux/file.h>
70a30e16 37#include <linux/fs.h>
5a0e3ad6 38#include <linux/slab.h>
8ada2c1c 39#include <linux/sched.h>
6b73597e 40
7c0f6ba6 41#include <linux/uaccess.h>
bc38a6ab 42
fd3c7904
MB		 43#include <rdma/uverbs_types.h>
44#include <rdma/uverbs_std_types.h>
45#include "rdma_core.h"
46
bc38a6ab 47#include "uverbs.h"
ed4c54e5 48#include "core_priv.h"
bc38a6ab 49
931373a1
JG		 50/*
51 * Copy a response to userspace. If the provided 'resp' is larger than the
52 * user buffer it is silently truncated. If the user provided a larger buffer
53 * then the trailing portion is zero filled.
54 *
55 * These semantics are intended to support future extension of the output
56 * structures.
57 */
58static int uverbs_response(struct uverbs_attr_bundle *attrs, const void *resp,
59 size_t resp_len)
60{
931373a1
JG		 61 int ret;
62
63 if (copy_to_user(attrs->ucore.outbuf, resp,
64 min(attrs->ucore.outlen, resp_len)))
65 return -EFAULT;
66
9435ef4c
LR		 67 if (resp_len < attrs->ucore.outlen) {
68 /*
69 * Zero fill any extra memory that user
70 * space might have provided.
71 */
72 ret = clear_user(attrs->ucore.outbuf + resp_len,
73 attrs->ucore.outlen - resp_len);
931373a1 74 if (ret)
9435ef4c 75 return -EFAULT;
931373a1
JG		 76 }
77
78 return 0;
79}
80
3c2c2094
JG		 81/*
82 * Copy a request from userspace. If the provided 'req' is larger than the
83 * user buffer then the user buffer is zero extended into the 'req'. If 'req'
84 * is smaller than the user buffer then the uncopied bytes in the user buffer
85 * must be zero.
86 */
87static int uverbs_request(struct uverbs_attr_bundle *attrs, void *req,
88 size_t req_len)
89{
90 if (copy_from_user(req, attrs->ucore.inbuf,
91 min(attrs->ucore.inlen, req_len)))
92 return -EFAULT;
93
94 if (attrs->ucore.inlen < req_len) {
95 memset(req + attrs->ucore.inlen, 0,
96 req_len - attrs->ucore.inlen);
97 } else if (attrs->ucore.inlen > req_len) {
98 if (!ib_is_buffer_cleared(attrs->ucore.inbuf + req_len,
99 attrs->ucore.inlen - req_len))
100 return -EOPNOTSUPP;
101 }
102 return 0;
103}
104
29a29d18
JG		 105/*
106 * Generate the value for the 'response_length' protocol used by write_ex.
107 * This is the number of bytes the kernel actually wrote. Userspace can use
108 * this to detect what structure members in the response the kernel
109 * understood.
110 */
111static u32 uverbs_response_length(struct uverbs_attr_bundle *attrs,
112 size_t resp_len)
113{
114 return min_t(size_t, attrs->ucore.outlen, resp_len);
115}
116
335708c7
JG		 117/*
118 * The iterator version of the request interface is for handlers that need to
119 * step over a flex array at the end of a command header.
120 */
121struct uverbs_req_iter {
122 const void __user *cur;
123 const void __user *end;
124};
125
126static int uverbs_request_start(struct uverbs_attr_bundle *attrs,
127 struct uverbs_req_iter *iter,
128 void *req,
129 size_t req_len)
130{
131 if (attrs->ucore.inlen < req_len)
132 return -ENOSPC;
133
134 if (copy_from_user(req, attrs->ucore.inbuf, req_len))
135 return -EFAULT;
136
137 iter->cur = attrs->ucore.inbuf + req_len;
138 iter->end = attrs->ucore.inbuf + attrs->ucore.inlen;
139 return 0;
140}
141
142static int uverbs_request_next(struct uverbs_req_iter *iter, void *val,
143 size_t len)
144{
145 if (iter->cur + len > iter->end)
146 return -ENOSPC;
147
148 if (copy_from_user(val, iter->cur, len))
149 return -EFAULT;
150
151 iter->cur += len;
152 return 0;
153}
154
c3bea3d2
JG		 155static const void __user *uverbs_request_next_ptr(struct uverbs_req_iter *iter,
156 size_t len)
157{
158 const void __user *res = iter->cur;
159
160 if (iter->cur + len > iter->end)
161 return ERR_PTR(-ENOSPC);
162 iter->cur += len;
163 return res;
164}
165
335708c7
JG		 166static int uverbs_request_finish(struct uverbs_req_iter *iter)
167{
168 if (!ib_is_buffer_cleared(iter->cur, iter->end - iter->cur))
169 return -EOPNOTSUPP;
170 return 0;
171}
172
1e7710f3 173static struct ib_uverbs_completion_event_file *
8313c10f 174_ib_uverbs_lookup_comp_file(s32 fd, const struct uverbs_attr_bundle *attrs)
1e7710f3 175{
1250c304 176 struct ib_uobject *uobj = ufd_get_read(UVERBS_OBJECT_COMP_CHANNEL,
8313c10f 177 fd, attrs);
1e7710f3
MB		 178
179 if (IS_ERR(uobj))
180 return (void *)uobj;
181
182 uverbs_uobject_get(uobj);
183 uobj_put_read(uobj);
184
d0259e82
JG		 185 return container_of(uobj, struct ib_uverbs_completion_event_file,
186 uobj);
1e7710f3 187}
1250c304
JG		 188#define ib_uverbs_lookup_comp_file(_fd, _ufile) \
189 _ib_uverbs_lookup_comp_file((_fd)*typecheck(s32, _fd), _ufile)
1e7710f3 190
974d6b4b 191static int ib_uverbs_get_context(struct uverbs_attr_bundle *attrs)
bc38a6ab 192{
8313c10f 193 struct ib_uverbs_file *file = attrs->ufile;
bc38a6ab
RD		 194 struct ib_uverbs_get_context cmd;
195 struct ib_uverbs_get_context_resp resp;
63c47c28 196 struct ib_ucontext *ucontext;
6b73597e 197 struct file *filp;
43579b5f 198 struct ib_rdmacg_object cg_obj;
bbd51e88 199 struct ib_device *ib_dev;
63c47c28 200 int ret;
bc38a6ab 201
3c2c2094
JG		 202 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
203 if (ret)
204 return ret;
bc38a6ab 205
e951747a 206 mutex_lock(&file->ucontext_lock);
bbd51e88
JG		 207 ib_dev = srcu_dereference(file->device->ib_dev,
208 &file->device->disassociate_srcu);
209 if (!ib_dev) {
210 ret = -EIO;
211 goto err;
212 }
63c47c28
RD		 213
214 if (file->ucontext) {
215 ret = -EINVAL;
216 goto err;
217 }
218
43579b5f
PP		 219 ret = ib_rdmacg_try_charge(&cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE);
220 if (ret)
221 goto err;
222
3023a1e9 223 ucontext = ib_dev->ops.alloc_ucontext(ib_dev, &attrs->driver_udata);
77f76013 224 if (IS_ERR(ucontext)) {
df42245a 225 ret = PTR_ERR(ucontext);
43579b5f 226 goto err_alloc;
77f76013 227 }
bc38a6ab 228
057aec0d 229 ucontext->device = ib_dev;
43579b5f 230 ucontext->cg_obj = cg_obj;
771addf6
MB		 231 /* ufile is required when some objects are released */
232 ucontext->ufile = file;
fd3c7904 233
6ceb6331 234 ucontext->closing = false;
1c77483e 235 ucontext->cleanup_retryable = false;
bc38a6ab 236
882214e2 237#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
f27a0d50
JG		 238 mutex_init(&ucontext->per_mm_list_lock);
239 INIT_LIST_HEAD(&ucontext->per_mm_list);
86bee4c9 240 if (!(ib_dev->attrs.device_cap_flags & IB_DEVICE_ON_DEMAND_PAGING))
882214e2
HE		 241 ucontext->invalidate_range = NULL;
242
243#endif
244
6b73597e
RD		 245 resp.num_comp_vectors = file->device->num_comp_vectors;
246
da183c7a 247 ret = get_unused_fd_flags(O_CLOEXEC);
b1e4594b
AV		 248 if (ret < 0)
249 goto err_free;
250 resp.async_fd = ret;
251
1e7710f3 252 filp = ib_uverbs_alloc_async_event_file(file, ib_dev);
6b73597e
RD		 253 if (IS_ERR(filp)) {
254 ret = PTR_ERR(filp);
b1e4594b 255 goto err_fd;
6b73597e 256 }
bc38a6ab 257
9a073857
JG		 258 ret = uverbs_response(attrs, &resp, sizeof(resp));
259 if (ret)
6b73597e 260 goto err_file;
63c47c28 261
6b73597e
RD		 262 fd_install(resp.async_fd, filp);
263
12d23a91 264 ucontext->res.type = RDMA_RESTRACK_CTX;
af8d7037 265 rdma_restrack_uadd(&ucontext->res);
12d23a91 266
22fa27fb
JG		 267 /*
268 * Make sure that ib_uverbs_get_ucontext() sees the pointer update
269 * only after all writes to setup the ucontext have completed
270 */
271 smp_store_release(&file->ucontext, ucontext);
272
e951747a 273 mutex_unlock(&file->ucontext_lock);
bc38a6ab 274
7106a976 275 return 0;
bc38a6ab 276
6b73597e 277err_file:
03c40442 278 ib_uverbs_free_async_event_file(file);
6b73597e
RD		 279 fput(filp);
280
b1e4594b
AV		 281err_fd:
282 put_unused_fd(resp.async_fd);
283
63c47c28 284err_free:
3023a1e9 285 ib_dev->ops.dealloc_ucontext(ucontext);
bc38a6ab 286
43579b5f
PP		 287err_alloc:
288 ib_rdmacg_uncharge(&cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE);
289
63c47c28 290err:
e951747a 291 mutex_unlock(&file->ucontext_lock);
63c47c28 292 return ret;
bc38a6ab
RD		 293}
294
bbd51e88 295static void copy_query_dev_fields(struct ib_ucontext *ucontext,
02d1aa7a
EC		 296 struct ib_uverbs_query_device_resp *resp,
297 struct ib_device_attr *attr)
298{
bbd51e88
JG		 299 struct ib_device *ib_dev = ucontext->device;
300
02d1aa7a 301 resp->fw_ver = attr->fw_ver;
057aec0d 302 resp->node_guid = ib_dev->node_guid;
02d1aa7a
EC		 303 resp->sys_image_guid = attr->sys_image_guid;
304 resp->max_mr_size = attr->max_mr_size;
305 resp->page_size_cap = attr->page_size_cap;
306 resp->vendor_id = attr->vendor_id;
307 resp->vendor_part_id = attr->vendor_part_id;
308 resp->hw_ver = attr->hw_ver;
309 resp->max_qp = attr->max_qp;
310 resp->max_qp_wr = attr->max_qp_wr;
fb532d6a 311 resp->device_cap_flags = lower_32_bits(attr->device_cap_flags);
33023fb8 312 resp->max_sge = min(attr->max_send_sge, attr->max_recv_sge);
02d1aa7a
EC		 313 resp->max_sge_rd = attr->max_sge_rd;
314 resp->max_cq = attr->max_cq;
315 resp->max_cqe = attr->max_cqe;
316 resp->max_mr = attr->max_mr;
317 resp->max_pd = attr->max_pd;
318 resp->max_qp_rd_atom = attr->max_qp_rd_atom;
319 resp->max_ee_rd_atom = attr->max_ee_rd_atom;
320 resp->max_res_rd_atom = attr->max_res_rd_atom;
321 resp->max_qp_init_rd_atom = attr->max_qp_init_rd_atom;
322 resp->max_ee_init_rd_atom = attr->max_ee_init_rd_atom;
323 resp->atomic_cap = attr->atomic_cap;
324 resp->max_ee = attr->max_ee;
325 resp->max_rdd = attr->max_rdd;
326 resp->max_mw = attr->max_mw;
327 resp->max_raw_ipv6_qp = attr->max_raw_ipv6_qp;
328 resp->max_raw_ethy_qp = attr->max_raw_ethy_qp;
329 resp->max_mcast_grp = attr->max_mcast_grp;
330 resp->max_mcast_qp_attach = attr->max_mcast_qp_attach;
331 resp->max_total_mcast_qp_attach = attr->max_total_mcast_qp_attach;
332 resp->max_ah = attr->max_ah;
333 resp->max_fmr = attr->max_fmr;
334 resp->max_map_per_fmr = attr->max_map_per_fmr;
335 resp->max_srq = attr->max_srq;
336 resp->max_srq_wr = attr->max_srq_wr;
337 resp->max_srq_sge = attr->max_srq_sge;
338 resp->max_pkeys = attr->max_pkeys;
339 resp->local_ca_ack_delay = attr->local_ca_ack_delay;
057aec0d 340 resp->phys_port_cnt = ib_dev->phys_port_cnt;
02d1aa7a
EC		 341}
342
974d6b4b 343static int ib_uverbs_query_device(struct uverbs_attr_bundle *attrs)
bc38a6ab
RD		 344{
345 struct ib_uverbs_query_device cmd;
346 struct ib_uverbs_query_device_resp resp;
bbd51e88 347 struct ib_ucontext *ucontext;
3c2c2094 348 int ret;
bbd51e88 349
8313c10f 350 ucontext = ib_uverbs_get_ucontext(attrs);
bbd51e88
JG		 351 if (IS_ERR(ucontext))
352 return PTR_ERR(ucontext);
bc38a6ab 353
3c2c2094
JG		 354 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
355 if (ret)
356 return ret;
bc38a6ab 357
bc38a6ab 358 memset(&resp, 0, sizeof resp);
bbd51e88 359 copy_query_dev_fields(ucontext, &resp, &ucontext->device->attrs);
bc38a6ab 360
9a073857 361 return uverbs_response(attrs, &resp, sizeof(resp));
bc38a6ab
RD		 362}
363
2f944c0f
JG		 364/*
365 * ib_uverbs_query_port_resp.port_cap_flags started out as just a copy of the
366 * PortInfo CapabilityMask, but was extended with unique bits.
367 */
368static u32 make_port_cap_flags(const struct ib_port_attr *attr)
369{
370 u32 res;
371
372 /* All IBA CapabilityMask bits are passed through here, except bit 26,
373 * which is overridden with IP_BASED_GIDS. This is due to a historical
374 * mistake in the implementation of IP_BASED_GIDS. Otherwise all other
375 * bits match the IBA definition across all kernel versions.
376 */
377 res = attr->port_cap_flags & ~(u32)IB_UVERBS_PCF_IP_BASED_GIDS;
378
379 if (attr->ip_gids)
380 res |= IB_UVERBS_PCF_IP_BASED_GIDS;
381
382 return res;
383}
384
974d6b4b 385static int ib_uverbs_query_port(struct uverbs_attr_bundle *attrs)
bc38a6ab
RD		 386{
387 struct ib_uverbs_query_port cmd;
388 struct ib_uverbs_query_port_resp resp;
389 struct ib_port_attr attr;
390 int ret;
bbd51e88
JG		 391 struct ib_ucontext *ucontext;
392 struct ib_device *ib_dev;
393
8313c10f 394 ucontext = ib_uverbs_get_ucontext(attrs);
bbd51e88
JG		 395 if (IS_ERR(ucontext))
396 return PTR_ERR(ucontext);
397 ib_dev = ucontext->device;
bc38a6ab 398
3c2c2094
JG		 399 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
400 if (ret)
401 return ret;
bc38a6ab 402
057aec0d 403 ret = ib_query_port(ib_dev, cmd.port_num, &attr);
bc38a6ab
RD		 404 if (ret)
405 return ret;
406
407 memset(&resp, 0, sizeof resp);
408
409 resp.state = attr.state;
410 resp.max_mtu = attr.max_mtu;
411 resp.active_mtu = attr.active_mtu;
412 resp.gid_tbl_len = attr.gid_tbl_len;
2f944c0f 413 resp.port_cap_flags = make_port_cap_flags(&attr);
bc38a6ab
RD		 414 resp.max_msg_sz = attr.max_msg_sz;
415 resp.bad_pkey_cntr = attr.bad_pkey_cntr;
416 resp.qkey_viol_cntr = attr.qkey_viol_cntr;
417 resp.pkey_tbl_len = attr.pkey_tbl_len;
62ede777 418
8942acea
AK		 419 if (rdma_is_grh_required(ib_dev, cmd.port_num))
420 resp.flags |= IB_UVERBS_QPF_GRH_REQUIRED;
421
db58540b 422 if (rdma_cap_opa_ah(ib_dev, cmd.port_num)) {
62ede777 423 resp.lid = OPA_TO_IB_UCAST_LID(attr.lid);
db58540b
DC		 424 resp.sm_lid = OPA_TO_IB_UCAST_LID(attr.sm_lid);
425 } else {
62ede777
HD		 426 resp.lid = ib_lid_cpu16(attr.lid);
427 resp.sm_lid = ib_lid_cpu16(attr.sm_lid);
db58540b 428 }
bc38a6ab
RD		 429 resp.lmc = attr.lmc;
430 resp.max_vl_num = attr.max_vl_num;
431 resp.sm_sl = attr.sm_sl;
432 resp.subnet_timeout = attr.subnet_timeout;
433 resp.init_type_reply = attr.init_type_reply;
434 resp.active_width = attr.active_width;
435 resp.active_speed = attr.active_speed;
436 resp.phys_state = attr.phys_state;
057aec0d 437 resp.link_layer = rdma_port_get_link_layer(ib_dev,
2420b60b 438 cmd.port_num);
bc38a6ab 439
9a073857 440 return uverbs_response(attrs, &resp, sizeof(resp));
bc38a6ab
RD		 441}
442
974d6b4b 443static int ib_uverbs_alloc_pd(struct uverbs_attr_bundle *attrs)
bc38a6ab
RD		 444{
445 struct ib_uverbs_alloc_pd cmd;
446 struct ib_uverbs_alloc_pd_resp resp;
bc38a6ab
RD		 447 struct ib_uobject *uobj;
448 struct ib_pd *pd;
449 int ret;
bbd51e88 450 struct ib_device *ib_dev;
bc38a6ab 451
3c2c2094
JG		 452 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
453 if (ret)
454 return ret;
bc38a6ab 455
8313c10f 456 uobj = uobj_alloc(UVERBS_OBJECT_PD, attrs, &ib_dev);
fd3c7904
MB		 457 if (IS_ERR(uobj))
458 return PTR_ERR(uobj);
bc38a6ab 459
3023a1e9 460 pd = ib_dev->ops.alloc_pd(ib_dev, uobj->context, &attrs->driver_udata);
bc38a6ab
RD		 461 if (IS_ERR(pd)) {
462 ret = PTR_ERR(pd);
463 goto err;
464 }
465
057aec0d 466 pd->device = ib_dev;
bc38a6ab 467 pd->uobject = uobj;
50d46335 468 pd->__internal_mr = NULL;
bc38a6ab
RD		 469 atomic_set(&pd->usecnt, 0);
470
9ead190b 471 uobj->object = pd;
bc38a6ab
RD		 472 memset(&resp, 0, sizeof resp);
473 resp.pd_handle = uobj->id;
9d5f8c20 474 pd->res.type = RDMA_RESTRACK_PD;
af8d7037 475 rdma_restrack_uadd(&pd->res);
bc38a6ab 476
9a073857
JG		 477 ret = uverbs_response(attrs, &resp, sizeof(resp));
478 if (ret)
9ead190b 479 goto err_copy;
bc38a6ab 480
7106a976 481 return uobj_alloc_commit(uobj);
eb9d3cd5 482
9ead190b 483err_copy:
bc38a6ab
RD		 484 ib_dealloc_pd(pd);
485
486err:
fd3c7904 487 uobj_alloc_abort(uobj);
bc38a6ab
RD		 488 return ret;
489}
490
974d6b4b 491static int ib_uverbs_dealloc_pd(struct uverbs_attr_bundle *attrs)
bc38a6ab
RD		 492{
493 struct ib_uverbs_dealloc_pd cmd;
3c2c2094 494 int ret;
bc38a6ab 495
3c2c2094
JG		 496 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
497 if (ret)
498 return ret;
bc38a6ab 499
7106a976 500 return uobj_perform_destroy(UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
bc38a6ab
RD		 501}
502
53d0bd1e
SH		 503struct xrcd_table_entry {
504 struct rb_node node;
505 struct ib_xrcd *xrcd;
506 struct inode *inode;
507};
508
509static int xrcd_table_insert(struct ib_uverbs_device *dev,
510 struct inode *inode,
511 struct ib_xrcd *xrcd)
512{
513 struct xrcd_table_entry *entry, *scan;
514 struct rb_node **p = &dev->xrcd_tree.rb_node;
515 struct rb_node *parent = NULL;
516
517 entry = kmalloc(sizeof *entry, GFP_KERNEL);
518 if (!entry)
519 return -ENOMEM;
520
521 entry->xrcd = xrcd;
522 entry->inode = inode;
523
524 while (*p) {
525 parent = *p;
526 scan = rb_entry(parent, struct xrcd_table_entry, node);
527
528 if (inode < scan->inode) {
529 p = &(*p)->rb_left;
530 } else if (inode > scan->inode) {
531 p = &(*p)->rb_right;
532 } else {
533 kfree(entry);
534 return -EEXIST;
535 }
536 }
537
538 rb_link_node(&entry->node, parent, p);
539 rb_insert_color(&entry->node, &dev->xrcd_tree);
540 igrab(inode);
541 return 0;
542}
543
544static struct xrcd_table_entry *xrcd_table_search(struct ib_uverbs_device *dev,
545 struct inode *inode)
546{
547 struct xrcd_table_entry *entry;
548 struct rb_node *p = dev->xrcd_tree.rb_node;
549
550 while (p) {
551 entry = rb_entry(p, struct xrcd_table_entry, node);
552
553 if (inode < entry->inode)
554 p = p->rb_left;
555 else if (inode > entry->inode)
556 p = p->rb_right;
557 else
558 return entry;
559 }
560
561 return NULL;
562}
563
564static struct ib_xrcd *find_xrcd(struct ib_uverbs_device *dev, struct inode *inode)
565{
566 struct xrcd_table_entry *entry;
567
568 entry = xrcd_table_search(dev, inode);
569 if (!entry)
570 return NULL;
571
572 return entry->xrcd;
573}
574
575static void xrcd_table_delete(struct ib_uverbs_device *dev,
576 struct inode *inode)
577{
578 struct xrcd_table_entry *entry;
579
580 entry = xrcd_table_search(dev, inode);
581 if (entry) {
582 iput(inode);
583 rb_erase(&entry->node, &dev->xrcd_tree);
584 kfree(entry);
585 }
586}
587
974d6b4b 588static int ib_uverbs_open_xrcd(struct uverbs_attr_bundle *attrs)
53d0bd1e 589{
8313c10f 590 struct ib_uverbs_device *ibudev = attrs->ufile->device;
53d0bd1e
SH		 591 struct ib_uverbs_open_xrcd cmd;
592 struct ib_uverbs_open_xrcd_resp resp;
53d0bd1e
SH		 593 struct ib_uxrcd_object *obj;
594 struct ib_xrcd *xrcd = NULL;
2903ff01 595 struct fd f = {NULL, 0};
53d0bd1e 596 struct inode *inode = NULL;
2903ff01 597 int ret = 0;
53d0bd1e 598 int new_xrcd = 0;
bbd51e88 599 struct ib_device *ib_dev;
53d0bd1e 600
3c2c2094
JG		 601 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
602 if (ret)
603 return ret;
53d0bd1e 604
8313c10f 605 mutex_lock(&ibudev->xrcd_tree_mutex);
53d0bd1e
SH		 606
607 if (cmd.fd != -1) {
608 /* search for file descriptor */
2903ff01
AV		 609 f = fdget(cmd.fd);
610 if (!f.file) {
53d0bd1e
SH		 611 ret = -EBADF;
612 goto err_tree_mutex_unlock;
613 }
614
496ad9aa 615 inode = file_inode(f.file);
8313c10f 616 xrcd = find_xrcd(ibudev, inode);
53d0bd1e
SH		 617 if (!xrcd && !(cmd.oflags & O_CREAT)) {
618 /* no file descriptor. Need CREATE flag */
619 ret = -EAGAIN;
620 goto err_tree_mutex_unlock;
621 }
622
623 if (xrcd && cmd.oflags & O_EXCL) {
624 ret = -EINVAL;
625 goto err_tree_mutex_unlock;
626 }
627 }
628
8313c10f 629 obj = (struct ib_uxrcd_object *)uobj_alloc(UVERBS_OBJECT_XRCD, attrs,
bbd51e88 630 &ib_dev);
fd3c7904
MB		 631 if (IS_ERR(obj)) {
632 ret = PTR_ERR(obj);
53d0bd1e
SH		 633 goto err_tree_mutex_unlock;
634 }
635
53d0bd1e 636 if (!xrcd) {
3023a1e9
KH		 637 xrcd = ib_dev->ops.alloc_xrcd(ib_dev, obj->uobject.context,
638 &attrs->driver_udata);
53d0bd1e
SH		 639 if (IS_ERR(xrcd)) {
640 ret = PTR_ERR(xrcd);
641 goto err;
642 }
643
644 xrcd->inode = inode;
057aec0d 645 xrcd->device = ib_dev;
53d0bd1e
SH		 646 atomic_set(&xrcd->usecnt, 0);
647 mutex_init(&xrcd->tgt_qp_mutex);
648 INIT_LIST_HEAD(&xrcd->tgt_qp_list);
649 new_xrcd = 1;
650 }
651
652 atomic_set(&obj->refcnt, 0);
653 obj->uobject.object = xrcd;
53d0bd1e
SH		 654 memset(&resp, 0, sizeof resp);
655 resp.xrcd_handle = obj->uobject.id;
656
657 if (inode) {
658 if (new_xrcd) {
659 /* create new inode/xrcd table entry */
8313c10f 660 ret = xrcd_table_insert(ibudev, inode, xrcd);
53d0bd1e 661 if (ret)
fd3c7904 662 goto err_dealloc_xrcd;
53d0bd1e
SH		 663 }
664 atomic_inc(&xrcd->usecnt);
665 }
666
9a073857
JG		 667 ret = uverbs_response(attrs, &resp, sizeof(resp));
668 if (ret)
53d0bd1e 669 goto err_copy;
53d0bd1e 670
2903ff01
AV		 671 if (f.file)
672 fdput(f);
53d0bd1e 673
8313c10f 674 mutex_unlock(&ibudev->xrcd_tree_mutex);
1ff5325c 675
7106a976 676 return uobj_alloc_commit(&obj->uobject);
53d0bd1e
SH		 677
678err_copy:
679 if (inode) {
680 if (new_xrcd)
8313c10f 681 xrcd_table_delete(ibudev, inode);
53d0bd1e
SH		 682 atomic_dec(&xrcd->usecnt);
683 }
684
fd3c7904 685err_dealloc_xrcd:
53d0bd1e
SH		 686 ib_dealloc_xrcd(xrcd);
687
688err:
fd3c7904 689 uobj_alloc_abort(&obj->uobject);
53d0bd1e
SH		 690
691err_tree_mutex_unlock:
2903ff01
AV		 692 if (f.file)
693 fdput(f);
53d0bd1e 694
8313c10f 695 mutex_unlock(&ibudev->xrcd_tree_mutex);
53d0bd1e
SH		 696
697 return ret;
698}
699
974d6b4b 700static int ib_uverbs_close_xrcd(struct uverbs_attr_bundle *attrs)
53d0bd1e
SH		 701{
702 struct ib_uverbs_close_xrcd cmd;
3c2c2094 703 int ret;
53d0bd1e 704
3c2c2094
JG		 705 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
706 if (ret)
707 return ret;
53d0bd1e 708
7106a976 709 return uobj_perform_destroy(UVERBS_OBJECT_XRCD, cmd.xrcd_handle, attrs);
53d0bd1e
SH		 710}
711
1c77483e 712int ib_uverbs_dealloc_xrcd(struct ib_uobject *uobject,
6be60aed
MB		 713 struct ib_xrcd *xrcd,
714 enum rdma_remove_reason why)
53d0bd1e
SH		 715{
716 struct inode *inode;
6be60aed 717 int ret;
1c77483e 718 struct ib_uverbs_device *dev = uobject->context->ufile->device;
53d0bd1e
SH		 719
720 inode = xrcd->inode;
721 if (inode && !atomic_dec_and_test(&xrcd->usecnt))
6be60aed 722 return 0;
53d0bd1e 723
6be60aed 724 ret = ib_dealloc_xrcd(xrcd);
53d0bd1e 725
1c77483e 726 if (ib_is_destroy_retryable(ret, why, uobject)) {
6be60aed 727 atomic_inc(&xrcd->usecnt);
1c77483e
YH		 728 return ret;
729 }
730
731 if (inode)
53d0bd1e 732 xrcd_table_delete(dev, inode);
6be60aed
MB		 733
734 return ret;
53d0bd1e
SH		 735}
736
974d6b4b 737static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs)
bc38a6ab
RD		 738{
739 struct ib_uverbs_reg_mr cmd;
740 struct ib_uverbs_reg_mr_resp resp;
f7c6a7b5 741 struct ib_uobject *uobj;
bc38a6ab
RD		 742 struct ib_pd *pd;
743 struct ib_mr *mr;
744 int ret;
bbd51e88 745 struct ib_device *ib_dev;
bc38a6ab 746
3c2c2094
JG		 747 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
748 if (ret)
749 return ret;
bc38a6ab 750
bc38a6ab
RD		 751 if ((cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK))
752 return -EINVAL;
753
1c636f80
EC		 754 ret = ib_check_mr_access(cmd.access_flags);
755 if (ret)
756 return ret;
f575394f 757
8313c10f 758 uobj = uobj_alloc(UVERBS_OBJECT_MR, attrs, &ib_dev);
fd3c7904
MB		 759 if (IS_ERR(uobj))
760 return PTR_ERR(uobj);
bc38a6ab 761
8313c10f 762 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
aaf1aef5
RD		 763 if (!pd) {
764 ret = -EINVAL;
f7c6a7b5 765 goto err_free;
aaf1aef5 766 }
bc38a6ab 767
860f10a7 768 if (cmd.access_flags & IB_ACCESS_ON_DEMAND) {
86bee4c9
OG		 769 if (!(pd->device->attrs.device_cap_flags &
770 IB_DEVICE_ON_DEMAND_PAGING)) {
860f10a7
SG		 771 pr_debug("ODP support not available\n");
772 ret = -EINVAL;
773 goto err_put;
774 }
775 }
776
3023a1e9
KH		 777 mr = pd->device->ops.reg_user_mr(pd, cmd.start, cmd.length, cmd.hca_va,
778 cmd.access_flags,
779 &attrs->driver_udata);
bc38a6ab
RD		 780 if (IS_ERR(mr)) {
781 ret = PTR_ERR(mr);
9ead190b 782 goto err_put;
bc38a6ab
RD		 783 }
784
785 mr->device = pd->device;
786 mr->pd = pd;
54e7e48b 787 mr->dm = NULL;
f7c6a7b5 788 mr->uobject = uobj;
bc38a6ab 789 atomic_inc(&pd->usecnt);
fccec5b8 790 mr->res.type = RDMA_RESTRACK_MR;
af8d7037 791 rdma_restrack_uadd(&mr->res);
bc38a6ab 792
f7c6a7b5 793 uobj->object = mr;
bc38a6ab 794
9ead190b
RD		 795 memset(&resp, 0, sizeof resp);
796 resp.lkey = mr->lkey;
797 resp.rkey = mr->rkey;
f7c6a7b5 798 resp.mr_handle = uobj->id;
bc38a6ab 799
9a073857
JG		 800 ret = uverbs_response(attrs, &resp, sizeof(resp));
801 if (ret)
9ead190b 802 goto err_copy;
bc38a6ab 803
fd3c7904 804 uobj_put_obj_read(pd);
eb9d3cd5 805
7106a976 806 return uobj_alloc_commit(uobj);
bc38a6ab 807
9ead190b 808err_copy:
bc38a6ab
RD		 809 ib_dereg_mr(mr);
810
9ead190b 811err_put:
fd3c7904 812 uobj_put_obj_read(pd);
bc38a6ab 813
bc38a6ab 814err_free:
fd3c7904 815 uobj_alloc_abort(uobj);
bc38a6ab
RD		 816 return ret;
817}
818
974d6b4b 819static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs)
7e6edb9b
MB		 820{
821 struct ib_uverbs_rereg_mr cmd;
822 struct ib_uverbs_rereg_mr_resp resp;
7e6edb9b
MB		 823 struct ib_pd *pd = NULL;
824 struct ib_mr *mr;
825 struct ib_pd *old_pd;
826 int ret;
827 struct ib_uobject *uobj;
828
3c2c2094
JG		 829 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
830 if (ret)
831 return ret;
7e6edb9b 832
7e6edb9b
MB		 833 if (cmd.flags & ~IB_MR_REREG_SUPPORTED || !cmd.flags)
834 return -EINVAL;
835
836 if ((cmd.flags & IB_MR_REREG_TRANS) &&
837 (!cmd.start || !cmd.hca_va || 0 >= cmd.length ||
838 (cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK)))
839 return -EINVAL;
840
8313c10f 841 uobj = uobj_get_write(UVERBS_OBJECT_MR, cmd.mr_handle, attrs);
fd3c7904
MB		 842 if (IS_ERR(uobj))
843 return PTR_ERR(uobj);
7e6edb9b
MB		 844
845 mr = uobj->object;
846
5ccbf63f
AL		 847 if (mr->dm) {
848 ret = -EINVAL;
849 goto put_uobjs;
850 }
851
7e6edb9b
MB		 852 if (cmd.flags & IB_MR_REREG_ACCESS) {
853 ret = ib_check_mr_access(cmd.access_flags);
854 if (ret)
855 goto put_uobjs;
856 }
857
858 if (cmd.flags & IB_MR_REREG_PD) {
2cc1e3b8 859 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle,
8313c10f 860 attrs);
7e6edb9b
MB		 861 if (!pd) {
862 ret = -EINVAL;
863 goto put_uobjs;
864 }
865 }
866
7e6edb9b 867 old_pd = mr->pd;
3023a1e9
KH		 868 ret = mr->device->ops.rereg_user_mr(mr, cmd.flags, cmd.start,
869 cmd.length, cmd.hca_va,
870 cmd.access_flags, pd,
871 &attrs->driver_udata);
7e6edb9b
MB		 872 if (!ret) {
873 if (cmd.flags & IB_MR_REREG_PD) {
874 atomic_inc(&pd->usecnt);
875 mr->pd = pd;
876 atomic_dec(&old_pd->usecnt);
877 }
878 } else {
879 goto put_uobj_pd;
880 }
881
882 memset(&resp, 0, sizeof(resp));
883 resp.lkey = mr->lkey;
884 resp.rkey = mr->rkey;
885
9a073857 886 ret = uverbs_response(attrs, &resp, sizeof(resp));
7e6edb9b
MB		 887
888put_uobj_pd:
889 if (cmd.flags & IB_MR_REREG_PD)
fd3c7904 890 uobj_put_obj_read(pd);
7e6edb9b
MB		 891
892put_uobjs:
fd3c7904 893 uobj_put_write(uobj);
7e6edb9b
MB		 894
895 return ret;
896}
897
974d6b4b 898static int ib_uverbs_dereg_mr(struct uverbs_attr_bundle *attrs)
bc38a6ab
RD		 899{
900 struct ib_uverbs_dereg_mr cmd;
3c2c2094 901 int ret;
bc38a6ab 902
3c2c2094
JG		 903 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
904 if (ret)
905 return ret;
bc38a6ab 906
7106a976 907 return uobj_perform_destroy(UVERBS_OBJECT_MR, cmd.mr_handle, attrs);
bc38a6ab
RD		 908}
909
974d6b4b 910static int ib_uverbs_alloc_mw(struct uverbs_attr_bundle *attrs)
6b52a12b
SM		 911{
912 struct ib_uverbs_alloc_mw cmd;
913 struct ib_uverbs_alloc_mw_resp resp;
914 struct ib_uobject *uobj;
915 struct ib_pd *pd;
916 struct ib_mw *mw;
917 int ret;
bbd51e88 918 struct ib_device *ib_dev;
6b52a12b 919
3c2c2094
JG		 920 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
921 if (ret)
922 return ret;
6b52a12b 923
8313c10f 924 uobj = uobj_alloc(UVERBS_OBJECT_MW, attrs, &ib_dev);
fd3c7904
MB		 925 if (IS_ERR(uobj))
926 return PTR_ERR(uobj);
6b52a12b 927
8313c10f 928 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
6b52a12b
SM		 929 if (!pd) {
930 ret = -EINVAL;
931 goto err_free;
932 }
933
3023a1e9 934 mw = pd->device->ops.alloc_mw(pd, cmd.mw_type, &attrs->driver_udata);
6b52a12b
SM		 935 if (IS_ERR(mw)) {
936 ret = PTR_ERR(mw);
937 goto err_put;
938 }
939
940 mw->device = pd->device;
941 mw->pd = pd;
942 mw->uobject = uobj;
943 atomic_inc(&pd->usecnt);
944
945 uobj->object = mw;
6b52a12b
SM		 946
947 memset(&resp, 0, sizeof(resp));
948 resp.rkey = mw->rkey;
949 resp.mw_handle = uobj->id;
950
9a073857
JG		 951 ret = uverbs_response(attrs, &resp, sizeof(resp));
952 if (ret)
6b52a12b 953 goto err_copy;
6b52a12b 954
fd3c7904 955 uobj_put_obj_read(pd);
7106a976 956 return uobj_alloc_commit(uobj);
6b52a12b
SM		 957
958err_copy:
feb7c1e3 959 uverbs_dealloc_mw(mw);
6b52a12b 960err_put:
fd3c7904 961 uobj_put_obj_read(pd);
6b52a12b 962err_free:
fd3c7904 963 uobj_alloc_abort(uobj);
6b52a12b
SM		 964 return ret;
965}
966
974d6b4b 967static int ib_uverbs_dealloc_mw(struct uverbs_attr_bundle *attrs)
6b52a12b
SM		 968{
969 struct ib_uverbs_dealloc_mw cmd;
3c2c2094 970 int ret;
6b52a12b 971
3c2c2094
JG		 972 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
973 if (ret)
974 return ret;
6b52a12b 975
7106a976 976 return uobj_perform_destroy(UVERBS_OBJECT_MW, cmd.mw_handle, attrs);
6b52a12b
SM		 977}
978
974d6b4b 979static int ib_uverbs_create_comp_channel(struct uverbs_attr_bundle *attrs)
6b73597e
RD		 980{
981 struct ib_uverbs_create_comp_channel cmd;
982 struct ib_uverbs_create_comp_channel_resp resp;
1e7710f3
MB		 983 struct ib_uobject *uobj;
984 struct ib_uverbs_completion_event_file *ev_file;
bbd51e88 985 struct ib_device *ib_dev;
9a073857 986 int ret;
6b73597e 987
3c2c2094
JG		 988 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
989 if (ret)
990 return ret;
6b73597e 991
8313c10f 992 uobj = uobj_alloc(UVERBS_OBJECT_COMP_CHANNEL, attrs, &ib_dev);
1e7710f3
MB		 993 if (IS_ERR(uobj))
994 return PTR_ERR(uobj);
b1e4594b 995
1e7710f3
MB		 996 resp.fd = uobj->id;
997
998 ev_file = container_of(uobj, struct ib_uverbs_completion_event_file,
d0259e82 999 uobj);
db1b5ddd 1000 ib_uverbs_init_event_queue(&ev_file->ev_queue);
6b73597e 1001
9a073857
JG		 1002 ret = uverbs_response(attrs, &resp, sizeof(resp));
1003 if (ret) {
1e7710f3 1004 uobj_alloc_abort(uobj);
9a073857 1005 return ret;
6b73597e
RD		 1006 }
1007
7106a976 1008 return uobj_alloc_commit(uobj);
6b73597e
RD		 1009}
1010
8313c10f 1011static struct ib_ucq_object *create_cq(struct uverbs_attr_bundle *attrs,
ece9ca97 1012 struct ib_uverbs_ex_create_cq *cmd)
bc38a6ab 1013{
9ead190b 1014 struct ib_ucq_object *obj;
1e7710f3 1015 struct ib_uverbs_completion_event_file *ev_file = NULL;
bc38a6ab
RD		 1016 struct ib_cq *cq;
1017 int ret;
565197dd 1018 struct ib_uverbs_ex_create_cq_resp resp;
bcf4c1ea 1019 struct ib_cq_init_attr attr = {};
bbd51e88 1020 struct ib_device *ib_dev;
21885586 1021
8313c10f 1022 if (cmd->comp_vector >= attrs->ufile->device->num_comp_vectors)
565197dd 1023 return ERR_PTR(-EINVAL);
bc38a6ab 1024
8313c10f 1025 obj = (struct ib_ucq_object *)uobj_alloc(UVERBS_OBJECT_CQ, attrs,
bbd51e88 1026 &ib_dev);
fd3c7904
MB		 1027 if (IS_ERR(obj))
1028 return obj;
9ead190b 1029
565197dd 1030 if (cmd->comp_channel >= 0) {
8313c10f 1031 ev_file = ib_uverbs_lookup_comp_file(cmd->comp_channel, attrs);
1e7710f3
MB		 1032 if (IS_ERR(ev_file)) {
1033 ret = PTR_ERR(ev_file);
ac4e7b35
JM		 1034 goto err;
1035 }
1036 }
1037
fd3c7904 1038 obj->uobject.user_handle = cmd->user_handle;
9ead190b
RD		 1039 obj->comp_events_reported = 0;
1040 obj->async_events_reported = 0;
1041 INIT_LIST_HEAD(&obj->comp_list);
1042 INIT_LIST_HEAD(&obj->async_list);
bc38a6ab 1043
565197dd
MB		 1044 attr.cqe = cmd->cqe;
1045 attr.comp_vector = cmd->comp_vector;
ece9ca97 1046 attr.flags = cmd->flags;
565197dd 1047
3023a1e9
KH		 1048 cq = ib_dev->ops.create_cq(ib_dev, &attr, obj->uobject.context,
1049 &attrs->driver_udata);
bc38a6ab
RD		 1050 if (IS_ERR(cq)) {
1051 ret = PTR_ERR(cq);
9ead190b 1052 goto err_file;
bc38a6ab
RD		 1053 }
1054
057aec0d 1055 cq->device = ib_dev;
9ead190b 1056 cq->uobject = &obj->uobject;
bc38a6ab
RD		 1057 cq->comp_handler = ib_uverbs_comp_handler;
1058 cq->event_handler = ib_uverbs_cq_event_handler;
699a2d5b 1059 cq->cq_context = ev_file ? &ev_file->ev_queue : NULL;
bc38a6ab
RD		 1060 atomic_set(&cq->usecnt, 0);
1061
9ead190b 1062 obj->uobject.object = cq;
bc38a6ab 1063 memset(&resp, 0, sizeof resp);
565197dd
MB		 1064 resp.base.cq_handle = obj->uobject.id;
1065 resp.base.cqe = cq->cqe;
29a29d18 1066 resp.response_length = uverbs_response_length(attrs, sizeof(resp));
565197dd 1067
0cba0efc 1068 cq->res.type = RDMA_RESTRACK_CQ;
af8d7037 1069 rdma_restrack_uadd(&cq->res);
0cba0efc 1070
931373a1 1071 ret = uverbs_response(attrs, &resp, sizeof(resp));
565197dd
MB		 1072 if (ret)
1073 goto err_cb;
bc38a6ab 1074
7106a976 1075 ret = uobj_alloc_commit(&obj->uobject);
2c96eb7d
JG		 1076 if (ret)
1077 return ERR_PTR(ret);
565197dd 1078 return obj;
eb9d3cd5 1079
565197dd 1080err_cb:
bc38a6ab
RD		 1081 ib_destroy_cq(cq);
1082
9ead190b 1083err_file:
ac4e7b35 1084 if (ev_file)
8313c10f 1085 ib_uverbs_release_ucq(attrs->ufile, ev_file, obj);
9ead190b
RD		 1086
1087err:
fd3c7904 1088 uobj_alloc_abort(&obj->uobject);
565197dd
MB		 1089
1090 return ERR_PTR(ret);
1091}
1092
974d6b4b 1093static int ib_uverbs_create_cq(struct uverbs_attr_bundle *attrs)
565197dd
MB		 1094{
1095 struct ib_uverbs_create_cq cmd;
1096 struct ib_uverbs_ex_create_cq cmd_ex;
565197dd 1097 struct ib_ucq_object *obj;
3c2c2094 1098 int ret;
565197dd 1099
3c2c2094
JG		 1100 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1101 if (ret)
1102 return ret;
565197dd 1103
565197dd
MB		 1104 memset(&cmd_ex, 0, sizeof(cmd_ex));
1105 cmd_ex.user_handle = cmd.user_handle;
1106 cmd_ex.cqe = cmd.cqe;
1107 cmd_ex.comp_vector = cmd.comp_vector;
1108 cmd_ex.comp_channel = cmd.comp_channel;
1109
ece9ca97 1110 obj = create_cq(attrs, &cmd_ex);
90849f4d 1111 return PTR_ERR_OR_ZERO(obj);
565197dd
MB		 1112}
1113
974d6b4b 1114static int ib_uverbs_ex_create_cq(struct uverbs_attr_bundle *attrs)
565197dd 1115{
565197dd
MB		 1116 struct ib_uverbs_ex_create_cq cmd;
1117 struct ib_ucq_object *obj;
29a29d18 1118 int ret;
565197dd 1119
29a29d18
JG		 1120 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1121 if (ret)
1122 return ret;
565197dd
MB		 1123
1124 if (cmd.comp_mask)
1125 return -EINVAL;
1126
1127 if (cmd.reserved)
1128 return -EINVAL;
1129
ece9ca97 1130 obj = create_cq(attrs, &cmd);
f4cd9d58 1131 return PTR_ERR_OR_ZERO(obj);
bc38a6ab
RD		 1132}
1133
974d6b4b 1134static int ib_uverbs_resize_cq(struct uverbs_attr_bundle *attrs)
33b9b3ee
RD		 1135{
1136 struct ib_uverbs_resize_cq cmd;
f7a6cb7b 1137 struct ib_uverbs_resize_cq_resp resp = {};
33b9b3ee
RD		 1138 struct ib_cq *cq;
1139 int ret = -EINVAL;
1140
3c2c2094
JG		 1141 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1142 if (ret)
1143 return ret;
33b9b3ee 1144
8313c10f 1145 cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
9ead190b
RD		 1146 if (!cq)
1147 return -EINVAL;
33b9b3ee 1148
3023a1e9 1149 ret = cq->device->ops.resize_cq(cq, cmd.cqe, &attrs->driver_udata);
33b9b3ee
RD		 1150 if (ret)
1151 goto out;
1152
33b9b3ee
RD		 1153 resp.cqe = cq->cqe;
1154
9a073857 1155 ret = uverbs_response(attrs, &resp, sizeof(resp));
33b9b3ee 1156out:
fd3c7904 1157 uobj_put_obj_read(cq);
33b9b3ee 1158
7106a976 1159 return ret;
33b9b3ee
RD		 1160}
1161
7db20ecd
HD		 1162static int copy_wc_to_user(struct ib_device *ib_dev, void __user *dest,
1163 struct ib_wc *wc)
7182afea
DC		 1164{
1165 struct ib_uverbs_wc tmp;
1166
1167 tmp.wr_id = wc->wr_id;
1168 tmp.status = wc->status;
1169 tmp.opcode = wc->opcode;
1170 tmp.vendor_err = wc->vendor_err;
1171 tmp.byte_len = wc->byte_len;
c966ea12 1172 tmp.ex.imm_data = wc->ex.imm_data;
7182afea
DC		 1173 tmp.qp_num = wc->qp->qp_num;
1174 tmp.src_qp = wc->src_qp;
1175 tmp.wc_flags = wc->wc_flags;
1176 tmp.pkey_index = wc->pkey_index;
7db20ecd 1177 if (rdma_cap_opa_ah(ib_dev, wc->port_num))
62ede777 1178 tmp.slid = OPA_TO_IB_UCAST_LID(wc->slid);
7db20ecd 1179 else
62ede777 1180 tmp.slid = ib_lid_cpu16(wc->slid);
7182afea
DC		 1181 tmp.sl = wc->sl;
1182 tmp.dlid_path_bits = wc->dlid_path_bits;
1183 tmp.port_num = wc->port_num;
1184 tmp.reserved = 0;
1185
1186 if (copy_to_user(dest, &tmp, sizeof tmp))
1187 return -EFAULT;
1188
1189 return 0;
1190}
1191
974d6b4b 1192static int ib_uverbs_poll_cq(struct uverbs_attr_bundle *attrs)
67cdb40c
RD		 1193{
1194 struct ib_uverbs_poll_cq cmd;
7182afea
DC		 1195 struct ib_uverbs_poll_cq_resp resp;
1196 u8 __user *header_ptr;
1197 u8 __user *data_ptr;
67cdb40c 1198 struct ib_cq *cq;
7182afea
DC		 1199 struct ib_wc wc;
1200 int ret;
67cdb40c 1201
3c2c2094
JG		 1202 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1203 if (ret)
1204 return ret;
67cdb40c 1205
8313c10f 1206 cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
7182afea
DC		 1207 if (!cq)
1208 return -EINVAL;
67cdb40c 1209
7182afea 1210 /* we copy a struct ib_uverbs_poll_cq_resp to user space */
c2a939fd 1211 header_ptr = attrs->ucore.outbuf;
7182afea 1212 data_ptr = header_ptr + sizeof resp;
9ead190b 1213
7182afea
DC		 1214 memset(&resp, 0, sizeof resp);
1215 while (resp.count < cmd.ne) {
1216 ret = ib_poll_cq(cq, 1, &wc);
1217 if (ret < 0)
1218 goto out_put;
1219 if (!ret)
1220 break;
1221
bbd51e88 1222 ret = copy_wc_to_user(cq->device, data_ptr, &wc);
7182afea
DC		 1223 if (ret)
1224 goto out_put;
1225
1226 data_ptr += sizeof(struct ib_uverbs_wc);
1227 ++resp.count;
67cdb40c
RD		 1228 }
1229
7182afea 1230 if (copy_to_user(header_ptr, &resp, sizeof resp)) {
67cdb40c 1231 ret = -EFAULT;
7182afea
DC		 1232 goto out_put;
1233 }
67cdb40c 1234
7106a976 1235 ret = 0;
67cdb40c 1236
7182afea 1237out_put:
fd3c7904 1238 uobj_put_obj_read(cq);
7182afea 1239 return ret;
67cdb40c
RD		 1240}
1241
974d6b4b 1242static int ib_uverbs_req_notify_cq(struct uverbs_attr_bundle *attrs)
67cdb40c
RD		 1243{
1244 struct ib_uverbs_req_notify_cq cmd;
1245 struct ib_cq *cq;
3c2c2094 1246 int ret;
67cdb40c 1247
3c2c2094
JG		 1248 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1249 if (ret)
1250 return ret;
67cdb40c 1251
8313c10f 1252 cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
ab108676 1253 if (!cq)
9ead190b 1254 return -EINVAL;
67cdb40c 1255
9ead190b
RD		 1256 ib_req_notify_cq(cq, cmd.solicited_only ?
1257 IB_CQ_SOLICITED : IB_CQ_NEXT_COMP);
1258
fd3c7904 1259 uobj_put_obj_read(cq);
9ead190b 1260
7106a976 1261 return 0;
67cdb40c
RD		 1262}
1263
974d6b4b 1264static int ib_uverbs_destroy_cq(struct uverbs_attr_bundle *attrs)
bc38a6ab 1265{
63aaf647
RD		 1266 struct ib_uverbs_destroy_cq cmd;
1267 struct ib_uverbs_destroy_cq_resp resp;
9ead190b 1268 struct ib_uobject *uobj;
9ead190b 1269 struct ib_ucq_object *obj;
3c2c2094 1270 int ret;
bc38a6ab 1271
3c2c2094
JG		 1272 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1273 if (ret)
1274 return ret;
bc38a6ab 1275
8313c10f 1276 uobj = uobj_get_destroy(UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
fd3c7904
MB		 1277 if (IS_ERR(uobj))
1278 return PTR_ERR(uobj);
1279
32ed5c00 1280 obj = container_of(uobj, struct ib_ucq_object, uobject);
fd3c7904 1281 memset(&resp, 0, sizeof(resp));
9ead190b
RD		 1282 resp.comp_events_reported = obj->comp_events_reported;
1283 resp.async_events_reported = obj->async_events_reported;
63aaf647 1284
32ed5c00
JG		 1285 uobj_put_destroy(uobj);
1286
9a073857 1287 return uverbs_response(attrs, &resp, sizeof(resp));
bc38a6ab
RD		 1288}
1289
8313c10f 1290static int create_qp(struct uverbs_attr_bundle *attrs,
ece9ca97 1291 struct ib_uverbs_ex_create_qp *cmd)
bc38a6ab 1292{
6d8a7497
EBE		 1293 struct ib_uqp_object *obj;
1294 struct ib_device *device;
1295 struct ib_pd *pd = NULL;
1296 struct ib_xrcd *xrcd = NULL;
fd3c7904 1297 struct ib_uobject *xrcd_uobj = ERR_PTR(-ENOENT);
6d8a7497
EBE		 1298 struct ib_cq *scq = NULL, *rcq = NULL;
1299 struct ib_srq *srq = NULL;
1300 struct ib_qp *qp;
c70285f8 1301 struct ib_qp_init_attr attr = {};
6d8a7497
EBE		 1302 struct ib_uverbs_ex_create_qp_resp resp;
1303 int ret;
c70285f8
YH		 1304 struct ib_rwq_ind_table *ind_tbl = NULL;
1305 bool has_sq = true;
bbd51e88 1306 struct ib_device *ib_dev;
6d8a7497
EBE		 1307
1308 if (cmd->qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW))
c938a616
OG		 1309 return -EPERM;
1310
8313c10f 1311 obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, attrs,
bbd51e88 1312 &ib_dev);
fd3c7904
MB		 1313 if (IS_ERR(obj))
1314 return PTR_ERR(obj);
1315 obj->uxrcd = NULL;
1316 obj->uevent.uobject.user_handle = cmd->user_handle;
f48b7269 1317 mutex_init(&obj->mcast_lock);
bc38a6ab 1318
ece9ca97 1319 if (cmd->comp_mask & IB_UVERBS_CREATE_QP_MASK_IND_TABLE) {
2cc1e3b8
JG		 1320 ind_tbl = uobj_get_obj_read(rwq_ind_table,
1321 UVERBS_OBJECT_RWQ_IND_TBL,
8313c10f 1322 cmd->rwq_ind_tbl_handle, attrs);
c70285f8
YH		 1323 if (!ind_tbl) {
1324 ret = -EINVAL;
1325 goto err_put;
1326 }
1327
1328 attr.rwq_ind_tbl = ind_tbl;
1329 }
1330
c70285f8
YH		 1331 if (ind_tbl && (cmd->max_recv_wr || cmd->max_recv_sge || cmd->is_srq)) {
1332 ret = -EINVAL;
1333 goto err_put;
1334 }
1335
1336 if (ind_tbl && !cmd->max_send_wr)
1337 has_sq = false;
bc38a6ab 1338
6d8a7497 1339 if (cmd->qp_type == IB_QPT_XRC_TGT) {
1f7ff9d5 1340 xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd->pd_handle,
8313c10f 1341 attrs);
fd3c7904
MB		 1342
1343 if (IS_ERR(xrcd_uobj)) {
1344 ret = -EINVAL;
1345 goto err_put;
1346 }
1347
1348 xrcd = (struct ib_xrcd *)xrcd_uobj->object;
b93f3c18
SH		 1349 if (!xrcd) {
1350 ret = -EINVAL;
1351 goto err_put;
1352 }
1353 device = xrcd->device;
9977f4f6 1354 } else {
6d8a7497
EBE		 1355 if (cmd->qp_type == IB_QPT_XRC_INI) {
1356 cmd->max_recv_wr = 0;
1357 cmd->max_recv_sge = 0;
b93f3c18 1358 } else {
6d8a7497 1359 if (cmd->is_srq) {
2cc1e3b8 1360 srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ,
8313c10f 1361 cmd->srq_handle, attrs);
38eb44fa 1362 if (!srq || srq->srq_type == IB_SRQT_XRC) {
b93f3c18
SH		 1363 ret = -EINVAL;
1364 goto err_put;
1365 }
1366 }
5909ce54 1367
c70285f8
YH		 1368 if (!ind_tbl) {
1369 if (cmd->recv_cq_handle != cmd->send_cq_handle) {
2cc1e3b8
JG		 1370 rcq = uobj_get_obj_read(
1371 cq, UVERBS_OBJECT_CQ,
8313c10f 1372 cmd->recv_cq_handle, attrs);
c70285f8
YH		 1373 if (!rcq) {
1374 ret = -EINVAL;
1375 goto err_put;
1376 }
5909ce54 1377 }
9977f4f6
SH		 1378 }
1379 }
5909ce54 1380
c70285f8 1381 if (has_sq)
2cc1e3b8 1382 scq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ,
8313c10f 1383 cmd->send_cq_handle, attrs);
c70285f8
YH		 1384 if (!ind_tbl)
1385 rcq = rcq ?: scq;
2cc1e3b8 1386 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle,
8313c10f 1387 attrs);
c70285f8 1388 if (!pd || (!scq && has_sq)) {
5909ce54
RD		 1389 ret = -EINVAL;
1390 goto err_put;
1391 }
1392
b93f3c18 1393 device = pd->device;
9977f4f6
SH		 1394 }
1395
bc38a6ab 1396 attr.event_handler = ib_uverbs_qp_event_handler;
8313c10f 1397 attr.qp_context = attrs->ufile;
bc38a6ab
RD		 1398 attr.send_cq = scq;
1399 attr.recv_cq = rcq;
f520ba5a 1400 attr.srq = srq;
b93f3c18 1401 attr.xrcd = xrcd;
6d8a7497
EBE		 1402 attr.sq_sig_type = cmd->sq_sig_all ? IB_SIGNAL_ALL_WR :
1403 IB_SIGNAL_REQ_WR;
1404 attr.qp_type = cmd->qp_type;
b846f25a 1405 attr.create_flags = 0;
bc38a6ab 1406
6d8a7497
EBE		 1407 attr.cap.max_send_wr = cmd->max_send_wr;
1408 attr.cap.max_recv_wr = cmd->max_recv_wr;
1409 attr.cap.max_send_sge = cmd->max_send_sge;
1410 attr.cap.max_recv_sge = cmd->max_recv_sge;
1411 attr.cap.max_inline_data = cmd->max_inline_data;
bc38a6ab 1412
9ead190b
RD		 1413 obj->uevent.events_reported = 0;
1414 INIT_LIST_HEAD(&obj->uevent.event_list);
1415 INIT_LIST_HEAD(&obj->mcast_list);
bc38a6ab 1416
ece9ca97 1417 attr.create_flags = cmd->create_flags;
8a06ce59
LR		 1418 if (attr.create_flags & ~(IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK |
1419 IB_QP_CREATE_CROSS_CHANNEL |
1420 IB_QP_CREATE_MANAGED_SEND |
b531b909 1421 IB_QP_CREATE_MANAGED_RECV |
9e1b161f 1422 IB_QP_CREATE_SCATTER_FCS |
2dee0e54 1423 IB_QP_CREATE_CVLAN_STRIPPING |
e1d2e887
NO		 1424 IB_QP_CREATE_SOURCE_QPN |
1425 IB_QP_CREATE_PCI_WRITE_END_PADDING)) {
6d8a7497
EBE		 1426 ret = -EINVAL;
1427 goto err_put;
1428 }
1429
2dee0e54
YH		 1430 if (attr.create_flags & IB_QP_CREATE_SOURCE_QPN) {
1431 if (!capable(CAP_NET_RAW)) {
1432 ret = -EPERM;
1433 goto err_put;
1434 }
1435
1436 attr.source_qpn = cmd->source_qpn;
1437 }
1438
6d8a7497 1439 if (cmd->qp_type == IB_QPT_XRC_TGT)
b93f3c18
SH		 1440 qp = ib_create_qp(pd, &attr);
1441 else
ef87df2c 1442 qp = _ib_create_qp(device, pd, &attr, &attrs->driver_udata,
2f08ee36 1443 &obj->uevent.uobject);
b93f3c18 1444
bc38a6ab
RD		 1445 if (IS_ERR(qp)) {
1446 ret = PTR_ERR(qp);
fd3c7904 1447 goto err_put;
bc38a6ab
RD		 1448 }
1449
6d8a7497 1450 if (cmd->qp_type != IB_QPT_XRC_TGT) {
d291f1a6
DJ		 1451 ret = ib_create_qp_security(qp, device);
1452 if (ret)
1453 goto err_cb;
1454
0e0ec7e0 1455 qp->real_qp = qp;
b93f3c18
SH		 1456 qp->pd = pd;
1457 qp->send_cq = attr.send_cq;
1458 qp->recv_cq = attr.recv_cq;
1459 qp->srq = attr.srq;
c70285f8 1460 qp->rwq_ind_tbl = ind_tbl;
b93f3c18
SH		 1461 qp->event_handler = attr.event_handler;
1462 qp->qp_context = attr.qp_context;
1463 qp->qp_type = attr.qp_type;
e47e321a 1464 atomic_set(&qp->usecnt, 0);
b93f3c18 1465 atomic_inc(&pd->usecnt);
498ca3c8 1466 qp->port = 0;
c70285f8
YH		 1467 if (attr.send_cq)
1468 atomic_inc(&attr.send_cq->usecnt);
b93f3c18
SH		 1469 if (attr.recv_cq)
1470 atomic_inc(&attr.recv_cq->usecnt);
1471 if (attr.srq)
1472 atomic_inc(&attr.srq->usecnt);
c70285f8
YH		 1473 if (ind_tbl)
1474 atomic_inc(&ind_tbl->usecnt);
f4576587
LR		 1475 } else {
1476 /* It is done in _ib_create_qp for other QP types */
1477 qp->uobject = &obj->uevent.uobject;
b93f3c18 1478 }
bc38a6ab 1479
9ead190b 1480 obj->uevent.uobject.object = qp;
bc38a6ab 1481
9ead190b 1482 memset(&resp, 0, sizeof resp);
6d8a7497
EBE		 1483 resp.base.qpn = qp->qp_num;
1484 resp.base.qp_handle = obj->uevent.uobject.id;
1485 resp.base.max_recv_sge = attr.cap.max_recv_sge;
1486 resp.base.max_send_sge = attr.cap.max_send_sge;
1487 resp.base.max_recv_wr = attr.cap.max_recv_wr;
1488 resp.base.max_send_wr = attr.cap.max_send_wr;
1489 resp.base.max_inline_data = attr.cap.max_inline_data;
29a29d18 1490 resp.response_length = uverbs_response_length(attrs, sizeof(resp));
bc38a6ab 1491
931373a1 1492 ret = uverbs_response(attrs, &resp, sizeof(resp));
6d8a7497
EBE		 1493 if (ret)
1494 goto err_cb;
bc38a6ab 1495
846be90d
YH		 1496 if (xrcd) {
1497 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object,
1498 uobject);
1499 atomic_inc(&obj->uxrcd->refcnt);
fd3c7904 1500 uobj_put_read(xrcd_uobj);
846be90d
YH		 1501 }
1502
b93f3c18 1503 if (pd)
fd3c7904 1504 uobj_put_obj_read(pd);
b93f3c18 1505 if (scq)
fd3c7904 1506 uobj_put_obj_read(scq);
9977f4f6 1507 if (rcq && rcq != scq)
fd3c7904 1508 uobj_put_obj_read(rcq);
9ead190b 1509 if (srq)
fd3c7904 1510 uobj_put_obj_read(srq);
c70285f8 1511 if (ind_tbl)
fd3c7904 1512 uobj_put_obj_read(ind_tbl);
9ead190b 1513
7106a976 1514 return uobj_alloc_commit(&obj->uevent.uobject);
6d8a7497 1515err_cb:
bc38a6ab
RD		 1516 ib_destroy_qp(qp);
1517
9ead190b 1518err_put:
fd3c7904
MB		 1519 if (!IS_ERR(xrcd_uobj))
1520 uobj_put_read(xrcd_uobj);
9ead190b 1521 if (pd)
fd3c7904 1522 uobj_put_obj_read(pd);
9ead190b 1523 if (scq)
fd3c7904 1524 uobj_put_obj_read(scq);
43db2bc0 1525 if (rcq && rcq != scq)
fd3c7904 1526 uobj_put_obj_read(rcq);
9ead190b 1527 if (srq)
fd3c7904 1528 uobj_put_obj_read(srq);
c70285f8 1529 if (ind_tbl)
fd3c7904 1530 uobj_put_obj_read(ind_tbl);
9ead190b 1531
fd3c7904 1532 uobj_alloc_abort(&obj->uevent.uobject);
bc38a6ab
RD		 1533 return ret;
1534}
1535
974d6b4b 1536static int ib_uverbs_create_qp(struct uverbs_attr_bundle *attrs)
6d8a7497
EBE		 1537{
1538 struct ib_uverbs_create_qp cmd;
1539 struct ib_uverbs_ex_create_qp cmd_ex;
3c2c2094 1540 int ret;
6d8a7497 1541
3c2c2094
JG		 1542 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1543 if (ret)
1544 return ret;
6d8a7497 1545
6d8a7497
EBE		 1546 memset(&cmd_ex, 0, sizeof(cmd_ex));
1547 cmd_ex.user_handle = cmd.user_handle;
1548 cmd_ex.pd_handle = cmd.pd_handle;
1549 cmd_ex.send_cq_handle = cmd.send_cq_handle;
1550 cmd_ex.recv_cq_handle = cmd.recv_cq_handle;
1551 cmd_ex.srq_handle = cmd.srq_handle;
1552 cmd_ex.max_send_wr = cmd.max_send_wr;
1553 cmd_ex.max_recv_wr = cmd.max_recv_wr;
1554 cmd_ex.max_send_sge = cmd.max_send_sge;
1555 cmd_ex.max_recv_sge = cmd.max_recv_sge;
1556 cmd_ex.max_inline_data = cmd.max_inline_data;
1557 cmd_ex.sq_sig_all = cmd.sq_sig_all;
1558 cmd_ex.qp_type = cmd.qp_type;
1559 cmd_ex.is_srq = cmd.is_srq;
1560
ece9ca97 1561 return create_qp(attrs, &cmd_ex);
6d8a7497
EBE		 1562}
1563
974d6b4b 1564static int ib_uverbs_ex_create_qp(struct uverbs_attr_bundle *attrs)
6d8a7497 1565{
29a29d18
JG		 1566 struct ib_uverbs_ex_create_qp cmd;
1567 int ret;
6d8a7497 1568
29a29d18
JG		 1569 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1570 if (ret)
1571 return ret;
6d8a7497 1572
c70285f8 1573 if (cmd.comp_mask & ~IB_UVERBS_CREATE_QP_SUP_COMP_MASK)
6d8a7497
EBE		 1574 return -EINVAL;
1575
1576 if (cmd.reserved)
1577 return -EINVAL;
1578
ece9ca97 1579 return create_qp(attrs, &cmd);
6d8a7497
EBE		 1580}
1581
974d6b4b 1582static int ib_uverbs_open_qp(struct uverbs_attr_bundle *attrs)
42849b26
SH		 1583{
1584 struct ib_uverbs_open_qp cmd;
1585 struct ib_uverbs_create_qp_resp resp;
42849b26
SH		 1586 struct ib_uqp_object *obj;
1587 struct ib_xrcd *xrcd;
1588 struct ib_uobject *uninitialized_var(xrcd_uobj);
1589 struct ib_qp *qp;
1590 struct ib_qp_open_attr attr;
1591 int ret;
bbd51e88 1592 struct ib_device *ib_dev;
42849b26 1593
3c2c2094
JG		 1594 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1595 if (ret)
1596 return ret;
42849b26 1597
8313c10f 1598 obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, attrs,
bbd51e88 1599 &ib_dev);
fd3c7904
MB		 1600 if (IS_ERR(obj))
1601 return PTR_ERR(obj);
42849b26 1602
8313c10f 1603 xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd.pd_handle, attrs);
fd3c7904
MB		 1604 if (IS_ERR(xrcd_uobj)) {
1605 ret = -EINVAL;
1606 goto err_put;
1607 }
42849b26 1608
fd3c7904 1609 xrcd = (struct ib_xrcd *)xrcd_uobj->object;
42849b26
SH		 1610 if (!xrcd) {
1611 ret = -EINVAL;
fd3c7904 1612 goto err_xrcd;
42849b26
SH		 1613 }
1614
1615 attr.event_handler = ib_uverbs_qp_event_handler;
8313c10f 1616 attr.qp_context = attrs->ufile;
42849b26
SH		 1617 attr.qp_num = cmd.qpn;
1618 attr.qp_type = cmd.qp_type;
1619
1620 obj->uevent.events_reported = 0;
1621 INIT_LIST_HEAD(&obj->uevent.event_list);
1622 INIT_LIST_HEAD(&obj->mcast_list);
1623
1624 qp = ib_open_qp(xrcd, &attr);
1625 if (IS_ERR(qp)) {
1626 ret = PTR_ERR(qp);
fd3c7904 1627 goto err_xrcd;
42849b26
SH		 1628 }
1629
42849b26 1630 obj->uevent.uobject.object = qp;
fd3c7904 1631 obj->uevent.uobject.user_handle = cmd.user_handle;
42849b26
SH		 1632
1633 memset(&resp, 0, sizeof resp);
1634 resp.qpn = qp->qp_num;
1635 resp.qp_handle = obj->uevent.uobject.id;
1636
9a073857
JG		 1637 ret = uverbs_response(attrs, &resp, sizeof(resp));
1638 if (ret)
fd3c7904 1639 goto err_destroy;
42849b26 1640
846be90d
YH		 1641 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);
1642 atomic_inc(&obj->uxrcd->refcnt);
fd3c7904
MB		 1643 qp->uobject = &obj->uevent.uobject;
1644 uobj_put_read(xrcd_uobj);
42849b26 1645
7106a976 1646 return uobj_alloc_commit(&obj->uevent.uobject);
42849b26 1647
42849b26
SH		 1648err_destroy:
1649 ib_destroy_qp(qp);
fd3c7904
MB		 1650err_xrcd:
1651 uobj_put_read(xrcd_uobj);
42849b26 1652err_put:
fd3c7904 1653 uobj_alloc_abort(&obj->uevent.uobject);
42849b26
SH		 1654 return ret;
1655}
1656
89caa053
PP		 1657static void copy_ah_attr_to_uverbs(struct ib_uverbs_qp_dest *uverb_attr,
1658 struct rdma_ah_attr *rdma_attr)
1659{
1660 const struct ib_global_route *grh;
1661
1662 uverb_attr->dlid = rdma_ah_get_dlid(rdma_attr);
1663 uverb_attr->sl = rdma_ah_get_sl(rdma_attr);
1664 uverb_attr->src_path_bits = rdma_ah_get_path_bits(rdma_attr);
1665 uverb_attr->static_rate = rdma_ah_get_static_rate(rdma_attr);
1666 uverb_attr->is_global = !!(rdma_ah_get_ah_flags(rdma_attr) &
1667 IB_AH_GRH);
1668 if (uverb_attr->is_global) {
1669 grh = rdma_ah_read_grh(rdma_attr);
1670 memcpy(uverb_attr->dgid, grh->dgid.raw, 16);
1671 uverb_attr->flow_label = grh->flow_label;
1672 uverb_attr->sgid_index = grh->sgid_index;
1673 uverb_attr->hop_limit = grh->hop_limit;
1674 uverb_attr->traffic_class = grh->traffic_class;
1675 }
1676 uverb_attr->port_num = rdma_ah_get_port_num(rdma_attr);
1677}
1678
974d6b4b 1679static int ib_uverbs_query_qp(struct uverbs_attr_bundle *attrs)
7ccc9a24
DB		 1680{
1681 struct ib_uverbs_query_qp cmd;
1682 struct ib_uverbs_query_qp_resp resp;
1683 struct ib_qp *qp;
1684 struct ib_qp_attr *attr;
1685 struct ib_qp_init_attr *init_attr;
1686 int ret;
1687
3c2c2094
JG		 1688 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1689 if (ret)
1690 return ret;
7ccc9a24
DB		 1691
1692 attr = kmalloc(sizeof *attr, GFP_KERNEL);
1693 init_attr = kmalloc(sizeof *init_attr, GFP_KERNEL);
1694 if (!attr || !init_attr) {
1695 ret = -ENOMEM;
1696 goto out;
1697 }
1698
8313c10f 1699 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
9ead190b 1700 if (!qp) {
7ccc9a24 1701 ret = -EINVAL;
9ead190b
RD		 1702 goto out;
1703 }
1704
1705 ret = ib_query_qp(qp, attr, cmd.attr_mask, init_attr);
7ccc9a24 1706
fd3c7904 1707 uobj_put_obj_read(qp);
7ccc9a24
DB		 1708
1709 if (ret)
1710 goto out;
1711
1712 memset(&resp, 0, sizeof resp);
1713
1714 resp.qp_state = attr->qp_state;
1715 resp.cur_qp_state = attr->cur_qp_state;
1716 resp.path_mtu = attr->path_mtu;
1717 resp.path_mig_state = attr->path_mig_state;
1718 resp.qkey = attr->qkey;
1719 resp.rq_psn = attr->rq_psn;
1720 resp.sq_psn = attr->sq_psn;
1721 resp.dest_qp_num = attr->dest_qp_num;
1722 resp.qp_access_flags = attr->qp_access_flags;
1723 resp.pkey_index = attr->pkey_index;
1724 resp.alt_pkey_index = attr->alt_pkey_index;
0b26c88f 1725 resp.sq_draining = attr->sq_draining;
7ccc9a24
DB		 1726 resp.max_rd_atomic = attr->max_rd_atomic;
1727 resp.max_dest_rd_atomic = attr->max_dest_rd_atomic;
1728 resp.min_rnr_timer = attr->min_rnr_timer;
1729 resp.port_num = attr->port_num;
1730 resp.timeout = attr->timeout;
1731 resp.retry_cnt = attr->retry_cnt;
1732 resp.rnr_retry = attr->rnr_retry;
1733 resp.alt_port_num = attr->alt_port_num;
1734 resp.alt_timeout = attr->alt_timeout;
1735
89caa053
PP		 1736 copy_ah_attr_to_uverbs(&resp.dest, &attr->ah_attr);
1737 copy_ah_attr_to_uverbs(&resp.alt_dest, &attr->alt_ah_attr);
7ccc9a24
DB		 1738
1739 resp.max_send_wr = init_attr->cap.max_send_wr;
1740 resp.max_recv_wr = init_attr->cap.max_recv_wr;
1741 resp.max_send_sge = init_attr->cap.max_send_sge;
1742 resp.max_recv_sge = init_attr->cap.max_recv_sge;
1743 resp.max_inline_data = init_attr->cap.max_inline_data;
27d56300 1744 resp.sq_sig_all = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR;
7ccc9a24 1745
9a073857 1746 ret = uverbs_response(attrs, &resp, sizeof(resp));
7ccc9a24
DB		 1747
1748out:
1749 kfree(attr);
1750 kfree(init_attr);
1751
7106a976 1752 return ret;
7ccc9a24
DB		 1753}
1754
9977f4f6
SH		 1755/* Remove ignored fields set in the attribute mask */
1756static int modify_qp_mask(enum ib_qp_type qp_type, int mask)
1757{
1758 switch (qp_type) {
1759 case IB_QPT_XRC_INI:
1760 return mask & ~(IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER);
b93f3c18
SH		 1761 case IB_QPT_XRC_TGT:
1762 return mask & ~(IB_QP_MAX_QP_RD_ATOMIC | IB_QP_RETRY_CNT |
1763 IB_QP_RNR_RETRY);
9977f4f6
SH		 1764 default:
1765 return mask;
1766 }
1767}
1768
89caa053
PP		 1769static void copy_ah_attr_from_uverbs(struct ib_device *dev,
1770 struct rdma_ah_attr *rdma_attr,
1771 struct ib_uverbs_qp_dest *uverb_attr)
1772{
1773 rdma_attr->type = rdma_ah_find_type(dev, uverb_attr->port_num);
1774 if (uverb_attr->is_global) {
1775 rdma_ah_set_grh(rdma_attr, NULL,
1776 uverb_attr->flow_label,
1777 uverb_attr->sgid_index,
1778 uverb_attr->hop_limit,
1779 uverb_attr->traffic_class);
1780 rdma_ah_set_dgid_raw(rdma_attr, uverb_attr->dgid);
1781 } else {
1782 rdma_ah_set_ah_flags(rdma_attr, 0);
1783 }
1784 rdma_ah_set_dlid(rdma_attr, uverb_attr->dlid);
1785 rdma_ah_set_sl(rdma_attr, uverb_attr->sl);
1786 rdma_ah_set_path_bits(rdma_attr, uverb_attr->src_path_bits);
1787 rdma_ah_set_static_rate(rdma_attr, uverb_attr->static_rate);
1788 rdma_ah_set_port_num(rdma_attr, uverb_attr->port_num);
1789 rdma_ah_set_make_grd(rdma_attr, false);
1790}
1791
8313c10f 1792static int modify_qp(struct uverbs_attr_bundle *attrs,
ef87df2c 1793 struct ib_uverbs_ex_modify_qp *cmd)
bc38a6ab 1794{
189aba99
BW		 1795 struct ib_qp_attr *attr;
1796 struct ib_qp *qp;
1797 int ret;
9bc57e2d 1798
fb51eeca 1799 attr = kzalloc(sizeof(*attr), GFP_KERNEL);
bc38a6ab
RD		 1800 if (!attr)
1801 return -ENOMEM;
1802
8313c10f
JG		 1803 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd->base.qp_handle,
1804 attrs);
9ead190b 1805 if (!qp) {
bc38a6ab
RD		 1806 ret = -EINVAL;
1807 goto out;
1808 }
1809
5a7a88f1
IM		 1810 if ((cmd->base.attr_mask & IB_QP_PORT) &&
1811 !rdma_is_port_valid(qp->device, cmd->base.port_num)) {
5ecce4c9
BP		 1812 ret = -EINVAL;
1813 goto release_qp;
1814 }
1815
addb8a65
JM		 1816 if ((cmd->base.attr_mask & IB_QP_AV)) {
1817 if (!rdma_is_port_valid(qp->device, cmd->base.dest.port_num)) {
1818 ret = -EINVAL;
1819 goto release_qp;
1820 }
1821
1822 if (cmd->base.attr_mask & IB_QP_STATE &&
1823 cmd->base.qp_state == IB_QPS_RTR) {
1824 /* We are in INIT->RTR TRANSITION (if we are not,
1825 * this transition will be rejected in subsequent checks).
1826 * In the INIT->RTR transition, we cannot have IB_QP_PORT set,
1827 * but the IB_QP_STATE flag is required.
1828 *
1829 * Since kernel 3.14 (commit dbf727de7440), the uverbs driver,
1830 * when IB_QP_AV is set, has required inclusion of a valid
1831 * port number in the primary AV. (AVs are created and handled
1832 * differently for infiniband and ethernet (RoCE) ports).
1833 *
1834 * Check the port number included in the primary AV against
1835 * the port number in the qp struct, which was set (and saved)
1836 * in the RST->INIT transition.
1837 */
1838 if (cmd->base.dest.port_num != qp->real_qp->port) {
1839 ret = -EINVAL;
1840 goto release_qp;
1841 }
1842 } else {
1843 /* We are in SQD->SQD. (If we are not, this transition will
1844 * be rejected later in the verbs layer checks).
1845 * Check for both IB_QP_PORT and IB_QP_AV, these can be set
1846 * together in the SQD->SQD transition.
1847 *
1848 * If only IP_QP_AV was set, add in IB_QP_PORT as well (the
1849 * verbs layer driver does not track primary port changes
1850 * resulting from path migration. Thus, in SQD, if the primary
1851 * AV is modified, the primary port should also be modified).
1852 *
1853 * Note that in this transition, the IB_QP_STATE flag
1854 * is not allowed.
1855 */
1856 if (((cmd->base.attr_mask & (IB_QP_AV | IB_QP_PORT))
1857 == (IB_QP_AV | IB_QP_PORT)) &&
1858 cmd->base.port_num != cmd->base.dest.port_num) {
1859 ret = -EINVAL;
1860 goto release_qp;
1861 }
1862 if ((cmd->base.attr_mask & (IB_QP_AV | IB_QP_PORT))
1863 == IB_QP_AV) {
1864 cmd->base.attr_mask |= IB_QP_PORT;
1865 cmd->base.port_num = cmd->base.dest.port_num;
1866 }
1867 }
5d4c05c3
LR		 1868 }
1869
4cae8ff1 1870 if ((cmd->base.attr_mask & IB_QP_ALT_PATH) &&
5d4c05c3 1871 (!rdma_is_port_valid(qp->device, cmd->base.alt_port_num) ||
addb8a65
JM		 1872 !rdma_is_port_valid(qp->device, cmd->base.alt_dest.port_num) ||
1873 cmd->base.alt_port_num != cmd->base.alt_dest.port_num)) {
4cae8ff1
DJ		 1874 ret = -EINVAL;
1875 goto release_qp;
1876 }
1877
88de869b
LR		 1878 if ((cmd->base.attr_mask & IB_QP_CUR_STATE &&
1879 cmd->base.cur_qp_state > IB_QPS_ERR) ||
4eeed368
MD		 1880 (cmd->base.attr_mask & IB_QP_STATE &&
1881 cmd->base.qp_state > IB_QPS_ERR)) {
88de869b
LR		 1882 ret = -EINVAL;
1883 goto release_qp;
1884 }
1885
4eeed368
MD		 1886 if (cmd->base.attr_mask & IB_QP_STATE)
1887 attr->qp_state = cmd->base.qp_state;
1888 if (cmd->base.attr_mask & IB_QP_CUR_STATE)
1889 attr->cur_qp_state = cmd->base.cur_qp_state;
1890 if (cmd->base.attr_mask & IB_QP_PATH_MTU)
1891 attr->path_mtu = cmd->base.path_mtu;
1892 if (cmd->base.attr_mask & IB_QP_PATH_MIG_STATE)
1893 attr->path_mig_state = cmd->base.path_mig_state;
1894 if (cmd->base.attr_mask & IB_QP_QKEY)
1895 attr->qkey = cmd->base.qkey;
1896 if (cmd->base.attr_mask & IB_QP_RQ_PSN)
1897 attr->rq_psn = cmd->base.rq_psn;
1898 if (cmd->base.attr_mask & IB_QP_SQ_PSN)
1899 attr->sq_psn = cmd->base.sq_psn;
1900 if (cmd->base.attr_mask & IB_QP_DEST_QPN)
1901 attr->dest_qp_num = cmd->base.dest_qp_num;
1902 if (cmd->base.attr_mask & IB_QP_ACCESS_FLAGS)
1903 attr->qp_access_flags = cmd->base.qp_access_flags;
1904 if (cmd->base.attr_mask & IB_QP_PKEY_INDEX)
1905 attr->pkey_index = cmd->base.pkey_index;
1906 if (cmd->base.attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
1907 attr->en_sqd_async_notify = cmd->base.en_sqd_async_notify;
1908 if (cmd->base.attr_mask & IB_QP_MAX_QP_RD_ATOMIC)
1909 attr->max_rd_atomic = cmd->base.max_rd_atomic;
1910 if (cmd->base.attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
1911 attr->max_dest_rd_atomic = cmd->base.max_dest_rd_atomic;
1912 if (cmd->base.attr_mask & IB_QP_MIN_RNR_TIMER)
1913 attr->min_rnr_timer = cmd->base.min_rnr_timer;
1914 if (cmd->base.attr_mask & IB_QP_PORT)
1915 attr->port_num = cmd->base.port_num;
1916 if (cmd->base.attr_mask & IB_QP_TIMEOUT)
1917 attr->timeout = cmd->base.timeout;
1918 if (cmd->base.attr_mask & IB_QP_RETRY_CNT)
1919 attr->retry_cnt = cmd->base.retry_cnt;
1920 if (cmd->base.attr_mask & IB_QP_RNR_RETRY)
1921 attr->rnr_retry = cmd->base.rnr_retry;
1922 if (cmd->base.attr_mask & IB_QP_ALT_PATH) {
1923 attr->alt_port_num = cmd->base.alt_port_num;
1924 attr->alt_timeout = cmd->base.alt_timeout;
1925 attr->alt_pkey_index = cmd->base.alt_pkey_index;
1926 }
1927 if (cmd->base.attr_mask & IB_QP_RATE_LIMIT)
1928 attr->rate_limit = cmd->rate_limit;
189aba99 1929
498ca3c8 1930 if (cmd->base.attr_mask & IB_QP_AV)
89caa053
PP		 1931 copy_ah_attr_from_uverbs(qp->device, &attr->ah_attr,
1932 &cmd->base.dest);
189aba99 1933
498ca3c8 1934 if (cmd->base.attr_mask & IB_QP_ALT_PATH)
89caa053
PP		 1935 copy_ah_attr_from_uverbs(qp->device, &attr->alt_ah_attr,
1936 &cmd->base.alt_dest);
bc38a6ab 1937
f7c8f2e9
PP		 1938 ret = ib_modify_qp_with_udata(qp, attr,
1939 modify_qp_mask(qp->qp_type,
1940 cmd->base.attr_mask),
ef87df2c 1941 &attrs->driver_udata);
9ead190b 1942
0fb8bcf0 1943release_qp:
fd3c7904 1944 uobj_put_obj_read(qp);
bc38a6ab 1945out:
bc38a6ab
RD		 1946 kfree(attr);
1947
1948 return ret;
1949}
1950
974d6b4b 1951static int ib_uverbs_modify_qp(struct uverbs_attr_bundle *attrs)
189aba99 1952{
29a29d18 1953 struct ib_uverbs_ex_modify_qp cmd;
3c2c2094 1954 int ret;
189aba99 1955
3c2c2094
JG		 1956 ret = uverbs_request(attrs, &cmd.base, sizeof(cmd.base));
1957 if (ret)
1958 return ret;
189aba99
BW		 1959
1960 if (cmd.base.attr_mask &
1961 ~((IB_USER_LEGACY_LAST_QP_ATTR_MASK << 1) - 1))
1962 return -EOPNOTSUPP;
1963
ef87df2c 1964 return modify_qp(attrs, &cmd);
189aba99
BW		 1965}
1966
974d6b4b 1967static int ib_uverbs_ex_modify_qp(struct uverbs_attr_bundle *attrs)
189aba99 1968{
29a29d18 1969 struct ib_uverbs_ex_modify_qp cmd;
40efca7a
JG		 1970 struct ib_uverbs_ex_modify_qp_resp resp = {
1971 .response_length = uverbs_response_length(attrs, sizeof(resp))
1972 };
189aba99
BW		 1973 int ret;
1974
29a29d18
JG		 1975 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1976 if (ret)
1977 return ret;
1978
189aba99
BW		 1979 /*
1980 * Last bit is reserved for extending the attr_mask by
1981 * using another field.
1982 */
1983 BUILD_BUG_ON(IB_USER_LAST_QP_ATTR_MASK == (1 << 31));
1984
189aba99
BW		 1985 if (cmd.base.attr_mask &
1986 ~((IB_USER_LAST_QP_ATTR_MASK << 1) - 1))
1987 return -EOPNOTSUPP;
1988
40efca7a
JG		 1989 ret = modify_qp(attrs, &cmd);
1990 if (ret)
1991 return ret;
1992
1993 return uverbs_response(attrs, &resp, sizeof(resp));
189aba99
BW		 1994}
1995
974d6b4b 1996static int ib_uverbs_destroy_qp(struct uverbs_attr_bundle *attrs)
bc38a6ab 1997{
63aaf647
RD		 1998 struct ib_uverbs_destroy_qp cmd;
1999 struct ib_uverbs_destroy_qp_resp resp;
9ead190b 2000 struct ib_uobject *uobj;
9ead190b 2001 struct ib_uqp_object *obj;
3c2c2094 2002 int ret;
bc38a6ab 2003
3c2c2094
JG		 2004 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
2005 if (ret)
2006 return ret;
bc38a6ab 2007
8313c10f 2008 uobj = uobj_get_destroy(UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
fd3c7904
MB		 2009 if (IS_ERR(uobj))
2010 return PTR_ERR(uobj);
2011
9ead190b 2012 obj = container_of(uobj, struct ib_uqp_object, uevent.uobject);
32ed5c00 2013 memset(&resp, 0, sizeof(resp));
9ead190b 2014 resp.events_reported = obj->uevent.events_reported;
32ed5c00
JG		 2015
2016 uobj_put_destroy(uobj);
bc38a6ab 2017
9a073857 2018 return uverbs_response(attrs, &resp, sizeof(resp));
bc38a6ab
RD		 2019}
2020
e622f2f4
CH		 2021static void *alloc_wr(size_t wr_size, __u32 num_sge)
2022{
4f7f4dcf
VT		 2023 if (num_sge >= (U32_MAX - ALIGN(wr_size, sizeof (struct ib_sge))) /
2024 sizeof (struct ib_sge))
2025 return NULL;
2026
e622f2f4
CH		 2027 return kmalloc(ALIGN(wr_size, sizeof (struct ib_sge)) +
2028 num_sge * sizeof (struct ib_sge), GFP_KERNEL);
4f7f4dcf 2029}
e622f2f4 2030
974d6b4b 2031static int ib_uverbs_post_send(struct uverbs_attr_bundle *attrs)
67cdb40c
RD		 2032{
2033 struct ib_uverbs_post_send cmd;
2034 struct ib_uverbs_post_send_resp resp;
2035 struct ib_uverbs_send_wr *user_wr;
d34ac5cd
BVA		 2036 struct ib_send_wr *wr = NULL, *last, *next;
2037 const struct ib_send_wr *bad_wr;
67cdb40c
RD		 2038 struct ib_qp *qp;
2039 int i, sg_ind;
9ead190b 2040 int is_ud;
9a073857 2041 int ret, ret2;
1d784b89 2042 size_t next_size;
c3bea3d2
JG		 2043 const struct ib_sge __user *sgls;
2044 const void __user *wqes;
2045 struct uverbs_req_iter iter;
67cdb40c 2046
c3bea3d2
JG		 2047 ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
2048 if (ret)
2049 return ret;
2050 wqes = uverbs_request_next_ptr(&iter, cmd.wqe_size * cmd.wr_count);
2051 if (IS_ERR(wqes))
2052 return PTR_ERR(wqes);
2053 sgls = uverbs_request_next_ptr(
2054 &iter, cmd.sge_count * sizeof(struct ib_uverbs_sge));
2055 if (IS_ERR(sgls))
2056 return PTR_ERR(sgls);
2057 ret = uverbs_request_finish(&iter);
2058 if (ret)
2059 return ret;
67cdb40c
RD		 2060
2061 user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL);
2062 if (!user_wr)
2063 return -ENOMEM;
2064
8313c10f 2065 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
9ead190b 2066 if (!qp)
67cdb40c
RD		 2067 goto out;
2068
9ead190b 2069 is_ud = qp->qp_type == IB_QPT_UD;
67cdb40c
RD		 2070 sg_ind = 0;
2071 last = NULL;
2072 for (i = 0; i < cmd.wr_count; ++i) {
c3bea3d2 2073 if (copy_from_user(user_wr, wqes + i * cmd.wqe_size,
67cdb40c
RD		 2074 cmd.wqe_size)) {
2075 ret = -EFAULT;
9ead190b 2076 goto out_put;
67cdb40c
RD		 2077 }
2078
2079 if (user_wr->num_sge + sg_ind > cmd.sge_count) {
2080 ret = -EINVAL;
9ead190b 2081 goto out_put;
67cdb40c
RD		 2082 }
2083
e622f2f4
CH		 2084 if (is_ud) {
2085 struct ib_ud_wr *ud;
2086
2087 if (user_wr->opcode != IB_WR_SEND &&
2088 user_wr->opcode != IB_WR_SEND_WITH_IMM) {
2089 ret = -EINVAL;
2090 goto out_put;
2091 }
2092
1d784b89
MM		 2093 next_size = sizeof(*ud);
2094 ud = alloc_wr(next_size, user_wr->num_sge);
e622f2f4
CH		 2095 if (!ud) {
2096 ret = -ENOMEM;
2097 goto out_put;
2098 }
2099
2cc1e3b8 2100 ud->ah = uobj_get_obj_read(ah, UVERBS_OBJECT_AH,
8313c10f 2101 user_wr->wr.ud.ah, attrs);
e622f2f4
CH		 2102 if (!ud->ah) {
2103 kfree(ud);
2104 ret = -EINVAL;
2105 goto out_put;
2106 }
2107 ud->remote_qpn = user_wr->wr.ud.remote_qpn;
2108 ud->remote_qkey = user_wr->wr.ud.remote_qkey;
2109
2110 next = &ud->wr;
2111 } else if (user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
2112 user_wr->opcode == IB_WR_RDMA_WRITE ||
2113 user_wr->opcode == IB_WR_RDMA_READ) {
2114 struct ib_rdma_wr *rdma;
2115
1d784b89
MM		 2116 next_size = sizeof(*rdma);
2117 rdma = alloc_wr(next_size, user_wr->num_sge);
e622f2f4
CH		 2118 if (!rdma) {
2119 ret = -ENOMEM;
2120 goto out_put;
2121 }
2122
2123 rdma->remote_addr = user_wr->wr.rdma.remote_addr;
2124 rdma->rkey = user_wr->wr.rdma.rkey;
2125
2126 next = &rdma->wr;
2127 } else if (user_wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
2128 user_wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
2129 struct ib_atomic_wr *atomic;
2130
1d784b89
MM		 2131 next_size = sizeof(*atomic);
2132 atomic = alloc_wr(next_size, user_wr->num_sge);
e622f2f4
CH		 2133 if (!atomic) {
2134 ret = -ENOMEM;
2135 goto out_put;
2136 }
2137
2138 atomic->remote_addr = user_wr->wr.atomic.remote_addr;
2139 atomic->compare_add = user_wr->wr.atomic.compare_add;
2140 atomic->swap = user_wr->wr.atomic.swap;
2141 atomic->rkey = user_wr->wr.atomic.rkey;
2142
2143 next = &atomic->wr;
2144 } else if (user_wr->opcode == IB_WR_SEND ||
2145 user_wr->opcode == IB_WR_SEND_WITH_IMM ||
2146 user_wr->opcode == IB_WR_SEND_WITH_INV) {
1d784b89
MM		 2147 next_size = sizeof(*next);
2148 next = alloc_wr(next_size, user_wr->num_sge);
e622f2f4
CH		 2149 if (!next) {
2150 ret = -ENOMEM;
2151 goto out_put;
2152 }
2153 } else {
2154 ret = -EINVAL;
9ead190b 2155 goto out_put;
67cdb40c
RD		 2156 }
2157
e622f2f4
CH		 2158 if (user_wr->opcode == IB_WR_SEND_WITH_IMM ||
2159 user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) {
2160 next->ex.imm_data =
2161 (__be32 __force) user_wr->ex.imm_data;
2162 } else if (user_wr->opcode == IB_WR_SEND_WITH_INV) {
2163 next->ex.invalidate_rkey = user_wr->ex.invalidate_rkey;
2164 }
2165
67cdb40c
RD		 2166 if (!last)
2167 wr = next;
2168 else
2169 last->next = next;
2170 last = next;
2171
2172 next->next = NULL;
2173 next->wr_id = user_wr->wr_id;
2174 next->num_sge = user_wr->num_sge;
2175 next->opcode = user_wr->opcode;
2176 next->send_flags = user_wr->send_flags;
67cdb40c 2177
67cdb40c
RD		 2178 if (next->num_sge) {
2179 next->sg_list = (void *) next +
1d784b89 2180 ALIGN(next_size, sizeof(struct ib_sge));
c3bea3d2
JG		 2181 if (copy_from_user(next->sg_list, sgls + sg_ind,
2182 next->num_sge *
2183 sizeof(struct ib_sge))) {
67cdb40c 2184 ret = -EFAULT;
9ead190b 2185 goto out_put;
67cdb40c
RD		 2186 }
2187 sg_ind += next->num_sge;
2188 } else
2189 next->sg_list = NULL;
2190 }
2191
2192 resp.bad_wr = 0;
3023a1e9 2193 ret = qp->device->ops.post_send(qp->real_qp, wr, &bad_wr);
67cdb40c
RD		 2194 if (ret)
2195 for (next = wr; next; next = next->next) {
2196 ++resp.bad_wr;
2197 if (next == bad_wr)
2198 break;
2199 }
2200
9a073857
JG		 2201 ret2 = uverbs_response(attrs, &resp, sizeof(resp));
2202 if (ret2)
2203 ret = ret2;
67cdb40c 2204
9ead190b 2205out_put:
fd3c7904 2206 uobj_put_obj_read(qp);
67cdb40c
RD		 2207
2208 while (wr) {
e622f2f4 2209 if (is_ud && ud_wr(wr)->ah)
fd3c7904 2210 uobj_put_obj_read(ud_wr(wr)->ah);
67cdb40c
RD		 2211 next = wr->next;
2212 kfree(wr);
2213 wr = next;
2214 }
2215
18320828 2216out:
67cdb40c
RD		 2217 kfree(user_wr);
2218
7106a976 2219 return ret;
67cdb40c
RD		 2220}
2221
c3bea3d2
JG		 2222static struct ib_recv_wr *
2223ib_uverbs_unmarshall_recv(struct uverbs_req_iter *iter, u32 wr_count,
2224 u32 wqe_size, u32 sge_count)
67cdb40c
RD		 2225{
2226 struct ib_uverbs_recv_wr *user_wr;
2227 struct ib_recv_wr *wr = NULL, *last, *next;
2228 int sg_ind;
2229 int i;
2230 int ret;
c3bea3d2
JG		 2231 const struct ib_sge __user *sgls;
2232 const void __user *wqes;
67cdb40c
RD		 2233
2234 if (wqe_size < sizeof (struct ib_uverbs_recv_wr))
2235 return ERR_PTR(-EINVAL);
2236
c3bea3d2
JG		 2237 wqes = uverbs_request_next_ptr(iter, wqe_size * wr_count);
2238 if (IS_ERR(wqes))
2239 return ERR_CAST(wqes);
2240 sgls = uverbs_request_next_ptr(
2241 iter, sge_count * sizeof(struct ib_uverbs_sge));
2242 if (IS_ERR(sgls))
2243 return ERR_CAST(sgls);
2244 ret = uverbs_request_finish(iter);
2245 if (ret)
2246 return ERR_PTR(ret);
2247
67cdb40c
RD		 2248 user_wr = kmalloc(wqe_size, GFP_KERNEL);
2249 if (!user_wr)
2250 return ERR_PTR(-ENOMEM);
2251
2252 sg_ind = 0;
2253 last = NULL;
2254 for (i = 0; i < wr_count; ++i) {
c3bea3d2 2255 if (copy_from_user(user_wr, wqes + i * wqe_size,
67cdb40c
RD		 2256 wqe_size)) {
2257 ret = -EFAULT;
2258 goto err;
2259 }
2260
2261 if (user_wr->num_sge + sg_ind > sge_count) {
2262 ret = -EINVAL;
2263 goto err;
2264 }
2265
4f7f4dcf
VT		 2266 if (user_wr->num_sge >=
2267 (U32_MAX - ALIGN(sizeof *next, sizeof (struct ib_sge))) /
2268 sizeof (struct ib_sge)) {
2269 ret = -EINVAL;
2270 goto err;
2271 }
2272
67cdb40c
RD		 2273 next = kmalloc(ALIGN(sizeof *next, sizeof (struct ib_sge)) +
2274 user_wr->num_sge * sizeof (struct ib_sge),
2275 GFP_KERNEL);
2276 if (!next) {
2277 ret = -ENOMEM;
2278 goto err;
2279 }
2280
2281 if (!last)
2282 wr = next;
2283 else
2284 last->next = next;
2285 last = next;
2286
2287 next->next = NULL;
2288 next->wr_id = user_wr->wr_id;
2289 next->num_sge = user_wr->num_sge;
2290
2291 if (next->num_sge) {
2292 next->sg_list = (void *) next +
2293 ALIGN(sizeof *next, sizeof (struct ib_sge));
c3bea3d2
JG		 2294 if (copy_from_user(next->sg_list, sgls + sg_ind,
2295 next->num_sge *
2296 sizeof(struct ib_sge))) {
67cdb40c
RD		 2297 ret = -EFAULT;
2298 goto err;
2299 }
2300 sg_ind += next->num_sge;
2301 } else
2302 next->sg_list = NULL;
2303 }
2304
2305 kfree(user_wr);
2306 return wr;
2307
2308err:
2309 kfree(user_wr);
2310
2311 while (wr) {
2312 next = wr->next;
2313 kfree(wr);
2314 wr = next;
2315 }
2316
2317 return ERR_PTR(ret);
2318}
2319
974d6b4b 2320static int ib_uverbs_post_recv(struct uverbs_attr_bundle *attrs)
67cdb40c
RD		 2321{
2322 struct ib_uverbs_post_recv cmd;
2323 struct ib_uverbs_post_recv_resp resp;
d34ac5cd
BVA		 2324 struct ib_recv_wr *wr, *next;
2325 const struct ib_recv_wr *bad_wr;
67cdb40c 2326 struct ib_qp *qp;
9a073857 2327 int ret, ret2;
c3bea3d2 2328 struct uverbs_req_iter iter;
67cdb40c 2329
c3bea3d2
JG		 2330 ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
2331 if (ret)
2332 return ret;
67cdb40c 2333
c3bea3d2
JG		 2334 wr = ib_uverbs_unmarshall_recv(&iter, cmd.wr_count, cmd.wqe_size,
2335 cmd.sge_count);
67cdb40c
RD		 2336 if (IS_ERR(wr))
2337 return PTR_ERR(wr);
2338
8313c10f 2339 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
9a073857
JG		 2340 if (!qp) {
2341 ret = -EINVAL;
67cdb40c 2342 goto out;
9a073857 2343 }
67cdb40c
RD		 2344
2345 resp.bad_wr = 0;
3023a1e9 2346 ret = qp->device->ops.post_recv(qp->real_qp, wr, &bad_wr);
9ead190b 2347
fd3c7904
MB		 2348 uobj_put_obj_read(qp);
2349 if (ret) {
67cdb40c
RD		 2350 for (next = wr; next; next = next->next) {
2351 ++resp.bad_wr;
2352 if (next == bad_wr)
2353 break;
2354 }
fd3c7904 2355 }
67cdb40c 2356
9a073857
JG		 2357 ret2 = uverbs_response(attrs, &resp, sizeof(resp));
2358 if (ret2)
2359 ret = ret2;
67cdb40c 2360out:
67cdb40c
RD		 2361 while (wr) {
2362 next = wr->next;
2363 kfree(wr);
2364 wr = next;
2365 }
2366
7106a976 2367 return ret;
67cdb40c
RD		 2368}
2369
974d6b4b 2370static int ib_uverbs_post_srq_recv(struct uverbs_attr_bundle *attrs)
67cdb40c
RD		 2371{
2372 struct ib_uverbs_post_srq_recv cmd;
2373 struct ib_uverbs_post_srq_recv_resp resp;
d34ac5cd
BVA		 2374 struct ib_recv_wr *wr, *next;
2375 const struct ib_recv_wr *bad_wr;
67cdb40c 2376 struct ib_srq *srq;
9a073857 2377 int ret, ret2;
c3bea3d2 2378 struct uverbs_req_iter iter;
67cdb40c 2379
c3bea3d2
JG		 2380 ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
2381 if (ret)
2382 return ret;
67cdb40c 2383
c3bea3d2
JG		 2384 wr = ib_uverbs_unmarshall_recv(&iter, cmd.wr_count, cmd.wqe_size,
2385 cmd.sge_count);
67cdb40c
RD		 2386 if (IS_ERR(wr))
2387 return PTR_ERR(wr);
2388
8313c10f 2389 srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);
9a073857
JG		 2390 if (!srq) {
2391 ret = -EINVAL;
67cdb40c 2392 goto out;
9a073857 2393 }
67cdb40c
RD		 2394
2395 resp.bad_wr = 0;
3023a1e9 2396 ret = srq->device->ops.post_srq_recv(srq, wr, &bad_wr);
9ead190b 2397
fd3c7904 2398 uobj_put_obj_read(srq);
9ead190b 2399
67cdb40c
RD		 2400 if (ret)
2401 for (next = wr; next; next = next->next) {
2402 ++resp.bad_wr;
2403 if (next == bad_wr)
2404 break;
2405 }
2406
9a073857
JG		 2407 ret2 = uverbs_response(attrs, &resp, sizeof(resp));
2408 if (ret2)
2409 ret = ret2;
67cdb40c
RD		 2410
2411out:
67cdb40c
RD		 2412 while (wr) {
2413 next = wr->next;
2414 kfree(wr);
2415 wr = next;
2416 }
2417
7106a976 2418 return ret;
67cdb40c
RD		 2419}
2420
974d6b4b 2421static int ib_uverbs_create_ah(struct uverbs_attr_bundle *attrs)
67cdb40c
RD		 2422{
2423 struct ib_uverbs_create_ah cmd;
2424 struct ib_uverbs_create_ah_resp resp;
2425 struct ib_uobject *uobj;
2426 struct ib_pd *pd;
2427 struct ib_ah *ah;
fb51eeca 2428 struct rdma_ah_attr attr = {};
67cdb40c 2429 int ret;
bbd51e88 2430 struct ib_device *ib_dev;
67cdb40c 2431
3c2c2094
JG		 2432 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
2433 if (ret)
2434 return ret;
67cdb40c 2435
8313c10f 2436 uobj = uobj_alloc(UVERBS_OBJECT_AH, attrs, &ib_dev);
fd3c7904
MB		 2437 if (IS_ERR(uobj))
2438 return PTR_ERR(uobj);
67cdb40c 2439
bbd51e88
JG		 2440 if (!rdma_is_port_valid(ib_dev, cmd.attr.port_num)) {
2441 ret = -EINVAL;
2442 goto err;
2443 }
2444
8313c10f 2445 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
9ead190b 2446 if (!pd) {
67cdb40c 2447 ret = -EINVAL;
9ead190b 2448 goto err;
67cdb40c
RD		 2449 }
2450
44c58487 2451 attr.type = rdma_ah_find_type(ib_dev, cmd.attr.port_num);
d98bb7f7 2452 rdma_ah_set_make_grd(&attr, false);
d8966fcd
DC		 2453 rdma_ah_set_dlid(&attr, cmd.attr.dlid);
2454 rdma_ah_set_sl(&attr, cmd.attr.sl);
2455 rdma_ah_set_path_bits(&attr, cmd.attr.src_path_bits);
2456 rdma_ah_set_static_rate(&attr, cmd.attr.static_rate);
2457 rdma_ah_set_port_num(&attr, cmd.attr.port_num);
2458
4ba66093 2459 if (cmd.attr.is_global) {
d8966fcd
DC		 2460 rdma_ah_set_grh(&attr, NULL, cmd.attr.grh.flow_label,
2461 cmd.attr.grh.sgid_index,
2462 cmd.attr.grh.hop_limit,
2463 cmd.attr.grh.traffic_class);
2464 rdma_ah_set_dgid_raw(&attr, cmd.attr.grh.dgid);
4ba66093 2465 } else {
d8966fcd 2466 rdma_ah_set_ah_flags(&attr, 0);
4ba66093 2467 }
477864c8 2468
ef87df2c 2469 ah = rdma_create_user_ah(pd, &attr, &attrs->driver_udata);
67cdb40c
RD		 2470 if (IS_ERR(ah)) {
2471 ret = PTR_ERR(ah);
fd3c7904 2472 goto err_put;
67cdb40c
RD		 2473 }
2474
9ead190b 2475 ah->uobject = uobj;
fd3c7904 2476 uobj->user_handle = cmd.user_handle;
9ead190b 2477 uobj->object = ah;
67cdb40c 2478
67cdb40c
RD		 2479 resp.ah_handle = uobj->id;
2480
9a073857
JG		 2481 ret = uverbs_response(attrs, &resp, sizeof(resp));
2482 if (ret)
9ead190b 2483 goto err_copy;
67cdb40c 2484
fd3c7904 2485 uobj_put_obj_read(pd);
7106a976 2486 return uobj_alloc_commit(uobj);
67cdb40c 2487
9ead190b 2488err_copy:
36523159 2489 rdma_destroy_ah(ah);
67cdb40c 2490
fd3c7904
MB		 2491err_put:
2492 uobj_put_obj_read(pd);
ec924b47 2493
9ead190b 2494err:
fd3c7904 2495 uobj_alloc_abort(uobj);
67cdb40c
RD		 2496 return ret;
2497}
2498
974d6b4b 2499static int ib_uverbs_destroy_ah(struct uverbs_attr_bundle *attrs)
67cdb40c
RD		 2500{
2501 struct ib_uverbs_destroy_ah cmd;
3c2c2094 2502 int ret;
67cdb40c 2503
3c2c2094
JG		 2504 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
2505 if (ret)
2506 return ret;
67cdb40c 2507
7106a976 2508 return uobj_perform_destroy(UVERBS_OBJECT_AH, cmd.ah_handle, attrs);
67cdb40c
RD		 2509}
2510
974d6b4b 2511static int ib_uverbs_attach_mcast(struct uverbs_attr_bundle *attrs)
bc38a6ab
RD		 2512{
2513 struct ib_uverbs_attach_mcast cmd;
2514 struct ib_qp *qp;
9ead190b 2515 struct ib_uqp_object *obj;
f4e40156 2516 struct ib_uverbs_mcast_entry *mcast;
9ead190b 2517 int ret;
bc38a6ab 2518
3c2c2094
JG		 2519 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
2520 if (ret)
2521 return ret;
bc38a6ab 2522
8313c10f 2523 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
9ead190b
RD		 2524 if (!qp)
2525 return -EINVAL;
f4e40156 2526
9ead190b 2527 obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject);
f4e40156 2528
f48b7269 2529 mutex_lock(&obj->mcast_lock);
9ead190b 2530 list_for_each_entry(mcast, &obj->mcast_list, list)
f4e40156
JM		 2531 if (cmd.mlid == mcast->lid &&
2532 !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) {
2533 ret = 0;
9ead190b 2534 goto out_put;
f4e40156
JM		 2535 }
2536
2537 mcast = kmalloc(sizeof *mcast, GFP_KERNEL);
2538 if (!mcast) {
2539 ret = -ENOMEM;
9ead190b 2540 goto out_put;
f4e40156
JM		 2541 }
2542
2543 mcast->lid = cmd.mlid;
2544 memcpy(mcast->gid.raw, cmd.gid, sizeof mcast->gid.raw);
bc38a6ab 2545
f4e40156 2546 ret = ib_attach_mcast(qp, &mcast->gid, cmd.mlid);
9ead190b
RD		 2547 if (!ret)
2548 list_add_tail(&mcast->list, &obj->mcast_list);
2549 else
f4e40156
JM		 2550 kfree(mcast);
2551
9ead190b 2552out_put:
f48b7269 2553 mutex_unlock(&obj->mcast_lock);
fd3c7904 2554 uobj_put_obj_read(qp);
bc38a6ab 2555
7106a976 2556 return ret;
bc38a6ab
RD		 2557}
2558
974d6b4b 2559static int ib_uverbs_detach_mcast(struct uverbs_attr_bundle *attrs)
bc38a6ab
RD		 2560{
2561 struct ib_uverbs_detach_mcast cmd;
9ead190b 2562 struct ib_uqp_object *obj;
bc38a6ab 2563 struct ib_qp *qp;
f4e40156 2564 struct ib_uverbs_mcast_entry *mcast;
bc38a6ab 2565 int ret = -EINVAL;
20c7840a 2566 bool found = false;
bc38a6ab 2567
3c2c2094
JG		 2568 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
2569 if (ret)
2570 return ret;
bc38a6ab 2571
8313c10f 2572 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
9ead190b
RD		 2573 if (!qp)
2574 return -EINVAL;
bc38a6ab 2575
fd3c7904 2576 obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject);
f48b7269 2577 mutex_lock(&obj->mcast_lock);
fd3c7904 2578
9ead190b 2579 list_for_each_entry(mcast, &obj->mcast_list, list)
f4e40156
JM		 2580 if (cmd.mlid == mcast->lid &&
2581 !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) {
2582 list_del(&mcast->list);
2583 kfree(mcast);
20c7840a 2584 found = true;
f4e40156
JM		 2585 break;
2586 }
2587
20c7840a
MR		 2588 if (!found) {
2589 ret = -EINVAL;
2590 goto out_put;
2591 }
2592
2593 ret = ib_detach_mcast(qp, (union ib_gid *)cmd.gid, cmd.mlid);
2594
9ead190b 2595out_put:
f48b7269 2596 mutex_unlock(&obj->mcast_lock);
fd3c7904 2597 uobj_put_obj_read(qp);
7106a976 2598 return ret;
bc38a6ab 2599}
f520ba5a 2600
fa76d24e 2601struct ib_uflow_resources *flow_resources_alloc(size_t num_specs)
9b828441
MB		 2602{
2603 struct ib_uflow_resources *resources;
2604
b6ba4a9a 2605 resources = kzalloc(sizeof(*resources), GFP_KERNEL);
9b828441
MB		 2606
2607 if (!resources)
de749814 2608 return NULL;
b6ba4a9a 2609
a5cc9831
LR		 2610 if (!num_specs)
2611 goto out;
2612
b6ba4a9a
RS		 2613 resources->counters =
2614 kcalloc(num_specs, sizeof(*resources->counters), GFP_KERNEL);
b6ba4a9a
RS		 2615 resources->collection =
2616 kcalloc(num_specs, sizeof(*resources->collection), GFP_KERNEL);
2617
de749814
LR		 2618 if (!resources->counters || !resources->collection)
2619 goto err;
9b828441 2620
a5cc9831 2621out:
9b828441 2622 resources->max = num_specs;
9b828441 2623 return resources;
b6ba4a9a 2624
de749814 2625err:
b6ba4a9a 2626 kfree(resources->counters);
b6ba4a9a 2627 kfree(resources);
de749814 2628
b6ba4a9a 2629 return NULL;
9b828441 2630}
fa76d24e 2631EXPORT_SYMBOL(flow_resources_alloc);
9b828441
MB		 2632
2633void ib_uverbs_flow_resources_free(struct ib_uflow_resources *uflow_res)
2634{
2635 unsigned int i;
2636
6cd080a6
YH		 2637 if (!uflow_res)
2638 return;
2639
b6ba4a9a 2640 for (i = 0; i < uflow_res->collection_num; i++)
9b828441
MB		 2641 atomic_dec(&uflow_res->collection[i]->usecnt);
2642
b6ba4a9a
RS		 2643 for (i = 0; i < uflow_res->counters_num; i++)
2644 atomic_dec(&uflow_res->counters[i]->usecnt);
2645
2646 kfree(uflow_res->collection);
2647 kfree(uflow_res->counters);
9b828441
MB		 2648 kfree(uflow_res);
2649}
fa76d24e 2650EXPORT_SYMBOL(ib_uverbs_flow_resources_free);
9b828441 2651
fa76d24e
MB		 2652void flow_resources_add(struct ib_uflow_resources *uflow_res,
2653 enum ib_flow_spec_type type,
2654 void *ibobj)
9b828441
MB		 2655{
2656 WARN_ON(uflow_res->num >= uflow_res->max);
2657
b6ba4a9a
RS		 2658 switch (type) {
2659 case IB_FLOW_SPEC_ACTION_HANDLE:
2660 atomic_inc(&((struct ib_flow_action *)ibobj)->usecnt);
2661 uflow_res->collection[uflow_res->collection_num++] =
2662 (struct ib_flow_action *)ibobj;
2663 break;
2664 case IB_FLOW_SPEC_ACTION_COUNT:
2665 atomic_inc(&((struct ib_counters *)ibobj)->usecnt);
2666 uflow_res->counters[uflow_res->counters_num++] =
2667 (struct ib_counters *)ibobj;
2668 break;
2669 default:
2670 WARN_ON(1);
2671 }
2672
2673 uflow_res->num++;
9b828441 2674}
fa76d24e 2675EXPORT_SYMBOL(flow_resources_add);
9b828441 2676
8313c10f 2677static int kern_spec_to_ib_spec_action(const struct uverbs_attr_bundle *attrs,
9b828441
MB		 2678 struct ib_uverbs_flow_spec *kern_spec,
2679 union ib_flow_spec *ib_spec,
2680 struct ib_uflow_resources *uflow_res)
94e03f11
MR		 2681{
2682 ib_spec->type = kern_spec->type;
2683 switch (ib_spec->type) {
2684 case IB_FLOW_SPEC_ACTION_TAG:
2685 if (kern_spec->flow_tag.size !=
2686 sizeof(struct ib_uverbs_flow_spec_action_tag))
2687 return -EINVAL;
2688
2689 ib_spec->flow_tag.size = sizeof(struct ib_flow_spec_action_tag);
2690 ib_spec->flow_tag.tag_id = kern_spec->flow_tag.tag_id;
2691 break;
483a3966
SS		 2692 case IB_FLOW_SPEC_ACTION_DROP:
2693 if (kern_spec->drop.size !=
2694 sizeof(struct ib_uverbs_flow_spec_action_drop))
2695 return -EINVAL;
2696
2697 ib_spec->drop.size = sizeof(struct ib_flow_spec_action_drop);
2698 break;
9b828441
MB		 2699 case IB_FLOW_SPEC_ACTION_HANDLE:
2700 if (kern_spec->action.size !=
2701 sizeof(struct ib_uverbs_flow_spec_action_handle))
2702 return -EOPNOTSUPP;
2703 ib_spec->action.act = uobj_get_obj_read(flow_action,
2704 UVERBS_OBJECT_FLOW_ACTION,
2705 kern_spec->action.handle,
8313c10f 2706 attrs);
9b828441
MB		 2707 if (!ib_spec->action.act)
2708 return -EINVAL;
2709 ib_spec->action.size =
2710 sizeof(struct ib_flow_spec_action_handle);
b6ba4a9a
RS		 2711 flow_resources_add(uflow_res,
2712 IB_FLOW_SPEC_ACTION_HANDLE,
2713 ib_spec->action.act);
9b828441
MB		 2714 uobj_put_obj_read(ib_spec->action.act);
2715 break;
b6ba4a9a
RS		 2716 case IB_FLOW_SPEC_ACTION_COUNT:
2717 if (kern_spec->flow_count.size !=
2718 sizeof(struct ib_uverbs_flow_spec_action_count))
2719 return -EINVAL;
2720 ib_spec->flow_count.counters =
2721 uobj_get_obj_read(counters,
2722 UVERBS_OBJECT_COUNTERS,
2723 kern_spec->flow_count.handle,
8313c10f 2724 attrs);
b6ba4a9a
RS		 2725 if (!ib_spec->flow_count.counters)
2726 return -EINVAL;
2727 ib_spec->flow_count.size =
2728 sizeof(struct ib_flow_spec_action_count);
2729 flow_resources_add(uflow_res,
2730 IB_FLOW_SPEC_ACTION_COUNT,
2731 ib_spec->flow_count.counters);
2732 uobj_put_obj_read(ib_spec->flow_count.counters);
2733 break;
94e03f11
MR		 2734 default:
2735 return -EINVAL;
2736 }
2737 return 0;
2738}
2739
766d8551 2740static size_t kern_spec_filter_sz(const struct ib_uverbs_flow_spec_hdr *spec)
15dfbd6b
MG		 2741{
2742 /* Returns user space filter size, includes padding */
2743 return (spec->size - sizeof(struct ib_uverbs_flow_spec_hdr)) / 2;
2744}
2745
766d8551 2746static ssize_t spec_filter_size(const void *kern_spec_filter, u16 kern_filter_size,
15dfbd6b
MG		 2747 u16 ib_real_filter_sz)
2748{
2749 /*
2750 * User space filter structures must be 64 bit aligned, otherwise this
2751 * may pass, but we won't handle additional new attributes.
2752 */
2753
2754 if (kern_filter_size > ib_real_filter_sz) {
2755 if (memchr_inv(kern_spec_filter +
2756 ib_real_filter_sz, 0,
2757 kern_filter_size - ib_real_filter_sz))
2758 return -EINVAL;
2759 return ib_real_filter_sz;
2760 }
2761 return kern_filter_size;
2762}
2763
766d8551
MB		 2764int ib_uverbs_kern_spec_to_ib_spec_filter(enum ib_flow_spec_type type,
2765 const void *kern_spec_mask,
2766 const void *kern_spec_val,
2767 size_t kern_filter_sz,
2768 union ib_flow_spec *ib_spec)
436f2ad0 2769{
15dfbd6b 2770 ssize_t actual_filter_sz;
15dfbd6b 2771 ssize_t ib_filter_sz;
15dfbd6b 2772
15dfbd6b
MG		 2773 /* User flow spec size must be aligned to 4 bytes */
2774 if (kern_filter_sz != ALIGN(kern_filter_sz, 4))
2775 return -EINVAL;
2776
766d8551
MB		 2777 ib_spec->type = type;
2778
fbf46860
MR		 2779 if (ib_spec->type == (IB_FLOW_SPEC_INNER | IB_FLOW_SPEC_VXLAN_TUNNEL))
2780 return -EINVAL;
15dfbd6b 2781
fbf46860 2782 switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) {
436f2ad0 2783 case IB_FLOW_SPEC_ETH:
15dfbd6b
MG		 2784 ib_filter_sz = offsetof(struct ib_flow_eth_filter, real_sz);
2785 actual_filter_sz = spec_filter_size(kern_spec_mask,
2786 kern_filter_sz,
2787 ib_filter_sz);
2788 if (actual_filter_sz <= 0)
436f2ad0 2789 return -EINVAL;
15dfbd6b
MG		 2790 ib_spec->size = sizeof(struct ib_flow_spec_eth);
2791 memcpy(&ib_spec->eth.val, kern_spec_val, actual_filter_sz);
2792 memcpy(&ib_spec->eth.mask, kern_spec_mask, actual_filter_sz);
436f2ad0
HHZ		 2793 break;
2794 case IB_FLOW_SPEC_IPV4:
15dfbd6b
MG		 2795 ib_filter_sz = offsetof(struct ib_flow_ipv4_filter, real_sz);
2796 actual_filter_sz = spec_filter_size(kern_spec_mask,
2797 kern_filter_sz,
2798 ib_filter_sz);
2799 if (actual_filter_sz <= 0)
436f2ad0 2800 return -EINVAL;
15dfbd6b
MG		 2801 ib_spec->size = sizeof(struct ib_flow_spec_ipv4);
2802 memcpy(&ib_spec->ipv4.val, kern_spec_val, actual_filter_sz);
2803 memcpy(&ib_spec->ipv4.mask, kern_spec_mask, actual_filter_sz);
436f2ad0 2804 break;
4c2aae71 2805 case IB_FLOW_SPEC_IPV6:
15dfbd6b
MG		 2806 ib_filter_sz = offsetof(struct ib_flow_ipv6_filter, real_sz);
2807 actual_filter_sz = spec_filter_size(kern_spec_mask,
2808 kern_filter_sz,
2809 ib_filter_sz);
2810 if (actual_filter_sz <= 0)
4c2aae71 2811 return -EINVAL;
15dfbd6b
MG		 2812 ib_spec->size = sizeof(struct ib_flow_spec_ipv6);
2813 memcpy(&ib_spec->ipv6.val, kern_spec_val, actual_filter_sz);
2814 memcpy(&ib_spec->ipv6.mask, kern_spec_mask, actual_filter_sz);
a72c6a2b
MG		 2815
2816 if ((ntohl(ib_spec->ipv6.mask.flow_label)) >= BIT(20) ||
2817 (ntohl(ib_spec->ipv6.val.flow_label)) >= BIT(20))
2818 return -EINVAL;
4c2aae71 2819 break;
436f2ad0
HHZ		 2820 case IB_FLOW_SPEC_TCP:
2821 case IB_FLOW_SPEC_UDP:
15dfbd6b
MG		 2822 ib_filter_sz = offsetof(struct ib_flow_tcp_udp_filter, real_sz);
2823 actual_filter_sz = spec_filter_size(kern_spec_mask,
2824 kern_filter_sz,
2825 ib_filter_sz);
2826 if (actual_filter_sz <= 0)
436f2ad0 2827 return -EINVAL;
15dfbd6b
MG		 2828 ib_spec->size = sizeof(struct ib_flow_spec_tcp_udp);
2829 memcpy(&ib_spec->tcp_udp.val, kern_spec_val, actual_filter_sz);
2830 memcpy(&ib_spec->tcp_udp.mask, kern_spec_mask, actual_filter_sz);
436f2ad0 2831 break;
0dbf3332
MR		 2832 case IB_FLOW_SPEC_VXLAN_TUNNEL:
2833 ib_filter_sz = offsetof(struct ib_flow_tunnel_filter, real_sz);
2834 actual_filter_sz = spec_filter_size(kern_spec_mask,
2835 kern_filter_sz,
2836 ib_filter_sz);
2837 if (actual_filter_sz <= 0)
2838 return -EINVAL;
2839 ib_spec->tunnel.size = sizeof(struct ib_flow_spec_tunnel);
2840 memcpy(&ib_spec->tunnel.val, kern_spec_val, actual_filter_sz);
2841 memcpy(&ib_spec->tunnel.mask, kern_spec_mask, actual_filter_sz);
2842
2843 if ((ntohl(ib_spec->tunnel.mask.tunnel_id)) >= BIT(24) ||
2844 (ntohl(ib_spec->tunnel.val.tunnel_id)) >= BIT(24))
2845 return -EINVAL;
2846 break;
56ab0b38
MB		 2847 case IB_FLOW_SPEC_ESP:
2848 ib_filter_sz = offsetof(struct ib_flow_esp_filter, real_sz);
2849 actual_filter_sz = spec_filter_size(kern_spec_mask,
2850 kern_filter_sz,
2851 ib_filter_sz);
2852 if (actual_filter_sz <= 0)
2853 return -EINVAL;
2854 ib_spec->esp.size = sizeof(struct ib_flow_spec_esp);
2855 memcpy(&ib_spec->esp.val, kern_spec_val, actual_filter_sz);
2856 memcpy(&ib_spec->esp.mask, kern_spec_mask, actual_filter_sz);
2857 break;
d90e5e50
AL		 2858 case IB_FLOW_SPEC_GRE:
2859 ib_filter_sz = offsetof(struct ib_flow_gre_filter, real_sz);
2860 actual_filter_sz = spec_filter_size(kern_spec_mask,
2861 kern_filter_sz,
2862 ib_filter_sz);
2863 if (actual_filter_sz <= 0)
2864 return -EINVAL;
2865 ib_spec->gre.size = sizeof(struct ib_flow_spec_gre);
2866 memcpy(&ib_spec->gre.val, kern_spec_val, actual_filter_sz);
2867 memcpy(&ib_spec->gre.mask, kern_spec_mask, actual_filter_sz);
2868 break;
b04f0f03
AL		 2869 case IB_FLOW_SPEC_MPLS:
2870 ib_filter_sz = offsetof(struct ib_flow_mpls_filter, real_sz);
2871 actual_filter_sz = spec_filter_size(kern_spec_mask,
2872 kern_filter_sz,
2873 ib_filter_sz);
2874 if (actual_filter_sz <= 0)
2875 return -EINVAL;
2876 ib_spec->mpls.size = sizeof(struct ib_flow_spec_mpls);
2877 memcpy(&ib_spec->mpls.val, kern_spec_val, actual_filter_sz);
2878 memcpy(&ib_spec->mpls.mask, kern_spec_mask, actual_filter_sz);
2879 break;
436f2ad0
HHZ		 2880 default:
2881 return -EINVAL;
2882 }
2883 return 0;
2884}
2885
766d8551
MB		 2886static int kern_spec_to_ib_spec_filter(struct ib_uverbs_flow_spec *kern_spec,
2887 union ib_flow_spec *ib_spec)
2888{
2889 ssize_t kern_filter_sz;
2890 void *kern_spec_mask;
2891 void *kern_spec_val;
2892
766d8551
MB		 2893 kern_filter_sz = kern_spec_filter_sz(&kern_spec->hdr);
2894
2895 kern_spec_val = (void *)kern_spec +
2896 sizeof(struct ib_uverbs_flow_spec_hdr);
2897 kern_spec_mask = kern_spec_val + kern_filter_sz;
2898
2899 return ib_uverbs_kern_spec_to_ib_spec_filter(kern_spec->type,
2900 kern_spec_mask,
2901 kern_spec_val,
2902 kern_filter_sz, ib_spec);
2903}
2904
8313c10f 2905static int kern_spec_to_ib_spec(struct uverbs_attr_bundle *attrs,
9b828441
MB		 2906 struct ib_uverbs_flow_spec *kern_spec,
2907 union ib_flow_spec *ib_spec,
2908 struct ib_uflow_resources *uflow_res)
94e03f11
MR		 2909{
2910 if (kern_spec->reserved)
2911 return -EINVAL;
2912
2913 if (kern_spec->type >= IB_FLOW_SPEC_ACTION_TAG)
8313c10f 2914 return kern_spec_to_ib_spec_action(attrs, kern_spec, ib_spec,
9b828441 2915 uflow_res);
94e03f11
MR		 2916 else
2917 return kern_spec_to_ib_spec_filter(kern_spec, ib_spec);
2918}
2919
974d6b4b 2920static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle *attrs)
f213c052 2921{
29a29d18 2922 struct ib_uverbs_ex_create_wq cmd;
f213c052
YH		 2923 struct ib_uverbs_ex_create_wq_resp resp = {};
2924 struct ib_uwq_object *obj;
2925 int err = 0;
2926 struct ib_cq *cq;
2927 struct ib_pd *pd;
2928 struct ib_wq *wq;
2929 struct ib_wq_init_attr wq_init_attr = {};
bbd51e88 2930 struct ib_device *ib_dev;
f213c052 2931
29a29d18 2932 err = uverbs_request(attrs, &cmd, sizeof(cmd));
f213c052
YH		 2933 if (err)
2934 return err;
2935
2936 if (cmd.comp_mask)
2937 return -EOPNOTSUPP;
2938
8313c10f 2939 obj = (struct ib_uwq_object *)uobj_alloc(UVERBS_OBJECT_WQ, attrs,
bbd51e88 2940 &ib_dev);
fd3c7904
MB		 2941 if (IS_ERR(obj))
2942 return PTR_ERR(obj);
f213c052 2943
8313c10f 2944 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
f213c052
YH		 2945 if (!pd) {
2946 err = -EINVAL;
2947 goto err_uobj;
2948 }
2949
8313c10f 2950 cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
f213c052
YH		 2951 if (!cq) {
2952 err = -EINVAL;
2953 goto err_put_pd;
2954 }
2955
2956 wq_init_attr.cq = cq;
2957 wq_init_attr.max_sge = cmd.max_sge;
2958 wq_init_attr.max_wr = cmd.max_wr;
8313c10f 2959 wq_init_attr.wq_context = attrs->ufile;
f213c052
YH		 2960 wq_init_attr.wq_type = cmd.wq_type;
2961 wq_init_attr.event_handler = ib_uverbs_wq_event_handler;
29a29d18 2962 wq_init_attr.create_flags = cmd.create_flags;
f213c052
YH		 2963 obj->uevent.events_reported = 0;
2964 INIT_LIST_HEAD(&obj->uevent.event_list);
21885586 2965
3023a1e9 2966 wq = pd->device->ops.create_wq(pd, &wq_init_attr, &attrs->driver_udata);
f213c052
YH		 2967 if (IS_ERR(wq)) {
2968 err = PTR_ERR(wq);
2969 goto err_put_cq;
2970 }
2971
2972 wq->uobject = &obj->uevent.uobject;
2973 obj->uevent.uobject.object = wq;
2974 wq->wq_type = wq_init_attr.wq_type;
2975 wq->cq = cq;
2976 wq->pd = pd;
2977 wq->device = pd->device;
2978 wq->wq_context = wq_init_attr.wq_context;
2979 atomic_set(&wq->usecnt, 0);
2980 atomic_inc(&pd->usecnt);
2981 atomic_inc(&cq->usecnt);
2982 wq->uobject = &obj->uevent.uobject;
2983 obj->uevent.uobject.object = wq;
f213c052
YH		 2984
2985 memset(&resp, 0, sizeof(resp));
2986 resp.wq_handle = obj->uevent.uobject.id;
2987 resp.max_sge = wq_init_attr.max_sge;
2988 resp.max_wr = wq_init_attr.max_wr;
2989 resp.wqn = wq->wq_num;
29a29d18 2990 resp.response_length = uverbs_response_length(attrs, sizeof(resp));
9a073857 2991 err = uverbs_response(attrs, &resp, sizeof(resp));
f213c052
YH		 2992 if (err)
2993 goto err_copy;
2994
fd3c7904
MB		 2995 uobj_put_obj_read(pd);
2996 uobj_put_obj_read(cq);
7106a976 2997 return uobj_alloc_commit(&obj->uevent.uobject);
f213c052
YH		 2998
2999err_copy:
f213c052
YH		 3000 ib_destroy_wq(wq);
3001err_put_cq:
fd3c7904 3002 uobj_put_obj_read(cq);
f213c052 3003err_put_pd:
fd3c7904 3004 uobj_put_obj_read(pd);
f213c052 3005err_uobj:
fd3c7904 3006 uobj_alloc_abort(&obj->uevent.uobject);
f213c052
YH		 3007
3008 return err;
3009}
3010
974d6b4b 3011static int ib_uverbs_ex_destroy_wq(struct uverbs_attr_bundle *attrs)
f213c052 3012{
29a29d18 3013 struct ib_uverbs_ex_destroy_wq cmd;
f213c052 3014 struct ib_uverbs_ex_destroy_wq_resp resp = {};
f213c052
YH		 3015 struct ib_uobject *uobj;
3016 struct ib_uwq_object *obj;
f213c052
YH		 3017 int ret;
3018
29a29d18 3019 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
f213c052
YH		 3020 if (ret)
3021 return ret;
3022
3023 if (cmd.comp_mask)
3024 return -EOPNOTSUPP;
3025
29a29d18 3026 resp.response_length = uverbs_response_length(attrs, sizeof(resp));
8313c10f 3027 uobj = uobj_get_destroy(UVERBS_OBJECT_WQ, cmd.wq_handle, attrs);
fd3c7904
MB		 3028 if (IS_ERR(uobj))
3029 return PTR_ERR(uobj);
f213c052 3030
f213c052 3031 obj = container_of(uobj, struct ib_uwq_object, uevent.uobject);
f213c052 3032 resp.events_reported = obj->uevent.events_reported;
32ed5c00
JG		 3033
3034 uobj_put_destroy(uobj);
f213c052 3035
9a073857 3036 return uverbs_response(attrs, &resp, sizeof(resp));
f213c052
YH		 3037}
3038
974d6b4b 3039static int ib_uverbs_ex_modify_wq(struct uverbs_attr_bundle *attrs)
f213c052 3040{
29a29d18 3041 struct ib_uverbs_ex_modify_wq cmd;
f213c052
YH		 3042 struct ib_wq *wq;
3043 struct ib_wq_attr wq_attr = {};
f213c052
YH		 3044 int ret;
3045
29a29d18 3046 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
f213c052
YH		 3047 if (ret)
3048 return ret;
3049
3050 if (!cmd.attr_mask)
3051 return -EINVAL;
3052
af1cb95d 3053 if (cmd.attr_mask > (IB_WQ_STATE | IB_WQ_CUR_STATE | IB_WQ_FLAGS))
f213c052
YH		 3054 return -EINVAL;
3055
8313c10f 3056 wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ, cmd.wq_handle, attrs);
f213c052
YH		 3057 if (!wq)
3058 return -EINVAL;
3059
3060 wq_attr.curr_wq_state = cmd.curr_wq_state;
3061 wq_attr.wq_state = cmd.wq_state;
af1cb95d
NO		 3062 if (cmd.attr_mask & IB_WQ_FLAGS) {
3063 wq_attr.flags = cmd.flags;
3064 wq_attr.flags_mask = cmd.flags_mask;
3065 }
3023a1e9
KH		 3066 ret = wq->device->ops.modify_wq(wq, &wq_attr, cmd.attr_mask,
3067 &attrs->driver_udata);
fd3c7904 3068 uobj_put_obj_read(wq);
f213c052
YH		 3069 return ret;
3070}
3071
974d6b4b 3072static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs)
de019a94 3073{
335708c7 3074 struct ib_uverbs_ex_create_rwq_ind_table cmd;
de019a94
YH		 3075 struct ib_uverbs_ex_create_rwq_ind_table_resp resp = {};
3076 struct ib_uobject *uobj;
335708c7 3077 int err;
de019a94
YH		 3078 struct ib_rwq_ind_table_init_attr init_attr = {};
3079 struct ib_rwq_ind_table *rwq_ind_tbl;
3080 struct ib_wq **wqs = NULL;
3081 u32 *wqs_handles = NULL;
3082 struct ib_wq *wq = NULL;
3083 int i, j, num_read_wqs;
3084 u32 num_wq_handles;
335708c7 3085 struct uverbs_req_iter iter;
bbd51e88 3086 struct ib_device *ib_dev;
de019a94 3087
335708c7 3088 err = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
de019a94
YH		 3089 if (err)
3090 return err;
3091
de019a94
YH		 3092 if (cmd.comp_mask)
3093 return -EOPNOTSUPP;
3094
3095 if (cmd.log_ind_tbl_size > IB_USER_VERBS_MAX_LOG_IND_TBL_SIZE)
3096 return -EINVAL;
3097
3098 num_wq_handles = 1 << cmd.log_ind_tbl_size;
de019a94
YH		 3099 wqs_handles = kcalloc(num_wq_handles, sizeof(*wqs_handles),
3100 GFP_KERNEL);
3101 if (!wqs_handles)
3102 return -ENOMEM;
3103
335708c7
JG		 3104 err = uverbs_request_next(&iter, wqs_handles,
3105 num_wq_handles * sizeof(__u32));
3106 if (err)
3107 goto err_free;
3108
3109 err = uverbs_request_finish(&iter);
de019a94
YH		 3110 if (err)
3111 goto err_free;
3112
3113 wqs = kcalloc(num_wq_handles, sizeof(*wqs), GFP_KERNEL);
3114 if (!wqs) {
3115 err = -ENOMEM;
3116 goto err_free;
3117 }
3118
3119 for (num_read_wqs = 0; num_read_wqs < num_wq_handles;
3120 num_read_wqs++) {
2cc1e3b8 3121 wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ,
8313c10f 3122 wqs_handles[num_read_wqs], attrs);
de019a94
YH		 3123 if (!wq) {
3124 err = -EINVAL;
3125 goto put_wqs;
3126 }
3127
3128 wqs[num_read_wqs] = wq;
3129 }
3130
8313c10f 3131 uobj = uobj_alloc(UVERBS_OBJECT_RWQ_IND_TBL, attrs, &ib_dev);
fd3c7904
MB		 3132 if (IS_ERR(uobj)) {
3133 err = PTR_ERR(uobj);
de019a94
YH		 3134 goto put_wqs;
3135 }
3136
de019a94
YH		 3137 init_attr.log_ind_tbl_size = cmd.log_ind_tbl_size;
3138 init_attr.ind_tbl = wqs;
21885586 3139
3023a1e9
KH		 3140 rwq_ind_tbl = ib_dev->ops.create_rwq_ind_table(ib_dev, &init_attr,
3141 &attrs->driver_udata);
de019a94
YH		 3142
3143 if (IS_ERR(rwq_ind_tbl)) {
3144 err = PTR_ERR(rwq_ind_tbl);
3145 goto err_uobj;
3146 }
3147
3148 rwq_ind_tbl->ind_tbl = wqs;
3149 rwq_ind_tbl->log_ind_tbl_size = init_attr.log_ind_tbl_size;
3150 rwq_ind_tbl->uobject = uobj;
3151 uobj->object = rwq_ind_tbl;
3152 rwq_ind_tbl->device = ib_dev;
3153 atomic_set(&rwq_ind_tbl->usecnt, 0);
3154
3155 for (i = 0; i < num_wq_handles; i++)
3156 atomic_inc(&wqs[i]->usecnt);
3157
de019a94
YH		 3158 resp.ind_tbl_handle = uobj->id;
3159 resp.ind_tbl_num = rwq_ind_tbl->ind_tbl_num;
29a29d18 3160 resp.response_length = uverbs_response_length(attrs, sizeof(resp));
de019a94 3161
9a073857 3162 err = uverbs_response(attrs, &resp, sizeof(resp));
de019a94
YH		 3163 if (err)
3164 goto err_copy;
3165
3166 kfree(wqs_handles);
3167
3168 for (j = 0; j < num_read_wqs; j++)
fd3c7904 3169 uobj_put_obj_read(wqs[j]);
de019a94 3170
7106a976 3171 return uobj_alloc_commit(uobj);
de019a94
YH		 3172
3173err_copy:
de019a94
YH		 3174 ib_destroy_rwq_ind_table(rwq_ind_tbl);
3175err_uobj:
fd3c7904 3176 uobj_alloc_abort(uobj);
de019a94
YH		 3177put_wqs:
3178 for (j = 0; j < num_read_wqs; j++)
fd3c7904 3179 uobj_put_obj_read(wqs[j]);
de019a94
YH		 3180err_free:
3181 kfree(wqs_handles);
3182 kfree(wqs);
3183 return err;
3184}
3185
974d6b4b 3186static int ib_uverbs_ex_destroy_rwq_ind_table(struct uverbs_attr_bundle *attrs)
de019a94 3187{
29a29d18
JG		 3188 struct ib_uverbs_ex_destroy_rwq_ind_table cmd;
3189 int ret;
de019a94 3190
29a29d18 3191 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
de019a94
YH		 3192 if (ret)
3193 return ret;
3194
3195 if (cmd.comp_mask)
3196 return -EOPNOTSUPP;
3197
c33e73af 3198 return uobj_perform_destroy(UVERBS_OBJECT_RWQ_IND_TBL,
7106a976 3199 cmd.ind_tbl_handle, attrs);
de019a94
YH		 3200}
3201
974d6b4b 3202static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle *attrs)
436f2ad0
HHZ		 3203{
3204 struct ib_uverbs_create_flow cmd;
3205 struct ib_uverbs_create_flow_resp resp;
3206 struct ib_uobject *uobj;
3207 struct ib_flow *flow_id;
d82693da 3208 struct ib_uverbs_flow_attr *kern_flow_attr;
436f2ad0
HHZ		 3209 struct ib_flow_attr *flow_attr;
3210 struct ib_qp *qp;
9b828441 3211 struct ib_uflow_resources *uflow_res;
4fae7f17 3212 struct ib_uverbs_flow_spec_hdr *kern_spec;
335708c7
JG		 3213 struct uverbs_req_iter iter;
3214 int err;
436f2ad0
HHZ		 3215 void *ib_spec;
3216 int i;
bbd51e88 3217 struct ib_device *ib_dev;
436f2ad0 3218
335708c7 3219 err = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
f21519b2
YD		 3220 if (err)
3221 return err;
3222
22878dbc
MB		 3223 if (cmd.comp_mask)
3224 return -EINVAL;
3225
e3b6d8cf 3226 if (!capable(CAP_NET_RAW))
436f2ad0
HHZ		 3227 return -EPERM;
3228
a3100a78
MV		 3229 if (cmd.flow_attr.flags >= IB_FLOW_ATTR_FLAGS_RESERVED)
3230 return -EINVAL;
3231
3232 if ((cmd.flow_attr.flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) &&
3233 ((cmd.flow_attr.type == IB_FLOW_ATTR_ALL_DEFAULT) ||
3234 (cmd.flow_attr.type == IB_FLOW_ATTR_MC_DEFAULT)))
3235 return -EINVAL;
3236
f8848274 3237 if (cmd.flow_attr.num_of_specs > IB_FLOW_SPEC_SUPPORT_LAYERS)
22878dbc
MB		 3238 return -EINVAL;
3239
335708c7 3240 if (cmd.flow_attr.size >
b68c9560 3241 (cmd.flow_attr.num_of_specs * sizeof(struct ib_uverbs_flow_spec)))
22878dbc
MB		 3242 return -EINVAL;
3243
c780d82a
YD		 3244 if (cmd.flow_attr.reserved[0] ||
3245 cmd.flow_attr.reserved[1])
3246 return -EINVAL;
3247
436f2ad0 3248 if (cmd.flow_attr.num_of_specs) {
f8848274
MB		 3249 kern_flow_attr = kmalloc(sizeof(*kern_flow_attr) + cmd.flow_attr.size,
3250 GFP_KERNEL);
436f2ad0
HHZ		 3251 if (!kern_flow_attr)
3252 return -ENOMEM;
3253
4fae7f17 3254 *kern_flow_attr = cmd.flow_attr;
335708c7
JG		 3255 err = uverbs_request_next(&iter, &kern_flow_attr->flow_specs,
3256 cmd.flow_attr.size);
f21519b2 3257 if (err)
436f2ad0 3258 goto err_free_attr;
436f2ad0
HHZ		 3259 } else {
3260 kern_flow_attr = &cmd.flow_attr;
436f2ad0
HHZ		 3261 }
3262
335708c7
JG		 3263 err = uverbs_request_finish(&iter);
3264 if (err)
3265 goto err_free_attr;
3266
8313c10f 3267 uobj = uobj_alloc(UVERBS_OBJECT_FLOW, attrs, &ib_dev);
fd3c7904
MB		 3268 if (IS_ERR(uobj)) {
3269 err = PTR_ERR(uobj);
436f2ad0
HHZ		 3270 goto err_free_attr;
3271 }
436f2ad0 3272
8313c10f 3273 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
436f2ad0
HHZ		 3274 if (!qp) {
3275 err = -EINVAL;
3276 goto err_uobj;
3277 }
3278
940efcc8
LR		 3279 if (qp->qp_type != IB_QPT_UD && qp->qp_type != IB_QPT_RAW_PACKET) {
3280 err = -EINVAL;
3281 goto err_put;
3282 }
3283
7654cb1b
MW		 3284 flow_attr = kzalloc(struct_size(flow_attr, flows,
3285 cmd.flow_attr.num_of_specs), GFP_KERNEL);
436f2ad0
HHZ		 3286 if (!flow_attr) {
3287 err = -ENOMEM;
3288 goto err_put;
3289 }
9b828441
MB		 3290 uflow_res = flow_resources_alloc(cmd.flow_attr.num_of_specs);
3291 if (!uflow_res) {
3292 err = -ENOMEM;
3293 goto err_free_flow_attr;
3294 }
436f2ad0
HHZ		 3295
3296 flow_attr->type = kern_flow_attr->type;
3297 flow_attr->priority = kern_flow_attr->priority;
3298 flow_attr->num_of_specs = kern_flow_attr->num_of_specs;
3299 flow_attr->port = kern_flow_attr->port;
3300 flow_attr->flags = kern_flow_attr->flags;
3301 flow_attr->size = sizeof(*flow_attr);
3302
4fae7f17 3303 kern_spec = kern_flow_attr->flow_specs;
436f2ad0 3304 ib_spec = flow_attr + 1;
f8848274 3305 for (i = 0; i < flow_attr->num_of_specs &&
fe48aecb 3306 cmd.flow_attr.size >= sizeof(*kern_spec) &&
4fae7f17
LR		 3307 cmd.flow_attr.size >= kern_spec->size;
3308 i++) {
3309 err = kern_spec_to_ib_spec(
8313c10f 3310 attrs, (struct ib_uverbs_flow_spec *)kern_spec,
4fae7f17 3311 ib_spec, uflow_res);
436f2ad0
HHZ		 3312 if (err)
3313 goto err_free;
b04f0f03 3314
436f2ad0
HHZ		 3315 flow_attr->size +=
3316 ((union ib_flow_spec *) ib_spec)->size;
4fae7f17
LR		 3317 cmd.flow_attr.size -= kern_spec->size;
3318 kern_spec = ((void *)kern_spec) + kern_spec->size;
436f2ad0
HHZ		 3319 ib_spec += ((union ib_flow_spec *) ib_spec)->size;
3320 }
f8848274
MB		 3321 if (cmd.flow_attr.size || (i != flow_attr->num_of_specs)) {
3322 pr_warn("create flow failed, flow %d: %d bytes left from uverb cmd\n",
3323 i, cmd.flow_attr.size);
98a37510 3324 err = -EINVAL;
436f2ad0
HHZ		 3325 goto err_free;
3326 }
59082a32 3327
3023a1e9
KH		 3328 flow_id = qp->device->ops.create_flow(
3329 qp, flow_attr, IB_FLOW_DOMAIN_USER, &attrs->driver_udata);
59082a32 3330
436f2ad0
HHZ		 3331 if (IS_ERR(flow_id)) {
3332 err = PTR_ERR(flow_id);
fd3c7904 3333 goto err_free;
436f2ad0 3334 }
86e1d464
MB		 3335
3336 ib_set_flow(uobj, flow_id, qp, qp->device, uflow_res);
436f2ad0 3337
436f2ad0
HHZ		 3338 memset(&resp, 0, sizeof(resp));
3339 resp.flow_handle = uobj->id;
3340
9a073857 3341 err = uverbs_response(attrs, &resp, sizeof(resp));
f21519b2 3342 if (err)
436f2ad0 3343 goto err_copy;
436f2ad0 3344
fd3c7904 3345 uobj_put_obj_read(qp);
436f2ad0
HHZ		 3346 kfree(flow_attr);
3347 if (cmd.flow_attr.num_of_specs)
3348 kfree(kern_flow_attr);
7106a976 3349 return uobj_alloc_commit(uobj);
436f2ad0 3350err_copy:
3023a1e9 3351 if (!qp->device->ops.destroy_flow(flow_id))
1ccddc42 3352 atomic_dec(&qp->usecnt);
436f2ad0 3353err_free:
9b828441
MB		 3354 ib_uverbs_flow_resources_free(uflow_res);
3355err_free_flow_attr:
436f2ad0
HHZ		 3356 kfree(flow_attr);
3357err_put:
fd3c7904 3358 uobj_put_obj_read(qp);
436f2ad0 3359err_uobj:
fd3c7904 3360 uobj_alloc_abort(uobj);
436f2ad0
HHZ		 3361err_free_attr:
3362 if (cmd.flow_attr.num_of_specs)
3363 kfree(kern_flow_attr);
3364 return err;
3365}
3366
974d6b4b 3367static int ib_uverbs_ex_destroy_flow(struct uverbs_attr_bundle *attrs)
f21519b2 3368{
436f2ad0 3369 struct ib_uverbs_destroy_flow cmd;
436f2ad0
HHZ		 3370 int ret;
3371
29a29d18 3372 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
f21519b2
YD		 3373 if (ret)
3374 return ret;
436f2ad0 3375
2782c2d3
YD		 3376 if (cmd.comp_mask)
3377 return -EINVAL;
3378
7106a976 3379 return uobj_perform_destroy(UVERBS_OBJECT_FLOW, cmd.flow_handle, attrs);
436f2ad0
HHZ		 3380}
3381
8313c10f 3382static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs,
c89d1bed
SH		 3383 struct ib_uverbs_create_xsrq *cmd,
3384 struct ib_udata *udata)
f520ba5a 3385{
f520ba5a 3386 struct ib_uverbs_create_srq_resp resp;
8541f8de 3387 struct ib_usrq_object *obj;
f520ba5a
RD		 3388 struct ib_pd *pd;
3389 struct ib_srq *srq;
8541f8de 3390 struct ib_uobject *uninitialized_var(xrcd_uobj);
f520ba5a
RD		 3391 struct ib_srq_init_attr attr;
3392 int ret;
bbd51e88 3393 struct ib_device *ib_dev;
f520ba5a 3394
8313c10f 3395 obj = (struct ib_usrq_object *)uobj_alloc(UVERBS_OBJECT_SRQ, attrs,
bbd51e88 3396 &ib_dev);
fd3c7904
MB		 3397 if (IS_ERR(obj))
3398 return PTR_ERR(obj);
f520ba5a 3399
38eb44fa
AK		 3400 if (cmd->srq_type == IB_SRQT_TM)
3401 attr.ext.tag_matching.max_num_tags = cmd->max_num_tags;
3402
8541f8de 3403 if (cmd->srq_type == IB_SRQT_XRC) {
1f7ff9d5 3404 xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd->xrcd_handle,
8313c10f 3405 attrs);
fd3c7904 3406 if (IS_ERR(xrcd_uobj)) {
8541f8de 3407 ret = -EINVAL;
5909ce54 3408 goto err;
8541f8de
SH		 3409 }
3410
fd3c7904
MB		 3411 attr.ext.xrc.xrcd = (struct ib_xrcd *)xrcd_uobj->object;
3412 if (!attr.ext.xrc.xrcd) {
3413 ret = -EINVAL;
3414 goto err_put_xrcd;
3415 }
3416
8541f8de
SH		 3417 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);
3418 atomic_inc(&obj->uxrcd->refcnt);
1a56ff6d 3419 }
5909ce54 3420
1a56ff6d 3421 if (ib_srq_has_cq(cmd->srq_type)) {
2cc1e3b8 3422 attr.ext.cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ,
8313c10f 3423 cmd->cq_handle, attrs);
1a56ff6d 3424 if (!attr.ext.cq) {
5909ce54
RD		 3425 ret = -EINVAL;
3426 goto err_put_xrcd;
3427 }
3428 }
3429
8313c10f 3430 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle, attrs);
5909ce54
RD		 3431 if (!pd) {
3432 ret = -EINVAL;
3433 goto err_put_cq;
8541f8de
SH		 3434 }
3435
f520ba5a 3436 attr.event_handler = ib_uverbs_srq_event_handler;
8313c10f 3437 attr.srq_context = attrs->ufile;
8541f8de
SH		 3438 attr.srq_type = cmd->srq_type;
3439 attr.attr.max_wr = cmd->max_wr;
3440 attr.attr.max_sge = cmd->max_sge;
3441 attr.attr.srq_limit = cmd->srq_limit;
f520ba5a 3442
8541f8de
SH		 3443 obj->uevent.events_reported = 0;
3444 INIT_LIST_HEAD(&obj->uevent.event_list);
f520ba5a 3445
3023a1e9 3446 srq = pd->device->ops.create_srq(pd, &attr, udata);
f520ba5a
RD		 3447 if (IS_ERR(srq)) {
3448 ret = PTR_ERR(srq);
ec924b47 3449 goto err_put;
f520ba5a
RD		 3450 }
3451
8541f8de
SH		 3452 srq->device = pd->device;
3453 srq->pd = pd;
3454 srq->srq_type = cmd->srq_type;
3455 srq->uobject = &obj->uevent.uobject;
f520ba5a
RD		 3456 srq->event_handler = attr.event_handler;
3457 srq->srq_context = attr.srq_context;
8541f8de 3458
1a56ff6d
AK		 3459 if (ib_srq_has_cq(cmd->srq_type)) {
3460 srq->ext.cq = attr.ext.cq;
3461 atomic_inc(&attr.ext.cq->usecnt);
3462 }
3463
8541f8de 3464 if (cmd->srq_type == IB_SRQT_XRC) {
8541f8de 3465 srq->ext.xrc.xrcd = attr.ext.xrc.xrcd;
8541f8de
SH		 3466 atomic_inc(&attr.ext.xrc.xrcd->usecnt);
3467 }
3468
f520ba5a
RD		 3469 atomic_inc(&pd->usecnt);
3470 atomic_set(&srq->usecnt, 0);
3471
8541f8de 3472 obj->uevent.uobject.object = srq;
fd3c7904 3473 obj->uevent.uobject.user_handle = cmd->user_handle;
f520ba5a 3474
9ead190b 3475 memset(&resp, 0, sizeof resp);
8541f8de 3476 resp.srq_handle = obj->uevent.uobject.id;
ea88fd16
DB		 3477 resp.max_wr = attr.attr.max_wr;
3478 resp.max_sge = attr.attr.max_sge;
8541f8de
SH		 3479 if (cmd->srq_type == IB_SRQT_XRC)
3480 resp.srqn = srq->ext.xrc.srq_num;
f520ba5a 3481
9a073857
JG		 3482 ret = uverbs_response(attrs, &resp, sizeof(resp));
3483 if (ret)
9ead190b 3484 goto err_copy;
f520ba5a 3485
1a56ff6d 3486 if (cmd->srq_type == IB_SRQT_XRC)
fd3c7904 3487 uobj_put_read(xrcd_uobj);
1a56ff6d
AK		 3488
3489 if (ib_srq_has_cq(cmd->srq_type))
3490 uobj_put_obj_read(attr.ext.cq);
3491
fd3c7904 3492 uobj_put_obj_read(pd);
7106a976 3493 return uobj_alloc_commit(&obj->uevent.uobject);
f520ba5a 3494
9ead190b 3495err_copy:
f520ba5a
RD		 3496 ib_destroy_srq(srq);
3497
ec924b47 3498err_put:
fd3c7904 3499 uobj_put_obj_read(pd);
8541f8de
SH		 3500
3501err_put_cq:
1a56ff6d
AK		 3502 if (ib_srq_has_cq(cmd->srq_type))
3503 uobj_put_obj_read(attr.ext.cq);
8541f8de 3504
5909ce54
RD		 3505err_put_xrcd:
3506 if (cmd->srq_type == IB_SRQT_XRC) {
3507 atomic_dec(&obj->uxrcd->refcnt);
fd3c7904 3508 uobj_put_read(xrcd_uobj);
5909ce54 3509 }
ec924b47 3510
9ead190b 3511err:
fd3c7904 3512 uobj_alloc_abort(&obj->uevent.uobject);
f520ba5a
RD		 3513 return ret;
3514}
3515
974d6b4b 3516static int ib_uverbs_create_srq(struct uverbs_attr_bundle *attrs)
8541f8de
SH		 3517{
3518 struct ib_uverbs_create_srq cmd;
3519 struct ib_uverbs_create_xsrq xcmd;
3c2c2094 3520 int ret;
8541f8de 3521
3c2c2094
JG		 3522 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3523 if (ret)
3524 return ret;
8541f8de 3525
38eb44fa 3526 memset(&xcmd, 0, sizeof(xcmd));
8541f8de
SH		 3527 xcmd.response = cmd.response;
3528 xcmd.user_handle = cmd.user_handle;
3529 xcmd.srq_type = IB_SRQT_BASIC;
3530 xcmd.pd_handle = cmd.pd_handle;
3531 xcmd.max_wr = cmd.max_wr;
3532 xcmd.max_sge = cmd.max_sge;
3533 xcmd.srq_limit = cmd.srq_limit;
3534
ef87df2c 3535 return __uverbs_create_xsrq(attrs, &xcmd, &attrs->driver_udata);
8541f8de
SH		 3536}
3537
974d6b4b 3538static int ib_uverbs_create_xsrq(struct uverbs_attr_bundle *attrs)
8541f8de
SH		 3539{
3540 struct ib_uverbs_create_xsrq cmd;
3c2c2094 3541 int ret;
8541f8de 3542
3c2c2094
JG		 3543 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3544 if (ret)
3545 return ret;
8541f8de 3546
ef87df2c 3547 return __uverbs_create_xsrq(attrs, &cmd, &attrs->driver_udata);
8541f8de
SH		 3548}
3549
974d6b4b 3550static int ib_uverbs_modify_srq(struct uverbs_attr_bundle *attrs)
f520ba5a
RD		 3551{
3552 struct ib_uverbs_modify_srq cmd;
3553 struct ib_srq *srq;
3554 struct ib_srq_attr attr;
3555 int ret;
3556
3c2c2094
JG		 3557 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3558 if (ret)
3559 return ret;
f520ba5a 3560
8313c10f 3561 srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);
9ead190b
RD		 3562 if (!srq)
3563 return -EINVAL;
f520ba5a
RD		 3564
3565 attr.max_wr = cmd.max_wr;
f520ba5a
RD		 3566 attr.srq_limit = cmd.srq_limit;
3567
3023a1e9
KH		 3568 ret = srq->device->ops.modify_srq(srq, &attr, cmd.attr_mask,
3569 &attrs->driver_udata);
f520ba5a 3570
fd3c7904 3571 uobj_put_obj_read(srq);
f520ba5a 3572
7106a976 3573 return ret;
f520ba5a
RD		 3574}
3575
974d6b4b 3576static int ib_uverbs_query_srq(struct uverbs_attr_bundle *attrs)
8bdb0e86
DB		 3577{
3578 struct ib_uverbs_query_srq cmd;
3579 struct ib_uverbs_query_srq_resp resp;
3580 struct ib_srq_attr attr;
3581 struct ib_srq *srq;
3582 int ret;
3583
3c2c2094
JG		 3584 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3585 if (ret)
3586 return ret;
8bdb0e86 3587
8313c10f 3588 srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);
9ead190b
RD		 3589 if (!srq)
3590 return -EINVAL;
8bdb0e86 3591
9ead190b 3592 ret = ib_query_srq(srq, &attr);
8bdb0e86 3593
fd3c7904 3594 uobj_put_obj_read(srq);
8bdb0e86
DB		 3595
3596 if (ret)
9ead190b 3597 return ret;
8bdb0e86
DB		 3598
3599 memset(&resp, 0, sizeof resp);
3600
3601 resp.max_wr = attr.max_wr;
3602 resp.max_sge = attr.max_sge;
3603 resp.srq_limit = attr.srq_limit;
3604
9a073857 3605 return uverbs_response(attrs, &resp, sizeof(resp));
8bdb0e86
DB		 3606}
3607
974d6b4b 3608static int ib_uverbs_destroy_srq(struct uverbs_attr_bundle *attrs)
f520ba5a 3609{
63aaf647
RD		 3610 struct ib_uverbs_destroy_srq cmd;
3611 struct ib_uverbs_destroy_srq_resp resp;
9ead190b 3612 struct ib_uobject *uobj;
9ead190b 3613 struct ib_uevent_object *obj;
3c2c2094 3614 int ret;
f520ba5a 3615
3c2c2094
JG		 3616 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3617 if (ret)
3618 return ret;
f520ba5a 3619
8313c10f 3620 uobj = uobj_get_destroy(UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);
fd3c7904
MB		 3621 if (IS_ERR(uobj))
3622 return PTR_ERR(uobj);
3623
9ead190b 3624 obj = container_of(uobj, struct ib_uevent_object, uobject);
fd3c7904 3625 memset(&resp, 0, sizeof(resp));
9ead190b 3626 resp.events_reported = obj->events_reported;
32ed5c00
JG		 3627
3628 uobj_put_destroy(uobj);
3629
9a073857 3630 return uverbs_response(attrs, &resp, sizeof(resp));
f520ba5a 3631}
02d1aa7a 3632
974d6b4b 3633static int ib_uverbs_ex_query_device(struct uverbs_attr_bundle *attrs)
02d1aa7a 3634{
7eebced1 3635 struct ib_uverbs_ex_query_device_resp resp = {};
02d1aa7a 3636 struct ib_uverbs_ex_query_device cmd;
2953f425 3637 struct ib_device_attr attr = {0};
bbd51e88
JG		 3638 struct ib_ucontext *ucontext;
3639 struct ib_device *ib_dev;
02d1aa7a
EC		 3640 int err;
3641
8313c10f 3642 ucontext = ib_uverbs_get_ucontext(attrs);
bbd51e88
JG		 3643 if (IS_ERR(ucontext))
3644 return PTR_ERR(ucontext);
3645 ib_dev = ucontext->device;
3646
7eebced1 3647 err = uverbs_request(attrs, &cmd, sizeof(cmd));
02d1aa7a
EC		 3648 if (err)
3649 return err;
3650
3651 if (cmd.comp_mask)
3652 return -EINVAL;
3653
3654 if (cmd.reserved)
3655 return -EINVAL;
3656
3023a1e9 3657 err = ib_dev->ops.query_device(ib_dev, &attr, &attrs->driver_udata);
02d1aa7a
EC		 3658 if (err)
3659 return err;
3660
bbd51e88 3661 copy_query_dev_fields(ucontext, &resp.base, &attr);
02d1aa7a 3662
f4056bfd
HE		 3663#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
3664 resp.odp_caps.general_caps = attr.odp_caps.general_caps;
3665 resp.odp_caps.per_transport_caps.rc_odp_caps =
3666 attr.odp_caps.per_transport_caps.rc_odp_caps;
3667 resp.odp_caps.per_transport_caps.uc_odp_caps =
3668 attr.odp_caps.per_transport_caps.uc_odp_caps;
3669 resp.odp_caps.per_transport_caps.ud_odp_caps =
3670 attr.odp_caps.per_transport_caps.ud_odp_caps;
f4056bfd 3671#endif
24306dc6
MB		 3672
3673 resp.timestamp_mask = attr.timestamp_mask;
24306dc6 3674 resp.hca_core_clock = attr.hca_core_clock;
0b24e5ac 3675 resp.device_cap_flags_ex = attr.device_cap_flags;
47adf2f4
YH		 3676 resp.rss_caps.supported_qpts = attr.rss_caps.supported_qpts;
3677 resp.rss_caps.max_rwq_indirection_tables =
3678 attr.rss_caps.max_rwq_indirection_tables;
3679 resp.rss_caps.max_rwq_indirection_table_size =
3680 attr.rss_caps.max_rwq_indirection_table_size;
47adf2f4 3681 resp.max_wq_type_rq = attr.max_wq_type_rq;
5f23d426 3682 resp.raw_packet_caps = attr.raw_packet_caps;
78b1beb0
LR		 3683 resp.tm_caps.max_rndv_hdr_size = attr.tm_caps.max_rndv_hdr_size;
3684 resp.tm_caps.max_num_tags = attr.tm_caps.max_num_tags;
3685 resp.tm_caps.max_ops = attr.tm_caps.max_ops;
3686 resp.tm_caps.max_sge = attr.tm_caps.max_sge;
3687 resp.tm_caps.flags = attr.tm_caps.flags;
18bd9072
YC		 3688 resp.cq_moderation_caps.max_cq_moderation_count =
3689 attr.cq_caps.max_cq_moderation_count;
3690 resp.cq_moderation_caps.max_cq_moderation_period =
3691 attr.cq_caps.max_cq_moderation_period;
1d8eeb9f 3692 resp.max_dm_size = attr.max_dm_size;
7eebced1
JG		 3693 resp.response_length = uverbs_response_length(attrs, sizeof(resp));
3694
9a073857 3695 return uverbs_response(attrs, &resp, sizeof(resp));
02d1aa7a 3696}
869ddcf8 3697
974d6b4b 3698static int ib_uverbs_ex_modify_cq(struct uverbs_attr_bundle *attrs)
869ddcf8 3699{
29a29d18 3700 struct ib_uverbs_ex_modify_cq cmd;
869ddcf8 3701 struct ib_cq *cq;
869ddcf8
YC		 3702 int ret;
3703
29a29d18 3704 ret = uverbs_request(attrs, &cmd, sizeof(cmd));
869ddcf8
YC		 3705 if (ret)
3706 return ret;
3707
3708 if (!cmd.attr_mask || cmd.reserved)
3709 return -EINVAL;
3710
3711 if (cmd.attr_mask > IB_CQ_MODERATE)
3712 return -EOPNOTSUPP;
3713
8313c10f 3714 cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
869ddcf8
YC		 3715 if (!cq)
3716 return -EINVAL;
3717
4190b4e9 3718 ret = rdma_set_cq_moderation(cq, cmd.attr.cq_count, cmd.attr.cq_period);
869ddcf8
YC		 3719
3720 uobj_put_obj_read(cq);
3721
3722 return ret;
3723}
d120c3c9 3724
669dac1e
JG		 3725/*
3726 * Describe the input structs for write(). Some write methods have an input
3727 * only struct, most have an input and output. If the struct has an output then
3728 * the 'response' u64 must be the first field in the request structure.
3729 *
3730 * If udata is present then both the request and response structs have a
3731 * trailing driver_data flex array. In this case the size of the base struct
3732 * cannot be changed.
3733 */
3734#define offsetof_after(_struct, _member) \
3735 (offsetof(_struct, _member) + sizeof(((_struct *)NULL)->_member))
3736
3737#define UAPI_DEF_WRITE_IO(req, resp) \
3738 .write.has_resp = 1 + \
3739 BUILD_BUG_ON_ZERO(offsetof(req, response) != 0) + \
3740 BUILD_BUG_ON_ZERO(sizeof(((req *)0)->response) != \
3741 sizeof(u64)), \
3742 .write.req_size = sizeof(req), .write.resp_size = sizeof(resp)
3743
3744#define UAPI_DEF_WRITE_I(req) .write.req_size = sizeof(req)
3745
3746#define UAPI_DEF_WRITE_UDATA_IO(req, resp) \
3747 UAPI_DEF_WRITE_IO(req, resp), \
3748 .write.has_udata = \
3749 1 + \
3750 BUILD_BUG_ON_ZERO(offsetof(req, driver_data) != \
3751 sizeof(req)) + \
3752 BUILD_BUG_ON_ZERO(offsetof(resp, driver_data) != \
3753 sizeof(resp))
3754
3755#define UAPI_DEF_WRITE_UDATA_I(req) \
3756 UAPI_DEF_WRITE_I(req), \
3757 .write.has_udata = \
3758 1 + BUILD_BUG_ON_ZERO(offsetof(req, driver_data) != \
3759 sizeof(req))
3760
3761/*
3762 * The _EX versions are for use with WRITE_EX and allow the last struct member
3763 * to be specified. Buffers that do not include that member will be rejected.
3764 */
3765#define UAPI_DEF_WRITE_IO_EX(req, req_last_member, resp, resp_last_member) \
3766 .write.has_resp = 1, \
3767 .write.req_size = offsetof_after(req, req_last_member), \
3768 .write.resp_size = offsetof_after(resp, resp_last_member)
3769
3770#define UAPI_DEF_WRITE_I_EX(req, req_last_member) \
3771 .write.req_size = offsetof_after(req, req_last_member)
3772
d120c3c9 3773const struct uapi_definition uverbs_def_write_intf[] = {
a140692a
JG		 3774 DECLARE_UVERBS_OBJECT(
3775 UVERBS_OBJECT_AH,
3776 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_AH,
3777 ib_uverbs_create_ah,
669dac1e
JG		 3778 UAPI_DEF_WRITE_UDATA_IO(
3779 struct ib_uverbs_create_ah,
3780 struct ib_uverbs_create_ah_resp),
a140692a 3781 UAPI_DEF_METHOD_NEEDS_FN(create_ah)),
669dac1e
JG		 3782 DECLARE_UVERBS_WRITE(
3783 IB_USER_VERBS_CMD_DESTROY_AH,
3784 ib_uverbs_destroy_ah,
3785 UAPI_DEF_WRITE_I(struct ib_uverbs_destroy_ah),
3786 UAPI_DEF_METHOD_NEEDS_FN(destroy_ah))),
d120c3c9
JG		 3787
3788 DECLARE_UVERBS_OBJECT(
3789 UVERBS_OBJECT_COMP_CHANNEL,
669dac1e
JG		 3790 DECLARE_UVERBS_WRITE(
3791 IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL,
3792 ib_uverbs_create_comp_channel,
3793 UAPI_DEF_WRITE_IO(
3794 struct ib_uverbs_create_comp_channel,
3795 struct ib_uverbs_create_comp_channel_resp))),
d120c3c9
JG		 3796
3797 DECLARE_UVERBS_OBJECT(
3798 UVERBS_OBJECT_CQ,
3799 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_CQ,
a140692a 3800 ib_uverbs_create_cq,
669dac1e
JG		 3801 UAPI_DEF_WRITE_UDATA_IO(
3802 struct ib_uverbs_create_cq,
3803 struct ib_uverbs_create_cq_resp),
a140692a 3804 UAPI_DEF_METHOD_NEEDS_FN(create_cq)),
669dac1e
JG		 3805 DECLARE_UVERBS_WRITE(
3806 IB_USER_VERBS_CMD_DESTROY_CQ,
3807 ib_uverbs_destroy_cq,
3808 UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_cq,
3809 struct ib_uverbs_destroy_cq_resp),
3810 UAPI_DEF_METHOD_NEEDS_FN(destroy_cq)),
3811 DECLARE_UVERBS_WRITE(
3812 IB_USER_VERBS_CMD_POLL_CQ,
3813 ib_uverbs_poll_cq,
3814 UAPI_DEF_WRITE_IO(struct ib_uverbs_poll_cq,
3815 struct ib_uverbs_poll_cq_resp),
3816 UAPI_DEF_METHOD_NEEDS_FN(poll_cq)),
3817 DECLARE_UVERBS_WRITE(
3818 IB_USER_VERBS_CMD_REQ_NOTIFY_CQ,
3819 ib_uverbs_req_notify_cq,
3820 UAPI_DEF_WRITE_I(struct ib_uverbs_req_notify_cq),
3821 UAPI_DEF_METHOD_NEEDS_FN(req_notify_cq)),
d120c3c9 3822 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_RESIZE_CQ,
a140692a 3823 ib_uverbs_resize_cq,
669dac1e
JG		 3824 UAPI_DEF_WRITE_UDATA_IO(
3825 struct ib_uverbs_resize_cq,
3826 struct ib_uverbs_resize_cq_resp),
a140692a 3827 UAPI_DEF_METHOD_NEEDS_FN(resize_cq)),
669dac1e
JG		 3828 DECLARE_UVERBS_WRITE_EX(
3829 IB_USER_VERBS_EX_CMD_CREATE_CQ,
3830 ib_uverbs_ex_create_cq,
3831 UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_cq,
3832 reserved,
3833 struct ib_uverbs_ex_create_cq_resp,
3834 response_length),
3835 UAPI_DEF_METHOD_NEEDS_FN(create_cq)),
3836 DECLARE_UVERBS_WRITE_EX(
3837 IB_USER_VERBS_EX_CMD_MODIFY_CQ,
3838 ib_uverbs_ex_modify_cq,
3839 UAPI_DEF_WRITE_I(struct ib_uverbs_ex_modify_cq),
3840 UAPI_DEF_METHOD_NEEDS_FN(create_cq))),
d120c3c9
JG		 3841
3842 DECLARE_UVERBS_OBJECT(
3843 UVERBS_OBJECT_DEVICE,
3844 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_GET_CONTEXT,
669dac1e
JG		 3845 ib_uverbs_get_context,
3846 UAPI_DEF_WRITE_UDATA_IO(
3847 struct ib_uverbs_get_context,
3848 struct ib_uverbs_get_context_resp)),
3849 DECLARE_UVERBS_WRITE(
3850 IB_USER_VERBS_CMD_QUERY_DEVICE,
3851 ib_uverbs_query_device,
3852 UAPI_DEF_WRITE_IO(struct ib_uverbs_query_device,
3853 struct ib_uverbs_query_device_resp)),
3854 DECLARE_UVERBS_WRITE(
3855 IB_USER_VERBS_CMD_QUERY_PORT,
3856 ib_uverbs_query_port,
3857 UAPI_DEF_WRITE_IO(struct ib_uverbs_query_port,
3858 struct ib_uverbs_query_port_resp),
3859 UAPI_DEF_METHOD_NEEDS_FN(query_port)),
3860 DECLARE_UVERBS_WRITE_EX(
3861 IB_USER_VERBS_EX_CMD_QUERY_DEVICE,
3862 ib_uverbs_ex_query_device,
3863 UAPI_DEF_WRITE_IO_EX(
3864 struct ib_uverbs_ex_query_device,
3865 reserved,
3866 struct ib_uverbs_ex_query_device_resp,
3867 response_length),
3868 UAPI_DEF_METHOD_NEEDS_FN(query_device)),
a140692a
JG		 3869 UAPI_DEF_OBJ_NEEDS_FN(alloc_ucontext),
3870 UAPI_DEF_OBJ_NEEDS_FN(dealloc_ucontext)),
d120c3c9
JG		 3871
3872 DECLARE_UVERBS_OBJECT(
3873 UVERBS_OBJECT_FLOW,
669dac1e
JG		 3874 DECLARE_UVERBS_WRITE_EX(
3875 IB_USER_VERBS_EX_CMD_CREATE_FLOW,
3876 ib_uverbs_ex_create_flow,
3877 UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_create_flow,
3878 flow_attr,
3879 struct ib_uverbs_create_flow_resp,
3880 flow_handle),
3881 UAPI_DEF_METHOD_NEEDS_FN(create_flow)),
3882 DECLARE_UVERBS_WRITE_EX(
3883 IB_USER_VERBS_EX_CMD_DESTROY_FLOW,
3884 ib_uverbs_ex_destroy_flow,
3885 UAPI_DEF_WRITE_I(struct ib_uverbs_destroy_flow),
3886 UAPI_DEF_METHOD_NEEDS_FN(destroy_flow))),
a140692a
JG		 3887
3888 DECLARE_UVERBS_OBJECT(
3889 UVERBS_OBJECT_MR,
3890 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_DEREG_MR,
3891 ib_uverbs_dereg_mr,
669dac1e 3892 UAPI_DEF_WRITE_I(struct ib_uverbs_dereg_mr),
a140692a 3893 UAPI_DEF_METHOD_NEEDS_FN(dereg_mr)),
669dac1e
JG		 3894 DECLARE_UVERBS_WRITE(
3895 IB_USER_VERBS_CMD_REG_MR,
3896 ib_uverbs_reg_mr,
3897 UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_reg_mr,
3898 struct ib_uverbs_reg_mr_resp),
3899 UAPI_DEF_METHOD_NEEDS_FN(reg_user_mr)),
3900 DECLARE_UVERBS_WRITE(
3901 IB_USER_VERBS_CMD_REREG_MR,
3902 ib_uverbs_rereg_mr,
3903 UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_rereg_mr,
3904 struct ib_uverbs_rereg_mr_resp),
3905 UAPI_DEF_METHOD_NEEDS_FN(rereg_user_mr))),
a140692a
JG		 3906
3907 DECLARE_UVERBS_OBJECT(
3908 UVERBS_OBJECT_MW,
669dac1e
JG		 3909 DECLARE_UVERBS_WRITE(
3910 IB_USER_VERBS_CMD_ALLOC_MW,
3911 ib_uverbs_alloc_mw,
3912 UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_alloc_mw,
3913 struct ib_uverbs_alloc_mw_resp),
3914 UAPI_DEF_METHOD_NEEDS_FN(alloc_mw)),
3915 DECLARE_UVERBS_WRITE(
3916 IB_USER_VERBS_CMD_DEALLOC_MW,
3917 ib_uverbs_dealloc_mw,
3918 UAPI_DEF_WRITE_I(struct ib_uverbs_dealloc_mw),
3919 UAPI_DEF_METHOD_NEEDS_FN(dealloc_mw))),
a140692a
JG		 3920
3921 DECLARE_UVERBS_OBJECT(
3922 UVERBS_OBJECT_PD,
669dac1e
JG		 3923 DECLARE_UVERBS_WRITE(
3924 IB_USER_VERBS_CMD_ALLOC_PD,
3925 ib_uverbs_alloc_pd,
3926 UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_alloc_pd,
3927 struct ib_uverbs_alloc_pd_resp),
3928 UAPI_DEF_METHOD_NEEDS_FN(alloc_pd)),
3929 DECLARE_UVERBS_WRITE(
3930 IB_USER_VERBS_CMD_DEALLOC_PD,
3931 ib_uverbs_dealloc_pd,
3932 UAPI_DEF_WRITE_I(struct ib_uverbs_dealloc_pd),
3933 UAPI_DEF_METHOD_NEEDS_FN(dealloc_pd))),
d120c3c9
JG		 3934
3935 DECLARE_UVERBS_OBJECT(
3936 UVERBS_OBJECT_QP,
669dac1e
JG		 3937 DECLARE_UVERBS_WRITE(
3938 IB_USER_VERBS_CMD_ATTACH_MCAST,
3939 ib_uverbs_attach_mcast,
3940 UAPI_DEF_WRITE_I(struct ib_uverbs_attach_mcast),
3941 UAPI_DEF_METHOD_NEEDS_FN(attach_mcast),
3942 UAPI_DEF_METHOD_NEEDS_FN(detach_mcast)),
d120c3c9 3943 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_QP,
a140692a 3944 ib_uverbs_create_qp,
669dac1e
JG		 3945 UAPI_DEF_WRITE_UDATA_IO(
3946 struct ib_uverbs_create_qp,
3947 struct ib_uverbs_create_qp_resp),
a140692a 3948 UAPI_DEF_METHOD_NEEDS_FN(create_qp)),
669dac1e
JG		 3949 DECLARE_UVERBS_WRITE(
3950 IB_USER_VERBS_CMD_DESTROY_QP,
3951 ib_uverbs_destroy_qp,
3952 UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_qp,
3953 struct ib_uverbs_destroy_qp_resp),
3954 UAPI_DEF_METHOD_NEEDS_FN(destroy_qp)),
3955 DECLARE_UVERBS_WRITE(
3956 IB_USER_VERBS_CMD_DETACH_MCAST,
3957 ib_uverbs_detach_mcast,
3958 UAPI_DEF_WRITE_I(struct ib_uverbs_detach_mcast),
3959 UAPI_DEF_METHOD_NEEDS_FN(detach_mcast)),
3960 DECLARE_UVERBS_WRITE(
3961 IB_USER_VERBS_CMD_MODIFY_QP,
3962 ib_uverbs_modify_qp,
3963 UAPI_DEF_WRITE_I(struct ib_uverbs_modify_qp),
3964 UAPI_DEF_METHOD_NEEDS_FN(modify_qp)),
3965 DECLARE_UVERBS_WRITE(
3966 IB_USER_VERBS_CMD_POST_RECV,
3967 ib_uverbs_post_recv,
3968 UAPI_DEF_WRITE_IO(struct ib_uverbs_post_recv,
3969 struct ib_uverbs_post_recv_resp),
3970 UAPI_DEF_METHOD_NEEDS_FN(post_recv)),
3971 DECLARE_UVERBS_WRITE(
3972 IB_USER_VERBS_CMD_POST_SEND,
3973 ib_uverbs_post_send,
3974 UAPI_DEF_WRITE_IO(struct ib_uverbs_post_send,
3975 struct ib_uverbs_post_send_resp),
3976 UAPI_DEF_METHOD_NEEDS_FN(post_send)),
3977 DECLARE_UVERBS_WRITE(
3978 IB_USER_VERBS_CMD_QUERY_QP,
3979 ib_uverbs_query_qp,
3980 UAPI_DEF_WRITE_IO(struct ib_uverbs_query_qp,
3981 struct ib_uverbs_query_qp_resp),
3982 UAPI_DEF_METHOD_NEEDS_FN(query_qp)),
3983 DECLARE_UVERBS_WRITE_EX(
3984 IB_USER_VERBS_EX_CMD_CREATE_QP,
3985 ib_uverbs_ex_create_qp,
3986 UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_qp,
3987 comp_mask,
3988 struct ib_uverbs_ex_create_qp_resp,
3989 response_length),
3990 UAPI_DEF_METHOD_NEEDS_FN(create_qp)),
3991 DECLARE_UVERBS_WRITE_EX(
3992 IB_USER_VERBS_EX_CMD_MODIFY_QP,
3993 ib_uverbs_ex_modify_qp,
3994 UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_modify_qp,
3995 base,
3996 struct ib_uverbs_ex_modify_qp_resp,
3997 response_length),
3998 UAPI_DEF_METHOD_NEEDS_FN(modify_qp))),
d120c3c9
JG		 3999
4000 DECLARE_UVERBS_OBJECT(
4001 UVERBS_OBJECT_RWQ_IND_TBL,
a140692a
JG		 4002 DECLARE_UVERBS_WRITE_EX(
4003 IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL,
4004 ib_uverbs_ex_create_rwq_ind_table,
669dac1e
JG		 4005 UAPI_DEF_WRITE_IO_EX(
4006 struct ib_uverbs_ex_create_rwq_ind_table,
4007 log_ind_tbl_size,
4008 struct ib_uverbs_ex_create_rwq_ind_table_resp,
4009 ind_tbl_num),
a140692a
JG		 4010 UAPI_DEF_METHOD_NEEDS_FN(create_rwq_ind_table)),
4011 DECLARE_UVERBS_WRITE_EX(
4012 IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL,
4013 ib_uverbs_ex_destroy_rwq_ind_table,
669dac1e
JG		 4014 UAPI_DEF_WRITE_I(
4015 struct ib_uverbs_ex_destroy_rwq_ind_table),
a140692a 4016 UAPI_DEF_METHOD_NEEDS_FN(destroy_rwq_ind_table))),
d120c3c9
JG		 4017
4018 DECLARE_UVERBS_OBJECT(
4019 UVERBS_OBJECT_WQ,
669dac1e
JG		 4020 DECLARE_UVERBS_WRITE_EX(
4021 IB_USER_VERBS_EX_CMD_CREATE_WQ,
4022 ib_uverbs_ex_create_wq,
4023 UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_wq,
4024 max_sge,
4025 struct ib_uverbs_ex_create_wq_resp,
4026 wqn),
4027 UAPI_DEF_METHOD_NEEDS_FN(create_wq)),
4028 DECLARE_UVERBS_WRITE_EX(
4029 IB_USER_VERBS_EX_CMD_DESTROY_WQ,
4030 ib_uverbs_ex_destroy_wq,
4031 UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_destroy_wq,
4032 wq_handle,
4033 struct ib_uverbs_ex_destroy_wq_resp,
4034 reserved),
4035 UAPI_DEF_METHOD_NEEDS_FN(destroy_wq)),
4036 DECLARE_UVERBS_WRITE_EX(
4037 IB_USER_VERBS_EX_CMD_MODIFY_WQ,
4038 ib_uverbs_ex_modify_wq,
4039 UAPI_DEF_WRITE_I_EX(struct ib_uverbs_ex_modify_wq,
4040 curr_wq_state),
4041 UAPI_DEF_METHOD_NEEDS_FN(modify_wq))),
d120c3c9
JG		 4042
4043 DECLARE_UVERBS_OBJECT(
4044 UVERBS_OBJECT_SRQ,
4045 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_SRQ,
a140692a 4046 ib_uverbs_create_srq,
669dac1e
JG		 4047 UAPI_DEF_WRITE_UDATA_IO(
4048 struct ib_uverbs_create_srq,
4049 struct ib_uverbs_create_srq_resp),
a140692a 4050 UAPI_DEF_METHOD_NEEDS_FN(create_srq)),
d120c3c9 4051 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_XSRQ,
a140692a 4052 ib_uverbs_create_xsrq,
669dac1e
JG		 4053 UAPI_DEF_WRITE_UDATA_IO(
4054 struct ib_uverbs_create_xsrq,
4055 struct ib_uverbs_create_srq_resp),
a140692a 4056 UAPI_DEF_METHOD_NEEDS_FN(create_srq)),
669dac1e
JG		 4057 DECLARE_UVERBS_WRITE(
4058 IB_USER_VERBS_CMD_DESTROY_SRQ,
4059 ib_uverbs_destroy_srq,
4060 UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_srq,
4061 struct ib_uverbs_destroy_srq_resp),
4062 UAPI_DEF_METHOD_NEEDS_FN(destroy_srq)),
4063 DECLARE_UVERBS_WRITE(
4064 IB_USER_VERBS_CMD_MODIFY_SRQ,
4065 ib_uverbs_modify_srq,
4066 UAPI_DEF_WRITE_UDATA_I(struct ib_uverbs_modify_srq),
4067 UAPI_DEF_METHOD_NEEDS_FN(modify_srq)),
4068 DECLARE_UVERBS_WRITE(
4069 IB_USER_VERBS_CMD_POST_SRQ_RECV,
4070 ib_uverbs_post_srq_recv,
4071 UAPI_DEF_WRITE_IO(struct ib_uverbs_post_srq_recv,
4072 struct ib_uverbs_post_srq_recv_resp),
4073 UAPI_DEF_METHOD_NEEDS_FN(post_srq_recv)),
4074 DECLARE_UVERBS_WRITE(
4075 IB_USER_VERBS_CMD_QUERY_SRQ,
4076 ib_uverbs_query_srq,
4077 UAPI_DEF_WRITE_IO(struct ib_uverbs_query_srq,
4078 struct ib_uverbs_query_srq_resp),
4079 UAPI_DEF_METHOD_NEEDS_FN(query_srq))),
a140692a
JG		 4080
4081 DECLARE_UVERBS_OBJECT(
4082 UVERBS_OBJECT_XRCD,
669dac1e
JG		 4083 DECLARE_UVERBS_WRITE(
4084 IB_USER_VERBS_CMD_CLOSE_XRCD,
4085 ib_uverbs_close_xrcd,
4086 UAPI_DEF_WRITE_I(struct ib_uverbs_close_xrcd),
4087 UAPI_DEF_METHOD_NEEDS_FN(dealloc_xrcd)),
a140692a 4088 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_OPEN_QP,
669dac1e
JG		 4089 ib_uverbs_open_qp,
4090 UAPI_DEF_WRITE_UDATA_IO(
4091 struct ib_uverbs_open_qp,
4092 struct ib_uverbs_create_qp_resp)),
a140692a
JG		 4093 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_OPEN_XRCD,
4094 ib_uverbs_open_xrcd,
669dac1e
JG		 4095 UAPI_DEF_WRITE_UDATA_IO(
4096 struct ib_uverbs_open_xrcd,
4097 struct ib_uverbs_open_xrcd_resp),
a140692a 4098 UAPI_DEF_METHOD_NEEDS_FN(alloc_xrcd))),
d120c3c9
JG		 4099
4100 {},
4101};
Linux 6.x block layer and io_uring tree(s)