drivers/infiniband/core/uverbs_cmd.c

   1 /*
   2  * Copyright (c) 2005 Topspin Communications.  All rights reserved.
   3  * Copyright (c) 2005, 2006, 2007 Cisco Systems.  All rights reserved.
   4  * Copyright (c) 2005 PathScale, Inc.  All rights reserved.
   5  * Copyright (c) 2006 Mellanox Technologies.  All rights reserved.
   6  *
   7  * This software is available to you under a choice of one of two
   8  * licenses.  You may choose to be licensed under the terms of the GNU
   9  * General Public License (GPL) Version 2, available from the file
  10  * COPYING in the main directory of this source tree, or the
  11  * OpenIB.org BSD license below:
  12  *
  13  *     Redistribution and use in source and binary forms, with or
  14  *     without modification, are permitted provided that the following
  15  *     conditions are met:
  16  *
  17  *      - Redistributions of source code must retain the above
  18  *        copyright notice, this list of conditions and the following
  19  *        disclaimer.
  20  *
  21  *      - Redistributions in binary form must reproduce the above
  22  *        copyright notice, this list of conditions and the following
  23  *        disclaimer in the documentation and/or other materials
  24  *        provided with the distribution.
  25  *
  26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  33  * SOFTWARE.
  34  */
  35
  36 #include <linux/file.h>
  37 #include <linux/fs.h>
  38 #include <linux/slab.h>
  39 #include <linux/sched.h>
  40
  41 #include <linux/uaccess.h>
  42
  43 #include <rdma/uverbs_types.h>
  44 #include <rdma/uverbs_std_types.h>
  45 #include "rdma_core.h"
  46
  47 #include "uverbs.h"
  48 #include "core_priv.h"
  49
  50 static struct ib_uverbs_completion_event_file *
  51 ib_uverbs_lookup_comp_file(int fd, struct ib_ucontext *context)
  52 {
  53         struct ib_uobject *uobj = uobj_get_read(uobj_get_type(comp_channel),
  54                                                 fd, context);
  55         struct ib_uobject_file *uobj_file;
  56
  57         if (IS_ERR(uobj))
  58                 return (void *)uobj;
  59
  60         uverbs_uobject_get(uobj);
  61         uobj_put_read(uobj);
  62
  63         uobj_file = container_of(uobj, struct ib_uobject_file, uobj);
  64         return container_of(uobj_file, struct ib_uverbs_completion_event_file,
  65                             uobj_file);
  66 }
  67
  68 ssize_t ib_uverbs_get_context(struct ib_uverbs_file *file,
  69                               struct ib_device *ib_dev,
  70                               const char __user *buf,
  71                               int in_len, int out_len)
  72 {
  73         struct ib_uverbs_get_context      cmd;
  74         struct ib_uverbs_get_context_resp resp;
  75         struct ib_udata                   udata;
  76         struct ib_ucontext               *ucontext;
  77         struct file                      *filp;
  78         struct ib_rdmacg_object          cg_obj;
  79         int ret;
  80
  81         if (out_len < sizeof resp)
  82                 return -ENOSPC;
  83
  84         if (copy_from_user(&cmd, buf, sizeof cmd))
  85                 return -EFAULT;
  86
  87         mutex_lock(&file->mutex);
  88
  89         if (file->ucontext) {
  90                 ret = -EINVAL;
  91                 goto err;
  92         }
  93
  94         ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
  95                    u64_to_user_ptr(cmd.response) + sizeof(resp),
  96                    in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
  97                    out_len - sizeof(resp));
  98
  99         ret = ib_rdmacg_try_charge(&cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE);
 100         if (ret)
 101                 goto err;
 102
 103         ucontext = ib_dev->alloc_ucontext(ib_dev, &udata);
 104         if (IS_ERR(ucontext)) {
 105                 ret = PTR_ERR(ucontext);
 106                 goto err_alloc;
 107         }
 108
 109         ucontext->device = ib_dev;
 110         ucontext->cg_obj = cg_obj;
 111         /* ufile is required when some objects are released */
 112         ucontext->ufile = file;
 113         uverbs_initialize_ucontext(ucontext);
 114
 115         rcu_read_lock();
 116         ucontext->tgid = get_task_pid(current->group_leader, PIDTYPE_PID);
 117         rcu_read_unlock();
 118         ucontext->closing = 0;
 119
 120 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
 121         ucontext->umem_tree = RB_ROOT_CACHED;
 122         init_rwsem(&ucontext->umem_rwsem);
 123         ucontext->odp_mrs_count = 0;
 124         INIT_LIST_HEAD(&ucontext->no_private_counters);
 125
 126         if (!(ib_dev->attrs.device_cap_flags & IB_DEVICE_ON_DEMAND_PAGING))
 127                 ucontext->invalidate_range = NULL;
 128
 129 #endif
 130
 131         resp.num_comp_vectors = file->device->num_comp_vectors;
 132
 133         ret = get_unused_fd_flags(O_CLOEXEC);
 134         if (ret < 0)
 135                 goto err_free;
 136         resp.async_fd = ret;
 137
 138         filp = ib_uverbs_alloc_async_event_file(file, ib_dev);
 139         if (IS_ERR(filp)) {
 140                 ret = PTR_ERR(filp);
 141                 goto err_fd;
 142         }
 143
 144         if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) {
 145                 ret = -EFAULT;
 146                 goto err_file;
 147         }
 148
 149         file->ucontext = ucontext;
 150
 151         fd_install(resp.async_fd, filp);
 152
 153         mutex_unlock(&file->mutex);
 154
 155         return in_len;
 156
 157 err_file:
 158         ib_uverbs_free_async_event_file(file);
 159         fput(filp);
 160
 161 err_fd:
 162         put_unused_fd(resp.async_fd);
 163
 164 err_free:
 165         put_pid(ucontext->tgid);
 166         ib_dev->dealloc_ucontext(ucontext);
 167
 168 err_alloc:
 169         ib_rdmacg_uncharge(&cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE);
 170
 171 err:
 172         mutex_unlock(&file->mutex);
 173         return ret;
 174 }
 175
 176 static void copy_query_dev_fields(struct ib_uverbs_file *file,
 177                                   struct ib_device *ib_dev,
 178                                   struct ib_uverbs_query_device_resp *resp,
 179                                   struct ib_device_attr *attr)
 180 {
 181         resp->fw_ver            = attr->fw_ver;
 182         resp->node_guid         = ib_dev->node_guid;
 183         resp->sys_image_guid    = attr->sys_image_guid;
 184         resp->max_mr_size       = attr->max_mr_size;
 185         resp->page_size_cap     = attr->page_size_cap;
 186         resp->vendor_id         = attr->vendor_id;
 187         resp->vendor_part_id    = attr->vendor_part_id;
 188         resp->hw_ver            = attr->hw_ver;
 189         resp->max_qp            = attr->max_qp;
 190         resp->max_qp_wr         = attr->max_qp_wr;
 191         resp->device_cap_flags  = lower_32_bits(attr->device_cap_flags);
 192         resp->max_sge           = attr->max_sge;
 193         resp->max_sge_rd        = attr->max_sge_rd;
 194         resp->max_cq            = attr->max_cq;
 195         resp->max_cqe           = attr->max_cqe;
 196         resp->max_mr            = attr->max_mr;
 197         resp->max_pd            = attr->max_pd;
 198         resp->max_qp_rd_atom    = attr->max_qp_rd_atom;
 199         resp->max_ee_rd_atom    = attr->max_ee_rd_atom;
 200         resp->max_res_rd_atom   = attr->max_res_rd_atom;
 201         resp->max_qp_init_rd_atom       = attr->max_qp_init_rd_atom;
 202         resp->max_ee_init_rd_atom       = attr->max_ee_init_rd_atom;
 203         resp->atomic_cap                = attr->atomic_cap;
 204         resp->max_ee                    = attr->max_ee;
 205         resp->max_rdd                   = attr->max_rdd;
 206         resp->max_mw                    = attr->max_mw;
 207         resp->max_raw_ipv6_qp           = attr->max_raw_ipv6_qp;
 208         resp->max_raw_ethy_qp           = attr->max_raw_ethy_qp;
 209         resp->max_mcast_grp             = attr->max_mcast_grp;
 210         resp->max_mcast_qp_attach       = attr->max_mcast_qp_attach;
 211         resp->max_total_mcast_qp_attach = attr->max_total_mcast_qp_attach;
 212         resp->max_ah                    = attr->max_ah;
 213         resp->max_fmr                   = attr->max_fmr;
 214         resp->max_map_per_fmr           = attr->max_map_per_fmr;
 215         resp->max_srq                   = attr->max_srq;
 216         resp->max_srq_wr                = attr->max_srq_wr;
 217         resp->max_srq_sge               = attr->max_srq_sge;
 218         resp->max_pkeys                 = attr->max_pkeys;
 219         resp->local_ca_ack_delay        = attr->local_ca_ack_delay;
 220         resp->phys_port_cnt             = ib_dev->phys_port_cnt;
 221 }
 222
 223 ssize_t ib_uverbs_query_device(struct ib_uverbs_file *file,
 224                                struct ib_device *ib_dev,
 225                                const char __user *buf,
 226                                int in_len, int out_len)
 227 {
 228         struct ib_uverbs_query_device      cmd;
 229         struct ib_uverbs_query_device_resp resp;
 230
 231         if (out_len < sizeof resp)
 232                 return -ENOSPC;
 233
 234         if (copy_from_user(&cmd, buf, sizeof cmd))
 235                 return -EFAULT;
 236
 237         memset(&resp, 0, sizeof resp);
 238         copy_query_dev_fields(file, ib_dev, &resp, &ib_dev->attrs);
 239
 240         if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp))
 241                 return -EFAULT;
 242
 243         return in_len;
 244 }
 245
 246 ssize_t ib_uverbs_query_port(struct ib_uverbs_file *file,
 247                              struct ib_device *ib_dev,
 248                              const char __user *buf,
 249                              int in_len, int out_len)
 250 {
 251         struct ib_uverbs_query_port      cmd;
 252         struct ib_uverbs_query_port_resp resp;
 253         struct ib_port_attr              attr;
 254         int                              ret;
 255
 256         if (out_len < sizeof resp)
 257                 return -ENOSPC;
 258
 259         if (copy_from_user(&cmd, buf, sizeof cmd))
 260                 return -EFAULT;
 261
 262         ret = ib_query_port(ib_dev, cmd.port_num, &attr);
 263         if (ret)
 264                 return ret;
 265
 266         memset(&resp, 0, sizeof resp);
 267
 268         resp.state           = attr.state;
 269         resp.max_mtu         = attr.max_mtu;
 270         resp.active_mtu      = attr.active_mtu;
 271         resp.gid_tbl_len     = attr.gid_tbl_len;
 272         resp.port_cap_flags  = attr.port_cap_flags;
 273         resp.max_msg_sz      = attr.max_msg_sz;
 274         resp.bad_pkey_cntr   = attr.bad_pkey_cntr;
 275         resp.qkey_viol_cntr  = attr.qkey_viol_cntr;
 276         resp.pkey_tbl_len    = attr.pkey_tbl_len;
 277
 278         if (rdma_cap_opa_ah(ib_dev, cmd.port_num)) {
 279                 resp.lid     = OPA_TO_IB_UCAST_LID(attr.lid);
 280                 resp.sm_lid  = OPA_TO_IB_UCAST_LID(attr.sm_lid);
 281         } else {
 282                 resp.lid     = ib_lid_cpu16(attr.lid);
 283                 resp.sm_lid  = ib_lid_cpu16(attr.sm_lid);
 284         }
 285         resp.lmc             = attr.lmc;
 286         resp.max_vl_num      = attr.max_vl_num;
 287         resp.sm_sl           = attr.sm_sl;
 288         resp.subnet_timeout  = attr.subnet_timeout;
 289         resp.init_type_reply = attr.init_type_reply;
 290         resp.active_width    = attr.active_width;
 291         resp.active_speed    = attr.active_speed;
 292         resp.phys_state      = attr.phys_state;
 293         resp.link_layer      = rdma_port_get_link_layer(ib_dev,
 294                                                         cmd.port_num);
 295
 296         if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp))
 297                 return -EFAULT;
 298
 299         return in_len;
 300 }
 301
 302 ssize_t ib_uverbs_alloc_pd(struct ib_uverbs_file *file,
 303                            struct ib_device *ib_dev,
 304                            const char __user *buf,
 305                            int in_len, int out_len)
 306 {
 307         struct ib_uverbs_alloc_pd      cmd;
 308         struct ib_uverbs_alloc_pd_resp resp;
 309         struct ib_udata                udata;
 310         struct ib_uobject             *uobj;
 311         struct ib_pd                  *pd;
 312         int                            ret;
 313
 314         if (out_len < sizeof resp)
 315                 return -ENOSPC;
 316
 317         if (copy_from_user(&cmd, buf, sizeof cmd))
 318                 return -EFAULT;
 319
 320         ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
 321                    u64_to_user_ptr(cmd.response) + sizeof(resp),
 322                    in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
 323                    out_len - sizeof(resp));
 324
 325         uobj  = uobj_alloc(uobj_get_type(pd), file->ucontext);
 326         if (IS_ERR(uobj))
 327                 return PTR_ERR(uobj);
 328
 329         pd = ib_dev->alloc_pd(ib_dev, file->ucontext, &udata);
 330         if (IS_ERR(pd)) {
 331                 ret = PTR_ERR(pd);
 332                 goto err;
 333         }
 334
 335         pd->device  = ib_dev;
 336         pd->uobject = uobj;
 337         pd->__internal_mr = NULL;
 338         atomic_set(&pd->usecnt, 0);
 339
 340         uobj->object = pd;
 341         memset(&resp, 0, sizeof resp);
 342         resp.pd_handle = uobj->id;
 343         pd->res.type = RDMA_RESTRACK_PD;
 344         rdma_restrack_add(&pd->res);
 345
 346         if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) {
 347                 ret = -EFAULT;
 348                 goto err_copy;
 349         }
 350
 351         uobj_alloc_commit(uobj);
 352
 353         return in_len;
 354
 355 err_copy:
 356         ib_dealloc_pd(pd);
 357
 358 err:
 359         uobj_alloc_abort(uobj);
 360         return ret;
 361 }
 362
 363 ssize_t ib_uverbs_dealloc_pd(struct ib_uverbs_file *file,
 364                              struct ib_device *ib_dev,
 365                              const char __user *buf,
 366                              int in_len, int out_len)
 367 {
 368         struct ib_uverbs_dealloc_pd cmd;
 369         struct ib_uobject          *uobj;
 370         int                         ret;
 371
 372         if (copy_from_user(&cmd, buf, sizeof cmd))
 373                 return -EFAULT;
 374
 375         uobj  = uobj_get_write(uobj_get_type(pd), cmd.pd_handle,
 376                                file->ucontext);
 377         if (IS_ERR(uobj))
 378                 return PTR_ERR(uobj);
 379
 380         ret = uobj_remove_commit(uobj);
 381
 382         return ret ?: in_len;
 383 }
 384
 385 struct xrcd_table_entry {
 386         struct rb_node  node;
 387         struct ib_xrcd *xrcd;
 388         struct inode   *inode;
 389 };
 390
 391 static int xrcd_table_insert(struct ib_uverbs_device *dev,
 392                             struct inode *inode,
 393                             struct ib_xrcd *xrcd)
 394 {
 395         struct xrcd_table_entry *entry, *scan;
 396         struct rb_node **p = &dev->xrcd_tree.rb_node;
 397         struct rb_node *parent = NULL;
 398
 399         entry = kmalloc(sizeof *entry, GFP_KERNEL);
 400         if (!entry)
 401                 return -ENOMEM;
 402
 403         entry->xrcd  = xrcd;
 404         entry->inode = inode;
 405
 406         while (*p) {
 407                 parent = *p;
 408                 scan = rb_entry(parent, struct xrcd_table_entry, node);
 409
 410                 if (inode < scan->inode) {
 411                         p = &(*p)->rb_left;
 412                 } else if (inode > scan->inode) {
 413                         p = &(*p)->rb_right;
 414                 } else {
 415                         kfree(entry);
 416                         return -EEXIST;
 417                 }
 418         }
 419
 420         rb_link_node(&entry->node, parent, p);
 421         rb_insert_color(&entry->node, &dev->xrcd_tree);
 422         igrab(inode);
 423         return 0;
 424 }
 425
 426 static struct xrcd_table_entry *xrcd_table_search(struct ib_uverbs_device *dev,
 427                                                   struct inode *inode)
 428 {
 429         struct xrcd_table_entry *entry;
 430         struct rb_node *p = dev->xrcd_tree.rb_node;
 431
 432         while (p) {
 433                 entry = rb_entry(p, struct xrcd_table_entry, node);
 434
 435                 if (inode < entry->inode)
 436                         p = p->rb_left;
 437                 else if (inode > entry->inode)
 438                         p = p->rb_right;
 439                 else
 440                         return entry;
 441         }
 442
 443         return NULL;
 444 }
 445
 446 static struct ib_xrcd *find_xrcd(struct ib_uverbs_device *dev, struct inode *inode)
 447 {
 448         struct xrcd_table_entry *entry;
 449
 450         entry = xrcd_table_search(dev, inode);
 451         if (!entry)
 452                 return NULL;
 453
 454         return entry->xrcd;
 455 }
 456
 457 static void xrcd_table_delete(struct ib_uverbs_device *dev,
 458                               struct inode *inode)
 459 {
 460         struct xrcd_table_entry *entry;
 461
 462         entry = xrcd_table_search(dev, inode);
 463         if (entry) {
 464                 iput(inode);
 465                 rb_erase(&entry->node, &dev->xrcd_tree);
 466                 kfree(entry);
 467         }
 468 }
 469
 470 ssize_t ib_uverbs_open_xrcd(struct ib_uverbs_file *file,
 471                             struct ib_device *ib_dev,
 472                             const char __user *buf, int in_len,
 473                             int out_len)
 474 {
 475         struct ib_uverbs_open_xrcd      cmd;
 476         struct ib_uverbs_open_xrcd_resp resp;
 477         struct ib_udata                 udata;
 478         struct ib_uxrcd_object         *obj;
 479         struct ib_xrcd                 *xrcd = NULL;
 480         struct fd                       f = {NULL, 0};
 481         struct inode                   *inode = NULL;
 482         int                             ret = 0;
 483         int                             new_xrcd = 0;
 484
 485         if (out_len < sizeof resp)
 486                 return -ENOSPC;
 487
 488         if (copy_from_user(&cmd, buf, sizeof cmd))
 489                 return -EFAULT;
 490
 491         ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
 492                    u64_to_user_ptr(cmd.response) + sizeof(resp),
 493                    in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
 494                    out_len - sizeof(resp));
 495
 496         mutex_lock(&file->device->xrcd_tree_mutex);
 497
 498         if (cmd.fd != -1) {
 499                 /* search for file descriptor */
 500                 f = fdget(cmd.fd);
 501                 if (!f.file) {
 502                         ret = -EBADF;
 503                         goto err_tree_mutex_unlock;
 504                 }
 505
 506                 inode = file_inode(f.file);
 507                 xrcd = find_xrcd(file->device, inode);
 508                 if (!xrcd && !(cmd.oflags & O_CREAT)) {
 509                         /* no file descriptor. Need CREATE flag */
 510                         ret = -EAGAIN;
 511                         goto err_tree_mutex_unlock;
 512                 }
 513
 514                 if (xrcd && cmd.oflags & O_EXCL) {
 515                         ret = -EINVAL;
 516                         goto err_tree_mutex_unlock;
 517                 }
 518         }
 519
 520         obj  = (struct ib_uxrcd_object *)uobj_alloc(uobj_get_type(xrcd),
 521                                                     file->ucontext);
 522         if (IS_ERR(obj)) {
 523                 ret = PTR_ERR(obj);
 524                 goto err_tree_mutex_unlock;
 525         }
 526
 527         if (!xrcd) {
 528                 xrcd = ib_dev->alloc_xrcd(ib_dev, file->ucontext, &udata);
 529                 if (IS_ERR(xrcd)) {
 530                         ret = PTR_ERR(xrcd);
 531                         goto err;
 532                 }
 533
 534                 xrcd->inode   = inode;
 535                 xrcd->device  = ib_dev;
 536                 atomic_set(&xrcd->usecnt, 0);
 537                 mutex_init(&xrcd->tgt_qp_mutex);
 538                 INIT_LIST_HEAD(&xrcd->tgt_qp_list);
 539                 new_xrcd = 1;
 540         }
 541
 542         atomic_set(&obj->refcnt, 0);
 543         obj->uobject.object = xrcd;
 544         memset(&resp, 0, sizeof resp);
 545         resp.xrcd_handle = obj->uobject.id;
 546
 547         if (inode) {
 548                 if (new_xrcd) {
 549                         /* create new inode/xrcd table entry */
 550                         ret = xrcd_table_insert(file->device, inode, xrcd);
 551                         if (ret)
 552                                 goto err_dealloc_xrcd;
 553                 }
 554                 atomic_inc(&xrcd->usecnt);
 555         }
 556
 557         if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) {
 558                 ret = -EFAULT;
 559                 goto err_copy;
 560         }
 561
 562         if (f.file)
 563                 fdput(f);
 564
 565         mutex_unlock(&file->device->xrcd_tree_mutex);
 566
 567         uobj_alloc_commit(&obj->uobject);
 568
 569         return in_len;
 570
 571 err_copy:
 572         if (inode) {
 573                 if (new_xrcd)
 574                         xrcd_table_delete(file->device, inode);
 575                 atomic_dec(&xrcd->usecnt);
 576         }
 577
 578 err_dealloc_xrcd:
 579         ib_dealloc_xrcd(xrcd);
 580
 581 err:
 582         uobj_alloc_abort(&obj->uobject);
 583
 584 err_tree_mutex_unlock:
 585         if (f.file)
 586                 fdput(f);
 587
 588         mutex_unlock(&file->device->xrcd_tree_mutex);
 589
 590         return ret;
 591 }
 592
 593 ssize_t ib_uverbs_close_xrcd(struct ib_uverbs_file *file,
 594                              struct ib_device *ib_dev,
 595                              const char __user *buf, int in_len,
 596                              int out_len)
 597 {
 598         struct ib_uverbs_close_xrcd cmd;
 599         struct ib_uobject           *uobj;
 600         int                         ret = 0;
 601
 602         if (copy_from_user(&cmd, buf, sizeof cmd))
 603                 return -EFAULT;
 604
 605         uobj  = uobj_get_write(uobj_get_type(xrcd), cmd.xrcd_handle,
 606                                file->ucontext);
 607         if (IS_ERR(uobj))
 608                 return PTR_ERR(uobj);
 609
 610         ret = uobj_remove_commit(uobj);
 611         return ret ?: in_len;
 612 }
 613
 614 int ib_uverbs_dealloc_xrcd(struct ib_uverbs_device *dev,
 615                            struct ib_xrcd *xrcd,
 616                            enum rdma_remove_reason why)
 617 {
 618         struct inode *inode;
 619         int ret;
 620
 621         inode = xrcd->inode;
 622         if (inode && !atomic_dec_and_test(&xrcd->usecnt))
 623                 return 0;
 624
 625         ret = ib_dealloc_xrcd(xrcd);
 626
 627         if (why == RDMA_REMOVE_DESTROY && ret)
 628                 atomic_inc(&xrcd->usecnt);
 629         else if (inode)
 630                 xrcd_table_delete(dev, inode);
 631
 632         return ret;
 633 }
 634
 635 ssize_t ib_uverbs_reg_mr(struct ib_uverbs_file *file,
 636                          struct ib_device *ib_dev,
 637                          const char __user *buf, int in_len,
 638                          int out_len)
 639 {
 640         struct ib_uverbs_reg_mr      cmd;
 641         struct ib_uverbs_reg_mr_resp resp;
 642         struct ib_udata              udata;
 643         struct ib_uobject           *uobj;
 644         struct ib_pd                *pd;
 645         struct ib_mr                *mr;
 646         int                          ret;
 647
 648         if (out_len < sizeof resp)
 649                 return -ENOSPC;
 650
 651         if (copy_from_user(&cmd, buf, sizeof cmd))
 652                 return -EFAULT;
 653
 654         ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
 655                    u64_to_user_ptr(cmd.response) + sizeof(resp),
 656                    in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
 657                    out_len - sizeof(resp));
 658
 659         if ((cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK))
 660                 return -EINVAL;
 661
 662         ret = ib_check_mr_access(cmd.access_flags);
 663         if (ret)
 664                 return ret;
 665
 666         uobj  = uobj_alloc(uobj_get_type(mr), file->ucontext);
 667         if (IS_ERR(uobj))
 668                 return PTR_ERR(uobj);
 669
 670         pd = uobj_get_obj_read(pd, cmd.pd_handle, file->ucontext);
 671         if (!pd) {
 672                 ret = -EINVAL;
 673                 goto err_free;
 674         }
 675
 676         if (cmd.access_flags & IB_ACCESS_ON_DEMAND) {
 677                 if (!(pd->device->attrs.device_cap_flags &
 678                       IB_DEVICE_ON_DEMAND_PAGING)) {
 679                         pr_debug("ODP support not available\n");
 680                         ret = -EINVAL;
 681                         goto err_put;
 682                 }
 683         }
 684
 685         mr = pd->device->reg_user_mr(pd, cmd.start, cmd.length, cmd.hca_va,
 686                                      cmd.access_flags, &udata);
 687         if (IS_ERR(mr)) {
 688                 ret = PTR_ERR(mr);
 689                 goto err_put;
 690         }
 691
 692         mr->device  = pd->device;
 693         mr->pd      = pd;
 694         mr->uobject = uobj;
 695         atomic_inc(&pd->usecnt);
 696
 697         uobj->object = mr;
 698
 699         memset(&resp, 0, sizeof resp);
 700         resp.lkey      = mr->lkey;
 701         resp.rkey      = mr->rkey;
 702         resp.mr_handle = uobj->id;
 703
 704         if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) {
 705                 ret = -EFAULT;
 706                 goto err_copy;
 707         }
 708
 709         uobj_put_obj_read(pd);
 710
 711         uobj_alloc_commit(uobj);
 712
 713         return in_len;
 714
 715 err_copy:
 716         ib_dereg_mr(mr);
 717
 718 err_put:
 719         uobj_put_obj_read(pd);
 720
 721 err_free:
 722         uobj_alloc_abort(uobj);
 723         return ret;
 724 }
 725
 726 ssize_t ib_uverbs_rereg_mr(struct ib_uverbs_file *file,
 727                            struct ib_device *ib_dev,
 728                            const char __user *buf, int in_len,
 729                            int out_len)
 730 {
 731         struct ib_uverbs_rereg_mr      cmd;
 732         struct ib_uverbs_rereg_mr_resp resp;
 733         struct ib_udata              udata;
 734         struct ib_pd                *pd = NULL;
 735         struct ib_mr                *mr;
 736         struct ib_pd                *old_pd;
 737         int                          ret;
 738         struct ib_uobject           *uobj;
 739
 740         if (out_len < sizeof(resp))
 741                 return -ENOSPC;
 742
 743         if (copy_from_user(&cmd, buf, sizeof(cmd)))
 744                 return -EFAULT;
 745
 746         ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
 747                    u64_to_user_ptr(cmd.response) + sizeof(resp),
 748                    in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
 749                    out_len - sizeof(resp));
 750
 751         if (cmd.flags & ~IB_MR_REREG_SUPPORTED || !cmd.flags)
 752                 return -EINVAL;
 753
 754         if ((cmd.flags & IB_MR_REREG_TRANS) &&
 755             (!cmd.start || !cmd.hca_va || 0 >= cmd.length ||
 756              (cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK)))
 757                         return -EINVAL;
 758
 759         uobj  = uobj_get_write(uobj_get_type(mr), cmd.mr_handle,
 760                                file->ucontext);
 761         if (IS_ERR(uobj))
 762                 return PTR_ERR(uobj);
 763
 764         mr = uobj->object;
 765
 766         if (cmd.flags & IB_MR_REREG_ACCESS) {
 767                 ret = ib_check_mr_access(cmd.access_flags);
 768                 if (ret)
 769                         goto put_uobjs;
 770         }
 771
 772         if (cmd.flags & IB_MR_REREG_PD) {
 773                 pd = uobj_get_obj_read(pd, cmd.pd_handle, file->ucontext);
 774                 if (!pd) {
 775                         ret = -EINVAL;
 776                         goto put_uobjs;
 777                 }
 778         }
 779
 780         old_pd = mr->pd;
 781         ret = mr->device->rereg_user_mr(mr, cmd.flags, cmd.start,
 782                                         cmd.length, cmd.hca_va,
 783                                         cmd.access_flags, pd, &udata);
 784         if (!ret) {
 785                 if (cmd.flags & IB_MR_REREG_PD) {
 786                         atomic_inc(&pd->usecnt);
 787                         mr->pd = pd;
 788                         atomic_dec(&old_pd->usecnt);
 789                 }
 790         } else {
 791                 goto put_uobj_pd;
 792         }
 793
 794         memset(&resp, 0, sizeof(resp));
 795         resp.lkey      = mr->lkey;
 796         resp.rkey      = mr->rkey;
 797
 798         if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof(resp)))
 799                 ret = -EFAULT;
 800         else
 801                 ret = in_len;
 802
 803 put_uobj_pd:
 804         if (cmd.flags & IB_MR_REREG_PD)
 805                 uobj_put_obj_read(pd);
 806
 807 put_uobjs:
 808         uobj_put_write(uobj);
 809
 810         return ret;
 811 }
 812
 813 ssize_t ib_uverbs_dereg_mr(struct ib_uverbs_file *file,
 814                            struct ib_device *ib_dev,
 815                            const char __user *buf, int in_len,
 816                            int out_len)
 817 {
 818         struct ib_uverbs_dereg_mr cmd;
 819         struct ib_uobject        *uobj;
 820         int                       ret = -EINVAL;
 821
 822         if (copy_from_user(&cmd, buf, sizeof cmd))
 823                 return -EFAULT;
 824
 825         uobj  = uobj_get_write(uobj_get_type(mr), cmd.mr_handle,
 826                                file->ucontext);
 827         if (IS_ERR(uobj))
 828                 return PTR_ERR(uobj);
 829
 830         ret = uobj_remove_commit(uobj);
 831
 832         return ret ?: in_len;
 833 }
 834
 835 ssize_t ib_uverbs_alloc_mw(struct ib_uverbs_file *file,
 836                            struct ib_device *ib_dev,
 837                            const char __user *buf, int in_len,
 838                            int out_len)
 839 {
 840         struct ib_uverbs_alloc_mw      cmd;
 841         struct ib_uverbs_alloc_mw_resp resp;
 842         struct ib_uobject             *uobj;
 843         struct ib_pd                  *pd;
 844         struct ib_mw                  *mw;
 845         struct ib_udata                udata;
 846         int                            ret;
 847
 848         if (out_len < sizeof(resp))
 849                 return -ENOSPC;
 850
 851         if (copy_from_user(&cmd, buf, sizeof(cmd)))
 852                 return -EFAULT;
 853
 854         uobj  = uobj_alloc(uobj_get_type(mw), file->ucontext);
 855         if (IS_ERR(uobj))
 856                 return PTR_ERR(uobj);
 857
 858         pd = uobj_get_obj_read(pd, cmd.pd_handle, file->ucontext);
 859         if (!pd) {
 860                 ret = -EINVAL;
 861                 goto err_free;
 862         }
 863
 864         ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
 865                    u64_to_user_ptr(cmd.response) + sizeof(resp),
 866                    in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
 867                    out_len - sizeof(resp));
 868
 869         mw = pd->device->alloc_mw(pd, cmd.mw_type, &udata);
 870         if (IS_ERR(mw)) {
 871                 ret = PTR_ERR(mw);
 872                 goto err_put;
 873         }
 874
 875         mw->device  = pd->device;
 876         mw->pd      = pd;
 877         mw->uobject = uobj;
 878         atomic_inc(&pd->usecnt);
 879
 880         uobj->object = mw;
 881
 882         memset(&resp, 0, sizeof(resp));
 883         resp.rkey      = mw->rkey;
 884         resp.mw_handle = uobj->id;
 885
 886         if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof(resp))) {
 887                 ret = -EFAULT;
 888                 goto err_copy;
 889         }
 890
 891         uobj_put_obj_read(pd);
 892         uobj_alloc_commit(uobj);
 893
 894         return in_len;
 895
 896 err_copy:
 897         uverbs_dealloc_mw(mw);
 898 err_put:
 899         uobj_put_obj_read(pd);
 900 err_free:
 901         uobj_alloc_abort(uobj);
 902         return ret;
 903 }
 904
 905 ssize_t ib_uverbs_dealloc_mw(struct ib_uverbs_file *file,
 906                              struct ib_device *ib_dev,
 907                              const char __user *buf, int in_len,
 908                              int out_len)
 909 {
 910         struct ib_uverbs_dealloc_mw cmd;
 911         struct ib_uobject          *uobj;
 912         int                         ret = -EINVAL;
 913
 914         if (copy_from_user(&cmd, buf, sizeof(cmd)))
 915                 return -EFAULT;
 916
 917         uobj  = uobj_get_write(uobj_get_type(mw), cmd.mw_handle,
 918                                file->ucontext);
 919         if (IS_ERR(uobj))
 920                 return PTR_ERR(uobj);
 921
 922         ret = uobj_remove_commit(uobj);
 923         return ret ?: in_len;
 924 }
 925
 926 ssize_t ib_uverbs_create_comp_channel(struct ib_uverbs_file *file,
 927                                       struct ib_device *ib_dev,
 928                                       const char __user *buf, int in_len,
 929                                       int out_len)
 930 {
 931         struct ib_uverbs_create_comp_channel       cmd;
 932         struct ib_uverbs_create_comp_channel_resp  resp;
 933         struct ib_uobject                         *uobj;
 934         struct ib_uverbs_completion_event_file    *ev_file;
 935
 936         if (out_len < sizeof resp)
 937                 return -ENOSPC;
 938
 939         if (copy_from_user(&cmd, buf, sizeof cmd))
 940                 return -EFAULT;
 941
 942         uobj = uobj_alloc(uobj_get_type(comp_channel), file->ucontext);
 943         if (IS_ERR(uobj))
 944                 return PTR_ERR(uobj);
 945
 946         resp.fd = uobj->id;
 947
 948         ev_file = container_of(uobj, struct ib_uverbs_completion_event_file,
 949                                uobj_file.uobj);
 950         ib_uverbs_init_event_queue(&ev_file->ev_queue);
 951
 952         if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) {
 953                 uobj_alloc_abort(uobj);
 954                 return -EFAULT;
 955         }
 956
 957         uobj_alloc_commit(uobj);
 958         return in_len;
 959 }
 960
 961 static struct ib_ucq_object *create_cq(struct ib_uverbs_file *file,
 962                                         struct ib_device *ib_dev,
 963                                        struct ib_udata *ucore,
 964                                        struct ib_udata *uhw,
 965                                        struct ib_uverbs_ex_create_cq *cmd,
 966                                        size_t cmd_sz,
 967                                        int (*cb)(struct ib_uverbs_file *file,
 968                                                  struct ib_ucq_object *obj,
 969                                                  struct ib_uverbs_ex_create_cq_resp *resp,
 970                                                  struct ib_udata *udata,
 971                                                  void *context),
 972                                        void *context)
 973 {
 974         struct ib_ucq_object           *obj;
 975         struct ib_uverbs_completion_event_file    *ev_file = NULL;
 976         struct ib_cq                   *cq;
 977         int                             ret;
 978         struct ib_uverbs_ex_create_cq_resp resp;
 979         struct ib_cq_init_attr attr = {};
 980
 981         if (!ib_dev->create_cq)
 982                 return ERR_PTR(-EOPNOTSUPP);
 983
 984         if (cmd->comp_vector >= file->device->num_comp_vectors)
 985                 return ERR_PTR(-EINVAL);
 986
 987         obj  = (struct ib_ucq_object *)uobj_alloc(uobj_get_type(cq),
 988                                                   file->ucontext);
 989         if (IS_ERR(obj))
 990                 return obj;
 991
 992         if (cmd->comp_channel >= 0) {
 993                 ev_file = ib_uverbs_lookup_comp_file(cmd->comp_channel,
 994                                                      file->ucontext);
 995                 if (IS_ERR(ev_file)) {
 996                         ret = PTR_ERR(ev_file);
 997                         goto err;
 998                 }
 999         }
1000
1001         obj->uobject.user_handle = cmd->user_handle;
1002         obj->uverbs_file           = file;
1003         obj->comp_events_reported  = 0;
1004         obj->async_events_reported = 0;
1005         INIT_LIST_HEAD(&obj->comp_list);
1006         INIT_LIST_HEAD(&obj->async_list);
1007
1008         attr.cqe = cmd->cqe;
1009         attr.comp_vector = cmd->comp_vector;
1010
1011         if (cmd_sz > offsetof(typeof(*cmd), flags) + sizeof(cmd->flags))
1012                 attr.flags = cmd->flags;
1013
1014         cq = ib_dev->create_cq(ib_dev, &attr, file->ucontext, uhw);
1015         if (IS_ERR(cq)) {
1016                 ret = PTR_ERR(cq);
1017                 goto err_file;
1018         }
1019
1020         cq->device        = ib_dev;
1021         cq->uobject       = &obj->uobject;
1022         cq->comp_handler  = ib_uverbs_comp_handler;
1023         cq->event_handler = ib_uverbs_cq_event_handler;
1024         cq->cq_context    = ev_file ? &ev_file->ev_queue : NULL;
1025         atomic_set(&cq->usecnt, 0);
1026
1027         obj->uobject.object = cq;
1028         memset(&resp, 0, sizeof resp);
1029         resp.base.cq_handle = obj->uobject.id;
1030         resp.base.cqe       = cq->cqe;
1031
1032         resp.response_length = offsetof(typeof(resp), response_length) +
1033                 sizeof(resp.response_length);
1034
1035         cq->res.type = RDMA_RESTRACK_CQ;
1036         rdma_restrack_add(&cq->res);
1037
1038         ret = cb(file, obj, &resp, ucore, context);
1039         if (ret)
1040                 goto err_cb;
1041
1042         uobj_alloc_commit(&obj->uobject);
1043         return obj;
1044
1045 err_cb:
1046         ib_destroy_cq(cq);
1047
1048 err_file:
1049         if (ev_file)
1050                 ib_uverbs_release_ucq(file, ev_file, obj);
1051
1052 err:
1053         uobj_alloc_abort(&obj->uobject);
1054
1055         return ERR_PTR(ret);
1056 }
1057
1058 static int ib_uverbs_create_cq_cb(struct ib_uverbs_file *file,
1059                                   struct ib_ucq_object *obj,
1060                                   struct ib_uverbs_ex_create_cq_resp *resp,
1061                                   struct ib_udata *ucore, void *context)
1062 {
1063         if (ib_copy_to_udata(ucore, &resp->base, sizeof(resp->base)))
1064                 return -EFAULT;
1065
1066         return 0;
1067 }
1068
1069 ssize_t ib_uverbs_create_cq(struct ib_uverbs_file *file,
1070                             struct ib_device *ib_dev,
1071                             const char __user *buf, int in_len,
1072                             int out_len)
1073 {
1074         struct ib_uverbs_create_cq      cmd;
1075         struct ib_uverbs_ex_create_cq   cmd_ex;
1076         struct ib_uverbs_create_cq_resp resp;
1077         struct ib_udata                 ucore;
1078         struct ib_udata                 uhw;
1079         struct ib_ucq_object           *obj;
1080
1081         if (out_len < sizeof(resp))
1082                 return -ENOSPC;
1083
1084         if (copy_from_user(&cmd, buf, sizeof(cmd)))
1085                 return -EFAULT;
1086
1087         ib_uverbs_init_udata(&ucore, buf, u64_to_user_ptr(cmd.response),
1088                              sizeof(cmd), sizeof(resp));
1089
1090         ib_uverbs_init_udata(&uhw, buf + sizeof(cmd),
1091                    u64_to_user_ptr(cmd.response) + sizeof(resp),
1092                    in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
1093                    out_len - sizeof(resp));
1094
1095         memset(&cmd_ex, 0, sizeof(cmd_ex));
1096         cmd_ex.user_handle = cmd.user_handle;
1097         cmd_ex.cqe = cmd.cqe;
1098         cmd_ex.comp_vector = cmd.comp_vector;
1099         cmd_ex.comp_channel = cmd.comp_channel;
1100
1101         obj = create_cq(file, ib_dev, &ucore, &uhw, &cmd_ex,
1102                         offsetof(typeof(cmd_ex), comp_channel) +
1103                         sizeof(cmd.comp_channel), ib_uverbs_create_cq_cb,
1104                         NULL);
1105
1106         if (IS_ERR(obj))
1107                 return PTR_ERR(obj);
1108
1109         return in_len;
1110 }
1111
1112 static int ib_uverbs_ex_create_cq_cb(struct ib_uverbs_file *file,
1113                                      struct ib_ucq_object *obj,
1114                                      struct ib_uverbs_ex_create_cq_resp *resp,
1115                                      struct ib_udata *ucore, void *context)
1116 {
1117         if (ib_copy_to_udata(ucore, resp, resp->response_length))
1118                 return -EFAULT;
1119
1120         return 0;
1121 }
1122
1123 int ib_uverbs_ex_create_cq(struct ib_uverbs_file *file,
1124                          struct ib_device *ib_dev,
1125                            struct ib_udata *ucore,
1126                            struct ib_udata *uhw)
1127 {
1128         struct ib_uverbs_ex_create_cq_resp resp;
1129         struct ib_uverbs_ex_create_cq  cmd;
1130         struct ib_ucq_object           *obj;
1131         int err;
1132
1133         if (ucore->inlen < sizeof(cmd))
1134                 return -EINVAL;
1135
1136         err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd));
1137         if (err)
1138                 return err;
1139
1140         if (cmd.comp_mask)
1141                 return -EINVAL;
1142
1143         if (cmd.reserved)
1144                 return -EINVAL;
1145
1146         if (ucore->outlen < (offsetof(typeof(resp), response_length) +
1147                              sizeof(resp.response_length)))
1148                 return -ENOSPC;
1149
1150         obj = create_cq(file, ib_dev, ucore, uhw, &cmd,
1151                         min(ucore->inlen, sizeof(cmd)),
1152                         ib_uverbs_ex_create_cq_cb, NULL);
1153
1154         return PTR_ERR_OR_ZERO(obj);
1155 }
1156
1157 ssize_t ib_uverbs_resize_cq(struct ib_uverbs_file *file,
1158                             struct ib_device *ib_dev,
1159                             const char __user *buf, int in_len,
1160                             int out_len)
1161 {
1162         struct ib_uverbs_resize_cq      cmd;
1163         struct ib_uverbs_resize_cq_resp resp = {};
1164         struct ib_udata                 udata;
1165         struct ib_cq                    *cq;
1166         int                             ret = -EINVAL;
1167
1168         if (copy_from_user(&cmd, buf, sizeof cmd))
1169                 return -EFAULT;
1170
1171         ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
1172                    u64_to_user_ptr(cmd.response) + sizeof(resp),
1173                    in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
1174                    out_len - sizeof(resp));
1175
1176         cq = uobj_get_obj_read(cq, cmd.cq_handle, file->ucontext);
1177         if (!cq)
1178                 return -EINVAL;
1179
1180         ret = cq->device->resize_cq(cq, cmd.cqe, &udata);
1181         if (ret)
1182                 goto out;
1183
1184         resp.cqe = cq->cqe;
1185
1186         if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp.cqe))
1187                 ret = -EFAULT;
1188
1189 out:
1190         uobj_put_obj_read(cq);
1191
1192         return ret ? ret : in_len;
1193 }
1194
1195 static int copy_wc_to_user(struct ib_device *ib_dev, void __user *dest,
1196                            struct ib_wc *wc)
1197 {
1198         struct ib_uverbs_wc tmp;
1199
1200         tmp.wr_id               = wc->wr_id;
1201         tmp.status              = wc->status;
1202         tmp.opcode              = wc->opcode;
1203         tmp.vendor_err          = wc->vendor_err;
1204         tmp.byte_len            = wc->byte_len;
1205         tmp.ex.imm_data         = wc->ex.imm_data;
1206         tmp.qp_num              = wc->qp->qp_num;
1207         tmp.src_qp              = wc->src_qp;
1208         tmp.wc_flags            = wc->wc_flags;
1209         tmp.pkey_index          = wc->pkey_index;
1210         if (rdma_cap_opa_ah(ib_dev, wc->port_num))
1211                 tmp.slid        = OPA_TO_IB_UCAST_LID(wc->slid);
1212         else
1213                 tmp.slid        = ib_lid_cpu16(wc->slid);
1214         tmp.sl                  = wc->sl;
1215         tmp.dlid_path_bits      = wc->dlid_path_bits;
1216         tmp.port_num            = wc->port_num;
1217         tmp.reserved            = 0;
1218
1219         if (copy_to_user(dest, &tmp, sizeof tmp))
1220                 return -EFAULT;
1221
1222         return 0;
1223 }
1224
1225 ssize_t ib_uverbs_poll_cq(struct ib_uverbs_file *file,
1226                           struct ib_device *ib_dev,
1227                           const char __user *buf, int in_len,
1228                           int out_len)
1229 {
1230         struct ib_uverbs_poll_cq       cmd;
1231         struct ib_uverbs_poll_cq_resp  resp;
1232         u8 __user                     *header_ptr;
1233         u8 __user                     *data_ptr;
1234         struct ib_cq                  *cq;
1235         struct ib_wc                   wc;
1236         int                            ret;
1237
1238         if (copy_from_user(&cmd, buf, sizeof cmd))
1239                 return -EFAULT;
1240
1241         cq = uobj_get_obj_read(cq, cmd.cq_handle, file->ucontext);
1242         if (!cq)
1243                 return -EINVAL;
1244
1245         /* we copy a struct ib_uverbs_poll_cq_resp to user space */
1246         header_ptr = u64_to_user_ptr(cmd.response);
1247         data_ptr = header_ptr + sizeof resp;
1248
1249         memset(&resp, 0, sizeof resp);
1250         while (resp.count < cmd.ne) {
1251                 ret = ib_poll_cq(cq, 1, &wc);
1252                 if (ret < 0)
1253                         goto out_put;
1254                 if (!ret)
1255                         break;
1256
1257                 ret = copy_wc_to_user(ib_dev, data_ptr, &wc);
1258                 if (ret)
1259                         goto out_put;
1260
1261                 data_ptr += sizeof(struct ib_uverbs_wc);
1262                 ++resp.count;
1263         }
1264
1265         if (copy_to_user(header_ptr, &resp, sizeof resp)) {
1266                 ret = -EFAULT;
1267                 goto out_put;
1268         }
1269
1270         ret = in_len;
1271
1272 out_put:
1273         uobj_put_obj_read(cq);
1274         return ret;
1275 }
1276
1277 ssize_t ib_uverbs_req_notify_cq(struct ib_uverbs_file *file,
1278                                 struct ib_device *ib_dev,
1279                                 const char __user *buf, int in_len,
1280                                 int out_len)
1281 {
1282         struct ib_uverbs_req_notify_cq cmd;
1283         struct ib_cq                  *cq;
1284
1285         if (copy_from_user(&cmd, buf, sizeof cmd))
1286                 return -EFAULT;
1287
1288         cq = uobj_get_obj_read(cq, cmd.cq_handle, file->ucontext);
1289         if (!cq)
1290                 return -EINVAL;
1291
1292         ib_req_notify_cq(cq, cmd.solicited_only ?
1293                          IB_CQ_SOLICITED : IB_CQ_NEXT_COMP);
1294
1295         uobj_put_obj_read(cq);
1296
1297         return in_len;
1298 }
1299
1300 ssize_t ib_uverbs_destroy_cq(struct ib_uverbs_file *file,
1301                              struct ib_device *ib_dev,
1302                              const char __user *buf, int in_len,
1303                              int out_len)
1304 {
1305         struct ib_uverbs_destroy_cq      cmd;
1306         struct ib_uverbs_destroy_cq_resp resp;
1307         struct ib_uobject               *uobj;
1308         struct ib_cq                    *cq;
1309         struct ib_ucq_object            *obj;
1310         int                              ret = -EINVAL;
1311
1312         if (copy_from_user(&cmd, buf, sizeof cmd))
1313                 return -EFAULT;
1314
1315         uobj  = uobj_get_write(uobj_get_type(cq), cmd.cq_handle,
1316                                file->ucontext);
1317         if (IS_ERR(uobj))
1318                 return PTR_ERR(uobj);
1319
1320         /*
1321          * Make sure we don't free the memory in remove_commit as we still
1322          * needs the uobject memory to create the response.
1323          */
1324         uverbs_uobject_get(uobj);
1325         cq      = uobj->object;
1326         obj     = container_of(cq->uobject, struct ib_ucq_object, uobject);
1327
1328         memset(&resp, 0, sizeof(resp));
1329
1330         ret = uobj_remove_commit(uobj);
1331         if (ret) {
1332                 uverbs_uobject_put(uobj);
1333                 return ret;
1334         }
1335
1336         resp.comp_events_reported  = obj->comp_events_reported;
1337         resp.async_events_reported = obj->async_events_reported;
1338
1339         uverbs_uobject_put(uobj);
1340         if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp))
1341                 return -EFAULT;
1342
1343         return in_len;
1344 }
1345
1346 static int create_qp(struct ib_uverbs_file *file,
1347                      struct ib_udata *ucore,
1348                      struct ib_udata *uhw,
1349                      struct ib_uverbs_ex_create_qp *cmd,
1350                      size_t cmd_sz,
1351                      int (*cb)(struct ib_uverbs_file *file,
1352                                struct ib_uverbs_ex_create_qp_resp *resp,
1353                                struct ib_udata *udata),
1354                      void *context)
1355 {
1356         struct ib_uqp_object            *obj;
1357         struct ib_device                *device;
1358         struct ib_pd                    *pd = NULL;
1359         struct ib_xrcd                  *xrcd = NULL;
1360         struct ib_uobject               *xrcd_uobj = ERR_PTR(-ENOENT);
1361         struct ib_cq                    *scq = NULL, *rcq = NULL;
1362         struct ib_srq                   *srq = NULL;
1363         struct ib_qp                    *qp;
1364         char                            *buf;
1365         struct ib_qp_init_attr          attr = {};
1366         struct ib_uverbs_ex_create_qp_resp resp;
1367         int                             ret;
1368         struct ib_rwq_ind_table *ind_tbl = NULL;
1369         bool has_sq = true;
1370
1371         if (cmd->qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW))
1372                 return -EPERM;
1373
1374         obj  = (struct ib_uqp_object *)uobj_alloc(uobj_get_type(qp),
1375                                                   file->ucontext);
1376         if (IS_ERR(obj))
1377                 return PTR_ERR(obj);
1378         obj->uxrcd = NULL;
1379         obj->uevent.uobject.user_handle = cmd->user_handle;
1380         mutex_init(&obj->mcast_lock);
1381
1382         if (cmd_sz >= offsetof(typeof(*cmd), rwq_ind_tbl_handle) +
1383                       sizeof(cmd->rwq_ind_tbl_handle) &&
1384                       (cmd->comp_mask & IB_UVERBS_CREATE_QP_MASK_IND_TABLE)) {
1385                 ind_tbl = uobj_get_obj_read(rwq_ind_table,
1386                                             cmd->rwq_ind_tbl_handle,
1387                                             file->ucontext);
1388                 if (!ind_tbl) {
1389                         ret = -EINVAL;
1390                         goto err_put;
1391                 }
1392
1393                 attr.rwq_ind_tbl = ind_tbl;
1394         }
1395
1396         if (cmd_sz > sizeof(*cmd) &&
1397             !ib_is_udata_cleared(ucore, sizeof(*cmd),
1398                                  cmd_sz - sizeof(*cmd))) {
1399                 ret = -EOPNOTSUPP;
1400                 goto err_put;
1401         }
1402
1403         if (ind_tbl && (cmd->max_recv_wr || cmd->max_recv_sge || cmd->is_srq)) {
1404                 ret = -EINVAL;
1405                 goto err_put;
1406         }
1407
1408         if (ind_tbl && !cmd->max_send_wr)
1409                 has_sq = false;
1410
1411         if (cmd->qp_type == IB_QPT_XRC_TGT) {
1412                 xrcd_uobj = uobj_get_read(uobj_get_type(xrcd), cmd->pd_handle,
1413                                           file->ucontext);
1414
1415                 if (IS_ERR(xrcd_uobj)) {
1416                         ret = -EINVAL;
1417                         goto err_put;
1418                 }
1419
1420                 xrcd = (struct ib_xrcd *)xrcd_uobj->object;
1421                 if (!xrcd) {
1422                         ret = -EINVAL;
1423                         goto err_put;
1424                 }
1425                 device = xrcd->device;
1426         } else {
1427                 if (cmd->qp_type == IB_QPT_XRC_INI) {
1428                         cmd->max_recv_wr = 0;
1429                         cmd->max_recv_sge = 0;
1430                 } else {
1431                         if (cmd->is_srq) {
1432                                 srq = uobj_get_obj_read(srq, cmd->srq_handle,
1433                                                         file->ucontext);
1434                                 if (!srq || srq->srq_type == IB_SRQT_XRC) {
1435                                         ret = -EINVAL;
1436                                         goto err_put;
1437                                 }
1438                         }
1439
1440                         if (!ind_tbl) {
1441                                 if (cmd->recv_cq_handle != cmd->send_cq_handle) {
1442                                         rcq = uobj_get_obj_read(cq, cmd->recv_cq_handle,
1443                                                                 file->ucontext);
1444                                         if (!rcq) {
1445                                                 ret = -EINVAL;
1446                                                 goto err_put;
1447                                         }
1448                                 }
1449                         }
1450                 }
1451
1452                 if (has_sq)
1453                         scq = uobj_get_obj_read(cq, cmd->send_cq_handle,
1454                                                 file->ucontext);
1455                 if (!ind_tbl)
1456                         rcq = rcq ?: scq;
1457                 pd  = uobj_get_obj_read(pd, cmd->pd_handle, file->ucontext);
1458                 if (!pd || (!scq && has_sq)) {
1459                         ret = -EINVAL;
1460                         goto err_put;
1461                 }
1462
1463                 device = pd->device;
1464         }
1465
1466         attr.event_handler = ib_uverbs_qp_event_handler;
1467         attr.qp_context    = file;
1468         attr.send_cq       = scq;
1469         attr.recv_cq       = rcq;
1470         attr.srq           = srq;
1471         attr.xrcd          = xrcd;
1472         attr.sq_sig_type   = cmd->sq_sig_all ? IB_SIGNAL_ALL_WR :
1473                                               IB_SIGNAL_REQ_WR;
1474         attr.qp_type       = cmd->qp_type;
1475         attr.create_flags  = 0;
1476
1477         attr.cap.max_send_wr     = cmd->max_send_wr;
1478         attr.cap.max_recv_wr     = cmd->max_recv_wr;
1479         attr.cap.max_send_sge    = cmd->max_send_sge;
1480         attr.cap.max_recv_sge    = cmd->max_recv_sge;
1481         attr.cap.max_inline_data = cmd->max_inline_data;
1482
1483         obj->uevent.events_reported     = 0;
1484         INIT_LIST_HEAD(&obj->uevent.event_list);
1485         INIT_LIST_HEAD(&obj->mcast_list);
1486
1487         if (cmd_sz >= offsetof(typeof(*cmd), create_flags) +
1488                       sizeof(cmd->create_flags))
1489                 attr.create_flags = cmd->create_flags;
1490
1491         if (attr.create_flags & ~(IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK |
1492                                 IB_QP_CREATE_CROSS_CHANNEL |
1493                                 IB_QP_CREATE_MANAGED_SEND |
1494                                 IB_QP_CREATE_MANAGED_RECV |
1495                                 IB_QP_CREATE_SCATTER_FCS |
1496                                 IB_QP_CREATE_CVLAN_STRIPPING |
1497                                 IB_QP_CREATE_SOURCE_QPN |
1498                                 IB_QP_CREATE_PCI_WRITE_END_PADDING)) {
1499                 ret = -EINVAL;
1500                 goto err_put;
1501         }
1502
1503         if (attr.create_flags & IB_QP_CREATE_SOURCE_QPN) {
1504                 if (!capable(CAP_NET_RAW)) {
1505                         ret = -EPERM;
1506                         goto err_put;
1507                 }
1508
1509                 attr.source_qpn = cmd->source_qpn;
1510         }
1511
1512         buf = (void *)cmd + sizeof(*cmd);
1513         if (cmd_sz > sizeof(*cmd))
1514                 if (!(buf[0] == 0 && !memcmp(buf, buf + 1,
1515                                              cmd_sz - sizeof(*cmd) - 1))) {
1516                         ret = -EINVAL;
1517                         goto err_put;
1518                 }
1519
1520         if (cmd->qp_type == IB_QPT_XRC_TGT)
1521                 qp = ib_create_qp(pd, &attr);
1522         else
1523                 qp = _ib_create_qp(device, pd, &attr, uhw);
1524
1525         if (IS_ERR(qp)) {
1526                 ret = PTR_ERR(qp);
1527                 goto err_put;
1528         }
1529
1530         if (cmd->qp_type != IB_QPT_XRC_TGT) {
1531                 ret = ib_create_qp_security(qp, device);
1532                 if (ret)
1533                         goto err_cb;
1534
1535                 qp->real_qp       = qp;
1536                 qp->pd            = pd;
1537                 qp->send_cq       = attr.send_cq;
1538                 qp->recv_cq       = attr.recv_cq;
1539                 qp->srq           = attr.srq;
1540                 qp->rwq_ind_tbl   = ind_tbl;
1541                 qp->event_handler = attr.event_handler;
1542                 qp->qp_context    = attr.qp_context;
1543                 qp->qp_type       = attr.qp_type;
1544                 atomic_set(&qp->usecnt, 0);
1545                 atomic_inc(&pd->usecnt);
1546                 qp->port = 0;
1547                 if (attr.send_cq)
1548                         atomic_inc(&attr.send_cq->usecnt);
1549                 if (attr.recv_cq)
1550                         atomic_inc(&attr.recv_cq->usecnt);
1551                 if (attr.srq)
1552                         atomic_inc(&attr.srq->usecnt);
1553                 if (ind_tbl)
1554                         atomic_inc(&ind_tbl->usecnt);
1555         }
1556         qp->uobject = &obj->uevent.uobject;
1557
1558         obj->uevent.uobject.object = qp;
1559
1560         memset(&resp, 0, sizeof resp);
1561         resp.base.qpn             = qp->qp_num;
1562         resp.base.qp_handle       = obj->uevent.uobject.id;
1563         resp.base.max_recv_sge    = attr.cap.max_recv_sge;
1564         resp.base.max_send_sge    = attr.cap.max_send_sge;
1565         resp.base.max_recv_wr     = attr.cap.max_recv_wr;
1566         resp.base.max_send_wr     = attr.cap.max_send_wr;
1567         resp.base.max_inline_data = attr.cap.max_inline_data;
1568
1569         resp.response_length = offsetof(typeof(resp), response_length) +
1570                                sizeof(resp.response_length);
1571
1572         ret = cb(file, &resp, ucore);
1573         if (ret)
1574                 goto err_cb;
1575
1576         if (xrcd) {
1577                 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object,
1578                                           uobject);
1579                 atomic_inc(&obj->uxrcd->refcnt);
1580                 uobj_put_read(xrcd_uobj);
1581         }
1582
1583         if (pd)
1584                 uobj_put_obj_read(pd);
1585         if (scq)
1586                 uobj_put_obj_read(scq);
1587         if (rcq && rcq != scq)
1588                 uobj_put_obj_read(rcq);
1589         if (srq)
1590                 uobj_put_obj_read(srq);
1591         if (ind_tbl)
1592                 uobj_put_obj_read(ind_tbl);
1593
1594         uobj_alloc_commit(&obj->uevent.uobject);
1595
1596         return 0;
1597 err_cb:
1598         ib_destroy_qp(qp);
1599
1600 err_put:
1601         if (!IS_ERR(xrcd_uobj))
1602                 uobj_put_read(xrcd_uobj);
1603         if (pd)
1604                 uobj_put_obj_read(pd);
1605         if (scq)
1606                 uobj_put_obj_read(scq);
1607         if (rcq && rcq != scq)
1608                 uobj_put_obj_read(rcq);
1609         if (srq)
1610                 uobj_put_obj_read(srq);
1611         if (ind_tbl)
1612                 uobj_put_obj_read(ind_tbl);
1613
1614         uobj_alloc_abort(&obj->uevent.uobject);
1615         return ret;
1616 }
1617
1618 static int ib_uverbs_create_qp_cb(struct ib_uverbs_file *file,
1619                                   struct ib_uverbs_ex_create_qp_resp *resp,
1620                                   struct ib_udata *ucore)
1621 {
1622         if (ib_copy_to_udata(ucore, &resp->base, sizeof(resp->base)))
1623                 return -EFAULT;
1624
1625         return 0;
1626 }
1627
1628 ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file,
1629                             struct ib_device *ib_dev,
1630                             const char __user *buf, int in_len,
1631                             int out_len)
1632 {
1633         struct ib_uverbs_create_qp      cmd;
1634         struct ib_uverbs_ex_create_qp   cmd_ex;
1635         struct ib_udata                 ucore;
1636         struct ib_udata                 uhw;
1637         ssize_t resp_size = sizeof(struct ib_uverbs_create_qp_resp);
1638         int                             err;
1639
1640         if (out_len < resp_size)
1641                 return -ENOSPC;
1642
1643         if (copy_from_user(&cmd, buf, sizeof(cmd)))
1644                 return -EFAULT;
1645
1646         ib_uverbs_init_udata(&ucore, buf, u64_to_user_ptr(cmd.response),
1647                    sizeof(cmd), resp_size);
1648         ib_uverbs_init_udata(&uhw, buf + sizeof(cmd),
1649                    u64_to_user_ptr(cmd.response) + resp_size,
1650                    in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
1651                    out_len - resp_size);
1652
1653         memset(&cmd_ex, 0, sizeof(cmd_ex));
1654         cmd_ex.user_handle = cmd.user_handle;
1655         cmd_ex.pd_handle = cmd.pd_handle;
1656         cmd_ex.send_cq_handle = cmd.send_cq_handle;
1657         cmd_ex.recv_cq_handle = cmd.recv_cq_handle;
1658         cmd_ex.srq_handle = cmd.srq_handle;
1659         cmd_ex.max_send_wr = cmd.max_send_wr;
1660         cmd_ex.max_recv_wr = cmd.max_recv_wr;
1661         cmd_ex.max_send_sge = cmd.max_send_sge;
1662         cmd_ex.max_recv_sge = cmd.max_recv_sge;
1663         cmd_ex.max_inline_data = cmd.max_inline_data;
1664         cmd_ex.sq_sig_all = cmd.sq_sig_all;
1665         cmd_ex.qp_type = cmd.qp_type;
1666         cmd_ex.is_srq = cmd.is_srq;
1667
1668         err = create_qp(file, &ucore, &uhw, &cmd_ex,
1669                         offsetof(typeof(cmd_ex), is_srq) +
1670                         sizeof(cmd.is_srq), ib_uverbs_create_qp_cb,
1671                         NULL);
1672
1673         if (err)
1674                 return err;
1675
1676         return in_len;
1677 }
1678
1679 static int ib_uverbs_ex_create_qp_cb(struct ib_uverbs_file *file,
1680                                      struct ib_uverbs_ex_create_qp_resp *resp,
1681                                      struct ib_udata *ucore)
1682 {
1683         if (ib_copy_to_udata(ucore, resp, resp->response_length))
1684                 return -EFAULT;
1685
1686         return 0;
1687 }
1688
1689 int ib_uverbs_ex_create_qp(struct ib_uverbs_file *file,
1690                            struct ib_device *ib_dev,
1691                            struct ib_udata *ucore,
1692                            struct ib_udata *uhw)
1693 {
1694         struct ib_uverbs_ex_create_qp_resp resp;
1695         struct ib_uverbs_ex_create_qp cmd = {0};
1696         int err;
1697
1698         if (ucore->inlen < (offsetof(typeof(cmd), comp_mask) +
1699                             sizeof(cmd.comp_mask)))
1700                 return -EINVAL;
1701
1702         err = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen));
1703         if (err)
1704                 return err;
1705
1706         if (cmd.comp_mask & ~IB_UVERBS_CREATE_QP_SUP_COMP_MASK)
1707                 return -EINVAL;
1708
1709         if (cmd.reserved)
1710                 return -EINVAL;
1711
1712         if (ucore->outlen < (offsetof(typeof(resp), response_length) +
1713                              sizeof(resp.response_length)))
1714                 return -ENOSPC;
1715
1716         err = create_qp(file, ucore, uhw, &cmd,
1717                         min(ucore->inlen, sizeof(cmd)),
1718                         ib_uverbs_ex_create_qp_cb, NULL);
1719
1720         if (err)
1721                 return err;
1722
1723         return 0;
1724 }
1725
1726 ssize_t ib_uverbs_open_qp(struct ib_uverbs_file *file,
1727                           struct ib_device *ib_dev,
1728                           const char __user *buf, int in_len, int out_len)
1729 {
1730         struct ib_uverbs_open_qp        cmd;
1731         struct ib_uverbs_create_qp_resp resp;
1732         struct ib_udata                 udata;
1733         struct ib_uqp_object           *obj;
1734         struct ib_xrcd                 *xrcd;
1735         struct ib_uobject              *uninitialized_var(xrcd_uobj);
1736         struct ib_qp                   *qp;
1737         struct ib_qp_open_attr          attr;
1738         int ret;
1739
1740         if (out_len < sizeof resp)
1741                 return -ENOSPC;
1742
1743         if (copy_from_user(&cmd, buf, sizeof cmd))
1744                 return -EFAULT;
1745
1746         ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
1747                    u64_to_user_ptr(cmd.response) + sizeof(resp),
1748                    in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
1749                    out_len - sizeof(resp));
1750
1751         obj  = (struct ib_uqp_object *)uobj_alloc(uobj_get_type(qp),
1752                                                   file->ucontext);
1753         if (IS_ERR(obj))
1754                 return PTR_ERR(obj);
1755
1756         xrcd_uobj = uobj_get_read(uobj_get_type(xrcd), cmd.pd_handle,
1757                                   file->ucontext);
1758         if (IS_ERR(xrcd_uobj)) {
1759                 ret = -EINVAL;
1760                 goto err_put;
1761         }
1762
1763         xrcd = (struct ib_xrcd *)xrcd_uobj->object;
1764         if (!xrcd) {
1765                 ret = -EINVAL;
1766                 goto err_xrcd;
1767         }
1768
1769         attr.event_handler = ib_uverbs_qp_event_handler;
1770         attr.qp_context    = file;
1771         attr.qp_num        = cmd.qpn;
1772         attr.qp_type       = cmd.qp_type;
1773
1774         obj->uevent.events_reported = 0;
1775         INIT_LIST_HEAD(&obj->uevent.event_list);
1776         INIT_LIST_HEAD(&obj->mcast_list);
1777
1778         qp = ib_open_qp(xrcd, &attr);
1779         if (IS_ERR(qp)) {
1780                 ret = PTR_ERR(qp);
1781                 goto err_xrcd;
1782         }
1783
1784         obj->uevent.uobject.object = qp;
1785         obj->uevent.uobject.user_handle = cmd.user_handle;
1786
1787         memset(&resp, 0, sizeof resp);
1788         resp.qpn       = qp->qp_num;
1789         resp.qp_handle = obj->uevent.uobject.id;
1790
1791         if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) {
1792                 ret = -EFAULT;
1793                 goto err_destroy;
1794         }
1795
1796         obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);
1797         atomic_inc(&obj->uxrcd->refcnt);
1798         qp->uobject = &obj->uevent.uobject;
1799         uobj_put_read(xrcd_uobj);
1800
1801
1802         uobj_alloc_commit(&obj->uevent.uobject);
1803
1804         return in_len;
1805
1806 err_destroy:
1807         ib_destroy_qp(qp);
1808 err_xrcd:
1809         uobj_put_read(xrcd_uobj);
1810 err_put:
1811         uobj_alloc_abort(&obj->uevent.uobject);
1812         return ret;
1813 }
1814
1815 static void copy_ah_attr_to_uverbs(struct ib_uverbs_qp_dest *uverb_attr,
1816                                    struct rdma_ah_attr *rdma_attr)
1817 {
1818         const struct ib_global_route   *grh;
1819
1820         uverb_attr->dlid              = rdma_ah_get_dlid(rdma_attr);
1821         uverb_attr->sl                = rdma_ah_get_sl(rdma_attr);
1822         uverb_attr->src_path_bits     = rdma_ah_get_path_bits(rdma_attr);
1823         uverb_attr->static_rate       = rdma_ah_get_static_rate(rdma_attr);
1824         uverb_attr->is_global         = !!(rdma_ah_get_ah_flags(rdma_attr) &
1825                                          IB_AH_GRH);
1826         if (uverb_attr->is_global) {
1827                 grh = rdma_ah_read_grh(rdma_attr);
1828                 memcpy(uverb_attr->dgid, grh->dgid.raw, 16);
1829                 uverb_attr->flow_label        = grh->flow_label;
1830                 uverb_attr->sgid_index        = grh->sgid_index;
1831                 uverb_attr->hop_limit         = grh->hop_limit;
1832                 uverb_attr->traffic_class     = grh->traffic_class;
1833         }
1834         uverb_attr->port_num          = rdma_ah_get_port_num(rdma_attr);
1835 }
1836
1837 ssize_t ib_uverbs_query_qp(struct ib_uverbs_file *file,
1838                            struct ib_device *ib_dev,
1839                            const char __user *buf, int in_len,
1840                            int out_len)
1841 {
1842         struct ib_uverbs_query_qp      cmd;
1843         struct ib_uverbs_query_qp_resp resp;
1844         struct ib_qp                   *qp;
1845         struct ib_qp_attr              *attr;
1846         struct ib_qp_init_attr         *init_attr;
1847         int                            ret;
1848
1849         if (copy_from_user(&cmd, buf, sizeof cmd))
1850                 return -EFAULT;
1851
1852         attr      = kmalloc(sizeof *attr, GFP_KERNEL);
1853         init_attr = kmalloc(sizeof *init_attr, GFP_KERNEL);
1854         if (!attr || !init_attr) {
1855                 ret = -ENOMEM;
1856                 goto out;
1857         }
1858
1859         qp = uobj_get_obj_read(qp, cmd.qp_handle, file->ucontext);
1860         if (!qp) {
1861                 ret = -EINVAL;
1862                 goto out;
1863         }
1864
1865         ret = ib_query_qp(qp, attr, cmd.attr_mask, init_attr);
1866
1867         uobj_put_obj_read(qp);
1868
1869         if (ret)
1870                 goto out;
1871
1872         memset(&resp, 0, sizeof resp);
1873
1874         resp.qp_state               = attr->qp_state;
1875         resp.cur_qp_state           = attr->cur_qp_state;
1876         resp.path_mtu               = attr->path_mtu;
1877         resp.path_mig_state         = attr->path_mig_state;
1878         resp.qkey                   = attr->qkey;
1879         resp.rq_psn                 = attr->rq_psn;
1880         resp.sq_psn                 = attr->sq_psn;
1881         resp.dest_qp_num            = attr->dest_qp_num;
1882         resp.qp_access_flags        = attr->qp_access_flags;
1883         resp.pkey_index             = attr->pkey_index;
1884         resp.alt_pkey_index         = attr->alt_pkey_index;
1885         resp.sq_draining            = attr->sq_draining;
1886         resp.max_rd_atomic          = attr->max_rd_atomic;
1887         resp.max_dest_rd_atomic     = attr->max_dest_rd_atomic;
1888         resp.min_rnr_timer          = attr->min_rnr_timer;
1889         resp.port_num               = attr->port_num;
1890         resp.timeout                = attr->timeout;
1891         resp.retry_cnt              = attr->retry_cnt;
1892         resp.rnr_retry              = attr->rnr_retry;
1893         resp.alt_port_num           = attr->alt_port_num;
1894         resp.alt_timeout            = attr->alt_timeout;
1895
1896         copy_ah_attr_to_uverbs(&resp.dest, &attr->ah_attr);
1897         copy_ah_attr_to_uverbs(&resp.alt_dest, &attr->alt_ah_attr);
1898
1899         resp.max_send_wr            = init_attr->cap.max_send_wr;
1900         resp.max_recv_wr            = init_attr->cap.max_recv_wr;
1901         resp.max_send_sge           = init_attr->cap.max_send_sge;
1902         resp.max_recv_sge           = init_attr->cap.max_recv_sge;
1903         resp.max_inline_data        = init_attr->cap.max_inline_data;
1904         resp.sq_sig_all             = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR;
1905
1906         if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp))
1907                 ret = -EFAULT;
1908
1909 out:
1910         kfree(attr);
1911         kfree(init_attr);
1912
1913         return ret ? ret : in_len;
1914 }
1915
1916 /* Remove ignored fields set in the attribute mask */
1917 static int modify_qp_mask(enum ib_qp_type qp_type, int mask)
1918 {
1919         switch (qp_type) {
1920         case IB_QPT_XRC_INI:
1921                 return mask & ~(IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER);
1922         case IB_QPT_XRC_TGT:
1923                 return mask & ~(IB_QP_MAX_QP_RD_ATOMIC | IB_QP_RETRY_CNT |
1924                                 IB_QP_RNR_RETRY);
1925         default:
1926                 return mask;
1927         }
1928 }
1929
1930 static void copy_ah_attr_from_uverbs(struct ib_device *dev,
1931                                      struct rdma_ah_attr *rdma_attr,
1932                                      struct ib_uverbs_qp_dest *uverb_attr)
1933 {
1934         rdma_attr->type = rdma_ah_find_type(dev, uverb_attr->port_num);
1935         if (uverb_attr->is_global) {
1936                 rdma_ah_set_grh(rdma_attr, NULL,
1937                                 uverb_attr->flow_label,
1938                                 uverb_attr->sgid_index,
1939                                 uverb_attr->hop_limit,
1940                                 uverb_attr->traffic_class);
1941                 rdma_ah_set_dgid_raw(rdma_attr, uverb_attr->dgid);
1942         } else {
1943                 rdma_ah_set_ah_flags(rdma_attr, 0);
1944         }
1945         rdma_ah_set_dlid(rdma_attr, uverb_attr->dlid);
1946         rdma_ah_set_sl(rdma_attr, uverb_attr->sl);
1947         rdma_ah_set_path_bits(rdma_attr, uverb_attr->src_path_bits);
1948         rdma_ah_set_static_rate(rdma_attr, uverb_attr->static_rate);
1949         rdma_ah_set_port_num(rdma_attr, uverb_attr->port_num);
1950         rdma_ah_set_make_grd(rdma_attr, false);
1951 }
1952
1953 static int modify_qp(struct ib_uverbs_file *file,
1954                      struct ib_uverbs_ex_modify_qp *cmd, struct ib_udata *udata)
1955 {
1956         struct ib_qp_attr *attr;
1957         struct ib_qp *qp;
1958         int ret;
1959
1960         attr = kmalloc(sizeof *attr, GFP_KERNEL);
1961         if (!attr)
1962                 return -ENOMEM;
1963
1964         qp = uobj_get_obj_read(qp, cmd->base.qp_handle, file->ucontext);
1965         if (!qp) {
1966                 ret = -EINVAL;
1967                 goto out;
1968         }
1969
1970         if ((cmd->base.attr_mask & IB_QP_PORT) &&
1971             !rdma_is_port_valid(qp->device, cmd->base.port_num)) {
1972                 ret = -EINVAL;
1973                 goto release_qp;
1974         }
1975
1976         if ((cmd->base.attr_mask & IB_QP_AV) &&
1977             !rdma_is_port_valid(qp->device, cmd->base.dest.port_num)) {
1978                 ret = -EINVAL;
1979                 goto release_qp;
1980         }
1981
1982         if ((cmd->base.attr_mask & IB_QP_ALT_PATH) &&
1983             (!rdma_is_port_valid(qp->device, cmd->base.alt_port_num) ||
1984             !rdma_is_port_valid(qp->device, cmd->base.alt_dest.port_num))) {
1985                 ret = -EINVAL;
1986                 goto release_qp;
1987         }
1988
1989         attr->qp_state            = cmd->base.qp_state;
1990         attr->cur_qp_state        = cmd->base.cur_qp_state;
1991         attr->path_mtu            = cmd->base.path_mtu;
1992         attr->path_mig_state      = cmd->base.path_mig_state;
1993         attr->qkey                = cmd->base.qkey;
1994         attr->rq_psn              = cmd->base.rq_psn;
1995         attr->sq_psn              = cmd->base.sq_psn;
1996         attr->dest_qp_num         = cmd->base.dest_qp_num;
1997         attr->qp_access_flags     = cmd->base.qp_access_flags;
1998         attr->pkey_index          = cmd->base.pkey_index;
1999         attr->alt_pkey_index      = cmd->base.alt_pkey_index;
2000         attr->en_sqd_async_notify = cmd->base.en_sqd_async_notify;
2001         attr->max_rd_atomic       = cmd->base.max_rd_atomic;
2002         attr->max_dest_rd_atomic  = cmd->base.max_dest_rd_atomic;
2003         attr->min_rnr_timer       = cmd->base.min_rnr_timer;
2004         attr->port_num            = cmd->base.port_num;
2005         attr->timeout             = cmd->base.timeout;
2006         attr->retry_cnt           = cmd->base.retry_cnt;
2007         attr->rnr_retry           = cmd->base.rnr_retry;
2008         attr->alt_port_num        = cmd->base.alt_port_num;
2009         attr->alt_timeout         = cmd->base.alt_timeout;
2010         attr->rate_limit          = cmd->rate_limit;
2011
2012         if (cmd->base.attr_mask & IB_QP_AV)
2013                 copy_ah_attr_from_uverbs(qp->device, &attr->ah_attr,
2014                                          &cmd->base.dest);
2015
2016         if (cmd->base.attr_mask & IB_QP_ALT_PATH)
2017                 copy_ah_attr_from_uverbs(qp->device, &attr->alt_ah_attr,
2018                                          &cmd->base.alt_dest);
2019
2020         ret = ib_modify_qp_with_udata(qp, attr,
2021                                       modify_qp_mask(qp->qp_type,
2022                                                      cmd->base.attr_mask),
2023                                       udata);
2024
2025 release_qp:
2026         uobj_put_obj_read(qp);
2027 out:
2028         kfree(attr);
2029
2030         return ret;
2031 }
2032
2033 ssize_t ib_uverbs_modify_qp(struct ib_uverbs_file *file,
2034                             struct ib_device *ib_dev,
2035                             const char __user *buf, int in_len,
2036                             int out_len)
2037 {
2038         struct ib_uverbs_ex_modify_qp cmd = {};
2039         struct ib_udata udata;
2040         int ret;
2041
2042         if (copy_from_user(&cmd.base, buf, sizeof(cmd.base)))
2043                 return -EFAULT;
2044
2045         if (cmd.base.attr_mask &
2046             ~((IB_USER_LEGACY_LAST_QP_ATTR_MASK << 1) - 1))
2047                 return -EOPNOTSUPP;
2048
2049         ib_uverbs_init_udata(&udata, buf + sizeof(cmd.base), NULL,
2050                    in_len - sizeof(cmd.base) - sizeof(struct ib_uverbs_cmd_hdr),
2051                    out_len);
2052
2053         ret = modify_qp(file, &cmd, &udata);
2054         if (ret)
2055                 return ret;
2056
2057         return in_len;
2058 }
2059
2060 int ib_uverbs_ex_modify_qp(struct ib_uverbs_file *file,
2061                            struct ib_device *ib_dev,
2062                            struct ib_udata *ucore,
2063                            struct ib_udata *uhw)
2064 {
2065         struct ib_uverbs_ex_modify_qp cmd = {};
2066         int ret;
2067
2068         /*
2069          * Last bit is reserved for extending the attr_mask by
2070          * using another field.
2071          */
2072         BUILD_BUG_ON(IB_USER_LAST_QP_ATTR_MASK == (1 << 31));
2073
2074         if (ucore->inlen < sizeof(cmd.base))
2075                 return -EINVAL;
2076
2077         ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen));
2078         if (ret)
2079                 return ret;
2080
2081         if (cmd.base.attr_mask &
2082             ~((IB_USER_LAST_QP_ATTR_MASK << 1) - 1))
2083                 return -EOPNOTSUPP;
2084
2085         if (ucore->inlen > sizeof(cmd)) {
2086                 if (!ib_is_udata_cleared(ucore, sizeof(cmd),
2087                                          ucore->inlen - sizeof(cmd)))
2088                         return -EOPNOTSUPP;
2089         }
2090
2091         ret = modify_qp(file, &cmd, uhw);
2092
2093         return ret;
2094 }
2095
2096 ssize_t ib_uverbs_destroy_qp(struct ib_uverbs_file *file,
2097                              struct ib_device *ib_dev,
2098                              const char __user *buf, int in_len,
2099                              int out_len)
2100 {
2101         struct ib_uverbs_destroy_qp      cmd;
2102         struct ib_uverbs_destroy_qp_resp resp;
2103         struct ib_uobject               *uobj;
2104         struct ib_uqp_object            *obj;
2105         int                              ret = -EINVAL;
2106
2107         if (copy_from_user(&cmd, buf, sizeof cmd))
2108                 return -EFAULT;
2109
2110         memset(&resp, 0, sizeof resp);
2111
2112         uobj  = uobj_get_write(uobj_get_type(qp), cmd.qp_handle,
2113                                file->ucontext);
2114         if (IS_ERR(uobj))
2115                 return PTR_ERR(uobj);
2116
2117         obj = container_of(uobj, struct ib_uqp_object, uevent.uobject);
2118         /*
2119          * Make sure we don't free the memory in remove_commit as we still
2120          * needs the uobject memory to create the response.
2121          */
2122         uverbs_uobject_get(uobj);
2123
2124         ret = uobj_remove_commit(uobj);
2125         if (ret) {
2126                 uverbs_uobject_put(uobj);
2127                 return ret;
2128         }
2129
2130         resp.events_reported = obj->uevent.events_reported;
2131         uverbs_uobject_put(uobj);
2132
2133         if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp))
2134                 return -EFAULT;
2135
2136         return in_len;
2137 }
2138
2139 static void *alloc_wr(size_t wr_size, __u32 num_sge)
2140 {
2141         if (num_sge >= (U32_MAX - ALIGN(wr_size, sizeof (struct ib_sge))) /
2142                        sizeof (struct ib_sge))
2143                 return NULL;
2144
2145         return kmalloc(ALIGN(wr_size, sizeof (struct ib_sge)) +
2146                          num_sge * sizeof (struct ib_sge), GFP_KERNEL);
2147 }
2148
2149 ssize_t ib_uverbs_post_send(struct ib_uverbs_file *file,
2150                             struct ib_device *ib_dev,
2151                             const char __user *buf, int in_len,
2152                             int out_len)
2153 {
2154         struct ib_uverbs_post_send      cmd;
2155         struct ib_uverbs_post_send_resp resp;
2156         struct ib_uverbs_send_wr       *user_wr;
2157         struct ib_send_wr              *wr = NULL, *last, *next, *bad_wr;
2158         struct ib_qp                   *qp;
2159         int                             i, sg_ind;
2160         int                             is_ud;
2161         ssize_t                         ret = -EINVAL;
2162         size_t                          next_size;
2163
2164         if (copy_from_user(&cmd, buf, sizeof cmd))
2165                 return -EFAULT;
2166
2167         if (in_len < sizeof cmd + cmd.wqe_size * cmd.wr_count +
2168             cmd.sge_count * sizeof (struct ib_uverbs_sge))
2169                 return -EINVAL;
2170
2171         if (cmd.wqe_size < sizeof (struct ib_uverbs_send_wr))
2172                 return -EINVAL;
2173
2174         user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL);
2175         if (!user_wr)
2176                 return -ENOMEM;
2177
2178         qp = uobj_get_obj_read(qp, cmd.qp_handle, file->ucontext);
2179         if (!qp)
2180                 goto out;
2181
2182         is_ud = qp->qp_type == IB_QPT_UD;
2183         sg_ind = 0;
2184         last = NULL;
2185         for (i = 0; i < cmd.wr_count; ++i) {
2186                 if (copy_from_user(user_wr,
2187                                    buf + sizeof cmd + i * cmd.wqe_size,
2188                                    cmd.wqe_size)) {
2189                         ret = -EFAULT;
2190                         goto out_put;
2191                 }
2192
2193                 if (user_wr->num_sge + sg_ind > cmd.sge_count) {
2194                         ret = -EINVAL;
2195                         goto out_put;
2196                 }
2197
2198                 if (is_ud) {
2199                         struct ib_ud_wr *ud;
2200
2201                         if (user_wr->opcode != IB_WR_SEND &&
2202                             user_wr->opcode != IB_WR_SEND_WITH_IMM) {
2203                                 ret = -EINVAL;
2204                                 goto out_put;
2205                         }
2206
2207                         next_size = sizeof(*ud);
2208                         ud = alloc_wr(next_size, user_wr->num_sge);
2209                         if (!ud) {
2210                                 ret = -ENOMEM;
2211                                 goto out_put;
2212                         }
2213
2214                         ud->ah = uobj_get_obj_read(ah, user_wr->wr.ud.ah,
2215                                                    file->ucontext);
2216                         if (!ud->ah) {
2217                                 kfree(ud);
2218                                 ret = -EINVAL;
2219                                 goto out_put;
2220                         }
2221                         ud->remote_qpn = user_wr->wr.ud.remote_qpn;
2222                         ud->remote_qkey = user_wr->wr.ud.remote_qkey;
2223
2224                         next = &ud->wr;
2225                 } else if (user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
2226                            user_wr->opcode == IB_WR_RDMA_WRITE ||
2227                            user_wr->opcode == IB_WR_RDMA_READ) {
2228                         struct ib_rdma_wr *rdma;
2229
2230                         next_size = sizeof(*rdma);
2231                         rdma = alloc_wr(next_size, user_wr->num_sge);
2232                         if (!rdma) {
2233                                 ret = -ENOMEM;
2234                                 goto out_put;
2235                         }
2236
2237                         rdma->remote_addr = user_wr->wr.rdma.remote_addr;
2238                         rdma->rkey = user_wr->wr.rdma.rkey;
2239
2240                         next = &rdma->wr;
2241                 } else if (user_wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
2242                            user_wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
2243                         struct ib_atomic_wr *atomic;
2244
2245                         next_size = sizeof(*atomic);
2246                         atomic = alloc_wr(next_size, user_wr->num_sge);
2247                         if (!atomic) {
2248                                 ret = -ENOMEM;
2249                                 goto out_put;
2250                         }
2251
2252                         atomic->remote_addr = user_wr->wr.atomic.remote_addr;
2253                         atomic->compare_add = user_wr->wr.atomic.compare_add;
2254                         atomic->swap = user_wr->wr.atomic.swap;
2255                         atomic->rkey = user_wr->wr.atomic.rkey;
2256
2257                         next = &atomic->wr;
2258                 } else if (user_wr->opcode == IB_WR_SEND ||
2259                            user_wr->opcode == IB_WR_SEND_WITH_IMM ||
2260                            user_wr->opcode == IB_WR_SEND_WITH_INV) {
2261                         next_size = sizeof(*next);
2262                         next = alloc_wr(next_size, user_wr->num_sge);
2263                         if (!next) {
2264                                 ret = -ENOMEM;
2265                                 goto out_put;
2266                         }
2267                 } else {
2268                         ret = -EINVAL;
2269                         goto out_put;
2270                 }
2271
2272                 if (user_wr->opcode == IB_WR_SEND_WITH_IMM ||
2273                     user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) {
2274                         next->ex.imm_data =
2275                                         (__be32 __force) user_wr->ex.imm_data;
2276                 } else if (user_wr->opcode == IB_WR_SEND_WITH_INV) {
2277                         next->ex.invalidate_rkey = user_wr->ex.invalidate_rkey;
2278                 }
2279
2280                 if (!last)
2281                         wr = next;
2282                 else
2283                         last->next = next;
2284                 last = next;
2285
2286                 next->next       = NULL;
2287                 next->wr_id      = user_wr->wr_id;
2288                 next->num_sge    = user_wr->num_sge;
2289                 next->opcode     = user_wr->opcode;
2290                 next->send_flags = user_wr->send_flags;
2291
2292                 if (next->num_sge) {
2293                         next->sg_list = (void *) next +
2294                                 ALIGN(next_size, sizeof(struct ib_sge));
2295                         if (copy_from_user(next->sg_list,
2296                                            buf + sizeof cmd +
2297                                            cmd.wr_count * cmd.wqe_size +
2298                                            sg_ind * sizeof (struct ib_sge),
2299                                            next->num_sge * sizeof (struct ib_sge))) {
2300                                 ret = -EFAULT;
2301                                 goto out_put;
2302                         }
2303                         sg_ind += next->num_sge;
2304                 } else
2305                         next->sg_list = NULL;
2306         }
2307
2308         resp.bad_wr = 0;
2309         ret = qp->device->post_send(qp->real_qp, wr, &bad_wr);
2310         if (ret)
2311                 for (next = wr; next; next = next->next) {
2312                         ++resp.bad_wr;
2313                         if (next == bad_wr)
2314                                 break;
2315                 }
2316
2317         if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp))
2318                 ret = -EFAULT;
2319
2320 out_put:
2321         uobj_put_obj_read(qp);
2322
2323         while (wr) {
2324                 if (is_ud && ud_wr(wr)->ah)
2325                         uobj_put_obj_read(ud_wr(wr)->ah);
2326                 next = wr->next;
2327                 kfree(wr);
2328                 wr = next;
2329         }
2330
2331 out:
2332         kfree(user_wr);
2333
2334         return ret ? ret : in_len;
2335 }
2336
2337 static struct ib_recv_wr *ib_uverbs_unmarshall_recv(const char __user *buf,
2338                                                     int in_len,
2339                                                     u32 wr_count,
2340                                                     u32 sge_count,
2341                                                     u32 wqe_size)
2342 {
2343         struct ib_uverbs_recv_wr *user_wr;
2344         struct ib_recv_wr        *wr = NULL, *last, *next;
2345         int                       sg_ind;
2346         int                       i;
2347         int                       ret;
2348
2349         if (in_len < wqe_size * wr_count +
2350             sge_count * sizeof (struct ib_uverbs_sge))
2351                 return ERR_PTR(-EINVAL);
2352
2353         if (wqe_size < sizeof (struct ib_uverbs_recv_wr))
2354                 return ERR_PTR(-EINVAL);
2355
2356         user_wr = kmalloc(wqe_size, GFP_KERNEL);
2357         if (!user_wr)
2358                 return ERR_PTR(-ENOMEM);
2359
2360         sg_ind = 0;
2361         last = NULL;
2362         for (i = 0; i < wr_count; ++i) {
2363                 if (copy_from_user(user_wr, buf + i * wqe_size,
2364                                    wqe_size)) {
2365                         ret = -EFAULT;
2366                         goto err;
2367                 }
2368
2369                 if (user_wr->num_sge + sg_ind > sge_count) {
2370                         ret = -EINVAL;
2371                         goto err;
2372                 }
2373
2374                 if (user_wr->num_sge >=
2375                     (U32_MAX - ALIGN(sizeof *next, sizeof (struct ib_sge))) /
2376                     sizeof (struct ib_sge)) {
2377                         ret = -EINVAL;
2378                         goto err;
2379                 }
2380
2381                 next = kmalloc(ALIGN(sizeof *next, sizeof (struct ib_sge)) +
2382                                user_wr->num_sge * sizeof (struct ib_sge),
2383                                GFP_KERNEL);
2384                 if (!next) {
2385                         ret = -ENOMEM;
2386                         goto err;
2387                 }
2388
2389                 if (!last)
2390                         wr = next;
2391                 else
2392                         last->next = next;
2393                 last = next;
2394
2395                 next->next       = NULL;
2396                 next->wr_id      = user_wr->wr_id;
2397                 next->num_sge    = user_wr->num_sge;
2398
2399                 if (next->num_sge) {
2400                         next->sg_list = (void *) next +
2401                                 ALIGN(sizeof *next, sizeof (struct ib_sge));
2402                         if (copy_from_user(next->sg_list,
2403                                            buf + wr_count * wqe_size +
2404                                            sg_ind * sizeof (struct ib_sge),
2405                                            next->num_sge * sizeof (struct ib_sge))) {
2406                                 ret = -EFAULT;
2407                                 goto err;
2408                         }
2409                         sg_ind += next->num_sge;
2410                 } else
2411                         next->sg_list = NULL;
2412         }
2413
2414         kfree(user_wr);
2415         return wr;
2416
2417 err:
2418         kfree(user_wr);
2419
2420         while (wr) {
2421                 next = wr->next;
2422                 kfree(wr);
2423                 wr = next;
2424         }
2425
2426         return ERR_PTR(ret);
2427 }
2428
2429 ssize_t ib_uverbs_post_recv(struct ib_uverbs_file *file,
2430                             struct ib_device *ib_dev,
2431                             const char __user *buf, int in_len,
2432                             int out_len)
2433 {
2434         struct ib_uverbs_post_recv      cmd;
2435         struct ib_uverbs_post_recv_resp resp;
2436         struct ib_recv_wr              *wr, *next, *bad_wr;
2437         struct ib_qp                   *qp;
2438         ssize_t                         ret = -EINVAL;
2439
2440         if (copy_from_user(&cmd, buf, sizeof cmd))
2441                 return -EFAULT;
2442
2443         wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd,
2444                                        in_len - sizeof cmd, cmd.wr_count,
2445                                        cmd.sge_count, cmd.wqe_size);
2446         if (IS_ERR(wr))
2447                 return PTR_ERR(wr);
2448
2449         qp = uobj_get_obj_read(qp, cmd.qp_handle, file->ucontext);
2450         if (!qp)
2451                 goto out;
2452
2453         resp.bad_wr = 0;
2454         ret = qp->device->post_recv(qp->real_qp, wr, &bad_wr);
2455
2456         uobj_put_obj_read(qp);
2457         if (ret) {
2458                 for (next = wr; next; next = next->next) {
2459                         ++resp.bad_wr;
2460                         if (next == bad_wr)
2461                                 break;
2462                 }
2463         }
2464
2465         if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp))
2466                 ret = -EFAULT;
2467
2468 out:
2469         while (wr) {
2470                 next = wr->next;
2471                 kfree(wr);
2472                 wr = next;
2473         }
2474
2475         return ret ? ret : in_len;
2476 }
2477
2478 ssize_t ib_uverbs_post_srq_recv(struct ib_uverbs_file *file,
2479                                 struct ib_device *ib_dev,
2480                                 const char __user *buf, int in_len,
2481                                 int out_len)
2482 {
2483         struct ib_uverbs_post_srq_recv      cmd;
2484         struct ib_uverbs_post_srq_recv_resp resp;
2485         struct ib_recv_wr                  *wr, *next, *bad_wr;
2486         struct ib_srq                      *srq;
2487         ssize_t                             ret = -EINVAL;
2488
2489         if (copy_from_user(&cmd, buf, sizeof cmd))
2490                 return -EFAULT;
2491
2492         wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd,
2493                                        in_len - sizeof cmd, cmd.wr_count,
2494                                        cmd.sge_count, cmd.wqe_size);
2495         if (IS_ERR(wr))
2496                 return PTR_ERR(wr);
2497
2498         srq = uobj_get_obj_read(srq, cmd.srq_handle, file->ucontext);
2499         if (!srq)
2500                 goto out;
2501
2502         resp.bad_wr = 0;
2503         ret = srq->device->post_srq_recv(srq, wr, &bad_wr);
2504
2505         uobj_put_obj_read(srq);
2506
2507         if (ret)
2508                 for (next = wr; next; next = next->next) {
2509                         ++resp.bad_wr;
2510                         if (next == bad_wr)
2511                                 break;
2512                 }
2513
2514         if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp))
2515                 ret = -EFAULT;
2516
2517 out:
2518         while (wr) {
2519                 next = wr->next;
2520                 kfree(wr);
2521                 wr = next;
2522         }
2523
2524         return ret ? ret : in_len;
2525 }
2526
2527 ssize_t ib_uverbs_create_ah(struct ib_uverbs_file *file,
2528                             struct ib_device *ib_dev,
2529                             const char __user *buf, int in_len,
2530                             int out_len)
2531 {
2532         struct ib_uverbs_create_ah       cmd;
2533         struct ib_uverbs_create_ah_resp  resp;
2534         struct ib_uobject               *uobj;
2535         struct ib_pd                    *pd;
2536         struct ib_ah                    *ah;
2537         struct rdma_ah_attr             attr;
2538         int ret;
2539         struct ib_udata                   udata;
2540
2541         if (out_len < sizeof resp)
2542                 return -ENOSPC;
2543
2544         if (copy_from_user(&cmd, buf, sizeof cmd))
2545                 return -EFAULT;
2546
2547         if (!rdma_is_port_valid(ib_dev, cmd.attr.port_num))
2548                 return -EINVAL;
2549
2550         ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
2551                    u64_to_user_ptr(cmd.response) + sizeof(resp),
2552                    in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
2553                    out_len - sizeof(resp));
2554
2555         uobj  = uobj_alloc(uobj_get_type(ah), file->ucontext);
2556         if (IS_ERR(uobj))
2557                 return PTR_ERR(uobj);
2558
2559         pd = uobj_get_obj_read(pd, cmd.pd_handle, file->ucontext);
2560         if (!pd) {
2561                 ret = -EINVAL;
2562                 goto err;
2563         }
2564
2565         attr.type = rdma_ah_find_type(ib_dev, cmd.attr.port_num);
2566         rdma_ah_set_make_grd(&attr, false);
2567         rdma_ah_set_dlid(&attr, cmd.attr.dlid);
2568         rdma_ah_set_sl(&attr, cmd.attr.sl);
2569         rdma_ah_set_path_bits(&attr, cmd.attr.src_path_bits);
2570         rdma_ah_set_static_rate(&attr, cmd.attr.static_rate);
2571         rdma_ah_set_port_num(&attr, cmd.attr.port_num);
2572
2573         if (cmd.attr.is_global) {
2574                 rdma_ah_set_grh(&attr, NULL, cmd.attr.grh.flow_label,
2575                                 cmd.attr.grh.sgid_index,
2576                                 cmd.attr.grh.hop_limit,
2577                                 cmd.attr.grh.traffic_class);
2578                 rdma_ah_set_dgid_raw(&attr, cmd.attr.grh.dgid);
2579         } else {
2580                 rdma_ah_set_ah_flags(&attr, 0);
2581         }
2582
2583         ah = rdma_create_user_ah(pd, &attr, &udata);
2584         if (IS_ERR(ah)) {
2585                 ret = PTR_ERR(ah);
2586                 goto err_put;
2587         }
2588
2589         ah->uobject  = uobj;
2590         uobj->user_handle = cmd.user_handle;
2591         uobj->object = ah;
2592
2593         resp.ah_handle = uobj->id;
2594
2595         if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) {
2596                 ret = -EFAULT;
2597                 goto err_copy;
2598         }
2599
2600         uobj_put_obj_read(pd);
2601         uobj_alloc_commit(uobj);
2602
2603         return in_len;
2604
2605 err_copy:
2606         rdma_destroy_ah(ah);
2607
2608 err_put:
2609         uobj_put_obj_read(pd);
2610
2611 err:
2612         uobj_alloc_abort(uobj);
2613         return ret;
2614 }
2615
2616 ssize_t ib_uverbs_destroy_ah(struct ib_uverbs_file *file,
2617                              struct ib_device *ib_dev,
2618                              const char __user *buf, int in_len, int out_len)
2619 {
2620         struct ib_uverbs_destroy_ah cmd;
2621         struct ib_uobject          *uobj;
2622         int                         ret;
2623
2624         if (copy_from_user(&cmd, buf, sizeof cmd))
2625                 return -EFAULT;
2626
2627         uobj  = uobj_get_write(uobj_get_type(ah), cmd.ah_handle,
2628                                file->ucontext);
2629         if (IS_ERR(uobj))
2630                 return PTR_ERR(uobj);
2631
2632         ret = uobj_remove_commit(uobj);
2633         return ret ?: in_len;
2634 }
2635
2636 ssize_t ib_uverbs_attach_mcast(struct ib_uverbs_file *file,
2637                                struct ib_device *ib_dev,
2638                                const char __user *buf, int in_len,
2639                                int out_len)
2640 {
2641         struct ib_uverbs_attach_mcast cmd;
2642         struct ib_qp                 *qp;
2643         struct ib_uqp_object         *obj;
2644         struct ib_uverbs_mcast_entry *mcast;
2645         int                           ret;
2646
2647         if (copy_from_user(&cmd, buf, sizeof cmd))
2648                 return -EFAULT;
2649
2650         qp = uobj_get_obj_read(qp, cmd.qp_handle, file->ucontext);
2651         if (!qp)
2652                 return -EINVAL;
2653
2654         obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject);
2655
2656         mutex_lock(&obj->mcast_lock);
2657         list_for_each_entry(mcast, &obj->mcast_list, list)
2658                 if (cmd.mlid == mcast->lid &&
2659                     !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) {
2660                         ret = 0;
2661                         goto out_put;
2662                 }
2663
2664         mcast = kmalloc(sizeof *mcast, GFP_KERNEL);
2665         if (!mcast) {
2666                 ret = -ENOMEM;
2667                 goto out_put;
2668         }
2669
2670         mcast->lid = cmd.mlid;
2671         memcpy(mcast->gid.raw, cmd.gid, sizeof mcast->gid.raw);
2672
2673         ret = ib_attach_mcast(qp, &mcast->gid, cmd.mlid);
2674         if (!ret)
2675                 list_add_tail(&mcast->list, &obj->mcast_list);
2676         else
2677                 kfree(mcast);
2678
2679 out_put:
2680         mutex_unlock(&obj->mcast_lock);
2681         uobj_put_obj_read(qp);
2682
2683         return ret ? ret : in_len;
2684 }
2685
2686 ssize_t ib_uverbs_detach_mcast(struct ib_uverbs_file *file,
2687                                struct ib_device *ib_dev,
2688                                const char __user *buf, int in_len,
2689                                int out_len)
2690 {
2691         struct ib_uverbs_detach_mcast cmd;
2692         struct ib_uqp_object         *obj;
2693         struct ib_qp                 *qp;
2694         struct ib_uverbs_mcast_entry *mcast;
2695         int                           ret = -EINVAL;
2696         bool                          found = false;
2697
2698         if (copy_from_user(&cmd, buf, sizeof cmd))
2699                 return -EFAULT;
2700
2701         qp = uobj_get_obj_read(qp, cmd.qp_handle, file->ucontext);
2702         if (!qp)
2703                 return -EINVAL;
2704
2705         obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject);
2706         mutex_lock(&obj->mcast_lock);
2707
2708         list_for_each_entry(mcast, &obj->mcast_list, list)
2709                 if (cmd.mlid == mcast->lid &&
2710                     !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) {
2711                         list_del(&mcast->list);
2712                         kfree(mcast);
2713                         found = true;
2714                         break;
2715                 }
2716
2717         if (!found) {
2718                 ret = -EINVAL;
2719                 goto out_put;
2720         }
2721
2722         ret = ib_detach_mcast(qp, (union ib_gid *)cmd.gid, cmd.mlid);
2723
2724 out_put:
2725         mutex_unlock(&obj->mcast_lock);
2726         uobj_put_obj_read(qp);
2727         return ret ? ret : in_len;
2728 }
2729
2730 static int kern_spec_to_ib_spec_action(struct ib_uverbs_flow_spec *kern_spec,
2731                                        union ib_flow_spec *ib_spec)
2732 {
2733         ib_spec->type = kern_spec->type;
2734         switch (ib_spec->type) {
2735         case IB_FLOW_SPEC_ACTION_TAG:
2736                 if (kern_spec->flow_tag.size !=
2737                     sizeof(struct ib_uverbs_flow_spec_action_tag))
2738                         return -EINVAL;
2739
2740                 ib_spec->flow_tag.size = sizeof(struct ib_flow_spec_action_tag);
2741                 ib_spec->flow_tag.tag_id = kern_spec->flow_tag.tag_id;
2742                 break;
2743         case IB_FLOW_SPEC_ACTION_DROP:
2744                 if (kern_spec->drop.size !=
2745                     sizeof(struct ib_uverbs_flow_spec_action_drop))
2746                         return -EINVAL;
2747
2748                 ib_spec->drop.size = sizeof(struct ib_flow_spec_action_drop);
2749                 break;
2750         default:
2751                 return -EINVAL;
2752         }
2753         return 0;
2754 }
2755
2756 static size_t kern_spec_filter_sz(struct ib_uverbs_flow_spec_hdr *spec)
2757 {
2758         /* Returns user space filter size, includes padding */
2759         return (spec->size - sizeof(struct ib_uverbs_flow_spec_hdr)) / 2;
2760 }
2761
2762 static ssize_t spec_filter_size(void *kern_spec_filter, u16 kern_filter_size,
2763                                 u16 ib_real_filter_sz)
2764 {
2765         /*
2766          * User space filter structures must be 64 bit aligned, otherwise this
2767          * may pass, but we won't handle additional new attributes.
2768          */
2769
2770         if (kern_filter_size > ib_real_filter_sz) {
2771                 if (memchr_inv(kern_spec_filter +
2772                                ib_real_filter_sz, 0,
2773                                kern_filter_size - ib_real_filter_sz))
2774                         return -EINVAL;
2775                 return ib_real_filter_sz;
2776         }
2777         return kern_filter_size;
2778 }
2779
2780 static int kern_spec_to_ib_spec_filter(struct ib_uverbs_flow_spec *kern_spec,
2781                                        union ib_flow_spec *ib_spec)
2782 {
2783         ssize_t actual_filter_sz;
2784         ssize_t kern_filter_sz;
2785         ssize_t ib_filter_sz;
2786         void *kern_spec_mask;
2787         void *kern_spec_val;
2788
2789         if (kern_spec->reserved)
2790                 return -EINVAL;
2791
2792         ib_spec->type = kern_spec->type;
2793
2794         kern_filter_sz = kern_spec_filter_sz(&kern_spec->hdr);
2795         /* User flow spec size must be aligned to 4 bytes */
2796         if (kern_filter_sz != ALIGN(kern_filter_sz, 4))
2797                 return -EINVAL;
2798
2799         kern_spec_val = (void *)kern_spec +
2800                 sizeof(struct ib_uverbs_flow_spec_hdr);
2801         kern_spec_mask = kern_spec_val + kern_filter_sz;
2802         if (ib_spec->type == (IB_FLOW_SPEC_INNER | IB_FLOW_SPEC_VXLAN_TUNNEL))
2803                 return -EINVAL;
2804
2805         switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) {
2806         case IB_FLOW_SPEC_ETH:
2807                 ib_filter_sz = offsetof(struct ib_flow_eth_filter, real_sz);
2808                 actual_filter_sz = spec_filter_size(kern_spec_mask,
2809                                                     kern_filter_sz,
2810                                                     ib_filter_sz);
2811                 if (actual_filter_sz <= 0)
2812                         return -EINVAL;
2813                 ib_spec->size = sizeof(struct ib_flow_spec_eth);
2814                 memcpy(&ib_spec->eth.val, kern_spec_val, actual_filter_sz);
2815                 memcpy(&ib_spec->eth.mask, kern_spec_mask, actual_filter_sz);
2816                 break;
2817         case IB_FLOW_SPEC_IPV4:
2818                 ib_filter_sz = offsetof(struct ib_flow_ipv4_filter, real_sz);
2819                 actual_filter_sz = spec_filter_size(kern_spec_mask,
2820                                                     kern_filter_sz,
2821                                                     ib_filter_sz);
2822                 if (actual_filter_sz <= 0)
2823                         return -EINVAL;
2824                 ib_spec->size = sizeof(struct ib_flow_spec_ipv4);
2825                 memcpy(&ib_spec->ipv4.val, kern_spec_val, actual_filter_sz);
2826                 memcpy(&ib_spec->ipv4.mask, kern_spec_mask, actual_filter_sz);
2827                 break;
2828         case IB_FLOW_SPEC_IPV6:
2829                 ib_filter_sz = offsetof(struct ib_flow_ipv6_filter, real_sz);
2830                 actual_filter_sz = spec_filter_size(kern_spec_mask,
2831                                                     kern_filter_sz,
2832                                                     ib_filter_sz);
2833                 if (actual_filter_sz <= 0)
2834                         return -EINVAL;
2835                 ib_spec->size = sizeof(struct ib_flow_spec_ipv6);
2836                 memcpy(&ib_spec->ipv6.val, kern_spec_val, actual_filter_sz);
2837                 memcpy(&ib_spec->ipv6.mask, kern_spec_mask, actual_filter_sz);
2838
2839                 if ((ntohl(ib_spec->ipv6.mask.flow_label)) >= BIT(20) ||
2840                     (ntohl(ib_spec->ipv6.val.flow_label)) >= BIT(20))
2841                         return -EINVAL;
2842                 break;
2843         case IB_FLOW_SPEC_TCP:
2844         case IB_FLOW_SPEC_UDP:
2845                 ib_filter_sz = offsetof(struct ib_flow_tcp_udp_filter, real_sz);
2846                 actual_filter_sz = spec_filter_size(kern_spec_mask,
2847                                                     kern_filter_sz,
2848                                                     ib_filter_sz);
2849                 if (actual_filter_sz <= 0)
2850                         return -EINVAL;
2851                 ib_spec->size = sizeof(struct ib_flow_spec_tcp_udp);
2852                 memcpy(&ib_spec->tcp_udp.val, kern_spec_val, actual_filter_sz);
2853                 memcpy(&ib_spec->tcp_udp.mask, kern_spec_mask, actual_filter_sz);
2854                 break;
2855         case IB_FLOW_SPEC_VXLAN_TUNNEL:
2856                 ib_filter_sz = offsetof(struct ib_flow_tunnel_filter, real_sz);
2857                 actual_filter_sz = spec_filter_size(kern_spec_mask,
2858                                                     kern_filter_sz,
2859                                                     ib_filter_sz);
2860                 if (actual_filter_sz <= 0)
2861                         return -EINVAL;
2862                 ib_spec->tunnel.size = sizeof(struct ib_flow_spec_tunnel);
2863                 memcpy(&ib_spec->tunnel.val, kern_spec_val, actual_filter_sz);
2864                 memcpy(&ib_spec->tunnel.mask, kern_spec_mask, actual_filter_sz);
2865
2866                 if ((ntohl(ib_spec->tunnel.mask.tunnel_id)) >= BIT(24) ||
2867                     (ntohl(ib_spec->tunnel.val.tunnel_id)) >= BIT(24))
2868                         return -EINVAL;
2869                 break;
2870         default:
2871                 return -EINVAL;
2872         }
2873         return 0;
2874 }
2875
2876 static int kern_spec_to_ib_spec(struct ib_uverbs_flow_spec *kern_spec,
2877                                 union ib_flow_spec *ib_spec)
2878 {
2879         if (kern_spec->reserved)
2880                 return -EINVAL;
2881
2882         if (kern_spec->type >= IB_FLOW_SPEC_ACTION_TAG)
2883                 return kern_spec_to_ib_spec_action(kern_spec, ib_spec);
2884         else
2885                 return kern_spec_to_ib_spec_filter(kern_spec, ib_spec);
2886 }
2887
2888 int ib_uverbs_ex_create_wq(struct ib_uverbs_file *file,
2889                            struct ib_device *ib_dev,
2890                            struct ib_udata *ucore,
2891                            struct ib_udata *uhw)
2892 {
2893         struct ib_uverbs_ex_create_wq     cmd = {};
2894         struct ib_uverbs_ex_create_wq_resp resp = {};
2895         struct ib_uwq_object           *obj;
2896         int err = 0;
2897         struct ib_cq *cq;
2898         struct ib_pd *pd;
2899         struct ib_wq *wq;
2900         struct ib_wq_init_attr wq_init_attr = {};
2901         size_t required_cmd_sz;
2902         size_t required_resp_len;
2903
2904         required_cmd_sz = offsetof(typeof(cmd), max_sge) + sizeof(cmd.max_sge);
2905         required_resp_len = offsetof(typeof(resp), wqn) + sizeof(resp.wqn);
2906
2907         if (ucore->inlen < required_cmd_sz)
2908                 return -EINVAL;
2909
2910         if (ucore->outlen < required_resp_len)
2911                 return -ENOSPC;
2912
2913         if (ucore->inlen > sizeof(cmd) &&
2914             !ib_is_udata_cleared(ucore, sizeof(cmd),
2915                                  ucore->inlen - sizeof(cmd)))
2916                 return -EOPNOTSUPP;
2917
2918         err = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen));
2919         if (err)
2920                 return err;
2921
2922         if (cmd.comp_mask)
2923                 return -EOPNOTSUPP;
2924
2925         obj  = (struct ib_uwq_object *)uobj_alloc(uobj_get_type(wq),
2926                                                   file->ucontext);
2927         if (IS_ERR(obj))
2928                 return PTR_ERR(obj);
2929
2930         pd  = uobj_get_obj_read(pd, cmd.pd_handle, file->ucontext);
2931         if (!pd) {
2932                 err = -EINVAL;
2933                 goto err_uobj;
2934         }
2935
2936         cq = uobj_get_obj_read(cq, cmd.cq_handle, file->ucontext);
2937         if (!cq) {
2938                 err = -EINVAL;
2939                 goto err_put_pd;
2940         }
2941
2942         wq_init_attr.cq = cq;
2943         wq_init_attr.max_sge = cmd.max_sge;
2944         wq_init_attr.max_wr = cmd.max_wr;
2945         wq_init_attr.wq_context = file;
2946         wq_init_attr.wq_type = cmd.wq_type;
2947         wq_init_attr.event_handler = ib_uverbs_wq_event_handler;
2948         if (ucore->inlen >= (offsetof(typeof(cmd), create_flags) +
2949                              sizeof(cmd.create_flags)))
2950                 wq_init_attr.create_flags = cmd.create_flags;
2951         obj->uevent.events_reported = 0;
2952         INIT_LIST_HEAD(&obj->uevent.event_list);
2953
2954         if (!pd->device->create_wq) {
2955                 err = -EOPNOTSUPP;
2956                 goto err_put_cq;
2957         }
2958         wq = pd->device->create_wq(pd, &wq_init_attr, uhw);
2959         if (IS_ERR(wq)) {
2960                 err = PTR_ERR(wq);
2961                 goto err_put_cq;
2962         }
2963
2964         wq->uobject = &obj->uevent.uobject;
2965         obj->uevent.uobject.object = wq;
2966         wq->wq_type = wq_init_attr.wq_type;
2967         wq->cq = cq;
2968         wq->pd = pd;
2969         wq->device = pd->device;
2970         wq->wq_context = wq_init_attr.wq_context;
2971         atomic_set(&wq->usecnt, 0);
2972         atomic_inc(&pd->usecnt);
2973         atomic_inc(&cq->usecnt);
2974         wq->uobject = &obj->uevent.uobject;
2975         obj->uevent.uobject.object = wq;
2976
2977         memset(&resp, 0, sizeof(resp));
2978         resp.wq_handle = obj->uevent.uobject.id;
2979         resp.max_sge = wq_init_attr.max_sge;
2980         resp.max_wr = wq_init_attr.max_wr;
2981         resp.wqn = wq->wq_num;
2982         resp.response_length = required_resp_len;
2983         err = ib_copy_to_udata(ucore,
2984                                &resp, resp.response_length);
2985         if (err)
2986                 goto err_copy;
2987
2988         uobj_put_obj_read(pd);
2989         uobj_put_obj_read(cq);
2990         uobj_alloc_commit(&obj->uevent.uobject);
2991         return 0;
2992
2993 err_copy:
2994         ib_destroy_wq(wq);
2995 err_put_cq:
2996         uobj_put_obj_read(cq);
2997 err_put_pd:
2998         uobj_put_obj_read(pd);
2999 err_uobj:
3000         uobj_alloc_abort(&obj->uevent.uobject);
3001
3002         return err;
3003 }
3004
3005 int ib_uverbs_ex_destroy_wq(struct ib_uverbs_file *file,
3006                             struct ib_device *ib_dev,
3007                             struct ib_udata *ucore,
3008                             struct ib_udata *uhw)
3009 {
3010         struct ib_uverbs_ex_destroy_wq  cmd = {};
3011         struct ib_uverbs_ex_destroy_wq_resp     resp = {};
3012         struct ib_uobject               *uobj;
3013         struct ib_uwq_object            *obj;
3014         size_t required_cmd_sz;
3015         size_t required_resp_len;
3016         int                             ret;
3017
3018         required_cmd_sz = offsetof(typeof(cmd), wq_handle) + sizeof(cmd.wq_handle);
3019         required_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved);
3020
3021         if (ucore->inlen < required_cmd_sz)
3022                 return -EINVAL;
3023
3024         if (ucore->outlen < required_resp_len)
3025                 return -ENOSPC;
3026
3027         if (ucore->inlen > sizeof(cmd) &&
3028             !ib_is_udata_cleared(ucore, sizeof(cmd),
3029                                  ucore->inlen - sizeof(cmd)))
3030                 return -EOPNOTSUPP;
3031
3032         ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen));
3033         if (ret)
3034                 return ret;
3035
3036         if (cmd.comp_mask)
3037                 return -EOPNOTSUPP;
3038
3039         resp.response_length = required_resp_len;
3040         uobj  = uobj_get_write(uobj_get_type(wq), cmd.wq_handle,
3041                                file->ucontext);
3042         if (IS_ERR(uobj))
3043                 return PTR_ERR(uobj);
3044
3045         obj = container_of(uobj, struct ib_uwq_object, uevent.uobject);
3046         /*
3047          * Make sure we don't free the memory in remove_commit as we still
3048          * needs the uobject memory to create the response.
3049          */
3050         uverbs_uobject_get(uobj);
3051
3052         ret = uobj_remove_commit(uobj);
3053         resp.events_reported = obj->uevent.events_reported;
3054         uverbs_uobject_put(uobj);
3055         if (ret)
3056                 return ret;
3057
3058         return ib_copy_to_udata(ucore, &resp, resp.response_length);
3059 }
3060
3061 int ib_uverbs_ex_modify_wq(struct ib_uverbs_file *file,
3062                            struct ib_device *ib_dev,
3063                            struct ib_udata *ucore,
3064                            struct ib_udata *uhw)
3065 {
3066         struct ib_uverbs_ex_modify_wq cmd = {};
3067         struct ib_wq *wq;
3068         struct ib_wq_attr wq_attr = {};
3069         size_t required_cmd_sz;
3070         int ret;
3071
3072         required_cmd_sz = offsetof(typeof(cmd), curr_wq_state) + sizeof(cmd.curr_wq_state);
3073         if (ucore->inlen < required_cmd_sz)
3074                 return -EINVAL;
3075
3076         if (ucore->inlen > sizeof(cmd) &&
3077             !ib_is_udata_cleared(ucore, sizeof(cmd),
3078                                  ucore->inlen - sizeof(cmd)))
3079                 return -EOPNOTSUPP;
3080
3081         ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen));
3082         if (ret)
3083                 return ret;
3084
3085         if (!cmd.attr_mask)
3086                 return -EINVAL;
3087
3088         if (cmd.attr_mask > (IB_WQ_STATE | IB_WQ_CUR_STATE | IB_WQ_FLAGS))
3089                 return -EINVAL;
3090
3091         wq = uobj_get_obj_read(wq, cmd.wq_handle, file->ucontext);
3092         if (!wq)
3093                 return -EINVAL;
3094
3095         wq_attr.curr_wq_state = cmd.curr_wq_state;
3096         wq_attr.wq_state = cmd.wq_state;
3097         if (cmd.attr_mask & IB_WQ_FLAGS) {
3098                 wq_attr.flags = cmd.flags;
3099                 wq_attr.flags_mask = cmd.flags_mask;
3100         }
3101         if (!wq->device->modify_wq) {
3102                 ret = -EOPNOTSUPP;
3103                 goto out;
3104         }
3105         ret = wq->device->modify_wq(wq, &wq_attr, cmd.attr_mask, uhw);
3106 out:
3107         uobj_put_obj_read(wq);
3108         return ret;
3109 }
3110
3111 int ib_uverbs_ex_create_rwq_ind_table(struct ib_uverbs_file *file,
3112                                       struct ib_device *ib_dev,
3113                                       struct ib_udata *ucore,
3114                                       struct ib_udata *uhw)
3115 {
3116         struct ib_uverbs_ex_create_rwq_ind_table          cmd = {};
3117         struct ib_uverbs_ex_create_rwq_ind_table_resp  resp = {};
3118         struct ib_uobject                 *uobj;
3119         int err = 0;
3120         struct ib_rwq_ind_table_init_attr init_attr = {};
3121         struct ib_rwq_ind_table *rwq_ind_tbl;
3122         struct ib_wq    **wqs = NULL;
3123         u32 *wqs_handles = NULL;
3124         struct ib_wq    *wq = NULL;
3125         int i, j, num_read_wqs;
3126         u32 num_wq_handles;
3127         u32 expected_in_size;
3128         size_t required_cmd_sz_header;
3129         size_t required_resp_len;
3130
3131         required_cmd_sz_header = offsetof(typeof(cmd), log_ind_tbl_size) + sizeof(cmd.log_ind_tbl_size);
3132         required_resp_len = offsetof(typeof(resp), ind_tbl_num) + sizeof(resp.ind_tbl_num);
3133
3134         if (ucore->inlen < required_cmd_sz_header)
3135                 return -EINVAL;
3136
3137         if (ucore->outlen < required_resp_len)
3138                 return -ENOSPC;
3139
3140         err = ib_copy_from_udata(&cmd, ucore, required_cmd_sz_header);
3141         if (err)
3142                 return err;
3143
3144         ucore->inbuf += required_cmd_sz_header;
3145         ucore->inlen -= required_cmd_sz_header;
3146
3147         if (cmd.comp_mask)
3148                 return -EOPNOTSUPP;
3149
3150         if (cmd.log_ind_tbl_size > IB_USER_VERBS_MAX_LOG_IND_TBL_SIZE)
3151                 return -EINVAL;
3152
3153         num_wq_handles = 1 << cmd.log_ind_tbl_size;
3154         expected_in_size = num_wq_handles * sizeof(__u32);
3155         if (num_wq_handles == 1)
3156                 /* input size for wq handles is u64 aligned */
3157                 expected_in_size += sizeof(__u32);
3158
3159         if (ucore->inlen < expected_in_size)
3160                 return -EINVAL;
3161
3162         if (ucore->inlen > expected_in_size &&
3163             !ib_is_udata_cleared(ucore, expected_in_size,
3164                                  ucore->inlen - expected_in_size))
3165                 return -EOPNOTSUPP;
3166
3167         wqs_handles = kcalloc(num_wq_handles, sizeof(*wqs_handles),
3168                               GFP_KERNEL);
3169         if (!wqs_handles)
3170                 return -ENOMEM;
3171
3172         err = ib_copy_from_udata(wqs_handles, ucore,
3173                                  num_wq_handles * sizeof(__u32));
3174         if (err)
3175                 goto err_free;
3176
3177         wqs = kcalloc(num_wq_handles, sizeof(*wqs), GFP_KERNEL);
3178         if (!wqs) {
3179                 err = -ENOMEM;
3180                 goto  err_free;
3181         }
3182
3183         for (num_read_wqs = 0; num_read_wqs < num_wq_handles;
3184                         num_read_wqs++) {
3185                 wq = uobj_get_obj_read(wq, wqs_handles[num_read_wqs],
3186                                        file->ucontext);
3187                 if (!wq) {
3188                         err = -EINVAL;
3189                         goto put_wqs;
3190                 }
3191
3192                 wqs[num_read_wqs] = wq;
3193         }
3194
3195         uobj  = uobj_alloc(uobj_get_type(rwq_ind_table), file->ucontext);
3196         if (IS_ERR(uobj)) {
3197                 err = PTR_ERR(uobj);
3198                 goto put_wqs;
3199         }
3200
3201         init_attr.log_ind_tbl_size = cmd.log_ind_tbl_size;
3202         init_attr.ind_tbl = wqs;
3203
3204         if (!ib_dev->create_rwq_ind_table) {
3205                 err = -EOPNOTSUPP;
3206                 goto err_uobj;
3207         }
3208         rwq_ind_tbl = ib_dev->create_rwq_ind_table(ib_dev, &init_attr, uhw);
3209
3210         if (IS_ERR(rwq_ind_tbl)) {
3211                 err = PTR_ERR(rwq_ind_tbl);
3212                 goto err_uobj;
3213         }
3214
3215         rwq_ind_tbl->ind_tbl = wqs;
3216         rwq_ind_tbl->log_ind_tbl_size = init_attr.log_ind_tbl_size;
3217         rwq_ind_tbl->uobject = uobj;
3218         uobj->object = rwq_ind_tbl;
3219         rwq_ind_tbl->device = ib_dev;
3220         atomic_set(&rwq_ind_tbl->usecnt, 0);
3221
3222         for (i = 0; i < num_wq_handles; i++)
3223                 atomic_inc(&wqs[i]->usecnt);
3224
3225         resp.ind_tbl_handle = uobj->id;
3226         resp.ind_tbl_num = rwq_ind_tbl->ind_tbl_num;
3227         resp.response_length = required_resp_len;
3228
3229         err = ib_copy_to_udata(ucore,
3230                                &resp, resp.response_length);
3231         if (err)
3232                 goto err_copy;
3233
3234         kfree(wqs_handles);
3235
3236         for (j = 0; j < num_read_wqs; j++)
3237                 uobj_put_obj_read(wqs[j]);
3238
3239         uobj_alloc_commit(uobj);
3240         return 0;
3241
3242 err_copy:
3243         ib_destroy_rwq_ind_table(rwq_ind_tbl);
3244 err_uobj:
3245         uobj_alloc_abort(uobj);
3246 put_wqs:
3247         for (j = 0; j < num_read_wqs; j++)
3248                 uobj_put_obj_read(wqs[j]);
3249 err_free:
3250         kfree(wqs_handles);
3251         kfree(wqs);
3252         return err;
3253 }
3254
3255 int ib_uverbs_ex_destroy_rwq_ind_table(struct ib_uverbs_file *file,
3256                                        struct ib_device *ib_dev,
3257                                        struct ib_udata *ucore,
3258                                        struct ib_udata *uhw)
3259 {
3260         struct ib_uverbs_ex_destroy_rwq_ind_table       cmd = {};
3261         struct ib_uobject               *uobj;
3262         int                     ret;
3263         size_t required_cmd_sz;
3264
3265         required_cmd_sz = offsetof(typeof(cmd), ind_tbl_handle) + sizeof(cmd.ind_tbl_handle);
3266
3267         if (ucore->inlen < required_cmd_sz)
3268                 return -EINVAL;
3269
3270         if (ucore->inlen > sizeof(cmd) &&
3271             !ib_is_udata_cleared(ucore, sizeof(cmd),
3272                                  ucore->inlen - sizeof(cmd)))
3273                 return -EOPNOTSUPP;
3274
3275         ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen));
3276         if (ret)
3277                 return ret;
3278
3279         if (cmd.comp_mask)
3280                 return -EOPNOTSUPP;
3281
3282         uobj  = uobj_get_write(uobj_get_type(rwq_ind_table), cmd.ind_tbl_handle,
3283                                file->ucontext);
3284         if (IS_ERR(uobj))
3285                 return PTR_ERR(uobj);
3286
3287         return uobj_remove_commit(uobj);
3288 }
3289
3290 int ib_uverbs_ex_create_flow(struct ib_uverbs_file *file,
3291                              struct ib_device *ib_dev,
3292                              struct ib_udata *ucore,
3293                              struct ib_udata *uhw)
3294 {
3295         struct ib_uverbs_create_flow      cmd;
3296         struct ib_uverbs_create_flow_resp resp;
3297         struct ib_uobject                 *uobj;
3298         struct ib_flow                    *flow_id;
3299         struct ib_uverbs_flow_attr        *kern_flow_attr;
3300         struct ib_flow_attr               *flow_attr;
3301         struct ib_qp                      *qp;
3302         int err = 0;
3303         void *kern_spec;
3304         void *ib_spec;
3305         int i;
3306
3307         if (ucore->inlen < sizeof(cmd))
3308                 return -EINVAL;
3309
3310         if (ucore->outlen < sizeof(resp))
3311                 return -ENOSPC;
3312
3313         err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd));
3314         if (err)
3315                 return err;
3316
3317         ucore->inbuf += sizeof(cmd);
3318         ucore->inlen -= sizeof(cmd);
3319
3320         if (cmd.comp_mask)
3321                 return -EINVAL;
3322
3323         if (!capable(CAP_NET_RAW))
3324                 return -EPERM;
3325
3326         if (cmd.flow_attr.flags >= IB_FLOW_ATTR_FLAGS_RESERVED)
3327                 return -EINVAL;
3328
3329         if ((cmd.flow_attr.flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) &&
3330             ((cmd.flow_attr.type == IB_FLOW_ATTR_ALL_DEFAULT) ||
3331              (cmd.flow_attr.type == IB_FLOW_ATTR_MC_DEFAULT)))
3332                 return -EINVAL;
3333
3334         if (cmd.flow_attr.num_of_specs > IB_FLOW_SPEC_SUPPORT_LAYERS)
3335                 return -EINVAL;
3336
3337         if (cmd.flow_attr.size > ucore->inlen ||
3338             cmd.flow_attr.size >
3339             (cmd.flow_attr.num_of_specs * sizeof(struct ib_uverbs_flow_spec)))
3340                 return -EINVAL;
3341
3342         if (cmd.flow_attr.reserved[0] ||
3343             cmd.flow_attr.reserved[1])
3344                 return -EINVAL;
3345
3346         if (cmd.flow_attr.num_of_specs) {
3347                 kern_flow_attr = kmalloc(sizeof(*kern_flow_attr) + cmd.flow_attr.size,
3348                                          GFP_KERNEL);
3349                 if (!kern_flow_attr)
3350                         return -ENOMEM;
3351
3352                 memcpy(kern_flow_attr, &cmd.flow_attr, sizeof(*kern_flow_attr));
3353                 err = ib_copy_from_udata(kern_flow_attr + 1, ucore,
3354                                          cmd.flow_attr.size);
3355                 if (err)
3356                         goto err_free_attr;
3357         } else {
3358                 kern_flow_attr = &cmd.flow_attr;
3359         }
3360
3361         uobj  = uobj_alloc(uobj_get_type(flow), file->ucontext);
3362         if (IS_ERR(uobj)) {
3363                 err = PTR_ERR(uobj);
3364                 goto err_free_attr;
3365         }
3366
3367         qp = uobj_get_obj_read(qp, cmd.qp_handle, file->ucontext);
3368         if (!qp) {
3369                 err = -EINVAL;
3370                 goto err_uobj;
3371         }
3372
3373         flow_attr = kzalloc(sizeof(*flow_attr) + cmd.flow_attr.num_of_specs *
3374                             sizeof(union ib_flow_spec), GFP_KERNEL);
3375         if (!flow_attr) {
3376                 err = -ENOMEM;
3377                 goto err_put;
3378         }
3379
3380         flow_attr->type = kern_flow_attr->type;
3381         flow_attr->priority = kern_flow_attr->priority;
3382         flow_attr->num_of_specs = kern_flow_attr->num_of_specs;
3383         flow_attr->port = kern_flow_attr->port;
3384         flow_attr->flags = kern_flow_attr->flags;
3385         flow_attr->size = sizeof(*flow_attr);
3386
3387         kern_spec = kern_flow_attr + 1;
3388         ib_spec = flow_attr + 1;
3389         for (i = 0; i < flow_attr->num_of_specs &&
3390              cmd.flow_attr.size > offsetof(struct ib_uverbs_flow_spec, reserved) &&
3391              cmd.flow_attr.size >=
3392              ((struct ib_uverbs_flow_spec *)kern_spec)->size; i++) {
3393                 err = kern_spec_to_ib_spec(kern_spec, ib_spec);
3394                 if (err)
3395                         goto err_free;
3396                 flow_attr->size +=
3397                         ((union ib_flow_spec *) ib_spec)->size;
3398                 cmd.flow_attr.size -= ((struct ib_uverbs_flow_spec *)kern_spec)->size;
3399                 kern_spec += ((struct ib_uverbs_flow_spec *) kern_spec)->size;
3400                 ib_spec += ((union ib_flow_spec *) ib_spec)->size;
3401         }
3402         if (cmd.flow_attr.size || (i != flow_attr->num_of_specs)) {
3403                 pr_warn("create flow failed, flow %d: %d bytes left from uverb cmd\n",
3404                         i, cmd.flow_attr.size);
3405                 err = -EINVAL;
3406                 goto err_free;
3407         }
3408         flow_id = ib_create_flow(qp, flow_attr, IB_FLOW_DOMAIN_USER);
3409         if (IS_ERR(flow_id)) {
3410                 err = PTR_ERR(flow_id);
3411                 goto err_free;
3412         }
3413         flow_id->uobject = uobj;
3414         uobj->object = flow_id;
3415
3416         memset(&resp, 0, sizeof(resp));
3417         resp.flow_handle = uobj->id;
3418
3419         err = ib_copy_to_udata(ucore,
3420                                &resp, sizeof(resp));
3421         if (err)
3422                 goto err_copy;
3423
3424         uobj_put_obj_read(qp);
3425         uobj_alloc_commit(uobj);
3426         kfree(flow_attr);
3427         if (cmd.flow_attr.num_of_specs)
3428                 kfree(kern_flow_attr);
3429         return 0;
3430 err_copy:
3431         ib_destroy_flow(flow_id);
3432 err_free:
3433         kfree(flow_attr);
3434 err_put:
3435         uobj_put_obj_read(qp);
3436 err_uobj:
3437         uobj_alloc_abort(uobj);
3438 err_free_attr:
3439         if (cmd.flow_attr.num_of_specs)
3440                 kfree(kern_flow_attr);
3441         return err;
3442 }
3443
3444 int ib_uverbs_ex_destroy_flow(struct ib_uverbs_file *file,
3445                               struct ib_device *ib_dev,
3446                               struct ib_udata *ucore,
3447                               struct ib_udata *uhw)
3448 {
3449         struct ib_uverbs_destroy_flow   cmd;
3450         struct ib_uobject               *uobj;
3451         int                             ret;
3452
3453         if (ucore->inlen < sizeof(cmd))
3454                 return -EINVAL;
3455
3456         ret = ib_copy_from_udata(&cmd, ucore, sizeof(cmd));
3457         if (ret)
3458                 return ret;
3459
3460         if (cmd.comp_mask)
3461                 return -EINVAL;
3462
3463         uobj  = uobj_get_write(uobj_get_type(flow), cmd.flow_handle,
3464                                file->ucontext);
3465         if (IS_ERR(uobj))
3466                 return PTR_ERR(uobj);
3467
3468         ret = uobj_remove_commit(uobj);
3469         return ret;
3470 }
3471
3472 static int __uverbs_create_xsrq(struct ib_uverbs_file *file,
3473                                 struct ib_device *ib_dev,
3474                                 struct ib_uverbs_create_xsrq *cmd,
3475                                 struct ib_udata *udata)
3476 {
3477         struct ib_uverbs_create_srq_resp resp;
3478         struct ib_usrq_object           *obj;
3479         struct ib_pd                    *pd;
3480         struct ib_srq                   *srq;
3481         struct ib_uobject               *uninitialized_var(xrcd_uobj);
3482         struct ib_srq_init_attr          attr;
3483         int ret;
3484
3485         obj  = (struct ib_usrq_object *)uobj_alloc(uobj_get_type(srq),
3486                                                    file->ucontext);
3487         if (IS_ERR(obj))
3488                 return PTR_ERR(obj);
3489
3490         if (cmd->srq_type == IB_SRQT_TM)
3491                 attr.ext.tag_matching.max_num_tags = cmd->max_num_tags;
3492
3493         if (cmd->srq_type == IB_SRQT_XRC) {
3494                 xrcd_uobj = uobj_get_read(uobj_get_type(xrcd), cmd->xrcd_handle,
3495                                           file->ucontext);
3496                 if (IS_ERR(xrcd_uobj)) {
3497                         ret = -EINVAL;
3498                         goto err;
3499                 }
3500
3501                 attr.ext.xrc.xrcd = (struct ib_xrcd *)xrcd_uobj->object;
3502                 if (!attr.ext.xrc.xrcd) {
3503                         ret = -EINVAL;
3504                         goto err_put_xrcd;
3505                 }
3506
3507                 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);
3508                 atomic_inc(&obj->uxrcd->refcnt);
3509         }
3510
3511         if (ib_srq_has_cq(cmd->srq_type)) {
3512                 attr.ext.cq  = uobj_get_obj_read(cq, cmd->cq_handle,
3513                                                  file->ucontext);
3514                 if (!attr.ext.cq) {
3515                         ret = -EINVAL;
3516                         goto err_put_xrcd;
3517                 }
3518         }
3519
3520         pd  = uobj_get_obj_read(pd, cmd->pd_handle, file->ucontext);
3521         if (!pd) {
3522                 ret = -EINVAL;
3523                 goto err_put_cq;
3524         }
3525
3526         attr.event_handler  = ib_uverbs_srq_event_handler;
3527         attr.srq_context    = file;
3528         attr.srq_type       = cmd->srq_type;
3529         attr.attr.max_wr    = cmd->max_wr;
3530         attr.attr.max_sge   = cmd->max_sge;
3531         attr.attr.srq_limit = cmd->srq_limit;
3532
3533         obj->uevent.events_reported = 0;
3534         INIT_LIST_HEAD(&obj->uevent.event_list);
3535
3536         srq = pd->device->create_srq(pd, &attr, udata);
3537         if (IS_ERR(srq)) {
3538                 ret = PTR_ERR(srq);
3539                 goto err_put;
3540         }
3541
3542         srq->device        = pd->device;
3543         srq->pd            = pd;
3544         srq->srq_type      = cmd->srq_type;
3545         srq->uobject       = &obj->uevent.uobject;
3546         srq->event_handler = attr.event_handler;
3547         srq->srq_context   = attr.srq_context;
3548
3549         if (ib_srq_has_cq(cmd->srq_type)) {
3550                 srq->ext.cq       = attr.ext.cq;
3551                 atomic_inc(&attr.ext.cq->usecnt);
3552         }
3553
3554         if (cmd->srq_type == IB_SRQT_XRC) {
3555                 srq->ext.xrc.xrcd = attr.ext.xrc.xrcd;
3556                 atomic_inc(&attr.ext.xrc.xrcd->usecnt);
3557         }
3558
3559         atomic_inc(&pd->usecnt);
3560         atomic_set(&srq->usecnt, 0);
3561
3562         obj->uevent.uobject.object = srq;
3563         obj->uevent.uobject.user_handle = cmd->user_handle;
3564
3565         memset(&resp, 0, sizeof resp);
3566         resp.srq_handle = obj->uevent.uobject.id;
3567         resp.max_wr     = attr.attr.max_wr;
3568         resp.max_sge    = attr.attr.max_sge;
3569         if (cmd->srq_type == IB_SRQT_XRC)
3570                 resp.srqn = srq->ext.xrc.srq_num;
3571
3572         if (copy_to_user((void __user *) (unsigned long) cmd->response,
3573                          &resp, sizeof resp)) {
3574                 ret = -EFAULT;
3575                 goto err_copy;
3576         }
3577
3578         if (cmd->srq_type == IB_SRQT_XRC)
3579                 uobj_put_read(xrcd_uobj);
3580
3581         if (ib_srq_has_cq(cmd->srq_type))
3582                 uobj_put_obj_read(attr.ext.cq);
3583
3584         uobj_put_obj_read(pd);
3585         uobj_alloc_commit(&obj->uevent.uobject);
3586
3587         return 0;
3588
3589 err_copy:
3590         ib_destroy_srq(srq);
3591
3592 err_put:
3593         uobj_put_obj_read(pd);
3594
3595 err_put_cq:
3596         if (ib_srq_has_cq(cmd->srq_type))
3597                 uobj_put_obj_read(attr.ext.cq);
3598
3599 err_put_xrcd:
3600         if (cmd->srq_type == IB_SRQT_XRC) {
3601                 atomic_dec(&obj->uxrcd->refcnt);
3602                 uobj_put_read(xrcd_uobj);
3603         }
3604
3605 err:
3606         uobj_alloc_abort(&obj->uevent.uobject);
3607         return ret;
3608 }
3609
3610 ssize_t ib_uverbs_create_srq(struct ib_uverbs_file *file,
3611                              struct ib_device *ib_dev,
3612                              const char __user *buf, int in_len,
3613                              int out_len)
3614 {
3615         struct ib_uverbs_create_srq      cmd;
3616         struct ib_uverbs_create_xsrq     xcmd;
3617         struct ib_uverbs_create_srq_resp resp;
3618         struct ib_udata                  udata;
3619         int ret;
3620
3621         if (out_len < sizeof resp)
3622                 return -ENOSPC;
3623
3624         if (copy_from_user(&cmd, buf, sizeof cmd))
3625                 return -EFAULT;
3626
3627         memset(&xcmd, 0, sizeof(xcmd));
3628         xcmd.response    = cmd.response;
3629         xcmd.user_handle = cmd.user_handle;
3630         xcmd.srq_type    = IB_SRQT_BASIC;
3631         xcmd.pd_handle   = cmd.pd_handle;
3632         xcmd.max_wr      = cmd.max_wr;
3633         xcmd.max_sge     = cmd.max_sge;
3634         xcmd.srq_limit   = cmd.srq_limit;
3635
3636         ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
3637                    u64_to_user_ptr(cmd.response) + sizeof(resp),
3638                    in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
3639                    out_len - sizeof(resp));
3640
3641         ret = __uverbs_create_xsrq(file, ib_dev, &xcmd, &udata);
3642         if (ret)
3643                 return ret;
3644
3645         return in_len;
3646 }
3647
3648 ssize_t ib_uverbs_create_xsrq(struct ib_uverbs_file *file,
3649                               struct ib_device *ib_dev,
3650                               const char __user *buf, int in_len, int out_len)
3651 {
3652         struct ib_uverbs_create_xsrq     cmd;
3653         struct ib_uverbs_create_srq_resp resp;
3654         struct ib_udata                  udata;
3655         int ret;
3656
3657         if (out_len < sizeof resp)
3658                 return -ENOSPC;
3659
3660         if (copy_from_user(&cmd, buf, sizeof cmd))
3661                 return -EFAULT;
3662
3663         ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
3664                    u64_to_user_ptr(cmd.response) + sizeof(resp),
3665                    in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
3666                    out_len - sizeof(resp));
3667
3668         ret = __uverbs_create_xsrq(file, ib_dev, &cmd, &udata);
3669         if (ret)
3670                 return ret;
3671
3672         return in_len;
3673 }
3674
3675 ssize_t ib_uverbs_modify_srq(struct ib_uverbs_file *file,
3676                              struct ib_device *ib_dev,
3677                              const char __user *buf, int in_len,
3678                              int out_len)
3679 {
3680         struct ib_uverbs_modify_srq cmd;
3681         struct ib_udata             udata;
3682         struct ib_srq              *srq;
3683         struct ib_srq_attr          attr;
3684         int                         ret;
3685
3686         if (copy_from_user(&cmd, buf, sizeof cmd))
3687                 return -EFAULT;
3688
3689         ib_uverbs_init_udata(&udata, buf + sizeof cmd, NULL, in_len - sizeof cmd,
3690                    out_len);
3691
3692         srq = uobj_get_obj_read(srq, cmd.srq_handle, file->ucontext);
3693         if (!srq)
3694                 return -EINVAL;
3695
3696         attr.max_wr    = cmd.max_wr;
3697         attr.srq_limit = cmd.srq_limit;
3698
3699         ret = srq->device->modify_srq(srq, &attr, cmd.attr_mask, &udata);
3700
3701         uobj_put_obj_read(srq);
3702
3703         return ret ? ret : in_len;
3704 }
3705
3706 ssize_t ib_uverbs_query_srq(struct ib_uverbs_file *file,
3707                             struct ib_device *ib_dev,
3708                             const char __user *buf,
3709                             int in_len, int out_len)
3710 {
3711         struct ib_uverbs_query_srq      cmd;
3712         struct ib_uverbs_query_srq_resp resp;
3713         struct ib_srq_attr              attr;
3714         struct ib_srq                   *srq;
3715         int                             ret;
3716
3717         if (out_len < sizeof resp)
3718                 return -ENOSPC;
3719
3720         if (copy_from_user(&cmd, buf, sizeof cmd))
3721                 return -EFAULT;
3722
3723         srq = uobj_get_obj_read(srq, cmd.srq_handle, file->ucontext);
3724         if (!srq)
3725                 return -EINVAL;
3726
3727         ret = ib_query_srq(srq, &attr);
3728
3729         uobj_put_obj_read(srq);
3730
3731         if (ret)
3732                 return ret;
3733
3734         memset(&resp, 0, sizeof resp);
3735
3736         resp.max_wr    = attr.max_wr;
3737         resp.max_sge   = attr.max_sge;
3738         resp.srq_limit = attr.srq_limit;
3739
3740         if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp))
3741                 return -EFAULT;
3742
3743         return in_len;
3744 }
3745
3746 ssize_t ib_uverbs_destroy_srq(struct ib_uverbs_file *file,
3747                               struct ib_device *ib_dev,
3748                               const char __user *buf, int in_len,
3749                               int out_len)
3750 {
3751         struct ib_uverbs_destroy_srq      cmd;
3752         struct ib_uverbs_destroy_srq_resp resp;
3753         struct ib_uobject                *uobj;
3754         struct ib_uevent_object          *obj;
3755         int                               ret = -EINVAL;
3756
3757         if (copy_from_user(&cmd, buf, sizeof cmd))
3758                 return -EFAULT;
3759
3760         uobj  = uobj_get_write(uobj_get_type(srq), cmd.srq_handle,
3761                                file->ucontext);
3762         if (IS_ERR(uobj))
3763                 return PTR_ERR(uobj);
3764
3765         obj = container_of(uobj, struct ib_uevent_object, uobject);
3766         /*
3767          * Make sure we don't free the memory in remove_commit as we still
3768          * needs the uobject memory to create the response.
3769          */
3770         uverbs_uobject_get(uobj);
3771
3772         memset(&resp, 0, sizeof(resp));
3773
3774         ret = uobj_remove_commit(uobj);
3775         if (ret) {
3776                 uverbs_uobject_put(uobj);
3777                 return ret;
3778         }
3779         resp.events_reported = obj->events_reported;
3780         uverbs_uobject_put(uobj);
3781         if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof(resp)))
3782                 return -EFAULT;
3783
3784         return in_len;
3785 }
3786
3787 int ib_uverbs_ex_query_device(struct ib_uverbs_file *file,
3788                               struct ib_device *ib_dev,
3789                               struct ib_udata *ucore,
3790                               struct ib_udata *uhw)
3791 {
3792         struct ib_uverbs_ex_query_device_resp resp = { {0} };
3793         struct ib_uverbs_ex_query_device  cmd;
3794         struct ib_device_attr attr = {0};
3795         int err;
3796
3797         if (!ib_dev->query_device)
3798                 return -EOPNOTSUPP;
3799
3800         if (ucore->inlen < sizeof(cmd))
3801                 return -EINVAL;
3802
3803         err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd));
3804         if (err)
3805                 return err;
3806
3807         if (cmd.comp_mask)
3808                 return -EINVAL;
3809
3810         if (cmd.reserved)
3811                 return -EINVAL;
3812
3813         resp.response_length = offsetof(typeof(resp), odp_caps);
3814
3815         if (ucore->outlen < resp.response_length)
3816                 return -ENOSPC;
3817
3818         err = ib_dev->query_device(ib_dev, &attr, uhw);
3819         if (err)
3820                 return err;
3821
3822         copy_query_dev_fields(file, ib_dev, &resp.base, &attr);
3823
3824         if (ucore->outlen < resp.response_length + sizeof(resp.odp_caps))
3825                 goto end;
3826
3827 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
3828         resp.odp_caps.general_caps = attr.odp_caps.general_caps;
3829         resp.odp_caps.per_transport_caps.rc_odp_caps =
3830                 attr.odp_caps.per_transport_caps.rc_odp_caps;
3831         resp.odp_caps.per_transport_caps.uc_odp_caps =
3832                 attr.odp_caps.per_transport_caps.uc_odp_caps;
3833         resp.odp_caps.per_transport_caps.ud_odp_caps =
3834                 attr.odp_caps.per_transport_caps.ud_odp_caps;
3835 #endif
3836         resp.response_length += sizeof(resp.odp_caps);
3837
3838         if (ucore->outlen < resp.response_length + sizeof(resp.timestamp_mask))
3839                 goto end;
3840
3841         resp.timestamp_mask = attr.timestamp_mask;
3842         resp.response_length += sizeof(resp.timestamp_mask);
3843
3844         if (ucore->outlen < resp.response_length + sizeof(resp.hca_core_clock))
3845                 goto end;
3846
3847         resp.hca_core_clock = attr.hca_core_clock;
3848         resp.response_length += sizeof(resp.hca_core_clock);
3849
3850         if (ucore->outlen < resp.response_length + sizeof(resp.device_cap_flags_ex))
3851                 goto end;
3852
3853         resp.device_cap_flags_ex = attr.device_cap_flags;
3854         resp.response_length += sizeof(resp.device_cap_flags_ex);
3855
3856         if (ucore->outlen < resp.response_length + sizeof(resp.rss_caps))
3857                 goto end;
3858
3859         resp.rss_caps.supported_qpts = attr.rss_caps.supported_qpts;
3860         resp.rss_caps.max_rwq_indirection_tables =
3861                 attr.rss_caps.max_rwq_indirection_tables;
3862         resp.rss_caps.max_rwq_indirection_table_size =
3863                 attr.rss_caps.max_rwq_indirection_table_size;
3864
3865         resp.response_length += sizeof(resp.rss_caps);
3866
3867         if (ucore->outlen < resp.response_length + sizeof(resp.max_wq_type_rq))
3868                 goto end;
3869
3870         resp.max_wq_type_rq = attr.max_wq_type_rq;
3871         resp.response_length += sizeof(resp.max_wq_type_rq);
3872
3873         if (ucore->outlen < resp.response_length + sizeof(resp.raw_packet_caps))
3874                 goto end;
3875
3876         resp.raw_packet_caps = attr.raw_packet_caps;
3877         resp.response_length += sizeof(resp.raw_packet_caps);
3878
3879         if (ucore->outlen < resp.response_length + sizeof(resp.tm_caps))
3880                 goto end;
3881
3882         resp.tm_caps.max_rndv_hdr_size  = attr.tm_caps.max_rndv_hdr_size;
3883         resp.tm_caps.max_num_tags       = attr.tm_caps.max_num_tags;
3884         resp.tm_caps.max_ops            = attr.tm_caps.max_ops;
3885         resp.tm_caps.max_sge            = attr.tm_caps.max_sge;
3886         resp.tm_caps.flags              = attr.tm_caps.flags;
3887         resp.response_length += sizeof(resp.tm_caps);
3888
3889         if (ucore->outlen < resp.response_length + sizeof(resp.cq_moderation_caps))
3890                 goto end;
3891
3892         resp.cq_moderation_caps.max_cq_moderation_count  =
3893                 attr.cq_caps.max_cq_moderation_count;
3894         resp.cq_moderation_caps.max_cq_moderation_period =
3895                 attr.cq_caps.max_cq_moderation_period;
3896         resp.response_length += sizeof(resp.cq_moderation_caps);
3897 end:
3898         err = ib_copy_to_udata(ucore, &resp, resp.response_length);
3899         return err;
3900 }
3901
3902 int ib_uverbs_ex_modify_cq(struct ib_uverbs_file *file,
3903                            struct ib_device *ib_dev,
3904                            struct ib_udata *ucore,
3905                            struct ib_udata *uhw)
3906 {
3907         struct ib_uverbs_ex_modify_cq cmd = {};
3908         struct ib_cq *cq;
3909         size_t required_cmd_sz;
3910         int ret;
3911
3912         required_cmd_sz = offsetof(typeof(cmd), reserved) +
3913                                 sizeof(cmd.reserved);
3914         if (ucore->inlen < required_cmd_sz)
3915                 return -EINVAL;
3916
3917         /* sanity checks */
3918         if (ucore->inlen > sizeof(cmd) &&
3919             !ib_is_udata_cleared(ucore, sizeof(cmd),
3920                                  ucore->inlen - sizeof(cmd)))
3921                 return -EOPNOTSUPP;
3922
3923         ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen));
3924         if (ret)
3925                 return ret;
3926
3927         if (!cmd.attr_mask || cmd.reserved)
3928                 return -EINVAL;
3929
3930         if (cmd.attr_mask > IB_CQ_MODERATE)
3931                 return -EOPNOTSUPP;
3932
3933         cq = uobj_get_obj_read(cq, cmd.cq_handle, file->ucontext);
3934         if (!cq)
3935                 return -EINVAL;
3936
3937         ret = rdma_set_cq_moderation(cq, cmd.attr.cq_count, cmd.attr.cq_period);
3938
3939         uobj_put_obj_read(cq);
3940
3941         return ret;
3942 }