drivers/gpu/drm/msm/msm_gem_submit.c

   1 /*
   2  * Copyright (C) 2013 Red Hat
   3  * Author: Rob Clark <robdclark@gmail.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 as published by
   7  * the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program.  If not, see <http://www.gnu.org/licenses/>.
  16  */
  17
  18 #include "msm_drv.h"
  19 #include "msm_gpu.h"
  20 #include "msm_gem.h"
  21
  22 /*
  23  * Cmdstream submission:
  24  */
  25
  26 /* make sure these don't conflict w/ MSM_SUBMIT_BO_x */
  27 #define BO_VALID    0x8000   /* is current addr in cmdstream correct/valid? */
  28 #define BO_LOCKED   0x4000
  29 #define BO_PINNED   0x2000
  30
  31 static struct msm_gem_submit *submit_create(struct drm_device *dev,
  32                 struct msm_gpu *gpu, int nr)
  33 {
  34         struct msm_gem_submit *submit;
  35         int sz = sizeof(*submit) + (nr * sizeof(submit->bos[0]));
  36
  37         submit = kmalloc(sz, GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
  38         if (!submit)
  39                 return NULL;
  40
  41         submit->dev = dev;
  42         submit->gpu = gpu;
  43         submit->pid = get_pid(task_pid(current));
  44
  45         /* initially, until copy_from_user() and bo lookup succeeds: */
  46         submit->nr_bos = 0;
  47         submit->nr_cmds = 0;
  48
  49         INIT_LIST_HEAD(&submit->bo_list);
  50         ww_acquire_init(&submit->ticket, &reservation_ww_class);
  51
  52         return submit;
  53 }
  54
  55 void msm_gem_submit_free(struct msm_gem_submit *submit)
  56 {
  57         fence_put(submit->fence);
  58         list_del(&submit->node);
  59         put_pid(submit->pid);
  60         kfree(submit);
  61 }
  62
  63 static int submit_lookup_objects(struct msm_gem_submit *submit,
  64                 struct drm_msm_gem_submit *args, struct drm_file *file)
  65 {
  66         unsigned i;
  67         int ret = 0;
  68
  69         spin_lock(&file->table_lock);
  70
  71         for (i = 0; i < args->nr_bos; i++) {
  72                 struct drm_msm_gem_submit_bo submit_bo;
  73                 struct drm_gem_object *obj;
  74                 struct msm_gem_object *msm_obj;
  75                 void __user *userptr =
  76                         u64_to_user_ptr(args->bos + (i * sizeof(submit_bo)));
  77
  78                 ret = copy_from_user(&submit_bo, userptr, sizeof(submit_bo));
  79                 if (ret) {
  80                         ret = -EFAULT;
  81                         goto out_unlock;
  82                 }
  83
  84                 if (submit_bo.flags & ~MSM_SUBMIT_BO_FLAGS) {
  85                         DRM_ERROR("invalid flags: %x\n", submit_bo.flags);
  86                         ret = -EINVAL;
  87                         goto out_unlock;
  88                 }
  89
  90                 submit->bos[i].flags = submit_bo.flags;
  91                 /* in validate_objects() we figure out if this is true: */
  92                 submit->bos[i].iova  = submit_bo.presumed;
  93
  94                 /* normally use drm_gem_object_lookup(), but for bulk lookup
  95                  * all under single table_lock just hit object_idr directly:
  96                  */
  97                 obj = idr_find(&file->object_idr, submit_bo.handle);
  98                 if (!obj) {
  99                         DRM_ERROR("invalid handle %u at index %u\n", submit_bo.handle, i);
 100                         ret = -EINVAL;
 101                         goto out_unlock;
 102                 }
 103
 104                 msm_obj = to_msm_bo(obj);
 105
 106                 if (!list_empty(&msm_obj->submit_entry)) {
 107                         DRM_ERROR("handle %u at index %u already on submit list\n",
 108                                         submit_bo.handle, i);
 109                         ret = -EINVAL;
 110                         goto out_unlock;
 111                 }
 112
 113                 drm_gem_object_reference(obj);
 114
 115                 submit->bos[i].obj = msm_obj;
 116
 117                 list_add_tail(&msm_obj->submit_entry, &submit->bo_list);
 118         }
 119
 120 out_unlock:
 121         submit->nr_bos = i;
 122         spin_unlock(&file->table_lock);
 123
 124         return ret;
 125 }
 126
 127 static void submit_unlock_unpin_bo(struct msm_gem_submit *submit, int i)
 128 {
 129         struct msm_gem_object *msm_obj = submit->bos[i].obj;
 130
 131         if (submit->bos[i].flags & BO_PINNED)
 132                 msm_gem_put_iova(&msm_obj->base, submit->gpu->id);
 133
 134         if (submit->bos[i].flags & BO_LOCKED)
 135                 ww_mutex_unlock(&msm_obj->resv->lock);
 136
 137         if (!(submit->bos[i].flags & BO_VALID))
 138                 submit->bos[i].iova = 0;
 139
 140         submit->bos[i].flags &= ~(BO_LOCKED | BO_PINNED);
 141 }
 142
 143 /* This is where we make sure all the bo's are reserved and pin'd: */
 144 static int submit_lock_objects(struct msm_gem_submit *submit)
 145 {
 146         int contended, slow_locked = -1, i, ret = 0;
 147
 148 retry:
 149         for (i = 0; i < submit->nr_bos; i++) {
 150                 struct msm_gem_object *msm_obj = submit->bos[i].obj;
 151
 152                 if (slow_locked == i)
 153                         slow_locked = -1;
 154
 155                 contended = i;
 156
 157                 if (!(submit->bos[i].flags & BO_LOCKED)) {
 158                         ret = ww_mutex_lock_interruptible(&msm_obj->resv->lock,
 159                                         &submit->ticket);
 160                         if (ret)
 161                                 goto fail;
 162                         submit->bos[i].flags |= BO_LOCKED;
 163                 }
 164         }
 165
 166         ww_acquire_done(&submit->ticket);
 167
 168         return 0;
 169
 170 fail:
 171         for (; i >= 0; i--)
 172                 submit_unlock_unpin_bo(submit, i);
 173
 174         if (slow_locked > 0)
 175                 submit_unlock_unpin_bo(submit, slow_locked);
 176
 177         if (ret == -EDEADLK) {
 178                 struct msm_gem_object *msm_obj = submit->bos[contended].obj;
 179                 /* we lost out in a seqno race, lock and retry.. */
 180                 ret = ww_mutex_lock_slow_interruptible(&msm_obj->resv->lock,
 181                                 &submit->ticket);
 182                 if (!ret) {
 183                         submit->bos[contended].flags |= BO_LOCKED;
 184                         slow_locked = contended;
 185                         goto retry;
 186                 }
 187         }
 188
 189         return ret;
 190 }
 191
 192 static int submit_fence_sync(struct msm_gem_submit *submit)
 193 {
 194         int i, ret = 0;
 195
 196         for (i = 0; i < submit->nr_bos; i++) {
 197                 struct msm_gem_object *msm_obj = submit->bos[i].obj;
 198                 bool write = submit->bos[i].flags & MSM_SUBMIT_BO_WRITE;
 199
 200                 ret = msm_gem_sync_object(&msm_obj->base, submit->gpu->fctx, write);
 201                 if (ret)
 202                         break;
 203         }
 204
 205         return ret;
 206 }
 207
 208 static int submit_pin_objects(struct msm_gem_submit *submit)
 209 {
 210         int i, ret = 0;
 211
 212         submit->valid = true;
 213
 214         for (i = 0; i < submit->nr_bos; i++) {
 215                 struct msm_gem_object *msm_obj = submit->bos[i].obj;
 216                 uint32_t iova;
 217
 218                 /* if locking succeeded, pin bo: */
 219                 ret = msm_gem_get_iova_locked(&msm_obj->base,
 220                                 submit->gpu->id, &iova);
 221
 222                 if (ret)
 223                         break;
 224
 225                 submit->bos[i].flags |= BO_PINNED;
 226
 227                 if (iova == submit->bos[i].iova) {
 228                         submit->bos[i].flags |= BO_VALID;
 229                 } else {
 230                         submit->bos[i].iova = iova;
 231                         /* iova changed, so address in cmdstream is not valid: */
 232                         submit->bos[i].flags &= ~BO_VALID;
 233                         submit->valid = false;
 234                 }
 235         }
 236
 237         return ret;
 238 }
 239
 240 static int submit_bo(struct msm_gem_submit *submit, uint32_t idx,
 241                 struct msm_gem_object **obj, uint32_t *iova, bool *valid)
 242 {
 243         if (idx >= submit->nr_bos) {
 244                 DRM_ERROR("invalid buffer index: %u (out of %u)\n",
 245                                 idx, submit->nr_bos);
 246                 return -EINVAL;
 247         }
 248
 249         if (obj)
 250                 *obj = submit->bos[idx].obj;
 251         if (iova)
 252                 *iova = submit->bos[idx].iova;
 253         if (valid)
 254                 *valid = !!(submit->bos[idx].flags & BO_VALID);
 255
 256         return 0;
 257 }
 258
 259 /* process the reloc's and patch up the cmdstream as needed: */
 260 static int submit_reloc(struct msm_gem_submit *submit, struct msm_gem_object *obj,
 261                 uint32_t offset, uint32_t nr_relocs, uint64_t relocs)
 262 {
 263         uint32_t i, last_offset = 0;
 264         uint32_t *ptr;
 265         int ret;
 266
 267         if (offset % 4) {
 268                 DRM_ERROR("non-aligned cmdstream buffer: %u\n", offset);
 269                 return -EINVAL;
 270         }
 271
 272         /* For now, just map the entire thing.  Eventually we probably
 273          * to do it page-by-page, w/ kmap() if not vmap()d..
 274          */
 275         ptr = msm_gem_vaddr_locked(&obj->base);
 276
 277         if (IS_ERR(ptr)) {
 278                 ret = PTR_ERR(ptr);
 279                 DBG("failed to map: %d", ret);
 280                 return ret;
 281         }
 282
 283         for (i = 0; i < nr_relocs; i++) {
 284                 struct drm_msm_gem_submit_reloc submit_reloc;
 285                 void __user *userptr =
 286                         u64_to_user_ptr(relocs + (i * sizeof(submit_reloc)));
 287                 uint32_t iova, off;
 288                 bool valid;
 289
 290                 ret = copy_from_user(&submit_reloc, userptr, sizeof(submit_reloc));
 291                 if (ret)
 292                         return -EFAULT;
 293
 294                 if (submit_reloc.submit_offset % 4) {
 295                         DRM_ERROR("non-aligned reloc offset: %u\n",
 296                                         submit_reloc.submit_offset);
 297                         return -EINVAL;
 298                 }
 299
 300                 /* offset in dwords: */
 301                 off = submit_reloc.submit_offset / 4;
 302
 303                 if ((off >= (obj->base.size / 4)) ||
 304                                 (off < last_offset)) {
 305                         DRM_ERROR("invalid offset %u at reloc %u\n", off, i);
 306                         return -EINVAL;
 307                 }
 308
 309                 ret = submit_bo(submit, submit_reloc.reloc_idx, NULL, &iova, &valid);
 310                 if (ret)
 311                         return ret;
 312
 313                 if (valid)
 314                         continue;
 315
 316                 iova += submit_reloc.reloc_offset;
 317
 318                 if (submit_reloc.shift < 0)
 319                         iova >>= -submit_reloc.shift;
 320                 else
 321                         iova <<= submit_reloc.shift;
 322
 323                 ptr[off] = iova | submit_reloc.or;
 324
 325                 last_offset = off;
 326         }
 327
 328         return 0;
 329 }
 330
 331 static void submit_cleanup(struct msm_gem_submit *submit)
 332 {
 333         unsigned i;
 334
 335         for (i = 0; i < submit->nr_bos; i++) {
 336                 struct msm_gem_object *msm_obj = submit->bos[i].obj;
 337                 submit_unlock_unpin_bo(submit, i);
 338                 list_del_init(&msm_obj->submit_entry);
 339                 drm_gem_object_unreference(&msm_obj->base);
 340         }
 341
 342         ww_acquire_fini(&submit->ticket);
 343 }
 344
 345 int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
 346                 struct drm_file *file)
 347 {
 348         struct msm_drm_private *priv = dev->dev_private;
 349         struct drm_msm_gem_submit *args = data;
 350         struct msm_file_private *ctx = file->driver_priv;
 351         struct msm_gem_submit *submit;
 352         struct msm_gpu *gpu = priv->gpu;
 353         unsigned i;
 354         int ret;
 355
 356         if (!gpu)
 357                 return -ENXIO;
 358
 359         /* for now, we just have 3d pipe.. eventually this would need to
 360          * be more clever to dispatch to appropriate gpu module:
 361          */
 362         if (args->pipe != MSM_PIPE_3D0)
 363                 return -EINVAL;
 364
 365         if (args->nr_cmds > MAX_CMDS)
 366                 return -EINVAL;
 367
 368         submit = submit_create(dev, gpu, args->nr_bos);
 369         if (!submit)
 370                 return -ENOMEM;
 371
 372         mutex_lock(&dev->struct_mutex);
 373
 374         ret = submit_lookup_objects(submit, args, file);
 375         if (ret)
 376                 goto out;
 377
 378         ret = submit_lock_objects(submit);
 379         if (ret)
 380                 goto out;
 381
 382         ret = submit_fence_sync(submit);
 383         if (ret)
 384                 goto out;
 385
 386         ret = submit_pin_objects(submit);
 387         if (ret)
 388                 goto out;
 389
 390         for (i = 0; i < args->nr_cmds; i++) {
 391                 struct drm_msm_gem_submit_cmd submit_cmd;
 392                 void __user *userptr =
 393                         u64_to_user_ptr(args->cmds + (i * sizeof(submit_cmd)));
 394                 struct msm_gem_object *msm_obj;
 395                 uint32_t iova;
 396
 397                 ret = copy_from_user(&submit_cmd, userptr, sizeof(submit_cmd));
 398                 if (ret) {
 399                         ret = -EFAULT;
 400                         goto out;
 401                 }
 402
 403                 /* validate input from userspace: */
 404                 switch (submit_cmd.type) {
 405                 case MSM_SUBMIT_CMD_BUF:
 406                 case MSM_SUBMIT_CMD_IB_TARGET_BUF:
 407                 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
 408                         break;
 409                 default:
 410                         DRM_ERROR("invalid type: %08x\n", submit_cmd.type);
 411                         ret = -EINVAL;
 412                         goto out;
 413                 }
 414
 415                 ret = submit_bo(submit, submit_cmd.submit_idx,
 416                                 &msm_obj, &iova, NULL);
 417                 if (ret)
 418                         goto out;
 419
 420                 if (submit_cmd.size % 4) {
 421                         DRM_ERROR("non-aligned cmdstream buffer size: %u\n",
 422                                         submit_cmd.size);
 423                         ret = -EINVAL;
 424                         goto out;
 425                 }
 426
 427                 if ((submit_cmd.size + submit_cmd.submit_offset) >=
 428                                 msm_obj->base.size) {
 429                         DRM_ERROR("invalid cmdstream size: %u\n", submit_cmd.size);
 430                         ret = -EINVAL;
 431                         goto out;
 432                 }
 433
 434                 submit->cmd[i].type = submit_cmd.type;
 435                 submit->cmd[i].size = submit_cmd.size / 4;
 436                 submit->cmd[i].iova = iova + submit_cmd.submit_offset;
 437                 submit->cmd[i].idx  = submit_cmd.submit_idx;
 438
 439                 if (submit->valid)
 440                         continue;
 441
 442                 ret = submit_reloc(submit, msm_obj, submit_cmd.submit_offset,
 443                                 submit_cmd.nr_relocs, submit_cmd.relocs);
 444                 if (ret)
 445                         goto out;
 446         }
 447
 448         submit->nr_cmds = i;
 449
 450         ret = msm_gpu_submit(gpu, submit, ctx);
 451
 452         args->fence = submit->fence->seqno;
 453
 454 out:
 455         submit_cleanup(submit);
 456         if (ret)
 457                 msm_gem_submit_free(submit);
 458         mutex_unlock(&dev->struct_mutex);
 459         return ret;
 460 }