sys/dev/drm/i915/i915_gem_gtt.c

   1 /*
   2  * Copyright © 2010 Daniel Vetter
   3  * Copyright © 2011-2014 Intel Corporation
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <linux/seq_file.h>
  27 #include <drm/drmP.h>
  28 #include <drm/i915_drm.h>
  29 #include "i915_drv.h"
  30 #include "i915_vgpu.h"
  31 #include "i915_trace.h"
  32 #include "intel_drv.h"
  33
  34 #include <linux/bitmap.h>
  35 #include <linux/highmem.h>
  36
  37 /**
  38  * DOC: Global GTT views
  39  *
  40  * Background and previous state
  41  *
  42  * Historically objects could exists (be bound) in global GTT space only as
  43  * singular instances with a view representing all of the object's backing pages
  44  * in a linear fashion. This view will be called a normal view.
  45  *
  46  * To support multiple views of the same object, where the number of mapped
  47  * pages is not equal to the backing store, or where the layout of the pages
  48  * is not linear, concept of a GGTT view was added.
  49  *
  50  * One example of an alternative view is a stereo display driven by a single
  51  * image. In this case we would have a framebuffer looking like this
  52  * (2x2 pages):
  53  *
  54  *    12
  55  *    34
  56  *
  57  * Above would represent a normal GGTT view as normally mapped for GPU or CPU
  58  * rendering. In contrast, fed to the display engine would be an alternative
  59  * view which could look something like this:
  60  *
  61  *   1212
  62  *   3434
  63  *
  64  * In this example both the size and layout of pages in the alternative view is
  65  * different from the normal view.
  66  *
  67  * Implementation and usage
  68  *
  69  * GGTT views are implemented using VMAs and are distinguished via enum
  70  * i915_ggtt_view_type and struct i915_ggtt_view.
  71  *
  72  * A new flavour of core GEM functions which work with GGTT bound objects were
  73  * added with the _ggtt_ infix, and sometimes with _view postfix to avoid
  74  * renaming  in large amounts of code. They take the struct i915_ggtt_view
  75  * parameter encapsulating all metadata required to implement a view.
  76  *
  77  * As a helper for callers which are only interested in the normal view,
  78  * globally const i915_ggtt_view_normal singleton instance exists. All old core
  79  * GEM API functions, the ones not taking the view parameter, are operating on,
  80  * or with the normal GGTT view.
  81  *
  82  * Code wanting to add or use a new GGTT view needs to:
  83  *
  84  * 1. Add a new enum with a suitable name.
  85  * 2. Extend the metadata in the i915_ggtt_view structure if required.
  86  * 3. Add support to i915_get_vma_pages().
  87  *
  88  * New views are required to build a scatter-gather table from within the
  89  * i915_get_vma_pages function. This table is stored in the vma.ggtt_view and
  90  * exists for the lifetime of an VMA.
  91  *
  92  * Core API is designed to have copy semantics which means that passed in
  93  * struct i915_ggtt_view does not need to be persistent (left around after
  94  * calling the core API functions).
  95  *
  96  */
  97
  98 static int
  99 i915_get_ggtt_vma_pages(struct i915_vma *vma);
 100
 101 const struct i915_ggtt_view i915_ggtt_view_normal;
 102 const struct i915_ggtt_view i915_ggtt_view_rotated = {
 103         .type = I915_GGTT_VIEW_ROTATED
 104 };
 105
 106 static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)
 107 {
 108         bool has_aliasing_ppgtt;
 109         bool has_full_ppgtt;
 110
 111         has_aliasing_ppgtt = INTEL_INFO(dev)->gen >= 6;
 112         has_full_ppgtt = INTEL_INFO(dev)->gen >= 7;
 113
 114         if (intel_vgpu_active(dev))
 115                 has_full_ppgtt = false; /* emulation is too hard */
 116
 117         /*
 118          * We don't allow disabling PPGTT for gen9+ as it's a requirement for
 119          * execlists, the sole mechanism available to submit work.
 120          */
 121         if (INTEL_INFO(dev)->gen < 9 &&
 122             (enable_ppgtt == 0 || !has_aliasing_ppgtt))
 123                 return 0;
 124
 125         if (enable_ppgtt == 1)
 126                 return 1;
 127
 128         if (enable_ppgtt == 2 && has_full_ppgtt)
 129                 return 2;
 130
 131 #ifdef CONFIG_INTEL_IOMMU
 132         /* Disable ppgtt on SNB if VT-d is on. */
 133         if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) {
 134                 DRM_INFO("Disabling PPGTT because VT-d is on\n");
 135                 return 0;
 136         }
 137 #endif
 138
 139         /* Early VLV doesn't have this */
 140         if (IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) &&
 141             dev->pdev->revision < 0xb) {
 142                 DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n");
 143                 return 0;
 144         }
 145
 146         if (INTEL_INFO(dev)->gen >= 8 && i915.enable_execlists)
 147                 return 2;
 148         else
 149                 return has_aliasing_ppgtt ? 1 : 0;
 150 }
 151
 152 static int ppgtt_bind_vma(struct i915_vma *vma,
 153                           enum i915_cache_level cache_level,
 154                           u32 unused)
 155 {
 156         u32 pte_flags = 0;
 157
 158         /* Currently applicable only to VLV */
 159         if (vma->obj->gt_ro)
 160                 pte_flags |= PTE_READ_ONLY;
 161
 162         vma->vm->insert_entries(vma->vm, vma->obj->pages, vma->node.start,
 163                                 cache_level, pte_flags);
 164
 165         return 0;
 166 }
 167
 168 static void ppgtt_unbind_vma(struct i915_vma *vma)
 169 {
 170         vma->vm->clear_range(vma->vm,
 171                              vma->node.start,
 172                              vma->obj->base.size,
 173                              true);
 174 }
 175
 176 static gen8_pte_t gen8_pte_encode(dma_addr_t addr,
 177                                   enum i915_cache_level level,
 178                                   bool valid)
 179 {
 180         gen8_pte_t pte = valid ? _PAGE_PRESENT | _PAGE_RW : 0;
 181         pte |= addr;
 182
 183         switch (level) {
 184         case I915_CACHE_NONE:
 185                 pte |= PPAT_UNCACHED_INDEX;
 186                 break;
 187         case I915_CACHE_WT:
 188                 pte |= PPAT_DISPLAY_ELLC_INDEX;
 189                 break;
 190         default:
 191                 pte |= PPAT_CACHED_INDEX;
 192                 break;
 193         }
 194
 195         return pte;
 196 }
 197
 198 static gen8_pde_t gen8_pde_encode(const dma_addr_t addr,
 199                                   const enum i915_cache_level level)
 200 {
 201         gen8_pde_t pde = _PAGE_PRESENT | _PAGE_RW;
 202         pde |= addr;
 203         if (level != I915_CACHE_NONE)
 204                 pde |= PPAT_CACHED_PDE_INDEX;
 205         else
 206                 pde |= PPAT_UNCACHED_INDEX;
 207         return pde;
 208 }
 209
 210 #define gen8_pdpe_encode gen8_pde_encode
 211 #define gen8_pml4e_encode gen8_pde_encode
 212
 213 static gen6_pte_t snb_pte_encode(dma_addr_t addr,
 214                                  enum i915_cache_level level,
 215                                  bool valid, u32 unused)
 216 {
 217         gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
 218         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 219
 220         switch (level) {
 221         case I915_CACHE_L3_LLC:
 222         case I915_CACHE_LLC:
 223                 pte |= GEN6_PTE_CACHE_LLC;
 224                 break;
 225         case I915_CACHE_NONE:
 226                 pte |= GEN6_PTE_UNCACHED;
 227                 break;
 228         default:
 229                 MISSING_CASE(level);
 230         }
 231
 232         return pte;
 233 }
 234
 235 static gen6_pte_t ivb_pte_encode(dma_addr_t addr,
 236                                  enum i915_cache_level level,
 237                                  bool valid, u32 unused)
 238 {
 239         gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
 240         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 241
 242         switch (level) {
 243         case I915_CACHE_L3_LLC:
 244                 pte |= GEN7_PTE_CACHE_L3_LLC;
 245                 break;
 246         case I915_CACHE_LLC:
 247                 pte |= GEN6_PTE_CACHE_LLC;
 248                 break;
 249         case I915_CACHE_NONE:
 250                 pte |= GEN6_PTE_UNCACHED;
 251                 break;
 252         default:
 253                 MISSING_CASE(level);
 254         }
 255
 256         return pte;
 257 }
 258
 259 static gen6_pte_t byt_pte_encode(dma_addr_t addr,
 260                                  enum i915_cache_level level,
 261                                  bool valid, u32 flags)
 262 {
 263         gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
 264         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 265
 266         if (!(flags & PTE_READ_ONLY))
 267                 pte |= BYT_PTE_WRITEABLE;
 268
 269         if (level != I915_CACHE_NONE)
 270                 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
 271
 272         return pte;
 273 }
 274
 275 static gen6_pte_t hsw_pte_encode(dma_addr_t addr,
 276                                  enum i915_cache_level level,
 277                                  bool valid, u32 unused)
 278 {
 279         gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
 280         pte |= HSW_PTE_ADDR_ENCODE(addr);
 281
 282         if (level != I915_CACHE_NONE)
 283                 pte |= HSW_WB_LLC_AGE3;
 284
 285         return pte;
 286 }
 287
 288 static gen6_pte_t iris_pte_encode(dma_addr_t addr,
 289                                   enum i915_cache_level level,
 290                                   bool valid, u32 unused)
 291 {
 292         gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
 293         pte |= HSW_PTE_ADDR_ENCODE(addr);
 294
 295         switch (level) {
 296         case I915_CACHE_NONE:
 297                 break;
 298         case I915_CACHE_WT:
 299                 pte |= HSW_WT_ELLC_LLC_AGE3;
 300                 break;
 301         default:
 302                 pte |= HSW_WB_ELLC_LLC_AGE3;
 303                 break;
 304         }
 305
 306         return pte;
 307 }
 308
 309 static int __setup_page_dma(struct drm_device *dev,
 310                             struct i915_page_dma *p, gfp_t flags)
 311 {
 312         struct device *device = dev->pdev->dev;
 313
 314         p->page = alloc_page(flags);
 315         if (!p->page)
 316                 return -ENOMEM;
 317
 318         p->daddr = dma_map_page(device,
 319                                 p->page, 0, 4096, PCI_DMA_BIDIRECTIONAL);
 320
 321         if (dma_mapping_error(device, p->daddr)) {
 322                 __free_page(p->page);
 323                 return -EINVAL;
 324         }
 325
 326         return 0;
 327 }
 328
 329 static int setup_page_dma(struct drm_device *dev, struct i915_page_dma *p)
 330 {
 331         return __setup_page_dma(dev, p, GFP_KERNEL);
 332 }
 333
 334 static void cleanup_page_dma(struct drm_device *dev, struct i915_page_dma *p)
 335 {
 336         if (WARN_ON(!p->page))
 337                 return;
 338
 339         dma_unmap_page(dev->pdev->dev, p->daddr, 4096, PCI_DMA_BIDIRECTIONAL);
 340         __free_page(p->page);
 341         memset(p, 0, sizeof(*p));
 342 }
 343
 344 static void *kmap_page_dma(struct i915_page_dma *p)
 345 {
 346         return kmap_atomic(p->page);
 347 }
 348
 349 /* We use the flushing unmap only with ppgtt structures:
 350  * page directories, page tables and scratch pages.
 351  */
 352 static void kunmap_page_dma(struct drm_device *dev, void *vaddr)
 353 {
 354         /* There are only few exceptions for gen >=6. chv and bxt.
 355          * And we are not sure about the latter so play safe for now.
 356          */
 357         if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev))
 358                 drm_clflush_virt_range(vaddr, PAGE_SIZE);
 359
 360         kunmap_atomic(vaddr);
 361 }
 362
 363 #define kmap_px(px) kmap_page_dma(px_base(px))
 364 #define kunmap_px(ppgtt, vaddr) kunmap_page_dma((ppgtt)->base.dev, (vaddr))
 365
 366 #define setup_px(dev, px) setup_page_dma((dev), px_base(px))
 367 #define cleanup_px(dev, px) cleanup_page_dma((dev), px_base(px))
 368 #define fill_px(dev, px, v) fill_page_dma((dev), px_base(px), (v))
 369 #define fill32_px(dev, px, v) fill_page_dma_32((dev), px_base(px), (v))
 370
 371 static void fill_page_dma(struct drm_device *dev, struct i915_page_dma *p,
 372                           const uint64_t val)
 373 {
 374         int i;
 375         uint64_t * const vaddr = kmap_page_dma(p);
 376
 377         for (i = 0; i < 512; i++)
 378                 vaddr[i] = val;
 379
 380         kunmap_page_dma(dev, vaddr);
 381 }
 382
 383 static void fill_page_dma_32(struct drm_device *dev, struct i915_page_dma *p,
 384                              const uint32_t val32)
 385 {
 386         uint64_t v = val32;
 387
 388         v = v << 32 | val32;
 389
 390         fill_page_dma(dev, p, v);
 391 }
 392
 393 static struct i915_page_scratch *alloc_scratch_page(struct drm_device *dev)
 394 {
 395         struct i915_page_scratch *sp;
 396         int ret;
 397
 398         sp = kzalloc(sizeof(*sp), GFP_KERNEL);
 399         if (sp == NULL)
 400                 return ERR_PTR(-ENOMEM);
 401
 402         ret = __setup_page_dma(dev, px_base(sp), GFP_DMA32 | __GFP_ZERO);
 403         if (ret) {
 404                 kfree(sp);
 405                 return ERR_PTR(ret);
 406         }
 407
 408         set_pages_uc(px_page(sp), 1);
 409
 410         return sp;
 411 }
 412
 413 static void free_scratch_page(struct drm_device *dev,
 414                               struct i915_page_scratch *sp)
 415 {
 416         set_pages_wb(px_page(sp), 1);
 417
 418         cleanup_px(dev, sp);
 419         kfree(sp);
 420 }
 421
 422 static struct i915_page_table *alloc_pt(struct drm_device *dev)
 423 {
 424         struct i915_page_table *pt;
 425         const size_t count = INTEL_INFO(dev)->gen >= 8 ?
 426                 GEN8_PTES : GEN6_PTES;
 427         int ret = -ENOMEM;
 428
 429         pt = kzalloc(sizeof(*pt), GFP_KERNEL);
 430         if (!pt)
 431                 return ERR_PTR(-ENOMEM);
 432
 433         pt->used_ptes = kcalloc(BITS_TO_LONGS(count), sizeof(*pt->used_ptes),
 434                                 GFP_KERNEL);
 435
 436         if (!pt->used_ptes)
 437                 goto fail_bitmap;
 438
 439         ret = setup_px(dev, pt);
 440         if (ret)
 441                 goto fail_page_m;
 442
 443         return pt;
 444
 445 fail_page_m:
 446         kfree(pt->used_ptes);
 447 fail_bitmap:
 448         kfree(pt);
 449
 450         return ERR_PTR(ret);
 451 }
 452
 453 static void free_pt(struct drm_device *dev, struct i915_page_table *pt)
 454 {
 455         cleanup_px(dev, pt);
 456         kfree(pt->used_ptes);
 457         kfree(pt);
 458 }
 459
 460 /**
 461  * alloc_pt_range() - Allocate a multiple page tables
 462  * @pd:         The page directory which will have at least @count entries
 463  *              available to point to the allocated page tables.
 464  * @pde:        First page directory entry for which we are allocating.
 465  * @count:      Number of pages to allocate.
 466  * @dev:        DRM device.
 467  *
 468  * Allocates multiple page table pages and sets the appropriate entries in the
 469  * page table structure within the page directory. Function cleans up after
 470  * itself on any failures.
 471  *
 472  * Return: 0 if allocation succeeded.
 473  */
 474 static int alloc_pt_range(struct i915_page_directory *pd, uint16_t pde, size_t count,
 475                           struct drm_device *dev)
 476 {
 477         int i, ret;
 478
 479         /* 512 is the max page tables per page_directory on any platform. */
 480         if (WARN_ON(pde + count > I915_PDES))
 481                 return -EINVAL;
 482
 483         for (i = pde; i < pde + count; i++) {
 484                 struct i915_page_table *pt = alloc_pt(dev);
 485
 486                 if (IS_ERR(pt)) {
 487                         ret = PTR_ERR(pt);
 488                         goto err_out;
 489                 }
 490                 WARN(pd->page_table[i],
 491                      "Leaking page directory entry %d (%p)\n",
 492                      i, pd->page_table[i]);
 493                 pd->page_table[i] = pt;
 494         }
 495
 496         return 0;
 497
 498 err_out:
 499         while (i-- > pde)
 500                 free_pt(dev, pd->page_table[i]);
 501         return ret;
 502 }
 503
 504 static void gen8_initialize_pt(struct i915_address_space *vm,
 505                                struct i915_page_table *pt)
 506 {
 507         gen8_pte_t scratch_pte;
 508
 509         scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page),
 510                                       I915_CACHE_LLC, true);
 511
 512         fill_px(vm->dev, pt, scratch_pte);
 513 }
 514
 515 static void gen6_initialize_pt(struct i915_address_space *vm,
 516                                struct i915_page_table *pt)
 517 {
 518         gen6_pte_t scratch_pte;
 519
 520         WARN_ON(px_dma(vm->scratch_page) == 0);
 521
 522         scratch_pte = vm->pte_encode(px_dma(vm->scratch_page),
 523                                      I915_CACHE_LLC, true, 0);
 524
 525         fill32_px(vm->dev, pt, scratch_pte);
 526 }
 527
 528 static struct i915_page_directory *alloc_pd(struct drm_device *dev)
 529 {
 530         struct i915_page_directory *pd;
 531         int ret = -ENOMEM;
 532
 533         pd = kzalloc(sizeof(*pd), GFP_KERNEL);
 534         if (!pd)
 535                 return ERR_PTR(-ENOMEM);
 536
 537         pd->used_pdes = kcalloc(BITS_TO_LONGS(I915_PDES),
 538                                 sizeof(*pd->used_pdes), GFP_KERNEL);
 539         if (!pd->used_pdes)
 540                 goto fail_bitmap;
 541
 542         ret = setup_px(dev, pd);
 543         if (ret)
 544                 goto fail_page_m;
 545
 546         return pd;
 547
 548 fail_page_m:
 549         kfree(pd->used_pdes);
 550 fail_bitmap:
 551         kfree(pd);
 552
 553         return ERR_PTR(ret);
 554 }
 555
 556 static void free_pd(struct drm_device *dev, struct i915_page_directory *pd)
 557 {
 558         if (px_page(pd)) {
 559                 cleanup_px(dev, pd);
 560                 kfree(pd->used_pdes);
 561                 kfree(pd);
 562         }
 563 }
 564
 565 static void gen8_initialize_pd(struct i915_address_space *vm,
 566                                struct i915_page_directory *pd)
 567 {
 568         gen8_pde_t scratch_pde;
 569
 570         scratch_pde = gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC);
 571
 572         fill_px(vm->dev, pd, scratch_pde);
 573 }
 574
 575 static int __pdp_init(struct drm_device *dev,
 576                       struct i915_page_directory_pointer *pdp)
 577 {
 578         size_t pdpes = I915_PDPES_PER_PDP(dev);
 579
 580         pdp->used_pdpes = kcalloc(BITS_TO_LONGS(pdpes),
 581                                   sizeof(unsigned long),
 582                                   GFP_KERNEL);
 583         if (!pdp->used_pdpes)
 584                 return -ENOMEM;
 585
 586         pdp->page_directory = kcalloc(pdpes, sizeof(*pdp->page_directory),
 587                                       GFP_KERNEL);
 588         if (!pdp->page_directory) {
 589                 kfree(pdp->used_pdpes);
 590                 /* the PDP might be the statically allocated top level. Keep it
 591                  * as clean as possible */
 592                 pdp->used_pdpes = NULL;
 593                 return -ENOMEM;
 594         }
 595
 596         return 0;
 597 }
 598
 599 static void __pdp_fini(struct i915_page_directory_pointer *pdp)
 600 {
 601         kfree(pdp->used_pdpes);
 602         kfree(pdp->page_directory);
 603         pdp->page_directory = NULL;
 604 }
 605
 606 static struct
 607 i915_page_directory_pointer *alloc_pdp(struct drm_device *dev)
 608 {
 609         struct i915_page_directory_pointer *pdp;
 610         int ret = -ENOMEM;
 611
 612         WARN_ON(!USES_FULL_48BIT_PPGTT(dev));
 613
 614         pdp = kzalloc(sizeof(*pdp), GFP_KERNEL);
 615         if (!pdp)
 616                 return ERR_PTR(-ENOMEM);
 617
 618         ret = __pdp_init(dev, pdp);
 619         if (ret)
 620                 goto fail_bitmap;
 621
 622         ret = setup_px(dev, pdp);
 623         if (ret)
 624                 goto fail_page_m;
 625
 626         return pdp;
 627
 628 fail_page_m:
 629         __pdp_fini(pdp);
 630 fail_bitmap:
 631         kfree(pdp);
 632
 633         return ERR_PTR(ret);
 634 }
 635
 636 static void free_pdp(struct drm_device *dev,
 637                      struct i915_page_directory_pointer *pdp)
 638 {
 639         __pdp_fini(pdp);
 640         if (USES_FULL_48BIT_PPGTT(dev)) {
 641                 cleanup_px(dev, pdp);
 642                 kfree(pdp);
 643         }
 644 }
 645
 646 static void gen8_initialize_pdp(struct i915_address_space *vm,
 647                                 struct i915_page_directory_pointer *pdp)
 648 {
 649         gen8_ppgtt_pdpe_t scratch_pdpe;
 650
 651         scratch_pdpe = gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC);
 652
 653         fill_px(vm->dev, pdp, scratch_pdpe);
 654 }
 655
 656 static void gen8_initialize_pml4(struct i915_address_space *vm,
 657                                  struct i915_pml4 *pml4)
 658 {
 659         gen8_ppgtt_pml4e_t scratch_pml4e;
 660
 661         scratch_pml4e = gen8_pml4e_encode(px_dma(vm->scratch_pdp),
 662                                           I915_CACHE_LLC);
 663
 664         fill_px(vm->dev, pml4, scratch_pml4e);
 665 }
 666
 667 static void
 668 gen8_setup_page_directory(struct i915_hw_ppgtt *ppgtt,
 669                           struct i915_page_directory_pointer *pdp,
 670                           struct i915_page_directory *pd,
 671                           int index)
 672 {
 673         gen8_ppgtt_pdpe_t *page_directorypo;
 674
 675         if (!USES_FULL_48BIT_PPGTT(ppgtt->base.dev))
 676                 return;
 677
 678         page_directorypo = kmap_px(pdp);
 679         page_directorypo[index] = gen8_pdpe_encode(px_dma(pd), I915_CACHE_LLC);
 680         kunmap_px(ppgtt, page_directorypo);
 681 }
 682
 683 static void
 684 gen8_setup_page_directory_pointer(struct i915_hw_ppgtt *ppgtt,
 685                                   struct i915_pml4 *pml4,
 686                                   struct i915_page_directory_pointer *pdp,
 687                                   int index)
 688 {
 689         gen8_ppgtt_pml4e_t *pagemap = kmap_px(pml4);
 690
 691         WARN_ON(!USES_FULL_48BIT_PPGTT(ppgtt->base.dev));
 692         pagemap[index] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC);
 693         kunmap_px(ppgtt, pagemap);
 694 }
 695
 696 /* Broadwell Page Directory Pointer Descriptors */
 697 static int gen8_write_pdp(struct drm_i915_gem_request *req,
 698                           unsigned entry,
 699                           dma_addr_t addr)
 700 {
 701         struct intel_engine_cs *ring = req->ring;
 702         int ret;
 703
 704         BUG_ON(entry >= 4);
 705
 706         ret = intel_ring_begin(req, 6);
 707         if (ret)
 708                 return ret;
 709
 710         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
 711         intel_ring_emit(ring, GEN8_RING_PDP_UDW(ring, entry));
 712         intel_ring_emit(ring, upper_32_bits(addr));
 713         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
 714         intel_ring_emit(ring, GEN8_RING_PDP_LDW(ring, entry));
 715         intel_ring_emit(ring, lower_32_bits(addr));
 716         intel_ring_advance(ring);
 717
 718         return 0;
 719 }
 720
 721 static int gen8_legacy_mm_switch(struct i915_hw_ppgtt *ppgtt,
 722                                  struct drm_i915_gem_request *req)
 723 {
 724         int i, ret;
 725
 726         for (i = GEN8_LEGACY_PDPES - 1; i >= 0; i--) {
 727                 const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i);
 728
 729                 ret = gen8_write_pdp(req, i, pd_daddr);
 730                 if (ret)
 731                         return ret;
 732         }
 733
 734         return 0;
 735 }
 736
 737 static int gen8_48b_mm_switch(struct i915_hw_ppgtt *ppgtt,
 738                               struct drm_i915_gem_request *req)
 739 {
 740         return gen8_write_pdp(req, 0, px_dma(&ppgtt->pml4));
 741 }
 742
 743 static void gen8_ppgtt_clear_pte_range(struct i915_address_space *vm,
 744                                        struct i915_page_directory_pointer *pdp,
 745                                        uint64_t start,
 746                                        uint64_t length,
 747                                        gen8_pte_t scratch_pte)
 748 {
 749         struct i915_hw_ppgtt *ppgtt =
 750                 container_of(vm, struct i915_hw_ppgtt, base);
 751         gen8_pte_t *pt_vaddr;
 752         unsigned pdpe = gen8_pdpe_index(start);
 753         unsigned pde = gen8_pde_index(start);
 754         unsigned pte = gen8_pte_index(start);
 755         unsigned num_entries = length >> PAGE_SHIFT;
 756         unsigned last_pte, i;
 757
 758         if (WARN_ON(!pdp))
 759                 return;
 760
 761         while (num_entries) {
 762                 struct i915_page_directory *pd;
 763                 struct i915_page_table *pt;
 764
 765                 if (WARN_ON(!pdp->page_directory[pdpe]))
 766                         break;
 767
 768                 pd = pdp->page_directory[pdpe];
 769
 770                 if (WARN_ON(!pd->page_table[pde]))
 771                         break;
 772
 773                 pt = pd->page_table[pde];
 774
 775                 if (WARN_ON(!px_page(pt)))
 776                         break;
 777
 778                 last_pte = pte + num_entries;
 779                 if (last_pte > GEN8_PTES)
 780                         last_pte = GEN8_PTES;
 781
 782                 pt_vaddr = kmap_px(pt);
 783
 784                 for (i = pte; i < last_pte; i++) {
 785                         pt_vaddr[i] = scratch_pte;
 786                         num_entries--;
 787                 }
 788
 789                 kunmap_px(ppgtt, pt);
 790
 791                 pte = 0;
 792                 if (++pde == I915_PDES) {
 793                         if (++pdpe == I915_PDPES_PER_PDP(vm->dev))
 794                                 break;
 795                         pde = 0;
 796                 }
 797         }
 798 }
 799
 800 static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
 801                                    uint64_t start,
 802                                    uint64_t length,
 803                                    bool use_scratch)
 804 {
 805         struct i915_hw_ppgtt *ppgtt =
 806                 container_of(vm, struct i915_hw_ppgtt, base);
 807         gen8_pte_t scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page),
 808                                                  I915_CACHE_LLC, use_scratch);
 809
 810         if (!USES_FULL_48BIT_PPGTT(vm->dev)) {
 811                 gen8_ppgtt_clear_pte_range(vm, &ppgtt->pdp, start, length,
 812                                            scratch_pte);
 813         } else {
 814                 uint64_t templ4, pml4e;
 815                 struct i915_page_directory_pointer *pdp;
 816
 817                 gen8_for_each_pml4e(pdp, &ppgtt->pml4, start, length, templ4, pml4e) {
 818                         gen8_ppgtt_clear_pte_range(vm, pdp, start, length,
 819                                                    scratch_pte);
 820                 }
 821         }
 822 }
 823
 824 static void
 825 gen8_ppgtt_insert_pte_entries(struct i915_address_space *vm,
 826                               struct i915_page_directory_pointer *pdp,
 827                               struct sg_page_iter *sg_iter,
 828                               uint64_t start,
 829                               enum i915_cache_level cache_level)
 830 {
 831         struct i915_hw_ppgtt *ppgtt =
 832                 container_of(vm, struct i915_hw_ppgtt, base);
 833         gen8_pte_t *pt_vaddr;
 834         unsigned pdpe = gen8_pdpe_index(start);
 835         unsigned pde = gen8_pde_index(start);
 836         unsigned pte = gen8_pte_index(start);
 837
 838         pt_vaddr = NULL;
 839
 840         while (__sg_page_iter_next(sg_iter)) {
 841                 if (pt_vaddr == NULL) {
 842                         struct i915_page_directory *pd = pdp->page_directory[pdpe];
 843                         struct i915_page_table *pt = pd->page_table[pde];
 844
 845                         pt_vaddr = kmap_px(pt);
 846                 }
 847
 848                 pt_vaddr[pte] =
 849                         gen8_pte_encode(sg_page_iter_dma_address(sg_iter),
 850                                         cache_level, true);
 851                 if (++pte == GEN8_PTES) {
 852                         kunmap_px(ppgtt, pt_vaddr);
 853                         pt_vaddr = NULL;
 854                         if (++pde == I915_PDES) {
 855                                 if (++pdpe == I915_PDPES_PER_PDP(vm->dev))
 856                                         break;
 857                                 pde = 0;
 858                         }
 859                         pte = 0;
 860                 }
 861         }
 862
 863         if (pt_vaddr)
 864                 kunmap_px(ppgtt, pt_vaddr);
 865 }
 866
 867 static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
 868                                       struct sg_table *pages,
 869                                       uint64_t start,
 870                                       enum i915_cache_level cache_level,
 871                                       u32 unused)
 872 {
 873         struct i915_hw_ppgtt *ppgtt =
 874                 container_of(vm, struct i915_hw_ppgtt, base);
 875         struct sg_page_iter sg_iter;
 876
 877         __sg_page_iter_start(&sg_iter, pages->sgl, sg_nents(pages->sgl), 0);
 878
 879         if (!USES_FULL_48BIT_PPGTT(vm->dev)) {
 880                 gen8_ppgtt_insert_pte_entries(vm, &ppgtt->pdp, &sg_iter, start,
 881                                               cache_level);
 882         } else {
 883                 struct i915_page_directory_pointer *pdp;
 884                 uint64_t templ4, pml4e;
 885                 uint64_t length = (uint64_t)pages->orig_nents << PAGE_SHIFT;
 886
 887                 gen8_for_each_pml4e(pdp, &ppgtt->pml4, start, length, templ4, pml4e) {
 888                         gen8_ppgtt_insert_pte_entries(vm, pdp, &sg_iter,
 889                                                       start, cache_level);
 890                 }
 891         }
 892 }
 893
 894 static void gen8_free_page_tables(struct drm_device *dev,
 895                                   struct i915_page_directory *pd)
 896 {
 897         int i;
 898
 899         if (!px_page(pd))
 900                 return;
 901
 902         for_each_set_bit(i, pd->used_pdes, I915_PDES) {
 903                 if (WARN_ON(!pd->page_table[i]))
 904                         continue;
 905
 906                 free_pt(dev, pd->page_table[i]);
 907                 pd->page_table[i] = NULL;
 908         }
 909 }
 910
 911 static int gen8_init_scratch(struct i915_address_space *vm)
 912 {
 913         struct drm_device *dev = vm->dev;
 914
 915         vm->scratch_page = alloc_scratch_page(dev);
 916         if (IS_ERR(vm->scratch_page))
 917                 return PTR_ERR(vm->scratch_page);
 918
 919         vm->scratch_pt = alloc_pt(dev);
 920         if (IS_ERR(vm->scratch_pt)) {
 921                 free_scratch_page(dev, vm->scratch_page);
 922                 return PTR_ERR(vm->scratch_pt);
 923         }
 924
 925         vm->scratch_pd = alloc_pd(dev);
 926         if (IS_ERR(vm->scratch_pd)) {
 927                 free_pt(dev, vm->scratch_pt);
 928                 free_scratch_page(dev, vm->scratch_page);
 929                 return PTR_ERR(vm->scratch_pd);
 930         }
 931
 932         if (USES_FULL_48BIT_PPGTT(dev)) {
 933                 vm->scratch_pdp = alloc_pdp(dev);
 934                 if (IS_ERR(vm->scratch_pdp)) {
 935                         free_pd(dev, vm->scratch_pd);
 936                         free_pt(dev, vm->scratch_pt);
 937                         free_scratch_page(dev, vm->scratch_page);
 938                         return PTR_ERR(vm->scratch_pdp);
 939                 }
 940         }
 941
 942         gen8_initialize_pt(vm, vm->scratch_pt);
 943         gen8_initialize_pd(vm, vm->scratch_pd);
 944         if (USES_FULL_48BIT_PPGTT(dev))
 945                 gen8_initialize_pdp(vm, vm->scratch_pdp);
 946
 947         return 0;
 948 }
 949
 950 static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create)
 951 {
 952         enum vgt_g2v_type msg;
 953         struct drm_device *dev = ppgtt->base.dev;
 954         struct drm_i915_private *dev_priv = dev->dev_private;
 955         unsigned int offset = vgtif_reg(pdp0_lo);
 956         int i;
 957
 958         if (USES_FULL_48BIT_PPGTT(dev)) {
 959                 u64 daddr = px_dma(&ppgtt->pml4);
 960
 961                 I915_WRITE(offset, lower_32_bits(daddr));
 962                 I915_WRITE(offset + 4, upper_32_bits(daddr));
 963
 964                 msg = (create ? VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE :
 965                                 VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY);
 966         } else {
 967                 for (i = 0; i < GEN8_LEGACY_PDPES; i++) {
 968                         u64 daddr = i915_page_dir_dma_addr(ppgtt, i);
 969
 970                         I915_WRITE(offset, lower_32_bits(daddr));
 971                         I915_WRITE(offset + 4, upper_32_bits(daddr));
 972
 973                         offset += 8;
 974                 }
 975
 976                 msg = (create ? VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE :
 977                                 VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY);
 978         }
 979
 980         I915_WRITE(vgtif_reg(g2v_notify), msg);
 981
 982         return 0;
 983 }
 984
 985 static void gen8_free_scratch(struct i915_address_space *vm)
 986 {
 987         struct drm_device *dev = vm->dev;
 988
 989         if (USES_FULL_48BIT_PPGTT(dev))
 990                 free_pdp(dev, vm->scratch_pdp);
 991         free_pd(dev, vm->scratch_pd);
 992         free_pt(dev, vm->scratch_pt);
 993         free_scratch_page(dev, vm->scratch_page);
 994 }
 995
 996 static void gen8_ppgtt_cleanup_3lvl(struct drm_device *dev,
 997                                     struct i915_page_directory_pointer *pdp)
 998 {
 999         int i;
1000
1001         for_each_set_bit(i, pdp->used_pdpes, I915_PDPES_PER_PDP(dev)) {
1002                 if (WARN_ON(!pdp->page_directory[i]))
1003                         continue;
1004
1005                 gen8_free_page_tables(dev, pdp->page_directory[i]);
1006                 free_pd(dev, pdp->page_directory[i]);
1007         }
1008
1009         free_pdp(dev, pdp);
1010 }
1011
1012 static void gen8_ppgtt_cleanup_4lvl(struct i915_hw_ppgtt *ppgtt)
1013 {
1014         int i;
1015
1016         for_each_set_bit(i, ppgtt->pml4.used_pml4es, GEN8_PML4ES_PER_PML4) {
1017                 if (WARN_ON(!ppgtt->pml4.pdps[i]))
1018                         continue;
1019
1020                 gen8_ppgtt_cleanup_3lvl(ppgtt->base.dev, ppgtt->pml4.pdps[i]);
1021         }
1022
1023         cleanup_px(ppgtt->base.dev, &ppgtt->pml4);
1024 }
1025
1026 static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
1027 {
1028         struct i915_hw_ppgtt *ppgtt =
1029                 container_of(vm, struct i915_hw_ppgtt, base);
1030
1031         if (intel_vgpu_active(vm->dev))
1032                 gen8_ppgtt_notify_vgt(ppgtt, false);
1033
1034         if (!USES_FULL_48BIT_PPGTT(ppgtt->base.dev))
1035                 gen8_ppgtt_cleanup_3lvl(ppgtt->base.dev, &ppgtt->pdp);
1036         else
1037                 gen8_ppgtt_cleanup_4lvl(ppgtt);
1038
1039         gen8_free_scratch(vm);
1040 }
1041
1042 /**
1043  * gen8_ppgtt_alloc_pagetabs() - Allocate page tables for VA range.
1044  * @vm: Master vm structure.
1045  * @pd: Page directory for this address range.
1046  * @start:      Starting virtual address to begin allocations.
1047  * @length:     Size of the allocations.
1048  * @new_pts:    Bitmap set by function with new allocations. Likely used by the
1049  *              caller to free on error.
1050  *
1051  * Allocate the required number of page tables. Extremely similar to
1052  * gen8_ppgtt_alloc_page_directories(). The main difference is here we are limited by
1053  * the page directory boundary (instead of the page directory pointer). That
1054  * boundary is 1GB virtual. Therefore, unlike gen8_ppgtt_alloc_page_directories(), it is
1055  * possible, and likely that the caller will need to use multiple calls of this
1056  * function to achieve the appropriate allocation.
1057  *
1058  * Return: 0 if success; negative error code otherwise.
1059  */
1060 static int gen8_ppgtt_alloc_pagetabs(struct i915_address_space *vm,
1061                                      struct i915_page_directory *pd,
1062                                      uint64_t start,
1063                                      uint64_t length,
1064                                      unsigned long *new_pts)
1065 {
1066         struct drm_device *dev = vm->dev;
1067         struct i915_page_table *pt;
1068         uint64_t temp;
1069         uint32_t pde;
1070
1071         gen8_for_each_pde(pt, pd, start, length, temp, pde) {
1072                 /* Don't reallocate page tables */
1073                 if (test_bit(pde, pd->used_pdes)) {
1074                         /* Scratch is never allocated this way */
1075                         WARN_ON(pt == vm->scratch_pt);
1076                         continue;
1077                 }
1078
1079                 pt = alloc_pt(dev);
1080                 if (IS_ERR(pt))
1081                         goto unwind_out;
1082
1083                 gen8_initialize_pt(vm, pt);
1084                 pd->page_table[pde] = pt;
1085                 __set_bit(pde, new_pts);
1086                 trace_i915_page_table_entry_alloc(vm, pde, start, GEN8_PDE_SHIFT);
1087         }
1088
1089         return 0;
1090
1091 unwind_out:
1092         for_each_set_bit(pde, new_pts, I915_PDES)
1093                 free_pt(dev, pd->page_table[pde]);
1094
1095         return -ENOMEM;
1096 }
1097
1098 /**
1099  * gen8_ppgtt_alloc_page_directories() - Allocate page directories for VA range.
1100  * @vm: Master vm structure.
1101  * @pdp:        Page directory pointer for this address range.
1102  * @start:      Starting virtual address to begin allocations.
1103  * @length:     Size of the allocations.
1104  * @new_pds:    Bitmap set by function with new allocations. Likely used by the
1105  *              caller to free on error.
1106  *
1107  * Allocate the required number of page directories starting at the pde index of
1108  * @start, and ending at the pde index @start + @length. This function will skip
1109  * over already allocated page directories within the range, and only allocate
1110  * new ones, setting the appropriate pointer within the pdp as well as the
1111  * correct position in the bitmap @new_pds.
1112  *
1113  * The function will only allocate the pages within the range for a give page
1114  * directory pointer. In other words, if @start + @length straddles a virtually
1115  * addressed PDP boundary (512GB for 4k pages), there will be more allocations
1116  * required by the caller, This is not currently possible, and the BUG in the
1117  * code will prevent it.
1118  *
1119  * Return: 0 if success; negative error code otherwise.
1120  */
1121 static int
1122 gen8_ppgtt_alloc_page_directories(struct i915_address_space *vm,
1123                                   struct i915_page_directory_pointer *pdp,
1124                                   uint64_t start,
1125                                   uint64_t length,
1126                                   unsigned long *new_pds)
1127 {
1128         struct drm_device *dev = vm->dev;
1129         struct i915_page_directory *pd;
1130         uint64_t temp;
1131         uint32_t pdpe;
1132         uint32_t pdpes = I915_PDPES_PER_PDP(dev);
1133
1134         WARN_ON(!bitmap_empty(new_pds, pdpes));
1135
1136         gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {
1137                 if (test_bit(pdpe, pdp->used_pdpes))
1138                         continue;
1139
1140                 pd = alloc_pd(dev);
1141                 if (IS_ERR(pd))
1142                         goto unwind_out;
1143
1144                 gen8_initialize_pd(vm, pd);
1145                 pdp->page_directory[pdpe] = pd;
1146                 __set_bit(pdpe, new_pds);
1147                 trace_i915_page_directory_entry_alloc(vm, pdpe, start, GEN8_PDPE_SHIFT);
1148         }
1149
1150         return 0;
1151
1152 unwind_out:
1153         for_each_set_bit(pdpe, new_pds, pdpes)
1154                 free_pd(dev, pdp->page_directory[pdpe]);
1155
1156         return -ENOMEM;
1157 }
1158
1159 /**
1160  * gen8_ppgtt_alloc_page_dirpointers() - Allocate pdps for VA range.
1161  * @vm: Master vm structure.
1162  * @pml4:       Page map level 4 for this address range.
1163  * @start:      Starting virtual address to begin allocations.
1164  * @length:     Size of the allocations.
1165  * @new_pdps:   Bitmap set by function with new allocations. Likely used by the
1166  *              caller to free on error.
1167  *
1168  * Allocate the required number of page directory pointers. Extremely similar to
1169  * gen8_ppgtt_alloc_page_directories() and gen8_ppgtt_alloc_pagetabs().
1170  * The main difference is here we are limited by the pml4 boundary (instead of
1171  * the page directory pointer).
1172  *
1173  * Return: 0 if success; negative error code otherwise.
1174  */
1175 static int
1176 gen8_ppgtt_alloc_page_dirpointers(struct i915_address_space *vm,
1177                                   struct i915_pml4 *pml4,
1178                                   uint64_t start,
1179                                   uint64_t length,
1180                                   unsigned long *new_pdps)
1181 {
1182         struct drm_device *dev = vm->dev;
1183         struct i915_page_directory_pointer *pdp;
1184         uint64_t temp;
1185         uint32_t pml4e;
1186
1187         WARN_ON(!bitmap_empty(new_pdps, GEN8_PML4ES_PER_PML4));
1188
1189         gen8_for_each_pml4e(pdp, pml4, start, length, temp, pml4e) {
1190                 if (!test_bit(pml4e, pml4->used_pml4es)) {
1191                         pdp = alloc_pdp(dev);
1192                         if (IS_ERR(pdp))
1193                                 goto unwind_out;
1194
1195                         gen8_initialize_pdp(vm, pdp);
1196                         pml4->pdps[pml4e] = pdp;
1197                         __set_bit(pml4e, new_pdps);
1198                         trace_i915_page_directory_pointer_entry_alloc(vm,
1199                                                                       pml4e,
1200                                                                       start,
1201                                                                       GEN8_PML4E_SHIFT);
1202                 }
1203         }
1204
1205         return 0;
1206
1207 unwind_out:
1208         for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4)
1209                 free_pdp(dev, pml4->pdps[pml4e]);
1210
1211         return -ENOMEM;
1212 }
1213
1214 static void
1215 free_gen8_temp_bitmaps(unsigned long *new_pds, unsigned long *new_pts)
1216 {
1217         kfree(new_pts);
1218         kfree(new_pds);
1219 }
1220
1221 /* Fills in the page directory bitmap, and the array of page tables bitmap. Both
1222  * of these are based on the number of PDPEs in the system.
1223  */
1224 static
1225 int __must_check alloc_gen8_temp_bitmaps(unsigned long **new_pds,
1226                                          unsigned long **new_pts,
1227                                          uint32_t pdpes)
1228 {
1229         unsigned long *pds;
1230         unsigned long *pts;
1231
1232         pds = kcalloc(BITS_TO_LONGS(pdpes), sizeof(unsigned long), GFP_TEMPORARY);
1233         if (!pds)
1234                 return -ENOMEM;
1235
1236         pts = kcalloc(pdpes, BITS_TO_LONGS(I915_PDES) * sizeof(unsigned long),
1237                       GFP_TEMPORARY);
1238         if (!pts)
1239                 goto err_out;
1240
1241         *new_pds = pds;
1242         *new_pts = pts;
1243
1244         return 0;
1245
1246 err_out:
1247         free_gen8_temp_bitmaps(pds, pts);
1248         return -ENOMEM;
1249 }
1250
1251 /* PDE TLBs are a pain to invalidate on GEN8+. When we modify
1252  * the page table structures, we mark them dirty so that
1253  * context switching/execlist queuing code takes extra steps
1254  * to ensure that tlbs are flushed.
1255  */
1256 static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt)
1257 {
1258         ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->base.dev)->ring_mask;
1259 }
1260
1261 static int gen8_alloc_va_range_3lvl(struct i915_address_space *vm,
1262                                     struct i915_page_directory_pointer *pdp,
1263                                     uint64_t start,
1264                                     uint64_t length)
1265 {
1266         struct i915_hw_ppgtt *ppgtt =
1267                 container_of(vm, struct i915_hw_ppgtt, base);
1268         unsigned long *new_page_dirs, *new_page_tables;
1269         struct drm_device *dev = vm->dev;
1270         struct i915_page_directory *pd;
1271         const uint64_t orig_start = start;
1272         const uint64_t orig_length = length;
1273         uint64_t temp;
1274         uint32_t pdpe;
1275         uint32_t pdpes = I915_PDPES_PER_PDP(dev);
1276         int ret;
1277
1278         /* Wrap is never okay since we can only represent 48b, and we don't
1279          * actually use the other side of the canonical address space.
1280          */
1281         if (WARN_ON(start + length < start))
1282                 return -ENODEV;
1283
1284         if (WARN_ON(start + length > vm->total))
1285                 return -ENODEV;
1286
1287         ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables, pdpes);
1288         if (ret)
1289                 return ret;
1290
1291         /* Do the allocations first so we can easily bail out */
1292         ret = gen8_ppgtt_alloc_page_directories(vm, pdp, start, length,
1293                                                 new_page_dirs);
1294         if (ret) {
1295                 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
1296                 return ret;
1297         }
1298
1299         /* For every page directory referenced, allocate page tables */
1300         gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {
1301                 ret = gen8_ppgtt_alloc_pagetabs(vm, pd, start, length,
1302                                                 new_page_tables + pdpe * BITS_TO_LONGS(I915_PDES));
1303                 if (ret)
1304                         goto err_out;
1305         }
1306
1307         start = orig_start;
1308         length = orig_length;
1309
1310         /* Allocations have completed successfully, so set the bitmaps, and do
1311          * the mappings. */
1312         gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {
1313                 gen8_pde_t *const page_directory = kmap_px(pd);
1314                 struct i915_page_table *pt;
1315                 uint64_t pd_len = length;
1316                 uint64_t pd_start = start;
1317                 uint32_t pde;
1318
1319                 /* Every pd should be allocated, we just did that above. */
1320                 WARN_ON(!pd);
1321
1322                 gen8_for_each_pde(pt, pd, pd_start, pd_len, temp, pde) {
1323                         /* Same reasoning as pd */
1324                         WARN_ON(!pt);
1325                         WARN_ON(!pd_len);
1326                         WARN_ON(!gen8_pte_count(pd_start, pd_len));
1327
1328                         /* Set our used ptes within the page table */
1329                         bitmap_set(pt->used_ptes,
1330                                    gen8_pte_index(pd_start),
1331                                    gen8_pte_count(pd_start, pd_len));
1332
1333                         /* Our pde is now pointing to the pagetable, pt */
1334                         __set_bit(pde, pd->used_pdes);
1335
1336                         /* Map the PDE to the page table */
1337                         page_directory[pde] = gen8_pde_encode(px_dma(pt),
1338                                                               I915_CACHE_LLC);
1339                         trace_i915_page_table_entry_map(&ppgtt->base, pde, pt,
1340                                                         gen8_pte_index(start),
1341                                                         gen8_pte_count(start, length),
1342                                                         GEN8_PTES);
1343
1344                         /* NB: We haven't yet mapped ptes to pages. At this
1345                          * point we're still relying on insert_entries() */
1346                 }
1347
1348                 kunmap_px(ppgtt, page_directory);
1349                 __set_bit(pdpe, pdp->used_pdpes);
1350                 gen8_setup_page_directory(ppgtt, pdp, pd, pdpe);
1351         }
1352
1353         free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
1354         mark_tlbs_dirty(ppgtt);
1355         return 0;
1356
1357 err_out:
1358         while (pdpe--) {
1359                 for_each_set_bit(temp, new_page_tables + pdpe *
1360                                 BITS_TO_LONGS(I915_PDES), I915_PDES)
1361                         free_pt(dev, pdp->page_directory[pdpe]->page_table[temp]);
1362         }
1363
1364         for_each_set_bit(pdpe, new_page_dirs, pdpes)
1365                 free_pd(dev, pdp->page_directory[pdpe]);
1366
1367         free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
1368         mark_tlbs_dirty(ppgtt);
1369         return ret;
1370 }
1371
1372 static int gen8_alloc_va_range_4lvl(struct i915_address_space *vm,
1373                                     struct i915_pml4 *pml4,
1374                                     uint64_t start,
1375                                     uint64_t length)
1376 {
1377         DECLARE_BITMAP(new_pdps, GEN8_PML4ES_PER_PML4);
1378         struct i915_hw_ppgtt *ppgtt =
1379                         container_of(vm, struct i915_hw_ppgtt, base);
1380         struct i915_page_directory_pointer *pdp;
1381         uint64_t temp, pml4e;
1382         int ret = 0;
1383
1384         /* Do the pml4 allocations first, so we don't need to track the newly
1385          * allocated tables below the pdp */
1386         bitmap_zero(new_pdps, GEN8_PML4ES_PER_PML4);
1387
1388         /* The pagedirectory and pagetable allocations are done in the shared 3
1389          * and 4 level code. Just allocate the pdps.
1390          */
1391         ret = gen8_ppgtt_alloc_page_dirpointers(vm, pml4, start, length,
1392                                                 new_pdps);
1393         if (ret)
1394                 return ret;
1395
1396         WARN(bitmap_weight(new_pdps, GEN8_PML4ES_PER_PML4) > 2,
1397              "The allocation has spanned more than 512GB. "
1398              "It is highly likely this is incorrect.");
1399
1400         gen8_for_each_pml4e(pdp, pml4, start, length, temp, pml4e) {
1401                 WARN_ON(!pdp);
1402
1403                 ret = gen8_alloc_va_range_3lvl(vm, pdp, start, length);
1404                 if (ret)
1405                         goto err_out;
1406
1407                 gen8_setup_page_directory_pointer(ppgtt, pml4, pdp, pml4e);
1408         }
1409
1410         bitmap_or(pml4->used_pml4es, new_pdps, pml4->used_pml4es,
1411                   GEN8_PML4ES_PER_PML4);
1412
1413         return 0;
1414
1415 err_out:
1416         for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4)
1417                 gen8_ppgtt_cleanup_3lvl(vm->dev, pml4->pdps[pml4e]);
1418
1419         return ret;
1420 }
1421
1422 static int gen8_alloc_va_range(struct i915_address_space *vm,
1423                                uint64_t start, uint64_t length)
1424 {
1425         struct i915_hw_ppgtt *ppgtt =
1426                 container_of(vm, struct i915_hw_ppgtt, base);
1427
1428         if (USES_FULL_48BIT_PPGTT(vm->dev))
1429                 return gen8_alloc_va_range_4lvl(vm, &ppgtt->pml4, start, length);
1430         else
1431                 return gen8_alloc_va_range_3lvl(vm, &ppgtt->pdp, start, length);
1432 }
1433
1434 static void gen8_dump_pdp(struct i915_page_directory_pointer *pdp,
1435                           uint64_t start, uint64_t length,
1436                           gen8_pte_t scratch_pte,
1437                           struct seq_file *m)
1438 {
1439         struct i915_page_directory *pd;
1440         uint64_t temp;
1441         uint32_t pdpe;
1442
1443         gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {
1444                 struct i915_page_table *pt;
1445                 uint64_t pd_len = length;
1446                 uint64_t pd_start = start;
1447                 uint32_t pde;
1448
1449                 if (!test_bit(pdpe, pdp->used_pdpes))
1450                         continue;
1451
1452                 seq_printf(m, "\tPDPE #%d\n", pdpe);
1453                 gen8_for_each_pde(pt, pd, pd_start, pd_len, temp, pde) {
1454                         uint32_t  pte;
1455                         gen8_pte_t *pt_vaddr;
1456
1457                         if (!test_bit(pde, pd->used_pdes))
1458                                 continue;
1459
1460                         pt_vaddr = kmap_px(pt);
1461                         for (pte = 0; pte < GEN8_PTES; pte += 4) {
1462                                 uint64_t va =
1463                                         (pdpe << GEN8_PDPE_SHIFT) |
1464                                         (pde << GEN8_PDE_SHIFT) |
1465                                         (pte << GEN8_PTE_SHIFT);
1466                                 int i;
1467                                 bool found = false;
1468
1469                                 for (i = 0; i < 4; i++)
1470                                         if (pt_vaddr[pte + i] != scratch_pte)
1471                                                 found = true;
1472                                 if (!found)
1473                                         continue;
1474
1475                                 seq_printf(m, "\t\t0x%lx [%03d,%03d,%04d]: =", va, pdpe, pde, pte);
1476                                 for (i = 0; i < 4; i++) {
1477                                         if (pt_vaddr[pte + i] != scratch_pte)
1478                                                 seq_printf(m, " %lx", pt_vaddr[pte + i]);
1479                                         else
1480                                                 seq_puts(m, "  SCRATCH ");
1481                                 }
1482                                 seq_puts(m, "\n");
1483                         }
1484                         /* don't use kunmap_px, it could trigger
1485                          * an unnecessary flush.
1486                          */
1487                         kunmap_atomic(pt_vaddr);
1488                 }
1489         }
1490 }
1491
1492 static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
1493 {
1494         struct i915_address_space *vm = &ppgtt->base;
1495         uint64_t start = ppgtt->base.start;
1496         uint64_t length = ppgtt->base.total;
1497         gen8_pte_t scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page),
1498                                                  I915_CACHE_LLC, true);
1499
1500         if (!USES_FULL_48BIT_PPGTT(vm->dev)) {
1501                 gen8_dump_pdp(&ppgtt->pdp, start, length, scratch_pte, m);
1502         } else {
1503                 uint64_t templ4, pml4e;
1504                 struct i915_pml4 *pml4 = &ppgtt->pml4;
1505                 struct i915_page_directory_pointer *pdp;
1506
1507                 gen8_for_each_pml4e(pdp, pml4, start, length, templ4, pml4e) {
1508                         if (!test_bit(pml4e, pml4->used_pml4es))
1509                                 continue;
1510
1511                         seq_printf(m, "    PML4E #%lu\n", pml4e);
1512                         gen8_dump_pdp(pdp, start, length, scratch_pte, m);
1513                 }
1514         }
1515 }
1516
1517 static int gen8_preallocate_top_level_pdps(struct i915_hw_ppgtt *ppgtt)
1518 {
1519         unsigned long *new_page_dirs, *new_page_tables;
1520         uint32_t pdpes = I915_PDPES_PER_PDP(dev);
1521         int ret;
1522
1523         /* We allocate temp bitmap for page tables for no gain
1524          * but as this is for init only, lets keep the things simple
1525          */
1526         ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables, pdpes);
1527         if (ret)
1528                 return ret;
1529
1530         /* Allocate for all pdps regardless of how the ppgtt
1531          * was defined.
1532          */
1533         ret = gen8_ppgtt_alloc_page_directories(&ppgtt->base, &ppgtt->pdp,
1534                                                 0, 1ULL << 32,
1535                                                 new_page_dirs);
1536         if (!ret)
1537                 *ppgtt->pdp.used_pdpes = *new_page_dirs;
1538
1539         free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
1540
1541         return ret;
1542 }
1543
1544 /*
1545  * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers
1546  * with a net effect resembling a 2-level page table in normal x86 terms. Each
1547  * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address
1548  * space.
1549  *
1550  */
1551 static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
1552 {
1553         int ret;
1554
1555         ret = gen8_init_scratch(&ppgtt->base);
1556         if (ret)
1557                 return ret;
1558
1559         ppgtt->base.start = 0;
1560         ppgtt->base.cleanup = gen8_ppgtt_cleanup;
1561         ppgtt->base.allocate_va_range = gen8_alloc_va_range;
1562         ppgtt->base.insert_entries = gen8_ppgtt_insert_entries;
1563         ppgtt->base.clear_range = gen8_ppgtt_clear_range;
1564         ppgtt->base.unbind_vma = ppgtt_unbind_vma;
1565         ppgtt->base.bind_vma = ppgtt_bind_vma;
1566         ppgtt->debug_dump = gen8_dump_ppgtt;
1567
1568         if (USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) {
1569                 ret = setup_px(ppgtt->base.dev, &ppgtt->pml4);
1570                 if (ret)
1571                         goto free_scratch;
1572
1573                 gen8_initialize_pml4(&ppgtt->base, &ppgtt->pml4);
1574
1575                 ppgtt->base.total = 1ULL << 48;
1576                 ppgtt->switch_mm = gen8_48b_mm_switch;
1577         } else {
1578                 ret = __pdp_init(ppgtt->base.dev, &ppgtt->pdp);
1579                 if (ret)
1580                         goto free_scratch;
1581
1582 #define CONFIG_X86_32   0
1583                 ppgtt->base.total = 1ULL << 32;
1584                 ppgtt->switch_mm = gen8_legacy_mm_switch;
1585                 trace_i915_page_directory_pointer_entry_alloc(&ppgtt->base,
1586                                                               0, 0,
1587                                                               GEN8_PML4E_SHIFT);
1588
1589                 if (intel_vgpu_active(ppgtt->base.dev)) {
1590                         ret = gen8_preallocate_top_level_pdps(ppgtt);
1591                         if (ret)
1592                                 goto free_scratch;
1593                 }
1594         }
1595
1596         if (intel_vgpu_active(ppgtt->base.dev))
1597                 gen8_ppgtt_notify_vgt(ppgtt, true);
1598
1599         return 0;
1600
1601 free_scratch:
1602         gen8_free_scratch(&ppgtt->base);
1603         return ret;
1604 }
1605
1606 static int gen8_aliasing_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
1607 {
1608         struct drm_device *dev = ppgtt->base.dev;
1609         struct drm_i915_private *dev_priv = dev->dev_private;
1610         uint64_t start = 0, size = dev_priv->gtt.base.total;
1611         int ret;
1612
1613         ret = gen8_ppgtt_init(ppgtt);
1614         if (ret)
1615                 return ret;
1616
1617         /* Aliasing PPGTT has to always work and be mapped because of the way we
1618          * use RESTORE_INHIBIT in the context switch. This will be fixed
1619          * eventually. */
1620         ret = gen8_alloc_va_range(&ppgtt->base, start, size);
1621         if (ret) {
1622                 free_pd(ppgtt->base.dev, ppgtt->base.scratch_pd);
1623                 free_pt(ppgtt->base.dev, ppgtt->base.scratch_pt);
1624                 return ret;
1625         }
1626
1627         ppgtt->base.allocate_va_range = NULL;
1628         ppgtt->base.clear_range(&ppgtt->base, 0, ppgtt->base.total, true);
1629
1630         return 0;
1631 }
1632
1633 static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
1634 {
1635         struct i915_address_space *vm = &ppgtt->base;
1636         struct i915_page_table *unused;
1637         gen6_pte_t scratch_pte;
1638         uint32_t pd_entry;
1639         uint32_t  pte, pde, temp;
1640         uint32_t start = ppgtt->base.start, length = ppgtt->base.total;
1641
1642         scratch_pte = vm->pte_encode(px_dma(vm->scratch_page),
1643                                      I915_CACHE_LLC, true, 0);
1644
1645         gen6_for_each_pde(unused, &ppgtt->pd, start, length, temp, pde) {
1646                 u32 expected;
1647                 gen6_pte_t *pt_vaddr;
1648                 const dma_addr_t pt_addr = px_dma(ppgtt->pd.page_table[pde]);
1649                 pd_entry = readl(ppgtt->pd_addr + pde);
1650                 expected = (GEN6_PDE_ADDR_ENCODE(pt_addr) | GEN6_PDE_VALID);
1651
1652                 if (pd_entry != expected)
1653                         seq_printf(m, "\tPDE #%d mismatch: Actual PDE: %x Expected PDE: %x\n",
1654                                    pde,
1655                                    pd_entry,
1656                                    expected);
1657                 seq_printf(m, "\tPDE: %x\n", pd_entry);
1658
1659                 pt_vaddr = kmap_px(ppgtt->pd.page_table[pde]);
1660
1661                 for (pte = 0; pte < GEN6_PTES; pte+=4) {
1662                         unsigned long va =
1663                                 (pde * PAGE_SIZE * GEN6_PTES) +
1664                                 (pte * PAGE_SIZE);
1665                         int i;
1666                         bool found = false;
1667                         for (i = 0; i < 4; i++)
1668                                 if (pt_vaddr[pte + i] != scratch_pte)
1669                                         found = true;
1670                         if (!found)
1671                                 continue;
1672
1673                         seq_printf(m, "\t\t0x%lx [%03d,%04d]: =", va, pde, pte);
1674                         for (i = 0; i < 4; i++) {
1675                                 if (pt_vaddr[pte + i] != scratch_pte)
1676                                         seq_printf(m, " %08x", pt_vaddr[pte + i]);
1677                                 else
1678                                         seq_puts(m, "  SCRATCH ");
1679                         }
1680                         seq_puts(m, "\n");
1681                 }
1682                 kunmap_px(ppgtt, pt_vaddr);
1683         }
1684 }
1685
1686 /* Write pde (index) from the page directory @pd to the page table @pt */
1687 static void gen6_write_pde(struct i915_page_directory *pd,
1688                             const int pde, struct i915_page_table *pt)
1689 {
1690         /* Caller needs to make sure the write completes if necessary */
1691         struct i915_hw_ppgtt *ppgtt =
1692                 container_of(pd, struct i915_hw_ppgtt, pd);
1693         u32 pd_entry;
1694
1695         pd_entry = GEN6_PDE_ADDR_ENCODE(px_dma(pt));
1696         pd_entry |= GEN6_PDE_VALID;
1697
1698         writel(pd_entry, ppgtt->pd_addr + pde);
1699 }
1700
1701 /* Write all the page tables found in the ppgtt structure to incrementing page
1702  * directories. */
1703 static void gen6_write_page_range(struct drm_i915_private *dev_priv,
1704                                   struct i915_page_directory *pd,
1705                                   uint32_t start, uint32_t length)
1706 {
1707         struct i915_page_table *pt;
1708         uint32_t pde, temp;
1709
1710         gen6_for_each_pde(pt, pd, start, length, temp, pde)
1711                 gen6_write_pde(pd, pde, pt);
1712
1713         /* Make sure write is complete before other code can use this page
1714          * table. Also require for WC mapped PTEs */
1715         readl(dev_priv->gtt.gsm);
1716 }
1717
1718 static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt)
1719 {
1720         BUG_ON(ppgtt->pd.base.ggtt_offset & 0x3f);
1721
1722         return (ppgtt->pd.base.ggtt_offset / 64) << 16;
1723 }
1724
1725 static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,
1726                          struct drm_i915_gem_request *req)
1727 {
1728         struct intel_engine_cs *ring = req->ring;
1729         int ret;
1730
1731         /* NB: TLBs must be flushed and invalidated before a switch */
1732         ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
1733         if (ret)
1734                 return ret;
1735
1736         ret = intel_ring_begin(req, 6);
1737         if (ret)
1738                 return ret;
1739
1740         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
1741         intel_ring_emit(ring, RING_PP_DIR_DCLV(ring));
1742         intel_ring_emit(ring, PP_DIR_DCLV_2G);
1743         intel_ring_emit(ring, RING_PP_DIR_BASE(ring));
1744         intel_ring_emit(ring, get_pd_offset(ppgtt));
1745         intel_ring_emit(ring, MI_NOOP);
1746         intel_ring_advance(ring);
1747
1748         return 0;
1749 }
1750
1751 static int vgpu_mm_switch(struct i915_hw_ppgtt *ppgtt,
1752                           struct drm_i915_gem_request *req)
1753 {
1754         struct intel_engine_cs *ring = req->ring;
1755         struct drm_i915_private *dev_priv = to_i915(ppgtt->base.dev);
1756
1757         I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
1758         I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
1759         return 0;
1760 }
1761
1762 static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt,
1763                           struct drm_i915_gem_request *req)
1764 {
1765         struct intel_engine_cs *ring = req->ring;
1766         int ret;
1767
1768         /* NB: TLBs must be flushed and invalidated before a switch */
1769         ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
1770         if (ret)
1771                 return ret;
1772
1773         ret = intel_ring_begin(req, 6);
1774         if (ret)
1775                 return ret;
1776
1777         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
1778         intel_ring_emit(ring, RING_PP_DIR_DCLV(ring));
1779         intel_ring_emit(ring, PP_DIR_DCLV_2G);
1780         intel_ring_emit(ring, RING_PP_DIR_BASE(ring));
1781         intel_ring_emit(ring, get_pd_offset(ppgtt));
1782         intel_ring_emit(ring, MI_NOOP);
1783         intel_ring_advance(ring);
1784
1785         /* XXX: RCS is the only one to auto invalidate the TLBs? */
1786         if (ring->id != RCS) {
1787                 ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
1788                 if (ret)
1789                         return ret;
1790         }
1791
1792         return 0;
1793 }
1794
1795 static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt,
1796                           struct drm_i915_gem_request *req)
1797 {
1798         struct intel_engine_cs *ring = req->ring;
1799         struct drm_device *dev = ppgtt->base.dev;
1800         struct drm_i915_private *dev_priv = dev->dev_private;
1801
1802
1803         I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
1804         I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
1805
1806         POSTING_READ(RING_PP_DIR_DCLV(ring));
1807
1808         return 0;
1809 }
1810
1811 static void gen8_ppgtt_enable(struct drm_device *dev)
1812 {
1813         struct drm_i915_private *dev_priv = dev->dev_private;
1814         struct intel_engine_cs *ring;
1815         int j;
1816
1817         for_each_ring(ring, dev_priv, j) {
1818                 u32 four_level = USES_FULL_48BIT_PPGTT(dev) ? GEN8_GFX_PPGTT_48B : 0;
1819                 I915_WRITE(RING_MODE_GEN7(ring),
1820                            _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE | four_level));
1821         }
1822 }
1823
1824 static void gen7_ppgtt_enable(struct drm_device *dev)
1825 {
1826         struct drm_i915_private *dev_priv = dev->dev_private;
1827         struct intel_engine_cs *ring;
1828         uint32_t ecochk, ecobits;
1829         int i;
1830
1831         ecobits = I915_READ(GAC_ECO_BITS);
1832         I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
1833
1834         ecochk = I915_READ(GAM_ECOCHK);
1835         if (IS_HASWELL(dev)) {
1836                 ecochk |= ECOCHK_PPGTT_WB_HSW;
1837         } else {
1838                 ecochk |= ECOCHK_PPGTT_LLC_IVB;
1839                 ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
1840         }
1841         I915_WRITE(GAM_ECOCHK, ecochk);
1842
1843         for_each_ring(ring, dev_priv, i) {
1844                 /* GFX_MODE is per-ring on gen7+ */
1845                 I915_WRITE(RING_MODE_GEN7(ring),
1846                            _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
1847         }
1848 }
1849
1850 static void gen6_ppgtt_enable(struct drm_device *dev)
1851 {
1852         struct drm_i915_private *dev_priv = dev->dev_private;
1853         uint32_t ecochk, gab_ctl, ecobits;
1854
1855         ecobits = I915_READ(GAC_ECO_BITS);
1856         I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
1857                    ECOBITS_PPGTT_CACHE64B);
1858
1859         gab_ctl = I915_READ(GAB_CTL);
1860         I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
1861
1862         ecochk = I915_READ(GAM_ECOCHK);
1863         I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B);
1864
1865         I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
1866 }
1867
1868 /* PPGTT support for Sandybdrige/Gen6 and later */
1869 static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
1870                                    uint64_t start,
1871                                    uint64_t length,
1872                                    bool use_scratch)
1873 {
1874         struct i915_hw_ppgtt *ppgtt =
1875                 container_of(vm, struct i915_hw_ppgtt, base);
1876         gen6_pte_t *pt_vaddr, scratch_pte;
1877         unsigned first_entry = start >> PAGE_SHIFT;
1878         unsigned num_entries = length >> PAGE_SHIFT;
1879         unsigned act_pt = first_entry / GEN6_PTES;
1880         unsigned first_pte = first_entry % GEN6_PTES;
1881         unsigned last_pte, i;
1882
1883         scratch_pte = vm->pte_encode(px_dma(vm->scratch_page),
1884                                      I915_CACHE_LLC, true, 0);
1885
1886         while (num_entries) {
1887                 last_pte = first_pte + num_entries;
1888                 if (last_pte > GEN6_PTES)
1889                         last_pte = GEN6_PTES;
1890
1891                 pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]);
1892
1893                 for (i = first_pte; i < last_pte; i++)
1894                         pt_vaddr[i] = scratch_pte;
1895
1896                 kunmap_px(ppgtt, pt_vaddr);
1897
1898                 num_entries -= last_pte - first_pte;
1899                 first_pte = 0;
1900                 act_pt++;
1901         }
1902 }
1903
1904 static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
1905                                       struct sg_table *pages,
1906                                       uint64_t start,
1907                                       enum i915_cache_level cache_level, u32 flags)
1908 {
1909         struct i915_hw_ppgtt *ppgtt =
1910                 container_of(vm, struct i915_hw_ppgtt, base);
1911         gen6_pte_t *pt_vaddr;
1912         unsigned first_entry = start >> PAGE_SHIFT;
1913         unsigned act_pt = first_entry / GEN6_PTES;
1914         unsigned act_pte = first_entry % GEN6_PTES;
1915         struct sg_page_iter sg_iter;
1916
1917         pt_vaddr = NULL;
1918         for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
1919                 if (pt_vaddr == NULL)
1920                         pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]);
1921
1922                 pt_vaddr[act_pte] =
1923                         vm->pte_encode(sg_page_iter_dma_address(&sg_iter),
1924                                        cache_level, true, flags);
1925
1926                 if (++act_pte == GEN6_PTES) {
1927                         kunmap_px(ppgtt, pt_vaddr);
1928                         pt_vaddr = NULL;
1929                         act_pt++;
1930                         act_pte = 0;
1931                 }
1932         }
1933         if (pt_vaddr)
1934                 kunmap_px(ppgtt, pt_vaddr);
1935 }
1936
1937 static int gen6_alloc_va_range(struct i915_address_space *vm,
1938                                uint64_t start_in, uint64_t length_in)
1939 {
1940         DECLARE_BITMAP(new_page_tables, I915_PDES);
1941         struct drm_device *dev = vm->dev;
1942         struct drm_i915_private *dev_priv = dev->dev_private;
1943         struct i915_hw_ppgtt *ppgtt =
1944                                 container_of(vm, struct i915_hw_ppgtt, base);
1945         struct i915_page_table *pt;
1946         uint32_t start, length, start_save, length_save;
1947         uint32_t pde, temp;
1948         int ret;
1949
1950         if (WARN_ON(start_in + length_in > ppgtt->base.total))
1951                 return -ENODEV;
1952
1953         start = start_save = start_in;
1954         length = length_save = length_in;
1955
1956         bitmap_zero(new_page_tables, I915_PDES);
1957
1958         /* The allocation is done in two stages so that we can bail out with
1959          * minimal amount of pain. The first stage finds new page tables that
1960          * need allocation. The second stage marks use ptes within the page
1961          * tables.
1962          */
1963         gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) {
1964                 if (pt != vm->scratch_pt) {
1965 //                      WARN_ON(bitmap_empty(pt->used_ptes, GEN6_PTES));
1966                         continue;
1967                 }
1968
1969                 /* We've already allocated a page table */
1970                 WARN_ON(!bitmap_empty(pt->used_ptes, GEN6_PTES));
1971
1972                 pt = alloc_pt(dev);
1973                 if (IS_ERR(pt)) {
1974                         ret = PTR_ERR(pt);
1975                         goto unwind_out;
1976                 }
1977
1978                 gen6_initialize_pt(vm, pt);
1979
1980                 ppgtt->pd.page_table[pde] = pt;
1981                 __set_bit(pde, new_page_tables);
1982                 trace_i915_page_table_entry_alloc(vm, pde, start, GEN6_PDE_SHIFT);
1983         }
1984
1985         start = start_save;
1986         length = length_save;
1987
1988         gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) {
1989                 DECLARE_BITMAP(tmp_bitmap, GEN6_PTES);
1990
1991                 bitmap_zero(tmp_bitmap, GEN6_PTES);
1992                 bitmap_set(tmp_bitmap, gen6_pte_index(start),
1993                            gen6_pte_count(start, length));
1994
1995                 if (__test_and_clear_bit(pde, new_page_tables))
1996                         gen6_write_pde(&ppgtt->pd, pde, pt);
1997
1998                 trace_i915_page_table_entry_map(vm, pde, pt,
1999                                          gen6_pte_index(start),
2000                                          gen6_pte_count(start, length),
2001                                          GEN6_PTES);
2002                 bitmap_or(pt->used_ptes, tmp_bitmap, pt->used_ptes,
2003                                 GEN6_PTES);
2004         }
2005
2006         WARN_ON(!bitmap_empty(new_page_tables, I915_PDES));
2007
2008         /* Make sure write is complete before other code can use this page
2009          * table. Also require for WC mapped PTEs */
2010         readl(dev_priv->gtt.gsm);
2011
2012         mark_tlbs_dirty(ppgtt);
2013         return 0;
2014
2015 unwind_out:
2016         for_each_set_bit(pde, new_page_tables, I915_PDES) {
2017                 struct i915_page_table *pt = ppgtt->pd.page_table[pde];
2018
2019                 ppgtt->pd.page_table[pde] = vm->scratch_pt;
2020                 free_pt(vm->dev, pt);
2021         }
2022
2023         mark_tlbs_dirty(ppgtt);
2024         return ret;
2025 }
2026
2027 static int gen6_init_scratch(struct i915_address_space *vm)
2028 {
2029         struct drm_device *dev = vm->dev;
2030
2031         vm->scratch_page = alloc_scratch_page(dev);
2032         if (IS_ERR(vm->scratch_page))
2033                 return PTR_ERR(vm->scratch_page);
2034
2035         vm->scratch_pt = alloc_pt(dev);
2036         if (IS_ERR(vm->scratch_pt)) {
2037                 free_scratch_page(dev, vm->scratch_page);
2038                 return PTR_ERR(vm->scratch_pt);
2039         }
2040
2041         gen6_initialize_pt(vm, vm->scratch_pt);
2042
2043         return 0;
2044 }
2045
2046 static void gen6_free_scratch(struct i915_address_space *vm)
2047 {
2048         struct drm_device *dev = vm->dev;
2049
2050         free_pt(dev, vm->scratch_pt);
2051         free_scratch_page(dev, vm->scratch_page);
2052 }
2053
2054 static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
2055 {
2056         struct i915_hw_ppgtt *ppgtt =
2057                 container_of(vm, struct i915_hw_ppgtt, base);
2058         struct i915_page_table *pt;
2059         uint32_t pde;
2060
2061         drm_mm_remove_node(&ppgtt->node);
2062
2063         gen6_for_all_pdes(pt, ppgtt, pde) {
2064                 if (pt != vm->scratch_pt)
2065                         free_pt(ppgtt->base.dev, pt);
2066         }
2067
2068         gen6_free_scratch(vm);
2069 }
2070
2071 static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt)
2072 {
2073         struct i915_address_space *vm = &ppgtt->base;
2074         struct drm_device *dev = ppgtt->base.dev;
2075         struct drm_i915_private *dev_priv = dev->dev_private;
2076         bool retried = false;
2077         int ret;
2078
2079         /* PPGTT PDEs reside in the GGTT and consists of 512 entries. The
2080          * allocator works in address space sizes, so it's multiplied by page
2081          * size. We allocate at the top of the GTT to avoid fragmentation.
2082          */
2083         BUG_ON(!drm_mm_initialized(&dev_priv->gtt.base.mm));
2084
2085         ret = gen6_init_scratch(vm);
2086         if (ret)
2087                 return ret;
2088
2089 alloc:
2090         ret = drm_mm_insert_node_in_range_generic(&dev_priv->gtt.base.mm,
2091                                                   &ppgtt->node, GEN6_PD_SIZE,
2092                                                   GEN6_PD_ALIGN, 0,
2093                                                   0, dev_priv->gtt.base.total,
2094                                                   DRM_MM_TOPDOWN);
2095         if (ret == -ENOSPC && !retried) {
2096                 ret = i915_gem_evict_something(dev, &dev_priv->gtt.base,
2097                                                GEN6_PD_SIZE, GEN6_PD_ALIGN,
2098                                                I915_CACHE_NONE,
2099                                                0, dev_priv->gtt.base.total,
2100                                                0);
2101                 if (ret)
2102                         goto err_out;
2103
2104                 retried = true;
2105                 goto alloc;
2106         }
2107
2108         if (ret)
2109                 goto err_out;
2110
2111
2112         if (ppgtt->node.start < dev_priv->gtt.mappable_end)
2113                 DRM_DEBUG("Forced to use aperture for PDEs\n");
2114
2115         return 0;
2116
2117 err_out:
2118         gen6_free_scratch(vm);
2119         return ret;
2120 }
2121
2122 static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)
2123 {
2124         return gen6_ppgtt_allocate_page_directories(ppgtt);
2125 }
2126
2127 static void gen6_scratch_va_range(struct i915_hw_ppgtt *ppgtt,
2128                                   uint64_t start, uint64_t length)
2129 {
2130         struct i915_page_table *unused;
2131         uint32_t pde, temp;
2132
2133         gen6_for_each_pde(unused, &ppgtt->pd, start, length, temp, pde)
2134                 ppgtt->pd.page_table[pde] = ppgtt->base.scratch_pt;
2135 }
2136
2137 static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt, bool aliasing)
2138 {
2139         struct drm_device *dev = ppgtt->base.dev;
2140         struct drm_i915_private *dev_priv = dev->dev_private;
2141         int ret;
2142
2143         ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode;
2144         if (IS_GEN6(dev)) {
2145                 ppgtt->switch_mm = gen6_mm_switch;
2146         } else if (IS_HASWELL(dev)) {
2147                 ppgtt->switch_mm = hsw_mm_switch;
2148         } else if (IS_GEN7(dev)) {
2149                 ppgtt->switch_mm = gen7_mm_switch;
2150         } else
2151                 BUG();
2152
2153         if (intel_vgpu_active(dev))
2154                 ppgtt->switch_mm = vgpu_mm_switch;
2155
2156         ret = gen6_ppgtt_alloc(ppgtt);
2157         if (ret)
2158                 return ret;
2159
2160         if (aliasing) {
2161                 /* preallocate all pts */
2162                 ret = alloc_pt_range(&ppgtt->pd, 0, I915_PDES,
2163                                 ppgtt->base.dev);
2164
2165                 if (ret) {
2166                         gen6_ppgtt_cleanup(&ppgtt->base);
2167                         return ret;
2168                 }
2169         }
2170
2171         ppgtt->base.allocate_va_range = gen6_alloc_va_range;
2172         ppgtt->base.clear_range = gen6_ppgtt_clear_range;
2173         ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
2174         ppgtt->base.unbind_vma = ppgtt_unbind_vma;
2175         ppgtt->base.bind_vma = ppgtt_bind_vma;
2176         ppgtt->base.cleanup = gen6_ppgtt_cleanup;
2177         ppgtt->base.start = 0;
2178         ppgtt->base.total = I915_PDES * GEN6_PTES * PAGE_SIZE;
2179         ppgtt->debug_dump = gen6_dump_ppgtt;
2180
2181         ppgtt->pd.base.ggtt_offset =
2182                 ppgtt->node.start / PAGE_SIZE * sizeof(gen6_pte_t);
2183
2184         ppgtt->pd_addr = (gen6_pte_t __iomem *)dev_priv->gtt.gsm +
2185                 ppgtt->pd.base.ggtt_offset / sizeof(gen6_pte_t);
2186
2187         if (aliasing)
2188                 ppgtt->base.clear_range(&ppgtt->base, 0, ppgtt->base.total, true);
2189         else
2190                 gen6_scratch_va_range(ppgtt, 0, ppgtt->base.total);
2191
2192         gen6_write_page_range(dev_priv, &ppgtt->pd, 0, ppgtt->base.total);
2193
2194         DRM_DEBUG_DRIVER("Allocated pde space (%ldM) at GTT entry: %lx\n",
2195                          ppgtt->node.size >> 20,
2196                          ppgtt->node.start / PAGE_SIZE);
2197
2198         DRM_DEBUG("Adding PPGTT at offset %x\n",
2199                   ppgtt->pd.base.ggtt_offset << 10);
2200
2201         return 0;
2202 }
2203
2204 static int __hw_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt,
2205                 bool aliasing)
2206 {
2207         ppgtt->base.dev = dev;
2208
2209         if (INTEL_INFO(dev)->gen < 8)
2210                 return gen6_ppgtt_init(ppgtt, aliasing);
2211         else if (aliasing)
2212                 return gen8_aliasing_ppgtt_init(ppgtt);
2213         else
2214                 return gen8_ppgtt_init(ppgtt);
2215 }
2216
2217 static void i915_address_space_init(struct i915_address_space *vm,
2218                                     struct drm_i915_private *dev_priv)
2219 {
2220         drm_mm_init(&vm->mm, vm->start, vm->total);
2221         vm->dev = dev_priv->dev;
2222         INIT_LIST_HEAD(&vm->active_list);
2223         INIT_LIST_HEAD(&vm->inactive_list);
2224         list_add_tail(&vm->global_link, &dev_priv->vm_list);
2225 }
2226
2227 int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
2228 {
2229         struct drm_i915_private *dev_priv = dev->dev_private;
2230         int ret = 0;
2231
2232         ret = __hw_ppgtt_init(dev, ppgtt, false);
2233         if (ret == 0) {
2234                 kref_init(&ppgtt->ref);
2235                 i915_address_space_init(&ppgtt->base, dev_priv);
2236         }
2237
2238         return ret;
2239 }
2240
2241 int i915_ppgtt_init_hw(struct drm_device *dev)
2242 {
2243         /* In the case of execlists, PPGTT is enabled by the context descriptor
2244          * and the PDPs are contained within the context itself.  We don't
2245          * need to do anything here. */
2246         if (i915.enable_execlists)
2247                 return 0;
2248
2249         if (!USES_PPGTT(dev))
2250                 return 0;
2251
2252         if (IS_GEN6(dev))
2253                 gen6_ppgtt_enable(dev);
2254         else if (IS_GEN7(dev))
2255                 gen7_ppgtt_enable(dev);
2256         else if (INTEL_INFO(dev)->gen >= 8)
2257                 gen8_ppgtt_enable(dev);
2258         else
2259                 MISSING_CASE(INTEL_INFO(dev)->gen);
2260
2261         return 0;
2262 }
2263
2264 int i915_ppgtt_init_ring(struct drm_i915_gem_request *req)
2265 {
2266         struct drm_i915_private *dev_priv = req->ring->dev->dev_private;
2267         struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
2268
2269         if (i915.enable_execlists)
2270                 return 0;
2271
2272         if (!ppgtt)
2273                 return 0;
2274
2275         return ppgtt->switch_mm(ppgtt, req);
2276 }
2277
2278 struct i915_hw_ppgtt *
2279 i915_ppgtt_create(struct drm_device *dev, struct drm_i915_file_private *fpriv)
2280 {
2281         struct i915_hw_ppgtt *ppgtt;
2282         int ret;
2283
2284         ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
2285         if (!ppgtt)
2286                 return ERR_PTR(-ENOMEM);
2287
2288         ret = i915_ppgtt_init(dev, ppgtt);
2289         if (ret) {
2290                 kfree(ppgtt);
2291                 return ERR_PTR(ret);
2292         }
2293
2294         ppgtt->file_priv = fpriv;
2295
2296         trace_i915_ppgtt_create(&ppgtt->base);
2297
2298         return ppgtt;
2299 }
2300
2301 void  i915_ppgtt_release(struct kref *kref)
2302 {
2303         struct i915_hw_ppgtt *ppgtt =
2304                 container_of(kref, struct i915_hw_ppgtt, ref);
2305
2306         trace_i915_ppgtt_release(&ppgtt->base);
2307
2308         /* vmas should already be unbound */
2309         WARN_ON(!list_empty(&ppgtt->base.active_list));
2310         WARN_ON(!list_empty(&ppgtt->base.inactive_list));
2311
2312         list_del(&ppgtt->base.global_link);
2313         drm_mm_takedown(&ppgtt->base.mm);
2314
2315         ppgtt->base.cleanup(&ppgtt->base);
2316         kfree(ppgtt);
2317 }
2318
2319 extern int intel_iommu_gfx_mapped;
2320 /* Certain Gen5 chipsets require require idling the GPU before
2321  * unmapping anything from the GTT when VT-d is enabled.
2322  */
2323 static bool needs_idle_maps(struct drm_device *dev)
2324 {
2325 #ifdef CONFIG_INTEL_IOMMU
2326         /* Query intel_iommu to see if we need the workaround. Presumably that
2327          * was loaded first.
2328          */
2329         if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped)
2330                 return true;
2331 #endif
2332         return false;
2333 }
2334
2335 static bool do_idling(struct drm_i915_private *dev_priv)
2336 {
2337         bool ret = dev_priv->mm.interruptible;
2338
2339         if (unlikely(dev_priv->gtt.do_idle_maps)) {
2340                 dev_priv->mm.interruptible = false;
2341                 if (i915_gpu_idle(dev_priv->dev)) {
2342                         DRM_ERROR("Couldn't idle GPU\n");
2343                         /* Wait a bit, in hopes it avoids the hang */
2344                         udelay(10);
2345                 }
2346         }
2347
2348         return ret;
2349 }
2350
2351 static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
2352 {
2353         if (unlikely(dev_priv->gtt.do_idle_maps))
2354                 dev_priv->mm.interruptible = interruptible;
2355 }
2356
2357 void i915_check_and_clear_faults(struct drm_device *dev)
2358 {
2359         struct drm_i915_private *dev_priv = dev->dev_private;
2360         struct intel_engine_cs *ring;
2361         int i;
2362
2363         if (INTEL_INFO(dev)->gen < 6)
2364                 return;
2365
2366         for_each_ring(ring, dev_priv, i) {
2367                 u32 fault_reg;
2368                 fault_reg = I915_READ(RING_FAULT_REG(ring));
2369                 if (fault_reg & RING_FAULT_VALID) {
2370 #if 0
2371                         DRM_DEBUG_DRIVER("Unexpected fault\n"
2372                                          "\tAddr: 0x%08lx\n"
2373                                          "\tAddress space: %s\n"
2374                                          "\tSource ID: %d\n"
2375                                          "\tType: %d\n",
2376                                          fault_reg & PAGE_MASK,
2377                                          fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT",
2378                                          RING_FAULT_SRCID(fault_reg),
2379                                          RING_FAULT_FAULT_TYPE(fault_reg));
2380 #endif
2381                         I915_WRITE(RING_FAULT_REG(ring),
2382                                    fault_reg & ~RING_FAULT_VALID);
2383                 }
2384         }
2385         POSTING_READ(RING_FAULT_REG(&dev_priv->ring[RCS]));
2386 }
2387
2388 static void i915_ggtt_flush(struct drm_i915_private *dev_priv)
2389 {
2390         if (INTEL_INFO(dev_priv->dev)->gen < 6) {
2391                 intel_gtt_chipset_flush();
2392         } else {
2393                 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
2394                 POSTING_READ(GFX_FLSH_CNTL_GEN6);
2395         }
2396 }
2397
2398 void i915_gem_suspend_gtt_mappings(struct drm_device *dev)
2399 {
2400         struct drm_i915_private *dev_priv = dev->dev_private;
2401
2402         /* Don't bother messing with faults pre GEN6 as we have little
2403          * documentation supporting that it's a good idea.
2404          */
2405         if (INTEL_INFO(dev)->gen < 6)
2406                 return;
2407
2408         i915_check_and_clear_faults(dev);
2409
2410         dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
2411                                        dev_priv->gtt.base.start,
2412                                        dev_priv->gtt.base.total,
2413                                        true);
2414
2415         i915_ggtt_flush(dev_priv);
2416 }
2417
2418 int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
2419 {
2420         if (!dma_map_sg(obj->base.dev->pdev->dev,
2421                         obj->pages->sgl, obj->pages->nents,
2422                         PCI_DMA_BIDIRECTIONAL))
2423                 return -ENOSPC;
2424
2425         return 0;
2426 }
2427
2428 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
2429 {
2430 #if 0
2431         writeq(pte, addr);
2432 #else
2433         iowrite32((u32)pte, addr);
2434         iowrite32(pte >> 32, addr + 4);
2435 #endif
2436 }
2437
2438 static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
2439                                      struct sg_table *st,
2440                                      uint64_t start,
2441                                      enum i915_cache_level level, u32 unused)
2442 {
2443         struct drm_i915_private *dev_priv = vm->dev->dev_private;
2444         unsigned first_entry = start >> PAGE_SHIFT;
2445         gen8_pte_t __iomem *gtt_entries =
2446                 (gen8_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
2447         int i = 0;
2448         struct sg_page_iter sg_iter;
2449         dma_addr_t addr = 0; /* shut up gcc */
2450
2451         for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
2452                 addr = sg_dma_address(sg_iter.sg) +
2453                         (sg_iter.sg_pgoffset << PAGE_SHIFT);
2454                 gen8_set_pte(&gtt_entries[i],
2455                              gen8_pte_encode(addr, level, true));
2456                 i++;
2457         }
2458
2459         /*
2460          * XXX: This serves as a posting read to make sure that the PTE has
2461          * actually been updated. There is some concern that even though
2462          * registers and PTEs are within the same BAR that they are potentially
2463          * of NUMA access patterns. Therefore, even with the way we assume
2464          * hardware should work, we must keep this posting read for paranoia.
2465          */
2466         if (i != 0)
2467                 WARN_ON(readq(&gtt_entries[i-1])
2468                         != gen8_pte_encode(addr, level, true));
2469
2470         /* This next bit makes the above posting read even more important. We
2471          * want to flush the TLBs only after we're certain all the PTE updates
2472          * have finished.
2473          */
2474         I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
2475         POSTING_READ(GFX_FLSH_CNTL_GEN6);
2476 }
2477
2478 /*
2479  * Binds an object into the global gtt with the specified cache level. The object
2480  * will be accessible to the GPU via commands whose operands reference offsets
2481  * within the global GTT as well as accessible by the GPU through the GMADR
2482  * mapped BAR (dev_priv->mm.gtt->gtt).
2483  */
2484 static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
2485                                      struct sg_table *st,
2486                                      uint64_t start,
2487                                      enum i915_cache_level level, u32 flags)
2488 {
2489         struct drm_i915_private *dev_priv = vm->dev->dev_private;
2490         unsigned first_entry = start >> PAGE_SHIFT;
2491         gen6_pte_t __iomem *gtt_entries =
2492                 (gen6_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
2493         int i = 0;
2494         struct sg_page_iter sg_iter;
2495         dma_addr_t addr = 0;
2496
2497         for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
2498                 addr = sg_page_iter_dma_address(&sg_iter);
2499                 iowrite32(vm->pte_encode(addr, level, true, flags), &gtt_entries[i]);
2500                 i++;
2501         }
2502
2503         /* XXX: This serves as a posting read to make sure that the PTE has
2504          * actually been updated. There is some concern that even though
2505          * registers and PTEs are within the same BAR that they are potentially
2506          * of NUMA access patterns. Therefore, even with the way we assume
2507          * hardware should work, we must keep this posting read for paranoia.
2508          */
2509         if (i != 0) {
2510                 unsigned long gtt = readl(&gtt_entries[i-1]);
2511                 WARN_ON(gtt != vm->pte_encode(addr, level, true, flags));
2512         }
2513
2514         /* This next bit makes the above posting read even more important. We
2515          * want to flush the TLBs only after we're certain all the PTE updates
2516          * have finished.
2517          */
2518         I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
2519         POSTING_READ(GFX_FLSH_CNTL_GEN6);
2520 }
2521
2522 static void gen8_ggtt_clear_range(struct i915_address_space *vm,
2523                                   uint64_t start,
2524                                   uint64_t length,
2525                                   bool use_scratch)
2526 {
2527         struct drm_i915_private *dev_priv = vm->dev->dev_private;
2528         unsigned first_entry = start >> PAGE_SHIFT;
2529         unsigned num_entries = length >> PAGE_SHIFT;
2530         gen8_pte_t scratch_pte, __iomem *gtt_base =
2531                 (gen8_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
2532         const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
2533         int i;
2534
2535         if (WARN(num_entries > max_entries,
2536                  "First entry = %d; Num entries = %d (max=%d)\n",
2537                  first_entry, num_entries, max_entries))
2538                 num_entries = max_entries;
2539
2540         scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page),
2541                                       I915_CACHE_LLC,
2542                                       use_scratch);
2543         for (i = 0; i < num_entries; i++)
2544                 gen8_set_pte(&gtt_base[i], scratch_pte);
2545         readl(gtt_base);
2546 }
2547
2548 static void gen6_ggtt_clear_range(struct i915_address_space *vm,
2549                                   uint64_t start,
2550                                   uint64_t length,
2551                                   bool use_scratch)
2552 {
2553         struct drm_i915_private *dev_priv = vm->dev->dev_private;
2554         unsigned first_entry = start >> PAGE_SHIFT;
2555         unsigned num_entries = length >> PAGE_SHIFT;
2556         gen6_pte_t scratch_pte, __iomem *gtt_base =
2557                 (gen6_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
2558         const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
2559         int i;
2560
2561         if (WARN(num_entries > max_entries,
2562                  "First entry = %d; Num entries = %d (max=%d)\n",
2563                  first_entry, num_entries, max_entries))
2564                 num_entries = max_entries;
2565
2566         scratch_pte = vm->pte_encode(px_dma(vm->scratch_page),
2567                                      I915_CACHE_LLC, use_scratch, 0);
2568
2569         for (i = 0; i < num_entries; i++)
2570                 iowrite32(scratch_pte, &gtt_base[i]);
2571         readl(gtt_base);
2572 }
2573
2574 static void i915_ggtt_insert_entries(struct i915_address_space *vm,
2575                                      struct sg_table *pages,
2576                                      uint64_t start,
2577                                      enum i915_cache_level cache_level, u32 unused)
2578 {
2579         unsigned int flags = (cache_level == I915_CACHE_NONE) ?
2580                 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
2581
2582         intel_gtt_insert_sg_entries(pages, start >> PAGE_SHIFT, flags);
2583 }
2584
2585 static void i915_ggtt_clear_range(struct i915_address_space *vm,
2586                                   uint64_t start,
2587                                   uint64_t length,
2588                                   bool unused)
2589 {
2590         unsigned first_entry = start >> PAGE_SHIFT;
2591         unsigned num_entries = length >> PAGE_SHIFT;
2592         intel_gtt_clear_range(first_entry, num_entries);
2593 }
2594
2595 static int ggtt_bind_vma(struct i915_vma *vma,
2596                          enum i915_cache_level cache_level,
2597                          u32 flags)
2598 {
2599         struct drm_i915_gem_object *obj = vma->obj;
2600         u32 pte_flags = 0;
2601         int ret;
2602
2603         ret = i915_get_ggtt_vma_pages(vma);
2604         if (ret)
2605                 return ret;
2606
2607         /* Currently applicable only to VLV */
2608         if (obj->gt_ro)
2609                 pte_flags |= PTE_READ_ONLY;
2610
2611         vma->vm->insert_entries(vma->vm, vma->ggtt_view.pages,
2612                                 vma->node.start,
2613                                 cache_level, pte_flags);
2614
2615         /*
2616          * Without aliasing PPGTT there's no difference between
2617          * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally
2618          * upgrade to both bound if we bind either to avoid double-binding.
2619          */
2620         vma->bound |= GLOBAL_BIND | LOCAL_BIND;
2621
2622         return 0;
2623 }
2624
2625 static int aliasing_gtt_bind_vma(struct i915_vma *vma,
2626                                  enum i915_cache_level cache_level,
2627                                  u32 flags)
2628 {
2629         struct drm_device *dev = vma->vm->dev;
2630         struct drm_i915_private *dev_priv = dev->dev_private;
2631         struct drm_i915_gem_object *obj = vma->obj;
2632         struct sg_table *pages = obj->pages;
2633         u32 pte_flags = 0;
2634         int ret;
2635
2636         ret = i915_get_ggtt_vma_pages(vma);
2637         if (ret)
2638                 return ret;
2639         pages = vma->ggtt_view.pages;
2640
2641         /* Currently applicable only to VLV */
2642         if (obj->gt_ro)
2643                 pte_flags |= PTE_READ_ONLY;
2644
2645
2646         if (flags & GLOBAL_BIND) {
2647                 vma->vm->insert_entries(vma->vm, pages,
2648                                         vma->node.start,
2649                                         cache_level, pte_flags);
2650         }
2651
2652         if (flags & LOCAL_BIND) {
2653                 struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt;
2654                 appgtt->base.insert_entries(&appgtt->base, pages,
2655                                             vma->node.start,
2656                                             cache_level, pte_flags);
2657         }
2658
2659         return 0;
2660 }
2661
2662 static void ggtt_unbind_vma(struct i915_vma *vma)
2663 {
2664         struct drm_device *dev = vma->vm->dev;
2665         struct drm_i915_private *dev_priv = dev->dev_private;
2666         struct drm_i915_gem_object *obj = vma->obj;
2667         const uint64_t size = min_t(uint64_t,
2668                                     obj->base.size,
2669                                     vma->node.size);
2670
2671         if (vma->bound & GLOBAL_BIND) {
2672                 vma->vm->clear_range(vma->vm,
2673                                      vma->node.start,
2674                                      size,
2675                                      true);
2676         }
2677
2678         if (dev_priv->mm.aliasing_ppgtt && vma->bound & LOCAL_BIND) {
2679                 struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt;
2680
2681                 appgtt->base.clear_range(&appgtt->base,
2682                                          vma->node.start,
2683                                          size,
2684                                          true);
2685         }
2686 }
2687
2688 void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
2689 {
2690         struct drm_device *dev = obj->base.dev;
2691         struct drm_i915_private *dev_priv = dev->dev_private;
2692         bool interruptible;
2693
2694         interruptible = do_idling(dev_priv);
2695
2696         dma_unmap_sg(dev->pdev->dev, obj->pages->sgl, obj->pages->nents,
2697                      PCI_DMA_BIDIRECTIONAL);
2698
2699         undo_idling(dev_priv, interruptible);
2700 }
2701
2702 static void i915_gtt_color_adjust(struct drm_mm_node *node,
2703                                   unsigned long color,
2704                                   u64 *start,
2705                                   u64 *end)
2706 {
2707         if (node->color != color)
2708                 *start += 4096;
2709
2710         if (!list_empty(&node->node_list)) {
2711                 node = list_entry(node->node_list.next,
2712                                   struct drm_mm_node,
2713                                   node_list);
2714                 if (node->allocated && node->color != color)
2715                         *end -= 4096;
2716         }
2717 }
2718
2719 static int i915_gem_setup_global_gtt(struct drm_device *dev,
2720                                      u64 start,
2721                                      u64 mappable_end,
2722                                      u64 end)
2723 {
2724         /* Let GEM Manage all of the aperture.
2725          *
2726          * However, leave one page at the end still bound to the scratch page.
2727          * There are a number of places where the hardware apparently prefetches
2728          * past the end of the object, and we've seen multiple hangs with the
2729          * GPU head pointer stuck in a batchbuffer bound at the last page of the
2730          * aperture.  One page should be enough to keep any prefetching inside
2731          * of the aperture.
2732          */
2733         struct drm_i915_private *dev_priv = dev->dev_private;
2734         struct i915_address_space *ggtt_vm = &dev_priv->gtt.base;
2735         unsigned long mappable;
2736         int error;
2737         struct drm_mm_node *entry;
2738         struct drm_i915_gem_object *obj;
2739         unsigned long hole_start, hole_end;
2740         int ret;
2741
2742         mappable = min(end, mappable_end) - start;
2743         BUG_ON(mappable_end > end);
2744
2745         ggtt_vm->start = start;
2746
2747         /* Subtract the guard page before address space initialization to
2748          * shrink the range used by drm_mm */
2749         ggtt_vm->total = end - start - PAGE_SIZE;
2750         i915_address_space_init(ggtt_vm, dev_priv);
2751         ggtt_vm->total += PAGE_SIZE;
2752
2753         if (intel_vgpu_active(dev)) {
2754                 ret = intel_vgt_balloon(dev);
2755                 if (ret)
2756                         return ret;
2757         }
2758
2759         if (!HAS_LLC(dev))
2760                 ggtt_vm->mm.color_adjust = i915_gtt_color_adjust;
2761
2762         /* Mark any preallocated objects as occupied */
2763         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
2764                 struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm);
2765
2766                 DRM_DEBUG_KMS("reserving preallocated space: %lx + %zx\n",
2767                               i915_gem_obj_ggtt_offset(obj), obj->base.size);
2768
2769                 WARN_ON(i915_gem_obj_ggtt_bound(obj));
2770                 ret = drm_mm_reserve_node(&ggtt_vm->mm, &vma->node);
2771                 if (ret) {
2772                         DRM_DEBUG_KMS("Reservation failed: %i\n", ret);
2773                         return ret;
2774                 }
2775                 vma->bound |= GLOBAL_BIND;
2776                 __i915_vma_set_map_and_fenceable(vma);
2777                 list_add_tail(&vma->mm_list, &ggtt_vm->inactive_list);
2778         }
2779
2780         /* Clear any non-preallocated blocks */
2781         drm_mm_for_each_hole(entry, &ggtt_vm->mm, hole_start, hole_end) {
2782                 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
2783                               hole_start, hole_end);
2784                 ggtt_vm->clear_range(ggtt_vm, hole_start,
2785                                      hole_end - hole_start, true);
2786         }
2787
2788 #ifdef __DragonFly__
2789         device_printf(dev->dev,
2790             "taking over the fictitious range 0x%lx-0x%lx\n",
2791             dev_priv->gtt.mappable_base + start, dev_priv->gtt.mappable_base + start + mappable);
2792         error = -vm_phys_fictitious_reg_range(dev_priv->gtt.mappable_base + start,
2793             dev_priv->gtt.mappable_base + start + mappable, VM_MEMATTR_WRITE_COMBINING);
2794 #endif
2795
2796         /* And finally clear the reserved guard page */
2797         ggtt_vm->clear_range(ggtt_vm, end - PAGE_SIZE, PAGE_SIZE, true);
2798
2799         if (USES_PPGTT(dev) && !USES_FULL_PPGTT(dev)) {
2800                 struct i915_hw_ppgtt *ppgtt;
2801
2802                 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
2803                 if (!ppgtt)
2804                         return -ENOMEM;
2805
2806                 ret = __hw_ppgtt_init(dev, ppgtt, true);
2807                 if (ret) {
2808                         ppgtt->base.cleanup(&ppgtt->base);
2809                         kfree(ppgtt);
2810                         return ret;
2811                 }
2812
2813                 if (ppgtt->base.allocate_va_range)
2814                         ret = ppgtt->base.allocate_va_range(&ppgtt->base, 0,
2815                                                             ppgtt->base.total);
2816                 if (ret) {
2817                         ppgtt->base.cleanup(&ppgtt->base);
2818                         kfree(ppgtt);
2819                         return ret;
2820                 }
2821
2822                 ppgtt->base.clear_range(&ppgtt->base,
2823                                         ppgtt->base.start,
2824                                         ppgtt->base.total,
2825                                         true);
2826
2827                 dev_priv->mm.aliasing_ppgtt = ppgtt;
2828                 WARN_ON(dev_priv->gtt.base.bind_vma != ggtt_bind_vma);
2829                 dev_priv->gtt.base.bind_vma = aliasing_gtt_bind_vma;
2830         }
2831
2832         return 0;
2833 }
2834
2835 void i915_gem_init_global_gtt(struct drm_device *dev)
2836 {
2837         struct drm_i915_private *dev_priv = dev->dev_private;
2838         u64 gtt_size, mappable_size;
2839
2840         gtt_size = dev_priv->gtt.base.total;
2841         mappable_size = dev_priv->gtt.mappable_end;
2842
2843         i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
2844 }
2845
2846 void i915_global_gtt_cleanup(struct drm_device *dev)
2847 {
2848         struct drm_i915_private *dev_priv = dev->dev_private;
2849         struct i915_address_space *vm = &dev_priv->gtt.base;
2850
2851         if (dev_priv->mm.aliasing_ppgtt) {
2852                 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
2853
2854                 ppgtt->base.cleanup(&ppgtt->base);
2855         }
2856
2857         if (drm_mm_initialized(&vm->mm)) {
2858                 if (intel_vgpu_active(dev))
2859                         intel_vgt_deballoon();
2860
2861                 drm_mm_takedown(&vm->mm);
2862                 list_del(&vm->global_link);
2863         }
2864
2865         vm->cleanup(vm);
2866 }
2867
2868 static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
2869 {
2870         snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
2871         snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
2872         return snb_gmch_ctl << 20;
2873 }
2874
2875 static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
2876 {
2877         bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
2878         bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
2879         if (bdw_gmch_ctl)
2880                 bdw_gmch_ctl = 1 << bdw_gmch_ctl;
2881
2882 #ifdef CONFIG_X86_32
2883         /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * PAGE_SIZE */
2884         if (bdw_gmch_ctl > 4)
2885                 bdw_gmch_ctl = 4;
2886 #endif
2887
2888         return bdw_gmch_ctl << 20;
2889 }
2890
2891 static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl)
2892 {
2893         gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT;
2894         gmch_ctrl &= SNB_GMCH_GGMS_MASK;
2895
2896         if (gmch_ctrl)
2897                 return 1 << (20 + gmch_ctrl);
2898
2899         return 0;
2900 }
2901
2902 static size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
2903 {
2904         snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
2905         snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
2906         return snb_gmch_ctl << 25; /* 32 MB units */
2907 }
2908
2909 static size_t gen8_get_stolen_size(u16 bdw_gmch_ctl)
2910 {
2911         bdw_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
2912         bdw_gmch_ctl &= BDW_GMCH_GMS_MASK;
2913         return bdw_gmch_ctl << 25; /* 32 MB units */
2914 }
2915
2916 static size_t chv_get_stolen_size(u16 gmch_ctrl)
2917 {
2918         gmch_ctrl >>= SNB_GMCH_GMS_SHIFT;
2919         gmch_ctrl &= SNB_GMCH_GMS_MASK;
2920
2921         /*
2922          * 0x0  to 0x10: 32MB increments starting at 0MB
2923          * 0x11 to 0x16: 4MB increments starting at 8MB
2924          * 0x17 to 0x1d: 4MB increments start at 36MB
2925          */
2926         if (gmch_ctrl < 0x11)
2927                 return gmch_ctrl << 25;
2928         else if (gmch_ctrl < 0x17)
2929                 return (gmch_ctrl - 0x11 + 2) << 22;
2930         else
2931                 return (gmch_ctrl - 0x17 + 9) << 22;
2932 }
2933
2934 static size_t gen9_get_stolen_size(u16 gen9_gmch_ctl)
2935 {
2936         gen9_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
2937         gen9_gmch_ctl &= BDW_GMCH_GMS_MASK;
2938
2939         if (gen9_gmch_ctl < 0xf0)
2940                 return gen9_gmch_ctl << 25; /* 32 MB units */
2941         else
2942                 /* 4MB increments starting at 0xf0 for 4MB */
2943                 return (gen9_gmch_ctl - 0xf0 + 1) << 22;
2944 }
2945
2946 static int ggtt_probe_common(struct drm_device *dev,
2947                              size_t gtt_size)
2948 {
2949         struct drm_i915_private *dev_priv = dev->dev_private;
2950         struct i915_page_scratch *scratch_page;
2951         phys_addr_t gtt_phys_addr;
2952
2953         /* For Modern GENs the PTEs and register space are split in the BAR */
2954         gtt_phys_addr = pci_resource_start(dev->pdev, 0) +
2955                 (pci_resource_len(dev->pdev, 0) / 2);
2956
2957         /*
2958          * On BXT writes larger than 64 bit to the GTT pagetable range will be
2959          * dropped. For WC mappings in general we have 64 byte burst writes
2960          * when the WC buffer is flushed, so we can't use it, but have to
2961          * resort to an uncached mapping. The WC issue is easily caught by the
2962          * readback check when writing GTT PTE entries.
2963          */
2964         if (IS_BROXTON(dev))
2965                 dev_priv->gtt.gsm = ioremap_nocache(gtt_phys_addr, gtt_size);
2966         else
2967                 dev_priv->gtt.gsm = ioremap_wc(gtt_phys_addr, gtt_size);
2968         if (!dev_priv->gtt.gsm) {
2969                 DRM_ERROR("Failed to map the gtt page table\n");
2970                 return -ENOMEM;
2971         }
2972
2973         scratch_page = alloc_scratch_page(dev);
2974         if (IS_ERR(scratch_page)) {
2975                 DRM_ERROR("Scratch setup failed\n");
2976                 /* iounmap will also get called at remove, but meh */
2977                 iounmap(dev_priv->gtt.gsm);
2978                 return PTR_ERR(scratch_page);
2979         }
2980
2981         dev_priv->gtt.base.scratch_page = scratch_page;
2982
2983         return 0;
2984 }
2985
2986 /* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability
2987  * bits. When using advanced contexts each context stores its own PAT, but
2988  * writing this data shouldn't be harmful even in those cases. */
2989 static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv)
2990 {
2991         uint64_t pat;
2992
2993         pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC)     | /* for normal objects, no eLLC */
2994               GEN8_PPAT(1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC) | /* for something pointing to ptes? */
2995               GEN8_PPAT(2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC) | /* for scanout with eLLC */
2996               GEN8_PPAT(3, GEN8_PPAT_UC)                     | /* Uncached objects, mostly for scanout */
2997               GEN8_PPAT(4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)) |
2998               GEN8_PPAT(5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)) |
2999               GEN8_PPAT(6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)) |
3000               GEN8_PPAT(7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
3001
3002         if (!USES_PPGTT(dev_priv->dev))
3003                 /* Spec: "For GGTT, there is NO pat_sel[2:0] from the entry,
3004                  * so RTL will always use the value corresponding to
3005                  * pat_sel = 000".
3006                  * So let's disable cache for GGTT to avoid screen corruptions.
3007                  * MOCS still can be used though.
3008                  * - System agent ggtt writes (i.e. cpu gtt mmaps) already work
3009                  * before this patch, i.e. the same uncached + snooping access
3010                  * like on gen6/7 seems to be in effect.
3011                  * - So this just fixes blitter/render access. Again it looks
3012                  * like it's not just uncached access, but uncached + snooping.
3013                  * So we can still hold onto all our assumptions wrt cpu
3014                  * clflushing on LLC machines.
3015                  */
3016                 pat = GEN8_PPAT(0, GEN8_PPAT_UC);
3017
3018         /* XXX: spec defines this as 2 distinct registers. It's unclear if a 64b
3019          * write would work. */
3020         I915_WRITE(GEN8_PRIVATE_PAT_LO, pat);
3021         I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32);
3022 }
3023
3024 static void chv_setup_private_ppat(struct drm_i915_private *dev_priv)
3025 {
3026         uint64_t pat;
3027
3028         /*
3029          * Map WB on BDW to snooped on CHV.
3030          *
3031          * Only the snoop bit has meaning for CHV, the rest is
3032          * ignored.
3033          *
3034          * The hardware will never snoop for certain types of accesses:
3035          * - CPU GTT (GMADR->GGTT->no snoop->memory)
3036          * - PPGTT page tables
3037          * - some other special cycles
3038          *
3039          * As with BDW, we also need to consider the following for GT accesses:
3040          * "For GGTT, there is NO pat_sel[2:0] from the entry,
3041          * so RTL will always use the value corresponding to
3042          * pat_sel = 000".
3043          * Which means we must set the snoop bit in PAT entry 0
3044          * in order to keep the global status page working.
3045          */
3046         pat = GEN8_PPAT(0, CHV_PPAT_SNOOP) |
3047               GEN8_PPAT(1, 0) |
3048               GEN8_PPAT(2, 0) |
3049               GEN8_PPAT(3, 0) |
3050               GEN8_PPAT(4, CHV_PPAT_SNOOP) |
3051               GEN8_PPAT(5, CHV_PPAT_SNOOP) |
3052               GEN8_PPAT(6, CHV_PPAT_SNOOP) |
3053               GEN8_PPAT(7, CHV_PPAT_SNOOP);
3054
3055         I915_WRITE(GEN8_PRIVATE_PAT_LO, pat);
3056         I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32);
3057 }
3058
3059 static int gen8_gmch_probe(struct drm_device *dev,
3060                            u64 *gtt_total,
3061                            size_t *stolen,
3062                            phys_addr_t *mappable_base,
3063                            u64 *mappable_end)
3064 {
3065         struct drm_i915_private *dev_priv = dev->dev_private;
3066         u64 gtt_size;
3067         u16 snb_gmch_ctl;
3068         int ret;
3069
3070         /* TODO: We're not aware of mappable constraints on gen8 yet */
3071         *mappable_base = pci_resource_start(dev->pdev, 2);
3072         *mappable_end = pci_resource_len(dev->pdev, 2);
3073
3074 #if 0
3075         if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(39)))
3076                 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(39));
3077 #endif
3078
3079         pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
3080
3081         if (INTEL_INFO(dev)->gen >= 9) {
3082                 *stolen = gen9_get_stolen_size(snb_gmch_ctl);
3083                 gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl);
3084         } else if (IS_CHERRYVIEW(dev)) {
3085                 *stolen = chv_get_stolen_size(snb_gmch_ctl);
3086                 gtt_size = chv_get_total_gtt_size(snb_gmch_ctl);
3087         } else {
3088                 *stolen = gen8_get_stolen_size(snb_gmch_ctl);
3089                 gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl);
3090         }
3091
3092         *gtt_total = (gtt_size / sizeof(gen8_pte_t)) << PAGE_SHIFT;
3093
3094         if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev))
3095                 chv_setup_private_ppat(dev_priv);
3096         else
3097                 bdw_setup_private_ppat(dev_priv);
3098
3099         ret = ggtt_probe_common(dev, gtt_size);
3100
3101         dev_priv->gtt.base.clear_range = gen8_ggtt_clear_range;
3102         dev_priv->gtt.base.insert_entries = gen8_ggtt_insert_entries;
3103         dev_priv->gtt.base.bind_vma = ggtt_bind_vma;
3104         dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma;
3105
3106         return ret;
3107 }
3108
3109 static int gen6_gmch_probe(struct drm_device *dev,
3110                            u64 *gtt_total,
3111                            size_t *stolen,
3112                            phys_addr_t *mappable_base,
3113                            u64 *mappable_end)
3114 {
3115         struct drm_i915_private *dev_priv = dev->dev_private;
3116         unsigned int gtt_size;
3117         u16 snb_gmch_ctl;
3118         int ret;
3119
3120         *mappable_base = pci_resource_start(dev->pdev, 2);
3121         *mappable_end = pci_resource_len(dev->pdev, 2);
3122
3123         /* 64/512MB is the current min/max we actually know of, but this is just
3124          * a coarse sanity check.
3125          */
3126         if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) {
3127                 DRM_ERROR("Unknown GMADR size (%lx)\n",
3128                           dev_priv->gtt.mappable_end);
3129                 return -ENXIO;
3130         }
3131
3132 #if 0
3133         if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40)))
3134                 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40));
3135 #endif
3136         pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
3137
3138         *stolen = gen6_get_stolen_size(snb_gmch_ctl);
3139
3140         gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl);
3141         *gtt_total = (gtt_size / sizeof(gen6_pte_t)) << PAGE_SHIFT;
3142
3143         ret = ggtt_probe_common(dev, gtt_size);
3144
3145         dev_priv->gtt.base.clear_range = gen6_ggtt_clear_range;
3146         dev_priv->gtt.base.insert_entries = gen6_ggtt_insert_entries;
3147         dev_priv->gtt.base.bind_vma = ggtt_bind_vma;
3148         dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma;
3149
3150         return ret;
3151 }
3152
3153 static void gen6_gmch_remove(struct i915_address_space *vm)
3154 {
3155         struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base);
3156
3157         iounmap(gtt->gsm);
3158         free_scratch_page(vm->dev, vm->scratch_page);
3159 }
3160
3161 static int i915_gmch_probe(struct drm_device *dev,
3162                            u64 *gtt_total,
3163                            size_t *stolen,
3164                            phys_addr_t *mappable_base,
3165                            u64 *mappable_end)
3166 {
3167         struct drm_i915_private *dev_priv = dev->dev_private;
3168 #if 0
3169         int ret;
3170
3171         ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL);
3172         if (!ret) {
3173                 DRM_ERROR("failed to set up gmch\n");
3174                 return -EIO;
3175         }
3176 #endif
3177
3178         intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end);
3179
3180         dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev);
3181         dev_priv->gtt.base.insert_entries = i915_ggtt_insert_entries;
3182         dev_priv->gtt.base.clear_range = i915_ggtt_clear_range;
3183         dev_priv->gtt.base.bind_vma = ggtt_bind_vma;
3184         dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma;
3185
3186         if (unlikely(dev_priv->gtt.do_idle_maps))
3187                 DRM_INFO("applying Ironlake quirks for intel_iommu\n");
3188
3189         return 0;
3190 }
3191
3192 static void i915_gmch_remove(struct i915_address_space *vm)
3193 {
3194         intel_gmch_remove();
3195 }
3196
3197 int i915_gem_gtt_init(struct drm_device *dev)
3198 {
3199         struct drm_i915_private *dev_priv = dev->dev_private;
3200         struct i915_gtt *gtt = &dev_priv->gtt;
3201         int ret;
3202
3203         if (INTEL_INFO(dev)->gen <= 5) {
3204                 gtt->gtt_probe = i915_gmch_probe;
3205                 gtt->base.cleanup = i915_gmch_remove;
3206         } else if (INTEL_INFO(dev)->gen < 8) {
3207                 gtt->gtt_probe = gen6_gmch_probe;
3208                 gtt->base.cleanup = gen6_gmch_remove;
3209                 if (IS_HASWELL(dev) && dev_priv->ellc_size)
3210                         gtt->base.pte_encode = iris_pte_encode;
3211                 else if (IS_HASWELL(dev))
3212                         gtt->base.pte_encode = hsw_pte_encode;
3213                 else if (IS_VALLEYVIEW(dev))
3214                         gtt->base.pte_encode = byt_pte_encode;
3215                 else if (INTEL_INFO(dev)->gen >= 7)
3216                         gtt->base.pte_encode = ivb_pte_encode;
3217                 else
3218                         gtt->base.pte_encode = snb_pte_encode;
3219         } else {
3220                 dev_priv->gtt.gtt_probe = gen8_gmch_probe;
3221                 dev_priv->gtt.base.cleanup = gen6_gmch_remove;
3222         }
3223
3224         gtt->base.dev = dev;
3225
3226         ret = gtt->gtt_probe(dev, &gtt->base.total, &gtt->stolen_size,
3227                              &gtt->mappable_base, &gtt->mappable_end);
3228         if (ret)
3229                 return ret;
3230
3231         /* GMADR is the PCI mmio aperture into the global GTT. */
3232         DRM_INFO("Memory usable by graphics device = %luM\n",
3233                  gtt->base.total >> 20);
3234         DRM_DEBUG_DRIVER("GMADR size = %ldM\n", gtt->mappable_end >> 20);
3235         DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20);
3236 #ifdef CONFIG_INTEL_IOMMU
3237         if (intel_iommu_gfx_mapped)
3238                 DRM_INFO("VT-d active for gfx access\n");
3239 #endif
3240         /*
3241          * i915.enable_ppgtt is read-only, so do an early pass to validate the
3242          * user's requested state against the hardware/driver capabilities.  We
3243          * do this now so that we can print out any log messages once rather
3244          * than every time we check intel_enable_ppgtt().
3245          */
3246         i915.enable_ppgtt = sanitize_enable_ppgtt(dev, i915.enable_ppgtt);
3247         DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt);
3248
3249         return 0;
3250 }
3251
3252 void i915_gem_restore_gtt_mappings(struct drm_device *dev)
3253 {
3254         struct drm_i915_private *dev_priv = dev->dev_private;
3255         struct drm_i915_gem_object *obj;
3256         struct i915_address_space *vm;
3257
3258         i915_check_and_clear_faults(dev);
3259
3260         /* First fill our portion of the GTT with scratch pages */
3261         dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
3262                                        dev_priv->gtt.base.start,
3263                                        dev_priv->gtt.base.total,
3264                                        true);
3265
3266         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
3267                 struct i915_vma *vma = i915_gem_obj_to_vma(obj,
3268                                                            &dev_priv->gtt.base);
3269                 if (!vma)
3270                         continue;
3271
3272                 i915_gem_clflush_object(obj, obj->pin_display);
3273                 WARN_ON(i915_vma_bind(vma, obj->cache_level, PIN_UPDATE));
3274         }
3275
3276
3277         if (INTEL_INFO(dev)->gen >= 8) {
3278                 if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev))
3279                         chv_setup_private_ppat(dev_priv);
3280                 else
3281                         bdw_setup_private_ppat(dev_priv);
3282
3283                 return;
3284         }
3285
3286         if (USES_PPGTT(dev)) {
3287                 list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
3288                         /* TODO: Perhaps it shouldn't be gen6 specific */
3289
3290                         struct i915_hw_ppgtt *ppgtt =
3291                                         container_of(vm, struct i915_hw_ppgtt,
3292                                                      base);
3293
3294                         if (i915_is_ggtt(vm))
3295                                 ppgtt = dev_priv->mm.aliasing_ppgtt;
3296
3297                         gen6_write_page_range(dev_priv, &ppgtt->pd,
3298                                               0, ppgtt->base.total);
3299                 }
3300         }
3301
3302         i915_ggtt_flush(dev_priv);
3303 }
3304
3305 static struct i915_vma *
3306 __i915_gem_vma_create(struct drm_i915_gem_object *obj,
3307                       struct i915_address_space *vm,
3308                       const struct i915_ggtt_view *ggtt_view)
3309 {
3310         struct i915_vma *vma;
3311
3312         if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view))
3313                 return ERR_PTR(-EINVAL);
3314
3315         vma = kzalloc(sizeof(*vma), GFP_KERNEL);
3316         if (vma == NULL)
3317                 return ERR_PTR(-ENOMEM);
3318
3319         INIT_LIST_HEAD(&vma->vma_link);
3320         INIT_LIST_HEAD(&vma->mm_list);
3321         INIT_LIST_HEAD(&vma->exec_list);
3322         vma->vm = vm;
3323         vma->obj = obj;
3324
3325         if (i915_is_ggtt(vm))
3326                 vma->ggtt_view = *ggtt_view;
3327
3328         list_add_tail(&vma->vma_link, &obj->vma_list);
3329         if (!i915_is_ggtt(vm))
3330                 i915_ppgtt_get(i915_vm_to_ppgtt(vm));
3331
3332         return vma;
3333 }
3334
3335 struct i915_vma *
3336 i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
3337                                   struct i915_address_space *vm)
3338 {
3339         struct i915_vma *vma;
3340
3341         vma = i915_gem_obj_to_vma(obj, vm);
3342         if (!vma)
3343                 vma = __i915_gem_vma_create(obj, vm,
3344                                             i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL);
3345
3346         return vma;
3347 }
3348
3349 struct i915_vma *
3350 i915_gem_obj_lookup_or_create_ggtt_vma(struct drm_i915_gem_object *obj,
3351                                        const struct i915_ggtt_view *view)
3352 {
3353         struct i915_address_space *ggtt = i915_obj_to_ggtt(obj);
3354         struct i915_vma *vma;
3355
3356         if (WARN_ON(!view))
3357                 return ERR_PTR(-EINVAL);
3358
3359         vma = i915_gem_obj_to_ggtt_view(obj, view);
3360
3361         if (IS_ERR(vma))
3362                 return vma;
3363
3364         if (!vma)
3365                 vma = __i915_gem_vma_create(obj, ggtt, view);
3366
3367         return vma;
3368
3369 }
3370
3371 static struct scatterlist *
3372 rotate_pages(dma_addr_t *in, unsigned int offset,
3373              unsigned int width, unsigned int height,
3374              struct sg_table *st, struct scatterlist *sg)
3375 {
3376         unsigned int column, row;
3377         unsigned int src_idx;
3378
3379         if (!sg) {
3380                 st->nents = 0;
3381                 sg = st->sgl;
3382         }
3383
3384         for (column = 0; column < width; column++) {
3385                 src_idx = width * (height - 1) + column;
3386                 for (row = 0; row < height; row++) {
3387                         st->nents++;
3388                         /* We don't need the pages, but need to initialize
3389                          * the entries so the sg list can be happily traversed.
3390                          * The only thing we need are DMA addresses.
3391                          */
3392                         sg_set_page(sg, NULL, PAGE_SIZE, 0);
3393                         sg_dma_address(sg) = in[offset + src_idx];
3394                         sg_dma_len(sg) = PAGE_SIZE;
3395                         sg = sg_next(sg);
3396                         src_idx -= width;
3397                 }
3398         }
3399
3400         return sg;
3401 }
3402
3403 static struct sg_table *
3404 intel_rotate_fb_obj_pages(struct i915_ggtt_view *ggtt_view,
3405                           struct drm_i915_gem_object *obj)
3406 {
3407         struct intel_rotation_info *rot_info = &ggtt_view->rotation_info;
3408         unsigned int size_pages = rot_info->size >> PAGE_SHIFT;
3409         unsigned int size_pages_uv;
3410         struct sg_page_iter sg_iter;
3411         unsigned long i;
3412         dma_addr_t *page_addr_list;
3413         struct sg_table *st;
3414         unsigned int uv_start_page;
3415         struct scatterlist *sg;
3416         int ret = -ENOMEM;
3417
3418         /* Allocate a temporary list of source pages for random access. */
3419         page_addr_list = drm_malloc_ab(obj->base.size / PAGE_SIZE,
3420                                        sizeof(dma_addr_t));
3421         if (!page_addr_list)
3422                 return ERR_PTR(ret);
3423
3424         /* Account for UV plane with NV12. */
3425         if (rot_info->pixel_format == DRM_FORMAT_NV12)
3426                 size_pages_uv = rot_info->size_uv >> PAGE_SHIFT;
3427         else
3428                 size_pages_uv = 0;
3429
3430         /* Allocate target SG list. */
3431         st = kmalloc(sizeof(*st), M_DRM, M_WAITOK);
3432         if (!st)
3433                 goto err_st_alloc;
3434
3435         ret = sg_alloc_table(st, size_pages + size_pages_uv, GFP_KERNEL);
3436         if (ret)
3437                 goto err_sg_alloc;
3438
3439         /* Populate source page list from the object. */
3440         i = 0;
3441         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
3442                 page_addr_list[i] = sg_page_iter_dma_address(&sg_iter);
3443                 i++;
3444         }
3445
3446         /* Rotate the pages. */
3447         sg = rotate_pages(page_addr_list, 0,
3448                      rot_info->width_pages, rot_info->height_pages,
3449                      st, NULL);
3450
3451         /* Append the UV plane if NV12. */
3452         if (rot_info->pixel_format == DRM_FORMAT_NV12) {
3453                 uv_start_page = size_pages;
3454
3455                 /* Check for tile-row un-alignment. */
3456                 if (offset_in_page(rot_info->uv_offset))
3457                         uv_start_page--;
3458
3459                 rot_info->uv_start_page = uv_start_page;
3460
3461                 rotate_pages(page_addr_list, uv_start_page,
3462                              rot_info->width_pages_uv,
3463                              rot_info->height_pages_uv,
3464                              st, sg);
3465         }
3466
3467         DRM_DEBUG_KMS(
3468                       "Created rotated page mapping for object size %zu (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %u pages (%u plane 0)).\n",
3469                       obj->base.size, rot_info->pitch, rot_info->height,
3470                       rot_info->pixel_format, rot_info->width_pages,
3471                       rot_info->height_pages, size_pages + size_pages_uv,
3472                       size_pages);
3473
3474         drm_free_large(page_addr_list);
3475
3476         return st;
3477
3478 err_sg_alloc:
3479         kfree(st);
3480 err_st_alloc:
3481         drm_free_large(page_addr_list);
3482
3483         DRM_DEBUG_KMS(
3484                       "Failed to create rotated mapping for object size %zu! (%d) (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %u pages (%u plane 0))\n",
3485                       obj->base.size, ret, rot_info->pitch, rot_info->height,
3486                       rot_info->pixel_format, rot_info->width_pages,
3487                       rot_info->height_pages, size_pages + size_pages_uv,
3488                       size_pages);
3489         return ERR_PTR(ret);
3490 }
3491
3492 static struct sg_table *
3493 intel_partial_pages(const struct i915_ggtt_view *view,
3494                     struct drm_i915_gem_object *obj)
3495 {
3496         struct sg_table *st;
3497         struct scatterlist *sg;
3498         struct sg_page_iter obj_sg_iter;
3499         int ret = -ENOMEM;
3500
3501         st = kmalloc(sizeof(*st), M_DRM, M_WAITOK);
3502         if (!st)
3503                 goto err_st_alloc;
3504
3505         ret = sg_alloc_table(st, view->params.partial.size, GFP_KERNEL);
3506         if (ret)
3507                 goto err_sg_alloc;
3508
3509         sg = st->sgl;
3510         st->nents = 0;
3511         for_each_sg_page(obj->pages->sgl, &obj_sg_iter, obj->pages->nents,
3512                 view->params.partial.offset)
3513         {
3514                 if (st->nents >= view->params.partial.size)
3515                         break;
3516
3517                 sg_set_page(sg, NULL, PAGE_SIZE, 0);
3518                 sg_dma_address(sg) = sg_page_iter_dma_address(&obj_sg_iter);
3519                 sg_dma_len(sg) = PAGE_SIZE;
3520
3521                 sg = sg_next(sg);
3522                 st->nents++;
3523         }
3524
3525         return st;
3526
3527 err_sg_alloc:
3528         kfree(st);
3529 err_st_alloc:
3530         return ERR_PTR(ret);
3531 }
3532
3533 static int
3534 i915_get_ggtt_vma_pages(struct i915_vma *vma)
3535 {
3536         int ret = 0;
3537
3538         if (vma->ggtt_view.pages)
3539                 return 0;
3540
3541         if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL)
3542                 vma->ggtt_view.pages = vma->obj->pages;
3543         else if (vma->ggtt_view.type == I915_GGTT_VIEW_ROTATED)
3544                 vma->ggtt_view.pages =
3545                         intel_rotate_fb_obj_pages(&vma->ggtt_view, vma->obj);
3546         else if (vma->ggtt_view.type == I915_GGTT_VIEW_PARTIAL)
3547                 vma->ggtt_view.pages =
3548                         intel_partial_pages(&vma->ggtt_view, vma->obj);
3549         else
3550                 WARN_ONCE(1, "GGTT view %u not implemented!\n",
3551                           vma->ggtt_view.type);
3552
3553         if (!vma->ggtt_view.pages) {
3554                 DRM_ERROR("Failed to get pages for GGTT view type %u!\n",
3555                           vma->ggtt_view.type);
3556                 ret = -EINVAL;
3557         } else if (IS_ERR(vma->ggtt_view.pages)) {
3558                 ret = PTR_ERR(vma->ggtt_view.pages);
3559                 vma->ggtt_view.pages = NULL;
3560                 DRM_ERROR("Failed to get pages for VMA view type %u (%d)!\n",
3561                           vma->ggtt_view.type, ret);
3562         }
3563
3564         return ret;
3565 }
3566
3567 /**
3568  * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space.
3569  * @vma: VMA to map
3570  * @cache_level: mapping cache level
3571  * @flags: flags like global or local mapping
3572  *
3573  * DMA addresses are taken from the scatter-gather table of this object (or of
3574  * this VMA in case of non-default GGTT views) and PTE entries set up.
3575  * Note that DMA addresses are also the only part of the SG table we care about.
3576  */
3577 int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
3578                   u32 flags)
3579 {
3580         int ret;
3581         u32 bind_flags;
3582
3583         if (WARN_ON(flags == 0))
3584                 return -EINVAL;
3585
3586         bind_flags = 0;
3587         if (flags & PIN_GLOBAL)
3588                 bind_flags |= GLOBAL_BIND;
3589         if (flags & PIN_USER)
3590                 bind_flags |= LOCAL_BIND;
3591
3592         if (flags & PIN_UPDATE)
3593                 bind_flags |= vma->bound;
3594         else
3595                 bind_flags &= ~vma->bound;
3596
3597         if (bind_flags == 0)
3598                 return 0;
3599
3600         if (vma->bound == 0 && vma->vm->allocate_va_range) {
3601                 trace_i915_va_alloc(vma->vm,
3602                                     vma->node.start,
3603                                     vma->node.size,
3604                                     VM_TO_TRACE_NAME(vma->vm));
3605
3606                 /* XXX: i915_vma_pin() will fix this +- hack */
3607                 vma->pin_count++;
3608                 ret = vma->vm->allocate_va_range(vma->vm,
3609                                                  vma->node.start,
3610                                                  vma->node.size);
3611                 vma->pin_count--;
3612                 if (ret)
3613                         return ret;
3614         }
3615
3616         ret = vma->vm->bind_vma(vma, cache_level, bind_flags);
3617         if (ret)
3618                 return ret;
3619
3620         vma->bound |= bind_flags;
3621
3622         return 0;
3623 }
3624
3625 /**
3626  * i915_ggtt_view_size - Get the size of a GGTT view.
3627  * @obj: Object the view is of.
3628  * @view: The view in question.
3629  *
3630  * @return The size of the GGTT view in bytes.
3631  */
3632 size_t
3633 i915_ggtt_view_size(struct drm_i915_gem_object *obj,
3634                     const struct i915_ggtt_view *view)
3635 {
3636         if (view->type == I915_GGTT_VIEW_NORMAL) {
3637                 return obj->base.size;
3638         } else if (view->type == I915_GGTT_VIEW_ROTATED) {
3639                 return view->rotation_info.size;
3640         } else if (view->type == I915_GGTT_VIEW_PARTIAL) {
3641                 return view->params.partial.size << PAGE_SHIFT;
3642         } else {
3643                 WARN_ONCE(1, "GGTT view %u not implemented!\n", view->type);
3644                 return obj->base.size;
3645         }
3646 }