sys/dev/drm/i915/i915_gem_gtt.c

   1 /*
   2  * Copyright © 2010 Daniel Vetter
   3  * Copyright © 2011-2014 Intel Corporation
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <linux/seq_file.h>
  27 #include <drm/drmP.h>
  28 #include <drm/i915_drm.h>
  29 #include "i915_drv.h"
  30 #include "intel_drv.h"
  31
  32 #include <linux/highmem.h>
  33
  34 /**
  35  * DOC: Global GTT views
  36  *
  37  * Background and previous state
  38  *
  39  * Historically objects could exists (be bound) in global GTT space only as
  40  * singular instances with a view representing all of the object's backing pages
  41  * in a linear fashion. This view will be called a normal view.
  42  *
  43  * To support multiple views of the same object, where the number of mapped
  44  * pages is not equal to the backing store, or where the layout of the pages
  45  * is not linear, concept of a GGTT view was added.
  46  *
  47  * One example of an alternative view is a stereo display driven by a single
  48  * image. In this case we would have a framebuffer looking like this
  49  * (2x2 pages):
  50  *
  51  *    12
  52  *    34
  53  *
  54  * Above would represent a normal GGTT view as normally mapped for GPU or CPU
  55  * rendering. In contrast, fed to the display engine would be an alternative
  56  * view which could look something like this:
  57  *
  58  *   1212
  59  *   3434
  60  *
  61  * In this example both the size and layout of pages in the alternative view is
  62  * different from the normal view.
  63  *
  64  * Implementation and usage
  65  *
  66  * GGTT views are implemented using VMAs and are distinguished via enum
  67  * i915_ggtt_view_type and struct i915_ggtt_view.
  68  *
  69  * A new flavour of core GEM functions which work with GGTT bound objects were
  70  * added with the _view suffix. They take the struct i915_ggtt_view parameter
  71  * encapsulating all metadata required to implement a view.
  72  *
  73  * As a helper for callers which are only interested in the normal view,
  74  * globally const i915_ggtt_view_normal singleton instance exists. All old core
  75  * GEM API functions, the ones not taking the view parameter, are operating on,
  76  * or with the normal GGTT view.
  77  *
  78  * Code wanting to add or use a new GGTT view needs to:
  79  *
  80  * 1. Add a new enum with a suitable name.
  81  * 2. Extend the metadata in the i915_ggtt_view structure if required.
  82  * 3. Add support to i915_get_vma_pages().
  83  *
  84  * New views are required to build a scatter-gather table from within the
  85  * i915_get_vma_pages function. This table is stored in the vma.ggtt_view and
  86  * exists for the lifetime of an VMA.
  87  *
  88  * Core API is designed to have copy semantics which means that passed in
  89  * struct i915_ggtt_view does not need to be persistent (left around after
  90  * calling the core API functions).
  91  *
  92  */
  93
  94 const struct i915_ggtt_view i915_ggtt_view_normal;
  95
  96 static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv);
  97 static void chv_setup_private_ppat(struct drm_i915_private *dev_priv);
  98
  99 static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)
 100 {
 101         bool has_aliasing_ppgtt;
 102         bool has_full_ppgtt;
 103
 104         has_aliasing_ppgtt = INTEL_INFO(dev)->gen >= 6;
 105         has_full_ppgtt = INTEL_INFO(dev)->gen >= 7;
 106
 107         /*
 108          * We don't allow disabling PPGTT for gen9+ as it's a requirement for
 109          * execlists, the sole mechanism available to submit work.
 110          */
 111         if (INTEL_INFO(dev)->gen < 9 &&
 112             (enable_ppgtt == 0 || !has_aliasing_ppgtt))
 113                 return 0;
 114
 115         if (enable_ppgtt == 1)
 116                 return 1;
 117
 118         if (enable_ppgtt == 2 && has_full_ppgtt)
 119                 return 2;
 120
 121 #ifdef CONFIG_INTEL_IOMMU
 122         /* Disable ppgtt on SNB if VT-d is on. */
 123         if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) {
 124                 DRM_INFO("Disabling PPGTT because VT-d is on\n");
 125                 return 0;
 126         }
 127 #endif
 128
 129         /* Early VLV doesn't have this */
 130         if (IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) &&
 131             dev->pdev->revision < 0xb) {
 132                 DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n");
 133                 return 0;
 134         }
 135
 136         if (INTEL_INFO(dev)->gen >= 8 && i915.enable_execlists)
 137                 return 2;
 138         else
 139                 return has_aliasing_ppgtt ? 1 : 0;
 140 }
 141
 142 static void ppgtt_bind_vma(struct i915_vma *vma,
 143                            enum i915_cache_level cache_level,
 144                            u32 flags);
 145 static void ppgtt_unbind_vma(struct i915_vma *vma);
 146
 147 static inline gen8_gtt_pte_t gen8_pte_encode(dma_addr_t addr,
 148                                              enum i915_cache_level level,
 149                                              bool valid)
 150 {
 151         gen8_gtt_pte_t pte = valid ? _PAGE_PRESENT | _PAGE_RW : 0;
 152         pte |= addr;
 153
 154         switch (level) {
 155         case I915_CACHE_NONE:
 156                 pte |= PPAT_UNCACHED_INDEX;
 157                 break;
 158         case I915_CACHE_WT:
 159                 pte |= PPAT_DISPLAY_ELLC_INDEX;
 160                 break;
 161         default:
 162                 pte |= PPAT_CACHED_INDEX;
 163                 break;
 164         }
 165
 166         return pte;
 167 }
 168
 169 static inline gen8_ppgtt_pde_t gen8_pde_encode(struct drm_device *dev,
 170                                              dma_addr_t addr,
 171                                              enum i915_cache_level level)
 172 {
 173         gen8_ppgtt_pde_t pde = _PAGE_PRESENT | _PAGE_RW;
 174         pde |= addr;
 175         if (level != I915_CACHE_NONE)
 176                 pde |= PPAT_CACHED_PDE_INDEX;
 177         else
 178                 pde |= PPAT_UNCACHED_INDEX;
 179         return pde;
 180 }
 181
 182 static gen6_gtt_pte_t snb_pte_encode(dma_addr_t addr,
 183                                      enum i915_cache_level level,
 184                                      bool valid, u32 unused)
 185 {
 186         gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
 187         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 188
 189         switch (level) {
 190         case I915_CACHE_L3_LLC:
 191         case I915_CACHE_LLC:
 192                 pte |= GEN6_PTE_CACHE_LLC;
 193                 break;
 194         case I915_CACHE_NONE:
 195                 pte |= GEN6_PTE_UNCACHED;
 196                 break;
 197         default:
 198                 MISSING_CASE(level);
 199         }
 200
 201         return pte;
 202 }
 203
 204 static gen6_gtt_pte_t ivb_pte_encode(dma_addr_t addr,
 205                                      enum i915_cache_level level,
 206                                      bool valid, u32 unused)
 207 {
 208         gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
 209         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 210
 211         switch (level) {
 212         case I915_CACHE_L3_LLC:
 213                 pte |= GEN7_PTE_CACHE_L3_LLC;
 214                 break;
 215         case I915_CACHE_LLC:
 216                 pte |= GEN6_PTE_CACHE_LLC;
 217                 break;
 218         case I915_CACHE_NONE:
 219                 pte |= GEN6_PTE_UNCACHED;
 220                 break;
 221         default:
 222                 MISSING_CASE(level);
 223         }
 224
 225         return pte;
 226 }
 227
 228 static gen6_gtt_pte_t byt_pte_encode(dma_addr_t addr,
 229                                      enum i915_cache_level level,
 230                                      bool valid, u32 flags)
 231 {
 232         gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
 233         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 234
 235         if (!(flags & PTE_READ_ONLY))
 236                 pte |= BYT_PTE_WRITEABLE;
 237
 238         if (level != I915_CACHE_NONE)
 239                 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
 240
 241         return pte;
 242 }
 243
 244 static gen6_gtt_pte_t hsw_pte_encode(dma_addr_t addr,
 245                                      enum i915_cache_level level,
 246                                      bool valid, u32 unused)
 247 {
 248         gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
 249         pte |= HSW_PTE_ADDR_ENCODE(addr);
 250
 251         if (level != I915_CACHE_NONE)
 252                 pte |= HSW_WB_LLC_AGE3;
 253
 254         return pte;
 255 }
 256
 257 static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,
 258                                       enum i915_cache_level level,
 259                                       bool valid, u32 unused)
 260 {
 261         gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
 262         pte |= HSW_PTE_ADDR_ENCODE(addr);
 263
 264         switch (level) {
 265         case I915_CACHE_NONE:
 266                 break;
 267         case I915_CACHE_WT:
 268                 pte |= HSW_WT_ELLC_LLC_AGE3;
 269                 break;
 270         default:
 271                 pte |= HSW_WB_ELLC_LLC_AGE3;
 272                 break;
 273         }
 274
 275         return pte;
 276 }
 277
 278 /* Broadwell Page Directory Pointer Descriptors */
 279 static int gen8_write_pdp(struct intel_engine_cs *ring, unsigned entry,
 280                            uint64_t val)
 281 {
 282         int ret;
 283
 284         BUG_ON(entry >= 4);
 285
 286         ret = intel_ring_begin(ring, 6);
 287         if (ret)
 288                 return ret;
 289
 290         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
 291         intel_ring_emit(ring, GEN8_RING_PDP_UDW(ring, entry));
 292         intel_ring_emit(ring, (u32)(val >> 32));
 293         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
 294         intel_ring_emit(ring, GEN8_RING_PDP_LDW(ring, entry));
 295         intel_ring_emit(ring, (u32)(val));
 296         intel_ring_advance(ring);
 297
 298         return 0;
 299 }
 300
 301 static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,
 302                           struct intel_engine_cs *ring)
 303 {
 304         int i, ret;
 305
 306         /* bit of a hack to find the actual last used pd */
 307         int used_pd = ppgtt->num_pd_entries / GEN8_PDES_PER_PAGE;
 308
 309         for (i = used_pd - 1; i >= 0; i--) {
 310                 dma_addr_t addr = ppgtt->pd_dma_addr[i];
 311                 ret = gen8_write_pdp(ring, i, addr);
 312                 if (ret)
 313                         return ret;
 314         }
 315
 316         return 0;
 317 }
 318
 319 static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
 320                                    uint64_t start,
 321                                    uint64_t length,
 322                                    bool use_scratch)
 323 {
 324         struct i915_hw_ppgtt *ppgtt =
 325                 container_of(vm, struct i915_hw_ppgtt, base);
 326         gen8_gtt_pte_t *pt_vaddr, scratch_pte;
 327         unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK;
 328         unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK;
 329         unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK;
 330         unsigned num_entries = length >> PAGE_SHIFT;
 331         unsigned last_pte, i;
 332
 333         scratch_pte = gen8_pte_encode(ppgtt->base.scratch.addr,
 334                                       I915_CACHE_LLC, use_scratch);
 335
 336         while (num_entries) {
 337                 struct vm_page *page_table = ppgtt->gen8_pt_pages[pdpe][pde];
 338
 339                 last_pte = pte + num_entries;
 340                 if (last_pte > GEN8_PTES_PER_PAGE)
 341                         last_pte = GEN8_PTES_PER_PAGE;
 342
 343                 pt_vaddr = kmap_atomic(page_table);
 344
 345                 for (i = pte; i < last_pte; i++) {
 346                         pt_vaddr[i] = scratch_pte;
 347                         num_entries--;
 348                 }
 349
 350                 if (!HAS_LLC(ppgtt->base.dev))
 351                         drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
 352                 kunmap_atomic(pt_vaddr);
 353
 354                 pte = 0;
 355                 if (++pde == GEN8_PDES_PER_PAGE) {
 356                         pdpe++;
 357                         pde = 0;
 358                 }
 359         }
 360 }
 361
 362 static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
 363                                       vm_page_t *pages,
 364                                       uint64_t start,
 365                                       unsigned int num_entries,
 366                                       enum i915_cache_level cache_level, u32 unused)
 367 {
 368         struct i915_hw_ppgtt *ppgtt =
 369                 container_of(vm, struct i915_hw_ppgtt, base);
 370         gen8_gtt_pte_t *pt_vaddr;
 371         unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK;
 372         unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK;
 373         unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK;
 374         int i;
 375
 376         pt_vaddr = NULL;
 377
 378         for (i=0;i<num_entries;i++) {
 379                 if (WARN_ON(pdpe >= GEN8_LEGACY_PDPS))
 380                         break;
 381
 382                 if (pt_vaddr == NULL)
 383                         pt_vaddr = kmap_atomic(ppgtt->gen8_pt_pages[pdpe][pde]);
 384
 385                 pt_vaddr[pte] =
 386                         gen8_pte_encode(VM_PAGE_TO_PHYS(pages[i]),
 387                                         cache_level, true);
 388                 if (++pte == GEN8_PTES_PER_PAGE) {
 389                         if (!HAS_LLC(ppgtt->base.dev))
 390                                 drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
 391                         kunmap_atomic(pt_vaddr);
 392                         pt_vaddr = NULL;
 393                         if (++pde == GEN8_PDES_PER_PAGE) {
 394                                 pdpe++;
 395                                 pde = 0;
 396                         }
 397                         pte = 0;
 398                 }
 399         }
 400         if (pt_vaddr) {
 401                 if (!HAS_LLC(ppgtt->base.dev))
 402                         drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
 403                 kunmap_atomic(pt_vaddr);
 404         }
 405 }
 406
 407 static void gen8_free_page_tables(struct vm_page **pt_pages)
 408 {
 409         int i;
 410
 411         if (pt_pages == NULL)
 412                 return;
 413
 414         for (i = 0; i < GEN8_PDES_PER_PAGE; i++)
 415                 if (pt_pages[i])
 416                         __free_pages(pt_pages[i], 0);
 417 }
 418
 419 static void gen8_ppgtt_free(const struct i915_hw_ppgtt *ppgtt)
 420 {
 421         int i;
 422
 423         for (i = 0; i < ppgtt->num_pd_pages; i++) {
 424                 gen8_free_page_tables(ppgtt->gen8_pt_pages[i]);
 425                 kfree(ppgtt->gen8_pt_pages[i]);
 426                 kfree(ppgtt->gen8_pt_dma_addr[i]);
 427         }
 428
 429         __free_pages(ppgtt->pd_pages, get_order(ppgtt->num_pd_pages << PAGE_SHIFT));
 430 }
 431
 432 static void gen8_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)
 433 {
 434         struct pci_dev *hwdev = ppgtt->base.dev->pdev;
 435         int i, j;
 436
 437         for (i = 0; i < ppgtt->num_pd_pages; i++) {
 438                 /* TODO: In the future we'll support sparse mappings, so this
 439                  * will have to change. */
 440                 if (!ppgtt->pd_dma_addr[i])
 441                         continue;
 442
 443                 pci_unmap_page(hwdev, ppgtt->pd_dma_addr[i], PAGE_SIZE,
 444                                PCI_DMA_BIDIRECTIONAL);
 445
 446                 for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
 447                         dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j];
 448                         if (addr)
 449                                 pci_unmap_page(hwdev, addr, PAGE_SIZE,
 450                                                PCI_DMA_BIDIRECTIONAL);
 451                 }
 452         }
 453 }
 454
 455 static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
 456 {
 457         struct i915_hw_ppgtt *ppgtt =
 458                 container_of(vm, struct i915_hw_ppgtt, base);
 459
 460         gen8_ppgtt_unmap_pages(ppgtt);
 461         gen8_ppgtt_free(ppgtt);
 462 }
 463
 464 static struct vm_page **__gen8_alloc_page_tables(void)
 465 {
 466         struct vm_page **pt_pages;
 467         int i;
 468
 469         pt_pages = kcalloc(GEN8_PDES_PER_PAGE, sizeof(struct vm_page *), GFP_KERNEL);
 470         if (!pt_pages)
 471                 return ERR_PTR(-ENOMEM);
 472
 473         for (i = 0; i < GEN8_PDES_PER_PAGE; i++) {
 474                 pt_pages[i] = alloc_page(GFP_KERNEL);
 475                 if (!pt_pages[i])
 476                         goto bail;
 477         }
 478
 479         return pt_pages;
 480
 481 bail:
 482         gen8_free_page_tables(pt_pages);
 483         kfree(pt_pages);
 484         return ERR_PTR(-ENOMEM);
 485 }
 486
 487 static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt,
 488                                            const int max_pdp)
 489 {
 490         struct vm_page **pt_pages[GEN8_LEGACY_PDPS];
 491         int i, ret;
 492
 493         for (i = 0; i < max_pdp; i++) {
 494                 pt_pages[i] = __gen8_alloc_page_tables();
 495                 if (IS_ERR(pt_pages[i])) {
 496                         ret = PTR_ERR(pt_pages[i]);
 497                         goto unwind_out;
 498                 }
 499         }
 500
 501         /* NB: Avoid touching gen8_pt_pages until last to keep the allocation,
 502          * "atomic" - for cleanup purposes.
 503          */
 504         for (i = 0; i < max_pdp; i++)
 505                 ppgtt->gen8_pt_pages[i] = pt_pages[i];
 506
 507         return 0;
 508
 509 unwind_out:
 510         while (i--) {
 511                 gen8_free_page_tables(pt_pages[i]);
 512                 kfree(pt_pages[i]);
 513         }
 514
 515         return ret;
 516 }
 517
 518 static int gen8_ppgtt_allocate_dma(struct i915_hw_ppgtt *ppgtt)
 519 {
 520         int i;
 521
 522         for (i = 0; i < ppgtt->num_pd_pages; i++) {
 523                 ppgtt->gen8_pt_dma_addr[i] = kcalloc(GEN8_PDES_PER_PAGE,
 524                                                      sizeof(dma_addr_t),
 525                                                      GFP_KERNEL);
 526                 if (!ppgtt->gen8_pt_dma_addr[i])
 527                         return -ENOMEM;
 528         }
 529
 530         return 0;
 531 }
 532
 533 static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,
 534                                                 const int max_pdp)
 535 {
 536         ppgtt->pd_pages = alloc_pages(GFP_KERNEL, get_order(max_pdp << PAGE_SHIFT));
 537         if (!ppgtt->pd_pages)
 538                 return -ENOMEM;
 539
 540         ppgtt->num_pd_pages = 1 << get_order(max_pdp << PAGE_SHIFT);
 541         BUG_ON(ppgtt->num_pd_pages > GEN8_LEGACY_PDPS);
 542
 543         return 0;
 544 }
 545
 546 static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,
 547                             const int max_pdp)
 548 {
 549         int ret;
 550
 551         ret = gen8_ppgtt_allocate_page_directories(ppgtt, max_pdp);
 552         if (ret)
 553                 return ret;
 554
 555         ret = gen8_ppgtt_allocate_page_tables(ppgtt, max_pdp);
 556         if (ret) {
 557                 __free_pages(ppgtt->pd_pages, get_order(max_pdp << PAGE_SHIFT));
 558                 return ret;
 559         }
 560
 561         ppgtt->num_pd_entries = max_pdp * GEN8_PDES_PER_PAGE;
 562
 563         ret = gen8_ppgtt_allocate_dma(ppgtt);
 564         if (ret)
 565                 gen8_ppgtt_free(ppgtt);
 566
 567         return ret;
 568 }
 569
 570 static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,
 571                                              const int pd)
 572 {
 573         dma_addr_t pd_addr;
 574         int ret;
 575
 576         pd_addr = pci_map_page(ppgtt->base.dev->pdev,
 577                                &ppgtt->pd_pages[pd], 0,
 578                                PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 579
 580         ret = pci_dma_mapping_error(ppgtt->base.dev->pdev, pd_addr);
 581         if (ret)
 582                 return ret;
 583
 584         ppgtt->pd_dma_addr[pd] = pd_addr;
 585
 586         return 0;
 587 }
 588
 589 static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt,
 590                                         const int pd,
 591                                         const int pt)
 592 {
 593         dma_addr_t pt_addr;
 594         struct vm_page *p;
 595         int ret;
 596
 597         p = ppgtt->gen8_pt_pages[pd][pt];
 598         pt_addr = pci_map_page(ppgtt->base.dev->pdev,
 599                                p, 0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 600         ret = pci_dma_mapping_error(ppgtt->base.dev->pdev, pt_addr);
 601         if (ret)
 602                 return ret;
 603
 604         ppgtt->gen8_pt_dma_addr[pd][pt] = pt_addr;
 605
 606         return 0;
 607 }
 608
 609 /**
 610  * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers
 611  * with a net effect resembling a 2-level page table in normal x86 terms. Each
 612  * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address
 613  * space.
 614  *
 615  * FIXME: split allocation into smaller pieces. For now we only ever do this
 616  * once, but with full PPGTT, the multiple contiguous allocations will be bad.
 617  * TODO: Do something with the size parameter
 618  */
 619 static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)
 620 {
 621         const int max_pdp = DIV_ROUND_UP(size, 1 << 30);
 622         const int min_pt_pages = GEN8_PDES_PER_PAGE * max_pdp;
 623         int i, j, ret;
 624
 625         if (size % (1<<30))
 626                 DRM_INFO("Pages will be wasted unless GTT size (%lu) is divisible by 1GB\n", size);
 627
 628         /* 1. Do all our allocations for page directories and page tables. */
 629         ret = gen8_ppgtt_alloc(ppgtt, max_pdp);
 630         if (ret)
 631                 return ret;
 632
 633         /*
 634          * 2. Create DMA mappings for the page directories and page tables.
 635          */
 636         for (i = 0; i < max_pdp; i++) {
 637                 ret = gen8_ppgtt_setup_page_directories(ppgtt, i);
 638                 if (ret)
 639                         goto bail;
 640
 641                 for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
 642                         ret = gen8_ppgtt_setup_page_tables(ppgtt, i, j);
 643                         if (ret)
 644                                 goto bail;
 645                 }
 646         }
 647
 648         /*
 649          * 3. Map all the page directory entires to point to the page tables
 650          * we've allocated.
 651          *
 652          * For now, the PPGTT helper functions all require that the PDEs are
 653          * plugged in correctly. So we do that now/here. For aliasing PPGTT, we
 654          * will never need to touch the PDEs again.
 655          */
 656         for (i = 0; i < max_pdp; i++) {
 657                 gen8_ppgtt_pde_t *pd_vaddr;
 658                 pd_vaddr = kmap_atomic(&ppgtt->pd_pages[i]);
 659                 for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
 660                         dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j];
 661                         pd_vaddr[j] = gen8_pde_encode(ppgtt->base.dev, addr,
 662                                                       I915_CACHE_LLC);
 663                 }
 664                 if (!HAS_LLC(ppgtt->base.dev))
 665                         drm_clflush_virt_range(pd_vaddr, PAGE_SIZE);
 666                 kunmap_atomic(pd_vaddr);
 667         }
 668
 669         ppgtt->switch_mm = gen8_mm_switch;
 670         ppgtt->base.clear_range = gen8_ppgtt_clear_range;
 671         ppgtt->base.insert_entries = gen8_ppgtt_insert_entries;
 672         ppgtt->base.cleanup = gen8_ppgtt_cleanup;
 673         ppgtt->base.start = 0;
 674         ppgtt->base.total = ppgtt->num_pd_entries * GEN8_PTES_PER_PAGE * PAGE_SIZE;
 675
 676         ppgtt->base.clear_range(&ppgtt->base, 0, ppgtt->base.total, true);
 677
 678         DRM_DEBUG_DRIVER("Allocated %d pages for page directories (%d wasted)\n",
 679                          ppgtt->num_pd_pages, ppgtt->num_pd_pages - max_pdp);
 680         DRM_DEBUG_DRIVER("Allocated %d pages for page tables (%ld wasted)\n",
 681                          ppgtt->num_pd_entries,
 682                          (ppgtt->num_pd_entries - min_pt_pages) + size % (1<<30));
 683         return 0;
 684
 685 bail:
 686         gen8_ppgtt_unmap_pages(ppgtt);
 687         gen8_ppgtt_free(ppgtt);
 688         return ret;
 689 }
 690
 691 static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
 692 {
 693         struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private;
 694         struct i915_address_space *vm = &ppgtt->base;
 695         gen6_gtt_pte_t __iomem *pd_addr;
 696         gen6_gtt_pte_t scratch_pte;
 697         uint32_t pd_entry;
 698         int pte, pde;
 699
 700         scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, true, 0);
 701
 702         pd_addr = (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm +
 703                 ppgtt->pd_offset / sizeof(gen6_gtt_pte_t);
 704
 705         seq_printf(m, "  VM %p (pd_offset %x-%x):\n", vm,
 706                    ppgtt->pd_offset, ppgtt->pd_offset + ppgtt->num_pd_entries);
 707         for (pde = 0; pde < ppgtt->num_pd_entries; pde++) {
 708                 u32 expected;
 709                 gen6_gtt_pte_t *pt_vaddr;
 710                 dma_addr_t pt_addr = ppgtt->pt_dma_addr[pde];
 711                 pd_entry = readl(pd_addr + pde);
 712                 expected = (GEN6_PDE_ADDR_ENCODE(pt_addr) | GEN6_PDE_VALID);
 713
 714                 if (pd_entry != expected)
 715                         seq_printf(m, "\tPDE #%d mismatch: Actual PDE: %x Expected PDE: %x\n",
 716                                    pde,
 717                                    pd_entry,
 718                                    expected);
 719                 seq_printf(m, "\tPDE: %x\n", pd_entry);
 720
 721                 pt_vaddr = kmap_atomic(ppgtt->pt_pages[pde]);
 722                 for (pte = 0; pte < I915_PPGTT_PT_ENTRIES; pte+=4) {
 723                         unsigned long va =
 724                                 (pde * PAGE_SIZE * I915_PPGTT_PT_ENTRIES) +
 725                                 (pte * PAGE_SIZE);
 726                         int i;
 727                         bool found = false;
 728                         for (i = 0; i < 4; i++)
 729                                 if (pt_vaddr[pte + i] != scratch_pte)
 730                                         found = true;
 731                         if (!found)
 732                                 continue;
 733
 734                         seq_printf(m, "\t\t0x%lx [%03d,%04d]: =", va, pde, pte);
 735                         for (i = 0; i < 4; i++) {
 736                                 if (pt_vaddr[pte + i] != scratch_pte)
 737                                         seq_printf(m, " %08x", pt_vaddr[pte + i]);
 738                                 else
 739                                         seq_printf(m, "  SCRATCH ");
 740                         }
 741                         seq_printf(m, "\n");
 742                 }
 743                 kunmap_atomic(pt_vaddr);
 744         }
 745 }
 746
 747 static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt)
 748 {
 749         struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private;
 750         gen6_gtt_pte_t __iomem *pd_addr;
 751         uint32_t pd_entry;
 752         int i;
 753
 754         WARN_ON(ppgtt->pd_offset & 0x3f);
 755         pd_addr = (gen6_gtt_pte_t __iomem*)dev_priv->gtt.gsm +
 756                 ppgtt->pd_offset / sizeof(gen6_gtt_pte_t);
 757         for (i = 0; i < ppgtt->num_pd_entries; i++) {
 758                 dma_addr_t pt_addr;
 759
 760                 pt_addr = ppgtt->pt_dma_addr[i];
 761                 pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);
 762                 pd_entry |= GEN6_PDE_VALID;
 763
 764                 writel(pd_entry, pd_addr + i);
 765         }
 766         readl(pd_addr);
 767 }
 768
 769 static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt)
 770 {
 771         BUG_ON(ppgtt->pd_offset & 0x3f);
 772
 773         return (ppgtt->pd_offset / 64) << 16;
 774 }
 775
 776 static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,
 777                          struct intel_engine_cs *ring)
 778 {
 779         int ret;
 780
 781         /* NB: TLBs must be flushed and invalidated before a switch */
 782         ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
 783         if (ret)
 784                 return ret;
 785
 786         ret = intel_ring_begin(ring, 6);
 787         if (ret)
 788                 return ret;
 789
 790         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
 791         intel_ring_emit(ring, RING_PP_DIR_DCLV(ring));
 792         intel_ring_emit(ring, PP_DIR_DCLV_2G);
 793         intel_ring_emit(ring, RING_PP_DIR_BASE(ring));
 794         intel_ring_emit(ring, get_pd_offset(ppgtt));
 795         intel_ring_emit(ring, MI_NOOP);
 796         intel_ring_advance(ring);
 797
 798         return 0;
 799 }
 800
 801 static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt,
 802                           struct intel_engine_cs *ring)
 803 {
 804         int ret;
 805
 806         /* NB: TLBs must be flushed and invalidated before a switch */
 807         ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
 808         if (ret)
 809                 return ret;
 810
 811         ret = intel_ring_begin(ring, 6);
 812         if (ret)
 813                 return ret;
 814
 815         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
 816         intel_ring_emit(ring, RING_PP_DIR_DCLV(ring));
 817         intel_ring_emit(ring, PP_DIR_DCLV_2G);
 818         intel_ring_emit(ring, RING_PP_DIR_BASE(ring));
 819         intel_ring_emit(ring, get_pd_offset(ppgtt));
 820         intel_ring_emit(ring, MI_NOOP);
 821         intel_ring_advance(ring);
 822
 823         /* XXX: RCS is the only one to auto invalidate the TLBs? */
 824         if (ring->id != RCS) {
 825                 ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
 826                 if (ret)
 827                         return ret;
 828         }
 829
 830         return 0;
 831 }
 832
 833 static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt,
 834                           struct intel_engine_cs *ring)
 835 {
 836         struct drm_device *dev = ppgtt->base.dev;
 837         struct drm_i915_private *dev_priv = dev->dev_private;
 838
 839
 840         I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
 841         I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
 842
 843         POSTING_READ(RING_PP_DIR_DCLV(ring));
 844
 845         return 0;
 846 }
 847
 848 static void gen8_ppgtt_enable(struct drm_device *dev)
 849 {
 850         struct drm_i915_private *dev_priv = dev->dev_private;
 851         struct intel_engine_cs *ring;
 852         int j;
 853
 854         for_each_ring(ring, dev_priv, j) {
 855                 I915_WRITE(RING_MODE_GEN7(ring),
 856                            _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
 857         }
 858 }
 859
 860 static void gen7_ppgtt_enable(struct drm_device *dev)
 861 {
 862         struct drm_i915_private *dev_priv = dev->dev_private;
 863         struct intel_engine_cs *ring;
 864         uint32_t ecochk, ecobits;
 865         int i;
 866
 867         ecobits = I915_READ(GAC_ECO_BITS);
 868         I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
 869
 870         ecochk = I915_READ(GAM_ECOCHK);
 871         if (IS_HASWELL(dev)) {
 872                 ecochk |= ECOCHK_PPGTT_WB_HSW;
 873         } else {
 874                 ecochk |= ECOCHK_PPGTT_LLC_IVB;
 875                 ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
 876         }
 877         I915_WRITE(GAM_ECOCHK, ecochk);
 878
 879         for_each_ring(ring, dev_priv, i) {
 880                 /* GFX_MODE is per-ring on gen7+ */
 881                 I915_WRITE(RING_MODE_GEN7(ring),
 882                            _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
 883         }
 884 }
 885
 886 static void gen6_ppgtt_enable(struct drm_device *dev)
 887 {
 888         struct drm_i915_private *dev_priv = dev->dev_private;
 889         uint32_t ecochk, gab_ctl, ecobits;
 890
 891         ecobits = I915_READ(GAC_ECO_BITS);
 892         I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
 893                    ECOBITS_PPGTT_CACHE64B);
 894
 895         gab_ctl = I915_READ(GAB_CTL);
 896         I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
 897
 898         ecochk = I915_READ(GAM_ECOCHK);
 899         I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B);
 900
 901         I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
 902 }
 903
 904 /* PPGTT support for Sandybdrige/Gen6 and later */
 905 static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
 906                                    uint64_t start,
 907                                    uint64_t length,
 908                                    bool use_scratch)
 909 {
 910         struct i915_hw_ppgtt *ppgtt =
 911                 container_of(vm, struct i915_hw_ppgtt, base);
 912         gen6_gtt_pte_t *pt_vaddr, scratch_pte;
 913         unsigned first_entry = start >> PAGE_SHIFT;
 914         unsigned num_entries = length >> PAGE_SHIFT;
 915         unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
 916         unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
 917         unsigned last_pte, i;
 918
 919         scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, true, 0);
 920
 921         while (num_entries) {
 922                 last_pte = first_pte + num_entries;
 923                 if (last_pte > I915_PPGTT_PT_ENTRIES)
 924                         last_pte = I915_PPGTT_PT_ENTRIES;
 925
 926                 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pt]);
 927
 928                 for (i = first_pte; i < last_pte; i++)
 929                         pt_vaddr[i] = scratch_pte;
 930
 931                 kunmap_atomic(pt_vaddr);
 932
 933                 num_entries -= last_pte - first_pte;
 934                 first_pte = 0;
 935                 act_pt++;
 936         }
 937 }
 938
 939 static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
 940                                       vm_page_t *pages,
 941                                       uint64_t start,
 942                                       unsigned num_entries,
 943                                       enum i915_cache_level cache_level, u32 flags)
 944 {
 945         struct i915_hw_ppgtt *ppgtt =
 946                 container_of(vm, struct i915_hw_ppgtt, base);
 947         gen6_gtt_pte_t *pt_vaddr;
 948         unsigned first_entry = start >> PAGE_SHIFT;
 949         unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
 950         unsigned act_pte = first_entry % I915_PPGTT_PT_ENTRIES;
 951
 952         pt_vaddr = NULL;
 953         for (int i=0;i<num_entries;i++) {
 954                 if (pt_vaddr == NULL)
 955                         pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pt]);
 956
 957                 pt_vaddr[act_pte] =
 958                         vm->pte_encode(VM_PAGE_TO_PHYS(pages[i]),
 959                                        cache_level, true, flags);
 960                 if (++act_pte == I915_PPGTT_PT_ENTRIES) {
 961                         kunmap_atomic(pt_vaddr);
 962                         pt_vaddr = NULL;
 963                         act_pt++;
 964                         act_pte = 0;
 965                 }
 966         }
 967         if (pt_vaddr)
 968                 kunmap_atomic(pt_vaddr);
 969 }
 970
 971 static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)
 972 {
 973         int i;
 974
 975         if (ppgtt->pt_dma_addr) {
 976                 for (i = 0; i < ppgtt->num_pd_entries; i++)
 977                         pci_unmap_page(ppgtt->base.dev->pdev,
 978                                        ppgtt->pt_dma_addr[i],
 979                                        4096, PCI_DMA_BIDIRECTIONAL);
 980         }
 981 }
 982
 983 static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)
 984 {
 985         int i;
 986
 987         kfree(ppgtt->pt_dma_addr);
 988         for (i = 0; i < ppgtt->num_pd_entries; i++)
 989                 __free_page(ppgtt->pt_pages[i]);
 990         kfree(ppgtt->pt_pages);
 991 }
 992
 993 static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
 994 {
 995         struct i915_hw_ppgtt *ppgtt =
 996                 container_of(vm, struct i915_hw_ppgtt, base);
 997
 998         drm_mm_remove_node(&ppgtt->node);
 999
1000         gen6_ppgtt_unmap_pages(ppgtt);
1001         gen6_ppgtt_free(ppgtt);
1002 }
1003
1004 static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt)
1005 {
1006         struct drm_device *dev = ppgtt->base.dev;
1007         struct drm_i915_private *dev_priv = dev->dev_private;
1008         bool retried = false;
1009         int ret;
1010
1011         /* PPGTT PDEs reside in the GGTT and consists of 512 entries. The
1012          * allocator works in address space sizes, so it's multiplied by page
1013          * size. We allocate at the top of the GTT to avoid fragmentation.
1014          */
1015         BUG_ON(!drm_mm_initialized(&dev_priv->gtt.base.mm));
1016 alloc:
1017         ret = drm_mm_insert_node_in_range_generic(&dev_priv->gtt.base.mm,
1018                                                   &ppgtt->node, GEN6_PD_SIZE,
1019                                                   GEN6_PD_ALIGN, 0,
1020                                                   0, dev_priv->gtt.base.total,
1021                                                   DRM_MM_TOPDOWN);
1022         if (ret == -ENOSPC && !retried) {
1023                 ret = i915_gem_evict_something(dev, &dev_priv->gtt.base,
1024                                                GEN6_PD_SIZE, GEN6_PD_ALIGN,
1025                                                I915_CACHE_NONE,
1026                                                0, dev_priv->gtt.base.total,
1027                                                0);
1028                 if (ret)
1029                         return ret;
1030
1031                 retried = true;
1032                 goto alloc;
1033         }
1034
1035         if (ppgtt->node.start < dev_priv->gtt.mappable_end)
1036                 DRM_DEBUG("Forced to use aperture for PDEs\n");
1037
1038         ppgtt->num_pd_entries = GEN6_PPGTT_PD_ENTRIES;
1039         return ret;
1040 }
1041
1042 static int gen6_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt)
1043 {
1044         int i;
1045
1046         ppgtt->pt_pages = kcalloc(ppgtt->num_pd_entries, sizeof(struct vm_page *),
1047                                   GFP_KERNEL);
1048
1049         if (!ppgtt->pt_pages)
1050                 return -ENOMEM;
1051
1052         for (i = 0; i < ppgtt->num_pd_entries; i++) {
1053                 ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL);
1054                 if (!ppgtt->pt_pages[i]) {
1055                         gen6_ppgtt_free(ppgtt);
1056                         return -ENOMEM;
1057                 }
1058         }
1059
1060         return 0;
1061 }
1062
1063 static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)
1064 {
1065         int ret;
1066
1067         ret = gen6_ppgtt_allocate_page_directories(ppgtt);
1068         if (ret)
1069                 return ret;
1070
1071         ret = gen6_ppgtt_allocate_page_tables(ppgtt);
1072         if (ret) {
1073                 drm_mm_remove_node(&ppgtt->node);
1074                 return ret;
1075         }
1076
1077         ppgtt->pt_dma_addr = kcalloc(ppgtt->num_pd_entries, sizeof(dma_addr_t),
1078                                      GFP_KERNEL);
1079         if (!ppgtt->pt_dma_addr) {
1080                 drm_mm_remove_node(&ppgtt->node);
1081                 gen6_ppgtt_free(ppgtt);
1082                 return -ENOMEM;
1083         }
1084
1085         return 0;
1086 }
1087
1088 static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)
1089 {
1090         struct drm_device *dev = ppgtt->base.dev;
1091         int i;
1092
1093         for (i = 0; i < ppgtt->num_pd_entries; i++) {
1094                 dma_addr_t pt_addr;
1095
1096                 pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], 0, 4096,
1097                                        PCI_DMA_BIDIRECTIONAL);
1098
1099                 if (pci_dma_mapping_error(dev->pdev, pt_addr)) {
1100                         gen6_ppgtt_unmap_pages(ppgtt);
1101                         return -EIO;
1102                 }
1103
1104                 ppgtt->pt_dma_addr[i] = pt_addr;
1105         }
1106
1107         return 0;
1108 }
1109
1110 static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
1111 {
1112         struct drm_device *dev = ppgtt->base.dev;
1113         struct drm_i915_private *dev_priv = dev->dev_private;
1114         int ret;
1115
1116         ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode;
1117         if (IS_GEN6(dev)) {
1118                 ppgtt->switch_mm = gen6_mm_switch;
1119         } else if (IS_HASWELL(dev)) {
1120                 ppgtt->switch_mm = hsw_mm_switch;
1121         } else if (IS_GEN7(dev)) {
1122                 ppgtt->switch_mm = gen7_mm_switch;
1123         } else
1124                 BUG();
1125
1126         ret = gen6_ppgtt_alloc(ppgtt);
1127         if (ret)
1128                 return ret;
1129
1130         ret = gen6_ppgtt_setup_page_tables(ppgtt);
1131         if (ret) {
1132                 gen6_ppgtt_free(ppgtt);
1133                 return ret;
1134         }
1135
1136         ppgtt->base.clear_range = gen6_ppgtt_clear_range;
1137         ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
1138         ppgtt->base.cleanup = gen6_ppgtt_cleanup;
1139         ppgtt->base.start = 0;
1140         ppgtt->base.total =  ppgtt->num_pd_entries * I915_PPGTT_PT_ENTRIES * PAGE_SIZE;
1141         ppgtt->debug_dump = gen6_dump_ppgtt;
1142
1143         ppgtt->pd_offset =
1144                 ppgtt->node.start / PAGE_SIZE * sizeof(gen6_gtt_pte_t);
1145
1146         ppgtt->base.clear_range(&ppgtt->base, 0, ppgtt->base.total, true);
1147
1148         DRM_DEBUG_DRIVER("Allocated pde space (%ldM) at GTT entry: %lx\n",
1149                          ppgtt->node.size >> 20,
1150                          ppgtt->node.start / PAGE_SIZE);
1151
1152         gen6_write_pdes(ppgtt);
1153         DRM_DEBUG("Adding PPGTT at offset %x\n",
1154                   ppgtt->pd_offset << 10);
1155
1156         return 0;
1157 }
1158
1159 static int __hw_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
1160 {
1161         struct drm_i915_private *dev_priv = dev->dev_private;
1162
1163         ppgtt->base.dev = dev;
1164         ppgtt->base.scratch = dev_priv->gtt.base.scratch;
1165
1166         if (INTEL_INFO(dev)->gen < 8)
1167                 return gen6_ppgtt_init(ppgtt);
1168         else
1169                 return gen8_ppgtt_init(ppgtt, dev_priv->gtt.base.total);
1170 }
1171 int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
1172 {
1173         struct drm_i915_private *dev_priv = dev->dev_private;
1174         int ret = 0;
1175
1176         ret = __hw_ppgtt_init(dev, ppgtt);
1177         if (ret == 0) {
1178                 kref_init(&ppgtt->ref);
1179                 drm_mm_init(&ppgtt->base.mm, ppgtt->base.start,
1180                             ppgtt->base.total);
1181                 i915_init_vm(dev_priv, &ppgtt->base);
1182         }
1183
1184         return ret;
1185 }
1186
1187 int i915_ppgtt_init_hw(struct drm_device *dev)
1188 {
1189         struct drm_i915_private *dev_priv = dev->dev_private;
1190         struct intel_engine_cs *ring;
1191         struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
1192         int i, ret = 0;
1193
1194         /* In the case of execlists, PPGTT is enabled by the context descriptor
1195          * and the PDPs are contained within the context itself.  We don't
1196          * need to do anything here. */
1197         if (i915.enable_execlists)
1198                 return 0;
1199
1200         if (!USES_PPGTT(dev))
1201                 return 0;
1202
1203         if (IS_GEN6(dev))
1204                 gen6_ppgtt_enable(dev);
1205         else if (IS_GEN7(dev))
1206                 gen7_ppgtt_enable(dev);
1207         else if (INTEL_INFO(dev)->gen >= 8)
1208                 gen8_ppgtt_enable(dev);
1209         else
1210                 MISSING_CASE(INTEL_INFO(dev)->gen);
1211
1212         if (ppgtt) {
1213                 for_each_ring(ring, dev_priv, i) {
1214                         ret = ppgtt->switch_mm(ppgtt, ring);
1215                         if (ret != 0)
1216                                 return ret;
1217                 }
1218         }
1219
1220         return ret;
1221 }
1222 struct i915_hw_ppgtt *
1223 i915_ppgtt_create(struct drm_device *dev, struct drm_i915_file_private *fpriv)
1224 {
1225         struct i915_hw_ppgtt *ppgtt;
1226         int ret;
1227
1228         ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
1229         if (!ppgtt)
1230                 return ERR_PTR(-ENOMEM);
1231
1232         ret = i915_ppgtt_init(dev, ppgtt);
1233         if (ret) {
1234                 kfree(ppgtt);
1235                 return ERR_PTR(ret);
1236         }
1237
1238         ppgtt->file_priv = fpriv;
1239
1240         trace_i915_ppgtt_create(&ppgtt->base);
1241
1242         return ppgtt;
1243 }
1244
1245 void  i915_ppgtt_release(struct kref *kref)
1246 {
1247         struct i915_hw_ppgtt *ppgtt =
1248                 container_of(kref, struct i915_hw_ppgtt, ref);
1249
1250         trace_i915_ppgtt_release(&ppgtt->base);
1251
1252         /* vmas should already be unbound */
1253         WARN_ON(!list_empty(&ppgtt->base.active_list));
1254         WARN_ON(!list_empty(&ppgtt->base.inactive_list));
1255
1256         list_del(&ppgtt->base.global_link);
1257         drm_mm_takedown(&ppgtt->base.mm);
1258
1259         ppgtt->base.cleanup(&ppgtt->base);
1260         kfree(ppgtt);
1261 }
1262
1263 static void
1264 ppgtt_bind_vma(struct i915_vma *vma,
1265                enum i915_cache_level cache_level,
1266                u32 flags)
1267 {
1268         const unsigned int num_entries = vma->obj->base.size >> PAGE_SHIFT;
1269
1270         /* Currently applicable only to VLV */
1271         if (vma->obj->gt_ro)
1272                 flags |= PTE_READ_ONLY;
1273
1274         vma->vm->insert_entries(vma->vm, vma->obj->pages, vma->node.start,
1275                                 num_entries,
1276                                 cache_level, flags);
1277 }
1278
1279 static void ppgtt_unbind_vma(struct i915_vma *vma)
1280 {
1281         vma->vm->clear_range(vma->vm,
1282                              vma->node.start,
1283                              vma->obj->base.size,
1284                              true);
1285 }
1286
1287 extern int intel_iommu_gfx_mapped;
1288 /* Certain Gen5 chipsets require require idling the GPU before
1289  * unmapping anything from the GTT when VT-d is enabled.
1290  */
1291 static inline bool needs_idle_maps(struct drm_device *dev)
1292 {
1293 #ifdef CONFIG_INTEL_IOMMU
1294         /* Query intel_iommu to see if we need the workaround. Presumably that
1295          * was loaded first.
1296          */
1297         if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped)
1298                 return true;
1299 #endif
1300         return false;
1301 }
1302
1303 static bool do_idling(struct drm_i915_private *dev_priv)
1304 {
1305         bool ret = dev_priv->mm.interruptible;
1306
1307         if (unlikely(dev_priv->gtt.do_idle_maps)) {
1308                 dev_priv->mm.interruptible = false;
1309                 if (i915_gpu_idle(dev_priv->dev)) {
1310                         DRM_ERROR("Couldn't idle GPU\n");
1311                         /* Wait a bit, in hopes it avoids the hang */
1312                         udelay(10);
1313                 }
1314         }
1315
1316         return ret;
1317 }
1318
1319 static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
1320 {
1321         if (unlikely(dev_priv->gtt.do_idle_maps))
1322                 dev_priv->mm.interruptible = interruptible;
1323 }
1324
1325 void i915_check_and_clear_faults(struct drm_device *dev)
1326 {
1327         struct drm_i915_private *dev_priv = dev->dev_private;
1328         struct intel_engine_cs *ring;
1329         int i;
1330
1331         if (INTEL_INFO(dev)->gen < 6)
1332                 return;
1333
1334         for_each_ring(ring, dev_priv, i) {
1335                 u32 fault_reg;
1336                 fault_reg = I915_READ(RING_FAULT_REG(ring));
1337                 if (fault_reg & RING_FAULT_VALID) {
1338 #if 0
1339                         DRM_DEBUG_DRIVER("Unexpected fault\n"
1340                                          "\tAddr: 0x%08lx\n"
1341                                          "\tAddress space: %s\n"
1342                                          "\tSource ID: %d\n"
1343                                          "\tType: %d\n",
1344                                          fault_reg & PAGE_MASK,
1345                                          fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT",
1346                                          RING_FAULT_SRCID(fault_reg),
1347                                          RING_FAULT_FAULT_TYPE(fault_reg));
1348 #endif
1349                         I915_WRITE(RING_FAULT_REG(ring),
1350                                    fault_reg & ~RING_FAULT_VALID);
1351                 }
1352         }
1353         POSTING_READ(RING_FAULT_REG(&dev_priv->ring[RCS]));
1354 }
1355
1356 static void i915_ggtt_flush(struct drm_i915_private *dev_priv)
1357 {
1358         if (INTEL_INFO(dev_priv->dev)->gen < 6) {
1359                 intel_gtt_chipset_flush();
1360         } else {
1361                 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
1362                 POSTING_READ(GFX_FLSH_CNTL_GEN6);
1363         }
1364 }
1365
1366 void i915_gem_suspend_gtt_mappings(struct drm_device *dev)
1367 {
1368         struct drm_i915_private *dev_priv = dev->dev_private;
1369
1370         /* Don't bother messing with faults pre GEN6 as we have little
1371          * documentation supporting that it's a good idea.
1372          */
1373         if (INTEL_INFO(dev)->gen < 6)
1374                 return;
1375
1376         i915_check_and_clear_faults(dev);
1377
1378         dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
1379                                        dev_priv->gtt.base.start,
1380                                        dev_priv->gtt.base.total,
1381                                        true);
1382
1383         i915_ggtt_flush(dev_priv);
1384 }
1385
1386 void i915_gem_restore_gtt_mappings(struct drm_device *dev)
1387 {
1388         struct drm_i915_private *dev_priv = dev->dev_private;
1389         struct drm_i915_gem_object *obj;
1390         struct i915_address_space *vm;
1391
1392         i915_check_and_clear_faults(dev);
1393
1394         /* First fill our portion of the GTT with scratch pages */
1395         dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
1396                                        dev_priv->gtt.base.start,
1397                                        dev_priv->gtt.base.total,
1398                                        true);
1399
1400         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
1401                 struct i915_vma *vma = i915_gem_obj_to_vma(obj,
1402                                                            &dev_priv->gtt.base);
1403                 if (!vma)
1404                         continue;
1405
1406                 i915_gem_clflush_object(obj, obj->pin_display);
1407                 /* The bind_vma code tries to be smart about tracking mappings.
1408                  * Unfortunately above, we've just wiped out the mappings
1409                  * without telling our object about it. So we need to fake it.
1410                  *
1411                  * Bind is not expected to fail since this is only called on
1412                  * resume and assumption is all requirements exist already.
1413                  */
1414                 vma->bound &= ~GLOBAL_BIND;
1415                 WARN_ON(i915_vma_bind(vma, obj->cache_level, GLOBAL_BIND));
1416         }
1417
1418
1419         if (INTEL_INFO(dev)->gen >= 8) {
1420                 if (IS_CHERRYVIEW(dev))
1421                         chv_setup_private_ppat(dev_priv);
1422                 else
1423                         bdw_setup_private_ppat(dev_priv);
1424
1425                 return;
1426         }
1427
1428         list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
1429                 /* TODO: Perhaps it shouldn't be gen6 specific */
1430                 if (i915_is_ggtt(vm)) {
1431                         if (dev_priv->mm.aliasing_ppgtt)
1432                                 gen6_write_pdes(dev_priv->mm.aliasing_ppgtt);
1433                         continue;
1434                 }
1435
1436                 gen6_write_pdes(container_of(vm, struct i915_hw_ppgtt, base));
1437         }
1438
1439         i915_ggtt_flush(dev_priv);
1440 }
1441
1442 int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
1443 {
1444         if (obj->has_dma_mapping)
1445                 return 0;
1446
1447 #if 0
1448         if (!dma_map_sg(&obj->base.dev->pdev->dev,
1449                         obj->pages->sgl, obj->pages->nents,
1450                         PCI_DMA_BIDIRECTIONAL))
1451                 return -ENOSPC;
1452 #endif
1453
1454         return 0;
1455 }
1456
1457 static inline void gen8_set_pte(void __iomem *addr, gen8_gtt_pte_t pte)
1458 {
1459 #if 0
1460         writeq(pte, addr);
1461 #else
1462         iowrite32((u32)pte, addr);
1463         iowrite32(pte >> 32, addr + 4);
1464 #endif
1465 }
1466
1467 static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
1468                                      vm_page_t *pages,
1469                                      uint64_t start,
1470                                      unsigned int num_entries,
1471                                      enum i915_cache_level level, u32 unused)
1472 {
1473         struct drm_i915_private *dev_priv = vm->dev->dev_private;
1474         unsigned first_entry = start >> PAGE_SHIFT;
1475         gen8_gtt_pte_t __iomem *gtt_entries =
1476                 (gen8_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
1477         int i = 0;
1478         dma_addr_t addr = 0;
1479
1480         for (i=0;i<num_entries;i++) {
1481                 addr = VM_PAGE_TO_PHYS(pages[i]);
1482                 gen8_set_pte(&gtt_entries[i],
1483                              gen8_pte_encode(addr, level, true));
1484         }
1485
1486         /*
1487          * XXX: This serves as a posting read to make sure that the PTE has
1488          * actually been updated. There is some concern that even though
1489          * registers and PTEs are within the same BAR that they are potentially
1490          * of NUMA access patterns. Therefore, even with the way we assume
1491          * hardware should work, we must keep this posting read for paranoia.
1492          */
1493         if (i != 0)
1494                 WARN_ON(readq(&gtt_entries[i-1])
1495                         != gen8_pte_encode(addr, level, true));
1496
1497         /* This next bit makes the above posting read even more important. We
1498          * want to flush the TLBs only after we're certain all the PTE updates
1499          * have finished.
1500          */
1501         I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
1502         POSTING_READ(GFX_FLSH_CNTL_GEN6);
1503 }
1504
1505 /*
1506  * Binds an object into the global gtt with the specified cache level. The object
1507  * will be accessible to the GPU via commands whose operands reference offsets
1508  * within the global GTT as well as accessible by the GPU through the GMADR
1509  * mapped BAR (dev_priv->mm.gtt->gtt).
1510  */
1511 static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
1512                                      vm_page_t *pages,
1513                                      uint64_t start,
1514                                      unsigned int num_entries,
1515                                      enum i915_cache_level level, u32 flags)
1516 {
1517         struct drm_i915_private *dev_priv = vm->dev->dev_private;
1518         unsigned first_entry = start >> PAGE_SHIFT;
1519         gen6_gtt_pte_t __iomem *gtt_entries =
1520                 (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
1521         int i = 0;
1522         dma_addr_t addr = 0; /* shut up gcc */
1523
1524         for (i = 0; i < num_entries; i++) {
1525                 addr = VM_PAGE_TO_PHYS(pages[i]);
1526                 iowrite32(vm->pte_encode(addr, level, true, flags), &gtt_entries[i]);
1527         }
1528
1529         /* XXX: This serves as a posting read to make sure that the PTE has
1530          * actually been updated. There is some concern that even though
1531          * registers and PTEs are within the same BAR that they are potentially
1532          * of NUMA access patterns. Therefore, even with the way we assume
1533          * hardware should work, we must keep this posting read for paranoia.
1534          */
1535         if (i != 0) {
1536                 unsigned long gtt = readl(&gtt_entries[i-1]);
1537                 WARN_ON(gtt != vm->pte_encode(addr, level, true, flags));
1538         }
1539
1540         /* This next bit makes the above posting read even more important. We
1541          * want to flush the TLBs only after we're certain all the PTE updates
1542          * have finished.
1543          */
1544         I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
1545         POSTING_READ(GFX_FLSH_CNTL_GEN6);
1546 }
1547
1548 static void gen8_ggtt_clear_range(struct i915_address_space *vm,
1549                                   uint64_t start,
1550                                   uint64_t length,
1551                                   bool use_scratch)
1552 {
1553         struct drm_i915_private *dev_priv = vm->dev->dev_private;
1554         unsigned first_entry = start >> PAGE_SHIFT;
1555         unsigned num_entries = length >> PAGE_SHIFT;
1556         gen8_gtt_pte_t scratch_pte, __iomem *gtt_base =
1557                 (gen8_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
1558         const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
1559         int i;
1560
1561         if (WARN(num_entries > max_entries,
1562                  "First entry = %d; Num entries = %d (max=%d)\n",
1563                  first_entry, num_entries, max_entries))
1564                 num_entries = max_entries;
1565
1566         scratch_pte = gen8_pte_encode(vm->scratch.addr,
1567                                       I915_CACHE_LLC,
1568                                       use_scratch);
1569         for (i = 0; i < num_entries; i++)
1570                 gen8_set_pte(&gtt_base[i], scratch_pte);
1571         readl(gtt_base);
1572 }
1573
1574 static void gen6_ggtt_clear_range(struct i915_address_space *vm,
1575                                   uint64_t start,
1576                                   uint64_t length,
1577                                   bool use_scratch)
1578 {
1579         struct drm_i915_private *dev_priv = vm->dev->dev_private;
1580         unsigned first_entry = start >> PAGE_SHIFT;
1581         unsigned num_entries = length >> PAGE_SHIFT;
1582         gen6_gtt_pte_t scratch_pte, __iomem *gtt_base =
1583                 (gen6_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
1584         const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
1585         int i;
1586
1587         if (WARN(num_entries > max_entries,
1588                  "First entry = %d; Num entries = %d (max=%d)\n",
1589                  first_entry, num_entries, max_entries))
1590                 num_entries = max_entries;
1591
1592         scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, use_scratch, 0);
1593
1594         for (i = 0; i < num_entries; i++)
1595                 iowrite32(scratch_pte, &gtt_base[i]);
1596         readl(gtt_base);
1597 }
1598
1599 static void i915_ggtt_bind_vma(struct i915_vma *vma,
1600                                enum i915_cache_level cache_level,
1601                                u32 unused)
1602 {
1603         const unsigned long entry = vma->node.start >> PAGE_SHIFT;
1604         const unsigned int num_entries = vma->obj->base.size >> PAGE_SHIFT;
1605         unsigned int flags = (cache_level == I915_CACHE_NONE) ?
1606                 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
1607
1608         BUG_ON(!i915_is_ggtt(vma->vm));
1609         intel_gtt_insert_pages(entry, num_entries, vma->ggtt_view.pages, flags);
1610         vma->bound = GLOBAL_BIND;
1611 }
1612
1613 static void i915_ggtt_clear_range(struct i915_address_space *vm,
1614                                   uint64_t start,
1615                                   uint64_t length,
1616                                   bool unused)
1617 {
1618         unsigned first_entry = start >> PAGE_SHIFT;
1619         unsigned num_entries = length >> PAGE_SHIFT;
1620         intel_gtt_clear_range(first_entry, num_entries);
1621 }
1622
1623 static void i915_ggtt_unbind_vma(struct i915_vma *vma)
1624 {
1625         const unsigned int first = vma->node.start >> PAGE_SHIFT;
1626         const unsigned int size = vma->obj->base.size >> PAGE_SHIFT;
1627
1628         BUG_ON(!i915_is_ggtt(vma->vm));
1629         vma->bound = 0;
1630         intel_gtt_clear_range(first, size);
1631 }
1632
1633 static void ggtt_bind_vma(struct i915_vma *vma,
1634                           enum i915_cache_level cache_level,
1635                           u32 flags)
1636 {
1637         struct drm_device *dev = vma->vm->dev;
1638         struct drm_i915_private *dev_priv = dev->dev_private;
1639         struct drm_i915_gem_object *obj = vma->obj;
1640
1641         /* Currently applicable only to VLV */
1642         if (obj->gt_ro)
1643                 flags |= PTE_READ_ONLY;
1644
1645         /* If there is no aliasing PPGTT, or the caller needs a global mapping,
1646          * or we have a global mapping already but the cacheability flags have
1647          * changed, set the global PTEs.
1648          *
1649          * If there is an aliasing PPGTT it is anecdotally faster, so use that
1650          * instead if none of the above hold true.
1651          *
1652          * NB: A global mapping should only be needed for special regions like
1653          * "gtt mappable", SNB errata, or if specified via special execbuf
1654          * flags. At all other times, the GPU will use the aliasing PPGTT.
1655          */
1656         if (!dev_priv->mm.aliasing_ppgtt || flags & GLOBAL_BIND) {
1657                 if (!(vma->bound & GLOBAL_BIND) ||
1658                     (cache_level != obj->cache_level)) {
1659                         vma->vm->insert_entries(vma->vm, vma->ggtt_view.pages,
1660                                                 vma->node.start,
1661                                                 obj->base.size >> PAGE_SHIFT,
1662                                                 cache_level, flags);
1663                         vma->bound |= GLOBAL_BIND;
1664                 }
1665         }
1666
1667         if (dev_priv->mm.aliasing_ppgtt &&
1668             (!(vma->bound & LOCAL_BIND) ||
1669              (cache_level != obj->cache_level))) {
1670                 struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt;
1671                 appgtt->base.insert_entries(&appgtt->base,
1672                                             vma->ggtt_view.pages,
1673                                             vma->node.start,
1674                                             obj->base.size >> PAGE_SHIFT,
1675                                             cache_level, flags);
1676                 vma->bound |= LOCAL_BIND;
1677         }
1678 }
1679
1680 static void ggtt_unbind_vma(struct i915_vma *vma)
1681 {
1682         struct drm_device *dev = vma->vm->dev;
1683         struct drm_i915_private *dev_priv = dev->dev_private;
1684         struct drm_i915_gem_object *obj = vma->obj;
1685
1686         if (vma->bound & GLOBAL_BIND) {
1687                 vma->vm->clear_range(vma->vm,
1688                                      vma->node.start,
1689                                      obj->base.size,
1690                                      true);
1691                 vma->bound &= ~GLOBAL_BIND;
1692         }
1693
1694         if (vma->bound & LOCAL_BIND) {
1695                 struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt;
1696                 appgtt->base.clear_range(&appgtt->base,
1697                                          vma->node.start,
1698                                          obj->base.size,
1699                                          true);
1700                 vma->bound &= ~LOCAL_BIND;
1701         }
1702 }
1703
1704 void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
1705 {
1706         struct drm_device *dev = obj->base.dev;
1707         struct drm_i915_private *dev_priv = dev->dev_private;
1708         bool interruptible;
1709
1710         interruptible = do_idling(dev_priv);
1711
1712 #if 0
1713         if (!obj->has_dma_mapping)
1714                 dma_unmap_sg(&dev->pdev->dev,
1715                              obj->pages->sgl, obj->pages->nents,
1716                              PCI_DMA_BIDIRECTIONAL);
1717 #endif
1718
1719         undo_idling(dev_priv, interruptible);
1720 }
1721
1722 static void i915_gtt_color_adjust(struct drm_mm_node *node,
1723                                   unsigned long color,
1724                                   u64 *start,
1725                                   u64 *end)
1726 {
1727         if (node->color != color)
1728                 *start += 4096;
1729
1730         if (!list_empty(&node->node_list)) {
1731                 node = list_entry(node->node_list.next,
1732                                   struct drm_mm_node,
1733                                   node_list);
1734                 if (node->allocated && node->color != color)
1735                         *end -= 4096;
1736         }
1737 }
1738
1739 static int i915_gem_setup_global_gtt(struct drm_device *dev,
1740                                      unsigned long start,
1741                                      unsigned long mappable_end,
1742                                      unsigned long end)
1743 {
1744         /* Let GEM Manage all of the aperture.
1745          *
1746          * However, leave one page at the end still bound to the scratch page.
1747          * There are a number of places where the hardware apparently prefetches
1748          * past the end of the object, and we've seen multiple hangs with the
1749          * GPU head pointer stuck in a batchbuffer bound at the last page of the
1750          * aperture.  One page should be enough to keep any prefetching inside
1751          * of the aperture.
1752          */
1753         struct drm_i915_private *dev_priv = dev->dev_private;
1754         struct i915_address_space *ggtt_vm = &dev_priv->gtt.base;
1755         unsigned long mappable;
1756         int error;
1757         struct drm_mm_node *entry;
1758         struct drm_i915_gem_object *obj;
1759         unsigned long hole_start, hole_end;
1760         int ret;
1761
1762         kprintf("MAPPABLE_END VS END %016jx %016jx\n", mappable_end, end);
1763         tsleep(&mappable_end, 0, "DELAY", hz); /* for kprintf */
1764         /*BUG_ON(mappable_end > end);*/
1765
1766         mappable = min(end, mappable_end) - start;
1767
1768         /* Subtract the guard page ... */
1769         drm_mm_init(&ggtt_vm->mm, start, end - start - PAGE_SIZE);
1770         if (!HAS_LLC(dev))
1771                 dev_priv->gtt.base.mm.color_adjust = i915_gtt_color_adjust;
1772
1773         /* Mark any preallocated objects as occupied */
1774         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
1775                 struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm);
1776
1777                 DRM_DEBUG_KMS("reserving preallocated space: %lx + %zx\n",
1778                               i915_gem_obj_ggtt_offset(obj), obj->base.size);
1779
1780                 WARN_ON(i915_gem_obj_ggtt_bound(obj));
1781                 ret = drm_mm_reserve_node(&ggtt_vm->mm, &vma->node);
1782                 if (ret) {
1783                         DRM_DEBUG_KMS("Reservation failed: %i\n", ret);
1784                         return ret;
1785                 }
1786                 vma->bound |= GLOBAL_BIND;
1787         }
1788
1789         dev_priv->gtt.base.start = start;
1790         dev_priv->gtt.base.total = end - start;
1791
1792         /* Clear any non-preallocated blocks */
1793         drm_mm_for_each_hole(entry, &ggtt_vm->mm, hole_start, hole_end) {
1794                 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
1795                               hole_start, hole_end);
1796                 ggtt_vm->clear_range(ggtt_vm, hole_start,
1797                                      hole_end - hole_start, true);
1798         }
1799
1800 #ifdef __DragonFly__
1801         device_printf(dev->dev,
1802             "taking over the fictitious range 0x%lx-0x%lx\n",
1803             dev_priv->gtt.mappable_base + start, dev_priv->gtt.mappable_base + start + mappable);
1804         error = -vm_phys_fictitious_reg_range(dev_priv->gtt.mappable_base + start,
1805             dev_priv->gtt.mappable_base + start + mappable, VM_MEMATTR_WRITE_COMBINING);
1806 #endif
1807
1808         /* And finally clear the reserved guard page */
1809         ggtt_vm->clear_range(ggtt_vm, end - PAGE_SIZE, PAGE_SIZE, true);
1810
1811         if (USES_PPGTT(dev) && !USES_FULL_PPGTT(dev)) {
1812                 struct i915_hw_ppgtt *ppgtt;
1813
1814                 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
1815                 if (!ppgtt)
1816                         return -ENOMEM;
1817
1818                 ret = __hw_ppgtt_init(dev, ppgtt);
1819                 if (ret != 0)
1820                         return ret;
1821
1822                 dev_priv->mm.aliasing_ppgtt = ppgtt;
1823         }
1824
1825         return 0;
1826 }
1827
1828 void i915_gem_init_global_gtt(struct drm_device *dev)
1829 {
1830         struct drm_i915_private *dev_priv = dev->dev_private;
1831         unsigned long gtt_size, mappable_size;
1832
1833         gtt_size = dev_priv->gtt.base.total;
1834         mappable_size = dev_priv->gtt.mappable_end;
1835
1836         i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
1837 }
1838
1839 void i915_global_gtt_cleanup(struct drm_device *dev)
1840 {
1841         struct drm_i915_private *dev_priv = dev->dev_private;
1842         struct i915_address_space *vm = &dev_priv->gtt.base;
1843
1844         if (dev_priv->mm.aliasing_ppgtt) {
1845                 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
1846
1847                 ppgtt->base.cleanup(&ppgtt->base);
1848         }
1849
1850         if (drm_mm_initialized(&vm->mm)) {
1851                 drm_mm_takedown(&vm->mm);
1852                 list_del(&vm->global_link);
1853         }
1854
1855         vm->cleanup(vm);
1856 }
1857
1858 static int setup_scratch_page(struct drm_device *dev)
1859 {
1860         struct drm_i915_private *dev_priv = dev->dev_private;
1861         struct vm_page *page;
1862         dma_addr_t dma_addr;
1863
1864         page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO);
1865         if (page == NULL)
1866                 return -ENOMEM;
1867         set_pages_uc(page, 1);
1868
1869 #ifdef CONFIG_INTEL_IOMMU
1870         dma_addr = pci_map_page(dev->pdev, page, 0, PAGE_SIZE,
1871                                 PCI_DMA_BIDIRECTIONAL);
1872         if (pci_dma_mapping_error(dev->pdev, dma_addr))
1873                 return -EINVAL;
1874 #else
1875         dma_addr = page_to_phys(page);
1876 #endif
1877         dev_priv->gtt.base.scratch.page = page;
1878         dev_priv->gtt.base.scratch.addr = dma_addr;
1879
1880         return 0;
1881 }
1882
1883 #if 0
1884 static void teardown_scratch_page(struct drm_device *dev)
1885 {
1886         struct drm_i915_private *dev_priv = dev->dev_private;
1887         struct vm_page *page = dev_priv->gtt.base.scratch.page;
1888
1889         set_pages_wb(page, 1);
1890         pci_unmap_page(dev->pdev, dev_priv->gtt.base.scratch.addr,
1891                        PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
1892         __free_page(page);
1893 }
1894 #endif
1895
1896 static inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
1897 {
1898         snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
1899         snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
1900         return snb_gmch_ctl << 20;
1901 }
1902
1903 static inline unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
1904 {
1905         bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
1906         bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
1907         if (bdw_gmch_ctl)
1908                 bdw_gmch_ctl = 1 << bdw_gmch_ctl;
1909
1910 #ifdef CONFIG_X86_32
1911         /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * PAGE_SIZE */
1912         if (bdw_gmch_ctl > 4)
1913                 bdw_gmch_ctl = 4;
1914 #endif
1915
1916         return bdw_gmch_ctl << 20;
1917 }
1918
1919 static inline unsigned int chv_get_total_gtt_size(u16 gmch_ctrl)
1920 {
1921         gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT;
1922         gmch_ctrl &= SNB_GMCH_GGMS_MASK;
1923
1924         if (gmch_ctrl)
1925                 return 1 << (20 + gmch_ctrl);
1926
1927         return 0;
1928 }
1929
1930 static inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
1931 {
1932         snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
1933         snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
1934         return snb_gmch_ctl << 25; /* 32 MB units */
1935 }
1936
1937 static inline size_t gen8_get_stolen_size(u16 bdw_gmch_ctl)
1938 {
1939         bdw_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
1940         bdw_gmch_ctl &= BDW_GMCH_GMS_MASK;
1941         return bdw_gmch_ctl << 25; /* 32 MB units */
1942 }
1943
1944 static size_t chv_get_stolen_size(u16 gmch_ctrl)
1945 {
1946         gmch_ctrl >>= SNB_GMCH_GMS_SHIFT;
1947         gmch_ctrl &= SNB_GMCH_GMS_MASK;
1948
1949         /*
1950          * 0x0  to 0x10: 32MB increments starting at 0MB
1951          * 0x11 to 0x16: 4MB increments starting at 8MB
1952          * 0x17 to 0x1d: 4MB increments start at 36MB
1953          */
1954         if (gmch_ctrl < 0x11)
1955                 return gmch_ctrl << 25;
1956         else if (gmch_ctrl < 0x17)
1957                 return (gmch_ctrl - 0x11 + 2) << 22;
1958         else
1959                 return (gmch_ctrl - 0x17 + 9) << 22;
1960 }
1961
1962 static size_t gen9_get_stolen_size(u16 gen9_gmch_ctl)
1963 {
1964         gen9_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
1965         gen9_gmch_ctl &= BDW_GMCH_GMS_MASK;
1966
1967         if (gen9_gmch_ctl < 0xf0)
1968                 return gen9_gmch_ctl << 25; /* 32 MB units */
1969         else
1970                 /* 4MB increments starting at 0xf0 for 4MB */
1971                 return (gen9_gmch_ctl - 0xf0 + 1) << 22;
1972 }
1973
1974 static int ggtt_probe_common(struct drm_device *dev,
1975                              size_t gtt_size)
1976 {
1977         struct drm_i915_private *dev_priv = dev->dev_private;
1978         phys_addr_t gtt_phys_addr;
1979         int ret;
1980
1981         /* For Modern GENs the PTEs and register space are split in the BAR */
1982         gtt_phys_addr = pci_resource_start(dev->pdev, 0) +
1983                 (pci_resource_len(dev->pdev, 0) / 2);
1984
1985         kprintf("gtt_probe_common: gtt_phys_addr=0x%lx\n", gtt_phys_addr);
1986         dev_priv->gtt.gsm = ioremap_wc(gtt_phys_addr, gtt_size);
1987         if (!dev_priv->gtt.gsm) {
1988                 DRM_ERROR("Failed to map the gtt page table\n");
1989                 return -ENOMEM;
1990         }
1991
1992         ret = setup_scratch_page(dev);
1993         if (ret) {
1994                 DRM_ERROR("Scratch setup failed\n");
1995                 /* iounmap will also get called at remove, but meh */
1996 #if 0
1997                 iounmap(dev_priv->gtt.gsm);
1998 #endif
1999         }
2000
2001         return ret;
2002 }
2003
2004 /* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability
2005  * bits. When using advanced contexts each context stores its own PAT, but
2006  * writing this data shouldn't be harmful even in those cases. */
2007 static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv)
2008 {
2009         uint64_t pat;
2010
2011         pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC)     | /* for normal objects, no eLLC */
2012               GEN8_PPAT(1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC) | /* for something pointing to ptes? */
2013               GEN8_PPAT(2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC) | /* for scanout with eLLC */
2014               GEN8_PPAT(3, GEN8_PPAT_UC)                     | /* Uncached objects, mostly for scanout */
2015               GEN8_PPAT(4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)) |
2016               GEN8_PPAT(5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)) |
2017               GEN8_PPAT(6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)) |
2018               GEN8_PPAT(7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
2019
2020         if (!USES_PPGTT(dev_priv->dev))
2021                 /* Spec: "For GGTT, there is NO pat_sel[2:0] from the entry,
2022                  * so RTL will always use the value corresponding to
2023                  * pat_sel = 000".
2024                  * So let's disable cache for GGTT to avoid screen corruptions.
2025                  * MOCS still can be used though.
2026                  * - System agent ggtt writes (i.e. cpu gtt mmaps) already work
2027                  * before this patch, i.e. the same uncached + snooping access
2028                  * like on gen6/7 seems to be in effect.
2029                  * - So this just fixes blitter/render access. Again it looks
2030                  * like it's not just uncached access, but uncached + snooping.
2031                  * So we can still hold onto all our assumptions wrt cpu
2032                  * clflushing on LLC machines.
2033                  */
2034                 pat = GEN8_PPAT(0, GEN8_PPAT_UC);
2035
2036         /* XXX: spec defines this as 2 distinct registers. It's unclear if a 64b
2037          * write would work. */
2038         I915_WRITE(GEN8_PRIVATE_PAT, pat);
2039         I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32);
2040 }
2041
2042 static void chv_setup_private_ppat(struct drm_i915_private *dev_priv)
2043 {
2044         uint64_t pat;
2045
2046         /*
2047          * Map WB on BDW to snooped on CHV.
2048          *
2049          * Only the snoop bit has meaning for CHV, the rest is
2050          * ignored.
2051          *
2052          * The hardware will never snoop for certain types of accesses:
2053          * - CPU GTT (GMADR->GGTT->no snoop->memory)
2054          * - PPGTT page tables
2055          * - some other special cycles
2056          *
2057          * As with BDW, we also need to consider the following for GT accesses:
2058          * "For GGTT, there is NO pat_sel[2:0] from the entry,
2059          * so RTL will always use the value corresponding to
2060          * pat_sel = 000".
2061          * Which means we must set the snoop bit in PAT entry 0
2062          * in order to keep the global status page working.
2063          */
2064         pat = GEN8_PPAT(0, CHV_PPAT_SNOOP) |
2065               GEN8_PPAT(1, 0) |
2066               GEN8_PPAT(2, 0) |
2067               GEN8_PPAT(3, 0) |
2068               GEN8_PPAT(4, CHV_PPAT_SNOOP) |
2069               GEN8_PPAT(5, CHV_PPAT_SNOOP) |
2070               GEN8_PPAT(6, CHV_PPAT_SNOOP) |
2071               GEN8_PPAT(7, CHV_PPAT_SNOOP);
2072
2073         I915_WRITE(GEN8_PRIVATE_PAT, pat);
2074         I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32);
2075 }
2076
2077 static int gen8_gmch_probe(struct drm_device *dev,
2078                            size_t *gtt_total,
2079                            size_t *stolen,
2080                            phys_addr_t *mappable_base,
2081                            unsigned long *mappable_end)
2082 {
2083         struct drm_i915_private *dev_priv = dev->dev_private;
2084         unsigned int gtt_size;
2085         u16 snb_gmch_ctl;
2086         int ret;
2087
2088         /* TODO: We're not aware of mappable constraints on gen8 yet */
2089         *mappable_base = pci_resource_start(dev->pdev, 2);
2090         *mappable_end = pci_resource_len(dev->pdev, 2);
2091
2092 #if 0
2093         if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(39)))
2094                 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(39));
2095 #endif
2096
2097         pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
2098
2099         if (INTEL_INFO(dev)->gen >= 9) {
2100                 *stolen = gen9_get_stolen_size(snb_gmch_ctl);
2101                 gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl);
2102         } else if (IS_CHERRYVIEW(dev)) {
2103                 *stolen = chv_get_stolen_size(snb_gmch_ctl);
2104                 gtt_size = chv_get_total_gtt_size(snb_gmch_ctl);
2105         } else {
2106                 *stolen = gen8_get_stolen_size(snb_gmch_ctl);
2107                 gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl);
2108         }
2109
2110         *gtt_total = (gtt_size / sizeof(gen8_gtt_pte_t)) << PAGE_SHIFT;
2111
2112         if (IS_CHERRYVIEW(dev))
2113                 chv_setup_private_ppat(dev_priv);
2114         else
2115                 bdw_setup_private_ppat(dev_priv);
2116
2117         ret = ggtt_probe_common(dev, gtt_size);
2118
2119         dev_priv->gtt.base.clear_range = gen8_ggtt_clear_range;
2120         dev_priv->gtt.base.insert_entries = gen8_ggtt_insert_entries;
2121
2122         return ret;
2123 }
2124
2125 static int gen6_gmch_probe(struct drm_device *dev,
2126                            size_t *gtt_total,
2127                            size_t *stolen,
2128                            phys_addr_t *mappable_base,
2129                            unsigned long *mappable_end)
2130 {
2131         struct drm_i915_private *dev_priv = dev->dev_private;
2132         unsigned int gtt_size;
2133         u16 snb_gmch_ctl;
2134         int ret;
2135
2136         *mappable_base = pci_resource_start(dev->pdev, 2);
2137         *mappable_end = pci_resource_len(dev->pdev, 2);
2138
2139         /* 64/512MB is the current min/max we actually know of, but this is just
2140          * a coarse sanity check.
2141          */
2142         if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) {
2143                 DRM_ERROR("Unknown GMADR size (%lx)\n",
2144                           dev_priv->gtt.mappable_end);
2145                 return -ENXIO;
2146         }
2147
2148 #if 0
2149         if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40)))
2150                 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40));
2151 #endif
2152         pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
2153
2154         *stolen = gen6_get_stolen_size(snb_gmch_ctl);
2155
2156         gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl);
2157         *gtt_total = (gtt_size / sizeof(gen6_gtt_pte_t)) << PAGE_SHIFT;
2158
2159         ret = ggtt_probe_common(dev, gtt_size);
2160
2161         dev_priv->gtt.base.clear_range = gen6_ggtt_clear_range;
2162         dev_priv->gtt.base.insert_entries = gen6_ggtt_insert_entries;
2163
2164         return ret;
2165 }
2166
2167 static void gen6_gmch_remove(struct i915_address_space *vm)
2168 {
2169 #if 0
2170         struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base);
2171
2172         iounmap(gtt->gsm);
2173         teardown_scratch_page(vm->dev);
2174 #endif
2175 }
2176
2177 static int i915_gmch_probe(struct drm_device *dev,
2178                            size_t *gtt_total,
2179                            size_t *stolen,
2180                            phys_addr_t *mappable_base,
2181                            unsigned long *mappable_end)
2182 {
2183         struct drm_i915_private *dev_priv = dev->dev_private;
2184 #if 0
2185         int ret;
2186
2187         ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL);
2188         if (!ret) {
2189                 DRM_ERROR("failed to set up gmch\n");
2190                 return -EIO;
2191         }
2192 #endif
2193
2194         intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end);
2195
2196         dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev);
2197         dev_priv->gtt.base.clear_range = i915_ggtt_clear_range;
2198
2199         if (unlikely(dev_priv->gtt.do_idle_maps))
2200                 DRM_INFO("applying Ironlake quirks for intel_iommu\n");
2201
2202         return 0;
2203 }
2204
2205 static void i915_gmch_remove(struct i915_address_space *vm)
2206 {
2207         intel_gmch_remove();
2208 }
2209
2210 int i915_gem_gtt_init(struct drm_device *dev)
2211 {
2212         struct drm_i915_private *dev_priv = dev->dev_private;
2213         struct i915_gtt *gtt = &dev_priv->gtt;
2214         int ret;
2215
2216         if (INTEL_INFO(dev)->gen <= 5) {
2217                 gtt->gtt_probe = i915_gmch_probe;
2218                 gtt->base.cleanup = i915_gmch_remove;
2219         } else if (INTEL_INFO(dev)->gen < 8) {
2220                 gtt->gtt_probe = gen6_gmch_probe;
2221                 gtt->base.cleanup = gen6_gmch_remove;
2222                 if (IS_HASWELL(dev) && dev_priv->ellc_size)
2223                         gtt->base.pte_encode = iris_pte_encode;
2224                 else if (IS_HASWELL(dev))
2225                         gtt->base.pte_encode = hsw_pte_encode;
2226                 else if (IS_VALLEYVIEW(dev))
2227                         gtt->base.pte_encode = byt_pte_encode;
2228                 else if (INTEL_INFO(dev)->gen >= 7)
2229                         gtt->base.pte_encode = ivb_pte_encode;
2230                 else
2231                         gtt->base.pte_encode = snb_pte_encode;
2232         } else {
2233                 dev_priv->gtt.gtt_probe = gen8_gmch_probe;
2234                 dev_priv->gtt.base.cleanup = gen6_gmch_remove;
2235         }
2236
2237         ret = gtt->gtt_probe(dev, &gtt->base.total, &gtt->stolen_size,
2238                              &gtt->mappable_base, &gtt->mappable_end);
2239         if (ret)
2240                 return ret;
2241
2242         gtt->base.dev = dev;
2243
2244         /* GMADR is the PCI mmio aperture into the global GTT. */
2245         DRM_INFO("Memory usable by graphics device = %zdM\n",
2246                  gtt->base.total >> 20);
2247         DRM_DEBUG_DRIVER("GMADR size = %ldM\n", gtt->mappable_end >> 20);
2248         DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20);
2249 #ifdef CONFIG_INTEL_IOMMU
2250         if (intel_iommu_gfx_mapped)
2251                 DRM_INFO("VT-d active for gfx access\n");
2252 #endif
2253         /*
2254          * i915.enable_ppgtt is read-only, so do an early pass to validate the
2255          * user's requested state against the hardware/driver capabilities.  We
2256          * do this now so that we can print out any log messages once rather
2257          * than every time we check intel_enable_ppgtt().
2258          */
2259         i915.enable_ppgtt = sanitize_enable_ppgtt(dev, i915.enable_ppgtt);
2260         DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt);
2261
2262         return 0;
2263 }
2264
2265 static struct i915_vma *__i915_gem_vma_create(struct drm_i915_gem_object *obj,
2266                                               struct i915_address_space *vm,
2267                                               const struct i915_ggtt_view *view)
2268 {
2269         struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);
2270         if (vma == NULL)
2271                 return ERR_PTR(-ENOMEM);
2272
2273         INIT_LIST_HEAD(&vma->vma_link);
2274         INIT_LIST_HEAD(&vma->mm_list);
2275         INIT_LIST_HEAD(&vma->exec_list);
2276         vma->vm = vm;
2277         vma->obj = obj;
2278         vma->ggtt_view = *view;
2279
2280         if (INTEL_INFO(vm->dev)->gen >= 6) {
2281                 if (i915_is_ggtt(vm)) {
2282                         vma->unbind_vma = ggtt_unbind_vma;
2283                         vma->bind_vma = ggtt_bind_vma;
2284                 } else {
2285                         vma->unbind_vma = ppgtt_unbind_vma;
2286                         vma->bind_vma = ppgtt_bind_vma;
2287                 }
2288         } else {
2289                 BUG_ON(!i915_is_ggtt(vm));
2290                 vma->unbind_vma = i915_ggtt_unbind_vma;
2291                 vma->bind_vma = i915_ggtt_bind_vma;
2292         }
2293
2294         list_add_tail(&vma->vma_link, &obj->vma_list);
2295         if (!i915_is_ggtt(vm))
2296                 i915_ppgtt_get(i915_vm_to_ppgtt(vm));
2297
2298         return vma;
2299 }
2300
2301 struct i915_vma *
2302 i915_gem_obj_lookup_or_create_vma_view(struct drm_i915_gem_object *obj,
2303                                        struct i915_address_space *vm,
2304                                        const struct i915_ggtt_view *view)
2305 {
2306         struct i915_vma *vma;
2307
2308         vma = i915_gem_obj_to_vma_view(obj, vm, view);
2309         if (!vma)
2310                 vma = __i915_gem_vma_create(obj, vm, view);
2311
2312         return vma;
2313 }
2314
2315 static inline
2316 int i915_get_vma_pages(struct i915_vma *vma)
2317 {
2318         if (vma->ggtt_view.pages)
2319                 return 0;
2320
2321         if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL)
2322                 vma->ggtt_view.pages = vma->obj->pages;
2323         else
2324                 WARN_ONCE(1, "GGTT view %u not implemented!\n",
2325                           vma->ggtt_view.type);
2326
2327         if (!vma->ggtt_view.pages) {
2328                 DRM_ERROR("Failed to get pages for VMA view type %u!\n",
2329                           vma->ggtt_view.type);
2330                 return -EINVAL;
2331         }
2332
2333         return 0;
2334 }
2335
2336 /**
2337  * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space.
2338  * @vma: VMA to map
2339  * @cache_level: mapping cache level
2340  * @flags: flags like global or local mapping
2341  *
2342  * DMA addresses are taken from the scatter-gather table of this object (or of
2343  * this VMA in case of non-default GGTT views) and PTE entries set up.
2344  * Note that DMA addresses are also the only part of the SG table we care about.
2345  */
2346 int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
2347                   u32 flags)
2348 {
2349         int ret = i915_get_vma_pages(vma);
2350
2351         if (ret)
2352                 return ret;
2353
2354         vma->bind_vma(vma, cache_level, flags);
2355
2356         return 0;
2357 }