sys/dev/drm/ttm/ttm_page_alloc.c

   1 /*
   2  * Copyright (c) Red Hat Inc.
   3
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sub license,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the
  12  * next paragraph) shall be included in all copies or substantial portions
  13  * of the Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors: Dave Airlie <airlied@redhat.com>
  24  *          Jerome Glisse <jglisse@redhat.com>
  25  *          Pauli Nieminen <suokkos@gmail.com>
  26  */
  27 /*
  28  * Copyright (c) 2013 The FreeBSD Foundation
  29  * All rights reserved.
  30  *
  31  * Portions of this software were developed by Konstantin Belousov
  32  * <kib@FreeBSD.org> under sponsorship from the FreeBSD Foundation.
  33  */
  34
  35 /* simple list based uncached page pool
  36  * - Pool collects resently freed pages for reuse
  37  * - Use page->lru to keep a free list
  38  * - doesn't track currently in use pages
  39  */
  40
  41 #define pr_fmt(fmt) "[TTM] " fmt
  42
  43 #include <linux/list.h>
  44 #include <linux/spinlock.h>
  45 #include <linux/highmem.h>
  46 #include <linux/mm_types.h>
  47 #include <linux/module.h>
  48 #include <linux/mm.h>
  49 #include <linux/seq_file.h> /* for seq_printf */
  50 #include <linux/dma-mapping.h>
  51
  52 #include <linux/atomic.h>
  53
  54 #include <drm/ttm/ttm_bo_driver.h>
  55 #include <drm/ttm/ttm_page_alloc.h>
  56
  57 #include <sys/eventhandler.h>
  58 #include <vm/vm_page2.h>
  59
  60 #if IS_ENABLED(CONFIG_AGP)
  61 #include <asm/agp.h>
  62 #endif
  63
  64 #define NUM_PAGES_TO_ALLOC              (PAGE_SIZE/sizeof(struct page *))
  65 #define SMALL_ALLOCATION                16
  66 #define FREE_ALL_PAGES                  (~0U)
  67 /* times are in msecs */
  68 #define PAGE_FREE_INTERVAL              1000
  69
  70 /**
  71  * struct ttm_page_pool - Pool to reuse recently allocated uc/wc pages.
  72  *
  73  * @lock: Protects the shared pool from concurrnet access. Must be used with
  74  * irqsave/irqrestore variants because pool allocator maybe called from
  75  * delayed work.
  76  * @fill_lock: Prevent concurrent calls to fill.
  77  * @list: Pool of free uc/wc pages for fast reuse.
  78  * @gfp_flags: Flags to pass for alloc_page.
  79  * @npages: Number of pages in pool.
  80  */
  81 struct ttm_page_pool {
  82         struct lock             lock;
  83         bool                    fill_lock;
  84         struct pglist           list;
  85         gfp_t                   gfp_flags;
  86         unsigned                npages;
  87         char                    *name;
  88         unsigned long           nfrees;
  89         unsigned long           nrefills;
  90 };
  91
  92 /**
  93  * Limits for the pool. They are handled without locks because only place where
  94  * they may change is in sysfs store. They won't have immediate effect anyway
  95  * so forcing serialization to access them is pointless.
  96  */
  97
  98 struct ttm_pool_opts {
  99         unsigned        alloc_size;
 100         unsigned        max_size;
 101         unsigned        small;
 102 };
 103
 104 #define NUM_POOLS 4
 105
 106 /**
 107  * struct ttm_pool_manager - Holds memory pools for fst allocation
 108  *
 109  * Manager is read only object for pool code so it doesn't need locking.
 110  *
 111  * @free_interval: minimum number of jiffies between freeing pages from pool.
 112  * @page_alloc_inited: reference counting for pool allocation.
 113  * @work: Work that is used to shrink the pool. Work is only run when there is
 114  * some pages to free.
 115  * @small_allocation: Limit in number of pages what is small allocation.
 116  *
 117  * @pools: All pool objects in use.
 118  **/
 119 struct ttm_pool_manager {
 120         struct kobject          kobj;
 121         struct shrinker         mm_shrink;
 122         eventhandler_tag lowmem_handler;
 123         struct ttm_pool_opts    options;
 124
 125         union {
 126                 struct ttm_page_pool    pools[NUM_POOLS];
 127                 struct {
 128                         struct ttm_page_pool    wc_pool;
 129                         struct ttm_page_pool    uc_pool;
 130                         struct ttm_page_pool    wc_pool_dma32;
 131                         struct ttm_page_pool    uc_pool_dma32;
 132                 } ;
 133         };
 134 };
 135
 136 static struct attribute ttm_page_pool_max = {
 137         .name = "pool_max_size",
 138         .mode = S_IRUGO | S_IWUSR
 139 };
 140 static struct attribute ttm_page_pool_small = {
 141         .name = "pool_small_allocation",
 142         .mode = S_IRUGO | S_IWUSR
 143 };
 144 static struct attribute ttm_page_pool_alloc_size = {
 145         .name = "pool_allocation_size",
 146         .mode = S_IRUGO | S_IWUSR
 147 };
 148
 149 static struct attribute *ttm_pool_attrs[] = {
 150         &ttm_page_pool_max,
 151         &ttm_page_pool_small,
 152         &ttm_page_pool_alloc_size,
 153         NULL
 154 };
 155
 156 static void ttm_pool_kobj_release(struct kobject *kobj)
 157 {
 158         struct ttm_pool_manager *m =
 159                 container_of(kobj, struct ttm_pool_manager, kobj);
 160         kfree(m);
 161 }
 162
 163 static ssize_t ttm_pool_store(struct kobject *kobj,
 164                 struct attribute *attr, const char *buffer, size_t size)
 165 {
 166         struct ttm_pool_manager *m =
 167                 container_of(kobj, struct ttm_pool_manager, kobj);
 168         int chars;
 169         unsigned val;
 170         chars = ksscanf(buffer, "%u", &val);
 171         if (chars == 0)
 172                 return size;
 173
 174         /* Convert kb to number of pages */
 175         val = val / (PAGE_SIZE >> 10);
 176
 177         if (attr == &ttm_page_pool_max)
 178                 m->options.max_size = val;
 179         else if (attr == &ttm_page_pool_small)
 180                 m->options.small = val;
 181         else if (attr == &ttm_page_pool_alloc_size) {
 182                 if (val > NUM_PAGES_TO_ALLOC*8) {
 183                         pr_err("Setting allocation size to %lu is not allowed. Recommended size is %lu\n",
 184                                NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7),
 185                                NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10));
 186                         return size;
 187                 } else if (val > NUM_PAGES_TO_ALLOC) {
 188                         pr_warn("Setting allocation size to larger than %lu is not recommended\n",
 189                                 NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10));
 190                 }
 191                 m->options.alloc_size = val;
 192         }
 193
 194         return size;
 195 }
 196
 197 static ssize_t ttm_pool_show(struct kobject *kobj,
 198                 struct attribute *attr, char *buffer)
 199 {
 200         struct ttm_pool_manager *m =
 201                 container_of(kobj, struct ttm_pool_manager, kobj);
 202         unsigned val = 0;
 203
 204         if (attr == &ttm_page_pool_max)
 205                 val = m->options.max_size;
 206         else if (attr == &ttm_page_pool_small)
 207                 val = m->options.small;
 208         else if (attr == &ttm_page_pool_alloc_size)
 209                 val = m->options.alloc_size;
 210
 211         val = val * (PAGE_SIZE >> 10);
 212
 213         return ksnprintf(buffer, PAGE_SIZE, "%u\n", val);
 214 }
 215
 216 static const struct sysfs_ops ttm_pool_sysfs_ops = {
 217         .show = &ttm_pool_show,
 218         .store = &ttm_pool_store,
 219 };
 220
 221 static struct kobj_type ttm_pool_kobj_type = {
 222         .release = &ttm_pool_kobj_release,
 223         .sysfs_ops = &ttm_pool_sysfs_ops,
 224         .default_attrs = ttm_pool_attrs,
 225 };
 226
 227 static struct ttm_pool_manager *_manager;
 228
 229 #ifndef CONFIG_X86
 230 static int set_pages_array_wb(struct page **pages, int addrinarray)
 231 {
 232 #if IS_ENABLED(CONFIG_AGP)
 233         int i;
 234
 235         for (i = 0; i < addrinarray; i++)
 236                 unmap_page_from_agp(pages[i]);
 237 #endif
 238         return 0;
 239 }
 240
 241 static int set_pages_array_wc(struct page **pages, int addrinarray)
 242 {
 243 #if IS_ENABLED(CONFIG_AGP)
 244         int i;
 245
 246         for (i = 0; i < addrinarray; i++)
 247                 map_page_into_agp(pages[i]);
 248 #endif
 249         return 0;
 250 }
 251
 252 static int set_pages_array_uc(struct page **pages, int addrinarray)
 253 {
 254 #if IS_ENABLED(CONFIG_AGP)
 255         int i;
 256
 257         for (i = 0; i < addrinarray; i++)
 258                 map_page_into_agp(pages[i]);
 259 #endif
 260         return 0;
 261 }
 262 #endif
 263
 264 /**
 265  * Select the right pool or requested caching state and ttm flags. */
 266 static struct ttm_page_pool *ttm_get_pool(int flags,
 267                 enum ttm_caching_state cstate)
 268 {
 269         int pool_index;
 270
 271         if (cstate == tt_cached)
 272                 return NULL;
 273
 274         if (cstate == tt_wc)
 275                 pool_index = 0x0;
 276         else
 277                 pool_index = 0x1;
 278
 279         if (flags & TTM_PAGE_FLAG_DMA32)
 280                 pool_index |= 0x2;
 281
 282         return &_manager->pools[pool_index];
 283 }
 284
 285 /* set memory back to wb and free the pages. */
 286 static void ttm_pages_put(struct page *pages[], unsigned npages)
 287 {
 288         unsigned i;
 289         if (set_pages_array_wb(pages, npages))
 290                 pr_err("Failed to set %d pages to wb!\n", npages);
 291         for (i = 0; i < npages; ++i) {
 292                 __free_page(pages[i]);
 293         }
 294 }
 295
 296 static void ttm_pool_update_free_locked(struct ttm_page_pool *pool,
 297                 unsigned freed_pages)
 298 {
 299         pool->npages -= freed_pages;
 300         pool->nfrees += freed_pages;
 301 }
 302
 303 /**
 304  * Free pages from pool.
 305  *
 306  * To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC
 307  * number of pages in one go.
 308  *
 309  * @pool: to free the pages from
 310  * @free_all: If set to true will free all pages in pool
 311  * @use_static: Safe to use static buffer
 312  **/
 313 static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free,
 314                               bool use_static)
 315 {
 316         static struct page *static_buf[NUM_PAGES_TO_ALLOC];
 317         unsigned long irq_flags;
 318         struct vm_page *p, *p1;
 319         struct page **pages_to_free;
 320         unsigned freed_pages = 0,
 321                  npages_to_free = nr_free;
 322         unsigned i;
 323
 324         if (NUM_PAGES_TO_ALLOC < nr_free)
 325                 npages_to_free = NUM_PAGES_TO_ALLOC;
 326
 327         if (use_static)
 328                 pages_to_free = static_buf;
 329         else
 330                 pages_to_free = kmalloc(npages_to_free * sizeof(struct page *),
 331                                         M_DRM, GFP_KERNEL);
 332         if (!pages_to_free) {
 333                 pr_err("Failed to allocate memory for pool free operation\n");
 334                 return 0;
 335         }
 336
 337 restart:
 338         spin_lock_irqsave(&pool->lock, irq_flags);
 339
 340         TAILQ_FOREACH_REVERSE_MUTABLE(p, &pool->list, pglist, pageq, p1) {
 341                 if (freed_pages >= npages_to_free)
 342                         break;
 343
 344                 pages_to_free[freed_pages++] = (struct page *)p;
 345                 /* We can only remove NUM_PAGES_TO_ALLOC at a time. */
 346                 if (freed_pages >= NUM_PAGES_TO_ALLOC) {
 347                         /* remove range of pages from the pool */
 348                         for (i = 0; i < freed_pages; i++)
 349                                 TAILQ_REMOVE(&pool->list, (struct vm_page *)pages_to_free[i], pageq);
 350
 351                         ttm_pool_update_free_locked(pool, freed_pages);
 352                         /**
 353                          * Because changing page caching is costly
 354                          * we unlock the pool to prevent stalling.
 355                          */
 356                         spin_unlock_irqrestore(&pool->lock, irq_flags);
 357
 358                         ttm_pages_put(pages_to_free, freed_pages);
 359                         if (likely(nr_free != FREE_ALL_PAGES))
 360                                 nr_free -= freed_pages;
 361
 362                         if (NUM_PAGES_TO_ALLOC >= nr_free)
 363                                 npages_to_free = nr_free;
 364                         else
 365                                 npages_to_free = NUM_PAGES_TO_ALLOC;
 366
 367                         freed_pages = 0;
 368
 369                         /* free all so restart the processing */
 370                         if (nr_free)
 371                                 goto restart;
 372
 373                         /* Not allowed to fall through or break because
 374                          * following context is inside spinlock while we are
 375                          * outside here.
 376                          */
 377                         goto out;
 378
 379                 }
 380         }
 381
 382         /* remove range of pages from the pool */
 383         if (freed_pages) {
 384                 for (i = 0; i < freed_pages; i++)
 385                         TAILQ_REMOVE(&pool->list, (struct vm_page *)pages_to_free[i], pageq);
 386
 387                 ttm_pool_update_free_locked(pool, freed_pages);
 388                 nr_free -= freed_pages;
 389         }
 390
 391         spin_unlock_irqrestore(&pool->lock, irq_flags);
 392
 393         if (freed_pages)
 394                 ttm_pages_put(pages_to_free, freed_pages);
 395 out:
 396         if (pages_to_free != static_buf)
 397                 kfree(pages_to_free);
 398         return nr_free;
 399 }
 400
 401 /**
 402  * Callback for mm to request pool to reduce number of page held.
 403  *
 404  * XXX: (dchinner) Deadlock warning!
 405  *
 406  * This code is crying out for a shrinker per pool....
 407  */
 408 static unsigned long
 409 ttm_pool_shrink_scan(void *arg)
 410 {
 411 #ifdef __DragonFly__
 412         static struct shrink_control __sc;
 413         struct shrink_control *sc = &__sc;
 414 #endif
 415         static DEFINE_MUTEX(lock);
 416         static unsigned start_pool;
 417         unsigned i;
 418         unsigned pool_offset;
 419         struct ttm_page_pool *pool;
 420         int shrink_pages = 100; /* XXXKIB */
 421         unsigned long freed = 0;
 422
 423 #ifdef __DragonFly__
 424         sc->gfp_mask = M_WAITOK;
 425 #endif
 426
 427         if (!mutex_trylock(&lock))
 428                 return SHRINK_STOP;
 429         pool_offset = ++start_pool % NUM_POOLS;
 430         /* select start pool in round robin fashion */
 431         for (i = 0; i < NUM_POOLS; ++i) {
 432                 unsigned nr_free = shrink_pages;
 433                 if (shrink_pages == 0)
 434                         break;
 435                 pool = &_manager->pools[(i + pool_offset)%NUM_POOLS];
 436                 /* OK to use static buffer since global mutex is held. */
 437                 shrink_pages = ttm_page_pool_free(pool, nr_free, true);
 438                 freed += nr_free - shrink_pages;
 439         }
 440         mutex_unlock(&lock);
 441         return freed;
 442 }
 443
 444
 445 static unsigned long
 446 ttm_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
 447 {
 448         unsigned i;
 449         unsigned long count = 0;
 450
 451         for (i = 0; i < NUM_POOLS; ++i)
 452                 count += _manager->pools[i].npages;
 453
 454         return count;
 455 }
 456
 457 static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager)
 458 {
 459         manager->mm_shrink.count_objects = ttm_pool_shrink_count;
 460         manager->lowmem_handler = EVENTHANDLER_REGISTER(vm_lowmem,
 461             ttm_pool_shrink_scan, manager, EVENTHANDLER_PRI_ANY);
 462 }
 463
 464 static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager)
 465 {
 466         EVENTHANDLER_DEREGISTER(vm_lowmem, manager->lowmem_handler);
 467 }
 468
 469 static int ttm_set_pages_caching(struct page **pages,
 470                 enum ttm_caching_state cstate, unsigned cpages)
 471 {
 472         int r = 0;
 473         /* Set page caching */
 474         switch (cstate) {
 475         case tt_uncached:
 476                 r = set_pages_array_uc(pages, cpages);
 477                 if (r)
 478                         pr_err("Failed to set %d pages to uc!\n", cpages);
 479                 break;
 480         case tt_wc:
 481                 r = set_pages_array_wc(pages, cpages);
 482                 if (r)
 483                         pr_err("Failed to set %d pages to wc!\n", cpages);
 484                 break;
 485         default:
 486                 break;
 487         }
 488         return r;
 489 }
 490
 491 /**
 492  * Free pages the pages that failed to change the caching state. If there is
 493  * any pages that have changed their caching state already put them to the
 494  * pool.
 495  */
 496 static void ttm_handle_caching_state_failure(struct pglist *pages,
 497                 int ttm_flags, enum ttm_caching_state cstate,
 498                 struct page **failed_pages, unsigned cpages)
 499 {
 500         unsigned i;
 501         /* Failed pages have to be freed */
 502         for (i = 0; i < cpages; ++i) {
 503                 TAILQ_REMOVE(pages, (struct vm_page *)failed_pages[i], pageq);
 504                 __free_page(failed_pages[i]);
 505         }
 506 }
 507
 508 /**
 509  * Allocate new pages with correct caching.
 510  *
 511  * This function is reentrant if caller updates count depending on number of
 512  * pages returned in pages array.
 513  */
 514 static int ttm_alloc_new_pages(struct pglist *pages, gfp_t gfp_flags,
 515                 int ttm_flags, enum ttm_caching_state cstate, unsigned count)
 516 {
 517         struct page **caching_array;
 518         struct page *p;
 519         int r = 0;
 520         unsigned i, cpages;
 521         unsigned max_cpages = min(count,
 522                         (unsigned)(PAGE_SIZE/sizeof(struct page *)));
 523
 524         /* allocate array for page caching change */
 525         caching_array = kmalloc(max_cpages*sizeof(struct page *), M_DRM, M_WAITOK);
 526
 527         if (!caching_array) {
 528                 pr_err("Unable to allocate table for new pages\n");
 529                 return -ENOMEM;
 530         }
 531
 532         for (i = 0, cpages = 0; i < count; ++i) {
 533                 p = alloc_page(gfp_flags);
 534
 535                 if (!p) {
 536                         pr_err("Unable to get page %u\n", i);
 537
 538                         /* store already allocated pages in the pool after
 539                          * setting the caching state */
 540                         if (cpages) {
 541                                 r = ttm_set_pages_caching(caching_array,
 542                                                           cstate, cpages);
 543                                 if (r)
 544                                         ttm_handle_caching_state_failure(pages,
 545                                                 ttm_flags, cstate,
 546                                                 caching_array, cpages);
 547                         }
 548                         r = -ENOMEM;
 549                         goto out;
 550                 }
 551
 552 #ifdef CONFIG_HIGHMEM
 553                 /* gfp flags of highmem page should never be dma32 so we
 554                  * we should be fine in such case
 555                  */
 556                 if (!PageHighMem(p))
 557 #endif
 558                 {
 559                         caching_array[cpages++] = p;
 560                         if (cpages == max_cpages) {
 561
 562                                 r = ttm_set_pages_caching(caching_array,
 563                                                 cstate, cpages);
 564                                 if (r) {
 565                                         ttm_handle_caching_state_failure(pages,
 566                                                 ttm_flags, cstate,
 567                                                 caching_array, cpages);
 568                                         goto out;
 569                                 }
 570                                 cpages = 0;
 571                         }
 572                 }
 573
 574                 TAILQ_INSERT_HEAD(pages, (struct vm_page *)p, pageq);
 575         }
 576
 577         if (cpages) {
 578                 r = ttm_set_pages_caching(caching_array, cstate, cpages);
 579                 if (r)
 580                         ttm_handle_caching_state_failure(pages,
 581                                         ttm_flags, cstate,
 582                                         caching_array, cpages);
 583         }
 584 out:
 585         kfree(caching_array);
 586
 587         return r;
 588 }
 589
 590 /**
 591  * Fill the given pool if there aren't enough pages and the requested number of
 592  * pages is small.
 593  */
 594 static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool,
 595                 int ttm_flags, enum ttm_caching_state cstate, unsigned count,
 596                 unsigned long *irq_flags)
 597 {
 598         vm_page_t p;
 599         int r;
 600         unsigned cpages = 0;
 601         /**
 602          * Only allow one pool fill operation at a time.
 603          * If pool doesn't have enough pages for the allocation new pages are
 604          * allocated from outside of pool.
 605          */
 606         if (pool->fill_lock)
 607                 return;
 608
 609         pool->fill_lock = true;
 610
 611         /* If allocation request is small and there are not enough
 612          * pages in a pool we fill the pool up first. */
 613         if (count < _manager->options.small
 614                 && count > pool->npages) {
 615                 struct pglist new_pages;
 616                 unsigned alloc_size = _manager->options.alloc_size;
 617
 618                 /**
 619                  * Can't change page caching if in irqsave context. We have to
 620                  * drop the pool->lock.
 621                  */
 622                 spin_unlock_irqrestore(&pool->lock, *irq_flags);
 623
 624                 TAILQ_INIT(&new_pages);
 625                 r = ttm_alloc_new_pages(&new_pages, pool->gfp_flags, ttm_flags,
 626                                 cstate, alloc_size);
 627                 spin_lock_irqsave(&pool->lock, *irq_flags);
 628
 629                 if (!r) {
 630                         TAILQ_CONCAT(&pool->list, &new_pages, pageq);
 631                         ++pool->nrefills;
 632                         pool->npages += alloc_size;
 633                 } else {
 634                         pr_err("Failed to fill pool (%p)\n", pool);
 635                         /* If we have any pages left put them to the pool. */
 636                         TAILQ_FOREACH(p, &pool->list, pageq) {
 637                                 ++cpages;
 638                         }
 639                         TAILQ_CONCAT(&pool->list, &new_pages, pageq);
 640                         pool->npages += cpages;
 641                 }
 642
 643         }
 644         pool->fill_lock = false;
 645 }
 646
 647 /**
 648  * Cut 'count' number of pages from the pool and put them on the return list.
 649  *
 650  * @return count of pages still required to fulfill the request.
 651  */
 652 static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool,
 653                                         struct pglist *pages,
 654                                         int ttm_flags,
 655                                         enum ttm_caching_state cstate,
 656                                         unsigned count)
 657 {
 658         unsigned long irq_flags;
 659         vm_page_t p;
 660         unsigned i;
 661
 662         spin_lock_irqsave(&pool->lock, irq_flags);
 663         ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, &irq_flags);
 664
 665         if (count >= pool->npages) {
 666                 /* take all pages from the pool */
 667                 TAILQ_CONCAT(pages, &pool->list, pageq);
 668                 count -= pool->npages;
 669                 pool->npages = 0;
 670                 goto out;
 671         }
 672         for (i = 0; i < count; i++) {
 673                 p = TAILQ_FIRST(&pool->list);
 674                 TAILQ_REMOVE(&pool->list, p, pageq);
 675                 TAILQ_INSERT_TAIL(pages, p, pageq);
 676         }
 677         pool->npages -= count;
 678         count = 0;
 679 out:
 680         spin_unlock_irqrestore(&pool->lock, irq_flags);
 681         return count;
 682 }
 683
 684 /* Put all pages in pages list to correct pool to wait for reuse */
 685 static void ttm_put_pages(struct page **pages, unsigned npages, int flags,
 686                           enum ttm_caching_state cstate)
 687 {
 688         unsigned long irq_flags;
 689         struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
 690         unsigned i;
 691         struct vm_page *page;
 692
 693         if (pool == NULL) {
 694                 /* No pool for this memory type so free the pages */
 695                 for (i = 0; i < npages; i++) {
 696                         if (pages[i]) {
 697 #if 0
 698                                 if (page_count(pages[i]) != 1)
 699                                         pr_err("Erroneous page count. Leaking pages.\n");
 700 #endif
 701                                 __free_page(pages[i]);
 702                                 pages[i] = NULL;
 703                         }
 704                 }
 705                 return;
 706         }
 707
 708         spin_lock_irqsave(&pool->lock, irq_flags);
 709         for (i = 0; i < npages; i++) {
 710                 if (pages[i]) {
 711                         page = (struct vm_page *)pages[i];
 712                         TAILQ_INSERT_TAIL(&pool->list, page, pageq);
 713                         pages[i] = NULL;
 714                         pool->npages++;
 715                 }
 716         }
 717         /* Check that we don't go over the pool limit */
 718         npages = 0;
 719         if (pool->npages > _manager->options.max_size) {
 720                 npages = pool->npages - _manager->options.max_size;
 721                 /* free at least NUM_PAGES_TO_ALLOC number of pages
 722                  * to reduce calls to set_memory_wb */
 723                 if (npages < NUM_PAGES_TO_ALLOC)
 724                         npages = NUM_PAGES_TO_ALLOC;
 725         }
 726         spin_unlock_irqrestore(&pool->lock, irq_flags);
 727         if (npages)
 728                 ttm_page_pool_free(pool, npages, false);
 729 }
 730
 731 /*
 732  * On success pages list will hold count number of correctly
 733  * cached pages.
 734  */
 735 static int ttm_get_pages(struct page **pages, unsigned npages, int flags,
 736                          enum ttm_caching_state cstate)
 737 {
 738         struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
 739         struct pglist plist;
 740         struct vm_page *p = NULL;
 741         gfp_t gfp_flags = GFP_USER;
 742         unsigned count;
 743         int r;
 744
 745         /* set zero flag for page allocation if required */
 746         if (flags & TTM_PAGE_FLAG_ZERO_ALLOC)
 747                 gfp_flags |= __GFP_ZERO;
 748
 749         /* No pool for cached pages */
 750         if (pool == NULL) {
 751                 if (flags & TTM_PAGE_FLAG_DMA32)
 752                         gfp_flags |= GFP_DMA32;
 753                 else
 754                         gfp_flags |= GFP_HIGHUSER;
 755
 756                 for (r = 0; r < npages; ++r) {
 757                         p = (struct vm_page *)alloc_page(gfp_flags);
 758                         if (!p) {
 759
 760                                 pr_err("Unable to allocate page\n");
 761                                 return -ENOMEM;
 762                         }
 763                         pages[r] = (struct page *)p;
 764                 }
 765                 return 0;
 766         }
 767
 768         /* combine zero flag to pool flags */
 769         gfp_flags |= pool->gfp_flags;
 770
 771         /* First we take pages from the pool */
 772         TAILQ_INIT(&plist);
 773         npages = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages);
 774         count = 0;
 775         TAILQ_FOREACH(p, &plist, pageq) {
 776                 pages[count++] = (struct page *)p;
 777         }
 778
 779         /* clear the pages coming from the pool if requested */
 780         if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) {
 781                 TAILQ_FOREACH(p, &plist, pageq) {
 782                         pmap_zero_page(VM_PAGE_TO_PHYS(p));
 783                 }
 784         }
 785
 786         /* If pool didn't have enough pages allocate new one. */
 787         if (npages > 0) {
 788                 /* ttm_alloc_new_pages doesn't reference pool so we can run
 789                  * multiple requests in parallel.
 790                  **/
 791                 TAILQ_INIT(&plist);
 792                 r = ttm_alloc_new_pages(&plist, gfp_flags, flags, cstate, npages);
 793                 TAILQ_FOREACH(p, &plist, pageq) {
 794                         pages[count++] = (struct page *)p;
 795                 }
 796                 if (r) {
 797                         /* If there is any pages in the list put them back to
 798                          * the pool. */
 799                         pr_err("Failed to allocate extra pages for large request\n");
 800                         ttm_put_pages(pages, count, flags, cstate);
 801                         return r;
 802                 }
 803         }
 804
 805         return 0;
 806 }
 807
 808 static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, gfp_t flags,
 809                 char *name)
 810 {
 811         lockinit(&pool->lock, "ttmpool", 0, LK_CANRECURSE);
 812         pool->fill_lock = false;
 813         TAILQ_INIT(&pool->list);
 814         pool->npages = pool->nfrees = 0;
 815         pool->gfp_flags = flags;
 816         pool->name = name;
 817 }
 818
 819 int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages)
 820 {
 821         int ret;
 822
 823         WARN_ON(_manager);
 824
 825         pr_info("Initializing pool allocator\n");
 826
 827         _manager = kzalloc(sizeof(*_manager), GFP_KERNEL);
 828
 829         ttm_page_pool_init_locked(&_manager->wc_pool, GFP_HIGHUSER, "wc");
 830
 831         ttm_page_pool_init_locked(&_manager->uc_pool, GFP_HIGHUSER, "uc");
 832
 833         ttm_page_pool_init_locked(&_manager->wc_pool_dma32,
 834                                   GFP_USER | GFP_DMA32, "wc dma");
 835
 836         ttm_page_pool_init_locked(&_manager->uc_pool_dma32,
 837                                   GFP_USER | GFP_DMA32, "uc dma");
 838
 839         _manager->options.max_size = max_pages;
 840         _manager->options.small = SMALL_ALLOCATION;
 841         _manager->options.alloc_size = NUM_PAGES_TO_ALLOC;
 842
 843         ret = kobject_init_and_add(&_manager->kobj, &ttm_pool_kobj_type,
 844                                    &glob->kobj, "pool");
 845         if (unlikely(ret != 0)) {
 846                 kobject_put(&_manager->kobj);
 847                 _manager = NULL;
 848                 return ret;
 849         }
 850
 851         ttm_pool_mm_shrink_init(_manager);
 852
 853         return 0;
 854 }
 855
 856 void ttm_page_alloc_fini(void)
 857 {
 858         int i;
 859
 860         pr_info("Finalizing pool allocator\n");
 861         ttm_pool_mm_shrink_fini(_manager);
 862
 863         /* OK to use static buffer since global mutex is no longer used. */
 864         for (i = 0; i < NUM_POOLS; ++i)
 865                 ttm_page_pool_free(&_manager->pools[i], FREE_ALL_PAGES, true);
 866
 867         kobject_put(&_manager->kobj);
 868         _manager = NULL;
 869 }
 870
 871 int ttm_pool_populate(struct ttm_tt *ttm)
 872 {
 873         struct ttm_mem_global *mem_glob = ttm->glob->mem_glob;
 874         unsigned i;
 875         int ret;
 876
 877         if (ttm->state != tt_unpopulated)
 878                 return 0;
 879
 880         for (i = 0; i < ttm->num_pages; ++i) {
 881                 ret = ttm_get_pages(&ttm->pages[i], 1,
 882                                     ttm->page_flags,
 883                                     ttm->caching_state);
 884                 if (ret != 0) {
 885                         ttm_pool_unpopulate(ttm);
 886                         return -ENOMEM;
 887                 }
 888
 889                 ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i],
 890                                                 false, false);
 891                 if (unlikely(ret != 0)) {
 892                         ttm_pool_unpopulate(ttm);
 893                         return -ENOMEM;
 894                 }
 895         }
 896
 897         if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) {
 898                 ret = ttm_tt_swapin(ttm);
 899                 if (unlikely(ret != 0)) {
 900                         ttm_pool_unpopulate(ttm);
 901                         return ret;
 902                 }
 903         }
 904
 905         ttm->state = tt_unbound;
 906         return 0;
 907 }
 908 EXPORT_SYMBOL(ttm_pool_populate);
 909
 910 void ttm_pool_unpopulate(struct ttm_tt *ttm)
 911 {
 912         unsigned i;
 913
 914         for (i = 0; i < ttm->num_pages; ++i) {
 915                 if (ttm->pages[i]) {
 916                         ttm_mem_global_free_page(ttm->glob->mem_glob,
 917                                                  ttm->pages[i]);
 918                         ttm_put_pages(&ttm->pages[i], 1,
 919                                       ttm->page_flags,
 920                                       ttm->caching_state);
 921                 }
 922         }
 923         ttm->state = tt_unpopulated;
 924 }
 925 EXPORT_SYMBOL(ttm_pool_unpopulate);
 926
 927 #if 0
 928 int ttm_page_alloc_debugfs(struct seq_file *m, void *data)
 929 {
 930         struct ttm_page_pool *p;
 931         unsigned i;
 932         char *h[] = {"pool", "refills", "pages freed", "size"};
 933         if (!_manager) {
 934                 seq_printf(m, "No pool allocator running.\n");
 935                 return 0;
 936         }
 937         seq_printf(m, "%6s %12s %13s %8s\n",
 938                         h[0], h[1], h[2], h[3]);
 939         for (i = 0; i < NUM_POOLS; ++i) {
 940                 p = &_manager->pools[i];
 941
 942                 seq_printf(m, "%6s %12ld %13ld %8d\n",
 943                                 p->name, p->nrefills,
 944                                 p->nfrees, p->npages);
 945         }
 946         return 0;
 947 }
 948 #endif
 949 EXPORT_SYMBOL(ttm_page_alloc_debugfs);