Revert "drm/i915: Sync i915_gem_pwrite_ioctl() with Linux 3.11"
[dragonfly.git] / sys / dev / drm / i915 / i915_gem.c
1 /*
2  * Copyright © 2008 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  *
23  * Authors:
24  *    Eric Anholt <eric@anholt.net>
25  *
26  * Copyright (c) 2011 The FreeBSD Foundation
27  * All rights reserved.
28  *
29  * This software was developed by Konstantin Belousov under sponsorship from
30  * the FreeBSD Foundation.
31  *
32  * Redistribution and use in source and binary forms, with or without
33  * modification, are permitted provided that the following conditions
34  * are met:
35  * 1. Redistributions of source code must retain the above copyright
36  *    notice, this list of conditions and the following disclaimer.
37  * 2. Redistributions in binary form must reproduce the above copyright
38  *    notice, this list of conditions and the following disclaimer in the
39  *    documentation and/or other materials provided with the distribution.
40  *
41  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
42  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
43  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
44  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
45  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
46  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
47  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
48  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
49  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
50  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
51  * SUCH DAMAGE.
52  *
53  */
54
55 #include <sys/resourcevar.h>
56 #include <sys/sfbuf.h>
57 #include <machine/md_var.h>
58
59 #include <drm/drmP.h>
60 #include <drm/i915_drm.h>
61 #include "i915_drv.h"
62 #include "i915_trace.h"
63 #include "intel_drv.h"
64 #include <linux/shmem_fs.h>
65 #include <linux/slab.h>
66 #include <linux/pci.h>
67
68 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
69 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
70 static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
71                                                     unsigned alignment,
72                                                     bool map_and_fenceable,
73                                                     bool nonblocking);
74 static int i915_gem_phys_pwrite(struct drm_device *dev,
75                                 struct drm_i915_gem_object *obj,
76                                 struct drm_i915_gem_pwrite *args,
77                                 struct drm_file *file);
78
79 static void i915_gem_write_fence(struct drm_device *dev, int reg,
80                                  struct drm_i915_gem_object *obj);
81 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
82                                          struct drm_i915_fence_reg *fence,
83                                          bool enable);
84
85 static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
86 static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
87
88 static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
89 {
90         if (obj->tiling_mode)
91                 i915_gem_release_mmap(obj);
92
93         /* As we do not have an associated fence register, we will force
94          * a tiling change if we ever need to acquire one.
95          */
96         obj->fence_dirty = false;
97         obj->fence_reg = I915_FENCE_REG_NONE;
98 }
99
100 static bool i915_gem_object_is_inactive(struct drm_i915_gem_object *obj);
101 static void i915_gem_lowmem(void *arg);
102
103 /* some bookkeeping */
104 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
105                                   size_t size)
106 {
107         dev_priv->mm.object_count++;
108         dev_priv->mm.object_memory += size;
109 }
110
111 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
112                                      size_t size)
113 {
114         dev_priv->mm.object_count--;
115         dev_priv->mm.object_memory -= size;
116 }
117
118 static int
119 i915_gem_wait_for_error(struct i915_gpu_error *error)
120 {
121         int ret;
122
123 #define EXIT_COND (!i915_reset_in_progress(error) || \
124                    i915_terminally_wedged(error))
125         if (EXIT_COND)
126                 return 0;
127
128         /*
129          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
130          * userspace. If it takes that long something really bad is going on and
131          * we should simply try to bail out and fail as gracefully as possible.
132          */
133         ret = wait_event_interruptible_timeout(error->reset_queue,
134                                                EXIT_COND,
135                                                10*HZ);
136         if (ret == 0) {
137                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
138                 return -EIO;
139         } else if (ret < 0) {
140                 return ret;
141         }
142 #undef EXIT_COND
143
144         return 0;
145 }
146
147 int i915_mutex_lock_interruptible(struct drm_device *dev)
148 {
149         struct drm_i915_private *dev_priv = dev->dev_private;
150         int ret;
151
152         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
153         if (ret)
154                 return ret;
155
156         ret = lockmgr(&dev->struct_mutex, LK_EXCLUSIVE|LK_SLEEPFAIL);
157         if (ret)
158                 return -EINTR;
159
160         WARN_ON(i915_verify_lists(dev));
161         return 0;
162 }
163
164 static inline bool
165 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
166 {
167         return !obj->active;
168 }
169
170 int
171 i915_gem_init_ioctl(struct drm_device *dev, void *data,
172                     struct drm_file *file)
173 {
174         struct drm_i915_private *dev_priv = dev->dev_private;
175         struct drm_i915_gem_init *args = data;
176
177         if (drm_core_check_feature(dev, DRIVER_MODESET))
178                 return -ENODEV;
179
180         if (args->gtt_start >= args->gtt_end ||
181             (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
182                 return -EINVAL;
183
184         /* GEM with user mode setting was never supported on ilk and later. */
185         if (INTEL_INFO(dev)->gen >= 5)
186                 return -ENODEV;
187
188         mutex_lock(&dev->struct_mutex);
189         i915_gem_setup_global_gtt(dev, args->gtt_start, args->gtt_end,
190                                   args->gtt_end);
191         dev_priv->gtt.mappable_end = args->gtt_end;
192         mutex_unlock(&dev->struct_mutex);
193
194         return 0;
195 }
196
197 int
198 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
199                             struct drm_file *file)
200 {
201         struct drm_i915_private *dev_priv = dev->dev_private;
202         struct drm_i915_gem_get_aperture *args = data;
203         struct drm_i915_gem_object *obj;
204         size_t pinned;
205
206         pinned = 0;
207         mutex_lock(&dev->struct_mutex);
208         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
209                 if (obj->pin_count)
210                         pinned += obj->gtt_space->size;
211         mutex_unlock(&dev->struct_mutex);
212
213         args->aper_size = dev_priv->gtt.total;
214         args->aper_available_size = args->aper_size - pinned;
215
216         return 0;
217 }
218
219 void i915_gem_object_free(struct drm_i915_gem_object *obj)
220 {
221         kfree(obj);
222 }
223
224 static int
225 i915_gem_create(struct drm_file *file,
226                 struct drm_device *dev,
227                 uint64_t size,
228                 uint32_t *handle_p)
229 {
230         struct drm_i915_gem_object *obj;
231         int ret;
232         u32 handle;
233
234         size = roundup(size, PAGE_SIZE);
235         if (size == 0)
236                 return -EINVAL;
237
238         /* Allocate the new object */
239         obj = i915_gem_alloc_object(dev, size);
240         if (obj == NULL)
241                 return -ENOMEM;
242
243         ret = drm_gem_handle_create(file, &obj->base, &handle);
244         if (ret) {
245                 drm_gem_object_release(&obj->base);
246                 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
247                 i915_gem_object_free(obj);
248                 return ret;
249         }
250
251         /* drop reference from allocate - handle holds it now */
252         drm_gem_object_unreference(&obj->base);
253         trace_i915_gem_object_create(obj);
254
255         *handle_p = handle;
256         return 0;
257 }
258
259 int
260 i915_gem_dumb_create(struct drm_file *file,
261                      struct drm_device *dev,
262                      struct drm_mode_create_dumb *args)
263 {
264
265         /* have to work out size/pitch and return them */
266         args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
267         args->size = args->pitch * args->height;
268         return i915_gem_create(file, dev,
269                                args->size, &args->handle);
270 }
271
272 int i915_gem_dumb_destroy(struct drm_file *file,
273                           struct drm_device *dev,
274                           uint32_t handle)
275 {
276
277         return drm_gem_handle_delete(file, handle);
278 }
279
280 /**
281  * Creates a new mm object and returns a handle to it.
282  */
283 int
284 i915_gem_create_ioctl(struct drm_device *dev, void *data,
285                       struct drm_file *file)
286 {
287         struct drm_i915_gem_create *args = data;
288
289         return i915_gem_create(file, dev,
290                                args->size, &args->handle);
291 }
292
293 static inline int
294 __copy_to_user_swizzled(char __user *cpu_vaddr,
295                         const char *gpu_vaddr, int gpu_offset,
296                         int length)
297 {
298         int ret, cpu_offset = 0;
299
300         while (length > 0) {
301                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
302                 int this_length = min(cacheline_end - gpu_offset, length);
303                 int swizzled_gpu_offset = gpu_offset ^ 64;
304
305                 ret = __copy_to_user(cpu_vaddr + cpu_offset,
306                                      gpu_vaddr + swizzled_gpu_offset,
307                                      this_length);
308                 if (ret)
309                         return ret + length;
310
311                 cpu_offset += this_length;
312                 gpu_offset += this_length;
313                 length -= this_length;
314         }
315
316         return 0;
317 }
318
319 static inline int
320 __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
321                           const char __user *cpu_vaddr,
322                           int length)
323 {
324         int ret, cpu_offset = 0;
325
326         while (length > 0) {
327                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
328                 int this_length = min(cacheline_end - gpu_offset, length);
329                 int swizzled_gpu_offset = gpu_offset ^ 64;
330
331                 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
332                                        cpu_vaddr + cpu_offset,
333                                        this_length);
334                 if (ret)
335                         return ret + length;
336
337                 cpu_offset += this_length;
338                 gpu_offset += this_length;
339                 length -= this_length;
340         }
341
342         return 0;
343 }
344
345 /* Per-page copy function for the shmem pread fastpath.
346  * Flushes invalid cachelines before reading the target if
347  * needs_clflush is set. */
348 static int
349 shmem_pread_fast(struct vm_page *page, int shmem_page_offset, int page_length,
350                  char __user *user_data,
351                  bool page_do_bit17_swizzling, bool needs_clflush)
352 {
353         char *vaddr;
354         int ret;
355
356         if (unlikely(page_do_bit17_swizzling))
357                 return -EINVAL;
358
359         vaddr = kmap_atomic(page);
360         if (needs_clflush)
361                 drm_clflush_virt_range(vaddr + shmem_page_offset,
362                                        page_length);
363         ret = __copy_to_user_inatomic(user_data,
364                                       vaddr + shmem_page_offset,
365                                       page_length);
366         kunmap_atomic(vaddr);
367
368         return ret ? -EFAULT : 0;
369 }
370
371 static void
372 shmem_clflush_swizzled_range(char *addr, unsigned long length,
373                              bool swizzled)
374 {
375         if (unlikely(swizzled)) {
376                 unsigned long start = (unsigned long) addr;
377                 unsigned long end = (unsigned long) addr + length;
378
379                 /* For swizzling simply ensure that we always flush both
380                  * channels. Lame, but simple and it works. Swizzled
381                  * pwrite/pread is far from a hotpath - current userspace
382                  * doesn't use it at all. */
383                 start = round_down(start, 128);
384                 end = round_up(end, 128);
385
386                 drm_clflush_virt_range((void *)start, end - start);
387         } else {
388                 drm_clflush_virt_range(addr, length);
389         }
390
391 }
392
393 /* Only difference to the fast-path function is that this can handle bit17
394  * and uses non-atomic copy and kmap functions. */
395 static int
396 shmem_pread_slow(struct vm_page *page, int shmem_page_offset, int page_length,
397                  char __user *user_data,
398                  bool page_do_bit17_swizzling, bool needs_clflush)
399 {
400         char *vaddr;
401         int ret;
402
403         vaddr = kmap(page);
404         if (needs_clflush)
405                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
406                                              page_length,
407                                              page_do_bit17_swizzling);
408
409         if (page_do_bit17_swizzling)
410                 ret = __copy_to_user_swizzled(user_data,
411                                               vaddr, shmem_page_offset,
412                                               page_length);
413         else
414                 ret = __copy_to_user(user_data,
415                                      vaddr + shmem_page_offset,
416                                      page_length);
417         kunmap(page);
418
419         return ret ? - EFAULT : 0;
420 }
421
422 static inline void vm_page_reference(vm_page_t m)
423 {
424         vm_page_flag_set(m, PG_REFERENCED);
425 }
426
427 static int
428 i915_gem_shmem_pread(struct drm_device *dev,
429                      struct drm_i915_gem_object *obj,
430                      struct drm_i915_gem_pread *args,
431                      struct drm_file *file)
432 {
433         char __user *user_data;
434         ssize_t remain;
435         off_t offset;
436         int shmem_page_offset, page_length, ret = 0;
437         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
438         int hit_slowpath = 0;
439         int needs_clflush = 0;
440         int i;
441
442         user_data = (char __user *) (uintptr_t) args->data_ptr;
443         remain = args->size;
444
445         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
446
447         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
448                 /* If we're not in the cpu read domain, set ourself into the gtt
449                  * read domain and manually flush cachelines (if required). This
450                  * optimizes for the case when the gpu will dirty the data
451                  * anyway again before the next pread happens. */
452                 if (obj->cache_level == I915_CACHE_NONE)
453                         needs_clflush = 1;
454                 if (obj->gtt_space) {
455                         ret = i915_gem_object_set_to_gtt_domain(obj, false);
456                         if (ret)
457                                 return ret;
458                 }
459         }
460
461         ret = i915_gem_object_get_pages(obj);
462         if (ret)
463                 return ret;
464
465         i915_gem_object_pin_pages(obj);
466
467         offset = args->offset;
468
469         for (i = 0; i < (obj->base.size >> PAGE_SHIFT); i++) {
470                 struct vm_page *page;
471
472                 if (i < offset >> PAGE_SHIFT)
473                         continue;
474
475                 if (remain <= 0)
476                         break;
477
478                 /* Operation in this page
479                  *
480                  * shmem_page_offset = offset within page in shmem file
481                  * page_length = bytes to copy for this page
482                  */
483                 shmem_page_offset = offset_in_page(offset);
484                 page_length = remain;
485                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
486                         page_length = PAGE_SIZE - shmem_page_offset;
487
488 #ifdef __linux__
489                 page = sg_page(sg);
490                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
491                         (page_to_phys(page) & (1 << 17)) != 0;
492 #else
493                 page = obj->pages[i];
494                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
495                         (VM_PAGE_TO_PHYS(page) & (1 << 17)) != 0;
496 #endif
497
498                 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
499                                        user_data, page_do_bit17_swizzling,
500                                        needs_clflush);
501                 if (ret == 0)
502                         goto next_page;
503
504                 hit_slowpath = 1;
505                 mutex_unlock(&dev->struct_mutex);
506
507 #ifdef __linux__
508                 if (!prefaulted) {
509                         ret = fault_in_multipages_writeable(user_data, remain);
510                         /* Userspace is tricking us, but we've already clobbered
511                          * its pages with the prefault and promised to write the
512                          * data up to the first fault. Hence ignore any errors
513                          * and just continue. */
514                         (void)ret;
515                         prefaulted = 1;
516                 }
517 #endif
518
519                 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
520                                        user_data, page_do_bit17_swizzling,
521                                        needs_clflush);
522
523                 mutex_lock(&dev->struct_mutex);
524
525 next_page:
526 #ifdef __linux__
527                 mark_page_accessed(page);
528 #endif
529
530                 if (ret)
531                         goto out;
532
533                 remain -= page_length;
534                 user_data += page_length;
535                 offset += page_length;
536         }
537
538 out:
539         i915_gem_object_unpin_pages(obj);
540
541         if (hit_slowpath) {
542                 /* Fixup: Kill any reinstated backing storage pages */
543                 if (obj->madv == __I915_MADV_PURGED)
544                         i915_gem_object_truncate(obj);
545         }
546
547         return ret;
548 }
549
550 /**
551  * Reads data from the object referenced by handle.
552  *
553  * On error, the contents of *data are undefined.
554  */
555 int
556 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
557                      struct drm_file *file)
558 {
559         struct drm_i915_gem_pread *args = data;
560         struct drm_i915_gem_object *obj;
561         int ret = 0;
562
563         if (args->size == 0)
564                 return 0;
565
566         ret = i915_mutex_lock_interruptible(dev);
567         if (ret)
568                 return ret;
569
570         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
571         if (&obj->base == NULL) {
572                 ret = -ENOENT;
573                 goto unlock;
574         }
575
576         /* Bounds check source.  */
577         if (args->offset > obj->base.size ||
578             args->size > obj->base.size - args->offset) {
579                 ret = -EINVAL;
580                 goto out;
581         }
582
583         ret = i915_gem_shmem_pread(dev, obj, args, file);
584 out:
585         drm_gem_object_unreference(&obj->base);
586 unlock:
587         mutex_unlock(&dev->struct_mutex);
588         return ret;
589 }
590
591 #if 0
592 /* This is the fast write path which cannot handle
593  * page faults in the source data
594  */
595
596 static inline int
597 fast_user_write(struct io_mapping *mapping,
598                 loff_t page_base, int page_offset,
599                 char __user *user_data,
600                 int length)
601 {
602         void __iomem *vaddr_atomic;
603         void *vaddr;
604         unsigned long unwritten;
605
606         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
607         /* We can use the cpu mem copy function because this is X86. */
608         vaddr = (void __force*)vaddr_atomic + page_offset;
609         unwritten = __copy_from_user_inatomic_nocache(vaddr,
610                                                       user_data, length);
611         io_mapping_unmap_atomic(vaddr_atomic);
612         return unwritten;
613 }
614
615 /**
616  * This is the fast pwrite path, where we copy the data directly from the
617  * user into the GTT, uncached.
618  */
619 static int
620 i915_gem_gtt_pwrite_fast(struct drm_device *dev,
621                          struct drm_i915_gem_object *obj,
622                          struct drm_i915_gem_pwrite *args,
623                          struct drm_file *file)
624 {
625         drm_i915_private_t *dev_priv = dev->dev_private;
626         ssize_t remain;
627         loff_t offset, page_base;
628         char __user *user_data;
629         int page_offset, page_length, ret;
630
631         ret = i915_gem_object_pin(obj, 0, true, true);
632         if (ret)
633                 goto out;
634
635         ret = i915_gem_object_set_to_gtt_domain(obj, true);
636         if (ret)
637                 goto out_unpin;
638
639         ret = i915_gem_object_put_fence(obj);
640         if (ret)
641                 goto out_unpin;
642
643         user_data = to_user_ptr(args->data_ptr);
644         remain = args->size;
645
646         offset = obj->gtt_offset + args->offset;
647
648         while (remain > 0) {
649                 /* Operation in this page
650                  *
651                  * page_base = page offset within aperture
652                  * page_offset = offset within page
653                  * page_length = bytes to copy for this page
654                  */
655                 page_base = offset & PAGE_MASK;
656                 page_offset = offset_in_page(offset);
657                 page_length = remain;
658                 if ((page_offset + remain) > PAGE_SIZE)
659                         page_length = PAGE_SIZE - page_offset;
660
661                 /* If we get a fault while copying data, then (presumably) our
662                  * source page isn't available.  Return the error and we'll
663                  * retry in the slow path.
664                  */
665                 if (fast_user_write(dev_priv->gtt.mappable, page_base,
666                                     page_offset, user_data, page_length)) {
667                         ret = -EFAULT;
668                         goto out_unpin;
669                 }
670
671                 remain -= page_length;
672                 user_data += page_length;
673                 offset += page_length;
674         }
675
676 out_unpin:
677         i915_gem_object_unpin(obj);
678 out:
679         return ret;
680 }
681 #endif
682
683 static int
684 i915_gem_gtt_write(struct drm_device *dev, struct drm_i915_gem_object *obj,
685     uint64_t data_ptr, uint64_t size, uint64_t offset, struct drm_file *file)
686 {
687         vm_offset_t mkva;
688         int ret;
689
690         /*
691          * Pass the unaligned physical address and size to pmap_mapdev_attr()
692          * so it can properly calculate whether an extra page needs to be
693          * mapped or not to cover the requested range.  The function will
694          * add the page offset into the returned mkva for us.
695          */
696         mkva = (vm_offset_t)pmap_mapdev_attr(dev->agp->base + obj->gtt_offset +
697             offset, size, PAT_WRITE_COMBINING);
698         ret = -copyin_nofault((void *)(uintptr_t)data_ptr, (char *)mkva, size);
699         pmap_unmapdev(mkva, size);
700         return ret;
701 }
702
703 #if 0
704 /* Per-page copy function for the shmem pwrite fastpath.
705  * Flushes invalid cachelines before writing to the target if
706  * needs_clflush_before is set and flushes out any written cachelines after
707  * writing if needs_clflush is set. */
708 static int
709 shmem_pwrite_fast(struct vm_page *page, int shmem_page_offset, int page_length,
710                   char __user *user_data,
711                   bool page_do_bit17_swizzling,
712                   bool needs_clflush_before,
713                   bool needs_clflush_after)
714 {
715         char *vaddr;
716         int ret;
717
718         if (unlikely(page_do_bit17_swizzling))
719                 return -EINVAL;
720
721         vaddr = kmap_atomic(page);
722         if (needs_clflush_before)
723                 drm_clflush_virt_range(vaddr + shmem_page_offset,
724                                        page_length);
725         ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset,
726                                                 user_data,
727                                                 page_length);
728         if (needs_clflush_after)
729                 drm_clflush_virt_range(vaddr + shmem_page_offset,
730                                        page_length);
731         kunmap_atomic(vaddr);
732
733         return ret ? -EFAULT : 0;
734 }
735
736 /* Only difference to the fast-path function is that this can handle bit17
737  * and uses non-atomic copy and kmap functions. */
738 static int
739 shmem_pwrite_slow(struct vm_page *page, int shmem_page_offset, int page_length,
740                   char __user *user_data,
741                   bool page_do_bit17_swizzling,
742                   bool needs_clflush_before,
743                   bool needs_clflush_after)
744 {
745         char *vaddr;
746         int ret;
747
748         vaddr = kmap(page);
749         if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
750                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
751                                              page_length,
752                                              page_do_bit17_swizzling);
753         if (page_do_bit17_swizzling)
754                 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
755                                                 user_data,
756                                                 page_length);
757         else
758                 ret = __copy_from_user(vaddr + shmem_page_offset,
759                                        user_data,
760                                        page_length);
761         if (needs_clflush_after)
762                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
763                                              page_length,
764                                              page_do_bit17_swizzling);
765         kunmap(page);
766
767         return ret ? -EFAULT : 0;
768 }
769 #endif
770
771 static int
772 i915_gem_shmem_pwrite(struct drm_device *dev,
773                       struct drm_i915_gem_object *obj,
774                       struct drm_i915_gem_pwrite *args,
775                       struct drm_file *file)
776 {
777         vm_object_t vm_obj;
778         vm_page_t m;
779         struct sf_buf *sf;
780         vm_offset_t mkva;
781         vm_pindex_t obj_pi;
782         int cnt, do_bit17_swizzling, length, obj_po, ret, swizzled_po;
783
784         do_bit17_swizzling = 0;
785
786         obj->dirty = 1;
787         vm_obj = obj->base.vm_obj;
788         ret = 0;
789
790         VM_OBJECT_LOCK(vm_obj);
791         vm_object_pip_add(vm_obj, 1);
792         while (args->size > 0) {
793                 obj_pi = OFF_TO_IDX(args->offset);
794                 obj_po = args->offset & PAGE_MASK;
795
796                 m = shmem_read_mapping_page(vm_obj, obj_pi);
797                 VM_OBJECT_UNLOCK(vm_obj);
798
799                 sf = sf_buf_alloc(m);
800                 mkva = sf_buf_kva(sf);
801                 length = min(args->size, PAGE_SIZE - obj_po);
802                 while (length > 0) {
803                         if (do_bit17_swizzling &&
804                             (VM_PAGE_TO_PHYS(m) & (1 << 17)) != 0) {
805                                 cnt = roundup2(obj_po + 1, 64);
806                                 cnt = min(cnt - obj_po, length);
807                                 swizzled_po = obj_po ^ 64;
808                         } else {
809                                 cnt = length;
810                                 swizzled_po = obj_po;
811                         }
812                         ret = -copyin_nofault(
813                             (void *)(uintptr_t)args->data_ptr,
814                             (char *)mkva + swizzled_po, cnt);
815                         if (ret != 0)
816                                 break;
817                         args->data_ptr += cnt;
818                         args->size -= cnt;
819                         length -= cnt;
820                         args->offset += cnt;
821                         obj_po += cnt;
822                 }
823                 sf_buf_free(sf);
824                 VM_OBJECT_LOCK(vm_obj);
825                 vm_page_dirty(m);
826                 vm_page_reference(m);
827                 vm_page_busy_wait(m, FALSE, "i915gem");
828                 vm_page_unwire(m, 1);
829                 vm_page_wakeup(m);
830
831                 if (ret != 0)
832                         break;
833         }
834         vm_object_pip_wakeup(vm_obj);
835         VM_OBJECT_UNLOCK(vm_obj);
836
837         return (ret);
838 }
839
840 /**
841  * Writes data to the object referenced by handle.
842  *
843  * On error, the contents of the buffer that were to be modified are undefined.
844  */
845 int
846 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
847                       struct drm_file *file)
848 {
849         struct drm_i915_gem_pwrite *args = data;
850         struct drm_i915_gem_object *obj;
851         vm_page_t *ma;
852         vm_offset_t start, end;
853         int npages, ret;
854
855         if (args->size == 0)
856                 return 0;
857
858         start = trunc_page(args->data_ptr);
859         end = round_page(args->data_ptr + args->size);
860         npages = howmany(end - start, PAGE_SIZE);
861         ma = kmalloc(npages * sizeof(vm_page_t), M_DRM, M_WAITOK |
862             M_ZERO);
863         npages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map,
864             (vm_offset_t)args->data_ptr, args->size,
865             VM_PROT_READ, ma, npages);
866         if (npages == -1) {
867                 ret = -EFAULT;
868                 goto free_ma;
869         }
870
871         ret = i915_mutex_lock_interruptible(dev);
872         if (ret != 0)
873                 goto unlocked;
874
875         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
876         if (&obj->base == NULL) {
877                 ret = -ENOENT;
878                 goto unlock;
879         }
880
881         /* Bounds check destination. */
882         if (args->offset > obj->base.size ||
883             args->size > obj->base.size - args->offset) {
884                 ret = -EINVAL;
885                 goto out;
886         }
887
888         if (obj->phys_obj) {
889                 ret = i915_gem_phys_pwrite(dev, obj, args, file);
890         } else if (obj->gtt_space &&
891                     obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
892                 ret = i915_gem_object_pin(obj, 0, true, false);
893                 if (ret != 0)
894                         goto out;
895                 ret = i915_gem_object_set_to_gtt_domain(obj, true);
896                 if (ret != 0)
897                         goto out_unpin;
898                 ret = i915_gem_object_put_fence(obj);
899                 if (ret != 0)
900                         goto out_unpin;
901                 ret = i915_gem_gtt_write(dev, obj, args->data_ptr, args->size,
902                     args->offset, file);
903 out_unpin:
904                 i915_gem_object_unpin(obj);
905         } else {
906                 ret = i915_gem_object_set_to_cpu_domain(obj, true);
907                 if (ret != 0)
908                         goto out;
909                 ret = i915_gem_shmem_pwrite(dev, obj, args, file);
910         }
911 out:
912         drm_gem_object_unreference(&obj->base);
913 unlock:
914         mutex_unlock(&dev->struct_mutex);
915 unlocked:
916         vm_page_unhold_pages(ma, npages);
917 free_ma:
918         drm_free(ma, M_DRM);
919         return ret;
920 }
921
922 int
923 i915_gem_check_wedge(struct i915_gpu_error *error,
924                      bool interruptible)
925 {
926         if (i915_reset_in_progress(error)) {
927                 /* Non-interruptible callers can't handle -EAGAIN, hence return
928                  * -EIO unconditionally for these. */
929                 if (!interruptible)
930                         return -EIO;
931
932                 /* Recovery complete, but the reset failed ... */
933                 if (i915_terminally_wedged(error))
934                         return -EIO;
935
936                 return -EAGAIN;
937         }
938
939         return 0;
940 }
941
942 /*
943  * Compare seqno against outstanding lazy request. Emit a request if they are
944  * equal.
945  */
946 static int
947 i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
948 {
949         int ret;
950
951         DRM_LOCK_ASSERT(ring->dev);
952
953         ret = 0;
954         if (seqno == ring->outstanding_lazy_request)
955                 ret = i915_add_request(ring, NULL);
956
957         return ret;
958 }
959
960 /**
961  * __wait_seqno - wait until execution of seqno has finished
962  * @ring: the ring expected to report seqno
963  * @seqno: duh!
964  * @reset_counter: reset sequence associated with the given seqno
965  * @interruptible: do an interruptible wait (normally yes)
966  * @timeout: in - how long to wait (NULL forever); out - how much time remaining
967  *
968  * Note: It is of utmost importance that the passed in seqno and reset_counter
969  * values have been read by the caller in an smp safe manner. Where read-side
970  * locks are involved, it is sufficient to read the reset_counter before
971  * unlocking the lock that protects the seqno. For lockless tricks, the
972  * reset_counter _must_ be read before, and an appropriate smp_rmb must be
973  * inserted.
974  *
975  * Returns 0 if the seqno was found within the alloted time. Else returns the
976  * errno with remaining time filled in timeout argument.
977  */
978 static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
979                         unsigned reset_counter,
980                         bool interruptible, struct timespec *timeout)
981 {
982         drm_i915_private_t *dev_priv = ring->dev->dev_private;
983         struct timespec before, now, wait_time={1,0};
984         unsigned long timeout_jiffies;
985         long end;
986         bool wait_forever = true;
987         int ret;
988
989         if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
990                 return 0;
991
992         if (timeout != NULL) {
993                 wait_time = *timeout;
994                 wait_forever = false;
995         }
996
997         timeout_jiffies = timespec_to_jiffies_timeout(&wait_time);
998
999         if (WARN_ON(!ring->irq_get(ring)))
1000                 return -ENODEV;
1001
1002         /* Record current time in case interrupted by signal, or wedged * */
1003         getrawmonotonic(&before);
1004
1005 #define EXIT_COND \
1006         (i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \
1007          i915_reset_in_progress(&dev_priv->gpu_error) || \
1008          reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
1009         do {
1010                 if (interruptible)
1011                         end = wait_event_interruptible_timeout(ring->irq_queue,
1012                                                                EXIT_COND,
1013                                                                timeout_jiffies);
1014                 else
1015                         end = wait_event_timeout(ring->irq_queue, EXIT_COND,
1016                                                  timeout_jiffies);
1017
1018                 /* We need to check whether any gpu reset happened in between
1019                  * the caller grabbing the seqno and now ... */
1020                 if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
1021                         end = -EAGAIN;
1022
1023                 /* ... but upgrade the -EGAIN to an -EIO if the gpu is truely
1024                  * gone. */
1025                 ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1026                 if (ret)
1027                         end = ret;
1028         } while (end == 0 && wait_forever);
1029
1030         getrawmonotonic(&now);
1031
1032         ring->irq_put(ring);
1033 #undef EXIT_COND
1034
1035         if (timeout) {
1036                 struct timespec sleep_time = timespec_sub(now, before);
1037                 *timeout = timespec_sub(*timeout, sleep_time);
1038                 if (!timespec_valid(timeout)) /* i.e. negative time remains */
1039                         set_normalized_timespec(timeout, 0, 0);
1040         }
1041
1042         switch (end) {
1043         case -EIO:
1044         case -EAGAIN: /* Wedged */
1045         case -ERESTARTSYS: /* Signal */
1046                 return (int)end;
1047         case 0: /* Timeout */
1048                 return -ETIMEDOUT;      /* -ETIME on Linux */
1049         default: /* Completed */
1050                 WARN_ON(end < 0); /* We're not aware of other errors */
1051                 return 0;
1052         }
1053 }
1054
1055 /**
1056  * Waits for a sequence number to be signaled, and cleans up the
1057  * request and object lists appropriately for that event.
1058  */
1059 int
1060 i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
1061 {
1062         struct drm_device *dev = ring->dev;
1063         struct drm_i915_private *dev_priv = dev->dev_private;
1064         bool interruptible = dev_priv->mm.interruptible;
1065         int ret;
1066
1067         DRM_LOCK_ASSERT(dev);
1068         BUG_ON(seqno == 0);
1069
1070         ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1071         if (ret)
1072                 return ret;
1073
1074         ret = i915_gem_check_olr(ring, seqno);
1075         if (ret)
1076                 return ret;
1077
1078         return __wait_seqno(ring, seqno,
1079                             atomic_read(&dev_priv->gpu_error.reset_counter),
1080                             interruptible, NULL);
1081 }
1082
1083 static int
1084 i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj,
1085                                      struct intel_ring_buffer *ring)
1086 {
1087         i915_gem_retire_requests_ring(ring);
1088
1089         /* Manually manage the write flush as we may have not yet
1090          * retired the buffer.
1091          *
1092          * Note that the last_write_seqno is always the earlier of
1093          * the two (read/write) seqno, so if we haved successfully waited,
1094          * we know we have passed the last write.
1095          */
1096         obj->last_write_seqno = 0;
1097         obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
1098
1099         return 0;
1100 }
1101
1102 /**
1103  * Ensures that all rendering to the object has completed and the object is
1104  * safe to unbind from the GTT or access from the CPU.
1105  */
1106 static __must_check int
1107 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
1108                                bool readonly)
1109 {
1110         struct intel_ring_buffer *ring = obj->ring;
1111         u32 seqno;
1112         int ret;
1113
1114         seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
1115         if (seqno == 0)
1116                 return 0;
1117
1118         ret = i915_wait_seqno(ring, seqno);
1119         if (ret)
1120                 return ret;
1121
1122         return i915_gem_object_wait_rendering__tail(obj, ring);
1123 }
1124
1125 /* A nonblocking variant of the above wait. This is a highly dangerous routine
1126  * as the object state may change during this call.
1127  */
1128 static __must_check int
1129 i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
1130                                             bool readonly)
1131 {
1132         struct drm_device *dev = obj->base.dev;
1133         struct drm_i915_private *dev_priv = dev->dev_private;
1134         struct intel_ring_buffer *ring = obj->ring;
1135         unsigned reset_counter;
1136         u32 seqno;
1137         int ret;
1138
1139         DRM_LOCK_ASSERT(dev);
1140         BUG_ON(!dev_priv->mm.interruptible);
1141
1142         seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
1143         if (seqno == 0)
1144                 return 0;
1145
1146         ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
1147         if (ret)
1148                 return ret;
1149
1150         ret = i915_gem_check_olr(ring, seqno);
1151         if (ret)
1152                 return ret;
1153
1154         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
1155         mutex_unlock(&dev->struct_mutex);
1156         ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
1157         mutex_lock(&dev->struct_mutex);
1158         if (ret)
1159                 return ret;
1160
1161         return i915_gem_object_wait_rendering__tail(obj, ring);
1162 }
1163
1164 /**
1165  * Called when user space prepares to use an object with the CPU, either
1166  * through the mmap ioctl's mapping or a GTT mapping.
1167  */
1168 int
1169 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1170                           struct drm_file *file)
1171 {
1172         struct drm_i915_gem_set_domain *args = data;
1173         struct drm_i915_gem_object *obj;
1174         uint32_t read_domains = args->read_domains;
1175         uint32_t write_domain = args->write_domain;
1176         int ret;
1177
1178         /* Only handle setting domains to types used by the CPU. */
1179         if (write_domain & I915_GEM_GPU_DOMAINS)
1180                 return -EINVAL;
1181
1182         if (read_domains & I915_GEM_GPU_DOMAINS)
1183                 return -EINVAL;
1184
1185         /* Having something in the write domain implies it's in the read
1186          * domain, and only that read domain.  Enforce that in the request.
1187          */
1188         if (write_domain != 0 && read_domains != write_domain)
1189                 return -EINVAL;
1190
1191         ret = i915_mutex_lock_interruptible(dev);
1192         if (ret)
1193                 return ret;
1194
1195         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1196         if (&obj->base == NULL) {
1197                 ret = -ENOENT;
1198                 goto unlock;
1199         }
1200
1201         /* Try to flush the object off the GPU without holding the lock.
1202          * We will repeat the flush holding the lock in the normal manner
1203          * to catch cases where we are gazumped.
1204          */
1205         ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain);
1206         if (ret)
1207                 goto unref;
1208
1209         if (read_domains & I915_GEM_DOMAIN_GTT) {
1210                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1211
1212                 /* Silently promote "you're not bound, there was nothing to do"
1213                  * to success, since the client was just asking us to
1214                  * make sure everything was done.
1215                  */
1216                 if (ret == -EINVAL)
1217                         ret = 0;
1218         } else {
1219                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1220         }
1221
1222 unref:
1223         drm_gem_object_unreference(&obj->base);
1224 unlock:
1225         mutex_unlock(&dev->struct_mutex);
1226         return ret;
1227 }
1228
1229 /**
1230  * Called when user space has done writes to this buffer
1231  */
1232 int
1233 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1234                          struct drm_file *file)
1235 {
1236         struct drm_i915_gem_sw_finish *args = data;
1237         struct drm_i915_gem_object *obj;
1238         int ret = 0;
1239
1240         ret = i915_mutex_lock_interruptible(dev);
1241         if (ret)
1242                 return ret;
1243         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1244         if (&obj->base == NULL) {
1245                 ret = -ENOENT;
1246                 goto unlock;
1247         }
1248
1249         /* Pinned buffers may be scanout, so flush the cache */
1250         if (obj->pin_count)
1251                 i915_gem_object_flush_cpu_write_domain(obj);
1252
1253         drm_gem_object_unreference(&obj->base);
1254 unlock:
1255         mutex_unlock(&dev->struct_mutex);
1256         return ret;
1257 }
1258
1259 /**
1260  * Maps the contents of an object, returning the address it is mapped
1261  * into.
1262  *
1263  * While the mapping holds a reference on the contents of the object, it doesn't
1264  * imply a ref on the object itself.
1265  */
1266 int
1267 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1268                     struct drm_file *file)
1269 {
1270         struct drm_i915_gem_mmap *args = data;
1271         struct drm_gem_object *obj;
1272         struct proc *p = curproc;
1273         vm_map_t map = &p->p_vmspace->vm_map;
1274         vm_offset_t addr;
1275         vm_size_t size;
1276         int error = 0, rv;
1277
1278         obj = drm_gem_object_lookup(dev, file, args->handle);
1279         if (obj == NULL)
1280                 return -ENOENT;
1281
1282         if (args->size == 0)
1283                 goto out;
1284
1285         size = round_page(args->size);
1286         if (map->size + size > p->p_rlimit[RLIMIT_VMEM].rlim_cur) {
1287                 error = ENOMEM;
1288                 goto out;
1289         }
1290
1291         addr = 0;
1292         vm_object_hold(obj->vm_obj);
1293         vm_object_reference_locked(obj->vm_obj);
1294         vm_object_drop(obj->vm_obj);
1295         rv = vm_map_find(map, obj->vm_obj, NULL,
1296                          args->offset, &addr, args->size,
1297                          PAGE_SIZE, /* align */
1298                          TRUE, /* fitit */
1299                          VM_MAPTYPE_NORMAL, /* maptype */
1300                          VM_PROT_READ | VM_PROT_WRITE, /* prot */
1301                          VM_PROT_READ | VM_PROT_WRITE, /* max */
1302                          MAP_SHARED /* cow */);
1303         if (rv != KERN_SUCCESS) {
1304                 vm_object_deallocate(obj->vm_obj);
1305                 error = -vm_mmap_to_errno(rv);
1306         } else {
1307                 args->addr_ptr = (uint64_t)addr;
1308         }
1309 out:
1310         drm_gem_object_unreference(obj);
1311         return (error);
1312 }
1313
1314 int i915_intr_pf;
1315
1316 /**
1317  * i915_gem_fault - fault a page into the GTT
1318  * vma: VMA in question
1319  * vmf: fault info
1320  *
1321  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1322  * from userspace.  The fault handler takes care of binding the object to
1323  * the GTT (if needed), allocating and programming a fence register (again,
1324  * only if needed based on whether the old reg is still valid or the object
1325  * is tiled) and inserting a new PTE into the faulting process.
1326  *
1327  * Note that the faulting process may involve evicting existing objects
1328  * from the GTT and/or fence registers to make room.  So performance may
1329  * suffer if the GTT working set is large or there are few fence registers
1330  * left.
1331  */
1332 int
1333 i915_gem_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot,
1334     vm_page_t *mres)
1335 {
1336         struct drm_gem_object *gem_obj;
1337         struct drm_i915_gem_object *obj;
1338         struct drm_device *dev;
1339         drm_i915_private_t *dev_priv;
1340         vm_page_t m, oldm;
1341         int cause, ret;
1342         bool write;
1343
1344         gem_obj = vm_obj->handle;
1345         obj = to_intel_bo(gem_obj);
1346         dev = obj->base.dev;
1347         dev_priv = dev->dev_private;
1348 #if 0
1349         write = (prot & VM_PROT_WRITE) != 0;
1350 #else
1351         write = true;
1352 #endif
1353         vm_object_pip_add(vm_obj, 1);
1354
1355         /*
1356          * Remove the placeholder page inserted by vm_fault() from the
1357          * object before dropping the object lock. If
1358          * i915_gem_release_mmap() is active in parallel on this gem
1359          * object, then it owns the drm device sx and might find the
1360          * placeholder already. Then, since the page is busy,
1361          * i915_gem_release_mmap() sleeps waiting for the busy state
1362          * of the page cleared. We will be not able to acquire drm
1363          * device lock until i915_gem_release_mmap() is able to make a
1364          * progress.
1365          */
1366         if (*mres != NULL) {
1367                 oldm = *mres;
1368                 vm_page_remove(oldm);
1369                 *mres = NULL;
1370         } else
1371                 oldm = NULL;
1372 retry:
1373         VM_OBJECT_UNLOCK(vm_obj);
1374 unlocked_vmobj:
1375         cause = ret = 0;
1376         m = NULL;
1377
1378         if (i915_intr_pf) {
1379                 ret = i915_mutex_lock_interruptible(dev);
1380                 if (ret != 0) {
1381                         cause = 10;
1382                         goto out;
1383                 }
1384         } else
1385                 mutex_lock(&dev->struct_mutex);
1386
1387         /*
1388          * Since the object lock was dropped, other thread might have
1389          * faulted on the same GTT address and instantiated the
1390          * mapping for the page.  Recheck.
1391          */
1392         VM_OBJECT_LOCK(vm_obj);
1393         m = vm_page_lookup(vm_obj, OFF_TO_IDX(offset));
1394         if (m != NULL) {
1395                 if ((m->flags & PG_BUSY) != 0) {
1396                         mutex_unlock(&dev->struct_mutex);
1397 #if 0 /* XXX */
1398                         vm_page_sleep(m, "915pee");
1399 #endif
1400                         goto retry;
1401                 }
1402                 goto have_page;
1403         } else
1404                 VM_OBJECT_UNLOCK(vm_obj);
1405
1406         /* Access to snoopable pages through the GTT is incoherent. */
1407         if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
1408                 ret = -EINVAL;
1409                 goto unlock;
1410         }
1411
1412         /* Now bind it into the GTT if needed */
1413         if (!obj->map_and_fenceable) {
1414                 ret = i915_gem_object_unbind(obj);
1415                 if (ret != 0) {
1416                         cause = 20;
1417                         goto unlock;
1418                 }
1419         }
1420         if (!obj->gtt_space) {
1421                 ret = i915_gem_object_bind_to_gtt(obj, 0, true, false);
1422                 if (ret != 0) {
1423                         cause = 30;
1424                         goto unlock;
1425                 }
1426
1427                 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1428                 if (ret != 0) {
1429                         cause = 40;
1430                         goto unlock;
1431                 }
1432         }
1433
1434         if (obj->tiling_mode == I915_TILING_NONE)
1435                 ret = i915_gem_object_put_fence(obj);
1436         else
1437                 ret = i915_gem_object_get_fence(obj);
1438         if (ret != 0) {
1439                 cause = 50;
1440                 goto unlock;
1441         }
1442
1443         if (i915_gem_object_is_inactive(obj))
1444                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1445
1446         obj->fault_mappable = true;
1447         VM_OBJECT_LOCK(vm_obj);
1448         m = vm_phys_fictitious_to_vm_page(dev->agp->base + obj->gtt_offset +
1449             offset);
1450         if (m == NULL) {
1451                 cause = 60;
1452                 ret = -EFAULT;
1453                 goto unlock;
1454         }
1455         KASSERT((m->flags & PG_FICTITIOUS) != 0,
1456             ("not fictitious %p", m));
1457         KASSERT(m->wire_count == 1, ("wire_count not 1 %p", m));
1458
1459         if ((m->flags & PG_BUSY) != 0) {
1460                 mutex_unlock(&dev->struct_mutex);
1461 #if 0 /* XXX */
1462                 vm_page_sleep(m, "915pbs");
1463 #endif
1464                 goto retry;
1465         }
1466         m->valid = VM_PAGE_BITS_ALL;
1467         vm_page_insert(m, vm_obj, OFF_TO_IDX(offset));
1468 have_page:
1469         *mres = m;
1470         vm_page_busy_try(m, false);
1471
1472         mutex_unlock(&dev->struct_mutex);
1473         if (oldm != NULL) {
1474                 vm_page_free(oldm);
1475         }
1476         vm_object_pip_wakeup(vm_obj);
1477         return (VM_PAGER_OK);
1478
1479 unlock:
1480         mutex_unlock(&dev->struct_mutex);
1481 out:
1482         KASSERT(ret != 0, ("i915_gem_pager_fault: wrong return"));
1483         if (ret == -EAGAIN || ret == -EIO || ret == -EINTR) {
1484                 goto unlocked_vmobj;
1485         }
1486         VM_OBJECT_LOCK(vm_obj);
1487         vm_object_pip_wakeup(vm_obj);
1488         return (VM_PAGER_ERROR);
1489 }
1490
1491 /**
1492  * i915_gem_release_mmap - remove physical page mappings
1493  * @obj: obj in question
1494  *
1495  * Preserve the reservation of the mmapping with the DRM core code, but
1496  * relinquish ownership of the pages back to the system.
1497  *
1498  * It is vital that we remove the page mapping if we have mapped a tiled
1499  * object through the GTT and then lose the fence register due to
1500  * resource pressure. Similarly if the object has been moved out of the
1501  * aperture, than pages mapped into userspace must be revoked. Removing the
1502  * mapping will then trigger a page fault on the next user access, allowing
1503  * fixup by i915_gem_fault().
1504  */
1505 void
1506 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1507 {
1508         vm_object_t devobj;
1509         vm_page_t m;
1510         int i, page_count;
1511
1512         if (!obj->fault_mappable)
1513                 return;
1514
1515         devobj = cdev_pager_lookup(obj);
1516         if (devobj != NULL) {
1517                 page_count = OFF_TO_IDX(obj->base.size);
1518
1519                 VM_OBJECT_LOCK(devobj);
1520                 for (i = 0; i < page_count; i++) {
1521                         m = vm_page_lookup_busy_wait(devobj, i, TRUE, "915unm");
1522                         if (m == NULL)
1523                                 continue;
1524                         cdev_pager_free_page(devobj, m);
1525                 }
1526                 VM_OBJECT_UNLOCK(devobj);
1527                 vm_object_deallocate(devobj);
1528         }
1529
1530         obj->fault_mappable = false;
1531 }
1532
1533 uint32_t
1534 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1535 {
1536         uint32_t gtt_size;
1537
1538         if (INTEL_INFO(dev)->gen >= 4 ||
1539             tiling_mode == I915_TILING_NONE)
1540                 return size;
1541
1542         /* Previous chips need a power-of-two fence region when tiling */
1543         if (INTEL_INFO(dev)->gen == 3)
1544                 gtt_size = 1024*1024;
1545         else
1546                 gtt_size = 512*1024;
1547
1548         while (gtt_size < size)
1549                 gtt_size <<= 1;
1550
1551         return gtt_size;
1552 }
1553
1554 /**
1555  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1556  * @obj: object to check
1557  *
1558  * Return the required GTT alignment for an object, taking into account
1559  * potential fence register mapping.
1560  */
1561 uint32_t
1562 i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
1563                            int tiling_mode, bool fenced)
1564 {
1565
1566         /*
1567          * Minimum alignment is 4k (GTT page size), but might be greater
1568          * if a fence register is needed for the object.
1569          */
1570         if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
1571             tiling_mode == I915_TILING_NONE)
1572                 return 4096;
1573
1574         /*
1575          * Previous chips need to be aligned to the size of the smallest
1576          * fence register that can contain the object.
1577          */
1578         return i915_gem_get_gtt_size(dev, size, tiling_mode);
1579 }
1580
1581 int
1582 i915_gem_mmap_gtt(struct drm_file *file,
1583                   struct drm_device *dev,
1584                   uint32_t handle,
1585                   uint64_t *offset)
1586 {
1587         struct drm_i915_private *dev_priv = dev->dev_private;
1588         struct drm_i915_gem_object *obj;
1589         int ret;
1590
1591         ret = i915_mutex_lock_interruptible(dev);
1592         if (ret)
1593                 return ret;
1594
1595         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
1596         if (&obj->base == NULL) {
1597                 ret = -ENOENT;
1598                 goto unlock;
1599         }
1600
1601         if (obj->base.size > dev_priv->gtt.mappable_end) {
1602                 ret = -E2BIG;
1603                 goto out;
1604         }
1605
1606         if (obj->madv != I915_MADV_WILLNEED) {
1607                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1608                 ret = -EINVAL;
1609                 goto out;
1610         }
1611
1612         ret = drm_gem_create_mmap_offset(&obj->base);
1613         if (ret)
1614                 goto out;
1615
1616         *offset = DRM_GEM_MAPPING_OFF(obj->base.map_list.key) |
1617             DRM_GEM_MAPPING_KEY;
1618 out:
1619         drm_gem_object_unreference(&obj->base);
1620 unlock:
1621         mutex_unlock(&dev->struct_mutex);
1622         return ret;
1623 }
1624
1625 /**
1626  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1627  * @dev: DRM device
1628  * @data: GTT mapping ioctl data
1629  * @file: GEM object info
1630  *
1631  * Simply returns the fake offset to userspace so it can mmap it.
1632  * The mmap call will end up in drm_gem_mmap(), which will set things
1633  * up so we can get faults in the handler above.
1634  *
1635  * The fault handler will take care of binding the object into the GTT
1636  * (since it may have been evicted to make room for something), allocating
1637  * a fence register, and mapping the appropriate aperture address into
1638  * userspace.
1639  */
1640 int
1641 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1642                         struct drm_file *file)
1643 {
1644         struct drm_i915_gem_mmap_gtt *args = data;
1645
1646         return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
1647 }
1648
1649 /* Immediately discard the backing storage */
1650 static void
1651 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
1652 {
1653         vm_object_t vm_obj;
1654
1655         vm_obj = obj->base.vm_obj;
1656         VM_OBJECT_LOCK(vm_obj);
1657         vm_object_page_remove(vm_obj, 0, 0, false);
1658         VM_OBJECT_UNLOCK(vm_obj);
1659         obj->madv = __I915_MADV_PURGED;
1660 }
1661
1662 static inline int
1663 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
1664 {
1665         return obj->madv == I915_MADV_DONTNEED;
1666 }
1667
1668 static void
1669 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
1670 {
1671         vm_page_t m;
1672         int page_count, i;
1673
1674         BUG_ON(obj->madv == __I915_MADV_PURGED);
1675
1676         if (obj->tiling_mode != I915_TILING_NONE)
1677                 i915_gem_object_save_bit_17_swizzle(obj);
1678         if (obj->madv == I915_MADV_DONTNEED)
1679                 obj->dirty = 0;
1680         page_count = obj->base.size / PAGE_SIZE;
1681         VM_OBJECT_LOCK(obj->base.vm_obj);
1682 #if GEM_PARANOID_CHECK_GTT
1683         i915_gem_assert_pages_not_mapped(obj->base.dev, obj->pages, page_count);
1684 #endif
1685         for (i = 0; i < page_count; i++) {
1686                 m = obj->pages[i];
1687                 if (obj->dirty)
1688                         vm_page_dirty(m);
1689                 if (obj->madv == I915_MADV_WILLNEED)
1690                         vm_page_reference(m);
1691                 vm_page_busy_wait(obj->pages[i], FALSE, "i915gem");
1692                 vm_page_unwire(obj->pages[i], 1);
1693                 vm_page_wakeup(obj->pages[i]);
1694         }
1695         VM_OBJECT_UNLOCK(obj->base.vm_obj);
1696         obj->dirty = 0;
1697         drm_free(obj->pages, M_DRM);
1698         obj->pages = NULL;
1699 }
1700
1701 int
1702 i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
1703 {
1704         const struct drm_i915_gem_object_ops *ops = obj->ops;
1705
1706         if (obj->pages == NULL)
1707                 return 0;
1708
1709         BUG_ON(obj->gtt_space);
1710
1711         if (obj->pages_pin_count)
1712                 return -EBUSY;
1713
1714         /* ->put_pages might need to allocate memory for the bit17 swizzle
1715          * array, hence protect them from being reaped by removing them from gtt
1716          * lists early. */
1717         list_del(&obj->global_list);
1718
1719         ops->put_pages(obj);
1720         obj->pages = NULL;
1721
1722         if (i915_gem_object_is_purgeable(obj))
1723                 i915_gem_object_truncate(obj);
1724
1725         return 0;
1726 }
1727
1728 static int
1729 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
1730 {
1731         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1732         struct drm_device *dev;
1733         vm_object_t vm_obj;
1734         int page_count, i, j;
1735         struct vm_page *page;
1736
1737         dev = obj->base.dev;
1738         KASSERT(obj->pages == NULL, ("Obj already has pages"));
1739         page_count = obj->base.size / PAGE_SIZE;
1740         obj->pages = kmalloc(page_count * sizeof(vm_page_t), M_DRM,
1741             M_WAITOK);
1742
1743         vm_obj = obj->base.vm_obj;
1744         VM_OBJECT_LOCK(vm_obj);
1745
1746         for (i = 0; i < page_count; i++) {
1747                 page = shmem_read_mapping_page(vm_obj, i);
1748                 if (IS_ERR(page)) {
1749                         i915_gem_purge(dev_priv, page_count);
1750                         goto err_pages;
1751                 }
1752
1753                 obj->pages[i] = page;
1754         }
1755
1756         VM_OBJECT_UNLOCK(vm_obj);
1757         if (i915_gem_object_needs_bit17_swizzle(obj))
1758                 i915_gem_object_do_bit_17_swizzle(obj);
1759
1760         return 0;
1761
1762 err_pages:
1763         for (j = 0; j < i; j++) {
1764                 page = obj->pages[j];
1765                 vm_page_busy_wait(page, FALSE, "i915gem");
1766                 vm_page_unwire(page, 0);
1767                 vm_page_wakeup(page);
1768         }
1769         VM_OBJECT_UNLOCK(vm_obj);
1770         drm_free(obj->pages, M_DRM);
1771         obj->pages = NULL;
1772         return (-EIO);
1773 }
1774
1775 /* Ensure that the associated pages are gathered from the backing storage
1776  * and pinned into our object. i915_gem_object_get_pages() may be called
1777  * multiple times before they are released by a single call to
1778  * i915_gem_object_put_pages() - once the pages are no longer referenced
1779  * either as a result of memory pressure (reaping pages under the shrinker)
1780  * or as the object is itself released.
1781  */
1782 int
1783 i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
1784 {
1785         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1786         const struct drm_i915_gem_object_ops *ops = obj->ops;
1787         int ret;
1788
1789         if (obj->pages)
1790                 return 0;
1791
1792         if (obj->madv != I915_MADV_WILLNEED) {
1793                 DRM_ERROR("Attempting to obtain a purgeable object\n");
1794                 return -EINVAL;
1795         }
1796
1797         BUG_ON(obj->pages_pin_count);
1798
1799         ret = ops->get_pages(obj);
1800         if (ret)
1801                 return ret;
1802
1803         list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
1804         return 0;
1805 }
1806
1807 void
1808 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
1809                                struct intel_ring_buffer *ring)
1810 {
1811         struct drm_device *dev = obj->base.dev;
1812         struct drm_i915_private *dev_priv = dev->dev_private;
1813         u32 seqno = intel_ring_get_seqno(ring);
1814
1815         BUG_ON(ring == NULL);
1816         if (obj->ring != ring && obj->last_write_seqno) {
1817                 /* Keep the seqno relative to the current ring */
1818                 obj->last_write_seqno = seqno;
1819         }
1820         obj->ring = ring;
1821
1822         /* Add a reference if we're newly entering the active list. */
1823         if (!obj->active) {
1824                 drm_gem_object_reference(&obj->base);
1825                 obj->active = 1;
1826         }
1827
1828         /* Move from whatever list we were on to the tail of execution. */
1829         list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
1830         list_move_tail(&obj->ring_list, &ring->active_list);
1831
1832         obj->last_read_seqno = seqno;
1833
1834         if (obj->fenced_gpu_access) {
1835                 obj->last_fenced_seqno = seqno;
1836
1837                 /* Bump MRU to take account of the delayed flush */
1838                 if (obj->fence_reg != I915_FENCE_REG_NONE) {
1839                         struct drm_i915_fence_reg *reg;
1840
1841                         reg = &dev_priv->fence_regs[obj->fence_reg];
1842                         list_move_tail(&reg->lru_list,
1843                                        &dev_priv->mm.fence_list);
1844                 }
1845         }
1846 }
1847
1848 static void
1849 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
1850 {
1851         struct drm_device *dev = obj->base.dev;
1852         struct drm_i915_private *dev_priv = dev->dev_private;
1853
1854         BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
1855         BUG_ON(!obj->active);
1856
1857         list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1858
1859         list_del_init(&obj->ring_list);
1860         obj->ring = NULL;
1861
1862         obj->last_read_seqno = 0;
1863         obj->last_write_seqno = 0;
1864         obj->base.write_domain = 0;
1865
1866         obj->last_fenced_seqno = 0;
1867         obj->fenced_gpu_access = false;
1868
1869         obj->active = 0;
1870         drm_gem_object_unreference(&obj->base);
1871
1872         WARN_ON(i915_verify_lists(dev));
1873 }
1874
1875 static int
1876 i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
1877 {
1878         struct drm_i915_private *dev_priv = dev->dev_private;
1879         struct intel_ring_buffer *ring;
1880         int ret, i, j;
1881
1882         /* Carefully retire all requests without writing to the rings */
1883         for_each_ring(ring, dev_priv, i) {
1884                 ret = intel_ring_idle(ring);
1885                 if (ret)
1886                         return ret;
1887         }
1888         i915_gem_retire_requests(dev);
1889
1890         /* Finally reset hw state */
1891         for_each_ring(ring, dev_priv, i) {
1892                 intel_ring_init_seqno(ring, seqno);
1893
1894                 for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
1895                         ring->sync_seqno[j] = 0;
1896         }
1897
1898         return 0;
1899 }
1900
1901 int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
1902 {
1903         struct drm_i915_private *dev_priv = dev->dev_private;
1904         int ret;
1905
1906         if (seqno == 0)
1907                 return -EINVAL;
1908
1909         /* HWS page needs to be set less than what we
1910          * will inject to ring
1911          */
1912         ret = i915_gem_init_seqno(dev, seqno - 1);
1913         if (ret)
1914                 return ret;
1915
1916         /* Carefully set the last_seqno value so that wrap
1917          * detection still works
1918          */
1919         dev_priv->next_seqno = seqno;
1920         dev_priv->last_seqno = seqno - 1;
1921         if (dev_priv->last_seqno == 0)
1922                 dev_priv->last_seqno--;
1923
1924         return 0;
1925 }
1926
1927 int
1928 i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
1929 {
1930         struct drm_i915_private *dev_priv = dev->dev_private;
1931
1932         /* reserve 0 for non-seqno */
1933         if (dev_priv->next_seqno == 0) {
1934                 int ret = i915_gem_init_seqno(dev, 0);
1935                 if (ret)
1936                         return ret;
1937
1938                 dev_priv->next_seqno = 1;
1939         }
1940
1941         *seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
1942         return 0;
1943 }
1944
1945 int __i915_add_request(struct intel_ring_buffer *ring,
1946                        struct drm_file *file,
1947                        struct drm_i915_gem_object *obj,
1948                        u32 *out_seqno)
1949 {
1950         drm_i915_private_t *dev_priv = ring->dev->dev_private;
1951         struct drm_i915_gem_request *request;
1952         u32 request_ring_position, request_start;
1953         int was_empty;
1954         int ret;
1955
1956         request_start = intel_ring_get_tail(ring);
1957         /*
1958          * Emit any outstanding flushes - execbuf can fail to emit the flush
1959          * after having emitted the batchbuffer command. Hence we need to fix
1960          * things up similar to emitting the lazy request. The difference here
1961          * is that the flush _must_ happen before the next request, no matter
1962          * what.
1963          */
1964         ret = intel_ring_flush_all_caches(ring);
1965         if (ret)
1966                 return ret;
1967
1968         request = kmalloc(sizeof(*request), M_DRM, M_WAITOK);
1969         if (request == NULL)
1970                 return -ENOMEM;
1971
1972
1973         /* Record the position of the start of the request so that
1974          * should we detect the updated seqno part-way through the
1975          * GPU processing the request, we never over-estimate the
1976          * position of the head.
1977          */
1978         request_ring_position = intel_ring_get_tail(ring);
1979
1980         ret = ring->add_request(ring);
1981         if (ret) {
1982                 kfree(request);
1983                 return ret;
1984         }
1985
1986         request->seqno = intel_ring_get_seqno(ring);
1987         request->ring = ring;
1988         request->head = request_start;
1989         request->tail = request_ring_position;
1990         request->ctx = ring->last_context;
1991         request->batch_obj = obj;
1992
1993         /* Whilst this request exists, batch_obj will be on the
1994          * active_list, and so will hold the active reference. Only when this
1995          * request is retired will the the batch_obj be moved onto the
1996          * inactive_list and lose its active reference. Hence we do not need
1997          * to explicitly hold another reference here.
1998          */
1999
2000         if (request->ctx)
2001                 i915_gem_context_reference(request->ctx);
2002
2003         request->emitted_jiffies = jiffies;
2004         was_empty = list_empty(&ring->request_list);
2005         list_add_tail(&request->list, &ring->request_list);
2006         request->file_priv = NULL;
2007
2008         if (file) {
2009                 struct drm_i915_file_private *file_priv = file->driver_priv;
2010
2011                 spin_lock(&file_priv->mm.lock);
2012                 request->file_priv = file_priv;
2013                 list_add_tail(&request->client_list,
2014                               &file_priv->mm.request_list);
2015                 spin_unlock(&file_priv->mm.lock);
2016         }
2017
2018         ring->outstanding_lazy_request = 0;
2019
2020         if (!dev_priv->mm.suspended) {
2021                 if (i915_enable_hangcheck) {
2022                         mod_timer(&dev_priv->gpu_error.hangcheck_timer,
2023                                   round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES));
2024                 }
2025                 if (was_empty) {
2026                         queue_delayed_work(dev_priv->wq,
2027                                            &dev_priv->mm.retire_work,
2028                                            round_jiffies_up_relative(hz));
2029                         intel_mark_busy(dev_priv->dev);
2030                 }
2031         }
2032
2033         if (out_seqno)
2034                 *out_seqno = request->seqno;
2035         return 0;
2036 }
2037
2038 static inline void
2039 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
2040 {
2041         struct drm_i915_file_private *file_priv = request->file_priv;
2042
2043         if (!file_priv)
2044                 return;
2045
2046         spin_lock(&file_priv->mm.lock);
2047         if (request->file_priv) {
2048                 list_del(&request->client_list);
2049                 request->file_priv = NULL;
2050         }
2051         spin_unlock(&file_priv->mm.lock);
2052 }
2053
2054 static bool i915_head_inside_object(u32 acthd, struct drm_i915_gem_object *obj)
2055 {
2056         if (acthd >= obj->gtt_offset &&
2057             acthd < obj->gtt_offset + obj->base.size)
2058                 return true;
2059
2060         return false;
2061 }
2062
2063 static bool i915_head_inside_request(const u32 acthd_unmasked,
2064                                      const u32 request_start,
2065                                      const u32 request_end)
2066 {
2067         const u32 acthd = acthd_unmasked & HEAD_ADDR;
2068
2069         if (request_start < request_end) {
2070                 if (acthd >= request_start && acthd < request_end)
2071                         return true;
2072         } else if (request_start > request_end) {
2073                 if (acthd >= request_start || acthd < request_end)
2074                         return true;
2075         }
2076
2077         return false;
2078 }
2079
2080 static bool i915_request_guilty(struct drm_i915_gem_request *request,
2081                                 const u32 acthd, bool *inside)
2082 {
2083         /* There is a possibility that unmasked head address
2084          * pointing inside the ring, matches the batch_obj address range.
2085          * However this is extremely unlikely.
2086          */
2087
2088         if (request->batch_obj) {
2089                 if (i915_head_inside_object(acthd, request->batch_obj)) {
2090                         *inside = true;
2091                         return true;
2092                 }
2093         }
2094
2095         if (i915_head_inside_request(acthd, request->head, request->tail)) {
2096                 *inside = false;
2097                 return true;
2098         }
2099
2100         return false;
2101 }
2102
2103 static void i915_set_reset_status(struct intel_ring_buffer *ring,
2104                                   struct drm_i915_gem_request *request,
2105                                   u32 acthd)
2106 {
2107         struct i915_ctx_hang_stats *hs = NULL;
2108         bool inside, guilty;
2109
2110         /* Innocent until proven guilty */
2111         guilty = false;
2112
2113         if (ring->hangcheck.action != wait &&
2114             i915_request_guilty(request, acthd, &inside)) {
2115                 DRM_ERROR("%s hung %s bo (0x%x ctx %d) at 0x%x\n",
2116                           ring->name,
2117                           inside ? "inside" : "flushing",
2118                           request->batch_obj ?
2119                           request->batch_obj->gtt_offset : 0,
2120                           request->ctx ? request->ctx->id : 0,
2121                           acthd);
2122
2123                 guilty = true;
2124         }
2125
2126         /* If contexts are disabled or this is the default context, use
2127          * file_priv->reset_state
2128          */
2129         if (request->ctx && request->ctx->id != DEFAULT_CONTEXT_ID)
2130                 hs = &request->ctx->hang_stats;
2131         else if (request->file_priv)
2132                 hs = &request->file_priv->hang_stats;
2133
2134         if (hs) {
2135                 if (guilty)
2136                         hs->batch_active++;
2137                 else
2138                         hs->batch_pending++;
2139         }
2140 }
2141
2142 static void i915_gem_free_request(struct drm_i915_gem_request *request)
2143 {
2144         list_del(&request->list);
2145         i915_gem_request_remove_from_client(request);
2146
2147         if (request->ctx)
2148                 i915_gem_context_unreference(request->ctx);
2149
2150         kfree(request);
2151 }
2152
2153 static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
2154                                       struct intel_ring_buffer *ring)
2155 {
2156         u32 completed_seqno;
2157         u32 acthd;
2158
2159         acthd = intel_ring_get_active_head(ring);
2160         completed_seqno = ring->get_seqno(ring, false);
2161
2162         while (!list_empty(&ring->request_list)) {
2163                 struct drm_i915_gem_request *request;
2164
2165                 request = list_first_entry(&ring->request_list,
2166                                            struct drm_i915_gem_request,
2167                                            list);
2168
2169                 if (request->seqno > completed_seqno)
2170                         i915_set_reset_status(ring, request, acthd);
2171
2172                 i915_gem_free_request(request);
2173         }
2174
2175         while (!list_empty(&ring->active_list)) {
2176                 struct drm_i915_gem_object *obj;
2177
2178                 obj = list_first_entry(&ring->active_list,
2179                                        struct drm_i915_gem_object,
2180                                        ring_list);
2181
2182                 i915_gem_object_move_to_inactive(obj);
2183         }
2184 }
2185
2186 void i915_gem_restore_fences(struct drm_device *dev)
2187 {
2188         struct drm_i915_private *dev_priv = dev->dev_private;
2189         int i;
2190
2191         for (i = 0; i < dev_priv->num_fence_regs; i++) {
2192                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
2193
2194                 /*
2195                  * Commit delayed tiling changes if we have an object still
2196                  * attached to the fence, otherwise just clear the fence.
2197                  */
2198                 if (reg->obj) {
2199                         i915_gem_object_update_fence(reg->obj, reg,
2200                                                      reg->obj->tiling_mode);
2201                 } else {
2202                         i915_gem_write_fence(dev, i, NULL);
2203                 }
2204         }
2205 }
2206
2207 void i915_gem_reset(struct drm_device *dev)
2208 {
2209         struct drm_i915_private *dev_priv = dev->dev_private;
2210         struct drm_i915_gem_object *obj;
2211         struct intel_ring_buffer *ring;
2212         int i;
2213
2214         for_each_ring(ring, dev_priv, i)
2215                 i915_gem_reset_ring_lists(dev_priv, ring);
2216
2217         /* Move everything out of the GPU domains to ensure we do any
2218          * necessary invalidation upon reuse.
2219          */
2220         list_for_each_entry(obj,
2221                             &dev_priv->mm.inactive_list,
2222                             mm_list)
2223         {
2224                 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
2225         }
2226
2227         i915_gem_restore_fences(dev);
2228 }
2229
2230 /**
2231  * This function clears the request list as sequence numbers are passed.
2232  */
2233 void
2234 i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
2235 {
2236         uint32_t seqno;
2237
2238         if (list_empty(&ring->request_list))
2239                 return;
2240
2241         WARN_ON(i915_verify_lists(ring->dev));
2242
2243         seqno = ring->get_seqno(ring, true);
2244
2245         while (!list_empty(&ring->request_list)) {
2246                 struct drm_i915_gem_request *request;
2247
2248                 request = list_first_entry(&ring->request_list,
2249                                            struct drm_i915_gem_request,
2250                                            list);
2251
2252                 if (!i915_seqno_passed(seqno, request->seqno))
2253                         break;
2254
2255                 /* We know the GPU must have read the request to have
2256                  * sent us the seqno + interrupt, so use the position
2257                  * of tail of the request to update the last known position
2258                  * of the GPU head.
2259                  */
2260                 ring->last_retired_head = request->tail;
2261
2262                 i915_gem_free_request(request);
2263         }
2264
2265         /* Move any buffers on the active list that are no longer referenced
2266          * by the ringbuffer to the flushing/inactive lists as appropriate.
2267          */
2268         while (!list_empty(&ring->active_list)) {
2269                 struct drm_i915_gem_object *obj;
2270
2271                 obj = list_first_entry(&ring->active_list,
2272                                       struct drm_i915_gem_object,
2273                                       ring_list);
2274
2275                 if (!i915_seqno_passed(seqno, obj->last_read_seqno))
2276                         break;
2277
2278                 i915_gem_object_move_to_inactive(obj);
2279         }
2280
2281         if (unlikely(ring->trace_irq_seqno &&
2282                      i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
2283                 ring->irq_put(ring);
2284                 ring->trace_irq_seqno = 0;
2285         }
2286
2287 }
2288
2289 void
2290 i915_gem_retire_requests(struct drm_device *dev)
2291 {
2292         drm_i915_private_t *dev_priv = dev->dev_private;
2293         struct intel_ring_buffer *ring;
2294         int i;
2295
2296         for_each_ring(ring, dev_priv, i)
2297                 i915_gem_retire_requests_ring(ring);
2298 }
2299
2300 static long
2301 __i915_gem_shrink(struct drm_i915_private *dev_priv, long target,
2302                   bool purgeable_only)
2303 {
2304         struct drm_i915_gem_object *obj, *next;
2305         long count = 0;
2306
2307         list_for_each_entry_safe(obj, next,
2308                                  &dev_priv->mm.unbound_list,
2309                                  global_list) {
2310 #if 0
2311                 if ((i915_gem_object_is_purgeable(obj) || !purgeable_only) &&
2312                     i915_gem_object_put_pages(obj) == 0) {
2313                         count += obj->base.size >> PAGE_SHIFT;
2314                         if (count >= target)
2315                                 return count;
2316                 }
2317 #endif
2318         }
2319
2320         list_for_each_entry_safe(obj, next,
2321                                  &dev_priv->mm.inactive_list,
2322                                  mm_list) {
2323 #if 0
2324                 if ((i915_gem_object_is_purgeable(obj) || !purgeable_only) &&
2325                     i915_gem_object_unbind(obj) == 0 &&
2326                     i915_gem_object_put_pages(obj) == 0) {
2327                         count += obj->base.size >> PAGE_SHIFT;
2328                         if (count >= target)
2329                                 return count;
2330                 }
2331 #endif
2332         }
2333
2334         return count;
2335 }
2336
2337 static long
2338 i915_gem_purge(struct drm_i915_private *dev_priv, long target)
2339 {
2340         return __i915_gem_shrink(dev_priv, target, true);
2341 }
2342
2343 static void
2344 i915_gem_retire_work_handler(struct work_struct *work)
2345 {
2346         drm_i915_private_t *dev_priv;
2347         struct drm_device *dev;
2348         struct intel_ring_buffer *ring;
2349         bool idle;
2350         int i;
2351
2352         dev_priv = container_of(work, drm_i915_private_t,
2353                                 mm.retire_work.work);
2354         dev = dev_priv->dev;
2355
2356         /* Come back later if the device is busy... */
2357         if (lockmgr(&dev->struct_mutex, LK_EXCLUSIVE|LK_NOWAIT)) {
2358                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
2359                                    round_jiffies_up_relative(hz));
2360                 return;
2361         }
2362
2363         i915_gem_retire_requests(dev);
2364
2365         /* Send a periodic flush down the ring so we don't hold onto GEM
2366          * objects indefinitely.
2367          */
2368         idle = true;
2369         for_each_ring(ring, dev_priv, i) {
2370                 if (ring->gpu_caches_dirty)
2371                         i915_add_request(ring, NULL);
2372
2373                 idle &= list_empty(&ring->request_list);
2374         }
2375
2376         if (!dev_priv->mm.suspended && !idle)
2377                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
2378                                    round_jiffies_up_relative(hz));
2379         if (idle)
2380                 intel_mark_idle(dev);
2381
2382         mutex_unlock(&dev->struct_mutex);
2383 }
2384 /**
2385  * Ensures that an object will eventually get non-busy by flushing any required
2386  * write domains, emitting any outstanding lazy request and retiring and
2387  * completed requests.
2388  */
2389 static int
2390 i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2391 {
2392         int ret;
2393
2394         if (obj->active) {
2395                 ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
2396                 if (ret)
2397                         return ret;
2398
2399                 i915_gem_retire_requests_ring(obj->ring);
2400         }
2401
2402         return 0;
2403 }
2404
2405 /**
2406  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
2407  * @DRM_IOCTL_ARGS: standard ioctl arguments
2408  *
2409  * Returns 0 if successful, else an error is returned with the remaining time in
2410  * the timeout parameter.
2411  *  -ETIME: object is still busy after timeout
2412  *  -ERESTARTSYS: signal interrupted the wait
2413  *  -ENONENT: object doesn't exist
2414  * Also possible, but rare:
2415  *  -EAGAIN: GPU wedged
2416  *  -ENOMEM: damn
2417  *  -ENODEV: Internal IRQ fail
2418  *  -E?: The add request failed
2419  *
2420  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
2421  * non-zero timeout parameter the wait ioctl will wait for the given number of
2422  * nanoseconds on an object becoming unbusy. Since the wait itself does so
2423  * without holding struct_mutex the object may become re-busied before this
2424  * function completes. A similar but shorter * race condition exists in the busy
2425  * ioctl
2426  */
2427 int
2428 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
2429 {
2430         drm_i915_private_t *dev_priv = dev->dev_private;
2431         struct drm_i915_gem_wait *args = data;
2432         struct drm_i915_gem_object *obj;
2433         struct intel_ring_buffer *ring = NULL;
2434         struct timespec timeout_stack, *timeout = NULL;
2435         unsigned reset_counter;
2436         u32 seqno = 0;
2437         int ret = 0;
2438
2439         if (args->timeout_ns >= 0) {
2440                 timeout_stack = ns_to_timespec(args->timeout_ns);
2441                 timeout = &timeout_stack;
2442         }
2443
2444         ret = i915_mutex_lock_interruptible(dev);
2445         if (ret)
2446                 return ret;
2447
2448         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
2449         if (&obj->base == NULL) {
2450                 mutex_unlock(&dev->struct_mutex);
2451                 return -ENOENT;
2452         }
2453
2454         /* Need to make sure the object gets inactive eventually. */
2455         ret = i915_gem_object_flush_active(obj);
2456         if (ret)
2457                 goto out;
2458
2459         if (obj->active) {
2460                 seqno = obj->last_read_seqno;
2461                 ring = obj->ring;
2462         }
2463
2464         if (seqno == 0)
2465                  goto out;
2466
2467         /* Do this after OLR check to make sure we make forward progress polling
2468          * on this IOCTL with a 0 timeout (like busy ioctl)
2469          */
2470         if (!args->timeout_ns) {
2471                 ret = -ETIMEDOUT;
2472                 goto out;
2473         }
2474
2475         drm_gem_object_unreference(&obj->base);
2476         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
2477         mutex_unlock(&dev->struct_mutex);
2478
2479         ret = __wait_seqno(ring, seqno, reset_counter, true, timeout);
2480         if (timeout)
2481                 args->timeout_ns = timespec_to_ns(timeout);
2482         return ret;
2483
2484 out:
2485         drm_gem_object_unreference(&obj->base);
2486         mutex_unlock(&dev->struct_mutex);
2487         return ret;
2488 }
2489
2490 /**
2491  * i915_gem_object_sync - sync an object to a ring.
2492  *
2493  * @obj: object which may be in use on another ring.
2494  * @to: ring we wish to use the object on. May be NULL.
2495  *
2496  * This code is meant to abstract object synchronization with the GPU.
2497  * Calling with NULL implies synchronizing the object with the CPU
2498  * rather than a particular GPU ring.
2499  *
2500  * Returns 0 if successful, else propagates up the lower layer error.
2501  */
2502 int
2503 i915_gem_object_sync(struct drm_i915_gem_object *obj,
2504                      struct intel_ring_buffer *to)
2505 {
2506         struct intel_ring_buffer *from = obj->ring;
2507         u32 seqno;
2508         int ret, idx;
2509
2510         if (from == NULL || to == from)
2511                 return 0;
2512
2513         if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
2514                 return i915_gem_object_wait_rendering(obj, false);
2515
2516         idx = intel_ring_sync_index(from, to);
2517
2518         seqno = obj->last_read_seqno;
2519         if (seqno <= from->sync_seqno[idx])
2520                 return 0;
2521
2522         ret = i915_gem_check_olr(obj->ring, seqno);
2523         if (ret)
2524                 return ret;
2525
2526         ret = to->sync_to(to, from, seqno);
2527         if (!ret)
2528                 /* We use last_read_seqno because sync_to()
2529                  * might have just caused seqno wrap under
2530                  * the radar.
2531                  */
2532                 from->sync_seqno[idx] = obj->last_read_seqno;
2533
2534         return ret;
2535 }
2536
2537 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
2538 {
2539         u32 old_write_domain, old_read_domains;
2540
2541         /* Force a pagefault for domain tracking on next user access */
2542         i915_gem_release_mmap(obj);
2543
2544         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
2545                 return;
2546
2547         /* Wait for any direct GTT access to complete */
2548         cpu_mfence();
2549
2550         old_read_domains = obj->base.read_domains;
2551         old_write_domain = obj->base.write_domain;
2552
2553         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
2554         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2555
2556 }
2557
2558 /**
2559  * Unbinds an object from the GTT aperture.
2560  */
2561 int
2562 i915_gem_object_unbind(struct drm_i915_gem_object *obj)
2563 {
2564         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2565         int ret;
2566
2567         if (obj->gtt_space == NULL)
2568                 return 0;
2569
2570         if (obj->pin_count)
2571                 return -EBUSY;
2572
2573         BUG_ON(obj->pages == NULL);
2574
2575         ret = i915_gem_object_finish_gpu(obj);
2576         if (ret)
2577                 return ret;
2578         /* Continue on if we fail due to EIO, the GPU is hung so we
2579          * should be safe and we need to cleanup or else we might
2580          * cause memory corruption through use-after-free.
2581          */
2582
2583         i915_gem_object_finish_gtt(obj);
2584
2585         /* Move the object to the CPU domain to ensure that
2586          * any possible CPU writes while it's not in the GTT
2587          * are flushed when we go to remap it.
2588          */
2589         if (ret == 0)
2590                 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
2591         if (ret == -ERESTARTSYS)
2592                 return ret;
2593         if (ret) {
2594                 /* In the event of a disaster, abandon all caches and
2595                  * hope for the best.
2596                  */
2597                 i915_gem_clflush_object(obj);
2598                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2599         }
2600
2601         /* release the fence reg _after_ flushing */
2602         ret = i915_gem_object_put_fence(obj);
2603         if (ret)
2604                 return ret;
2605
2606         if (obj->has_global_gtt_mapping)
2607                 i915_gem_gtt_unbind_object(obj);
2608         if (obj->has_aliasing_ppgtt_mapping) {
2609                 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
2610                 obj->has_aliasing_ppgtt_mapping = 0;
2611         }
2612         i915_gem_gtt_finish_object(obj);
2613
2614         i915_gem_object_put_pages_gtt(obj);
2615
2616         list_del_init(&obj->global_list);
2617         list_del_init(&obj->mm_list);
2618         /* Avoid an unnecessary call to unbind on rebind. */
2619         obj->map_and_fenceable = true;
2620
2621         drm_mm_put_block(obj->gtt_space);
2622         obj->gtt_space = NULL;
2623         obj->gtt_offset = 0;
2624
2625         if (i915_gem_object_is_purgeable(obj))
2626                 i915_gem_object_truncate(obj);
2627
2628         return ret;
2629 }
2630
2631 int i915_gpu_idle(struct drm_device *dev)
2632 {
2633         drm_i915_private_t *dev_priv = dev->dev_private;
2634         struct intel_ring_buffer *ring;
2635         int ret, i;
2636
2637         /* Flush everything onto the inactive list. */
2638         for_each_ring(ring, dev_priv, i) {
2639                 ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
2640                 if (ret)
2641                         return ret;
2642
2643                 ret = intel_ring_idle(ring);
2644                 if (ret)
2645                         return ret;
2646         }
2647
2648         return 0;
2649 }
2650
2651 static void i965_write_fence_reg(struct drm_device *dev, int reg,
2652                                  struct drm_i915_gem_object *obj)
2653 {
2654         drm_i915_private_t *dev_priv = dev->dev_private;
2655         int fence_reg;
2656         int fence_pitch_shift;
2657
2658         if (INTEL_INFO(dev)->gen >= 6) {
2659                 fence_reg = FENCE_REG_SANDYBRIDGE_0;
2660                 fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT;
2661         } else {
2662                 fence_reg = FENCE_REG_965_0;
2663                 fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
2664         }
2665
2666         fence_reg += reg * 8;
2667
2668         /* To w/a incoherency with non-atomic 64-bit register updates,
2669          * we split the 64-bit update into two 32-bit writes. In order
2670          * for a partial fence not to be evaluated between writes, we
2671          * precede the update with write to turn off the fence register,
2672          * and only enable the fence as the last step.
2673          *
2674          * For extra levels of paranoia, we make sure each step lands
2675          * before applying the next step.
2676          */
2677         I915_WRITE(fence_reg, 0);
2678         POSTING_READ(fence_reg);
2679
2680         if (obj) {
2681                 u32 size = obj->gtt_space->size;
2682                 uint64_t val;
2683
2684                 val = (uint64_t)((obj->gtt_offset + size - 4096) &
2685                                  0xfffff000) << 32;
2686                 val |= obj->gtt_offset & 0xfffff000;
2687                 val |= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift;
2688                 if (obj->tiling_mode == I915_TILING_Y)
2689                         val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2690                 val |= I965_FENCE_REG_VALID;
2691
2692                 I915_WRITE(fence_reg + 4, val >> 32);
2693                 POSTING_READ(fence_reg + 4);
2694
2695                 I915_WRITE(fence_reg + 0, val);
2696                 POSTING_READ(fence_reg);
2697         } else {
2698                 I915_WRITE(fence_reg + 4, 0);
2699                 POSTING_READ(fence_reg + 4);
2700         }
2701 }
2702
2703 static void i915_write_fence_reg(struct drm_device *dev, int reg,
2704                                  struct drm_i915_gem_object *obj)
2705 {
2706         drm_i915_private_t *dev_priv = dev->dev_private;
2707         u32 val;
2708
2709         if (obj) {
2710                 u32 size = obj->gtt_space->size;
2711                 int pitch_val;
2712                 int tile_width;
2713
2714                 WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
2715                      (size & -size) != size ||
2716                      (obj->gtt_offset & (size - 1)),
2717                      "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
2718                      obj->gtt_offset, obj->map_and_fenceable, size);
2719
2720                 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
2721                         tile_width = 128;
2722                 else
2723                         tile_width = 512;
2724
2725                 /* Note: pitch better be a power of two tile widths */
2726                 pitch_val = obj->stride / tile_width;
2727                 pitch_val = ffs(pitch_val) - 1;
2728
2729                 val = obj->gtt_offset;
2730                 if (obj->tiling_mode == I915_TILING_Y)
2731                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2732                 val |= I915_FENCE_SIZE_BITS(size);
2733                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2734                 val |= I830_FENCE_REG_VALID;
2735         } else
2736                 val = 0;
2737
2738         if (reg < 8)
2739                 reg = FENCE_REG_830_0 + reg * 4;
2740         else
2741                 reg = FENCE_REG_945_8 + (reg - 8) * 4;
2742
2743         I915_WRITE(reg, val);
2744         POSTING_READ(reg);
2745 }
2746
2747 static void i830_write_fence_reg(struct drm_device *dev, int reg,
2748                                 struct drm_i915_gem_object *obj)
2749 {
2750         drm_i915_private_t *dev_priv = dev->dev_private;
2751         uint32_t val;
2752
2753         if (obj) {
2754                 u32 size = obj->gtt_space->size;
2755                 uint32_t pitch_val;
2756
2757                 WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
2758                      (size & -size) != size ||
2759                      (obj->gtt_offset & (size - 1)),
2760                      "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
2761                      obj->gtt_offset, size);
2762
2763                 pitch_val = obj->stride / 128;
2764                 pitch_val = ffs(pitch_val) - 1;
2765
2766                 val = obj->gtt_offset;
2767                 if (obj->tiling_mode == I915_TILING_Y)
2768                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2769                 val |= I830_FENCE_SIZE_BITS(size);
2770                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2771                 val |= I830_FENCE_REG_VALID;
2772         } else
2773                 val = 0;
2774
2775         I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
2776         POSTING_READ(FENCE_REG_830_0 + reg * 4);
2777 }
2778
2779 inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj)
2780 {
2781         return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT;
2782 }
2783
2784 static void i915_gem_write_fence(struct drm_device *dev, int reg,
2785                                  struct drm_i915_gem_object *obj)
2786 {
2787         struct drm_i915_private *dev_priv = dev->dev_private;
2788
2789         /* Ensure that all CPU reads are completed before installing a fence
2790          * and all writes before removing the fence.
2791          */
2792         if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj))
2793                 cpu_mfence();
2794
2795         WARN(obj && (!obj->stride || !obj->tiling_mode),
2796              "bogus fence setup with stride: 0x%x, tiling mode: %i\n",
2797              obj->stride, obj->tiling_mode);
2798
2799         switch (INTEL_INFO(dev)->gen) {
2800         case 7:
2801         case 6:
2802         case 5:
2803         case 4: i965_write_fence_reg(dev, reg, obj); break;
2804         case 3: i915_write_fence_reg(dev, reg, obj); break;
2805         case 2: i830_write_fence_reg(dev, reg, obj); break;
2806         default: BUG();
2807         }
2808
2809         /* And similarly be paranoid that no direct access to this region
2810          * is reordered to before the fence is installed.
2811          */
2812         if (i915_gem_object_needs_mb(obj))
2813                 cpu_mfence();
2814 }
2815
2816 static inline int fence_number(struct drm_i915_private *dev_priv,
2817                                struct drm_i915_fence_reg *fence)
2818 {
2819         return fence - dev_priv->fence_regs;
2820 }
2821
2822 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
2823                                          struct drm_i915_fence_reg *fence,
2824                                          bool enable)
2825 {
2826         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2827         int reg = fence_number(dev_priv, fence);
2828
2829         i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
2830
2831         if (enable) {
2832                 obj->fence_reg = reg;
2833                 fence->obj = obj;
2834                 list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
2835         } else {
2836                 obj->fence_reg = I915_FENCE_REG_NONE;
2837                 fence->obj = NULL;
2838                 list_del_init(&fence->lru_list);
2839         }
2840         obj->fence_dirty = false;
2841 }
2842
2843 static int
2844 i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
2845 {
2846         if (obj->last_fenced_seqno) {
2847                 int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
2848                 if (ret)
2849                         return ret;
2850
2851                 obj->last_fenced_seqno = 0;
2852         }
2853
2854         obj->fenced_gpu_access = false;
2855         return 0;
2856 }
2857
2858 int
2859 i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2860 {
2861         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2862         struct drm_i915_fence_reg *fence;
2863         int ret;
2864
2865         ret = i915_gem_object_wait_fence(obj);
2866         if (ret)
2867                 return ret;
2868
2869         if (obj->fence_reg == I915_FENCE_REG_NONE)
2870                 return 0;
2871
2872         fence = &dev_priv->fence_regs[obj->fence_reg];
2873
2874         i915_gem_object_fence_lost(obj);
2875         i915_gem_object_update_fence(obj, fence, false);
2876
2877         return 0;
2878 }
2879
2880 static struct drm_i915_fence_reg *
2881 i915_find_fence_reg(struct drm_device *dev)
2882 {
2883         struct drm_i915_private *dev_priv = dev->dev_private;
2884         struct drm_i915_fence_reg *reg, *avail;
2885         int i;
2886
2887         /* First try to find a free reg */
2888         avail = NULL;
2889         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2890                 reg = &dev_priv->fence_regs[i];
2891                 if (!reg->obj)
2892                         return reg;
2893
2894                 if (!reg->pin_count)
2895                         avail = reg;
2896         }
2897
2898         if (avail == NULL)
2899                 return NULL;
2900
2901         /* None available, try to steal one or wait for a user to finish */
2902         list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
2903                 if (reg->pin_count)
2904                         continue;
2905
2906                 return reg;
2907         }
2908
2909         return NULL;
2910 }
2911
2912 /**
2913  * i915_gem_object_get_fence - set up fencing for an object
2914  * @obj: object to map through a fence reg
2915  *
2916  * When mapping objects through the GTT, userspace wants to be able to write
2917  * to them without having to worry about swizzling if the object is tiled.
2918  * This function walks the fence regs looking for a free one for @obj,
2919  * stealing one if it can't find any.
2920  *
2921  * It then sets up the reg based on the object's properties: address, pitch
2922  * and tiling format.
2923  *
2924  * For an untiled surface, this removes any existing fence.
2925  */
2926 int
2927 i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
2928 {
2929         struct drm_device *dev = obj->base.dev;
2930         struct drm_i915_private *dev_priv = dev->dev_private;
2931         bool enable = obj->tiling_mode != I915_TILING_NONE;
2932         struct drm_i915_fence_reg *reg;
2933         int ret;
2934
2935         /* Have we updated the tiling parameters upon the object and so
2936          * will need to serialise the write to the associated fence register?
2937          */
2938         if (obj->fence_dirty) {
2939                 ret = i915_gem_object_wait_fence(obj);
2940                 if (ret)
2941                         return ret;
2942         }
2943
2944         /* Just update our place in the LRU if our fence is getting reused. */
2945         if (obj->fence_reg != I915_FENCE_REG_NONE) {
2946                 reg = &dev_priv->fence_regs[obj->fence_reg];
2947                 if (!obj->fence_dirty) {
2948                         list_move_tail(&reg->lru_list,
2949                                        &dev_priv->mm.fence_list);
2950                         return 0;
2951                 }
2952         } else if (enable) {
2953                 reg = i915_find_fence_reg(dev);
2954                 if (reg == NULL)
2955                         return -EDEADLK;
2956
2957                 if (reg->obj) {
2958                         struct drm_i915_gem_object *old = reg->obj;
2959
2960                         ret = i915_gem_object_wait_fence(old);
2961                         if (ret)
2962                                 return ret;
2963
2964                         i915_gem_object_fence_lost(old);
2965                 }
2966         } else
2967                 return 0;
2968
2969         i915_gem_object_update_fence(obj, reg, enable);
2970
2971         return 0;
2972 }
2973
2974 static bool i915_gem_valid_gtt_space(struct drm_device *dev,
2975                                      struct drm_mm_node *gtt_space,
2976                                      unsigned long cache_level)
2977 {
2978         struct drm_mm_node *other;
2979
2980         /* On non-LLC machines we have to be careful when putting differing
2981          * types of snoopable memory together to avoid the prefetcher
2982          * crossing memory domains and dying.
2983          */
2984         if (HAS_LLC(dev))
2985                 return true;
2986
2987         if (gtt_space == NULL)
2988                 return true;
2989
2990         if (list_empty(&gtt_space->node_list))
2991                 return true;
2992
2993         other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
2994         if (other->allocated && !other->hole_follows && other->color != cache_level)
2995                 return false;
2996
2997         other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
2998         if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
2999                 return false;
3000
3001         return true;
3002 }
3003
3004 static void i915_gem_verify_gtt(struct drm_device *dev)
3005 {
3006 #if WATCH_GTT
3007         struct drm_i915_private *dev_priv = dev->dev_private;
3008         struct drm_i915_gem_object *obj;
3009         int err = 0;
3010
3011         list_for_each_entry(obj, &dev_priv->mm.global_list, global_list) {
3012                 if (obj->gtt_space == NULL) {
3013                         printk(KERN_ERR "object found on GTT list with no space reserved\n");
3014                         err++;
3015                         continue;
3016                 }
3017
3018                 if (obj->cache_level != obj->gtt_space->color) {
3019                         printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
3020                                obj->gtt_space->start,
3021                                obj->gtt_space->start + obj->gtt_space->size,
3022                                obj->cache_level,
3023                                obj->gtt_space->color);
3024                         err++;
3025                         continue;
3026                 }
3027
3028                 if (!i915_gem_valid_gtt_space(dev,
3029                                               obj->gtt_space,
3030                                               obj->cache_level)) {
3031                         printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
3032                                obj->gtt_space->start,
3033                                obj->gtt_space->start + obj->gtt_space->size,
3034                                obj->cache_level);
3035                         err++;
3036                         continue;
3037                 }
3038         }
3039
3040         WARN_ON(err);
3041 #endif
3042 }
3043
3044 /**
3045  * Finds free space in the GTT aperture and binds the object there.
3046  */
3047 static int
3048 i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
3049                             unsigned alignment,
3050                             bool map_and_fenceable,
3051                             bool nonblocking)
3052 {
3053         struct drm_device *dev = obj->base.dev;
3054         drm_i915_private_t *dev_priv = dev->dev_private;
3055         struct drm_mm_node *node;
3056         u32 size, fence_size, fence_alignment, unfenced_alignment;
3057         bool mappable, fenceable;
3058         size_t gtt_max = map_and_fenceable ?
3059                 dev_priv->gtt.mappable_end : dev_priv->gtt.total;
3060         int ret;
3061
3062         fence_size = i915_gem_get_gtt_size(dev,
3063                                            obj->base.size,
3064                                            obj->tiling_mode);
3065         fence_alignment = i915_gem_get_gtt_alignment(dev,
3066                                                      obj->base.size,
3067                                                      obj->tiling_mode, true);
3068         unfenced_alignment =
3069                 i915_gem_get_gtt_alignment(dev,
3070                                                     obj->base.size,
3071                                                     obj->tiling_mode, false);
3072
3073         if (alignment == 0)
3074                 alignment = map_and_fenceable ? fence_alignment :
3075                                                 unfenced_alignment;
3076         if (map_and_fenceable && alignment & (fence_alignment - 1)) {
3077                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
3078                 return -EINVAL;
3079         }
3080
3081         size = map_and_fenceable ? fence_size : obj->base.size;
3082
3083         /* If the object is bigger than the entire aperture, reject it early
3084          * before evicting everything in a vain attempt to find space.
3085          */
3086         if (obj->base.size > gtt_max) {
3087                 DRM_ERROR("Attempting to bind an object larger than the aperture: object=%zd > %s aperture=%zu\n",
3088                           obj->base.size,
3089                           map_and_fenceable ? "mappable" : "total",
3090                           gtt_max);
3091                 return -E2BIG;
3092         }
3093
3094  search_free:
3095         if (map_and_fenceable)
3096                 node = drm_mm_search_free_in_range_color(&dev_priv->mm.gtt_space,
3097                                                           size, alignment, obj->cache_level,
3098                                                           0, dev_priv->gtt.mappable_end,
3099                                                           false);
3100         else
3101                 node = drm_mm_search_free_color(&dev_priv->mm.gtt_space,
3102                                                       size, alignment, obj->cache_level,
3103                                                       false);
3104         if (node != NULL) {
3105                 if (map_and_fenceable)
3106                         obj->gtt_space =
3107                                 drm_mm_get_block_range_generic(node,
3108                                                                size, alignment, obj->cache_level,
3109                                                                0, dev_priv->gtt.mappable_end,
3110                                                                false);
3111                 else
3112                         obj->gtt_space =
3113                                 drm_mm_get_block_generic(node,
3114                                                          size, alignment, obj->cache_level,
3115                                                          false);
3116         }
3117         if (obj->gtt_space == NULL) {
3118                 ret = i915_gem_evict_something(dev, size, alignment,
3119                                                obj->cache_level,
3120                                                map_and_fenceable,
3121                                                nonblocking);
3122                 if (ret)
3123                         return ret;
3124
3125                 goto search_free;
3126         }
3127
3128         /*
3129          * NOTE: i915_gem_object_get_pages_gtt() cannot
3130          *       return ENOMEM, since we used VM_ALLOC_RETRY.
3131          */
3132         ret = i915_gem_object_get_pages_gtt(obj);
3133         if (ret != 0) {
3134                 drm_mm_put_block(obj->gtt_space);
3135                 obj->gtt_space = NULL;
3136                 return ret;
3137         }
3138
3139         i915_gem_gtt_bind_object(obj, obj->cache_level);
3140         if (ret != 0) {
3141                 i915_gem_object_put_pages_gtt(obj);
3142                 drm_mm_put_block(obj->gtt_space);
3143                 obj->gtt_space = NULL;
3144                 if (i915_gem_evict_everything(dev))
3145                         return (ret);
3146                 goto search_free;
3147         }
3148
3149         list_add_tail(&obj->global_list, &dev_priv->mm.bound_list);
3150         list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
3151
3152         obj->gtt_offset = obj->gtt_space->start;
3153
3154         fenceable =
3155                 obj->gtt_space->size == fence_size &&
3156                 (obj->gtt_space->start & (fence_alignment - 1)) == 0;
3157
3158         mappable =
3159                 obj->gtt_offset + obj->base.size <= dev_priv->gtt.mappable_end;
3160
3161         obj->map_and_fenceable = mappable && fenceable;
3162
3163         trace_i915_gem_object_bind(obj, map_and_fenceable);
3164         i915_gem_verify_gtt(dev);
3165         return 0;
3166 }
3167
3168 void
3169 i915_gem_clflush_object(struct drm_i915_gem_object *obj)
3170 {
3171
3172         /* If we don't have a page list set up, then we're not pinned
3173          * to GPU, and we can ignore the cache flush because it'll happen
3174          * again at bind time.
3175          */
3176         if (obj->pages == NULL)
3177                 return;
3178
3179         /*
3180          * Stolen memory is always coherent with the GPU as it is explicitly
3181          * marked as wc by the system, or the system is cache-coherent.
3182          */
3183         if (obj->stolen)
3184                 return;
3185
3186         /* If the GPU is snooping the contents of the CPU cache,
3187          * we do not need to manually clear the CPU cache lines.  However,
3188          * the caches are only snooped when the render cache is
3189          * flushed/invalidated.  As we always have to emit invalidations
3190          * and flushes when moving into and out of the RENDER domain, correct
3191          * snooping behaviour occurs naturally as the result of our domain
3192          * tracking.
3193          */
3194         if (obj->cache_level != I915_CACHE_NONE)
3195                 return;
3196
3197         drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
3198 }
3199
3200 /** Flushes the GTT write domain for the object if it's dirty. */
3201 static void
3202 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
3203 {
3204         uint32_t old_write_domain;
3205
3206         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
3207                 return;
3208
3209         /* No actual flushing is required for the GTT write domain.  Writes
3210          * to it immediately go to main memory as far as we know, so there's
3211          * no chipset flush.  It also doesn't land in render cache.
3212          *
3213          * However, we do have to enforce the order so that all writes through
3214          * the GTT land before any writes to the device, such as updates to
3215          * the GATT itself.
3216          */
3217         cpu_sfence();
3218
3219         old_write_domain = obj->base.write_domain;
3220         obj->base.write_domain = 0;
3221 }
3222
3223 /** Flushes the CPU write domain for the object if it's dirty. */
3224 static void
3225 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
3226 {
3227         uint32_t old_write_domain;
3228
3229         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
3230                 return;
3231
3232         i915_gem_clflush_object(obj);
3233         i915_gem_chipset_flush(obj->base.dev);
3234         old_write_domain = obj->base.write_domain;
3235         obj->base.write_domain = 0;
3236 }
3237
3238 /**
3239  * Moves a single object to the GTT read, and possibly write domain.
3240  *
3241  * This function returns when the move is complete, including waiting on
3242  * flushes to occur.
3243  */
3244 int
3245 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
3246 {
3247         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
3248         uint32_t old_write_domain, old_read_domains;
3249         int ret;
3250
3251         /* Not valid to be called on unbound objects. */
3252         if (obj->gtt_space == NULL)
3253                 return -EINVAL;
3254
3255         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
3256                 return 0;
3257
3258         ret = i915_gem_object_wait_rendering(obj, !write);
3259         if (ret)
3260                 return ret;
3261
3262         i915_gem_object_flush_cpu_write_domain(obj);
3263
3264         /* Serialise direct access to this object with the barriers for
3265          * coherent writes from the GPU, by effectively invalidating the
3266          * GTT domain upon first access.
3267          */
3268         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
3269                 cpu_mfence();
3270
3271         old_write_domain = obj->base.write_domain;
3272         old_read_domains = obj->base.read_domains;
3273
3274         /* It should now be out of any other write domains, and we can update
3275          * the domain values for our changes.
3276          */
3277         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
3278         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3279         if (write) {
3280                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
3281                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
3282                 obj->dirty = 1;
3283         }
3284
3285         /* And bump the LRU for this access */
3286         if (i915_gem_object_is_inactive(obj))
3287                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
3288
3289         return 0;
3290 }
3291
3292 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
3293                                     enum i915_cache_level cache_level)
3294 {
3295         struct drm_device *dev = obj->base.dev;
3296         drm_i915_private_t *dev_priv = dev->dev_private;
3297         int ret;
3298
3299         if (obj->cache_level == cache_level)
3300                 return 0;
3301
3302         if (obj->pin_count) {
3303                 DRM_DEBUG("can not change the cache level of pinned objects\n");
3304                 return -EBUSY;
3305         }
3306
3307         if (!i915_gem_valid_gtt_space(dev, obj->gtt_space, cache_level)) {
3308                 ret = i915_gem_object_unbind(obj);
3309                 if (ret)
3310                         return ret;
3311         }
3312
3313         if (obj->gtt_space) {
3314                 ret = i915_gem_object_finish_gpu(obj);
3315                 if (ret)
3316                         return ret;
3317
3318                 i915_gem_object_finish_gtt(obj);
3319
3320                 /* Before SandyBridge, you could not use tiling or fence
3321                  * registers with snooped memory, so relinquish any fences
3322                  * currently pointing to our region in the aperture.
3323                  */
3324                 if (INTEL_INFO(dev)->gen < 6) {
3325                         ret = i915_gem_object_put_fence(obj);
3326                         if (ret)
3327                                 return ret;
3328                 }
3329
3330                 if (obj->has_global_gtt_mapping)
3331                         i915_gem_gtt_bind_object(obj, cache_level);
3332                 if (obj->has_aliasing_ppgtt_mapping)
3333                         i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
3334                                                obj, cache_level);
3335
3336                 obj->gtt_space->color = cache_level;
3337         }
3338
3339         if (cache_level == I915_CACHE_NONE) {
3340                 u32 old_read_domains, old_write_domain;
3341
3342                 /* If we're coming from LLC cached, then we haven't
3343                  * actually been tracking whether the data is in the
3344                  * CPU cache or not, since we only allow one bit set
3345                  * in obj->write_domain and have been skipping the clflushes.
3346                  * Just set it to the CPU cache for now.
3347                  */
3348                 WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
3349                 WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
3350
3351                 old_read_domains = obj->base.read_domains;
3352                 old_write_domain = obj->base.write_domain;
3353
3354                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3355                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3356
3357         }
3358
3359         obj->cache_level = cache_level;
3360         i915_gem_verify_gtt(dev);
3361         return 0;
3362 }
3363
3364 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
3365                                struct drm_file *file)
3366 {
3367         struct drm_i915_gem_caching *args = data;
3368         struct drm_i915_gem_object *obj;
3369         int ret;
3370
3371         ret = i915_mutex_lock_interruptible(dev);
3372         if (ret)
3373                 return ret;
3374
3375         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3376         if (&obj->base == NULL) {
3377                 ret = -ENOENT;
3378                 goto unlock;
3379         }
3380
3381         args->caching = obj->cache_level != I915_CACHE_NONE;
3382
3383         drm_gem_object_unreference(&obj->base);
3384 unlock:
3385         mutex_unlock(&dev->struct_mutex);
3386         return ret;
3387 }
3388
3389 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
3390                                struct drm_file *file)
3391 {
3392         struct drm_i915_gem_caching *args = data;
3393         struct drm_i915_gem_object *obj;
3394         enum i915_cache_level level;
3395         int ret;
3396
3397         switch (args->caching) {
3398         case I915_CACHING_NONE:
3399                 level = I915_CACHE_NONE;
3400                 break;
3401         case I915_CACHING_CACHED:
3402                 level = I915_CACHE_LLC;
3403                 break;
3404         default:
3405                 return -EINVAL;
3406         }
3407
3408         ret = i915_mutex_lock_interruptible(dev);
3409         if (ret)
3410                 return ret;
3411
3412         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3413         if (&obj->base == NULL) {
3414                 ret = -ENOENT;
3415                 goto unlock;
3416         }
3417
3418         ret = i915_gem_object_set_cache_level(obj, level);
3419
3420         drm_gem_object_unreference(&obj->base);
3421 unlock:
3422         mutex_unlock(&dev->struct_mutex);
3423         return ret;
3424 }
3425
3426 /*
3427  * Prepare buffer for display plane (scanout, cursors, etc).
3428  * Can be called from an uninterruptible phase (modesetting) and allows
3429  * any flushes to be pipelined (for pageflips).
3430  */
3431 int
3432 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
3433                                      u32 alignment,
3434                                      struct intel_ring_buffer *pipelined)
3435 {
3436         u32 old_read_domains, old_write_domain;
3437         int ret;
3438
3439         if (pipelined != obj->ring) {
3440                 ret = i915_gem_object_sync(obj, pipelined);
3441                 if (ret)
3442                         return ret;
3443         }
3444
3445         /* The display engine is not coherent with the LLC cache on gen6.  As
3446          * a result, we make sure that the pinning that is about to occur is
3447          * done with uncached PTEs. This is lowest common denominator for all
3448          * chipsets.
3449          *
3450          * However for gen6+, we could do better by using the GFDT bit instead
3451          * of uncaching, which would allow us to flush all the LLC-cached data
3452          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
3453          */
3454         ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
3455         if (ret)
3456                 return ret;
3457
3458         /* As the user may map the buffer once pinned in the display plane
3459          * (e.g. libkms for the bootup splash), we have to ensure that we
3460          * always use map_and_fenceable for all scanout buffers.
3461          */
3462         ret = i915_gem_object_pin(obj, alignment, true, false);
3463         if (ret)
3464                 return ret;
3465
3466         i915_gem_object_flush_cpu_write_domain(obj);
3467
3468         old_write_domain = obj->base.write_domain;
3469         old_read_domains = obj->base.read_domains;
3470
3471         /* It should now be out of any other write domains, and we can update
3472          * the domain values for our changes.
3473          */
3474         obj->base.write_domain = 0;
3475         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3476
3477         return 0;
3478 }
3479
3480 int
3481 i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
3482 {
3483         int ret;
3484
3485         if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
3486                 return 0;
3487
3488         ret = i915_gem_object_wait_rendering(obj, false);
3489         if (ret)
3490                 return ret;
3491
3492         /* Ensure that we invalidate the GPU's caches and TLBs. */
3493         obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
3494         return 0;
3495 }
3496
3497 /**
3498  * Moves a single object to the CPU read, and possibly write domain.
3499  *
3500  * This function returns when the move is complete, including waiting on
3501  * flushes to occur.
3502  */
3503 int
3504 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
3505 {
3506         uint32_t old_write_domain, old_read_domains;
3507         int ret;
3508
3509         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
3510                 return 0;
3511
3512         ret = i915_gem_object_wait_rendering(obj, !write);
3513         if (ret)
3514                 return ret;
3515
3516         i915_gem_object_flush_gtt_write_domain(obj);
3517
3518         old_write_domain = obj->base.write_domain;
3519         old_read_domains = obj->base.read_domains;
3520
3521         /* Flush the CPU cache if it's still invalid. */
3522         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3523                 i915_gem_clflush_object(obj);
3524
3525                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3526         }
3527
3528         /* It should now be out of any other write domains, and we can update
3529          * the domain values for our changes.
3530          */
3531         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3532
3533         /* If we're writing through the CPU, then the GPU read domains will
3534          * need to be invalidated at next use.
3535          */
3536         if (write) {
3537                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3538                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3539         }
3540
3541         return 0;
3542 }
3543
3544 /* Throttle our rendering by waiting until the ring has completed our requests
3545  * emitted over 20 msec ago.
3546  *
3547  * Note that if we were to use the current jiffies each time around the loop,
3548  * we wouldn't escape the function with any frames outstanding if the time to
3549  * render a frame was over 20ms.
3550  *
3551  * This should get us reasonable parallelism between CPU and GPU but also
3552  * relatively low latency when blocking on a particular request to finish.
3553  */
3554 static int
3555 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3556 {
3557         struct drm_i915_private *dev_priv = dev->dev_private;
3558         struct drm_i915_file_private *file_priv = file->driver_priv;
3559         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3560         struct drm_i915_gem_request *request;
3561         struct intel_ring_buffer *ring = NULL;
3562         unsigned reset_counter;
3563         u32 seqno = 0;
3564         int ret;
3565
3566         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
3567         if (ret)
3568                 return ret;
3569
3570         ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
3571         if (ret)
3572                 return ret;
3573
3574         spin_lock(&file_priv->mm.lock);
3575         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3576                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3577                         break;
3578
3579                 ring = request->ring;
3580                 seqno = request->seqno;
3581         }
3582         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3583         spin_unlock(&file_priv->mm.lock);
3584
3585         if (seqno == 0)
3586                 return 0;
3587
3588         ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
3589         if (ret == 0)
3590                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
3591
3592         return ret;
3593 }
3594
3595 int
3596 i915_gem_object_pin(struct drm_i915_gem_object *obj,
3597                     uint32_t alignment,
3598                     bool map_and_fenceable,
3599                     bool nonblocking)
3600 {
3601         int ret;
3602
3603         if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
3604                 return -EBUSY;
3605
3606         if (obj->gtt_space != NULL) {
3607                 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
3608                     (map_and_fenceable && !obj->map_and_fenceable)) {
3609                         WARN(obj->pin_count,
3610                              "bo is already pinned with incorrect alignment:"
3611                              " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
3612                              " obj->map_and_fenceable=%d\n",
3613                              obj->gtt_offset, alignment,
3614                              map_and_fenceable,
3615                              obj->map_and_fenceable);
3616                         ret = i915_gem_object_unbind(obj);
3617                         if (ret)
3618                                 return ret;
3619                 }
3620         }
3621
3622         if (obj->gtt_space == NULL) {
3623                 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3624
3625                 ret = i915_gem_object_bind_to_gtt(obj, alignment,
3626                                                   map_and_fenceable,
3627                                                   nonblocking);
3628                 if (ret)
3629                         return ret;
3630
3631                 if (!dev_priv->mm.aliasing_ppgtt)
3632                         i915_gem_gtt_bind_object(obj, obj->cache_level);
3633         }
3634
3635         if (!obj->has_global_gtt_mapping && map_and_fenceable)
3636                 i915_gem_gtt_bind_object(obj, obj->cache_level);
3637
3638         obj->pin_count++;
3639         obj->pin_mappable |= map_and_fenceable;
3640
3641         return 0;
3642 }
3643
3644 void
3645 i915_gem_object_unpin(struct drm_i915_gem_object *obj)
3646 {
3647         BUG_ON(obj->pin_count == 0);
3648         BUG_ON(obj->gtt_space == NULL);
3649
3650         if (--obj->pin_count == 0)
3651                 obj->pin_mappable = false;
3652 }
3653
3654 int
3655 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
3656                    struct drm_file *file)
3657 {
3658         struct drm_i915_gem_pin *args = data;
3659         struct drm_i915_gem_object *obj;
3660         int ret;
3661
3662         ret = i915_mutex_lock_interruptible(dev);
3663         if (ret)
3664                 return ret;
3665
3666         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3667         if (&obj->base == NULL) {
3668                 ret = -ENOENT;
3669                 goto unlock;
3670         }
3671
3672         if (obj->madv != I915_MADV_WILLNEED) {
3673                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
3674                 ret = -EINVAL;
3675                 goto out;
3676         }
3677
3678         if (obj->pin_filp != NULL && obj->pin_filp != file) {
3679                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
3680                           args->handle);
3681                 ret = -EINVAL;
3682                 goto out;
3683         }
3684
3685         if (obj->user_pin_count == 0) {
3686                 ret = i915_gem_object_pin(obj, args->alignment, true, false);
3687                 if (ret)
3688                         goto out;
3689         }
3690
3691         obj->user_pin_count++;
3692         obj->pin_filp = file;
3693
3694         /* XXX - flush the CPU caches for pinned objects
3695          * as the X server doesn't manage domains yet
3696          */
3697         i915_gem_object_flush_cpu_write_domain(obj);
3698         args->offset = obj->gtt_offset;
3699 out:
3700         drm_gem_object_unreference(&obj->base);
3701 unlock:
3702         mutex_unlock(&dev->struct_mutex);
3703         return ret;
3704 }
3705
3706 int
3707 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
3708                      struct drm_file *file)
3709 {
3710         struct drm_i915_gem_pin *args = data;
3711         struct drm_i915_gem_object *obj;
3712         int ret;
3713
3714         ret = i915_mutex_lock_interruptible(dev);
3715         if (ret)
3716                 return ret;
3717
3718         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3719         if (&obj->base == NULL) {
3720                 ret = -ENOENT;
3721                 goto unlock;
3722         }
3723
3724         if (obj->pin_filp != file) {
3725                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
3726                           args->handle);
3727                 ret = -EINVAL;
3728                 goto out;
3729         }
3730         obj->user_pin_count--;
3731         if (obj->user_pin_count == 0) {
3732                 obj->pin_filp = NULL;
3733                 i915_gem_object_unpin(obj);
3734         }
3735
3736 out:
3737         drm_gem_object_unreference(&obj->base);
3738 unlock:
3739         mutex_unlock(&dev->struct_mutex);
3740         return ret;
3741 }
3742
3743 int
3744 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
3745                     struct drm_file *file)
3746 {
3747         struct drm_i915_gem_busy *args = data;
3748         struct drm_i915_gem_object *obj;
3749         int ret;
3750
3751         ret = i915_mutex_lock_interruptible(dev);
3752         if (ret)
3753                 return ret;
3754
3755         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3756         if (&obj->base == NULL) {
3757                 ret = -ENOENT;
3758                 goto unlock;
3759         }
3760
3761         /* Count all active objects as busy, even if they are currently not used
3762          * by the gpu. Users of this interface expect objects to eventually
3763          * become non-busy without any further actions, therefore emit any
3764          * necessary flushes here.
3765          */
3766         ret = i915_gem_object_flush_active(obj);
3767
3768         args->busy = obj->active;
3769         if (obj->ring) {
3770                 args->busy |= intel_ring_flag(obj->ring) << 16;
3771         }
3772
3773         drm_gem_object_unreference(&obj->base);
3774 unlock:
3775         mutex_unlock(&dev->struct_mutex);
3776         return ret;
3777 }
3778
3779 int
3780 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
3781                         struct drm_file *file_priv)
3782 {
3783         return i915_gem_ring_throttle(dev, file_priv);
3784 }
3785
3786 int
3787 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
3788                        struct drm_file *file_priv)
3789 {
3790         struct drm_i915_gem_madvise *args = data;
3791         struct drm_i915_gem_object *obj;
3792         int ret;
3793
3794         switch (args->madv) {
3795         case I915_MADV_DONTNEED:
3796         case I915_MADV_WILLNEED:
3797             break;
3798         default:
3799             return -EINVAL;
3800         }
3801
3802         ret = i915_mutex_lock_interruptible(dev);
3803         if (ret)
3804                 return ret;
3805
3806         obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
3807         if (&obj->base == NULL) {
3808                 ret = -ENOENT;
3809                 goto unlock;
3810         }
3811
3812         if (obj->pin_count) {
3813                 ret = -EINVAL;
3814                 goto out;
3815         }
3816
3817         if (obj->madv != __I915_MADV_PURGED)
3818                 obj->madv = args->madv;
3819
3820         /* if the object is no longer attached, discard its backing storage */
3821         if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
3822                 i915_gem_object_truncate(obj);
3823
3824         args->retained = obj->madv != __I915_MADV_PURGED;
3825
3826 out:
3827         drm_gem_object_unreference(&obj->base);
3828 unlock:
3829         mutex_unlock(&dev->struct_mutex);
3830         return ret;
3831 }
3832
3833 void i915_gem_object_init(struct drm_i915_gem_object *obj,
3834                           const struct drm_i915_gem_object_ops *ops)
3835 {
3836         INIT_LIST_HEAD(&obj->mm_list);
3837         INIT_LIST_HEAD(&obj->global_list);
3838         INIT_LIST_HEAD(&obj->ring_list);
3839         INIT_LIST_HEAD(&obj->exec_list);
3840
3841         obj->ops = ops;
3842
3843         obj->fence_reg = I915_FENCE_REG_NONE;
3844         obj->madv = I915_MADV_WILLNEED;
3845         /* Avoid an unnecessary call to unbind on the first bind. */
3846         obj->map_and_fenceable = true;
3847
3848         i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
3849 }
3850
3851 static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
3852         .get_pages = i915_gem_object_get_pages_gtt,
3853         .put_pages = i915_gem_object_put_pages_gtt,
3854 };
3855
3856 struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
3857                                                   size_t size)
3858 {
3859         struct drm_i915_gem_object *obj;
3860 #if 0
3861         struct address_space *mapping;
3862         u32 mask;
3863 #endif
3864
3865         obj = kmalloc(sizeof(*obj), M_DRM, M_WAITOK | M_ZERO);
3866         if (obj == NULL)
3867                 return NULL;
3868
3869         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
3870                 kfree(obj);
3871                 return NULL;
3872         }
3873
3874 #if 0
3875         mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
3876         if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
3877                 /* 965gm cannot relocate objects above 4GiB. */
3878                 mask &= ~__GFP_HIGHMEM;
3879                 mask |= __GFP_DMA32;
3880         }
3881
3882         mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
3883         mapping_set_gfp_mask(mapping, mask);
3884 #endif
3885
3886         i915_gem_object_init(obj, &i915_gem_object_ops);
3887
3888         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3889         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3890
3891         if (HAS_LLC(dev)) {
3892                 /* On some devices, we can have the GPU use the LLC (the CPU
3893                  * cache) for about a 10% performance improvement
3894                  * compared to uncached.  Graphics requests other than
3895                  * display scanout are coherent with the CPU in
3896                  * accessing this cache.  This means in this mode we
3897                  * don't need to clflush on the CPU side, and on the
3898                  * GPU side we only need to flush internal caches to
3899                  * get data visible to the CPU.
3900                  *
3901                  * However, we maintain the display planes as UC, and so
3902                  * need to rebind when first used as such.
3903                  */
3904                 obj->cache_level = I915_CACHE_LLC;
3905         } else
3906                 obj->cache_level = I915_CACHE_NONE;
3907
3908         return obj;
3909 }
3910
3911 int i915_gem_init_object(struct drm_gem_object *obj)
3912 {
3913         BUG();
3914
3915         return 0;
3916 }
3917
3918 void i915_gem_free_object(struct drm_gem_object *gem_obj)
3919 {
3920         struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
3921         struct drm_device *dev = obj->base.dev;
3922         drm_i915_private_t *dev_priv = dev->dev_private;
3923
3924         if (obj->phys_obj)
3925                 i915_gem_detach_phys_object(dev, obj);
3926
3927         obj->pin_count = 0;
3928         if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) {
3929                 bool was_interruptible;
3930
3931                 was_interruptible = dev_priv->mm.interruptible;
3932                 dev_priv->mm.interruptible = false;
3933
3934                 WARN_ON(i915_gem_object_unbind(obj));
3935
3936                 dev_priv->mm.interruptible = was_interruptible;
3937         }
3938
3939         /* Stolen objects don't hold a ref, but do hold pin count. Fix that up
3940          * before progressing. */
3941         if (obj->stolen)
3942                 i915_gem_object_unpin_pages(obj);
3943
3944         if (WARN_ON(obj->pages_pin_count))
3945                 obj->pages_pin_count = 0;
3946         i915_gem_object_put_pages(obj);
3947         drm_gem_free_mmap_offset(&obj->base);
3948
3949         BUG_ON(obj->pages);
3950
3951         drm_gem_object_release(&obj->base);
3952         i915_gem_info_remove_obj(dev_priv, obj->base.size);
3953
3954         kfree(obj->bit_17);
3955         i915_gem_object_free(obj);
3956 }
3957
3958 int
3959 i915_gem_idle(struct drm_device *dev)
3960 {
3961         drm_i915_private_t *dev_priv = dev->dev_private;
3962         int ret;
3963
3964         mutex_lock(&dev->struct_mutex);
3965
3966         if (dev_priv->mm.suspended) {
3967                 mutex_unlock(&dev->struct_mutex);
3968                 return 0;
3969         }
3970
3971         ret = i915_gpu_idle(dev);
3972         if (ret) {
3973                 mutex_unlock(&dev->struct_mutex);
3974                 return ret;
3975         }
3976         i915_gem_retire_requests(dev);
3977
3978         /* Under UMS, be paranoid and evict. */
3979         if (!drm_core_check_feature(dev, DRIVER_MODESET))
3980                 i915_gem_evict_everything(dev);
3981
3982         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
3983          * We need to replace this with a semaphore, or something.
3984          * And not confound mm.suspended!
3985          */
3986         dev_priv->mm.suspended = 1;
3987         del_timer_sync(&dev_priv->gpu_error.hangcheck_timer);
3988
3989         i915_kernel_lost_context(dev);
3990         i915_gem_cleanup_ringbuffer(dev);
3991
3992         mutex_unlock(&dev->struct_mutex);
3993
3994         /* Cancel the retire work handler, which should be idle now. */
3995         cancel_delayed_work_sync(&dev_priv->mm.retire_work);