2 * Copyright © 2008 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * Eric Anholt <eric@anholt.net>
26 * Copyright (c) 2011 The FreeBSD Foundation
27 * All rights reserved.
29 * This software was developed by Konstantin Belousov under sponsorship from
30 * the FreeBSD Foundation.
32 * Redistribution and use in source and binary forms, with or without
33 * modification, are permitted provided that the following conditions
35 * 1. Redistributions of source code must retain the above copyright
36 * notice, this list of conditions and the following disclaimer.
37 * 2. Redistributions in binary form must reproduce the above copyright
38 * notice, this list of conditions and the following disclaimer in the
39 * documentation and/or other materials provided with the distribution.
41 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
42 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
43 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
44 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
45 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
46 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
47 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
48 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
49 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
50 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
53 * $FreeBSD: head/sys/dev/drm2/i915/i915_gem.c 253497 2013-07-20 13:52:40Z kib $
56 #include <sys/resourcevar.h>
57 #include <sys/sfbuf.h>
60 #include <drm/i915_drm.h>
62 #include "intel_drv.h"
63 #include "intel_ringbuffer.h"
65 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
66 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
68 static uint32_t i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size,
70 static uint32_t i915_gem_get_gtt_alignment(struct drm_device *dev,
71 uint32_t size, int tiling_mode);
72 static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
73 unsigned alignment, bool map_and_fenceable);
74 static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
76 static void i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj);
77 static int i915_gem_object_set_cpu_read_domain_range(
78 struct drm_i915_gem_object *obj, uint64_t offset, uint64_t size);
79 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj);
80 static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
81 static int i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj);
82 static bool i915_gem_object_is_inactive(struct drm_i915_gem_object *obj);
83 static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj);
84 static vm_page_t i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex);
85 static void i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
86 uint32_t flush_domains);
87 static void i915_gem_clear_fence_reg(struct drm_device *dev,
88 struct drm_i915_fence_reg *reg);
89 static void i915_gem_reset_fences(struct drm_device *dev);
90 static int i915_gem_phys_pwrite(struct drm_device *dev,
91 struct drm_i915_gem_object *obj, uint64_t data_ptr, uint64_t offset,
92 uint64_t size, struct drm_file *file_priv);
93 static void i915_gem_lowmem(void *arg);
95 MALLOC_DEFINE(DRM_I915_GEM, "i915gem", "Allocations from i915 gem");
96 long i915_gem_wired_pages_cnt;
99 i915_gem_info_add_obj(struct drm_i915_private *dev_priv, size_t size)
102 dev_priv->mm.object_count++;
103 dev_priv->mm.object_memory += size;
107 i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, size_t size)
110 dev_priv->mm.object_count--;
111 dev_priv->mm.object_memory -= size;
115 i915_gem_wait_for_error(struct drm_device *dev)
117 struct drm_i915_private *dev_priv = dev->dev_private;
120 if (!atomic_read(&dev_priv->mm.wedged))
123 lockmgr(&dev_priv->error_completion_lock, LK_EXCLUSIVE);
124 while (dev_priv->error_completion == 0) {
125 ret = -lksleep(&dev_priv->error_completion,
126 &dev_priv->error_completion_lock, PCATCH, "915wco", 0);
128 lockmgr(&dev_priv->error_completion_lock, LK_RELEASE);
132 lockmgr(&dev_priv->error_completion_lock, LK_RELEASE);
134 if (atomic_read(&dev_priv->mm.wedged)) {
135 /* GPU is hung, bump the completion count to account for
136 * the token we just consumed so that we never hit zero and
137 * end up waiting upon a subsequent completion event that
140 lockmgr(&dev_priv->error_completion_lock, LK_EXCLUSIVE);
141 dev_priv->error_completion++;
142 lockmgr(&dev_priv->error_completion_lock, LK_RELEASE);
148 i915_mutex_lock_interruptible(struct drm_device *dev)
150 struct drm_i915_private *dev_priv;
153 dev_priv = dev->dev_private;
154 ret = i915_gem_wait_for_error(dev);
159 * interruptible shall it be. might indeed be if dev_lock is
162 ret = lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_SLEEPFAIL);
171 i915_gem_free_object_tail(struct drm_i915_gem_object *obj)
173 struct drm_device *dev;
174 drm_i915_private_t *dev_priv;
178 dev_priv = dev->dev_private;
180 ret = i915_gem_object_unbind(obj);
181 if (ret == -ERESTART) {
182 list_move(&obj->mm_list, &dev_priv->mm.deferred_free_list);
186 drm_gem_free_mmap_offset(&obj->base);
187 drm_gem_object_release(&obj->base);
188 i915_gem_info_remove_obj(dev_priv, obj->base.size);
190 drm_free(obj->page_cpu_valid, DRM_I915_GEM);
191 drm_free(obj->bit_17, DRM_I915_GEM);
192 drm_free(obj, DRM_I915_GEM);
196 i915_gem_free_object(struct drm_gem_object *gem_obj)
198 struct drm_i915_gem_object *obj;
199 struct drm_device *dev;
201 obj = to_intel_bo(gem_obj);
204 while (obj->pin_count > 0)
205 i915_gem_object_unpin(obj);
207 if (obj->phys_obj != NULL)
208 i915_gem_detach_phys_object(dev, obj);
210 i915_gem_free_object_tail(obj);
214 i915_gem_do_init(struct drm_device *dev, unsigned long start,
215 unsigned long mappable_end, unsigned long end)
217 drm_i915_private_t *dev_priv;
218 unsigned long mappable;
221 dev_priv = dev->dev_private;
222 mappable = min(end, mappable_end) - start;
224 drm_mm_init(&dev_priv->mm.gtt_space, start, end - start);
226 dev_priv->mm.gtt_start = start;
227 dev_priv->mm.gtt_mappable_end = mappable_end;
228 dev_priv->mm.gtt_end = end;
229 dev_priv->mm.gtt_total = end - start;
230 dev_priv->mm.mappable_gtt_total = mappable;
232 /* Take over this portion of the GTT */
233 intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);
234 device_printf(dev->dev,
235 "taking over the fictitious range 0x%lx-0x%lx\n",
236 dev->agp->base + start, dev->agp->base + start + mappable);
237 error = -vm_phys_fictitious_reg_range(dev->agp->base + start,
238 dev->agp->base + start + mappable, VM_MEMATTR_WRITE_COMBINING);
243 i915_gem_init_ioctl(struct drm_device *dev, void *data,
244 struct drm_file *file)
246 struct drm_i915_gem_init *args;
247 drm_i915_private_t *dev_priv;
249 dev_priv = dev->dev_private;
252 if (args->gtt_start >= args->gtt_end ||
253 (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
257 * XXXKIB. The second-time initialization should be guarded
260 lockmgr(&dev->dev_lock, LK_EXCLUSIVE|LK_RETRY|LK_CANRECURSE);
261 i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
262 lockmgr(&dev->dev_lock, LK_RELEASE);
268 i915_gem_idle(struct drm_device *dev)
270 drm_i915_private_t *dev_priv;
273 dev_priv = dev->dev_private;
274 if (dev_priv->mm.suspended)
277 ret = i915_gpu_idle(dev);
281 /* Under UMS, be paranoid and evict. */
282 if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
283 ret = i915_gem_evict_inactive(dev, false);
288 i915_gem_reset_fences(dev);
290 /* Hack! Don't let anybody do execbuf while we don't control the chip.
291 * We need to replace this with a semaphore, or something.
292 * And not confound mm.suspended!
294 dev_priv->mm.suspended = 1;
295 del_timer_sync(&dev_priv->hangcheck_timer);
297 i915_kernel_lost_context(dev);
298 i915_gem_cleanup_ringbuffer(dev);
300 /* Cancel the retire work handler, which should be idle now. */
301 cancel_delayed_work_sync(&dev_priv->mm.retire_work);
307 i915_gem_init_swizzling(struct drm_device *dev)
309 drm_i915_private_t *dev_priv;
311 dev_priv = dev->dev_private;
313 if (INTEL_INFO(dev)->gen < 5 ||
314 dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
317 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
318 DISP_TILE_SURFACE_SWIZZLING);
323 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
325 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
327 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
331 i915_gem_init_hw(struct drm_device *dev)
333 drm_i915_private_t *dev_priv;
336 dev_priv = dev->dev_private;
338 i915_gem_init_swizzling(dev);
340 ret = intel_init_render_ring_buffer(dev);
345 ret = intel_init_bsd_ring_buffer(dev);
347 goto cleanup_render_ring;
351 ret = intel_init_blt_ring_buffer(dev);
353 goto cleanup_bsd_ring;
356 dev_priv->next_seqno = 1;
357 i915_gem_init_ppgtt(dev);
361 intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
363 intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
368 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
369 struct drm_file *file)
371 struct drm_i915_private *dev_priv;
372 struct drm_i915_gem_get_aperture *args;
373 struct drm_i915_gem_object *obj;
376 dev_priv = dev->dev_private;
379 if (!(dev->driver->driver_features & DRIVER_GEM))
384 list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
385 pinned += obj->gtt_space->size;
388 args->aper_size = dev_priv->mm.gtt_total;
389 args->aper_available_size = args->aper_size - pinned;
395 i915_gem_object_pin(struct drm_i915_gem_object *obj, uint32_t alignment,
396 bool map_and_fenceable)
398 struct drm_device *dev;
399 struct drm_i915_private *dev_priv;
403 dev_priv = dev->dev_private;
405 KASSERT(obj->pin_count != DRM_I915_GEM_OBJECT_MAX_PIN_COUNT,
408 if (obj->gtt_space != NULL) {
409 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
410 (map_and_fenceable && !obj->map_and_fenceable)) {
411 DRM_DEBUG("bo is already pinned with incorrect alignment:"
412 " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
413 " obj->map_and_fenceable=%d\n",
414 obj->gtt_offset, alignment,
416 obj->map_and_fenceable);
417 ret = i915_gem_object_unbind(obj);
423 if (obj->gtt_space == NULL) {
424 ret = i915_gem_object_bind_to_gtt(obj, alignment,
430 if (obj->pin_count++ == 0 && !obj->active)
431 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
432 obj->pin_mappable |= map_and_fenceable;
437 WARN_ON(i915_verify_lists(dev));
443 i915_gem_object_unpin(struct drm_i915_gem_object *obj)
445 struct drm_device *dev;
446 drm_i915_private_t *dev_priv;
449 dev_priv = dev->dev_private;
454 WARN_ON(i915_verify_lists(dev));
457 KASSERT(obj->pin_count != 0, ("zero pin count"));
458 KASSERT(obj->gtt_space != NULL, ("No gtt mapping"));
460 if (--obj->pin_count == 0) {
462 list_move_tail(&obj->mm_list,
463 &dev_priv->mm.inactive_list);
464 obj->pin_mappable = false;
469 WARN_ON(i915_verify_lists(dev));
474 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
475 struct drm_file *file)
477 struct drm_i915_gem_pin *args;
478 struct drm_i915_gem_object *obj;
479 struct drm_gem_object *gobj;
484 ret = i915_mutex_lock_interruptible(dev);
488 gobj = drm_gem_object_lookup(dev, file, args->handle);
493 obj = to_intel_bo(gobj);
495 if (obj->madv != I915_MADV_WILLNEED) {
496 DRM_ERROR("Attempting to pin a purgeable buffer\n");
501 if (obj->pin_filp != NULL && obj->pin_filp != file) {
502 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
508 obj->user_pin_count++;
509 obj->pin_filp = file;
510 if (obj->user_pin_count == 1) {
511 ret = i915_gem_object_pin(obj, args->alignment, true);
516 /* XXX - flush the CPU caches for pinned objects
517 * as the X server doesn't manage domains yet
519 i915_gem_object_flush_cpu_write_domain(obj);
520 args->offset = obj->gtt_offset;
522 drm_gem_object_unreference(&obj->base);
529 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
530 struct drm_file *file)
532 struct drm_i915_gem_pin *args;
533 struct drm_i915_gem_object *obj;
537 ret = i915_mutex_lock_interruptible(dev);
541 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
542 if (&obj->base == NULL) {
547 if (obj->pin_filp != file) {
548 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
553 obj->user_pin_count--;
554 if (obj->user_pin_count == 0) {
555 obj->pin_filp = NULL;
556 i915_gem_object_unpin(obj);
560 drm_gem_object_unreference(&obj->base);
567 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
568 struct drm_file *file)
570 struct drm_i915_gem_busy *args;
571 struct drm_i915_gem_object *obj;
572 struct drm_i915_gem_request *request;
577 ret = i915_mutex_lock_interruptible(dev);
581 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
582 if (&obj->base == NULL) {
587 args->busy = obj->active;
589 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
590 ret = i915_gem_flush_ring(obj->ring,
591 0, obj->base.write_domain);
592 } else if (obj->ring->outstanding_lazy_request ==
593 obj->last_rendering_seqno) {
594 request = kmalloc(sizeof(*request), DRM_I915_GEM,
596 ret = i915_add_request(obj->ring, NULL, request);
598 drm_free(request, DRM_I915_GEM);
601 i915_gem_retire_requests_ring(obj->ring);
602 args->busy = obj->active;
605 drm_gem_object_unreference(&obj->base);
612 i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
615 if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
617 return (i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain));
621 * Moves a single object to the CPU read, and possibly write domain.
623 * This function returns when the move is complete, including waiting on
627 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
629 uint32_t old_write_domain, old_read_domains;
632 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
635 ret = i915_gem_object_flush_gpu_write_domain(obj);
639 ret = i915_gem_object_wait_rendering(obj);
643 i915_gem_object_flush_gtt_write_domain(obj);
645 old_write_domain = obj->base.write_domain;
646 old_read_domains = obj->base.read_domains;
648 /* Flush the CPU cache if it's still invalid. */
649 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
650 i915_gem_clflush_object(obj);
652 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
655 /* It should now be out of any other write domains, and we can update
656 * the domain values for our changes.
658 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
660 /* If we're writing through the CPU, then the GPU read domains will
661 * need to be invalidated at next use.
664 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
665 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
671 /* Throttle our rendering by waiting until the ring has completed our requests
672 * emitted over 20 msec ago.
674 * Note that if we were to use the current jiffies each time around the loop,
675 * we wouldn't escape the function with any frames outstanding if the time to
676 * render a frame was over 20ms.
678 * This should get us reasonable parallelism between CPU and GPU but also
679 * relatively low latency when blocking on a particular request to finish.
682 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
684 struct drm_i915_private *dev_priv = dev->dev_private;
685 struct drm_i915_file_private *file_priv = file->driver_priv;
686 unsigned long recent_enough = ticks - (20 * hz / 1000);
687 struct drm_i915_gem_request *request;
688 struct intel_ring_buffer *ring = NULL;
692 dev_priv = dev->dev_private;
693 if (atomic_read(&dev_priv->mm.wedged))
696 recent_enough = ticks - (20 * hz / 1000);
700 spin_lock(&file_priv->mm.lock);
701 list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
702 if (time_after_eq(request->emitted_jiffies, recent_enough))
705 ring = request->ring;
706 seqno = request->seqno;
708 spin_unlock(&file_priv->mm.lock);
714 lockmgr(&ring->irq_lock, LK_EXCLUSIVE);
715 if (!i915_seqno_passed(ring->get_seqno(ring,false), seqno)) {
716 if (ring->irq_get(ring)) {
718 !(i915_seqno_passed(ring->get_seqno(ring,false), seqno) ||
719 atomic_read(&dev_priv->mm.wedged)))
720 ret = -lksleep(ring, &ring->irq_lock, PCATCH,
723 if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
725 } else if (_intel_wait_for(dev,
726 i915_seqno_passed(ring->get_seqno(ring,false), seqno) ||
727 atomic_read(&dev_priv->mm.wedged), 3000, 0, "915rtr")) {
731 lockmgr(&ring->irq_lock, LK_RELEASE);
734 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
740 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
741 struct drm_file *file_priv)
744 return (i915_gem_ring_throttle(dev, file_priv));
748 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
749 struct drm_file *file_priv)
751 struct drm_i915_gem_madvise *args = data;
752 struct drm_i915_gem_object *obj;
755 switch (args->madv) {
756 case I915_MADV_DONTNEED:
757 case I915_MADV_WILLNEED:
763 ret = i915_mutex_lock_interruptible(dev);
767 obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
768 if (&obj->base == NULL) {
773 if (obj->pin_count) {
778 if (obj->madv != __I915_MADV_PURGED)
779 obj->madv = args->madv;
781 /* if the object is no longer attached, discard its backing storage */
782 if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
783 i915_gem_object_truncate(obj);
785 args->retained = obj->madv != __I915_MADV_PURGED;
788 drm_gem_object_unreference(&obj->base);
795 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
797 drm_i915_private_t *dev_priv;
800 dev_priv = dev->dev_private;
801 for (i = 0; i < I915_NUM_RINGS; i++)
802 intel_cleanup_ring_buffer(&dev_priv->ring[i]);
806 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
807 struct drm_file *file_priv)
809 drm_i915_private_t *dev_priv = dev->dev_private;
812 if (drm_core_check_feature(dev, DRIVER_MODESET))
815 if (atomic_read(&dev_priv->mm.wedged)) {
816 DRM_ERROR("Reenabling wedged hardware, good luck\n");
817 atomic_set(&dev_priv->mm.wedged, 0);
821 dev_priv->mm.suspended = 0;
823 ret = i915_gem_init_hw(dev);
829 KASSERT(list_empty(&dev_priv->mm.active_list), ("active list"));
832 ret = drm_irq_install(dev);
834 goto cleanup_ringbuffer;
840 i915_gem_cleanup_ringbuffer(dev);
841 dev_priv->mm.suspended = 1;
848 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
849 struct drm_file *file_priv)
852 if (drm_core_check_feature(dev, DRIVER_MODESET))
855 drm_irq_uninstall(dev);
856 return (i915_gem_idle(dev));
860 i915_gem_create(struct drm_file *file, struct drm_device *dev, uint64_t size,
863 struct drm_i915_gem_object *obj;
867 size = roundup(size, PAGE_SIZE);
871 obj = i915_gem_alloc_object(dev, size);
876 ret = drm_gem_handle_create(file, &obj->base, &handle);
878 drm_gem_object_release(&obj->base);
879 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
880 drm_free(obj, DRM_I915_GEM);
884 /* drop reference from allocate - handle holds it now */
885 drm_gem_object_unreference(&obj->base);
891 i915_gem_dumb_create(struct drm_file *file, struct drm_device *dev,
892 struct drm_mode_create_dumb *args)
895 /* have to work out size/pitch and return them */
896 args->pitch = roundup2(args->width * ((args->bpp + 7) / 8), 64);
897 args->size = args->pitch * args->height;
898 return (i915_gem_create(file, dev, args->size, &args->handle));
902 i915_gem_dumb_destroy(struct drm_file *file, struct drm_device *dev,
906 return (drm_gem_handle_delete(file, handle));
910 i915_gem_create_ioctl(struct drm_device *dev, void *data,
911 struct drm_file *file)
913 struct drm_i915_gem_create *args = data;
915 return (i915_gem_create(file, dev, args->size, &args->handle));
918 static inline void vm_page_reference(vm_page_t m)
920 vm_page_flag_set(m, PG_REFERENCED);
924 i915_gem_swap_io(struct drm_device *dev, struct drm_i915_gem_object *obj,
925 uint64_t data_ptr, uint64_t size, uint64_t offset, enum uio_rw rw,
926 struct drm_file *file)
933 int cnt, do_bit17_swizzling, length, obj_po, ret, swizzled_po;
935 if (obj->gtt_offset != 0 && rw == UIO_READ)
936 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
938 do_bit17_swizzling = 0;
941 vm_obj = obj->base.vm_obj;
944 VM_OBJECT_LOCK(vm_obj);
945 vm_object_pip_add(vm_obj, 1);
947 obj_pi = OFF_TO_IDX(offset);
948 obj_po = offset & PAGE_MASK;
950 m = i915_gem_wire_page(vm_obj, obj_pi);
951 VM_OBJECT_UNLOCK(vm_obj);
953 sf = sf_buf_alloc(m);
954 mkva = sf_buf_kva(sf);
955 length = min(size, PAGE_SIZE - obj_po);
957 if (do_bit17_swizzling &&
958 (VM_PAGE_TO_PHYS(m) & (1 << 17)) != 0) {
959 cnt = roundup2(obj_po + 1, 64);
960 cnt = min(cnt - obj_po, length);
961 swizzled_po = obj_po ^ 64;
964 swizzled_po = obj_po;
967 ret = -copyout_nofault(
968 (char *)mkva + swizzled_po,
969 (void *)(uintptr_t)data_ptr, cnt);
971 ret = -copyin_nofault(
972 (void *)(uintptr_t)data_ptr,
973 (char *)mkva + swizzled_po, cnt);
983 VM_OBJECT_LOCK(vm_obj);
986 vm_page_reference(m);
987 vm_page_busy_wait(m, FALSE, "i915gem");
988 vm_page_unwire(m, 1);
990 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
995 vm_object_pip_wakeup(vm_obj);
996 VM_OBJECT_UNLOCK(vm_obj);
1002 i915_gem_gtt_write(struct drm_device *dev, struct drm_i915_gem_object *obj,
1003 uint64_t data_ptr, uint64_t size, uint64_t offset, struct drm_file *file)
1009 * Pass the unaligned physical address and size to pmap_mapdev_attr()
1010 * so it can properly calculate whether an extra page needs to be
1011 * mapped or not to cover the requested range. The function will
1012 * add the page offset into the returned mkva for us.
1014 mkva = (vm_offset_t)pmap_mapdev_attr(dev->agp->base + obj->gtt_offset +
1015 offset, size, PAT_WRITE_COMBINING);
1016 ret = -copyin_nofault((void *)(uintptr_t)data_ptr, (char *)mkva, size);
1017 pmap_unmapdev(mkva, size);
1022 i915_gem_obj_io(struct drm_device *dev, uint32_t handle, uint64_t data_ptr,
1023 uint64_t size, uint64_t offset, enum uio_rw rw, struct drm_file *file)
1025 struct drm_i915_gem_object *obj;
1027 vm_offset_t start, end;
1032 start = trunc_page(data_ptr);
1033 end = round_page(data_ptr + size);
1034 npages = howmany(end - start, PAGE_SIZE);
1035 ma = kmalloc(npages * sizeof(vm_page_t), DRM_I915_GEM, M_WAITOK |
1037 npages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map,
1038 (vm_offset_t)data_ptr, size,
1039 (rw == UIO_READ ? VM_PROT_WRITE : 0 ) | VM_PROT_READ, ma, npages);
1045 ret = i915_mutex_lock_interruptible(dev);
1049 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
1050 if (&obj->base == NULL) {
1054 if (offset > obj->base.size || size > obj->base.size - offset) {
1059 if (rw == UIO_READ) {
1060 ret = i915_gem_object_set_cpu_read_domain_range(obj,
1064 ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset,
1067 if (obj->phys_obj) {
1068 ret = i915_gem_phys_pwrite(dev, obj, data_ptr, offset,
1070 } else if (obj->gtt_space &&
1071 obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1072 ret = i915_gem_object_pin(obj, 0, true);
1075 ret = i915_gem_object_set_to_gtt_domain(obj, true);
1078 ret = i915_gem_object_put_fence(obj);
1081 ret = i915_gem_gtt_write(dev, obj, data_ptr, size,
1084 i915_gem_object_unpin(obj);
1086 ret = i915_gem_object_set_to_cpu_domain(obj, true);
1089 ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset,
1094 drm_gem_object_unreference(&obj->base);
1098 vm_page_unhold_pages(ma, npages);
1100 drm_free(ma, DRM_I915_GEM);
1105 i915_gem_pread_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
1107 struct drm_i915_gem_pread *args;
1110 return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size,
1111 args->offset, UIO_READ, file));
1115 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
1117 struct drm_i915_gem_pwrite *args;
1120 return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size,
1121 args->offset, UIO_WRITE, file));
1125 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1126 struct drm_file *file)
1128 struct drm_i915_gem_set_domain *args;
1129 struct drm_i915_gem_object *obj;
1130 uint32_t read_domains;
1131 uint32_t write_domain;
1134 if ((dev->driver->driver_features & DRIVER_GEM) == 0)
1138 read_domains = args->read_domains;
1139 write_domain = args->write_domain;
1141 if ((write_domain & I915_GEM_GPU_DOMAINS) != 0 ||
1142 (read_domains & I915_GEM_GPU_DOMAINS) != 0 ||
1143 (write_domain != 0 && read_domains != write_domain))
1146 ret = i915_mutex_lock_interruptible(dev);
1150 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1151 if (&obj->base == NULL) {
1156 if ((read_domains & I915_GEM_DOMAIN_GTT) != 0) {
1157 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1161 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1163 drm_gem_object_unreference(&obj->base);
1170 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1171 struct drm_file *file)
1173 struct drm_i915_gem_sw_finish *args;
1174 struct drm_i915_gem_object *obj;
1179 if ((dev->driver->driver_features & DRIVER_GEM) == 0)
1181 ret = i915_mutex_lock_interruptible(dev);
1184 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1185 if (&obj->base == NULL) {
1189 if (obj->pin_count != 0)
1190 i915_gem_object_flush_cpu_write_domain(obj);
1191 drm_gem_object_unreference(&obj->base);
1198 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1199 struct drm_file *file)
1201 struct drm_i915_gem_mmap *args;
1202 struct drm_gem_object *obj;
1211 if ((dev->driver->driver_features & DRIVER_GEM) == 0)
1214 obj = drm_gem_object_lookup(dev, file, args->handle);
1218 if (args->size == 0)
1221 map = &p->p_vmspace->vm_map;
1222 size = round_page(args->size);
1224 if (map->size + size > p->p_rlimit[RLIMIT_VMEM].rlim_cur) {
1232 vm_object_hold(obj->vm_obj);
1233 vm_object_reference_locked(obj->vm_obj);
1234 vm_object_drop(obj->vm_obj);
1236 rv = vm_map_find(map, obj->vm_obj, args->offset, &addr, args->size,
1237 PAGE_SIZE, /* align */
1239 VM_MAPTYPE_NORMAL, /* maptype */
1240 VM_PROT_READ | VM_PROT_WRITE, /* prot */
1241 VM_PROT_READ | VM_PROT_WRITE, /* max */
1242 MAP_SHARED /* cow */);
1243 if (rv != KERN_SUCCESS) {
1244 vm_object_deallocate(obj->vm_obj);
1245 error = -vm_mmap_to_errno(rv);
1247 args->addr_ptr = (uint64_t)addr;
1251 drm_gem_object_unreference(obj);
1256 i915_gem_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot,
1257 vm_ooffset_t foff, struct ucred *cred, u_short *color)
1260 *color = 0; /* XXXKIB */
1267 i915_gem_pager_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot,
1270 struct drm_gem_object *gem_obj;
1271 struct drm_i915_gem_object *obj;
1272 struct drm_device *dev;
1273 drm_i915_private_t *dev_priv;
1278 gem_obj = vm_obj->handle;
1279 obj = to_intel_bo(gem_obj);
1280 dev = obj->base.dev;
1281 dev_priv = dev->dev_private;
1283 write = (prot & VM_PROT_WRITE) != 0;
1287 vm_object_pip_add(vm_obj, 1);
1290 * Remove the placeholder page inserted by vm_fault() from the
1291 * object before dropping the object lock. If
1292 * i915_gem_release_mmap() is active in parallel on this gem
1293 * object, then it owns the drm device sx and might find the
1294 * placeholder already. Then, since the page is busy,
1295 * i915_gem_release_mmap() sleeps waiting for the busy state
1296 * of the page cleared. We will be not able to acquire drm
1297 * device lock until i915_gem_release_mmap() is able to make a
1300 if (*mres != NULL) {
1302 vm_page_remove(oldm);
1307 VM_OBJECT_UNLOCK(vm_obj);
1313 ret = i915_mutex_lock_interruptible(dev);
1322 * Since the object lock was dropped, other thread might have
1323 * faulted on the same GTT address and instantiated the
1324 * mapping for the page. Recheck.
1326 VM_OBJECT_LOCK(vm_obj);
1327 m = vm_page_lookup(vm_obj, OFF_TO_IDX(offset));
1329 if ((m->flags & PG_BUSY) != 0) {
1332 vm_page_sleep(m, "915pee");
1338 VM_OBJECT_UNLOCK(vm_obj);
1340 /* Now bind it into the GTT if needed */
1341 if (!obj->map_and_fenceable) {
1342 ret = i915_gem_object_unbind(obj);
1348 if (!obj->gtt_space) {
1349 ret = i915_gem_object_bind_to_gtt(obj, 0, true);
1355 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1362 if (obj->tiling_mode == I915_TILING_NONE)
1363 ret = i915_gem_object_put_fence(obj);
1365 ret = i915_gem_object_get_fence(obj, NULL);
1371 if (i915_gem_object_is_inactive(obj))
1372 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1374 obj->fault_mappable = true;
1375 VM_OBJECT_LOCK(vm_obj);
1376 m = vm_phys_fictitious_to_vm_page(dev->agp->base + obj->gtt_offset +
1383 KASSERT((m->flags & PG_FICTITIOUS) != 0,
1384 ("not fictitious %p", m));
1385 KASSERT(m->wire_count == 1, ("wire_count not 1 %p", m));
1387 if ((m->flags & PG_BUSY) != 0) {
1390 vm_page_sleep(m, "915pbs");
1394 m->valid = VM_PAGE_BITS_ALL;
1395 vm_page_insert(m, vm_obj, OFF_TO_IDX(offset));
1398 vm_page_busy_try(m, false);
1404 vm_object_pip_wakeup(vm_obj);
1405 return (VM_PAGER_OK);
1410 KASSERT(ret != 0, ("i915_gem_pager_fault: wrong return"));
1411 if (ret == -EAGAIN || ret == -EIO || ret == -EINTR) {
1412 goto unlocked_vmobj;
1414 VM_OBJECT_LOCK(vm_obj);
1415 vm_object_pip_wakeup(vm_obj);
1416 return (VM_PAGER_ERROR);
1420 i915_gem_pager_dtor(void *handle)
1422 struct drm_gem_object *obj;
1423 struct drm_device *dev;
1429 drm_gem_free_mmap_offset(obj);
1430 i915_gem_release_mmap(to_intel_bo(obj));
1431 drm_gem_object_unreference(obj);
1435 struct cdev_pager_ops i915_gem_pager_ops = {
1436 .cdev_pg_fault = i915_gem_pager_fault,
1437 .cdev_pg_ctor = i915_gem_pager_ctor,
1438 .cdev_pg_dtor = i915_gem_pager_dtor
1442 i915_gem_mmap_gtt(struct drm_file *file, struct drm_device *dev,
1443 uint32_t handle, uint64_t *offset)
1445 struct drm_i915_private *dev_priv;
1446 struct drm_i915_gem_object *obj;
1449 if (!(dev->driver->driver_features & DRIVER_GEM))
1452 dev_priv = dev->dev_private;
1454 ret = i915_mutex_lock_interruptible(dev);
1458 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
1459 if (&obj->base == NULL) {
1464 if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
1469 if (obj->madv != I915_MADV_WILLNEED) {
1470 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1475 ret = drm_gem_create_mmap_offset(&obj->base);
1479 *offset = DRM_GEM_MAPPING_OFF(obj->base.map_list.key) |
1480 DRM_GEM_MAPPING_KEY;
1482 drm_gem_object_unreference(&obj->base);
1489 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1490 struct drm_file *file)
1492 struct drm_i915_private *dev_priv;
1493 struct drm_i915_gem_mmap_gtt *args;
1495 dev_priv = dev->dev_private;
1498 return (i915_gem_mmap_gtt(file, dev, args->handle, &args->offset));
1501 struct drm_i915_gem_object *
1502 i915_gem_alloc_object(struct drm_device *dev, size_t size)
1504 struct drm_i915_private *dev_priv;
1505 struct drm_i915_gem_object *obj;
1507 dev_priv = dev->dev_private;
1509 obj = kmalloc(sizeof(*obj), DRM_I915_GEM, M_WAITOK | M_ZERO);
1511 if (drm_gem_object_init(dev, &obj->base, size) != 0) {
1512 drm_free(obj, DRM_I915_GEM);
1516 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
1517 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
1520 obj->cache_level = I915_CACHE_LLC;
1522 obj->cache_level = I915_CACHE_NONE;
1523 obj->base.driver_private = NULL;
1524 obj->fence_reg = I915_FENCE_REG_NONE;
1525 INIT_LIST_HEAD(&obj->mm_list);
1526 INIT_LIST_HEAD(&obj->gtt_list);
1527 INIT_LIST_HEAD(&obj->ring_list);
1528 INIT_LIST_HEAD(&obj->exec_list);
1529 INIT_LIST_HEAD(&obj->gpu_write_list);
1530 obj->madv = I915_MADV_WILLNEED;
1531 /* Avoid an unnecessary call to unbind on the first bind. */
1532 obj->map_and_fenceable = true;
1534 i915_gem_info_add_obj(dev_priv, size);
1540 i915_gem_clflush_object(struct drm_i915_gem_object *obj)
1543 /* If we don't have a page list set up, then we're not pinned
1544 * to GPU, and we can ignore the cache flush because it'll happen
1545 * again at bind time.
1547 if (obj->pages == NULL)
1550 /* If the GPU is snooping the contents of the CPU cache,
1551 * we do not need to manually clear the CPU cache lines. However,
1552 * the caches are only snooped when the render cache is
1553 * flushed/invalidated. As we always have to emit invalidations
1554 * and flushes when moving into and out of the RENDER domain, correct
1555 * snooping behaviour occurs naturally as the result of our domain
1558 if (obj->cache_level != I915_CACHE_NONE)
1561 drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
1565 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
1567 uint32_t old_write_domain;
1569 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
1572 i915_gem_clflush_object(obj);
1573 intel_gtt_chipset_flush();
1574 old_write_domain = obj->base.write_domain;
1575 obj->base.write_domain = 0;
1578 /** Flushes the GTT write domain for the object if it's dirty. */
1580 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
1582 uint32_t old_write_domain;
1584 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
1587 /* No actual flushing is required for the GTT write domain. Writes
1588 * to it immediately go to main memory as far as we know, so there's
1589 * no chipset flush. It also doesn't land in render cache.
1591 * However, we do have to enforce the order so that all writes through
1592 * the GTT land before any writes to the device, such as updates to
1597 old_write_domain = obj->base.write_domain;
1598 obj->base.write_domain = 0;
1602 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
1604 uint32_t old_write_domain, old_read_domains;
1607 if (obj->gtt_space == NULL)
1610 if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
1613 ret = i915_gem_object_flush_gpu_write_domain(obj);
1617 if (obj->pending_gpu_write || write) {
1618 ret = i915_gem_object_wait_rendering(obj);
1623 i915_gem_object_flush_cpu_write_domain(obj);
1625 old_write_domain = obj->base.write_domain;
1626 old_read_domains = obj->base.read_domains;
1628 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0,
1629 ("In GTT write domain"));
1630 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
1632 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
1633 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
1640 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
1641 enum i915_cache_level cache_level)
1643 struct drm_device *dev = obj->base.dev;
1644 drm_i915_private_t *dev_priv = dev->dev_private;
1647 if (obj->cache_level == cache_level)
1650 if (obj->pin_count) {
1651 DRM_DEBUG("can not change the cache level of pinned objects\n");
1655 if (obj->gtt_space) {
1656 ret = i915_gem_object_finish_gpu(obj);
1660 i915_gem_object_finish_gtt(obj);
1662 /* Before SandyBridge, you could not use tiling or fence
1663 * registers with snooped memory, so relinquish any fences
1664 * currently pointing to our region in the aperture.
1666 if (INTEL_INFO(obj->base.dev)->gen < 6) {
1667 ret = i915_gem_object_put_fence(obj);
1672 if (obj->has_global_gtt_mapping)
1673 i915_gem_gtt_bind_object(obj, cache_level);
1674 if (obj->has_aliasing_ppgtt_mapping)
1675 i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
1679 if (cache_level == I915_CACHE_NONE) {
1680 u32 old_read_domains, old_write_domain;
1682 /* If we're coming from LLC cached, then we haven't
1683 * actually been tracking whether the data is in the
1684 * CPU cache or not, since we only allow one bit set
1685 * in obj->write_domain and have been skipping the clflushes.
1686 * Just set it to the CPU cache for now.
1688 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0,
1689 ("obj %p in CPU write domain", obj));
1690 KASSERT((obj->base.read_domains & ~I915_GEM_DOMAIN_CPU) == 0,
1691 ("obj %p in CPU read domain", obj));
1693 old_read_domains = obj->base.read_domains;
1694 old_write_domain = obj->base.write_domain;
1696 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
1697 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
1701 obj->cache_level = cache_level;
1706 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
1707 u32 alignment, struct intel_ring_buffer *pipelined)
1709 u32 old_read_domains, old_write_domain;
1712 ret = i915_gem_object_flush_gpu_write_domain(obj);
1716 if (pipelined != obj->ring) {
1717 ret = i915_gem_object_wait_rendering(obj);
1718 if (ret == -ERESTART || ret == -EINTR)
1722 ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
1726 ret = i915_gem_object_pin(obj, alignment, true);
1730 i915_gem_object_flush_cpu_write_domain(obj);
1732 old_write_domain = obj->base.write_domain;
1733 old_read_domains = obj->base.read_domains;
1735 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0,
1736 ("obj %p in GTT write domain", obj));
1737 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
1743 i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
1747 if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
1750 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
1751 ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
1756 ret = i915_gem_object_wait_rendering(obj);
1760 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1766 i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
1767 uint64_t offset, uint64_t size)
1769 uint32_t old_read_domains;
1772 if (offset == 0 && size == obj->base.size)
1773 return (i915_gem_object_set_to_cpu_domain(obj, 0));
1775 ret = i915_gem_object_flush_gpu_write_domain(obj);
1778 ret = i915_gem_object_wait_rendering(obj);
1782 i915_gem_object_flush_gtt_write_domain(obj);
1784 if (obj->page_cpu_valid == NULL &&
1785 (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0)
1788 if (obj->page_cpu_valid == NULL) {
1789 obj->page_cpu_valid = kmalloc(obj->base.size / PAGE_SIZE,
1790 DRM_I915_GEM, M_WAITOK | M_ZERO);
1791 } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
1792 memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE);
1794 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
1796 if (obj->page_cpu_valid[i])
1798 drm_clflush_pages(obj->pages + i, 1);
1799 obj->page_cpu_valid[i] = 1;
1802 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0,
1803 ("In gpu write domain"));
1805 old_read_domains = obj->base.read_domains;
1806 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
1812 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1816 if (INTEL_INFO(dev)->gen >= 4 ||
1817 tiling_mode == I915_TILING_NONE)
1820 /* Previous chips need a power-of-two fence region when tiling */
1821 if (INTEL_INFO(dev)->gen == 3)
1822 gtt_size = 1024*1024;
1824 gtt_size = 512*1024;
1826 while (gtt_size < size)
1833 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1834 * @obj: object to check
1836 * Return the required GTT alignment for an object, taking into account
1837 * potential fence register mapping.
1840 i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
1845 * Minimum alignment is 4k (GTT page size), but might be greater
1846 * if a fence register is needed for the object.
1848 if (INTEL_INFO(dev)->gen >= 4 ||
1849 tiling_mode == I915_TILING_NONE)
1853 * Previous chips need to be aligned to the size of the smallest
1854 * fence register that can contain the object.
1856 return (i915_gem_get_gtt_size(dev, size, tiling_mode));
1860 i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev, uint32_t size,
1864 if (tiling_mode == I915_TILING_NONE)
1868 * Minimum alignment is 4k (GTT page size) for sane hw.
1870 if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev))
1874 * Previous hardware however needs to be aligned to a power-of-two
1875 * tile height. The simplest method for determining this is to reuse
1876 * the power-of-tile object size.
1878 return (i915_gem_get_gtt_size(dev, size, tiling_mode));
1882 i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
1883 unsigned alignment, bool map_and_fenceable)
1885 struct drm_device *dev;
1886 struct drm_i915_private *dev_priv;
1887 struct drm_mm_node *free_space;
1888 uint32_t size, fence_size, fence_alignment, unfenced_alignment;
1889 bool mappable, fenceable;
1892 dev = obj->base.dev;
1893 dev_priv = dev->dev_private;
1895 if (obj->madv != I915_MADV_WILLNEED) {
1896 DRM_ERROR("Attempting to bind a purgeable object\n");
1900 fence_size = i915_gem_get_gtt_size(dev, obj->base.size,
1902 fence_alignment = i915_gem_get_gtt_alignment(dev, obj->base.size,
1904 unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(dev,
1905 obj->base.size, obj->tiling_mode);
1907 alignment = map_and_fenceable ? fence_alignment :
1909 if (map_and_fenceable && (alignment & (fence_alignment - 1)) != 0) {
1910 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
1914 size = map_and_fenceable ? fence_size : obj->base.size;
1916 /* If the object is bigger than the entire aperture, reject it early
1917 * before evicting everything in a vain attempt to find space.
1919 if (obj->base.size > (map_and_fenceable ?
1920 dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
1922 "Attempting to bind an object larger than the aperture\n");
1927 if (map_and_fenceable)
1928 free_space = drm_mm_search_free_in_range(
1929 &dev_priv->mm.gtt_space, size, alignment, 0,
1930 dev_priv->mm.gtt_mappable_end, 0);
1932 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
1933 size, alignment, 0);
1934 if (free_space != NULL) {
1936 if (map_and_fenceable)
1937 obj->gtt_space = drm_mm_get_block_range_generic(
1938 free_space, size, alignment, color, 0,
1939 dev_priv->mm.gtt_mappable_end, 1);
1941 obj->gtt_space = drm_mm_get_block_generic(free_space,
1942 size, alignment, color, 1);
1944 if (obj->gtt_space == NULL) {
1945 ret = i915_gem_evict_something(dev, size, alignment,
1951 ret = i915_gem_object_get_pages_gtt(obj, 0);
1953 drm_mm_put_block(obj->gtt_space);
1954 obj->gtt_space = NULL;
1956 * i915_gem_object_get_pages_gtt() cannot return
1957 * ENOMEM, since we use vm_page_grab(VM_ALLOC_RETRY)
1958 * (which does not support operation without a flag
1964 i915_gem_gtt_bind_object(obj, obj->cache_level);
1966 i915_gem_object_put_pages_gtt(obj);
1967 drm_mm_put_block(obj->gtt_space);
1968 obj->gtt_space = NULL;
1969 if (i915_gem_evict_everything(dev, false))
1974 list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
1975 list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1977 obj->gtt_offset = obj->gtt_space->start;
1980 obj->gtt_space->size == fence_size &&
1981 (obj->gtt_space->start & (fence_alignment - 1)) == 0;
1984 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
1985 obj->map_and_fenceable = mappable && fenceable;
1991 i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
1993 u32 old_write_domain, old_read_domains;
1995 /* Act a barrier for all accesses through the GTT */
1998 /* Force a pagefault for domain tracking on next user access */
1999 i915_gem_release_mmap(obj);
2001 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
2004 old_read_domains = obj->base.read_domains;
2005 old_write_domain = obj->base.write_domain;
2007 obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
2008 obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2013 i915_gem_object_unbind(struct drm_i915_gem_object *obj)
2015 drm_i915_private_t *dev_priv;
2018 dev_priv = obj->base.dev->dev_private;
2020 if (obj->gtt_space == NULL)
2022 if (obj->pin_count != 0) {
2023 DRM_ERROR("Attempting to unbind pinned buffer\n");
2027 ret = i915_gem_object_finish_gpu(obj);
2028 if (ret == -ERESTART || ret == -EINTR)
2031 i915_gem_object_finish_gtt(obj);
2034 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
2035 if (ret == -ERESTART || ret == -EINTR)
2038 i915_gem_clflush_object(obj);
2039 obj->base.read_domains = obj->base.write_domain =
2040 I915_GEM_DOMAIN_CPU;
2043 ret = i915_gem_object_put_fence(obj);
2044 if (ret == -ERESTART)
2047 i915_gem_gtt_unbind_object(obj);
2048 if (obj->has_aliasing_ppgtt_mapping) {
2049 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
2050 obj->has_aliasing_ppgtt_mapping = 0;
2052 i915_gem_object_put_pages_gtt(obj);
2054 list_del_init(&obj->gtt_list);
2055 list_del_init(&obj->mm_list);
2056 obj->map_and_fenceable = true;
2058 drm_mm_put_block(obj->gtt_space);
2059 obj->gtt_space = NULL;
2060 obj->gtt_offset = 0;
2062 if (i915_gem_object_is_purgeable(obj))
2063 i915_gem_object_truncate(obj);
2068 int i915_gpu_idle(struct drm_device *dev)
2070 drm_i915_private_t *dev_priv = dev->dev_private;
2071 struct intel_ring_buffer *ring;
2074 /* Flush everything onto the inactive list. */
2075 for_each_ring(ring, dev_priv, i) {
2076 ret = intel_ring_idle(ring);
2085 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
2088 struct drm_device *dev;
2091 int page_count, i, j;
2093 dev = obj->base.dev;
2094 KASSERT(obj->pages == NULL, ("Obj already has pages"));
2095 page_count = obj->base.size / PAGE_SIZE;
2096 obj->pages = kmalloc(page_count * sizeof(vm_page_t), DRM_I915_GEM,
2098 vm_obj = obj->base.vm_obj;
2099 VM_OBJECT_LOCK(vm_obj);
2100 for (i = 0; i < page_count; i++) {
2101 if ((obj->pages[i] = i915_gem_wire_page(vm_obj, i)) == NULL)
2104 VM_OBJECT_UNLOCK(vm_obj);
2105 if (i915_gem_object_needs_bit17_swizzle(obj))
2106 i915_gem_object_do_bit_17_swizzle(obj);
2110 for (j = 0; j < i; j++) {
2112 vm_page_busy_wait(m, FALSE, "i915gem");
2113 vm_page_unwire(m, 0);
2115 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
2117 VM_OBJECT_UNLOCK(vm_obj);
2118 drm_free(obj->pages, DRM_I915_GEM);
2123 #define GEM_PARANOID_CHECK_GTT 0
2124 #if GEM_PARANOID_CHECK_GTT
2126 i915_gem_assert_pages_not_mapped(struct drm_device *dev, vm_page_t *ma,
2129 struct drm_i915_private *dev_priv;
2131 unsigned long start, end;
2135 dev_priv = dev->dev_private;
2136 start = OFF_TO_IDX(dev_priv->mm.gtt_start);
2137 end = OFF_TO_IDX(dev_priv->mm.gtt_end);
2138 for (i = start; i < end; i++) {
2139 pa = intel_gtt_read_pte_paddr(i);
2140 for (j = 0; j < page_count; j++) {
2141 if (pa == VM_PAGE_TO_PHYS(ma[j])) {
2142 panic("Page %p in GTT pte index %d pte %x",
2143 ma[i], i, intel_gtt_read_pte(i));
2151 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
2156 BUG_ON(obj->madv == __I915_MADV_PURGED);
2158 if (obj->tiling_mode != I915_TILING_NONE)
2159 i915_gem_object_save_bit_17_swizzle(obj);
2160 if (obj->madv == I915_MADV_DONTNEED)
2162 page_count = obj->base.size / PAGE_SIZE;
2163 VM_OBJECT_LOCK(obj->base.vm_obj);
2164 #if GEM_PARANOID_CHECK_GTT
2165 i915_gem_assert_pages_not_mapped(obj->base.dev, obj->pages, page_count);
2167 for (i = 0; i < page_count; i++) {
2171 if (obj->madv == I915_MADV_WILLNEED)
2172 vm_page_reference(m);
2173 vm_page_busy_wait(obj->pages[i], FALSE, "i915gem");
2174 vm_page_unwire(obj->pages[i], 1);
2175 vm_page_wakeup(obj->pages[i]);
2176 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
2178 VM_OBJECT_UNLOCK(obj->base.vm_obj);
2180 drm_free(obj->pages, DRM_I915_GEM);
2185 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
2191 if (!obj->fault_mappable)
2194 devobj = cdev_pager_lookup(obj);
2195 if (devobj != NULL) {
2196 page_count = OFF_TO_IDX(obj->base.size);
2198 VM_OBJECT_LOCK(devobj);
2199 for (i = 0; i < page_count; i++) {
2200 m = vm_page_lookup_busy_wait(devobj, i, TRUE, "915unm");
2203 cdev_pager_free_page(devobj, m);
2205 VM_OBJECT_UNLOCK(devobj);
2206 vm_object_deallocate(devobj);
2209 obj->fault_mappable = false;
2213 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj)
2219 seqno = obj->last_rendering_seqno;
2224 ret = i915_wait_seqno(obj->ring, obj->last_rendering_seqno);
2229 /* Manually manage the write flush as we may have not yet
2230 * retired the buffer.
2232 if (obj->last_rendering_seqno &&
2233 i915_seqno_passed(seqno, obj->last_rendering_seqno)) {
2234 obj->last_rendering_seqno = 0;
2235 obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
2242 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
2243 struct intel_ring_buffer *ring, uint32_t seqno)
2245 struct drm_device *dev = obj->base.dev;
2246 struct drm_i915_private *dev_priv = dev->dev_private;
2247 struct drm_i915_fence_reg *reg;
2250 KASSERT(ring != NULL, ("NULL ring"));
2252 /* Add a reference if we're newly entering the active list. */
2254 drm_gem_object_reference(&obj->base);
2258 /* Move from whatever list we were on to the tail of execution. */
2259 list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
2260 list_move_tail(&obj->ring_list, &ring->active_list);
2262 obj->last_rendering_seqno = seqno;
2263 if (obj->fenced_gpu_access) {
2264 obj->last_fenced_seqno = seqno;
2265 obj->last_fenced_ring = ring;
2267 /* Bump MRU to take account of the delayed flush */
2268 if (obj->fence_reg != I915_FENCE_REG_NONE) {
2269 reg = &dev_priv->fence_regs[obj->fence_reg];
2270 list_move_tail(®->lru_list,
2271 &dev_priv->mm.fence_list);
2277 i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
2279 list_del_init(&obj->ring_list);
2280 obj->last_rendering_seqno = 0;
2281 obj->last_fenced_seqno = 0;
2285 i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
2287 struct drm_device *dev = obj->base.dev;
2288 drm_i915_private_t *dev_priv = dev->dev_private;
2290 KASSERT(obj->active, ("Object not active"));
2291 list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
2293 i915_gem_object_move_off_active(obj);
2297 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
2299 struct drm_device *dev = obj->base.dev;
2300 struct drm_i915_private *dev_priv = dev->dev_private;
2302 if (obj->pin_count != 0)
2303 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
2305 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2307 KASSERT(list_empty(&obj->gpu_write_list), ("On gpu_write_list"));
2308 KASSERT(obj->active, ("Object not active"));
2310 obj->last_fenced_ring = NULL;
2312 i915_gem_object_move_off_active(obj);
2313 obj->fenced_gpu_access = false;
2316 obj->pending_gpu_write = false;
2317 drm_gem_object_unreference(&obj->base);
2322 WARN_ON(i915_verify_lists(dev));
2326 /* Immediately discard the backing storage */
2328 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
2332 vm_obj = obj->base.vm_obj;
2333 VM_OBJECT_LOCK(vm_obj);
2334 vm_object_page_remove(vm_obj, 0, 0, false);
2335 VM_OBJECT_UNLOCK(vm_obj);
2336 obj->madv = __I915_MADV_PURGED;
2340 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
2343 return (obj->madv == I915_MADV_DONTNEED);
2347 i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
2348 uint32_t flush_domains)
2350 struct drm_i915_gem_object *obj, *next;
2351 uint32_t old_write_domain;
2353 list_for_each_entry_safe(obj, next, &ring->gpu_write_list,
2355 if (obj->base.write_domain & flush_domains) {
2356 old_write_domain = obj->base.write_domain;
2357 obj->base.write_domain = 0;
2358 list_del_init(&obj->gpu_write_list);
2359 i915_gem_object_move_to_active(obj, ring,
2360 i915_gem_next_request_seqno(ring));
2366 i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
2368 drm_i915_private_t *dev_priv;
2370 dev_priv = obj->base.dev->dev_private;
2371 return (dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
2372 obj->tiling_mode != I915_TILING_NONE);
2375 #define VM_OBJECT_LOCK_ASSERT_OWNED(object)
2378 i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex)
2383 VM_OBJECT_LOCK_ASSERT_OWNED(object);
2384 m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_RETRY);
2385 if (m->valid != VM_PAGE_BITS_ALL) {
2386 if (vm_pager_has_page(object, pindex)) {
2387 rv = vm_pager_get_page(object, &m, 1);
2388 m = vm_page_lookup(object, pindex);
2391 if (rv != VM_PAGER_OK) {
2396 pmap_zero_page(VM_PAGE_TO_PHYS(m));
2397 m->valid = VM_PAGE_BITS_ALL;
2403 atomic_add_long(&i915_gem_wired_pages_cnt, 1);
2408 i915_gem_flush_ring(struct intel_ring_buffer *ring, uint32_t invalidate_domains,
2409 uint32_t flush_domains)
2413 if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0)
2416 ret = ring->flush(ring, invalidate_domains, flush_domains);
2420 if (flush_domains & I915_GEM_GPU_DOMAINS)
2421 i915_gem_process_flushing_list(ring, flush_domains);
2426 * Waits for a sequence number to be signaled, and cleans up the
2427 * request and object lists appropriately for that event.
2430 i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
2432 drm_i915_private_t *dev_priv;
2433 struct drm_i915_gem_request *request;
2436 bool recovery_complete;
2437 bool do_retire = true;
2439 KASSERT(seqno != 0, ("Zero seqno"));
2441 dev_priv = ring->dev->dev_private;
2444 if (atomic_read(&dev_priv->mm.wedged) != 0) {
2445 /* Give the error handler a chance to run. */
2446 lockmgr(&dev_priv->error_completion_lock, LK_EXCLUSIVE);
2447 recovery_complete = (&dev_priv->error_completion) > 0;
2448 lockmgr(&dev_priv->error_completion_lock, LK_RELEASE);
2449 return (recovery_complete ? -EIO : -EAGAIN);
2452 if (seqno == ring->outstanding_lazy_request) {
2453 request = kmalloc(sizeof(*request), DRM_I915_GEM,
2455 if (request == NULL)
2458 ret = i915_add_request(ring, NULL, request);
2460 drm_free(request, DRM_I915_GEM);
2464 seqno = request->seqno;
2467 if (!i915_seqno_passed(ring->get_seqno(ring,false), seqno)) {
2468 if (HAS_PCH_SPLIT(ring->dev))
2469 ier = I915_READ(DEIER) | I915_READ(GTIER);
2471 ier = I915_READ(IER);
2473 DRM_ERROR("something (likely vbetool) disabled "
2474 "interrupts, re-enabling\n");
2475 ring->dev->driver->irq_preinstall(ring->dev);
2476 ring->dev->driver->irq_postinstall(ring->dev);
2479 lockmgr(&ring->irq_lock, LK_EXCLUSIVE);
2480 if (ring->irq_get(ring)) {
2481 flags = dev_priv->mm.interruptible ? PCATCH : 0;
2482 while (!i915_seqno_passed(ring->get_seqno(ring,false), seqno)
2483 && !atomic_read(&dev_priv->mm.wedged) &&
2485 ret = -lksleep(ring, &ring->irq_lock, flags,
2488 ring->irq_put(ring);
2489 lockmgr(&ring->irq_lock, LK_RELEASE);
2491 lockmgr(&ring->irq_lock, LK_RELEASE);
2492 if (_intel_wait_for(ring->dev,
2493 i915_seqno_passed(ring->get_seqno(ring,false), seqno) ||
2494 atomic_read(&dev_priv->mm.wedged), 3000,
2500 if (atomic_read(&dev_priv->mm.wedged))
2503 /* Directly dispatch request retiring. While we have the work queue
2504 * to handle this, the waiter on a request often wants an associated
2505 * buffer to have made it to the inactive list, and we would need
2506 * a separate wait queue to handle that.
2508 if (ret == 0 && do_retire)
2509 i915_gem_retire_requests_ring(ring);
2515 i915_gem_get_seqno(struct drm_device *dev)
2517 drm_i915_private_t *dev_priv = dev->dev_private;
2518 u32 seqno = dev_priv->next_seqno;
2520 /* reserve 0 for non-seqno */
2521 if (++dev_priv->next_seqno == 0)
2522 dev_priv->next_seqno = 1;
2528 i915_gem_next_request_seqno(struct intel_ring_buffer *ring)
2530 if (ring->outstanding_lazy_request == 0)
2531 ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev);
2533 return ring->outstanding_lazy_request;
2537 i915_add_request(struct intel_ring_buffer *ring, struct drm_file *file,
2538 struct drm_i915_gem_request *request)
2540 drm_i915_private_t *dev_priv;
2541 struct drm_i915_file_private *file_priv;
2543 u32 request_ring_position;
2547 KASSERT(request != NULL, ("NULL request in add"));
2548 DRM_LOCK_ASSERT(ring->dev);
2549 dev_priv = ring->dev->dev_private;
2551 seqno = i915_gem_next_request_seqno(ring);
2552 request_ring_position = intel_ring_get_tail(ring);
2554 ret = ring->add_request(ring, &seqno);
2558 request->seqno = seqno;
2559 request->ring = ring;
2560 request->tail = request_ring_position;
2561 request->emitted_jiffies = ticks;
2562 was_empty = list_empty(&ring->request_list);
2563 list_add_tail(&request->list, &ring->request_list);
2566 file_priv = file->driver_priv;
2568 spin_lock(&file_priv->mm.lock);
2569 request->file_priv = file_priv;
2570 list_add_tail(&request->client_list,
2571 &file_priv->mm.request_list);
2572 spin_unlock(&file_priv->mm.lock);
2575 ring->outstanding_lazy_request = 0;
2577 if (!dev_priv->mm.suspended) {
2578 if (i915_enable_hangcheck) {
2579 mod_timer(&dev_priv->hangcheck_timer,
2580 round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES));
2583 queue_delayed_work(dev_priv->wq,
2584 &dev_priv->mm.retire_work,
2585 round_jiffies_up_relative(hz));
2586 intel_mark_busy(dev_priv->dev);
2593 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
2595 struct drm_i915_file_private *file_priv = request->file_priv;
2600 DRM_LOCK_ASSERT(request->ring->dev);
2602 spin_lock(&file_priv->mm.lock);
2603 if (request->file_priv != NULL) {
2604 list_del(&request->client_list);
2605 request->file_priv = NULL;
2607 spin_unlock(&file_priv->mm.lock);
2611 i915_gem_release(struct drm_device *dev, struct drm_file *file)
2613 struct drm_i915_file_private *file_priv;
2614 struct drm_i915_gem_request *request;
2616 file_priv = file->driver_priv;
2618 /* Clean up our request list when the client is going away, so that
2619 * later retire_requests won't dereference our soon-to-be-gone
2622 spin_lock(&file_priv->mm.lock);
2623 while (!list_empty(&file_priv->mm.request_list)) {
2624 request = list_first_entry(&file_priv->mm.request_list,
2625 struct drm_i915_gem_request,
2627 list_del(&request->client_list);
2628 request->file_priv = NULL;
2630 spin_unlock(&file_priv->mm.lock);
2634 i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
2635 struct intel_ring_buffer *ring)
2638 if (ring->dev != NULL)
2639 DRM_LOCK_ASSERT(ring->dev);
2641 while (!list_empty(&ring->request_list)) {
2642 struct drm_i915_gem_request *request;
2644 request = list_first_entry(&ring->request_list,
2645 struct drm_i915_gem_request, list);
2647 list_del(&request->list);
2648 i915_gem_request_remove_from_client(request);
2649 drm_free(request, DRM_I915_GEM);
2652 while (!list_empty(&ring->active_list)) {
2653 struct drm_i915_gem_object *obj;
2655 obj = list_first_entry(&ring->active_list,
2656 struct drm_i915_gem_object, ring_list);
2658 obj->base.write_domain = 0;
2659 list_del_init(&obj->gpu_write_list);
2660 i915_gem_object_move_to_inactive(obj);
2665 i915_gem_reset_fences(struct drm_device *dev)
2667 struct drm_i915_private *dev_priv = dev->dev_private;
2670 for (i = 0; i < dev_priv->num_fence_regs; i++) {
2671 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
2672 struct drm_i915_gem_object *obj = reg->obj;
2677 if (obj->tiling_mode)
2678 i915_gem_release_mmap(obj);
2680 reg->obj->fence_reg = I915_FENCE_REG_NONE;
2681 reg->obj->fenced_gpu_access = false;
2682 reg->obj->last_fenced_seqno = 0;
2683 reg->obj->last_fenced_ring = NULL;
2684 i915_gem_clear_fence_reg(dev, reg);
2689 i915_gem_reset(struct drm_device *dev)
2691 struct drm_i915_private *dev_priv = dev->dev_private;
2692 struct drm_i915_gem_object *obj;
2695 for (i = 0; i < I915_NUM_RINGS; i++)
2696 i915_gem_reset_ring_lists(dev_priv, &dev_priv->ring[i]);
2698 /* Remove anything from the flushing lists. The GPU cache is likely
2699 * to be lost on reset along with the data, so simply move the
2700 * lost bo to the inactive list.
2702 while (!list_empty(&dev_priv->mm.flushing_list)) {
2703 obj = list_first_entry(&dev_priv->mm.flushing_list,
2704 struct drm_i915_gem_object,
2707 obj->base.write_domain = 0;
2708 list_del_init(&obj->gpu_write_list);
2709 i915_gem_object_move_to_inactive(obj);
2712 /* Move everything out of the GPU domains to ensure we do any
2713 * necessary invalidation upon reuse.
2715 list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) {
2716 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
2719 /* The fence registers are invalidated so clear them out */
2720 i915_gem_reset_fences(dev);
2724 * This function clears the request list as sequence numbers are passed.
2727 i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
2731 if (list_empty(&ring->request_list))
2734 seqno = ring->get_seqno(ring, true);
2736 while (!list_empty(&ring->request_list)) {
2737 struct drm_i915_gem_request *request;
2739 request = list_first_entry(&ring->request_list,
2740 struct drm_i915_gem_request,
2743 if (!i915_seqno_passed(seqno, request->seqno))
2746 /* We know the GPU must have read the request to have
2747 * sent us the seqno + interrupt, so use the position
2748 * of tail of the request to update the last known position
2751 ring->last_retired_head = request->tail;
2753 list_del(&request->list);
2754 i915_gem_request_remove_from_client(request);
2755 drm_free(request, DRM_I915_GEM);
2758 /* Move any buffers on the active list that are no longer referenced
2759 * by the ringbuffer to the flushing/inactive lists as appropriate.
2761 while (!list_empty(&ring->active_list)) {
2762 struct drm_i915_gem_object *obj;
2764 obj = list_first_entry(&ring->active_list,
2765 struct drm_i915_gem_object,
2768 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno))
2771 if (obj->base.write_domain != 0)
2772 i915_gem_object_move_to_flushing(obj);
2774 i915_gem_object_move_to_inactive(obj);
2777 if (unlikely(ring->trace_irq_seqno &&
2778 i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
2779 ring->irq_put(ring);
2780 ring->trace_irq_seqno = 0;
2786 i915_gem_retire_requests(struct drm_device *dev)
2788 drm_i915_private_t *dev_priv = dev->dev_private;
2789 struct drm_i915_gem_object *obj, *next;
2792 if (!list_empty(&dev_priv->mm.deferred_free_list)) {
2793 list_for_each_entry_safe(obj, next,
2794 &dev_priv->mm.deferred_free_list, mm_list)
2795 i915_gem_free_object_tail(obj);
2798 for (i = 0; i < I915_NUM_RINGS; i++)
2799 i915_gem_retire_requests_ring(&dev_priv->ring[i]);
2803 sandybridge_write_fence_reg(struct drm_i915_gem_object *obj,
2804 struct intel_ring_buffer *pipelined)
2806 struct drm_device *dev = obj->base.dev;
2807 drm_i915_private_t *dev_priv = dev->dev_private;
2808 u32 size = obj->gtt_space->size;
2809 int regnum = obj->fence_reg;
2812 val = (uint64_t)((obj->gtt_offset + size - 4096) &
2814 val |= obj->gtt_offset & 0xfffff000;
2815 val |= (uint64_t)((obj->stride / 128) - 1) <<
2816 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2818 if (obj->tiling_mode == I915_TILING_Y)
2819 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2820 val |= I965_FENCE_REG_VALID;
2823 int ret = intel_ring_begin(pipelined, 6);
2827 intel_ring_emit(pipelined, MI_NOOP);
2828 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
2829 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8);
2830 intel_ring_emit(pipelined, (u32)val);
2831 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4);
2832 intel_ring_emit(pipelined, (u32)(val >> 32));
2833 intel_ring_advance(pipelined);
2835 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val);
2841 i965_write_fence_reg(struct drm_i915_gem_object *obj,
2842 struct intel_ring_buffer *pipelined)
2844 struct drm_device *dev = obj->base.dev;
2845 drm_i915_private_t *dev_priv = dev->dev_private;
2846 u32 size = obj->gtt_space->size;
2847 int regnum = obj->fence_reg;
2850 val = (uint64_t)((obj->gtt_offset + size - 4096) &
2852 val |= obj->gtt_offset & 0xfffff000;
2853 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2854 if (obj->tiling_mode == I915_TILING_Y)
2855 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2856 val |= I965_FENCE_REG_VALID;
2859 int ret = intel_ring_begin(pipelined, 6);
2863 intel_ring_emit(pipelined, MI_NOOP);
2864 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
2865 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8);
2866 intel_ring_emit(pipelined, (u32)val);
2867 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4);
2868 intel_ring_emit(pipelined, (u32)(val >> 32));
2869 intel_ring_advance(pipelined);
2871 I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val);
2877 i915_write_fence_reg(struct drm_i915_gem_object *obj,
2878 struct intel_ring_buffer *pipelined)
2880 struct drm_device *dev = obj->base.dev;
2881 drm_i915_private_t *dev_priv = dev->dev_private;
2882 u32 size = obj->gtt_space->size;
2883 u32 fence_reg, val, pitch_val;
2886 if ((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
2887 (size & -size) != size || (obj->gtt_offset & (size - 1))) {
2889 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
2890 obj->gtt_offset, obj->map_and_fenceable, size);
2894 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
2899 /* Note: pitch better be a power of two tile widths */
2900 pitch_val = obj->stride / tile_width;
2901 pitch_val = ffs(pitch_val) - 1;
2903 val = obj->gtt_offset;
2904 if (obj->tiling_mode == I915_TILING_Y)
2905 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2906 val |= I915_FENCE_SIZE_BITS(size);
2907 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2908 val |= I830_FENCE_REG_VALID;
2910 fence_reg = obj->fence_reg;
2912 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
2914 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
2917 int ret = intel_ring_begin(pipelined, 4);
2921 intel_ring_emit(pipelined, MI_NOOP);
2922 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
2923 intel_ring_emit(pipelined, fence_reg);
2924 intel_ring_emit(pipelined, val);
2925 intel_ring_advance(pipelined);
2927 I915_WRITE(fence_reg, val);
2933 i830_write_fence_reg(struct drm_i915_gem_object *obj,
2934 struct intel_ring_buffer *pipelined)
2936 struct drm_device *dev = obj->base.dev;
2937 drm_i915_private_t *dev_priv = dev->dev_private;
2938 u32 size = obj->gtt_space->size;
2939 int regnum = obj->fence_reg;
2943 if ((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
2944 (size & -size) != size || (obj->gtt_offset & (size - 1))) {
2946 "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
2947 obj->gtt_offset, size);
2951 pitch_val = obj->stride / 128;
2952 pitch_val = ffs(pitch_val) - 1;
2954 val = obj->gtt_offset;
2955 if (obj->tiling_mode == I915_TILING_Y)
2956 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2957 val |= I830_FENCE_SIZE_BITS(size);
2958 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2959 val |= I830_FENCE_REG_VALID;
2962 int ret = intel_ring_begin(pipelined, 4);
2966 intel_ring_emit(pipelined, MI_NOOP);
2967 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
2968 intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4);
2969 intel_ring_emit(pipelined, val);
2970 intel_ring_advance(pipelined);
2972 I915_WRITE(FENCE_REG_830_0 + regnum * 4, val);
2977 static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)
2979 return i915_seqno_passed(ring->get_seqno(ring,false), seqno);
2983 i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
2984 struct intel_ring_buffer *pipelined)
2988 if (obj->fenced_gpu_access) {
2989 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
2990 ret = i915_gem_flush_ring(obj->last_fenced_ring, 0,
2991 obj->base.write_domain);
2996 obj->fenced_gpu_access = false;
2999 if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {
3000 if (!ring_passed_seqno(obj->last_fenced_ring,
3001 obj->last_fenced_seqno)) {
3002 ret = i915_wait_seqno(obj->last_fenced_ring,
3003 obj->last_fenced_seqno);
3008 obj->last_fenced_seqno = 0;
3009 obj->last_fenced_ring = NULL;
3012 /* Ensure that all CPU reads are completed before installing a fence
3013 * and all writes before removing the fence.
3015 if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
3022 i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
3026 if (obj->tiling_mode)
3027 i915_gem_release_mmap(obj);
3029 ret = i915_gem_object_flush_fence(obj, NULL);
3033 if (obj->fence_reg != I915_FENCE_REG_NONE) {
3034 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3036 if (dev_priv->fence_regs[obj->fence_reg].pin_count != 0)
3037 kprintf("%s: pin_count %d\n", __func__,
3038 dev_priv->fence_regs[obj->fence_reg].pin_count);
3039 i915_gem_clear_fence_reg(obj->base.dev,
3040 &dev_priv->fence_regs[obj->fence_reg]);
3042 obj->fence_reg = I915_FENCE_REG_NONE;
3048 static struct drm_i915_fence_reg *
3049 i915_find_fence_reg(struct drm_device *dev, struct intel_ring_buffer *pipelined)
3051 struct drm_i915_private *dev_priv = dev->dev_private;
3052 struct drm_i915_fence_reg *reg, *first, *avail;
3055 /* First try to find a free reg */
3057 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
3058 reg = &dev_priv->fence_regs[i];
3062 if (!reg->pin_count)
3069 /* None available, try to steal one or wait for a user to finish */
3070 avail = first = NULL;
3071 list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
3079 !reg->obj->last_fenced_ring ||
3080 reg->obj->last_fenced_ring == pipelined) {
3093 i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
3094 struct intel_ring_buffer *pipelined)
3096 struct drm_device *dev = obj->base.dev;
3097 struct drm_i915_private *dev_priv = dev->dev_private;
3098 struct drm_i915_fence_reg *reg;
3104 if (obj->fence_reg != I915_FENCE_REG_NONE) {
3105 reg = &dev_priv->fence_regs[obj->fence_reg];
3106 list_move_tail(®->lru_list, &dev_priv->mm.fence_list);
3108 if (obj->tiling_changed) {
3109 ret = i915_gem_object_flush_fence(obj, pipelined);
3113 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
3118 i915_gem_next_request_seqno(pipelined);
3119 obj->last_fenced_seqno = reg->setup_seqno;
3120 obj->last_fenced_ring = pipelined;
3127 if (reg->setup_seqno) {
3128 if (!ring_passed_seqno(obj->last_fenced_ring,
3129 reg->setup_seqno)) {
3130 ret = i915_wait_seqno(
3131 obj->last_fenced_ring,
3137 reg->setup_seqno = 0;
3139 } else if (obj->last_fenced_ring &&
3140 obj->last_fenced_ring != pipelined) {
3141 ret = i915_gem_object_flush_fence(obj, pipelined);
3146 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
3148 KASSERT(pipelined || reg->setup_seqno == 0, ("!pipelined"));
3150 if (obj->tiling_changed) {
3153 i915_gem_next_request_seqno(pipelined);
3154 obj->last_fenced_seqno = reg->setup_seqno;
3155 obj->last_fenced_ring = pipelined;
3163 reg = i915_find_fence_reg(dev, pipelined);
3167 ret = i915_gem_object_flush_fence(obj, pipelined);
3172 struct drm_i915_gem_object *old = reg->obj;
3174 drm_gem_object_reference(&old->base);
3176 if (old->tiling_mode)
3177 i915_gem_release_mmap(old);
3179 ret = i915_gem_object_flush_fence(old, pipelined);
3181 drm_gem_object_unreference(&old->base);
3185 if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0)
3188 old->fence_reg = I915_FENCE_REG_NONE;
3189 old->last_fenced_ring = pipelined;
3190 old->last_fenced_seqno =
3191 pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
3193 drm_gem_object_unreference(&old->base);
3194 } else if (obj->last_fenced_seqno == 0)
3198 list_move_tail(®->lru_list, &dev_priv->mm.fence_list);
3199 obj->fence_reg = reg - dev_priv->fence_regs;
3200 obj->last_fenced_ring = pipelined;
3203 pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
3204 obj->last_fenced_seqno = reg->setup_seqno;
3207 obj->tiling_changed = false;
3208 switch (INTEL_INFO(dev)->gen) {
3211 ret = sandybridge_write_fence_reg(obj, pipelined);
3215 ret = i965_write_fence_reg(obj, pipelined);
3218 ret = i915_write_fence_reg(obj, pipelined);
3221 ret = i830_write_fence_reg(obj, pipelined);
3229 i915_gem_clear_fence_reg(struct drm_device *dev, struct drm_i915_fence_reg *reg)
3231 drm_i915_private_t *dev_priv = dev->dev_private;
3232 uint32_t fence_reg = reg - dev_priv->fence_regs;
3234 switch (INTEL_INFO(dev)->gen) {
3237 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0);
3241 I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0);
3245 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
3248 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
3250 I915_WRITE(fence_reg, 0);
3254 list_del_init(®->lru_list);
3256 reg->setup_seqno = 0;
3261 i915_gem_init_object(struct drm_gem_object *obj)
3264 kprintf("i915_gem_init_object called\n");
3269 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
3272 return (obj->gtt_space && !obj->active && obj->pin_count == 0);
3276 i915_gem_retire_work_handler(struct work_struct *work)
3278 drm_i915_private_t *dev_priv;
3279 struct drm_device *dev;
3280 struct intel_ring_buffer *ring;
3284 dev_priv = container_of(work, drm_i915_private_t,
3285 mm.retire_work.work);
3286 dev = dev_priv->dev;
3288 /* Come back later if the device is busy... */
3289 if (lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_NOWAIT)) {
3290 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
3291 round_jiffies_up_relative(hz));
3295 i915_gem_retire_requests(dev);
3297 /* Send a periodic flush down the ring so we don't hold onto GEM
3298 * objects indefinitely.
3301 for_each_ring(ring, dev_priv, i) {
3302 if (ring->gpu_caches_dirty)
3303 i915_add_request(ring, NULL, NULL);
3305 idle &= list_empty(&ring->request_list);
3308 if (!dev_priv->mm.suspended && !idle)
3309 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
3310 round_jiffies_up_relative(hz));
3312 intel_mark_idle(dev);
3318 i915_gem_lastclose(struct drm_device *dev)
3322 if (drm_core_check_feature(dev, DRIVER_MODESET))
3325 ret = i915_gem_idle(dev);
3327 DRM_ERROR("failed to idle hardware: %d\n", ret);
3331 i915_gem_init_phys_object(struct drm_device *dev, int id, int size, int align)
3333 drm_i915_private_t *dev_priv;
3334 struct drm_i915_gem_phys_object *phys_obj;
3337 dev_priv = dev->dev_private;
3338 if (dev_priv->mm.phys_objs[id - 1] != NULL || size == 0)
3341 phys_obj = kmalloc(sizeof(struct drm_i915_gem_phys_object), DRM_I915_GEM,
3346 phys_obj->handle = drm_pci_alloc(dev, size, align, ~0);
3347 if (phys_obj->handle == NULL) {
3351 pmap_change_attr((vm_offset_t)phys_obj->handle->vaddr,
3352 size / PAGE_SIZE, PAT_WRITE_COMBINING);
3354 dev_priv->mm.phys_objs[id - 1] = phys_obj;
3359 drm_free(phys_obj, DRM_I915_GEM);
3364 i915_gem_free_phys_object(struct drm_device *dev, int id)
3366 drm_i915_private_t *dev_priv;
3367 struct drm_i915_gem_phys_object *phys_obj;
3369 dev_priv = dev->dev_private;
3370 if (dev_priv->mm.phys_objs[id - 1] == NULL)
3373 phys_obj = dev_priv->mm.phys_objs[id - 1];
3374 if (phys_obj->cur_obj != NULL)
3375 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
3377 drm_pci_free(dev, phys_obj->handle);
3378 drm_free(phys_obj, DRM_I915_GEM);
3379 dev_priv->mm.phys_objs[id - 1] = NULL;
3383 i915_gem_free_all_phys_object(struct drm_device *dev)
3387 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
3388 i915_gem_free_phys_object(dev, i);
3392 i915_gem_detach_phys_object(struct drm_device *dev,
3393 struct drm_i915_gem_object *obj)
3400 if (obj->phys_obj == NULL)
3402 vaddr = obj->phys_obj->handle->vaddr;
3404 page_count = obj->base.size / PAGE_SIZE;
3405 VM_OBJECT_LOCK(obj->base.vm_obj);
3406 for (i = 0; i < page_count; i++) {
3407 m = i915_gem_wire_page(obj->base.vm_obj, i);
3411 VM_OBJECT_UNLOCK(obj->base.vm_obj);
3412 sf = sf_buf_alloc(m);
3414 dst = (char *)sf_buf_kva(sf);
3415 memcpy(dst, vaddr + IDX_TO_OFF(i), PAGE_SIZE);
3418 drm_clflush_pages(&m, 1);
3420 VM_OBJECT_LOCK(obj->base.vm_obj);
3421 vm_page_reference(m);
3423 vm_page_busy_wait(m, FALSE, "i915gem");
3424 vm_page_unwire(m, 0);
3426 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
3428 VM_OBJECT_UNLOCK(obj->base.vm_obj);
3429 intel_gtt_chipset_flush();
3431 obj->phys_obj->cur_obj = NULL;
3432 obj->phys_obj = NULL;
3436 init_ring_lists(struct intel_ring_buffer *ring)
3439 INIT_LIST_HEAD(&ring->active_list);
3440 INIT_LIST_HEAD(&ring->request_list);
3441 INIT_LIST_HEAD(&ring->gpu_write_list);
3445 i915_gem_load(struct drm_device *dev)
3448 drm_i915_private_t *dev_priv = dev->dev_private;
3450 INIT_LIST_HEAD(&dev_priv->mm.active_list);
3451 INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
3452 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
3453 INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
3454 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
3455 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
3456 INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
3457 for (i = 0; i < I915_NUM_RINGS; i++)
3458 init_ring_lists(&dev_priv->ring[i]);
3459 for (i = 0; i < I915_MAX_NUM_FENCES; i++)
3460 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
3461 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
3462 i915_gem_retire_work_handler);
3463 dev_priv->error_completion = 0;
3465 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
3467 I915_WRITE(MI_ARB_STATE,
3468 _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
3471 dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
3473 /* Old X drivers will take 0-2 for front, back, depth buffers */
3474 if (!drm_core_check_feature(dev, DRIVER_MODESET))
3475 dev_priv->fence_reg_start = 3;
3477 if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
3478 dev_priv->num_fence_regs = 16;
3480 dev_priv->num_fence_regs = 8;
3482 /* Initialize fence registers to zero */
3483 i915_gem_reset_fences(dev);
3485 i915_gem_detect_bit_6_swizzle(dev);
3487 dev_priv->mm.interruptible = true;
3489 dev_priv->mm.i915_lowmem = EVENTHANDLER_REGISTER(vm_lowmem,
3490 i915_gem_lowmem, dev, EVENTHANDLER_PRI_ANY);
3494 i915_gem_attach_phys_object(struct drm_device *dev,
3495 struct drm_i915_gem_object *obj, int id, int align)
3497 drm_i915_private_t *dev_priv;
3501 int i, page_count, ret;
3503 if (id > I915_MAX_PHYS_OBJECT)
3506 if (obj->phys_obj != NULL) {
3507 if (obj->phys_obj->id == id)
3509 i915_gem_detach_phys_object(dev, obj);
3512 dev_priv = dev->dev_private;
3513 if (dev_priv->mm.phys_objs[id - 1] == NULL) {
3514 ret = i915_gem_init_phys_object(dev, id, obj->base.size, align);
3516 DRM_ERROR("failed to init phys object %d size: %zu\n",
3517 id, obj->base.size);
3522 /* bind to the object */
3523 obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
3524 obj->phys_obj->cur_obj = obj;
3526 page_count = obj->base.size / PAGE_SIZE;
3528 VM_OBJECT_LOCK(obj->base.vm_obj);
3530 for (i = 0; i < page_count; i++) {
3531 m = i915_gem_wire_page(obj->base.vm_obj, i);
3536 VM_OBJECT_UNLOCK(obj->base.vm_obj);
3537 sf = sf_buf_alloc(m);
3538 src = (char *)sf_buf_kva(sf);
3539 dst = (char *)obj->phys_obj->handle->vaddr + IDX_TO_OFF(i);
3540 memcpy(dst, src, PAGE_SIZE);
3543 VM_OBJECT_LOCK(obj->base.vm_obj);
3545 vm_page_reference(m);
3546 vm_page_busy_wait(m, FALSE, "i915gem");
3547 vm_page_unwire(m, 0);
3549 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
3551 VM_OBJECT_UNLOCK(obj->base.vm_obj);
3557 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_i915_gem_object *obj,
3558 uint64_t data_ptr, uint64_t offset, uint64_t size,
3559 struct drm_file *file_priv)
3561 char *user_data, *vaddr;
3564 vaddr = (char *)obj->phys_obj->handle->vaddr + offset;
3565 user_data = (char *)(uintptr_t)data_ptr;
3567 if (copyin_nofault(user_data, vaddr, size) != 0) {
3568 /* The physical object once assigned is fixed for the lifetime
3569 * of the obj, so we can safely drop the lock and continue
3573 ret = -copyin(user_data, vaddr, size);
3579 intel_gtt_chipset_flush();
3584 i915_gpu_is_active(struct drm_device *dev)
3586 drm_i915_private_t *dev_priv;
3588 dev_priv = dev->dev_private;
3589 return (!list_empty(&dev_priv->mm.flushing_list) ||
3590 !list_empty(&dev_priv->mm.active_list));
3594 i915_gem_lowmem(void *arg)
3596 struct drm_device *dev;
3597 struct drm_i915_private *dev_priv;
3598 struct drm_i915_gem_object *obj, *next;
3599 int cnt, cnt_fail, cnt_total;
3602 dev_priv = dev->dev_private;
3604 if (lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_NOWAIT))
3608 /* first scan for clean buffers */
3609 i915_gem_retire_requests(dev);
3611 cnt_total = cnt_fail = cnt = 0;
3613 list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list,
3615 if (i915_gem_object_is_purgeable(obj)) {
3616 if (i915_gem_object_unbind(obj) != 0)
3622 /* second pass, evict/count anything still on the inactive list */
3623 list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list,
3625 if (i915_gem_object_unbind(obj) == 0)
3631 if (cnt_fail > cnt_total / 100 && i915_gpu_is_active(dev)) {
3633 * We are desperate for pages, so as a last resort, wait
3634 * for the GPU to finish and discard whatever we can.
3635 * This has a dramatic impact to reduce the number of
3636 * OOM-killer events whilst running the GPU aggressively.
3638 if (i915_gpu_idle(dev) == 0)
3645 i915_gem_unload(struct drm_device *dev)
3647 struct drm_i915_private *dev_priv;
3649 dev_priv = dev->dev_private;
3650 EVENTHANDLER_DEREGISTER(vm_lowmem, dev_priv->mm.i915_lowmem);