sys/dev/agp/agp_i810.c

   1 /*
   2  * Copyright (c) 2000 Doug Rabson
   3  * Copyright (c) 2000 Ruslan Ermilov
   4  * Copyright (c) 2011 The FreeBSD Foundation
   5  * All rights reserved.
   6  *
   7  * Portions of this software were developed by Konstantin Belousov
   8  * under sponsorship from the FreeBSD Foundation.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  22  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  29  * SUCH DAMAGE.
  30  */
  31
  32 /*
  33  * Fixes for 830/845G support: David Dawes <dawes@xfree86.org>
  34  * 852GM/855GM/865G support added by David Dawes <dawes@xfree86.org>
  35  *
  36  * This is generic Intel GTT handling code, morphed from the AGP
  37  * bridge code.
  38  */
  39
  40 #if 0
  41 #define KTR_AGP_I810    KTR_DEV
  42 #else
  43 #define KTR_AGP_I810    0
  44 #endif
  45
  46 #include <sys/param.h>
  47 #include <sys/systm.h>
  48 #include <sys/malloc.h>
  49 #include <sys/kernel.h>
  50 #include <sys/bus.h>
  51 #include <sys/lock.h>
  52 #include <sys/rman.h>
  53
  54 #include "pcidevs.h"
  55 #include <bus/pci/pcivar.h>
  56 #include <bus/pci/pcireg.h>
  57 #include "agppriv.h"
  58 #include "agpreg.h"
  59 #include <drm/intel-gtt.h>
  60
  61 #include <vm/vm.h>
  62 #include <vm/vm_object.h>
  63 #include <vm/vm_page.h>
  64 #include <vm/vm_param.h>
  65 #include <vm/vm_pageout.h>
  66 #include <vm/pmap.h>
  67
  68 #include <vm/vm_page2.h>
  69
  70 #include <machine/md_var.h>
  71
  72 #define bus_read_1(r, o) \
  73                    bus_space_read_1((r)->r_bustag, (r)->r_bushandle, (o))
  74 #define bus_read_4(r, o) \
  75                    bus_space_read_4((r)->r_bustag, (r)->r_bushandle, (o))
  76 #define bus_write_4(r, o, v) \
  77                     bus_space_write_4((r)->r_bustag, (r)->r_bushandle, (o), (v))
  78
  79 MALLOC_DECLARE(M_AGP);
  80
  81 struct agp_i810_match;
  82
  83 static int agp_i915_check_active(device_t bridge_dev);
  84 static int agp_sb_check_active(device_t bridge_dev);
  85
  86 static void agp_i810_set_desc(device_t dev, const struct agp_i810_match *match);
  87
  88 static void agp_i915_dump_regs(device_t dev);
  89 static void agp_i965_dump_regs(device_t dev);
  90 static void agp_sb_dump_regs(device_t dev);
  91
  92 static int agp_i915_get_stolen_size(device_t dev);
  93 static int agp_sb_get_stolen_size(device_t dev);
  94 static int agp_gen8_get_stolen_size(device_t dev);
  95
  96 static int agp_i915_get_gtt_mappable_entries(device_t dev);
  97
  98 static int agp_i810_get_gtt_total_entries(device_t dev);
  99 static int agp_i965_get_gtt_total_entries(device_t dev);
 100 static int agp_gen5_get_gtt_total_entries(device_t dev);
 101 static int agp_sb_get_gtt_total_entries(device_t dev);
 102 static int agp_gen8_get_gtt_total_entries(device_t dev);
 103
 104 static int agp_i830_install_gatt(device_t dev);
 105 static int agp_sb_install_gatt(device_t dev);
 106 static int agp_vlv_install_gatt(device_t dev);
 107
 108 static void agp_i830_deinstall_gatt(device_t dev);
 109 static void agp_sb_deinstall_gatt(device_t dev);
 110
 111 static void agp_i915_install_gtt_pte(device_t dev, u_int index,
 112     vm_offset_t physical, int flags);
 113 static void agp_i965_install_gtt_pte(device_t dev, u_int index,
 114     vm_offset_t physical, int flags);
 115 static void agp_g4x_install_gtt_pte(device_t dev, u_int index,
 116     vm_offset_t physical, int flags);
 117 static void agp_sb_install_gtt_pte(device_t dev, u_int index,
 118     vm_offset_t physical, int flags);
 119 static void agp_vlv_install_gtt_pte(device_t dev, u_int index,
 120     vm_offset_t physical, int flags);
 121 static void agp_gen8_install_gtt_pte(device_t dev, u_int index,
 122     vm_offset_t physical, int flags);
 123
 124 static void agp_i915_write_gtt(device_t dev, u_int index, uint32_t pte);
 125 static void agp_i965_write_gtt(device_t dev, u_int index, uint32_t pte);
 126 static void agp_g4x_write_gtt(device_t dev, u_int index, uint32_t pte);
 127 static void agp_sb_write_gtt(device_t dev, u_int index, uint32_t pte);
 128
 129 static void agp_i915_sync_gtt_pte(device_t dev, u_int index);
 130 static void agp_i965_sync_gtt_pte(device_t dev, u_int index);
 131 static void agp_g4x_sync_gtt_pte(device_t dev, u_int index);
 132
 133 static int agp_i915_set_aperture(device_t dev, u_int32_t aperture);
 134
 135 static int agp_i810_chipset_flush_setup(device_t dev);
 136 static int agp_i915_chipset_flush_setup(device_t dev);
 137 static int agp_i965_chipset_flush_setup(device_t dev);
 138
 139 static void agp_i810_chipset_flush_teardown(device_t dev);
 140 static void agp_i915_chipset_flush_teardown(device_t dev);
 141 static void agp_i965_chipset_flush_teardown(device_t dev);
 142
 143 static void agp_i810_chipset_flush(device_t dev);
 144 static void agp_i915_chipset_flush(device_t dev);
 145
 146 enum {
 147         CHIP_I810,      /* i810/i815 */
 148         CHIP_I830,      /* 830M/845G */
 149         CHIP_I855,      /* 852GM/855GM/865G */
 150         CHIP_I915,      /* 915G/915GM */
 151         CHIP_I965,      /* G965 */
 152         CHIP_G33,       /* G33/Q33/Q35 */
 153         CHIP_IGD,       /* Pineview */
 154         CHIP_G4X,       /* G45/Q45 */
 155         CHIP_SB,        /* SandyBridge */
 156 };
 157
 158 /* The i810 through i855 have the registers at BAR 1, and the GATT gets
 159  * allocated by us.  The i915 has registers in BAR 0 and the GATT is at the
 160  * start of the stolen memory, and should only be accessed by the OS through
 161  * BAR 3.  The G965 has registers and GATT in the same BAR (0) -- first 512KB
 162  * is registers, second 512KB is GATT.
 163  */
 164 static struct resource_spec agp_i915_res_spec[] = {
 165         { SYS_RES_MEMORY, AGP_I915_MMADR, RF_ACTIVE | RF_SHAREABLE },
 166         { SYS_RES_MEMORY, AGP_I915_GTTADR, RF_ACTIVE | RF_SHAREABLE },
 167         { -1, 0 }
 168 };
 169
 170 static struct resource_spec agp_i965_res_spec[] = {
 171         { SYS_RES_MEMORY, AGP_I965_GTTMMADR, RF_ACTIVE | RF_SHAREABLE },
 172         { -1, 0 }
 173 };
 174
 175 static struct resource_spec agp_g4x_res_spec[] = {
 176         { SYS_RES_MEMORY, AGP_G4X_MMADR, RF_ACTIVE | RF_SHAREABLE },
 177         { SYS_RES_MEMORY, AGP_G4X_GTTADR, RF_ACTIVE | RF_SHAREABLE },
 178         { -1, 0 }
 179 };
 180
 181 struct agp_i810_softc {
 182         struct agp_softc agp;
 183         u_int32_t initial_aperture;     /* aperture size at startup */
 184         struct agp_gatt *gatt;
 185         u_int32_t dcache_size;          /* i810 only */
 186         u_int32_t stolen;               /* number of i830/845 gtt
 187                                            entries for stolen memory */
 188         u_int stolen_size;              /* BIOS-reserved graphics memory */
 189         u_int gtt_total_entries;        /* Total number of gtt ptes */
 190         u_int gtt_mappable_entries;     /* Number of gtt ptes mappable by CPU */
 191         device_t bdev;                  /* bridge device */
 192         void *argb_cursor;              /* contigmalloc area for ARGB cursor */
 193         struct resource *sc_res[3];
 194         const struct agp_i810_match *match;
 195         int sc_flush_page_rid;
 196         struct resource *sc_flush_page_res;
 197         void *sc_flush_page_vaddr;
 198         int sc_bios_allocated_flush_page;
 199 };
 200
 201 static device_t intel_agp;
 202
 203 struct agp_i810_driver {
 204         int chiptype;
 205         int gen;
 206         int busdma_addr_mask_sz;
 207         struct resource_spec *res_spec;
 208         int (*check_active)(device_t);
 209         void (*set_desc)(device_t, const struct agp_i810_match *);
 210         void (*dump_regs)(device_t);
 211         int (*get_stolen_size)(device_t);
 212         int (*get_gtt_total_entries)(device_t);
 213         int (*get_gtt_mappable_entries)(device_t);
 214         int (*install_gatt)(device_t);
 215         void (*deinstall_gatt)(device_t);
 216         void (*write_gtt)(device_t, u_int, uint32_t);
 217         void (*install_gtt_pte)(device_t, u_int, vm_offset_t, int);
 218         void (*sync_gtt_pte)(device_t, u_int);
 219         int (*set_aperture)(device_t, u_int32_t);
 220         int (*chipset_flush_setup)(device_t);
 221         void (*chipset_flush_teardown)(device_t);
 222         void (*chipset_flush)(device_t);
 223 };
 224
 225 static struct {
 226         struct intel_gtt base;
 227 } intel_private;
 228
 229 static const struct agp_i810_driver agp_i810_i915_driver = {
 230         .chiptype = CHIP_I915,
 231         .gen = 3,
 232         .busdma_addr_mask_sz = 32,
 233         .res_spec = agp_i915_res_spec,
 234         .check_active = agp_i915_check_active,
 235         .set_desc = agp_i810_set_desc,
 236         .dump_regs = agp_i915_dump_regs,
 237         .get_stolen_size = agp_i915_get_stolen_size,
 238         .get_gtt_mappable_entries = agp_i915_get_gtt_mappable_entries,
 239         .get_gtt_total_entries = agp_i810_get_gtt_total_entries,
 240         .install_gatt = agp_i830_install_gatt,
 241         .deinstall_gatt = agp_i830_deinstall_gatt,
 242         .write_gtt = agp_i915_write_gtt,
 243         .install_gtt_pte = agp_i915_install_gtt_pte,
 244         .sync_gtt_pte = agp_i915_sync_gtt_pte,
 245         .set_aperture = agp_i915_set_aperture,
 246         .chipset_flush_setup = agp_i915_chipset_flush_setup,
 247         .chipset_flush_teardown = agp_i915_chipset_flush_teardown,
 248         .chipset_flush = agp_i915_chipset_flush,
 249 };
 250
 251 static const struct agp_i810_driver agp_i810_g965_driver = {
 252         .chiptype = CHIP_I965,
 253         .gen = 4,
 254         .busdma_addr_mask_sz = 36,
 255         .res_spec = agp_i965_res_spec,
 256         .check_active = agp_i915_check_active,
 257         .set_desc = agp_i810_set_desc,
 258         .dump_regs = agp_i965_dump_regs,
 259         .get_stolen_size = agp_i915_get_stolen_size,
 260         .get_gtt_mappable_entries = agp_i915_get_gtt_mappable_entries,
 261         .get_gtt_total_entries = agp_i965_get_gtt_total_entries,
 262         .install_gatt = agp_i830_install_gatt,
 263         .deinstall_gatt = agp_i830_deinstall_gatt,
 264         .write_gtt = agp_i965_write_gtt,
 265         .install_gtt_pte = agp_i965_install_gtt_pte,
 266         .sync_gtt_pte = agp_i965_sync_gtt_pte,
 267         .set_aperture = agp_i915_set_aperture,
 268         .chipset_flush_setup = agp_i965_chipset_flush_setup,
 269         .chipset_flush_teardown = agp_i965_chipset_flush_teardown,
 270         .chipset_flush = agp_i915_chipset_flush,
 271 };
 272
 273 static const struct agp_i810_driver agp_i810_g33_driver = {
 274         .chiptype = CHIP_G33,
 275         .gen = 3,
 276         .busdma_addr_mask_sz = 36,
 277         .res_spec = agp_i915_res_spec,
 278         .check_active = agp_i915_check_active,
 279         .set_desc = agp_i810_set_desc,
 280         .dump_regs = agp_i965_dump_regs,
 281         .get_stolen_size = agp_i915_get_stolen_size,
 282         .get_gtt_mappable_entries = agp_i915_get_gtt_mappable_entries,
 283         .get_gtt_total_entries = agp_i965_get_gtt_total_entries,
 284         .install_gatt = agp_i830_install_gatt,
 285         .deinstall_gatt = agp_i830_deinstall_gatt,
 286         .write_gtt = agp_i915_write_gtt,
 287         .install_gtt_pte = agp_i965_install_gtt_pte,
 288         .sync_gtt_pte = agp_i915_sync_gtt_pte,
 289         .set_aperture = agp_i915_set_aperture,
 290         .chipset_flush_setup = agp_i965_chipset_flush_setup,
 291         .chipset_flush_teardown = agp_i965_chipset_flush_teardown,
 292         .chipset_flush = agp_i915_chipset_flush,
 293 };
 294
 295 static const struct agp_i810_driver pineview_gtt_driver = {
 296         .chiptype = CHIP_IGD,
 297         .gen = 3,
 298         .busdma_addr_mask_sz = 36,
 299         .res_spec = agp_i915_res_spec,
 300         .check_active = agp_i915_check_active,
 301         .set_desc = agp_i810_set_desc,
 302         .dump_regs = agp_i915_dump_regs,
 303         .get_stolen_size = agp_i915_get_stolen_size,
 304         .get_gtt_mappable_entries = agp_i915_get_gtt_mappable_entries,
 305         .get_gtt_total_entries = agp_i965_get_gtt_total_entries,
 306         .install_gatt = agp_i830_install_gatt,
 307         .deinstall_gatt = agp_i830_deinstall_gatt,
 308         .write_gtt = agp_i915_write_gtt,
 309         .install_gtt_pte = agp_i965_install_gtt_pte,
 310         .sync_gtt_pte = agp_i915_sync_gtt_pte,
 311         .set_aperture = agp_i915_set_aperture,
 312         .chipset_flush_setup = agp_i965_chipset_flush_setup,
 313         .chipset_flush_teardown = agp_i965_chipset_flush_teardown,
 314         .chipset_flush = agp_i915_chipset_flush,
 315 };
 316
 317 static const struct agp_i810_driver agp_i810_g4x_driver = {
 318         .chiptype = CHIP_G4X,
 319         .gen = 5,
 320         .busdma_addr_mask_sz = 36,
 321         .res_spec = agp_i965_res_spec,
 322         .check_active = agp_i915_check_active,
 323         .set_desc = agp_i810_set_desc,
 324         .dump_regs = agp_i965_dump_regs,
 325         .get_stolen_size = agp_i915_get_stolen_size,
 326         .get_gtt_mappable_entries = agp_i915_get_gtt_mappable_entries,
 327         .get_gtt_total_entries = agp_gen5_get_gtt_total_entries,
 328         .install_gatt = agp_i830_install_gatt,
 329         .deinstall_gatt = agp_i830_deinstall_gatt,
 330         .write_gtt = agp_g4x_write_gtt,
 331         .install_gtt_pte = agp_g4x_install_gtt_pte,
 332         .sync_gtt_pte = agp_g4x_sync_gtt_pte,
 333         .set_aperture = agp_i915_set_aperture,
 334         .chipset_flush_setup = agp_i965_chipset_flush_setup,
 335         .chipset_flush_teardown = agp_i965_chipset_flush_teardown,
 336         .chipset_flush = agp_i915_chipset_flush,
 337 };
 338
 339 static const struct agp_i810_driver agp_i810_sb_driver = {
 340         .chiptype = CHIP_SB,
 341         .gen = 6,
 342         .busdma_addr_mask_sz = 40,
 343         .res_spec = agp_g4x_res_spec,
 344         .check_active = agp_sb_check_active,
 345         .set_desc = agp_i810_set_desc,
 346         .dump_regs = agp_sb_dump_regs,
 347         .get_stolen_size = agp_sb_get_stolen_size,
 348         .get_gtt_mappable_entries = agp_i915_get_gtt_mappable_entries,
 349         .get_gtt_total_entries = agp_sb_get_gtt_total_entries,
 350         .install_gatt = agp_sb_install_gatt,
 351         .deinstall_gatt = agp_sb_deinstall_gatt,
 352         .write_gtt = agp_sb_write_gtt,
 353         .install_gtt_pte = agp_sb_install_gtt_pte,
 354         .sync_gtt_pte = agp_g4x_sync_gtt_pte,
 355         .set_aperture = agp_i915_set_aperture,
 356         .chipset_flush_setup = agp_i810_chipset_flush_setup,
 357         .chipset_flush_teardown = agp_i810_chipset_flush_teardown,
 358         .chipset_flush = agp_i810_chipset_flush,
 359 };
 360
 361 static const struct agp_i810_driver valleyview_gtt_driver = {
 362         .chiptype = CHIP_SB,
 363         .gen = 7,
 364         .busdma_addr_mask_sz = 40,
 365         .res_spec = agp_g4x_res_spec,
 366         .check_active = NULL,
 367         .set_desc = agp_i810_set_desc,
 368         .dump_regs = agp_sb_dump_regs,
 369         .get_stolen_size = agp_sb_get_stolen_size,
 370         .get_gtt_mappable_entries = agp_i915_get_gtt_mappable_entries,
 371         .get_gtt_total_entries = agp_sb_get_gtt_total_entries,
 372         .install_gatt = agp_vlv_install_gatt,
 373         .deinstall_gatt = agp_sb_deinstall_gatt,
 374         .write_gtt = agp_sb_write_gtt,
 375         .install_gtt_pte = agp_vlv_install_gtt_pte,
 376         .sync_gtt_pte = agp_g4x_sync_gtt_pte,
 377         .set_aperture = agp_i915_set_aperture,
 378         .chipset_flush_setup = agp_i810_chipset_flush_setup,
 379         .chipset_flush_teardown = agp_i810_chipset_flush_teardown,
 380         .chipset_flush = agp_i810_chipset_flush,
 381 };
 382
 383 static const struct agp_i810_driver broadwell_gtt_driver = {
 384         .chiptype = CHIP_SB,
 385         .gen = 8,
 386         .busdma_addr_mask_sz = 40,
 387         .res_spec = agp_g4x_res_spec,
 388         .check_active = agp_sb_check_active,
 389         .set_desc = agp_i810_set_desc,
 390         .dump_regs = agp_sb_dump_regs,
 391         .get_stolen_size = agp_gen8_get_stolen_size,
 392         .get_gtt_mappable_entries = agp_i915_get_gtt_mappable_entries,
 393         .get_gtt_total_entries = agp_gen8_get_gtt_total_entries,
 394         .install_gatt = agp_i830_install_gatt,
 395         .deinstall_gatt = agp_i830_deinstall_gatt,
 396         .write_gtt = NULL,
 397         .install_gtt_pte = agp_gen8_install_gtt_pte,
 398         .sync_gtt_pte = agp_g4x_sync_gtt_pte,
 399         .set_aperture = agp_i915_set_aperture,
 400         .chipset_flush_setup = agp_i810_chipset_flush_setup,
 401         .chipset_flush_teardown = agp_i810_chipset_flush_teardown,
 402         .chipset_flush = agp_i810_chipset_flush,
 403 };
 404
 405 /* For adding new devices, devid is the id of the graphics controller
 406  * (pci:0:2:0, for example).  The placeholder (usually at pci:0:2:1) for the
 407  * second head should never be added.  The bridge_offset is the offset to
 408  * subtract from devid to get the id of the hostb that the device is on.
 409  */
 410 static const struct agp_i810_match {
 411         uint16_t devid;
 412         char *name;
 413         const struct agp_i810_driver *driver;
 414 } agp_i810_matches[] = {
 415         {
 416                 .devid = 0x2582,
 417                 .name = "Intel 82915G (915G GMCH) SVGA controller",
 418                 .driver = &agp_i810_i915_driver
 419         },
 420         {
 421                 .devid = 0x258A,
 422                 .name = "Intel E7221 SVGA controller",
 423                 .driver = &agp_i810_i915_driver
 424         },
 425         {
 426                 .devid = 0x2592,
 427                 .name = "Intel 82915GM (915GM GMCH) SVGA controller",
 428                 .driver = &agp_i810_i915_driver
 429         },
 430         {
 431                 .devid = 0x2772,
 432                 .name = "Intel 82945G (945G GMCH) SVGA controller",
 433                 .driver = &agp_i810_i915_driver
 434         },
 435         {
 436                 .devid = 0x27A2,
 437                 .name = "Intel 82945GM (945GM GMCH) SVGA controller",
 438                 .driver = &agp_i810_i915_driver
 439         },
 440         {
 441                 .devid = 0x27AE,
 442                 .name = "Intel 945GME SVGA controller",
 443                 .driver = &agp_i810_i915_driver
 444         },
 445         {
 446                 .devid = 0x2972,
 447                 .name = "Intel 946GZ SVGA controller",
 448                 .driver = &agp_i810_g965_driver
 449         },
 450         {
 451                 .devid = 0x2982,
 452                 .name = "Intel G965 SVGA controller",
 453                 .driver = &agp_i810_g965_driver
 454         },
 455         {
 456                 .devid = 0x2992,
 457                 .name = "Intel Q965 SVGA controller",
 458                 .driver = &agp_i810_g965_driver
 459         },
 460         {
 461                 .devid = 0x29A2,
 462                 .name = "Intel G965 SVGA controller",
 463                 .driver = &agp_i810_g965_driver
 464         },
 465         {
 466                 .devid = 0x29B2,
 467                 .name = "Intel Q35 SVGA controller",
 468                 .driver = &agp_i810_g33_driver
 469         },
 470         {
 471                 .devid = 0x29C2,
 472                 .name = "Intel G33 SVGA controller",
 473                 .driver = &agp_i810_g33_driver
 474         },
 475         {
 476                 .devid = 0x29D2,
 477                 .name = "Intel Q33 SVGA controller",
 478                 .driver = &agp_i810_g33_driver
 479         },
 480         {
 481                 .devid = 0xA001,
 482                 .name = "Intel Pineview SVGA controller",
 483                 .driver = &pineview_gtt_driver
 484         },
 485         {
 486                 .devid = 0xA011,
 487                 .name = "Intel Pineview (M) SVGA controller",
 488                 .driver = &pineview_gtt_driver
 489         },
 490         {
 491                 .devid = 0x2A02,
 492                 .name = "Intel GM965 SVGA controller",
 493                 .driver = &agp_i810_g965_driver
 494         },
 495         {
 496                 .devid = 0x2A12,
 497                 .name = "Intel GME965 SVGA controller",
 498                 .driver = &agp_i810_g965_driver
 499         },
 500         {
 501                 .devid = 0x2A42,
 502                 .name = "Intel GM45 SVGA controller",
 503                 .driver = &agp_i810_g4x_driver
 504         },
 505         {
 506                 .devid = 0x2E02,
 507                 .name = "Intel Eaglelake SVGA controller",
 508                 .driver = &agp_i810_g4x_driver
 509         },
 510         {
 511                 .devid = 0x2E12,
 512                 .name = "Intel Q45 SVGA controller",
 513                 .driver = &agp_i810_g4x_driver
 514         },
 515         {
 516                 .devid = 0x2E22,
 517                 .name = "Intel G45 SVGA controller",
 518                 .driver = &agp_i810_g4x_driver
 519         },
 520         {
 521                 .devid = 0x2E32,
 522                 .name = "Intel G41 SVGA controller",
 523                 .driver = &agp_i810_g4x_driver
 524         },
 525         {
 526                 .devid = 0x0042,
 527                 .name = "Intel Ironlake (D) SVGA controller",
 528                 .driver = &agp_i810_g4x_driver
 529         },
 530         {
 531                 .devid = 0x0046,
 532                 .name = "Intel Ironlake (M) SVGA controller",
 533                 .driver = &agp_i810_g4x_driver
 534         },
 535         {
 536                 .devid = 0x0102,
 537                 .name = "SandyBridge desktop GT1 IG",
 538                 .driver = &agp_i810_sb_driver
 539         },
 540         {
 541                 .devid = 0x0112,
 542                 .name = "SandyBridge desktop GT2 IG",
 543                 .driver = &agp_i810_sb_driver
 544         },
 545         {
 546                 .devid = 0x0122,
 547                 .name = "SandyBridge desktop GT2+ IG",
 548                 .driver = &agp_i810_sb_driver
 549         },
 550         {
 551                 .devid = 0x0106,
 552                 .name = "SandyBridge mobile GT1 IG",
 553                 .driver = &agp_i810_sb_driver
 554         },
 555         {
 556                 .devid = 0x0116,
 557                 .name = "SandyBridge mobile GT2 IG",
 558                 .driver = &agp_i810_sb_driver
 559         },
 560         {
 561                 .devid = 0x0126,
 562                 .name = "SandyBridge mobile GT2+ IG",
 563                 .driver = &agp_i810_sb_driver
 564         },
 565         {
 566                 .devid = 0x010a,
 567                 .name = "SandyBridge server IG",
 568                 .driver = &agp_i810_sb_driver
 569         },
 570         {
 571                 .devid = 0x0152,
 572                 .name = "IvyBridge desktop GT1 IG",
 573                 .driver = &agp_i810_sb_driver
 574         },
 575         {
 576                 .devid = 0x0162,
 577                 .name = "IvyBridge desktop GT2 IG",
 578                 .driver = &agp_i810_sb_driver
 579         },
 580         {
 581                 .devid = 0x0156,
 582                 .name = "IvyBridge mobile GT1 IG",
 583                 .driver = &agp_i810_sb_driver
 584         },
 585         {
 586                 .devid = 0x0166,
 587                 .name = "IvyBridge mobile GT2 IG",
 588                 .driver = &agp_i810_sb_driver
 589         },
 590         {
 591                 .devid = 0x015a,
 592                 .name = "IvyBridge server GT1 IG",
 593                 .driver = &agp_i810_sb_driver
 594         },
 595         {
 596                 .devid = 0x016a,
 597                 .name = "IvyBridge server GT2 IG",
 598                 .driver = &agp_i810_sb_driver
 599         },
 600
 601         {       0x0f30, "ValleyView/Baytrail", &valleyview_gtt_driver },
 602         {       0x0f31, "ValleyView/Baytrail", &valleyview_gtt_driver },
 603         {       0x0f32, "ValleyView/Baytrail", &valleyview_gtt_driver },
 604         {       0x0f33, "ValleyView/Baytrail", &valleyview_gtt_driver },
 605         {       0x0155, "ValleyView/Baytrail", &valleyview_gtt_driver },
 606         {       0x0157, "ValleyView/Baytrail", &valleyview_gtt_driver },
 607
 608         {
 609                 .devid = 0x0402,
 610                 .name = "Haswell desktop GT1 IG",
 611                 .driver = &agp_i810_sb_driver
 612         },
 613         {
 614                 .devid = 0x0412,
 615                 .name = "Haswell desktop GT2 IG",
 616                 .driver = &agp_i810_sb_driver
 617         },
 618         {       0x041e, "Haswell", &agp_i810_sb_driver },
 619         {       0x0422, "Haswell", &agp_i810_sb_driver },
 620         {
 621                 .devid = 0x0406,
 622                 .name = "Haswell mobile GT1 IG",
 623                 .driver = &agp_i810_sb_driver
 624         },
 625         {
 626                 .devid = 0x0416,
 627                 .name = "Haswell mobile GT2 IG",
 628                 .driver = &agp_i810_sb_driver
 629         },
 630         {       0x0426, "Haswell", &agp_i810_sb_driver },
 631         {
 632                 .devid = 0x040a,
 633                 .name = "Haswell server GT1 IG",
 634                 .driver = &agp_i810_sb_driver
 635         },
 636         {
 637                 .devid = 0x041a,
 638                 .name = "Haswell server GT2 IG",
 639                 .driver = &agp_i810_sb_driver
 640         },
 641         {       0x042a, "Haswell", &agp_i810_sb_driver },
 642         {       0x0c02, "Haswell", &agp_i810_sb_driver },
 643         {       0x0c12, "Haswell", &agp_i810_sb_driver },
 644         {       0x0c22, "Haswell", &agp_i810_sb_driver },
 645         {       0x0c06, "Haswell", &agp_i810_sb_driver },
 646         {
 647                 .devid = 0x0c16,
 648                 .name = "Haswell SDV",
 649                 .driver = &agp_i810_sb_driver
 650         },
 651         {       0x0c26, "Haswell", &agp_i810_sb_driver },
 652         {       0x0c0a, "Haswell", &agp_i810_sb_driver },
 653         {       0x0c1a, "Haswell", &agp_i810_sb_driver },
 654         {       0x0c2a, "Haswell", &agp_i810_sb_driver },
 655         {       0x0a02, "Haswell", &agp_i810_sb_driver },
 656         {       0x0a12, "Haswell", &agp_i810_sb_driver },
 657         {       0x0a22, "Haswell", &agp_i810_sb_driver },
 658         {       0x0a06, "Haswell", &agp_i810_sb_driver },
 659         {       0x0a16, "Haswell", &agp_i810_sb_driver },
 660         {       0x0a26, "Haswell", &agp_i810_sb_driver },
 661         {       0x0a0a, "Haswell", &agp_i810_sb_driver },
 662         {       0x0a1a, "Haswell", &agp_i810_sb_driver },
 663         {       0x0a2a, "Haswell", &agp_i810_sb_driver },
 664         {       0x0d12, "Haswell", &agp_i810_sb_driver },
 665         {       0x0d22, "Haswell", &agp_i810_sb_driver },
 666         {       0x0d32, "Haswell", &agp_i810_sb_driver },
 667         {       0x0d16, "Haswell", &agp_i810_sb_driver },
 668         {       0x0d26, "Haswell", &agp_i810_sb_driver },
 669         {       0x0d36, "Haswell", &agp_i810_sb_driver },
 670         {       0x0d1a, "Haswell", &agp_i810_sb_driver },
 671         {       0x0d2a, "Haswell", &agp_i810_sb_driver },
 672         {       0x0d3a, "Haswell", &agp_i810_sb_driver },
 673
 674         {       0x1602, "Broadwell", &broadwell_gtt_driver }, /* m */
 675         {       0x1606, "Broadwell", &broadwell_gtt_driver },
 676         {       0x160B, "Broadwell", &broadwell_gtt_driver },
 677         {       0x160E, "Broadwell", &broadwell_gtt_driver },
 678         {       0x1616, "Broadwell", &broadwell_gtt_driver },
 679         {       0x161E, "Broadwell", &broadwell_gtt_driver },
 680
 681         {       0x160A, "Broadwell", &broadwell_gtt_driver }, /* d */
 682         {       0x160D, "Broadwell", &broadwell_gtt_driver },
 683         {
 684                 .devid = 0,
 685         }
 686 };
 687
 688 static const struct agp_i810_match*
 689 agp_i810_match(device_t dev)
 690 {
 691         int i, devid;
 692
 693         if (pci_get_vendor(dev) != PCI_VENDOR_INTEL)
 694                 return (NULL);
 695
 696         devid = pci_get_device(dev);
 697         for (i = 0; agp_i810_matches[i].devid != 0; i++) {
 698                 if (agp_i810_matches[i].devid == devid)
 699                         break;
 700         }
 701         if (agp_i810_matches[i].devid == 0)
 702                 return (NULL);
 703         else
 704                 return (&agp_i810_matches[i]);
 705 }
 706
 707 /*
 708  * Find bridge device.
 709  */
 710 static device_t
 711 agp_i810_find_bridge(device_t dev)
 712 {
 713
 714         return (pci_find_dbsf(0, 0, 0, 0));
 715 }
 716
 717 static void
 718 agp_i810_identify(driver_t *driver, device_t parent)
 719 {
 720
 721         if (device_find_child(parent, "agp", -1) == NULL &&
 722             agp_i810_match(parent))
 723                 device_add_child(parent, "agp", -1);
 724 }
 725
 726 static int
 727 agp_i915_check_active(device_t bridge_dev)
 728 {
 729         int deven;
 730
 731         deven = pci_read_config(bridge_dev, AGP_I915_DEVEN, 4);
 732         if ((deven & AGP_I915_DEVEN_D2F0) == AGP_I915_DEVEN_D2F0_DISABLED)
 733                 return (ENXIO);
 734         return (0);
 735 }
 736
 737 static int
 738 agp_sb_check_active(device_t bridge_dev)
 739 {
 740         int deven;
 741
 742         deven = pci_read_config(bridge_dev, AGP_I915_DEVEN, 4);
 743         if ((deven & AGP_SB_DEVEN_D2EN) == AGP_SB_DEVEN_D2EN_DISABLED)
 744                 return (ENXIO);
 745         return (0);
 746 }
 747
 748 static void
 749 agp_i810_set_desc(device_t dev, const struct agp_i810_match *match)
 750 {
 751
 752         device_set_desc(dev, match->name);
 753 }
 754
 755 static int
 756 agp_i810_probe(device_t dev)
 757 {
 758         device_t bdev;
 759         const struct agp_i810_match *match;
 760         int err = 0;
 761
 762         if (resource_disabled("agp", device_get_unit(dev)))
 763                 return (ENXIO);
 764         match = agp_i810_match(dev);
 765         if (match == NULL)
 766                 return (ENXIO);
 767
 768         bdev = agp_i810_find_bridge(dev);
 769         if (bdev == NULL) {
 770                 if (bootverbose)
 771                         kprintf("I810: can't find bridge device\n");
 772                 return (ENXIO);
 773         }
 774
 775         /*
 776          * checking whether internal graphics device has been activated.
 777          */
 778         if (match->driver->check_active != NULL) {
 779                 err = match->driver->check_active(bdev);
 780                 if (err != 0) {
 781                         if (bootverbose)
 782                                 kprintf("i810: disabled, not probing\n");
 783                         return (err);
 784                 }
 785         }
 786
 787         match->driver->set_desc(dev, match);
 788         return (BUS_PROBE_DEFAULT);
 789 }
 790
 791 static void
 792 agp_i915_dump_regs(device_t dev)
 793 {
 794         struct agp_i810_softc *sc = device_get_softc(dev);
 795
 796         device_printf(dev, "AGP_I810_PGTBL_CTL: %08x\n",
 797             bus_read_4(sc->sc_res[0], AGP_I810_PGTBL_CTL));
 798         device_printf(dev, "AGP_I855_GCC1: 0x%02x\n",
 799             pci_read_config(sc->bdev, AGP_I855_GCC1, 1));
 800         device_printf(dev, "AGP_I915_MSAC: 0x%02x\n",
 801             pci_read_config(sc->bdev, AGP_I915_MSAC, 1));
 802 }
 803
 804 static void
 805 agp_i965_dump_regs(device_t dev)
 806 {
 807         struct agp_i810_softc *sc = device_get_softc(dev);
 808
 809         device_printf(dev, "AGP_I965_PGTBL_CTL2: %08x\n",
 810             bus_read_4(sc->sc_res[0], AGP_I965_PGTBL_CTL2));
 811         device_printf(dev, "AGP_I855_GCC1: 0x%02x\n",
 812             pci_read_config(dev, AGP_I855_GCC1, 1));
 813         device_printf(dev, "AGP_I965_MSAC: 0x%02x\n",
 814             pci_read_config(dev, AGP_I965_MSAC, 1));
 815 }
 816
 817 static void
 818 agp_sb_dump_regs(device_t dev)
 819 {
 820         struct agp_i810_softc *sc = device_get_softc(dev);
 821
 822         device_printf(dev, "AGP_SNB_GFX_MODE: %08x\n",
 823             bus_read_4(sc->sc_res[0], AGP_SNB_GFX_MODE));
 824         device_printf(dev, "AGP_SNB_GCC1: 0x%04x\n",
 825             pci_read_config(dev, AGP_SNB_GCC1, 2));
 826 }
 827
 828 static int
 829 agp_i915_get_stolen_size(device_t dev)
 830 {
 831         struct agp_i810_softc *sc;
 832         unsigned int gcc1, stolen, gtt_size;
 833
 834         sc = device_get_softc(dev);
 835
 836         /*
 837          * Stolen memory is set up at the beginning of the aperture by
 838          * the BIOS, consisting of the GATT followed by 4kb for the
 839          * BIOS display.
 840          */
 841         switch (sc->match->driver->chiptype) {
 842         case CHIP_I855:
 843                 gtt_size = 128;
 844                 break;
 845         case CHIP_I915:
 846                 gtt_size = 256;
 847                 break;
 848         case CHIP_I965:
 849                 switch (bus_read_4(sc->sc_res[0], AGP_I810_PGTBL_CTL) &
 850                         AGP_I810_PGTBL_SIZE_MASK) {
 851                 case AGP_I810_PGTBL_SIZE_128KB:
 852                         gtt_size = 128;
 853                         break;
 854                 case AGP_I810_PGTBL_SIZE_256KB:
 855                         gtt_size = 256;
 856                         break;
 857                 case AGP_I810_PGTBL_SIZE_512KB:
 858                         gtt_size = 512;
 859                         break;
 860                 case AGP_I965_PGTBL_SIZE_1MB:
 861                         gtt_size = 1024;
 862                         break;
 863                 case AGP_I965_PGTBL_SIZE_2MB:
 864                         gtt_size = 2048;
 865                         break;
 866                 case AGP_I965_PGTBL_SIZE_1_5MB:
 867                         gtt_size = 1024 + 512;
 868                         break;
 869                 default:
 870                         device_printf(dev, "Bad PGTBL size\n");
 871                         return (EINVAL);
 872                 }
 873                 break;
 874         case CHIP_G33:
 875                 gcc1 = pci_read_config(sc->bdev, AGP_I855_GCC1, 2);
 876                 switch (gcc1 & AGP_G33_MGGC_GGMS_MASK) {
 877                 case AGP_G33_MGGC_GGMS_SIZE_1M:
 878                         gtt_size = 1024;
 879                         break;
 880                 case AGP_G33_MGGC_GGMS_SIZE_2M:
 881                         gtt_size = 2048;
 882                         break;
 883                 default:
 884                         device_printf(dev, "Bad PGTBL size\n");
 885                         return (EINVAL);
 886                 }
 887                 break;
 888         case CHIP_IGD:
 889         case CHIP_G4X:
 890                 gtt_size = 0;
 891                 break;
 892         default:
 893                 device_printf(dev, "Bad chiptype\n");
 894                 return (EINVAL);
 895         }
 896
 897         /* GCC1 is called MGGC on i915+ */
 898         gcc1 = pci_read_config(sc->bdev, AGP_I855_GCC1, 1);
 899         switch (gcc1 & AGP_I855_GCC1_GMS) {
 900         case AGP_I855_GCC1_GMS_STOLEN_1M:
 901                 stolen = 1024;
 902                 break;
 903         case AGP_I855_GCC1_GMS_STOLEN_4M:
 904                 stolen = 4 * 1024;
 905                 break;
 906         case AGP_I855_GCC1_GMS_STOLEN_8M:
 907                 stolen = 8 * 1024;
 908                 break;
 909         case AGP_I855_GCC1_GMS_STOLEN_16M:
 910                 stolen = 16 * 1024;
 911                 break;
 912         case AGP_I855_GCC1_GMS_STOLEN_32M:
 913                 stolen = 32 * 1024;
 914                 break;
 915         case AGP_I915_GCC1_GMS_STOLEN_48M:
 916                 stolen = sc->match->driver->gen > 2 ? 48 * 1024 : 0;
 917                 break;
 918         case AGP_I915_GCC1_GMS_STOLEN_64M:
 919                 stolen = sc->match->driver->gen > 2 ? 64 * 1024 : 0;
 920                 break;
 921         case AGP_G33_GCC1_GMS_STOLEN_128M:
 922                 stolen = sc->match->driver->gen > 2 ? 128 * 1024 : 0;
 923                 break;
 924         case AGP_G33_GCC1_GMS_STOLEN_256M:
 925                 stolen = sc->match->driver->gen > 2 ? 256 * 1024 : 0;
 926                 break;
 927         case AGP_G4X_GCC1_GMS_STOLEN_96M:
 928                 if (sc->match->driver->chiptype == CHIP_I965 ||
 929                     sc->match->driver->chiptype == CHIP_G4X)
 930                         stolen = 96 * 1024;
 931                 else
 932                         stolen = 0;
 933                 break;
 934         case AGP_G4X_GCC1_GMS_STOLEN_160M:
 935                 if (sc->match->driver->chiptype == CHIP_I965 ||
 936                     sc->match->driver->chiptype == CHIP_G4X)
 937                         stolen = 160 * 1024;
 938                 else
 939                         stolen = 0;
 940                 break;
 941         case AGP_G4X_GCC1_GMS_STOLEN_224M:
 942                 if (sc->match->driver->chiptype == CHIP_I965 ||
 943                     sc->match->driver->chiptype == CHIP_G4X)
 944                         stolen = 224 * 1024;
 945                 else
 946                         stolen = 0;
 947                 break;
 948         case AGP_G4X_GCC1_GMS_STOLEN_352M:
 949                 if (sc->match->driver->chiptype == CHIP_I965 ||
 950                     sc->match->driver->chiptype == CHIP_G4X)
 951                         stolen = 352 * 1024;
 952                 else
 953                         stolen = 0;
 954                 break;
 955         default:
 956                 device_printf(dev,
 957                     "unknown memory configuration, disabling (GCC1 %x)\n",
 958                     gcc1);
 959                 return (EINVAL);
 960         }
 961
 962         gtt_size += 4;
 963         sc->stolen_size = stolen * 1024;
 964         sc->stolen = (stolen - gtt_size) * 1024 / 4096;
 965
 966         return (0);
 967 }
 968
 969 static int
 970 agp_sb_get_stolen_size(device_t dev)
 971 {
 972         struct agp_i810_softc *sc;
 973         uint16_t gmch_ctl;
 974
 975         sc = device_get_softc(dev);
 976         gmch_ctl = pci_read_config(dev, AGP_SNB_GCC1, 2);
 977
 978         switch (gmch_ctl & AGP_SNB_GMCH_GMS_STOLEN_MASK) {
 979         case AGP_SNB_GMCH_GMS_STOLEN_32M:
 980                 sc->stolen_size = 32 * 1024 * 1024;
 981                 break;
 982         case AGP_SNB_GMCH_GMS_STOLEN_64M:
 983                 sc->stolen_size = 64 * 1024 * 1024;
 984                 break;
 985         case AGP_SNB_GMCH_GMS_STOLEN_96M:
 986                 sc->stolen_size = 96 * 1024 * 1024;
 987                 break;
 988         case AGP_SNB_GMCH_GMS_STOLEN_128M:
 989                 sc->stolen_size = 128 * 1024 * 1024;
 990                 break;
 991         case AGP_SNB_GMCH_GMS_STOLEN_160M:
 992                 sc->stolen_size = 160 * 1024 * 1024;
 993                 break;
 994         case AGP_SNB_GMCH_GMS_STOLEN_192M:
 995                 sc->stolen_size = 192 * 1024 * 1024;
 996                 break;
 997         case AGP_SNB_GMCH_GMS_STOLEN_224M:
 998                 sc->stolen_size = 224 * 1024 * 1024;
 999                 break;
1000         case AGP_SNB_GMCH_GMS_STOLEN_256M:
1001                 sc->stolen_size = 256 * 1024 * 1024;
1002                 break;
1003         case AGP_SNB_GMCH_GMS_STOLEN_288M:
1004                 sc->stolen_size = 288 * 1024 * 1024;
1005                 break;
1006         case AGP_SNB_GMCH_GMS_STOLEN_320M:
1007                 sc->stolen_size = 320 * 1024 * 1024;
1008                 break;
1009         case AGP_SNB_GMCH_GMS_STOLEN_352M:
1010                 sc->stolen_size = 352 * 1024 * 1024;
1011                 break;
1012         case AGP_SNB_GMCH_GMS_STOLEN_384M:
1013                 sc->stolen_size = 384 * 1024 * 1024;
1014                 break;
1015         case AGP_SNB_GMCH_GMS_STOLEN_416M:
1016                 sc->stolen_size = 416 * 1024 * 1024;
1017                 break;
1018         case AGP_SNB_GMCH_GMS_STOLEN_448M:
1019                 sc->stolen_size = 448 * 1024 * 1024;
1020                 break;
1021         case AGP_SNB_GMCH_GMS_STOLEN_480M:
1022                 sc->stolen_size = 480 * 1024 * 1024;
1023                 break;
1024         case AGP_SNB_GMCH_GMS_STOLEN_512M:
1025                 sc->stolen_size = 512 * 1024 * 1024;
1026                 break;
1027         }
1028         sc->stolen = (sc->stolen_size - 4) / 4096;
1029         return (0);
1030 }
1031
1032 static int
1033 agp_gen8_get_stolen_size(device_t dev)
1034 {
1035         struct agp_i810_softc *sc;
1036         uint16_t gcc1;
1037         int v;
1038
1039         sc = device_get_softc(dev);
1040         gcc1 = pci_read_config(dev, AGP_SNB_GCC1, 2);
1041         v = (gcc1 >> 8) & 0xFF;
1042         sc->stolen_size = v * (32L * 1024 * 1024);      /* 32MB increments */
1043
1044         return 0;
1045 }
1046
1047 static int
1048 agp_i915_get_gtt_mappable_entries(device_t dev)
1049 {
1050         struct agp_i810_softc *sc;
1051         uint32_t ap;
1052
1053         sc = device_get_softc(dev);
1054         ap = AGP_GET_APERTURE(dev);
1055         sc->gtt_mappable_entries = ap >> AGP_PAGE_SHIFT;
1056         return (0);
1057 }
1058
1059 static int
1060 agp_i810_get_gtt_total_entries(device_t dev)
1061 {
1062         struct agp_i810_softc *sc;
1063
1064         sc = device_get_softc(dev);
1065         sc->gtt_total_entries = sc->gtt_mappable_entries;
1066         return (0);
1067 }
1068
1069 static int
1070 agp_i965_get_gtt_total_entries(device_t dev)
1071 {
1072         struct agp_i810_softc *sc;
1073         uint32_t pgetbl_ctl;
1074         int error;
1075
1076         sc = device_get_softc(dev);
1077         error = 0;
1078         pgetbl_ctl = bus_read_4(sc->sc_res[0], AGP_I810_PGTBL_CTL);
1079         switch (pgetbl_ctl & AGP_I810_PGTBL_SIZE_MASK) {
1080         case AGP_I810_PGTBL_SIZE_128KB:
1081                 sc->gtt_total_entries = 128 * 1024 / 4;
1082                 break;
1083         case AGP_I810_PGTBL_SIZE_256KB:
1084                 sc->gtt_total_entries = 256 * 1024 / 4;
1085                 break;
1086         case AGP_I810_PGTBL_SIZE_512KB:
1087                 sc->gtt_total_entries = 512 * 1024 / 4;
1088                 break;
1089         /* GTT pagetable sizes bigger than 512KB are not possible on G33! */
1090         case AGP_I810_PGTBL_SIZE_1MB:
1091                 sc->gtt_total_entries = 1024 * 1024 / 4;
1092                 break;
1093         case AGP_I810_PGTBL_SIZE_2MB:
1094                 sc->gtt_total_entries = 2 * 1024 * 1024 / 4;
1095                 break;
1096         case AGP_I810_PGTBL_SIZE_1_5MB:
1097                 sc->gtt_total_entries = (1024 + 512) * 1024 / 4;
1098                 break;
1099         default:
1100                 device_printf(dev, "Unknown page table size\n");
1101                 error = ENXIO;
1102         }
1103         return (error);
1104 }
1105
1106 static void
1107 agp_gen5_adjust_pgtbl_size(device_t dev, uint32_t sz)
1108 {
1109         struct agp_i810_softc *sc;
1110         uint32_t pgetbl_ctl, pgetbl_ctl2;
1111
1112         sc = device_get_softc(dev);
1113
1114         /* Disable per-process page table. */
1115         pgetbl_ctl2 = bus_read_4(sc->sc_res[0], AGP_I965_PGTBL_CTL2);
1116         pgetbl_ctl2 &= ~AGP_I810_PGTBL_ENABLED;
1117         bus_write_4(sc->sc_res[0], AGP_I965_PGTBL_CTL2, pgetbl_ctl2);
1118
1119         /* Write the new ggtt size. */
1120         pgetbl_ctl = bus_read_4(sc->sc_res[0], AGP_I810_PGTBL_CTL);
1121         pgetbl_ctl &= ~AGP_I810_PGTBL_SIZE_MASK;
1122         pgetbl_ctl |= sz;
1123         bus_write_4(sc->sc_res[0], AGP_I810_PGTBL_CTL, pgetbl_ctl);
1124 }
1125
1126 static int
1127 agp_gen5_get_gtt_total_entries(device_t dev)
1128 {
1129         struct agp_i810_softc *sc;
1130         uint16_t gcc1;
1131
1132         sc = device_get_softc(dev);
1133
1134         gcc1 = pci_read_config(sc->bdev, AGP_I830_GCC1, 2);
1135         switch (gcc1 & AGP_G4x_GCC1_SIZE_MASK) {
1136         case AGP_G4x_GCC1_SIZE_1M:
1137         case AGP_G4x_GCC1_SIZE_VT_1M:
1138                 agp_gen5_adjust_pgtbl_size(dev, AGP_I810_PGTBL_SIZE_1MB);
1139                 break;
1140         case AGP_G4x_GCC1_SIZE_VT_1_5M:
1141                 agp_gen5_adjust_pgtbl_size(dev, AGP_I810_PGTBL_SIZE_1_5MB);
1142                 break;
1143         case AGP_G4x_GCC1_SIZE_2M:
1144         case AGP_G4x_GCC1_SIZE_VT_2M:
1145                 agp_gen5_adjust_pgtbl_size(dev, AGP_I810_PGTBL_SIZE_2MB);
1146                 break;
1147         default:
1148                 device_printf(dev, "Unknown page table size\n");
1149                 return (ENXIO);
1150         }
1151
1152         return (agp_i965_get_gtt_total_entries(dev));
1153 }
1154
1155 static int
1156 agp_sb_get_gtt_total_entries(device_t dev)
1157 {
1158         struct agp_i810_softc *sc;
1159         uint16_t gcc1;
1160
1161         sc = device_get_softc(dev);
1162
1163         gcc1 = pci_read_config(dev, AGP_SNB_GCC1, 2);
1164         switch (gcc1 & AGP_SNB_GTT_SIZE_MASK) {
1165         default:
1166         case AGP_SNB_GTT_SIZE_0M:
1167                 kprintf("Bad GTT size mask: 0x%04x\n", gcc1);
1168                 return (ENXIO);
1169         case AGP_SNB_GTT_SIZE_1M:
1170                 sc->gtt_total_entries = 1024 * 1024 / 4;
1171                 break;
1172         case AGP_SNB_GTT_SIZE_2M:
1173                 sc->gtt_total_entries = 2 * 1024 * 1024 / 4;
1174                 break;
1175         }
1176         return (0);
1177 }
1178
1179 static int
1180 agp_gen8_get_gtt_total_entries(device_t dev)
1181 {
1182         struct agp_i810_softc *sc;
1183         uint16_t gcc1;
1184         int v;
1185
1186         sc = device_get_softc(dev);
1187
1188         gcc1 = pci_read_config(dev, AGP_SNB_GCC1, 2);
1189         v = (gcc1 >> 6) & 3;
1190         if (v)
1191                 v = 1 << v;
1192         sc->gtt_total_entries = (v << 20) / 8;
1193
1194         /*
1195          * XXX limit to 2GB due to misc integer overflows calculated on
1196          * this field.
1197          */
1198         while ((long)sc->gtt_total_entries * PAGE_SIZE >= 4LL*1024*1024*1024)
1199                 sc->gtt_total_entries >>= 1;
1200
1201         kprintf("GTT SIZE %ld representing %ldM vmap\n",
1202                 (long)sc->gtt_total_entries * 8,
1203                 (long)sc->gtt_total_entries * PAGE_SIZE);
1204
1205         return 0;
1206 }
1207
1208 static int
1209 agp_i830_install_gatt(device_t dev)
1210 {
1211         struct agp_i810_softc *sc;
1212         uint32_t pgtblctl;
1213
1214         sc = device_get_softc(dev);
1215
1216         /*
1217          * The i830 automatically initializes the 128k gatt on boot.
1218          * GATT address is already in there, make sure it's enabled.
1219          */
1220         pgtblctl = bus_read_4(sc->sc_res[0], AGP_I810_PGTBL_CTL);
1221         pgtblctl |= 1;
1222         bus_write_4(sc->sc_res[0], AGP_I810_PGTBL_CTL, pgtblctl);
1223
1224         sc->gatt->ag_physical = pgtblctl & ~1;
1225         return (0);
1226 }
1227
1228 static int
1229 agp_sb_install_gatt(device_t dev)
1230 {
1231         struct agp_i810_softc *sc;
1232         uint32_t pgtblctl;
1233
1234         sc = device_get_softc(dev);
1235
1236         /*
1237          * The i830 automatically initializes the 128k gatt on boot.
1238          * GATT address is already in there, make sure it's enabled.
1239          */
1240         pgtblctl = bus_read_4(sc->sc_res[0], AGP_I810_PGTBL_CTL);
1241         pgtblctl |= 1;
1242         bus_write_4(sc->sc_res[0], AGP_I810_PGTBL_CTL, pgtblctl);
1243
1244         sc->gatt->ag_physical = pgtblctl & ~1;
1245         return (0);
1246 }
1247
1248 static int
1249 agp_vlv_install_gatt(device_t dev)
1250 {
1251         void *pg = contigmalloc(4096, M_AGP, M_WAITOK | M_ZERO,
1252                                 0x10000, 0xFFFFFFFFU,
1253                                 PAGE_SIZE, PAGE_SIZE);
1254         struct agp_i810_softc *sc;
1255         uint32_t pte;
1256         int i;
1257
1258         /*
1259          * Make the entire aperture valid by pointing to a page of junk
1260          * memory.
1261          *
1262          * NOTE: valleyview has not PGTBL_CTL
1263          */
1264         sc = device_get_softc(dev);
1265         pte = vtophys(pg) | 0x03;
1266         for (i = 0; i < 2 * 1024 * 1024; i += 4)
1267                 bus_write_4(sc->sc_res[0], i + (2 * 1024 * 1024), pte);
1268
1269         return 0;
1270 }
1271
1272 static int
1273 agp_i810_attach(device_t dev)
1274 {
1275         struct agp_i810_softc *sc;
1276         int error;
1277
1278         sc = device_get_softc(dev);
1279         sc->bdev = agp_i810_find_bridge(dev);
1280         if (sc->bdev == NULL)
1281                 return (ENOENT);
1282
1283         sc->match = agp_i810_match(dev);
1284
1285         agp_set_aperture_resource(dev, sc->match->driver->gen <= 2 ?
1286             AGP_APBASE : AGP_I915_GMADR);
1287         error = agp_generic_attach(dev);
1288         if (error)
1289                 return (error);
1290
1291         if (ptoa((vm_paddr_t)Maxmem) >
1292             (1ULL << sc->match->driver->busdma_addr_mask_sz) - 1) {
1293                 device_printf(dev, "agp_i810 does not support physical "
1294                     "memory above %ju.\n", (uintmax_t)(1ULL <<
1295                     sc->match->driver->busdma_addr_mask_sz) - 1);
1296                 return (ENOENT);
1297         }
1298
1299         if (bus_alloc_resources(dev, sc->match->driver->res_spec, sc->sc_res)) {
1300                 agp_generic_detach(dev);
1301                 return (ENODEV);
1302         }
1303
1304         sc->initial_aperture = AGP_GET_APERTURE(dev);
1305         sc->gatt = kmalloc(sizeof(struct agp_gatt), M_AGP, M_WAITOK);
1306         sc->gatt->ag_entries = AGP_GET_APERTURE(dev) >> AGP_PAGE_SHIFT;
1307
1308         if ((error = sc->match->driver->get_stolen_size(dev)) != 0 ||
1309             (error = sc->match->driver->install_gatt(dev)) != 0 ||
1310             (error = sc->match->driver->get_gtt_mappable_entries(dev)) != 0 ||
1311             (error = sc->match->driver->get_gtt_total_entries(dev)) != 0 ||
1312             (error = sc->match->driver->chipset_flush_setup(dev)) != 0) {
1313                 bus_release_resources(dev, sc->match->driver->res_spec,
1314                     sc->sc_res);
1315                 kfree(sc->gatt, M_AGP);
1316                 agp_generic_detach(dev);
1317                 return (error);
1318         }
1319
1320         intel_agp = dev;
1321         device_printf(dev, "aperture size is %dM",
1322             sc->initial_aperture / 1024 / 1024);
1323         if (sc->stolen > 0)
1324                 kprintf(", detected %dk stolen memory\n", sc->stolen * 4);
1325         else
1326                 kprintf("\n");
1327         if (bootverbose) {
1328                 sc->match->driver->dump_regs(dev);
1329                 device_printf(dev, "Mappable GTT entries: %d\n",
1330                     sc->gtt_mappable_entries);
1331                 device_printf(dev, "Total GTT entries: %d\n",
1332                     sc->gtt_total_entries);
1333         }
1334         return (0);
1335 }
1336
1337 static void
1338 agp_i830_deinstall_gatt(device_t dev)
1339 {
1340         struct agp_i810_softc *sc;
1341         unsigned int pgtblctl;
1342
1343         sc = device_get_softc(dev);
1344         pgtblctl = bus_read_4(sc->sc_res[0], AGP_I810_PGTBL_CTL);
1345         pgtblctl &= ~1;
1346         bus_write_4(sc->sc_res[0], AGP_I810_PGTBL_CTL, pgtblctl);
1347 }
1348
1349 static void
1350 agp_sb_deinstall_gatt(device_t dev)
1351 {
1352         struct agp_i810_softc *sc;
1353         unsigned int pgtblctl;
1354
1355         sc = device_get_softc(dev);
1356         if (sc->sc_res[0]) {
1357                 pgtblctl = bus_read_4(sc->sc_res[0], AGP_I810_PGTBL_CTL);
1358                 pgtblctl &= ~1;
1359                 bus_write_4(sc->sc_res[0], AGP_I810_PGTBL_CTL, pgtblctl);
1360         }
1361 }
1362
1363 static int
1364 agp_i810_detach(device_t dev)
1365 {
1366         struct agp_i810_softc *sc;
1367
1368         sc = device_get_softc(dev);
1369         agp_free_cdev(dev);
1370
1371         /* Clear the GATT base. */
1372         sc->match->driver->deinstall_gatt(dev);
1373
1374         sc->match->driver->chipset_flush_teardown(dev);
1375
1376         /* Put the aperture back the way it started. */
1377         AGP_SET_APERTURE(dev, sc->initial_aperture);
1378
1379         kfree(sc->gatt, M_AGP);
1380         bus_release_resources(dev, sc->match->driver->res_spec, sc->sc_res);
1381         agp_free_res(dev);
1382
1383         return (0);
1384 }
1385
1386 static int
1387 agp_i810_resume(device_t dev)
1388 {
1389         struct agp_i810_softc *sc;
1390         sc = device_get_softc(dev);
1391
1392         AGP_SET_APERTURE(dev, sc->initial_aperture);
1393
1394         /* Install the GATT. */
1395         bus_write_4(sc->sc_res[0], AGP_I810_PGTBL_CTL,
1396         sc->gatt->ag_physical | 1);
1397
1398         return (bus_generic_resume(dev));
1399 }
1400
1401 /**
1402  * Sets the PCI resource size of the aperture on i830-class and below chipsets,
1403  * while returning failure on later chipsets when an actual change is
1404  * requested.
1405  *
1406  * This whole function is likely bogus, as the kernel would probably need to
1407  * reconfigure the placement of the AGP aperture if a larger size is requested,
1408  * which doesn't happen currently.
1409  */
1410
1411 static int
1412 agp_i915_set_aperture(device_t dev, u_int32_t aperture)
1413 {
1414
1415         return (agp_generic_set_aperture(dev, aperture));
1416 }
1417
1418 static int
1419 agp_i810_method_set_aperture(device_t dev, u_int32_t aperture)
1420 {
1421         struct agp_i810_softc *sc;
1422
1423         sc = device_get_softc(dev);
1424         return (sc->match->driver->set_aperture(dev, aperture));
1425 }
1426
1427 /**
1428  * Writes a GTT entry mapping the page at the given offset from the
1429  * beginning of the aperture to the given physical address.  Setup the
1430  * caching mode according to flags.
1431  *
1432  * For gen 1, 2 and 3, GTT start is located at AGP_I810_GTT offset
1433  * from corresponding BAR start. For gen 4, offset is 512KB +
1434  * AGP_I810_GTT, for gen 5 and 6 it is 2MB + AGP_I810_GTT.
1435  *
1436  * Also, the bits of the physical page address above 4GB needs to be
1437  * placed into bits 40-32 of PTE.
1438  */
1439 static void
1440 agp_i915_install_gtt_pte(device_t dev, u_int index, vm_offset_t physical,
1441     int flags)
1442 {
1443         uint32_t pte;
1444
1445         pte = (u_int32_t)physical | I810_PTE_VALID;
1446         if (flags == AGP_USER_CACHED_MEMORY)
1447                 pte |= I830_PTE_SYSTEM_CACHED;
1448
1449         agp_i915_write_gtt(dev, index, pte);
1450 }
1451
1452 static void
1453 agp_i915_write_gtt(device_t dev, u_int index, uint32_t pte)
1454 {
1455         struct agp_i810_softc *sc;
1456
1457         sc = device_get_softc(dev);
1458         bus_write_4(sc->sc_res[0], index * 4, pte);
1459 }
1460
1461 static void
1462 agp_i965_install_gtt_pte(device_t dev, u_int index, vm_offset_t physical,
1463     int flags)
1464 {
1465         uint32_t pte;
1466
1467         pte = (u_int32_t)physical | I810_PTE_VALID;
1468         if (flags == AGP_USER_CACHED_MEMORY)
1469                 pte |= I830_PTE_SYSTEM_CACHED;
1470
1471         pte |= (physical >> 28) & 0xf0;
1472         agp_i965_write_gtt(dev, index, pte);
1473 }
1474
1475 static void
1476 agp_i965_write_gtt(device_t dev, u_int index, uint32_t pte)
1477 {
1478         struct agp_i810_softc *sc;
1479
1480         sc = device_get_softc(dev);
1481         bus_write_4(sc->sc_res[0], index * 4 + (512 * 1024), pte);
1482 }
1483
1484 static void
1485 agp_g4x_install_gtt_pte(device_t dev, u_int index, vm_offset_t physical,
1486     int flags)
1487 {
1488         uint32_t pte;
1489
1490         pte = (u_int32_t)physical | I810_PTE_VALID;
1491         if (flags == AGP_USER_CACHED_MEMORY)
1492                 pte |= I830_PTE_SYSTEM_CACHED;
1493
1494         pte |= (physical >> 28) & 0xf0;
1495         agp_g4x_write_gtt(dev, index, pte);
1496 }
1497
1498 static void
1499 agp_g4x_write_gtt(device_t dev, u_int index, uint32_t pte)
1500 {
1501         struct agp_i810_softc *sc;
1502
1503         sc = device_get_softc(dev);
1504         bus_write_4(sc->sc_res[0], index * 4 + (2 * 1024 * 1024), pte);
1505 }
1506
1507 static void
1508 agp_sb_install_gtt_pte(device_t dev, u_int index,
1509                        vm_offset_t physical, int flags)
1510 {
1511         int type_mask, gfdt;
1512         uint32_t pte;
1513
1514         pte = (u_int32_t)physical | I810_PTE_VALID;
1515         type_mask = flags & ~AGP_USER_CACHED_MEMORY_GFDT;
1516         gfdt = (flags & AGP_USER_CACHED_MEMORY_GFDT) != 0 ? GEN6_PTE_GFDT : 0;
1517
1518         if (type_mask == AGP_USER_MEMORY)
1519                 pte |= GEN6_PTE_UNCACHED;
1520         else if (type_mask == AGP_USER_CACHED_MEMORY_LLC_MLC)
1521                 pte |= GEN6_PTE_LLC_MLC | gfdt;
1522         else
1523                 pte |= GEN6_PTE_LLC | gfdt;
1524
1525         pte |= (physical & 0x000000ff00000000ull) >> 28;
1526         agp_sb_write_gtt(dev, index, pte);
1527 }
1528
1529 #define GEN6_PTE_VALID                  (1 << 0)
1530 #define BYT_PTE_WRITEABLE               (1 << 1)
1531 #define BYT_PTE_SNOOPED_BY_CPU_CACHES   (1 << 2)
1532 #define GEN6_PTE_ADDR_ENCODE(addr)      GEN6_GTT_ADDR_ENCODE(addr)
1533 #define GEN6_GTT_ADDR_ENCODE(addr)      ((addr) | (((addr) >> 28) & 0xff0))
1534
1535 static void
1536 agp_vlv_install_gtt_pte(device_t dev, u_int index,
1537                         vm_offset_t physical, int flags)
1538 {
1539         int type_mask;
1540         uint32_t pte;
1541
1542         pte = GEN6_PTE_ADDR_ENCODE(physical) | GEN6_PTE_VALID;
1543         pte |= BYT_PTE_WRITEABLE;
1544
1545         type_mask = flags & ~AGP_USER_CACHED_MEMORY_GFDT;
1546         if (type_mask != AGP_USER_MEMORY) {
1547                  pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
1548                  if (flags & AGP_USER_CACHED_MEMORY_GFDT)
1549                         pte |= GEN6_PTE_GFDT;
1550         }
1551         agp_sb_write_gtt(dev, index, pte);
1552 }
1553
1554 static void
1555 agp_gen8_install_gtt_pte(device_t dev, u_int index,
1556                        vm_offset_t physical, int flags)
1557 {
1558         struct agp_i810_softc *sc;
1559         int type_mask;
1560         uint64_t pte;
1561
1562         pte = (u_int64_t)physical | GEN8_PTE_PRESENT | GEN8_PTE_RW;
1563         type_mask = flags & ~AGP_USER_CACHED_MEMORY_GFDT;
1564
1565         if (type_mask == AGP_USER_MEMORY)
1566                 pte |= GEN8_PTE_PWT;    /* XXX */
1567         else if (type_mask == AGP_USER_CACHED_MEMORY_LLC_MLC)
1568                 pte |= GEN8_PTE_PWT;    /* XXX */
1569         else
1570                 pte |= GEN8_PTE_PWT;    /* XXX */
1571
1572         sc = device_get_softc(dev);
1573         bus_write_4(sc->sc_res[0], index * 8 + (2 * 1024 * 1024),
1574                     (uint32_t)pte);
1575         bus_write_4(sc->sc_res[0], index * 8 + (2 * 1024 * 1024) + 4,
1576                     (uint32_t)(pte >> 32));
1577 }
1578
1579 static void
1580 agp_sb_write_gtt(device_t dev, u_int index, uint32_t pte)
1581 {
1582         struct agp_i810_softc *sc;
1583
1584         sc = device_get_softc(dev);
1585         bus_write_4(sc->sc_res[0], index * 4 + (2 * 1024 * 1024), pte);
1586 }
1587
1588 static int
1589 agp_i810_bind_page(device_t dev, vm_offset_t offset, vm_offset_t physical)
1590 {
1591         struct agp_i810_softc *sc = device_get_softc(dev);
1592         u_int index;
1593
1594         if (offset >= (sc->gatt->ag_entries << AGP_PAGE_SHIFT)) {
1595                 device_printf(dev, "failed: offset is 0x%08jx, "
1596                     "shift is %d, entries is %d\n", (intmax_t)offset,
1597                     AGP_PAGE_SHIFT, sc->gatt->ag_entries);
1598                 return (EINVAL);
1599         }
1600         index = offset >> AGP_PAGE_SHIFT;
1601         if (sc->stolen != 0 && index < sc->stolen) {
1602                 device_printf(dev, "trying to bind into stolen memory\n");
1603                 return (EINVAL);
1604         }
1605         sc->match->driver->install_gtt_pte(dev, index, physical, 0);
1606         return (0);
1607 }
1608
1609 static int
1610 agp_i810_unbind_page(device_t dev, vm_offset_t offset)
1611 {
1612         struct agp_i810_softc *sc;
1613         u_int index;
1614
1615         sc = device_get_softc(dev);
1616         if (offset >= (sc->gatt->ag_entries << AGP_PAGE_SHIFT))
1617                 return (EINVAL);
1618         index = offset >> AGP_PAGE_SHIFT;
1619         if (sc->stolen != 0 && index < sc->stolen) {
1620                 device_printf(dev, "trying to unbind from stolen memory\n");
1621                 return (EINVAL);
1622         }
1623         sc->match->driver->install_gtt_pte(dev, index, 0, 0);
1624         return (0);
1625 }
1626
1627 static void
1628 agp_i915_sync_gtt_pte(device_t dev, u_int index)
1629 {
1630         struct agp_i810_softc *sc;
1631
1632         sc = device_get_softc(dev);
1633         bus_read_4(sc->sc_res[1], index * 4);
1634 }
1635
1636 static void
1637 agp_i965_sync_gtt_pte(device_t dev, u_int index)
1638 {
1639         struct agp_i810_softc *sc;
1640
1641         sc = device_get_softc(dev);
1642         bus_read_4(sc->sc_res[0], index * 4 + (512 * 1024));
1643 }
1644
1645 static void
1646 agp_g4x_sync_gtt_pte(device_t dev, u_int index)
1647 {
1648         struct agp_i810_softc *sc;
1649
1650         sc = device_get_softc(dev);
1651         bus_read_4(sc->sc_res[0], index * 4 + (2 * 1024 * 1024));
1652 }
1653
1654 /*
1655  * Writing via memory mapped registers already flushes all TLBs.
1656  */
1657 static void
1658 agp_i810_flush_tlb(device_t dev)
1659 {
1660 }
1661
1662 static int
1663 agp_i810_enable(device_t dev, u_int32_t mode)
1664 {
1665
1666         return (0);
1667 }
1668
1669 static struct agp_memory *
1670 agp_i810_alloc_memory(device_t dev, int type, vm_size_t size)
1671 {
1672         struct agp_i810_softc *sc;
1673         struct agp_memory *mem;
1674         vm_page_t m;
1675
1676         sc = device_get_softc(dev);
1677
1678         if ((size & (AGP_PAGE_SIZE - 1)) != 0 ||
1679             sc->agp.as_allocated + size > sc->agp.as_maxmem)
1680                 return (0);
1681
1682         if (type == 1) {
1683                 /*
1684                  * Mapping local DRAM into GATT.
1685                  */
1686                 if (sc->match->driver->chiptype != CHIP_I810)
1687                         return (0);
1688                 if (size != sc->dcache_size)
1689                         return (0);
1690         } else if (type == 2) {
1691                 /*
1692                  * Type 2 is the contiguous physical memory type, that hands
1693                  * back a physical address.  This is used for cursors on i810.
1694                  * Hand back as many single pages with physical as the user
1695                  * wants, but only allow one larger allocation (ARGB cursor)
1696                  * for simplicity.
1697                  */
1698                 if (size != AGP_PAGE_SIZE) {
1699                         if (sc->argb_cursor != NULL)
1700                                 return (0);
1701
1702                         /* Allocate memory for ARGB cursor, if we can. */
1703                         sc->argb_cursor = contigmalloc(size, M_AGP,
1704                            0, 0, ~0, PAGE_SIZE, 0);
1705                         if (sc->argb_cursor == NULL)
1706                                 return (0);
1707                 }
1708         }
1709
1710         mem = kmalloc(sizeof *mem, M_AGP, M_INTWAIT);
1711         mem->am_id = sc->agp.as_nextid++;
1712         mem->am_size = size;
1713         mem->am_type = type;
1714         if (type != 1 && (type != 2 || size == AGP_PAGE_SIZE))
1715                 mem->am_obj = vm_object_allocate(OBJT_DEFAULT,
1716                     atop(round_page(size)));
1717         else
1718                 mem->am_obj = 0;
1719
1720         if (type == 2) {
1721                 if (size == AGP_PAGE_SIZE) {
1722                         /*
1723                          * Allocate and wire down the page now so that we can
1724                          * get its physical address.
1725                          */
1726                         VM_OBJECT_LOCK(mem->am_obj);
1727                         m = vm_page_grab(mem->am_obj, 0, VM_ALLOC_NORMAL |
1728                                                          VM_ALLOC_ZERO |
1729                                                          VM_ALLOC_RETRY);
1730                         vm_page_wire(m);
1731                         VM_OBJECT_UNLOCK(mem->am_obj);
1732                         mem->am_physical = VM_PAGE_TO_PHYS(m);
1733                         vm_page_wakeup(m);
1734                 } else {
1735                         /* Our allocation is already nicely wired down for us.
1736                          * Just grab the physical address.
1737                          */
1738                         mem->am_physical = vtophys(sc->argb_cursor);
1739                 }
1740         } else
1741                 mem->am_physical = 0;
1742
1743         mem->am_offset = 0;
1744         mem->am_is_bound = 0;
1745         TAILQ_INSERT_TAIL(&sc->agp.as_memory, mem, am_link);
1746         sc->agp.as_allocated += size;
1747
1748         return (mem);
1749 }
1750
1751 static int
1752 agp_i810_free_memory(device_t dev, struct agp_memory *mem)
1753 {
1754         struct agp_i810_softc *sc;
1755
1756         if (mem->am_is_bound)
1757                 return (EBUSY);
1758
1759         sc = device_get_softc(dev);
1760
1761         if (mem->am_type == 2) {
1762                 if (mem->am_size == AGP_PAGE_SIZE) {
1763                         /*
1764                          * Unwire the page which we wired in alloc_memory.
1765                          */
1766                         vm_page_t m;
1767
1768                         vm_object_hold(mem->am_obj);
1769                         m = vm_page_lookup_busy_wait(mem->am_obj, 0,
1770                                                      FALSE, "agppg");
1771                         vm_object_drop(mem->am_obj);
1772                         vm_page_unwire(m, 0);
1773                         vm_page_wakeup(m);
1774                 } else {
1775                         contigfree(sc->argb_cursor, mem->am_size, M_AGP);
1776                         sc->argb_cursor = NULL;
1777                 }
1778         }
1779
1780         sc->agp.as_allocated -= mem->am_size;
1781         TAILQ_REMOVE(&sc->agp.as_memory, mem, am_link);
1782         if (mem->am_obj)
1783                 vm_object_deallocate(mem->am_obj);
1784         kfree(mem, M_AGP);
1785         return (0);
1786 }
1787
1788 static int
1789 agp_i810_bind_memory(device_t dev, struct agp_memory *mem, vm_offset_t offset)
1790 {
1791         struct agp_i810_softc *sc;
1792         vm_offset_t i;
1793
1794         /* Do some sanity checks first. */
1795         if ((offset & (AGP_PAGE_SIZE - 1)) != 0 ||
1796             offset + mem->am_size > AGP_GET_APERTURE(dev)) {
1797                 device_printf(dev, "binding memory at bad offset %#x\n",
1798                     (int)offset);
1799                 return (EINVAL);
1800         }
1801
1802         sc = device_get_softc(dev);
1803         if (mem->am_type == 2 && mem->am_size != AGP_PAGE_SIZE) {
1804                 lockmgr(&sc->agp.as_lock, LK_EXCLUSIVE);
1805                 if (mem->am_is_bound) {
1806                         lockmgr(&sc->agp.as_lock, LK_RELEASE);
1807                         return EINVAL;
1808                 }
1809                 /* The memory's already wired down, just stick it in the GTT. */
1810                 for (i = 0; i < mem->am_size; i += AGP_PAGE_SIZE) {
1811                         sc->match->driver->install_gtt_pte(dev, (offset + i) >>
1812                             AGP_PAGE_SHIFT, mem->am_physical + i, 0);
1813                 }
1814                 agp_flush_cache();
1815                 mem->am_offset = offset;
1816                 mem->am_is_bound = 1;
1817                 lockmgr(&sc->agp.as_lock, LK_RELEASE);
1818                 return (0);
1819         }
1820
1821         if (mem->am_type != 1)
1822                 return (agp_generic_bind_memory(dev, mem, offset));
1823
1824         /*
1825          * Mapping local DRAM into GATT.
1826          */
1827         if (sc->match->driver->chiptype != CHIP_I810)
1828                 return (EINVAL);
1829         for (i = 0; i < mem->am_size; i += AGP_PAGE_SIZE)
1830                 bus_write_4(sc->sc_res[0],
1831                     AGP_I810_GTT + (i >> AGP_PAGE_SHIFT) * 4, i | 3);
1832
1833         return (0);
1834 }
1835
1836 static int
1837 agp_i810_unbind_memory(device_t dev, struct agp_memory *mem)
1838 {
1839         struct agp_i810_softc *sc;
1840         vm_offset_t i;
1841
1842         sc = device_get_softc(dev);
1843
1844         if (mem->am_type == 2 && mem->am_size != AGP_PAGE_SIZE) {
1845                 lockmgr(&sc->agp.as_lock, LK_EXCLUSIVE);
1846                 if (!mem->am_is_bound) {
1847                         lockmgr(&sc->agp.as_lock, LK_RELEASE);
1848                         return (EINVAL);
1849                 }
1850
1851                 for (i = 0; i < mem->am_size; i += AGP_PAGE_SIZE) {
1852                         sc->match->driver->install_gtt_pte(dev,
1853                             (mem->am_offset + i) >> AGP_PAGE_SHIFT, 0, 0);
1854                 }
1855                 agp_flush_cache();
1856                 mem->am_is_bound = 0;
1857                 lockmgr(&sc->agp.as_lock, LK_RELEASE);
1858                 return (0);
1859         }
1860
1861         if (mem->am_type != 1)
1862                 return (agp_generic_unbind_memory(dev, mem));
1863
1864         if (sc->match->driver->chiptype != CHIP_I810)
1865                 return (EINVAL);
1866         for (i = 0; i < mem->am_size; i += AGP_PAGE_SIZE) {
1867                 sc->match->driver->install_gtt_pte(dev, i >> AGP_PAGE_SHIFT,
1868                     0, 0);
1869         }
1870         return (0);
1871 }
1872
1873 static device_method_t agp_i810_methods[] = {
1874         /* Device interface */
1875         DEVMETHOD(device_identify,      agp_i810_identify),
1876         DEVMETHOD(device_probe,         agp_i810_probe),
1877         DEVMETHOD(device_attach,        agp_i810_attach),
1878         DEVMETHOD(device_detach,        agp_i810_detach),
1879         DEVMETHOD(device_suspend,       bus_generic_suspend),
1880         DEVMETHOD(device_resume,        agp_i810_resume),
1881
1882         /* AGP interface */
1883         DEVMETHOD(agp_get_aperture,     agp_generic_get_aperture),
1884         DEVMETHOD(agp_set_aperture,     agp_i810_method_set_aperture),
1885         DEVMETHOD(agp_bind_page,        agp_i810_bind_page),
1886         DEVMETHOD(agp_unbind_page,      agp_i810_unbind_page),
1887         DEVMETHOD(agp_flush_tlb,        agp_i810_flush_tlb),
1888         DEVMETHOD(agp_enable,           agp_i810_enable),
1889         DEVMETHOD(agp_alloc_memory,     agp_i810_alloc_memory),
1890         DEVMETHOD(agp_free_memory,      agp_i810_free_memory),
1891         DEVMETHOD(agp_bind_memory,      agp_i810_bind_memory),
1892         DEVMETHOD(agp_unbind_memory,    agp_i810_unbind_memory),
1893         DEVMETHOD(agp_chipset_flush,    agp_intel_gtt_chipset_flush),
1894
1895         DEVMETHOD_END
1896 };
1897
1898 static driver_t agp_i810_driver = {
1899         "agp",
1900         agp_i810_methods,
1901         sizeof(struct agp_i810_softc),
1902 };
1903
1904 static devclass_t agp_devclass;
1905
1906 DRIVER_MODULE(agp_i810, vgapci, agp_i810_driver, agp_devclass, NULL, NULL);
1907 MODULE_DEPEND(agp_i810, agp, 1, 1, 1);
1908 MODULE_DEPEND(agp_i810, pci, 1, 1, 1);
1909
1910 extern vm_page_t bogus_page;
1911
1912 void
1913 agp_intel_gtt_clear_range(device_t dev, u_int first_entry, u_int num_entries)
1914 {
1915         struct agp_i810_softc *sc;
1916         u_int i;
1917
1918         sc = device_get_softc(dev);
1919         for (i = 0; i < num_entries; i++)
1920                 sc->match->driver->install_gtt_pte(dev, first_entry + i,
1921                     VM_PAGE_TO_PHYS(bogus_page), 0);
1922         sc->match->driver->sync_gtt_pte(dev, first_entry + num_entries - 1);
1923 }
1924
1925 void
1926 agp_intel_gtt_insert_pages(device_t dev, u_int first_entry, u_int num_entries,
1927     vm_page_t *pages, u_int flags)
1928 {
1929         struct agp_i810_softc *sc;
1930         u_int i;
1931
1932         sc = device_get_softc(dev);
1933         for (i = 0; i < num_entries; i++) {
1934                 KKASSERT(pages[i]->valid == VM_PAGE_BITS_ALL);
1935                 KKASSERT(pages[i]->wire_count > 0);
1936                 sc->match->driver->install_gtt_pte(dev, first_entry + i,
1937                     VM_PAGE_TO_PHYS(pages[i]), flags);
1938         }
1939         sc->match->driver->sync_gtt_pte(dev, first_entry + num_entries - 1);
1940 }
1941
1942 struct intel_gtt
1943 agp_intel_gtt_get(device_t dev)
1944 {
1945         struct agp_i810_softc *sc;
1946         struct intel_gtt res;
1947
1948         sc = device_get_softc(dev);
1949         res.stolen_size = sc->stolen_size;
1950         res.gtt_total_entries = sc->gtt_total_entries;
1951         res.gtt_mappable_entries = sc->gtt_mappable_entries;
1952         res.do_idle_maps = 0;
1953         res.scratch_page_dma = VM_PAGE_TO_PHYS(bogus_page);
1954         return (res);
1955 }
1956
1957 static int
1958 agp_i810_chipset_flush_setup(device_t dev)
1959 {
1960
1961         return (0);
1962 }
1963
1964 static void
1965 agp_i810_chipset_flush_teardown(device_t dev)
1966 {
1967
1968         /* Nothing to do. */
1969 }
1970
1971 static void
1972 agp_i810_chipset_flush(device_t dev)
1973 {
1974
1975         /* Nothing to do. */
1976 }
1977
1978 static int
1979 agp_i915_chipset_flush_alloc_page(device_t dev, uint64_t start, uint64_t end)
1980 {
1981         struct agp_i810_softc *sc;
1982         device_t vga;
1983
1984         sc = device_get_softc(dev);
1985         vga = device_get_parent(dev);
1986         sc->sc_flush_page_rid = 100;
1987         sc->sc_flush_page_res = BUS_ALLOC_RESOURCE(device_get_parent(vga), dev,
1988             SYS_RES_MEMORY, &sc->sc_flush_page_rid, start, end, PAGE_SIZE,
1989             RF_ACTIVE, -1);
1990         if (sc->sc_flush_page_res == NULL) {
1991                 device_printf(dev, "Failed to allocate flush page at 0x%jx\n",
1992                     (uintmax_t)start);
1993                 return (EINVAL);
1994         }
1995         sc->sc_flush_page_vaddr = rman_get_virtual(sc->sc_flush_page_res);
1996         if (bootverbose) {
1997                 device_printf(dev, "Allocated flush page phys 0x%jx virt %p\n",
1998                     (uintmax_t)rman_get_start(sc->sc_flush_page_res),
1999                     sc->sc_flush_page_vaddr);
2000         }
2001         return (0);
2002 }
2003
2004 static void
2005 agp_i915_chipset_flush_free_page(device_t dev)
2006 {
2007         struct agp_i810_softc *sc;
2008         device_t vga;
2009
2010         sc = device_get_softc(dev);
2011         vga = device_get_parent(dev);
2012         if (sc->sc_flush_page_res == NULL)
2013                 return;
2014         BUS_DEACTIVATE_RESOURCE(device_get_parent(vga), dev, SYS_RES_MEMORY,
2015             sc->sc_flush_page_rid, sc->sc_flush_page_res);
2016         BUS_RELEASE_RESOURCE(device_get_parent(vga), dev, SYS_RES_MEMORY,
2017             sc->sc_flush_page_rid, sc->sc_flush_page_res);
2018 }
2019
2020 static int
2021 agp_i915_chipset_flush_setup(device_t dev)
2022 {
2023         struct agp_i810_softc *sc;
2024         uint32_t temp;
2025         int error;
2026
2027         sc = device_get_softc(dev);
2028         temp = pci_read_config(sc->bdev, AGP_I915_IFPADDR, 4);
2029         if ((temp & 1) != 0) {
2030                 temp &= ~1;
2031                 if (bootverbose)
2032                         device_printf(dev,
2033                             "Found already configured flush page at 0x%jx\n",
2034                             (uintmax_t)temp);
2035                 sc->sc_bios_allocated_flush_page = 1;
2036                 /*
2037                  * In the case BIOS initialized the flush pointer (?)
2038                  * register, expect that BIOS also set up the resource
2039                  * for the page.
2040                  */
2041                 error = agp_i915_chipset_flush_alloc_page(dev, temp,
2042                     temp + PAGE_SIZE - 1);
2043                 if (error != 0)
2044                         return (error);
2045         } else {
2046                 sc->sc_bios_allocated_flush_page = 0;
2047                 error = agp_i915_chipset_flush_alloc_page(dev, 0, 0xffffffff);
2048                 if (error != 0)
2049                         return (error);
2050                 temp = rman_get_start(sc->sc_flush_page_res);
2051                 pci_write_config(sc->bdev, AGP_I915_IFPADDR, temp | 1, 4);
2052         }
2053         return (0);
2054 }
2055
2056 static void
2057 agp_i915_chipset_flush_teardown(device_t dev)
2058 {
2059         struct agp_i810_softc *sc;
2060         uint32_t temp;
2061
2062         sc = device_get_softc(dev);
2063         if (sc->sc_flush_page_res == NULL)
2064                 return;
2065         if (!sc->sc_bios_allocated_flush_page) {
2066                 temp = pci_read_config(sc->bdev, AGP_I915_IFPADDR, 4);
2067                 temp &= ~1;
2068                 pci_write_config(sc->bdev, AGP_I915_IFPADDR, temp, 4);
2069         }
2070         agp_i915_chipset_flush_free_page(dev);
2071 }
2072
2073 static int
2074 agp_i965_chipset_flush_setup(device_t dev)
2075 {
2076         struct agp_i810_softc *sc;
2077         uint64_t temp;
2078         uint32_t temp_hi, temp_lo;
2079         int error;
2080
2081         sc = device_get_softc(dev);
2082
2083         temp_hi = pci_read_config(sc->bdev, AGP_I965_IFPADDR + 4, 4);
2084         temp_lo = pci_read_config(sc->bdev, AGP_I965_IFPADDR, 4);
2085
2086         if ((temp_lo & 1) != 0) {
2087                 temp = ((uint64_t)temp_hi << 32) | (temp_lo & ~1);
2088                 if (bootverbose)
2089                         device_printf(dev,
2090                             "Found already configured flush page at 0x%jx\n",
2091                             (uintmax_t)temp);
2092                 sc->sc_bios_allocated_flush_page = 1;
2093                 /*
2094                  * In the case BIOS initialized the flush pointer (?)
2095                  * register, expect that BIOS also set up the resource
2096                  * for the page.
2097                  */
2098                 error = agp_i915_chipset_flush_alloc_page(dev, temp,
2099                     temp + PAGE_SIZE - 1);
2100                 if (error != 0)
2101                         return (error);
2102         } else {
2103                 sc->sc_bios_allocated_flush_page = 0;
2104                 error = agp_i915_chipset_flush_alloc_page(dev, 0, ~0);
2105                 if (error != 0)
2106                         return (error);
2107                 temp = rman_get_start(sc->sc_flush_page_res);
2108                 pci_write_config(sc->bdev, AGP_I965_IFPADDR + 4,
2109                     (temp >> 32) & UINT32_MAX, 4);
2110                 pci_write_config(sc->bdev, AGP_I965_IFPADDR,
2111                     (temp & UINT32_MAX) | 1, 4);
2112         }
2113         return (0);
2114 }
2115
2116 static void
2117 agp_i965_chipset_flush_teardown(device_t dev)
2118 {
2119         struct agp_i810_softc *sc;
2120         uint32_t temp_lo;
2121
2122         sc = device_get_softc(dev);
2123         if (sc->sc_flush_page_res == NULL)
2124                 return;
2125         if (!sc->sc_bios_allocated_flush_page) {
2126                 temp_lo = pci_read_config(sc->bdev, AGP_I965_IFPADDR, 4);
2127                 temp_lo &= ~1;
2128                 pci_write_config(sc->bdev, AGP_I965_IFPADDR, temp_lo, 4);
2129         }
2130         agp_i915_chipset_flush_free_page(dev);
2131 }
2132
2133 static void
2134 agp_i915_chipset_flush(device_t dev)
2135 {
2136         struct agp_i810_softc *sc;
2137
2138         sc = device_get_softc(dev);
2139         *(uint32_t *)sc->sc_flush_page_vaddr = 1;
2140 }
2141
2142 int
2143 agp_intel_gtt_chipset_flush(device_t dev)
2144 {
2145         struct agp_i810_softc *sc;
2146
2147         sc = device_get_softc(dev);
2148         sc->match->driver->chipset_flush(dev);
2149         return (0);
2150 }
2151
2152 void
2153 agp_intel_gtt_unmap_memory(device_t dev, struct sglist *sg_list)
2154 {
2155 }
2156
2157 int
2158 agp_intel_gtt_map_memory(device_t dev, vm_page_t *pages, u_int num_entries,
2159     struct sglist **sg_list)
2160 {
2161 #if 0
2162         struct agp_i810_softc *sc;
2163 #endif
2164         struct sglist *sg;
2165         int i;
2166 #if 0
2167         int error;
2168         bus_dma_tag_t dmat;
2169 #endif
2170
2171         if (*sg_list != NULL)
2172                 return (0);
2173 #if 0
2174         sc = device_get_softc(dev);
2175 #endif
2176         sg = sglist_alloc(num_entries, M_WAITOK /* XXXKIB */);
2177         for (i = 0; i < num_entries; i++) {
2178                 sg->sg_segs[i].ss_paddr = VM_PAGE_TO_PHYS(pages[i]);
2179                 sg->sg_segs[i].ss_len = PAGE_SIZE;
2180         }
2181
2182 #if 0
2183         error = bus_dma_tag_create(bus_get_dma_tag(dev),
2184             1 /* alignment */, 0 /* boundary */,
2185             1ULL << sc->match->busdma_addr_mask_sz /* lowaddr */,
2186             BUS_SPACE_MAXADDR /* highaddr */,
2187             NULL /* filtfunc */, NULL /* filtfuncarg */,
2188             BUS_SPACE_MAXADDR /* maxsize */,
2189             BUS_SPACE_UNRESTRICTED /* nsegments */,
2190             BUS_SPACE_MAXADDR /* maxsegsz */,
2191             0 /* flags */, NULL /* lockfunc */, NULL /* lockfuncarg */,
2192             &dmat);
2193         if (error != 0) {
2194                 sglist_free(sg);
2195                 return (error);
2196         }
2197         /* XXXKIB */
2198 #endif
2199         *sg_list = sg;
2200         return (0);
2201 }
2202
2203 void
2204 agp_intel_gtt_insert_sg_entries(device_t dev, struct sglist *sg_list,
2205     u_int first_entry, u_int flags)
2206 {
2207         struct agp_i810_softc *sc;
2208         vm_paddr_t spaddr;
2209         size_t slen;
2210         u_int i, j;
2211
2212         sc = device_get_softc(dev);
2213         for (i = j = 0; j < sg_list->sg_nseg; j++) {
2214                 spaddr = sg_list->sg_segs[i].ss_paddr;
2215                 slen = sg_list->sg_segs[i].ss_len;
2216                 for (; slen > 0; i++) {
2217                         sc->match->driver->install_gtt_pte(dev, first_entry + i,
2218                             spaddr, flags);
2219                         spaddr += AGP_PAGE_SIZE;
2220                         slen -= AGP_PAGE_SIZE;
2221                 }
2222         }
2223         sc->match->driver->sync_gtt_pte(dev, first_entry + i - 1);
2224 }
2225
2226 void
2227 intel_gtt_clear_range(u_int first_entry, u_int num_entries)
2228 {
2229
2230         agp_intel_gtt_clear_range(intel_agp, first_entry, num_entries);
2231 }
2232
2233 void
2234 intel_gtt_insert_pages(u_int first_entry, u_int num_entries, vm_page_t *pages,
2235     u_int flags)
2236 {
2237
2238         agp_intel_gtt_insert_pages(intel_agp, first_entry, num_entries,
2239             pages, flags);
2240 }
2241
2242 void intel_gtt_get(size_t *gtt_total, size_t *stolen_size,
2243                    phys_addr_t *mappable_base, unsigned long *mappable_end)
2244 {
2245         struct agp_info ainfo;
2246
2247         intel_private.base = agp_intel_gtt_get(intel_agp);
2248
2249         *gtt_total = intel_private.base.gtt_total_entries << PAGE_SHIFT;
2250         *stolen_size = intel_private.base.stolen_size;
2251         agp_get_info(intel_agp, &ainfo);
2252         *mappable_base = ainfo.ai_aperture_base;
2253         *mappable_end = intel_private.base.gtt_mappable_entries << PAGE_SHIFT;
2254 }
2255
2256 int
2257 intel_gtt_chipset_flush(void)
2258 {
2259
2260         return (agp_intel_gtt_chipset_flush(intel_agp));
2261 }
2262
2263 void
2264 intel_gtt_unmap_memory(struct sglist *sg_list)
2265 {
2266
2267         agp_intel_gtt_unmap_memory(intel_agp, sg_list);
2268 }
2269
2270 int
2271 intel_gtt_map_memory(vm_page_t *pages, u_int num_entries,
2272     struct sglist **sg_list)
2273 {
2274
2275         return (agp_intel_gtt_map_memory(intel_agp, pages, num_entries,
2276             sg_list));
2277 }
2278
2279 void
2280 intel_gtt_insert_sg_entries(struct sglist *sg_list, u_int first_entry,
2281     u_int flags)
2282 {
2283
2284         agp_intel_gtt_insert_sg_entries(intel_agp, sg_list, first_entry, flags);
2285 }
2286
2287 /*
2288  * Only used by gen6
2289  */
2290 void
2291 intel_gtt_sync_pte(u_int entry)
2292 {
2293         struct agp_i810_softc *sc;
2294
2295         sc = device_get_softc(intel_agp);
2296         sc->match->driver->sync_gtt_pte(intel_agp, entry);
2297 }
2298
2299 /*
2300  * Only used by gen6
2301  */
2302 void
2303 intel_gtt_write(u_int entry, uint32_t val)
2304 {
2305         struct agp_i810_softc *sc;
2306
2307         sc = device_get_softc(intel_agp);
2308         sc->match->driver->write_gtt(intel_agp, entry, val);
2309 }