sys/dev/agp/intel-gtt.c

   1 /*
   2  * Copyright (c) 2000 Doug Rabson
   3  * Copyright (c) 2000 Ruslan Ermilov
   4  * Copyright (c) 2011 The FreeBSD Foundation
   5  * Copyright (c) 2017 François Tigeot
   6  * All rights reserved.
   7  *
   8  * Portions of this software were developed by Konstantin Belousov
   9  * under sponsorship from the FreeBSD Foundation.
  10  *
  11  * Redistribution and use in source and binary forms, with or without
  12  * modification, are permitted provided that the following conditions
  13  * are met:
  14  * 1. Redistributions of source code must retain the above copyright
  15  *    notice, this list of conditions and the following disclaimer.
  16  * 2. Redistributions in binary form must reproduce the above copyright
  17  *    notice, this list of conditions and the following disclaimer in the
  18  *    documentation and/or other materials provided with the distribution.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  24  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  25  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  26  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  27  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  30  * SUCH DAMAGE.
  31  */
  32
  33 /*
  34  * Fixes for 830/845G support: David Dawes <dawes@xfree86.org>
  35  * 852GM/855GM/865G support added by David Dawes <dawes@xfree86.org>
  36  *
  37  * This is generic Intel GTT handling code, morphed from the AGP
  38  * bridge code.
  39  */
  40
  41 #if 0
  42 #define KTR_AGP_I810    KTR_DEV
  43 #else
  44 #define KTR_AGP_I810    0
  45 #endif
  46
  47 #include <sys/param.h>
  48 #include <sys/systm.h>
  49 #include <sys/malloc.h>
  50 #include <sys/kernel.h>
  51 #include <sys/bus.h>
  52 #include <sys/lock.h>
  53 #include <sys/rman.h>
  54
  55 #include "pcidevs.h"
  56 #include <bus/pci/pcivar.h>
  57 #include <bus/pci/pcireg.h>
  58 #include "agppriv.h"
  59 #include "agpreg.h"
  60
  61 #include <vm/vm.h>
  62 #include <vm/vm_object.h>
  63 #include <vm/vm_page.h>
  64 #include <vm/vm_param.h>
  65 #include <vm/vm_pageout.h>
  66 #include <vm/pmap.h>
  67
  68 #include <vm/vm_page2.h>
  69
  70 #include <machine/md_var.h>
  71
  72 #include <linux/slab.h>
  73 #include <linux/scatterlist.h>
  74 #include <drm/intel-gtt.h>
  75
  76 struct agp_i810_match;
  77
  78 static int agp_i915_check_active(device_t bridge_dev);
  79
  80 static void agp_i810_set_desc(device_t dev, const struct agp_i810_match *match);
  81
  82 static void agp_i915_dump_regs(device_t dev);
  83 static void agp_i965_dump_regs(device_t dev);
  84
  85 static int agp_i915_get_stolen_size(device_t dev);
  86
  87 static int agp_i915_get_gtt_mappable_entries(device_t dev);
  88
  89 static int agp_i810_get_gtt_total_entries(device_t dev);
  90 static int agp_i965_get_gtt_total_entries(device_t dev);
  91 static int agp_gen5_get_gtt_total_entries(device_t dev);
  92
  93 static int agp_i830_install_gatt(device_t dev);
  94
  95 static void agp_i830_deinstall_gatt(device_t dev);
  96
  97 static void agp_i915_install_gtt_pte(device_t dev, u_int index,
  98     vm_offset_t physical, int flags);
  99 static void agp_i965_install_gtt_pte(device_t dev, u_int index,
 100     vm_offset_t physical, int flags);
 101 static void agp_g4x_install_gtt_pte(device_t dev, u_int index,
 102     vm_offset_t physical, int flags);
 103
 104 static void agp_i915_write_gtt(device_t dev, u_int index, uint32_t pte);
 105 static void agp_i965_write_gtt(device_t dev, u_int index, uint32_t pte);
 106 static void agp_g4x_write_gtt(device_t dev, u_int index, uint32_t pte);
 107
 108 static void agp_i915_sync_gtt_pte(device_t dev, u_int index);
 109 static void agp_i965_sync_gtt_pte(device_t dev, u_int index);
 110 static void agp_g4x_sync_gtt_pte(device_t dev, u_int index);
 111
 112 static int agp_i915_set_aperture(device_t dev, u_int32_t aperture);
 113
 114 static int agp_i915_chipset_flush_setup(device_t dev);
 115 static int agp_i965_chipset_flush_setup(device_t dev);
 116
 117 static void agp_i915_chipset_flush_teardown(device_t dev);
 118 static void agp_i965_chipset_flush_teardown(device_t dev);
 119
 120 static void agp_i915_chipset_flush(device_t dev);
 121
 122 enum {
 123         CHIP_I810,      /* i810/i815 */
 124         CHIP_I830,      /* 830M/845G */
 125         CHIP_I855,      /* 852GM/855GM/865G */
 126         CHIP_I915,      /* 915G/915GM */
 127         CHIP_I965,      /* G965 */
 128         CHIP_G33,       /* G33/Q33/Q35 */
 129         CHIP_IGD,       /* Pineview */
 130         CHIP_G4X,       /* G45/Q45 */
 131 };
 132
 133 /* The i810 through i855 have the registers at BAR 1, and the GATT gets
 134  * allocated by us.  The i915 has registers in BAR 0 and the GATT is at the
 135  * start of the stolen memory, and should only be accessed by the OS through
 136  * BAR 3.  The G965 has registers and GATT in the same BAR (0) -- first 512KB
 137  * is registers, second 512KB is GATT.
 138  */
 139 static struct resource_spec agp_i915_res_spec[] = {
 140         { SYS_RES_MEMORY, AGP_I915_MMADR, RF_ACTIVE | RF_SHAREABLE },
 141         { SYS_RES_MEMORY, AGP_I915_GTTADR, RF_ACTIVE | RF_SHAREABLE },
 142         { -1, 0 }
 143 };
 144
 145 static struct resource_spec agp_i965_res_spec[] = {
 146         { SYS_RES_MEMORY, AGP_I965_GTTMMADR, RF_ACTIVE | RF_SHAREABLE },
 147         { -1, 0 }
 148 };
 149
 150 struct agp_i810_softc {
 151         struct agp_softc agp;
 152         u_int32_t initial_aperture;     /* aperture size at startup */
 153         struct agp_gatt *gatt;
 154         u_int32_t dcache_size;          /* i810 only */
 155         u_int32_t stolen;               /* number of i830/845 gtt
 156                                            entries for stolen memory */
 157         u_int stolen_size;              /* BIOS-reserved graphics memory */
 158         u_int gtt_total_entries;        /* Total number of gtt ptes */
 159         u_int gtt_mappable_entries;     /* Number of gtt ptes mappable by CPU */
 160         device_t bdev;                  /* bridge device */
 161         void *argb_cursor;              /* contigmalloc area for ARGB cursor */
 162         struct resource *sc_res[3];
 163         const struct agp_i810_match *match;
 164         int sc_flush_page_rid;
 165         struct resource *sc_flush_page_res;
 166         void *sc_flush_page_vaddr;
 167         int sc_bios_allocated_flush_page;
 168 };
 169
 170 static device_t intel_agp;
 171
 172 struct agp_i810_driver {
 173         int chiptype;
 174         int gen;
 175         int busdma_addr_mask_sz;
 176         struct resource_spec *res_spec;
 177         int (*check_active)(device_t);
 178         void (*set_desc)(device_t, const struct agp_i810_match *);
 179         void (*dump_regs)(device_t);
 180         int (*get_stolen_size)(device_t);
 181         int (*get_gtt_total_entries)(device_t);
 182         int (*get_gtt_mappable_entries)(device_t);
 183         int (*install_gatt)(device_t);
 184         void (*deinstall_gatt)(device_t);
 185         void (*write_gtt)(device_t, u_int, uint32_t);
 186         void (*install_gtt_pte)(device_t, u_int, vm_offset_t, int);
 187         void (*sync_gtt_pte)(device_t, u_int);
 188         int (*set_aperture)(device_t, u_int32_t);
 189         int (*chipset_flush_setup)(device_t);
 190         void (*chipset_flush_teardown)(device_t);
 191         void (*chipset_flush)(device_t);
 192 };
 193
 194 static struct {
 195         struct intel_gtt base;
 196 } intel_private;
 197
 198 static const struct agp_i810_driver agp_i810_i915_driver = {
 199         .chiptype = CHIP_I915,
 200         .gen = 3,
 201         .busdma_addr_mask_sz = 32,
 202         .res_spec = agp_i915_res_spec,
 203         .check_active = agp_i915_check_active,
 204         .set_desc = agp_i810_set_desc,
 205         .dump_regs = agp_i915_dump_regs,
 206         .get_stolen_size = agp_i915_get_stolen_size,
 207         .get_gtt_mappable_entries = agp_i915_get_gtt_mappable_entries,
 208         .get_gtt_total_entries = agp_i810_get_gtt_total_entries,
 209         .install_gatt = agp_i830_install_gatt,
 210         .deinstall_gatt = agp_i830_deinstall_gatt,
 211         .write_gtt = agp_i915_write_gtt,
 212         .install_gtt_pte = agp_i915_install_gtt_pte,
 213         .sync_gtt_pte = agp_i915_sync_gtt_pte,
 214         .set_aperture = agp_i915_set_aperture,
 215         .chipset_flush_setup = agp_i915_chipset_flush_setup,
 216         .chipset_flush_teardown = agp_i915_chipset_flush_teardown,
 217         .chipset_flush = agp_i915_chipset_flush,
 218 };
 219
 220 static const struct agp_i810_driver agp_i810_g965_driver = {
 221         .chiptype = CHIP_I965,
 222         .gen = 4,
 223         .busdma_addr_mask_sz = 36,
 224         .res_spec = agp_i965_res_spec,
 225         .check_active = agp_i915_check_active,
 226         .set_desc = agp_i810_set_desc,
 227         .dump_regs = agp_i965_dump_regs,
 228         .get_stolen_size = agp_i915_get_stolen_size,
 229         .get_gtt_mappable_entries = agp_i915_get_gtt_mappable_entries,
 230         .get_gtt_total_entries = agp_i965_get_gtt_total_entries,
 231         .install_gatt = agp_i830_install_gatt,
 232         .deinstall_gatt = agp_i830_deinstall_gatt,
 233         .write_gtt = agp_i965_write_gtt,
 234         .install_gtt_pte = agp_i965_install_gtt_pte,
 235         .sync_gtt_pte = agp_i965_sync_gtt_pte,
 236         .set_aperture = agp_i915_set_aperture,
 237         .chipset_flush_setup = agp_i965_chipset_flush_setup,
 238         .chipset_flush_teardown = agp_i965_chipset_flush_teardown,
 239         .chipset_flush = agp_i915_chipset_flush,
 240 };
 241
 242 static const struct agp_i810_driver agp_i810_g33_driver = {
 243         .chiptype = CHIP_G33,
 244         .gen = 3,
 245         .busdma_addr_mask_sz = 36,
 246         .res_spec = agp_i915_res_spec,
 247         .check_active = agp_i915_check_active,
 248         .set_desc = agp_i810_set_desc,
 249         .dump_regs = agp_i965_dump_regs,
 250         .get_stolen_size = agp_i915_get_stolen_size,
 251         .get_gtt_mappable_entries = agp_i915_get_gtt_mappable_entries,
 252         .get_gtt_total_entries = agp_i965_get_gtt_total_entries,
 253         .install_gatt = agp_i830_install_gatt,
 254         .deinstall_gatt = agp_i830_deinstall_gatt,
 255         .write_gtt = agp_i915_write_gtt,
 256         .install_gtt_pte = agp_i965_install_gtt_pte,
 257         .sync_gtt_pte = agp_i915_sync_gtt_pte,
 258         .set_aperture = agp_i915_set_aperture,
 259         .chipset_flush_setup = agp_i965_chipset_flush_setup,
 260         .chipset_flush_teardown = agp_i965_chipset_flush_teardown,
 261         .chipset_flush = agp_i915_chipset_flush,
 262 };
 263
 264 static const struct agp_i810_driver pineview_gtt_driver = {
 265         .chiptype = CHIP_IGD,
 266         .gen = 3,
 267         .busdma_addr_mask_sz = 36,
 268         .res_spec = agp_i915_res_spec,
 269         .check_active = agp_i915_check_active,
 270         .set_desc = agp_i810_set_desc,
 271         .dump_regs = agp_i915_dump_regs,
 272         .get_stolen_size = agp_i915_get_stolen_size,
 273         .get_gtt_mappable_entries = agp_i915_get_gtt_mappable_entries,
 274         .get_gtt_total_entries = agp_i965_get_gtt_total_entries,
 275         .install_gatt = agp_i830_install_gatt,
 276         .deinstall_gatt = agp_i830_deinstall_gatt,
 277         .write_gtt = agp_i915_write_gtt,
 278         .install_gtt_pte = agp_i965_install_gtt_pte,
 279         .sync_gtt_pte = agp_i915_sync_gtt_pte,
 280         .set_aperture = agp_i915_set_aperture,
 281         .chipset_flush_setup = agp_i965_chipset_flush_setup,
 282         .chipset_flush_teardown = agp_i965_chipset_flush_teardown,
 283         .chipset_flush = agp_i915_chipset_flush,
 284 };
 285
 286 static const struct agp_i810_driver agp_i810_g4x_driver = {
 287         .chiptype = CHIP_G4X,
 288         .gen = 5,
 289         .busdma_addr_mask_sz = 36,
 290         .res_spec = agp_i965_res_spec,
 291         .check_active = agp_i915_check_active,
 292         .set_desc = agp_i810_set_desc,
 293         .dump_regs = agp_i965_dump_regs,
 294         .get_stolen_size = agp_i915_get_stolen_size,
 295         .get_gtt_mappable_entries = agp_i915_get_gtt_mappable_entries,
 296         .get_gtt_total_entries = agp_gen5_get_gtt_total_entries,
 297         .install_gatt = agp_i830_install_gatt,
 298         .deinstall_gatt = agp_i830_deinstall_gatt,
 299         .write_gtt = agp_g4x_write_gtt,
 300         .install_gtt_pte = agp_g4x_install_gtt_pte,
 301         .sync_gtt_pte = agp_g4x_sync_gtt_pte,
 302         .set_aperture = agp_i915_set_aperture,
 303         .chipset_flush_setup = agp_i965_chipset_flush_setup,
 304         .chipset_flush_teardown = agp_i965_chipset_flush_teardown,
 305         .chipset_flush = agp_i915_chipset_flush,
 306 };
 307
 308 /* For adding new devices, devid is the id of the graphics controller
 309  * (pci:0:2:0, for example).  The placeholder (usually at pci:0:2:1) for the
 310  * second head should never be added.  The bridge_offset is the offset to
 311  * subtract from devid to get the id of the hostb that the device is on.
 312  */
 313 static const struct agp_i810_match {
 314         uint16_t devid;
 315         char *name;
 316         const struct agp_i810_driver *driver;
 317 } agp_i810_matches[] = {
 318         {
 319                 .devid = 0x2582,
 320                 .name = "Intel 82915G (915G GMCH) SVGA controller",
 321                 .driver = &agp_i810_i915_driver
 322         },
 323         {
 324                 .devid = 0x258A,
 325                 .name = "Intel E7221 SVGA controller",
 326                 .driver = &agp_i810_i915_driver
 327         },
 328         {
 329                 .devid = 0x2592,
 330                 .name = "Intel 82915GM (915GM GMCH) SVGA controller",
 331                 .driver = &agp_i810_i915_driver
 332         },
 333         {
 334                 .devid = 0x2772,
 335                 .name = "Intel 82945G (945G GMCH) SVGA controller",
 336                 .driver = &agp_i810_i915_driver
 337         },
 338         {
 339                 .devid = 0x27A2,
 340                 .name = "Intel 82945GM (945GM GMCH) SVGA controller",
 341                 .driver = &agp_i810_i915_driver
 342         },
 343         {
 344                 .devid = 0x27AE,
 345                 .name = "Intel 945GME SVGA controller",
 346                 .driver = &agp_i810_i915_driver
 347         },
 348         {
 349                 .devid = 0x2972,
 350                 .name = "Intel 946GZ SVGA controller",
 351                 .driver = &agp_i810_g965_driver
 352         },
 353         {
 354                 .devid = 0x2982,
 355                 .name = "Intel G965 SVGA controller",
 356                 .driver = &agp_i810_g965_driver
 357         },
 358         {
 359                 .devid = 0x2992,
 360                 .name = "Intel Q965 SVGA controller",
 361                 .driver = &agp_i810_g965_driver
 362         },
 363         {
 364                 .devid = 0x29A2,
 365                 .name = "Intel G965 SVGA controller",
 366                 .driver = &agp_i810_g965_driver
 367         },
 368         {
 369                 .devid = 0x29B2,
 370                 .name = "Intel Q35 SVGA controller",
 371                 .driver = &agp_i810_g33_driver
 372         },
 373         {
 374                 .devid = 0x29C2,
 375                 .name = "Intel G33 SVGA controller",
 376                 .driver = &agp_i810_g33_driver
 377         },
 378         {
 379                 .devid = 0x29D2,
 380                 .name = "Intel Q33 SVGA controller",
 381                 .driver = &agp_i810_g33_driver
 382         },
 383         {
 384                 .devid = 0xA001,
 385                 .name = "Intel Pineview SVGA controller",
 386                 .driver = &pineview_gtt_driver
 387         },
 388         {
 389                 .devid = 0xA011,
 390                 .name = "Intel Pineview (M) SVGA controller",
 391                 .driver = &pineview_gtt_driver
 392         },
 393         {
 394                 .devid = 0x2A02,
 395                 .name = "Intel GM965 SVGA controller",
 396                 .driver = &agp_i810_g965_driver
 397         },
 398         {
 399                 .devid = 0x2A12,
 400                 .name = "Intel GME965 SVGA controller",
 401                 .driver = &agp_i810_g965_driver
 402         },
 403         {
 404                 .devid = 0x2A42,
 405                 .name = "Intel GM45 SVGA controller",
 406                 .driver = &agp_i810_g4x_driver
 407         },
 408         {
 409                 .devid = 0x2E02,
 410                 .name = "Intel Eaglelake SVGA controller",
 411                 .driver = &agp_i810_g4x_driver
 412         },
 413         {
 414                 .devid = 0x2E12,
 415                 .name = "Intel Q45 SVGA controller",
 416                 .driver = &agp_i810_g4x_driver
 417         },
 418         {
 419                 .devid = 0x2E22,
 420                 .name = "Intel G45 SVGA controller",
 421                 .driver = &agp_i810_g4x_driver
 422         },
 423         {
 424                 .devid = 0x2E32,
 425                 .name = "Intel G41 SVGA controller",
 426                 .driver = &agp_i810_g4x_driver
 427         },
 428         {
 429                 .devid = 0x0042,
 430                 .name = "Intel Ironlake (D) SVGA controller",
 431                 .driver = &agp_i810_g4x_driver
 432         },
 433         {
 434                 .devid = 0x0046,
 435                 .name = "Intel Ironlake (M) SVGA controller",
 436                 .driver = &agp_i810_g4x_driver
 437         },
 438
 439         {
 440                 .devid = 0,
 441         }
 442 };
 443
 444 static const struct agp_i810_match*
 445 agp_i810_match(device_t dev)
 446 {
 447         int i, devid;
 448
 449         if (pci_get_vendor(dev) != PCI_VENDOR_INTEL)
 450                 return (NULL);
 451
 452         devid = pci_get_device(dev);
 453         for (i = 0; agp_i810_matches[i].devid != 0; i++) {
 454                 if (agp_i810_matches[i].devid == devid)
 455                         break;
 456         }
 457         if (agp_i810_matches[i].devid == 0)
 458                 return (NULL);
 459         else
 460                 return (&agp_i810_matches[i]);
 461 }
 462
 463 /*
 464  * Find bridge device.
 465  */
 466 static device_t
 467 agp_i810_find_bridge(device_t dev)
 468 {
 469
 470         return (pci_find_dbsf(0, 0, 0, 0));
 471 }
 472
 473 static void
 474 agp_i810_identify(driver_t *driver, device_t parent)
 475 {
 476
 477         if (device_find_child(parent, "agp", -1) == NULL &&
 478             agp_i810_match(parent))
 479                 device_add_child(parent, "agp", -1);
 480 }
 481
 482 static int
 483 agp_i915_check_active(device_t bridge_dev)
 484 {
 485         int deven;
 486
 487         deven = pci_read_config(bridge_dev, AGP_I915_DEVEN, 4);
 488         if ((deven & AGP_I915_DEVEN_D2F0) == AGP_I915_DEVEN_D2F0_DISABLED)
 489                 return (ENXIO);
 490         return (0);
 491 }
 492
 493 static void
 494 agp_i810_set_desc(device_t dev, const struct agp_i810_match *match)
 495 {
 496
 497         device_set_desc(dev, match->name);
 498 }
 499
 500 static int
 501 agp_i810_probe(device_t dev)
 502 {
 503         device_t bdev;
 504         const struct agp_i810_match *match;
 505         int err = 0;
 506
 507         if (resource_disabled("agp", device_get_unit(dev)))
 508                 return (ENXIO);
 509         match = agp_i810_match(dev);
 510         if (match == NULL)
 511                 return (ENXIO);
 512
 513         bdev = agp_i810_find_bridge(dev);
 514         if (bdev == NULL) {
 515                 if (bootverbose)
 516                         kprintf("I810: can't find bridge device\n");
 517                 return (ENXIO);
 518         }
 519
 520         /*
 521          * checking whether internal graphics device has been activated.
 522          */
 523         if (match->driver->check_active != NULL) {
 524                 err = match->driver->check_active(bdev);
 525                 if (err != 0) {
 526                         if (bootverbose)
 527                                 kprintf("i810: disabled, not probing\n");
 528                         return (err);
 529                 }
 530         }
 531
 532         match->driver->set_desc(dev, match);
 533         return (BUS_PROBE_DEFAULT);
 534 }
 535
 536 static void
 537 agp_i915_dump_regs(device_t dev)
 538 {
 539         struct agp_i810_softc *sc = device_get_softc(dev);
 540
 541         device_printf(dev, "AGP_I810_PGTBL_CTL: %08x\n",
 542             bus_read_4(sc->sc_res[0], AGP_I810_PGTBL_CTL));
 543         device_printf(dev, "AGP_I855_GCC1: 0x%02x\n",
 544             pci_read_config(sc->bdev, AGP_I855_GCC1, 1));
 545         device_printf(dev, "AGP_I915_MSAC: 0x%02x\n",
 546             pci_read_config(sc->bdev, AGP_I915_MSAC, 1));
 547 }
 548
 549 static void
 550 agp_i965_dump_regs(device_t dev)
 551 {
 552         struct agp_i810_softc *sc = device_get_softc(dev);
 553
 554         device_printf(dev, "AGP_I965_PGTBL_CTL2: %08x\n",
 555             bus_read_4(sc->sc_res[0], AGP_I965_PGTBL_CTL2));
 556         device_printf(dev, "AGP_I855_GCC1: 0x%02x\n",
 557             pci_read_config(dev, AGP_I855_GCC1, 1));
 558         device_printf(dev, "AGP_I965_MSAC: 0x%02x\n",
 559             pci_read_config(dev, AGP_I965_MSAC, 1));
 560 }
 561
 562 static int
 563 agp_i915_get_stolen_size(device_t dev)
 564 {
 565         struct agp_i810_softc *sc;
 566         unsigned int gcc1, stolen, gtt_size;
 567
 568         sc = device_get_softc(dev);
 569
 570         /*
 571          * Stolen memory is set up at the beginning of the aperture by
 572          * the BIOS, consisting of the GATT followed by 4kb for the
 573          * BIOS display.
 574          */
 575         switch (sc->match->driver->chiptype) {
 576         case CHIP_I855:
 577                 gtt_size = 128;
 578                 break;
 579         case CHIP_I915:
 580                 gtt_size = 256;
 581                 break;
 582         case CHIP_I965:
 583                 switch (bus_read_4(sc->sc_res[0], AGP_I810_PGTBL_CTL) &
 584                         AGP_I810_PGTBL_SIZE_MASK) {
 585                 case AGP_I810_PGTBL_SIZE_128KB:
 586                         gtt_size = 128;
 587                         break;
 588                 case AGP_I810_PGTBL_SIZE_256KB:
 589                         gtt_size = 256;
 590                         break;
 591                 case AGP_I810_PGTBL_SIZE_512KB:
 592                         gtt_size = 512;
 593                         break;
 594                 case AGP_I965_PGTBL_SIZE_1MB:
 595                         gtt_size = 1024;
 596                         break;
 597                 case AGP_I965_PGTBL_SIZE_2MB:
 598                         gtt_size = 2048;
 599                         break;
 600                 case AGP_I965_PGTBL_SIZE_1_5MB:
 601                         gtt_size = 1024 + 512;
 602                         break;
 603                 default:
 604                         device_printf(dev, "Bad PGTBL size\n");
 605                         return (EINVAL);
 606                 }
 607                 break;
 608         case CHIP_G33:
 609                 gcc1 = pci_read_config(sc->bdev, AGP_I855_GCC1, 2);
 610                 switch (gcc1 & AGP_G33_MGGC_GGMS_MASK) {
 611                 case AGP_G33_MGGC_GGMS_SIZE_1M:
 612                         gtt_size = 1024;
 613                         break;
 614                 case AGP_G33_MGGC_GGMS_SIZE_2M:
 615                         gtt_size = 2048;
 616                         break;
 617                 default:
 618                         device_printf(dev, "Bad PGTBL size\n");
 619                         return (EINVAL);
 620                 }
 621                 break;
 622         case CHIP_IGD:
 623         case CHIP_G4X:
 624                 gtt_size = 0;
 625                 break;
 626         default:
 627                 device_printf(dev, "Bad chiptype\n");
 628                 return (EINVAL);
 629         }
 630
 631         /* GCC1 is called MGGC on i915+ */
 632         gcc1 = pci_read_config(sc->bdev, AGP_I855_GCC1, 1);
 633         switch (gcc1 & AGP_I855_GCC1_GMS) {
 634         case AGP_I855_GCC1_GMS_STOLEN_1M:
 635                 stolen = 1024;
 636                 break;
 637         case AGP_I855_GCC1_GMS_STOLEN_4M:
 638                 stolen = 4 * 1024;
 639                 break;
 640         case AGP_I855_GCC1_GMS_STOLEN_8M:
 641                 stolen = 8 * 1024;
 642                 break;
 643         case AGP_I855_GCC1_GMS_STOLEN_16M:
 644                 stolen = 16 * 1024;
 645                 break;
 646         case AGP_I855_GCC1_GMS_STOLEN_32M:
 647                 stolen = 32 * 1024;
 648                 break;
 649         case AGP_I915_GCC1_GMS_STOLEN_48M:
 650                 stolen = sc->match->driver->gen > 2 ? 48 * 1024 : 0;
 651                 break;
 652         case AGP_I915_GCC1_GMS_STOLEN_64M:
 653                 stolen = sc->match->driver->gen > 2 ? 64 * 1024 : 0;
 654                 break;
 655         case AGP_G33_GCC1_GMS_STOLEN_128M:
 656                 stolen = sc->match->driver->gen > 2 ? 128 * 1024 : 0;
 657                 break;
 658         case AGP_G33_GCC1_GMS_STOLEN_256M:
 659                 stolen = sc->match->driver->gen > 2 ? 256 * 1024 : 0;
 660                 break;
 661         case AGP_G4X_GCC1_GMS_STOLEN_96M:
 662                 if (sc->match->driver->chiptype == CHIP_I965 ||
 663                     sc->match->driver->chiptype == CHIP_G4X)
 664                         stolen = 96 * 1024;
 665                 else
 666                         stolen = 0;
 667                 break;
 668         case AGP_G4X_GCC1_GMS_STOLEN_160M:
 669                 if (sc->match->driver->chiptype == CHIP_I965 ||
 670                     sc->match->driver->chiptype == CHIP_G4X)
 671                         stolen = 160 * 1024;
 672                 else
 673                         stolen = 0;
 674                 break;
 675         case AGP_G4X_GCC1_GMS_STOLEN_224M:
 676                 if (sc->match->driver->chiptype == CHIP_I965 ||
 677                     sc->match->driver->chiptype == CHIP_G4X)
 678                         stolen = 224 * 1024;
 679                 else
 680                         stolen = 0;
 681                 break;
 682         case AGP_G4X_GCC1_GMS_STOLEN_352M:
 683                 if (sc->match->driver->chiptype == CHIP_I965 ||
 684                     sc->match->driver->chiptype == CHIP_G4X)
 685                         stolen = 352 * 1024;
 686                 else
 687                         stolen = 0;
 688                 break;
 689         default:
 690                 device_printf(dev,
 691                     "unknown memory configuration, disabling (GCC1 %x)\n",
 692                     gcc1);
 693                 return (EINVAL);
 694         }
 695
 696         gtt_size += 4;
 697         sc->stolen_size = stolen * 1024;
 698         sc->stolen = (stolen - gtt_size) * 1024 / 4096;
 699
 700         return (0);
 701 }
 702
 703 static int
 704 agp_i915_get_gtt_mappable_entries(device_t dev)
 705 {
 706         struct agp_i810_softc *sc;
 707         uint32_t ap;
 708
 709         sc = device_get_softc(dev);
 710         ap = AGP_GET_APERTURE(dev);
 711         sc->gtt_mappable_entries = ap >> AGP_PAGE_SHIFT;
 712         return (0);
 713 }
 714
 715 static int
 716 agp_i810_get_gtt_total_entries(device_t dev)
 717 {
 718         struct agp_i810_softc *sc;
 719
 720         sc = device_get_softc(dev);
 721         sc->gtt_total_entries = sc->gtt_mappable_entries;
 722         return (0);
 723 }
 724
 725 static int
 726 agp_i965_get_gtt_total_entries(device_t dev)
 727 {
 728         struct agp_i810_softc *sc;
 729         uint32_t pgetbl_ctl;
 730         int error;
 731
 732         sc = device_get_softc(dev);
 733         error = 0;
 734         pgetbl_ctl = bus_read_4(sc->sc_res[0], AGP_I810_PGTBL_CTL);
 735         switch (pgetbl_ctl & AGP_I810_PGTBL_SIZE_MASK) {
 736         case AGP_I810_PGTBL_SIZE_128KB:
 737                 sc->gtt_total_entries = 128 * 1024 / 4;
 738                 break;
 739         case AGP_I810_PGTBL_SIZE_256KB:
 740                 sc->gtt_total_entries = 256 * 1024 / 4;
 741                 break;
 742         case AGP_I810_PGTBL_SIZE_512KB:
 743                 sc->gtt_total_entries = 512 * 1024 / 4;
 744                 break;
 745         /* GTT pagetable sizes bigger than 512KB are not possible on G33! */
 746         case AGP_I810_PGTBL_SIZE_1MB:
 747                 sc->gtt_total_entries = 1024 * 1024 / 4;
 748                 break;
 749         case AGP_I810_PGTBL_SIZE_2MB:
 750                 sc->gtt_total_entries = 2 * 1024 * 1024 / 4;
 751                 break;
 752         case AGP_I810_PGTBL_SIZE_1_5MB:
 753                 sc->gtt_total_entries = (1024 + 512) * 1024 / 4;
 754                 break;
 755         default:
 756                 device_printf(dev, "Unknown page table size\n");
 757                 error = ENXIO;
 758         }
 759         return (error);
 760 }
 761
 762 static void
 763 agp_gen5_adjust_pgtbl_size(device_t dev, uint32_t sz)
 764 {
 765         struct agp_i810_softc *sc;
 766         uint32_t pgetbl_ctl, pgetbl_ctl2;
 767
 768         sc = device_get_softc(dev);
 769
 770         /* Disable per-process page table. */
 771         pgetbl_ctl2 = bus_read_4(sc->sc_res[0], AGP_I965_PGTBL_CTL2);
 772         pgetbl_ctl2 &= ~AGP_I810_PGTBL_ENABLED;
 773         bus_write_4(sc->sc_res[0], AGP_I965_PGTBL_CTL2, pgetbl_ctl2);
 774
 775         /* Write the new ggtt size. */
 776         pgetbl_ctl = bus_read_4(sc->sc_res[0], AGP_I810_PGTBL_CTL);
 777         pgetbl_ctl &= ~AGP_I810_PGTBL_SIZE_MASK;
 778         pgetbl_ctl |= sz;
 779         bus_write_4(sc->sc_res[0], AGP_I810_PGTBL_CTL, pgetbl_ctl);
 780 }
 781
 782 static int
 783 agp_gen5_get_gtt_total_entries(device_t dev)
 784 {
 785         struct agp_i810_softc *sc;
 786         uint16_t gcc1;
 787
 788         sc = device_get_softc(dev);
 789
 790         gcc1 = pci_read_config(sc->bdev, AGP_I830_GCC1, 2);
 791         switch (gcc1 & AGP_G4x_GCC1_SIZE_MASK) {
 792         case AGP_G4x_GCC1_SIZE_1M:
 793         case AGP_G4x_GCC1_SIZE_VT_1M:
 794                 agp_gen5_adjust_pgtbl_size(dev, AGP_I810_PGTBL_SIZE_1MB);
 795                 break;
 796         case AGP_G4x_GCC1_SIZE_VT_1_5M:
 797                 agp_gen5_adjust_pgtbl_size(dev, AGP_I810_PGTBL_SIZE_1_5MB);
 798                 break;
 799         case AGP_G4x_GCC1_SIZE_2M:
 800         case AGP_G4x_GCC1_SIZE_VT_2M:
 801                 agp_gen5_adjust_pgtbl_size(dev, AGP_I810_PGTBL_SIZE_2MB);
 802                 break;
 803         default:
 804                 device_printf(dev, "Unknown page table size\n");
 805                 return (ENXIO);
 806         }
 807
 808         return (agp_i965_get_gtt_total_entries(dev));
 809 }
 810
 811 static int
 812 agp_i830_install_gatt(device_t dev)
 813 {
 814         struct agp_i810_softc *sc;
 815         uint32_t pgtblctl;
 816
 817         sc = device_get_softc(dev);
 818
 819         /*
 820          * The i830 automatically initializes the 128k gatt on boot.
 821          * GATT address is already in there, make sure it's enabled.
 822          */
 823         pgtblctl = bus_read_4(sc->sc_res[0], AGP_I810_PGTBL_CTL);
 824         pgtblctl |= 1;
 825         bus_write_4(sc->sc_res[0], AGP_I810_PGTBL_CTL, pgtblctl);
 826
 827         sc->gatt->ag_physical = pgtblctl & ~1;
 828         return (0);
 829 }
 830
 831 static int
 832 agp_i810_attach(device_t dev)
 833 {
 834         struct agp_i810_softc *sc;
 835         int error;
 836
 837         sc = device_get_softc(dev);
 838         sc->bdev = agp_i810_find_bridge(dev);
 839         if (sc->bdev == NULL)
 840                 return (ENOENT);
 841
 842         sc->match = agp_i810_match(dev);
 843
 844         agp_set_aperture_resource(dev, sc->match->driver->gen <= 2 ?
 845             AGP_APBASE : AGP_I915_GMADR);
 846         error = agp_generic_attach(dev);
 847         if (error)
 848                 return (error);
 849
 850         if (ptoa((vm_paddr_t)Maxmem) >
 851             (1ULL << sc->match->driver->busdma_addr_mask_sz) - 1) {
 852                 device_printf(dev, "agp_i810 does not support physical "
 853                     "memory above %ju.\n", (uintmax_t)(1ULL <<
 854                     sc->match->driver->busdma_addr_mask_sz) - 1);
 855                 return (ENOENT);
 856         }
 857
 858         if (bus_alloc_resources(dev, sc->match->driver->res_spec, sc->sc_res)) {
 859                 agp_generic_detach(dev);
 860                 return (ENODEV);
 861         }
 862
 863         sc->initial_aperture = AGP_GET_APERTURE(dev);
 864         sc->gatt = kmalloc(sizeof(struct agp_gatt), M_DRM, M_WAITOK);
 865         sc->gatt->ag_entries = AGP_GET_APERTURE(dev) >> AGP_PAGE_SHIFT;
 866
 867         if ((error = sc->match->driver->get_stolen_size(dev)) != 0 ||
 868             (error = sc->match->driver->install_gatt(dev)) != 0 ||
 869             (error = sc->match->driver->get_gtt_mappable_entries(dev)) != 0 ||
 870             (error = sc->match->driver->get_gtt_total_entries(dev)) != 0 ||
 871             (error = sc->match->driver->chipset_flush_setup(dev)) != 0) {
 872                 bus_release_resources(dev, sc->match->driver->res_spec,
 873                     sc->sc_res);
 874                 kfree(sc->gatt);
 875                 agp_generic_detach(dev);
 876                 return (error);
 877         }
 878
 879         intel_agp = dev;
 880         device_printf(dev, "aperture size is %dM",
 881             sc->initial_aperture / 1024 / 1024);
 882         if (sc->stolen > 0)
 883                 kprintf(", detected %dk stolen memory\n", sc->stolen * 4);
 884         else
 885                 kprintf("\n");
 886         if (bootverbose) {
 887                 sc->match->driver->dump_regs(dev);
 888                 device_printf(dev, "Mappable GTT entries: %d\n",
 889                     sc->gtt_mappable_entries);
 890                 device_printf(dev, "Total GTT entries: %d\n",
 891                     sc->gtt_total_entries);
 892         }
 893         return (0);
 894 }
 895
 896 static void
 897 agp_i830_deinstall_gatt(device_t dev)
 898 {
 899         struct agp_i810_softc *sc;
 900         unsigned int pgtblctl;
 901
 902         sc = device_get_softc(dev);
 903         pgtblctl = bus_read_4(sc->sc_res[0], AGP_I810_PGTBL_CTL);
 904         pgtblctl &= ~1;
 905         bus_write_4(sc->sc_res[0], AGP_I810_PGTBL_CTL, pgtblctl);
 906 }
 907
 908 static int
 909 agp_i810_detach(device_t dev)
 910 {
 911         struct agp_i810_softc *sc;
 912
 913         sc = device_get_softc(dev);
 914         agp_free_cdev(dev);
 915
 916         /* Clear the GATT base. */
 917         sc->match->driver->deinstall_gatt(dev);
 918
 919         sc->match->driver->chipset_flush_teardown(dev);
 920
 921         /* Put the aperture back the way it started. */
 922         AGP_SET_APERTURE(dev, sc->initial_aperture);
 923
 924         kfree(sc->gatt);
 925         bus_release_resources(dev, sc->match->driver->res_spec, sc->sc_res);
 926         agp_free_res(dev);
 927
 928         return (0);
 929 }
 930
 931 static int
 932 agp_i810_resume(device_t dev)
 933 {
 934         struct agp_i810_softc *sc;
 935         sc = device_get_softc(dev);
 936
 937         AGP_SET_APERTURE(dev, sc->initial_aperture);
 938
 939         /* Install the GATT. */
 940         bus_write_4(sc->sc_res[0], AGP_I810_PGTBL_CTL,
 941         sc->gatt->ag_physical | 1);
 942
 943         return (bus_generic_resume(dev));
 944 }
 945
 946 /**
 947  * Sets the PCI resource size of the aperture on i830-class and below chipsets,
 948  * while returning failure on later chipsets when an actual change is
 949  * requested.
 950  *
 951  * This whole function is likely bogus, as the kernel would probably need to
 952  * reconfigure the placement of the AGP aperture if a larger size is requested,
 953  * which doesn't happen currently.
 954  */
 955
 956 static int
 957 agp_i915_set_aperture(device_t dev, u_int32_t aperture)
 958 {
 959
 960         return (agp_generic_set_aperture(dev, aperture));
 961 }
 962
 963 static int
 964 agp_i810_method_set_aperture(device_t dev, u_int32_t aperture)
 965 {
 966         struct agp_i810_softc *sc;
 967
 968         sc = device_get_softc(dev);
 969         return (sc->match->driver->set_aperture(dev, aperture));
 970 }
 971
 972 /**
 973  * Writes a GTT entry mapping the page at the given offset from the
 974  * beginning of the aperture to the given physical address.  Setup the
 975  * caching mode according to flags.
 976  *
 977  * For gen 1, 2 and 3, GTT start is located at AGP_I810_GTT offset
 978  * from corresponding BAR start. For gen 4, offset is 512KB +
 979  * AGP_I810_GTT, for gen 5 and 6 it is 2MB + AGP_I810_GTT.
 980  *
 981  * Also, the bits of the physical page address above 4GB needs to be
 982  * placed into bits 40-32 of PTE.
 983  */
 984 static void
 985 agp_i915_install_gtt_pte(device_t dev, u_int index, vm_offset_t physical,
 986     int flags)
 987 {
 988         uint32_t pte;
 989
 990         pte = (u_int32_t)physical | I810_PTE_VALID;
 991         if (flags == AGP_USER_CACHED_MEMORY)
 992                 pte |= I830_PTE_SYSTEM_CACHED;
 993
 994         agp_i915_write_gtt(dev, index, pte);
 995 }
 996
 997 static void
 998 agp_i915_write_gtt(device_t dev, u_int index, uint32_t pte)
 999 {
1000         struct agp_i810_softc *sc;
1001
1002         sc = device_get_softc(dev);
1003         bus_write_4(sc->sc_res[0], index * 4, pte);
1004 }
1005
1006 static void
1007 agp_i965_install_gtt_pte(device_t dev, u_int index, vm_offset_t physical,
1008     int flags)
1009 {
1010         uint32_t pte;
1011
1012         pte = (u_int32_t)physical | I810_PTE_VALID;
1013         if (flags == AGP_USER_CACHED_MEMORY)
1014                 pte |= I830_PTE_SYSTEM_CACHED;
1015
1016         pte |= (physical >> 28) & 0xf0;
1017         agp_i965_write_gtt(dev, index, pte);
1018 }
1019
1020 static void
1021 agp_i965_write_gtt(device_t dev, u_int index, uint32_t pte)
1022 {
1023         struct agp_i810_softc *sc;
1024
1025         sc = device_get_softc(dev);
1026         bus_write_4(sc->sc_res[0], index * 4 + (512 * 1024), pte);
1027 }
1028
1029 static void
1030 agp_g4x_install_gtt_pte(device_t dev, u_int index, vm_offset_t physical,
1031     int flags)
1032 {
1033         uint32_t pte;
1034
1035         pte = (u_int32_t)physical | I810_PTE_VALID;
1036         if (flags == AGP_USER_CACHED_MEMORY)
1037                 pte |= I830_PTE_SYSTEM_CACHED;
1038
1039         pte |= (physical >> 28) & 0xf0;
1040         agp_g4x_write_gtt(dev, index, pte);
1041 }
1042
1043 static void
1044 agp_g4x_write_gtt(device_t dev, u_int index, uint32_t pte)
1045 {
1046         struct agp_i810_softc *sc;
1047
1048         sc = device_get_softc(dev);
1049         bus_write_4(sc->sc_res[0], index * 4 + (2 * 1024 * 1024), pte);
1050 }
1051
1052 static int
1053 agp_i810_bind_page(device_t dev, vm_offset_t offset, vm_offset_t physical)
1054 {
1055         struct agp_i810_softc *sc = device_get_softc(dev);
1056         u_int index;
1057
1058         if (offset >= (sc->gatt->ag_entries << AGP_PAGE_SHIFT)) {
1059                 device_printf(dev, "failed: offset is 0x%08jx, "
1060                     "shift is %d, entries is %d\n", (intmax_t)offset,
1061                     AGP_PAGE_SHIFT, sc->gatt->ag_entries);
1062                 return (EINVAL);
1063         }
1064         index = offset >> AGP_PAGE_SHIFT;
1065         if (sc->stolen != 0 && index < sc->stolen) {
1066                 device_printf(dev, "trying to bind into stolen memory\n");
1067                 return (EINVAL);
1068         }
1069         sc->match->driver->install_gtt_pte(dev, index, physical, 0);
1070         return (0);
1071 }
1072
1073 static int
1074 agp_i810_unbind_page(device_t dev, vm_offset_t offset)
1075 {
1076         struct agp_i810_softc *sc;
1077         u_int index;
1078
1079         sc = device_get_softc(dev);
1080         if (offset >= (sc->gatt->ag_entries << AGP_PAGE_SHIFT))
1081                 return (EINVAL);
1082         index = offset >> AGP_PAGE_SHIFT;
1083         if (sc->stolen != 0 && index < sc->stolen) {
1084                 device_printf(dev, "trying to unbind from stolen memory\n");
1085                 return (EINVAL);
1086         }
1087         sc->match->driver->install_gtt_pte(dev, index, 0, 0);
1088         return (0);
1089 }
1090
1091 static void
1092 agp_i915_sync_gtt_pte(device_t dev, u_int index)
1093 {
1094         struct agp_i810_softc *sc;
1095
1096         sc = device_get_softc(dev);
1097         bus_read_4(sc->sc_res[1], index * 4);
1098 }
1099
1100 static void
1101 agp_i965_sync_gtt_pte(device_t dev, u_int index)
1102 {
1103         struct agp_i810_softc *sc;
1104
1105         sc = device_get_softc(dev);
1106         bus_read_4(sc->sc_res[0], index * 4 + (512 * 1024));
1107 }
1108
1109 static void
1110 agp_g4x_sync_gtt_pte(device_t dev, u_int index)
1111 {
1112         struct agp_i810_softc *sc;
1113
1114         sc = device_get_softc(dev);
1115         bus_read_4(sc->sc_res[0], index * 4 + (2 * 1024 * 1024));
1116 }
1117
1118 /*
1119  * Writing via memory mapped registers already flushes all TLBs.
1120  */
1121 static void
1122 agp_i810_flush_tlb(device_t dev)
1123 {
1124 }
1125
1126 static int
1127 agp_i810_enable(device_t dev, u_int32_t mode)
1128 {
1129
1130         return (0);
1131 }
1132
1133 static struct agp_memory *
1134 agp_i810_alloc_memory(device_t dev, int type, vm_size_t size)
1135 {
1136         struct agp_i810_softc *sc;
1137         struct agp_memory *mem;
1138         vm_page_t m;
1139
1140         sc = device_get_softc(dev);
1141
1142         if ((size & (AGP_PAGE_SIZE - 1)) != 0 ||
1143             sc->agp.as_allocated + size > sc->agp.as_maxmem)
1144                 return (0);
1145
1146         if (type == 1) {
1147                 /*
1148                  * Mapping local DRAM into GATT.
1149                  */
1150                 if (sc->match->driver->chiptype != CHIP_I810)
1151                         return (0);
1152                 if (size != sc->dcache_size)
1153                         return (0);
1154         } else if (type == 2) {
1155                 /*
1156                  * Type 2 is the contiguous physical memory type, that hands
1157                  * back a physical address.  This is used for cursors on i810.
1158                  * Hand back as many single pages with physical as the user
1159                  * wants, but only allow one larger allocation (ARGB cursor)
1160                  * for simplicity.
1161                  */
1162                 if (size != AGP_PAGE_SIZE) {
1163                         if (sc->argb_cursor != NULL)
1164                                 return (0);
1165
1166                         /* Allocate memory for ARGB cursor, if we can. */
1167                         sc->argb_cursor = contigmalloc(size, M_DRM,
1168                            0, 0, ~0, PAGE_SIZE, 0);
1169                         if (sc->argb_cursor == NULL)
1170                                 return (0);
1171                 }
1172         }
1173
1174         mem = kmalloc(sizeof *mem, M_DRM, M_INTWAIT);
1175         mem->am_id = sc->agp.as_nextid++;
1176         mem->am_size = size;
1177         mem->am_type = type;
1178         if (type != 1 && (type != 2 || size == AGP_PAGE_SIZE))
1179                 mem->am_obj = vm_object_allocate(OBJT_DEFAULT,
1180                     atop(round_page(size)));
1181         else
1182                 mem->am_obj = 0;
1183
1184         if (type == 2) {
1185                 if (size == AGP_PAGE_SIZE) {
1186                         /*
1187                          * Allocate and wire down the page now so that we can
1188                          * get its physical address.
1189                          */
1190                         VM_OBJECT_LOCK(mem->am_obj);
1191                         m = vm_page_grab(mem->am_obj, 0, VM_ALLOC_NORMAL |
1192                                                          VM_ALLOC_ZERO |
1193                                                          VM_ALLOC_RETRY);
1194                         vm_page_wire(m);
1195                         VM_OBJECT_UNLOCK(mem->am_obj);
1196                         mem->am_physical = VM_PAGE_TO_PHYS(m);
1197                         vm_page_wakeup(m);
1198                 } else {
1199                         /* Our allocation is already nicely wired down for us.
1200                          * Just grab the physical address.
1201                          */
1202                         mem->am_physical = vtophys(sc->argb_cursor);
1203                 }
1204         } else
1205                 mem->am_physical = 0;
1206
1207         mem->am_offset = 0;
1208         mem->am_is_bound = 0;
1209         TAILQ_INSERT_TAIL(&sc->agp.as_memory, mem, am_link);
1210         sc->agp.as_allocated += size;
1211
1212         return (mem);
1213 }
1214
1215 static int
1216 agp_i810_free_memory(device_t dev, struct agp_memory *mem)
1217 {
1218         struct agp_i810_softc *sc;
1219
1220         if (mem->am_is_bound)
1221                 return (EBUSY);
1222
1223         sc = device_get_softc(dev);
1224
1225         if (mem->am_type == 2) {
1226                 if (mem->am_size == AGP_PAGE_SIZE) {
1227                         /*
1228                          * Unwire the page which we wired in alloc_memory.
1229                          */
1230                         vm_page_t m;
1231
1232                         vm_object_hold(mem->am_obj);
1233                         m = vm_page_lookup_busy_wait(mem->am_obj, 0,
1234                                                      FALSE, "agppg");
1235                         vm_object_drop(mem->am_obj);
1236                         vm_page_unwire(m, 0);
1237                         vm_page_wakeup(m);
1238                 } else {
1239                         contigfree(sc->argb_cursor, mem->am_size, M_DRM);
1240                         sc->argb_cursor = NULL;
1241                 }
1242         }
1243
1244         sc->agp.as_allocated -= mem->am_size;
1245         TAILQ_REMOVE(&sc->agp.as_memory, mem, am_link);
1246         if (mem->am_obj)
1247                 vm_object_deallocate(mem->am_obj);
1248         kfree(mem);
1249         return (0);
1250 }
1251
1252 static int
1253 agp_i810_bind_memory(device_t dev, struct agp_memory *mem, vm_offset_t offset)
1254 {
1255         struct agp_i810_softc *sc;
1256         vm_offset_t i;
1257
1258         /* Do some sanity checks first. */
1259         if ((offset & (AGP_PAGE_SIZE - 1)) != 0 ||
1260             offset + mem->am_size > AGP_GET_APERTURE(dev)) {
1261                 device_printf(dev, "binding memory at bad offset %#x\n",
1262                     (int)offset);
1263                 return (EINVAL);
1264         }
1265
1266         sc = device_get_softc(dev);
1267         if (mem->am_type == 2 && mem->am_size != AGP_PAGE_SIZE) {
1268                 lockmgr(&sc->agp.as_lock, LK_EXCLUSIVE);
1269                 if (mem->am_is_bound) {
1270                         lockmgr(&sc->agp.as_lock, LK_RELEASE);
1271                         return EINVAL;
1272                 }
1273                 /* The memory's already wired down, just stick it in the GTT. */
1274                 for (i = 0; i < mem->am_size; i += AGP_PAGE_SIZE) {
1275                         sc->match->driver->install_gtt_pte(dev, (offset + i) >>
1276                             AGP_PAGE_SHIFT, mem->am_physical + i, 0);
1277                 }
1278                 agp_flush_cache();
1279                 mem->am_offset = offset;
1280                 mem->am_is_bound = 1;
1281                 lockmgr(&sc->agp.as_lock, LK_RELEASE);
1282                 return (0);
1283         }
1284
1285         if (mem->am_type != 1)
1286                 return (agp_generic_bind_memory(dev, mem, offset));
1287
1288         /*
1289          * Mapping local DRAM into GATT.
1290          */
1291         if (sc->match->driver->chiptype != CHIP_I810)
1292                 return (EINVAL);
1293         for (i = 0; i < mem->am_size; i += AGP_PAGE_SIZE)
1294                 bus_write_4(sc->sc_res[0],
1295                     AGP_I810_GTT + (i >> AGP_PAGE_SHIFT) * 4, i | 3);
1296
1297         return (0);
1298 }
1299
1300 static int
1301 agp_i810_unbind_memory(device_t dev, struct agp_memory *mem)
1302 {
1303         struct agp_i810_softc *sc;
1304         vm_offset_t i;
1305
1306         sc = device_get_softc(dev);
1307
1308         if (mem->am_type == 2 && mem->am_size != AGP_PAGE_SIZE) {
1309                 lockmgr(&sc->agp.as_lock, LK_EXCLUSIVE);
1310                 if (!mem->am_is_bound) {
1311                         lockmgr(&sc->agp.as_lock, LK_RELEASE);
1312                         return (EINVAL);
1313                 }
1314
1315                 for (i = 0; i < mem->am_size; i += AGP_PAGE_SIZE) {
1316                         sc->match->driver->install_gtt_pte(dev,
1317                             (mem->am_offset + i) >> AGP_PAGE_SHIFT, 0, 0);
1318                 }
1319                 agp_flush_cache();
1320                 mem->am_is_bound = 0;
1321                 lockmgr(&sc->agp.as_lock, LK_RELEASE);
1322                 return (0);
1323         }
1324
1325         if (mem->am_type != 1)
1326                 return (agp_generic_unbind_memory(dev, mem));
1327
1328         if (sc->match->driver->chiptype != CHIP_I810)
1329                 return (EINVAL);
1330         for (i = 0; i < mem->am_size; i += AGP_PAGE_SIZE) {
1331                 sc->match->driver->install_gtt_pte(dev, i >> AGP_PAGE_SHIFT,
1332                     0, 0);
1333         }
1334         return (0);
1335 }
1336
1337 static device_method_t agp_i810_methods[] = {
1338         /* Device interface */
1339         DEVMETHOD(device_identify,      agp_i810_identify),
1340         DEVMETHOD(device_probe,         agp_i810_probe),
1341         DEVMETHOD(device_attach,        agp_i810_attach),
1342         DEVMETHOD(device_detach,        agp_i810_detach),
1343         DEVMETHOD(device_suspend,       bus_generic_suspend),
1344         DEVMETHOD(device_resume,        agp_i810_resume),
1345
1346         /* AGP interface */
1347         DEVMETHOD(agp_get_aperture,     agp_generic_get_aperture),
1348         DEVMETHOD(agp_set_aperture,     agp_i810_method_set_aperture),
1349         DEVMETHOD(agp_bind_page,        agp_i810_bind_page),
1350         DEVMETHOD(agp_unbind_page,      agp_i810_unbind_page),
1351         DEVMETHOD(agp_flush_tlb,        agp_i810_flush_tlb),
1352         DEVMETHOD(agp_enable,           agp_i810_enable),
1353         DEVMETHOD(agp_alloc_memory,     agp_i810_alloc_memory),
1354         DEVMETHOD(agp_free_memory,      agp_i810_free_memory),
1355         DEVMETHOD(agp_bind_memory,      agp_i810_bind_memory),
1356         DEVMETHOD(agp_unbind_memory,    agp_i810_unbind_memory),
1357         DEVMETHOD(agp_chipset_flush,    agp_intel_gtt_chipset_flush),
1358
1359         DEVMETHOD_END
1360 };
1361
1362 static driver_t agp_i810_driver = {
1363         "agp",
1364         agp_i810_methods,
1365         sizeof(struct agp_i810_softc),
1366 };
1367
1368 static devclass_t agp_devclass;
1369
1370 DRIVER_MODULE(agp_i810, vgapci, agp_i810_driver, agp_devclass, NULL, NULL);
1371 MODULE_DEPEND(agp_i810, agp, 1, 1, 1);
1372 MODULE_DEPEND(agp_i810, pci, 1, 1, 1);
1373
1374 extern vm_page_t bogus_page;
1375
1376 void
1377 agp_intel_gtt_clear_range(device_t dev, u_int first_entry, u_int num_entries)
1378 {
1379         struct agp_i810_softc *sc;
1380         u_int i;
1381
1382         sc = device_get_softc(dev);
1383         for (i = 0; i < num_entries; i++)
1384                 sc->match->driver->install_gtt_pte(dev, first_entry + i,
1385                     VM_PAGE_TO_PHYS(bogus_page), 0);
1386         sc->match->driver->sync_gtt_pte(dev, first_entry + num_entries - 1);
1387 }
1388
1389 void
1390 agp_intel_gtt_insert_pages(device_t dev, u_int first_entry, u_int num_entries,
1391     vm_page_t *pages, u_int flags)
1392 {
1393         struct agp_i810_softc *sc;
1394         u_int i;
1395
1396         sc = device_get_softc(dev);
1397         for (i = 0; i < num_entries; i++) {
1398                 KKASSERT(pages[i]->valid == VM_PAGE_BITS_ALL);
1399                 KKASSERT(pages[i]->wire_count > 0);
1400                 sc->match->driver->install_gtt_pte(dev, first_entry + i,
1401                     VM_PAGE_TO_PHYS(pages[i]), flags);
1402         }
1403         sc->match->driver->sync_gtt_pte(dev, first_entry + num_entries - 1);
1404 }
1405
1406 void
1407 intel_gtt_insert_sg_entries(struct sg_table *st,
1408                             unsigned int pg_start,
1409                             unsigned int flags)
1410 {
1411         struct agp_i810_softc *sc = device_get_softc(intel_agp);
1412         struct scatterlist *sg;
1413         dma_addr_t page;
1414         int i, j, npages, subpage;
1415
1416         i = 0;
1417         for_each_sg(st->sgl, sg, st->nents, j) {
1418                 npages = sg_dma_len(sg) / PAGE_SIZE;
1419                 for (subpage = 0; subpage < npages; subpage++) {
1420                         page = sg_dma_address(sg) + subpage * PAGE_SIZE;
1421                         sc->match->driver->install_gtt_pte(intel_agp,
1422                                 pg_start + i, page, flags);
1423                         i++;
1424                 }
1425         }
1426         sc->match->driver->sync_gtt_pte(intel_agp, pg_start + i - 1);
1427 }
1428
1429
1430 struct intel_gtt
1431 agp_intel_gtt_get(device_t dev)
1432 {
1433         struct agp_i810_softc *sc;
1434         struct intel_gtt res;
1435
1436         sc = device_get_softc(dev);
1437         res.stolen_size = sc->stolen_size;
1438         res.gtt_total_entries = sc->gtt_total_entries;
1439         res.gtt_mappable_entries = sc->gtt_mappable_entries;
1440         res.do_idle_maps = 0;
1441         res.scratch_page_dma = VM_PAGE_TO_PHYS(bogus_page);
1442         return (res);
1443 }
1444
1445 static int
1446 agp_i915_chipset_flush_alloc_page(device_t dev, uint64_t start, uint64_t end)
1447 {
1448         struct agp_i810_softc *sc;
1449         device_t vga;
1450
1451         sc = device_get_softc(dev);
1452         vga = device_get_parent(dev);
1453         sc->sc_flush_page_rid = 100;
1454         sc->sc_flush_page_res = BUS_ALLOC_RESOURCE(device_get_parent(vga), dev,
1455             SYS_RES_MEMORY, &sc->sc_flush_page_rid, start, end, PAGE_SIZE,
1456             RF_ACTIVE, -1);
1457         if (sc->sc_flush_page_res == NULL) {
1458                 device_printf(dev, "Failed to allocate flush page at 0x%jx\n",
1459                     (uintmax_t)start);
1460                 return (EINVAL);
1461         }
1462         sc->sc_flush_page_vaddr = rman_get_virtual(sc->sc_flush_page_res);
1463         if (bootverbose) {
1464                 device_printf(dev, "Allocated flush page phys 0x%jx virt %p\n",
1465                     (uintmax_t)rman_get_start(sc->sc_flush_page_res),
1466                     sc->sc_flush_page_vaddr);
1467         }
1468         return (0);
1469 }
1470
1471 static void
1472 agp_i915_chipset_flush_free_page(device_t dev)
1473 {
1474         struct agp_i810_softc *sc;
1475         device_t vga;
1476
1477         sc = device_get_softc(dev);
1478         vga = device_get_parent(dev);
1479         if (sc->sc_flush_page_res == NULL)
1480                 return;
1481         BUS_DEACTIVATE_RESOURCE(device_get_parent(vga), dev, SYS_RES_MEMORY,
1482             sc->sc_flush_page_rid, sc->sc_flush_page_res);
1483         BUS_RELEASE_RESOURCE(device_get_parent(vga), dev, SYS_RES_MEMORY,
1484             sc->sc_flush_page_rid, sc->sc_flush_page_res);
1485 }
1486
1487 static int
1488 agp_i915_chipset_flush_setup(device_t dev)
1489 {
1490         struct agp_i810_softc *sc;
1491         uint32_t temp;
1492         int error;
1493
1494         sc = device_get_softc(dev);
1495         temp = pci_read_config(sc->bdev, AGP_I915_IFPADDR, 4);
1496         if ((temp & 1) != 0) {
1497                 temp &= ~1;
1498                 if (bootverbose)
1499                         device_printf(dev,
1500                             "Found already configured flush page at 0x%jx\n",
1501                             (uintmax_t)temp);
1502                 sc->sc_bios_allocated_flush_page = 1;
1503                 /*
1504                  * In the case BIOS initialized the flush pointer (?)
1505                  * register, expect that BIOS also set up the resource
1506                  * for the page.
1507                  */
1508                 error = agp_i915_chipset_flush_alloc_page(dev, temp,
1509                     temp + PAGE_SIZE - 1);
1510                 if (error != 0)
1511                         return (error);
1512         } else {
1513                 sc->sc_bios_allocated_flush_page = 0;
1514                 error = agp_i915_chipset_flush_alloc_page(dev, 0, 0xffffffff);
1515                 if (error != 0)
1516                         return (error);
1517                 temp = rman_get_start(sc->sc_flush_page_res);
1518                 pci_write_config(sc->bdev, AGP_I915_IFPADDR, temp | 1, 4);
1519         }
1520         return (0);
1521 }
1522
1523 static void
1524 agp_i915_chipset_flush_teardown(device_t dev)
1525 {
1526         struct agp_i810_softc *sc;
1527         uint32_t temp;
1528
1529         sc = device_get_softc(dev);
1530         if (sc->sc_flush_page_res == NULL)
1531                 return;
1532         if (!sc->sc_bios_allocated_flush_page) {
1533                 temp = pci_read_config(sc->bdev, AGP_I915_IFPADDR, 4);
1534                 temp &= ~1;
1535                 pci_write_config(sc->bdev, AGP_I915_IFPADDR, temp, 4);
1536         }
1537         agp_i915_chipset_flush_free_page(dev);
1538 }
1539
1540 static int
1541 agp_i965_chipset_flush_setup(device_t dev)
1542 {
1543         struct agp_i810_softc *sc;
1544         uint64_t temp;
1545         uint32_t temp_hi, temp_lo;
1546         int error;
1547
1548         sc = device_get_softc(dev);
1549
1550         temp_hi = pci_read_config(sc->bdev, AGP_I965_IFPADDR + 4, 4);
1551         temp_lo = pci_read_config(sc->bdev, AGP_I965_IFPADDR, 4);
1552
1553         if ((temp_lo & 1) != 0) {
1554                 temp = ((uint64_t)temp_hi << 32) | (temp_lo & ~1);
1555                 if (bootverbose)
1556                         device_printf(dev,
1557                             "Found already configured flush page at 0x%jx\n",
1558                             (uintmax_t)temp);
1559                 sc->sc_bios_allocated_flush_page = 1;
1560                 /*
1561                  * In the case BIOS initialized the flush pointer (?)
1562                  * register, expect that BIOS also set up the resource
1563                  * for the page.
1564                  */
1565                 error = agp_i915_chipset_flush_alloc_page(dev, temp,
1566                     temp + PAGE_SIZE - 1);
1567                 if (error != 0)
1568                         return (error);
1569         } else {
1570                 sc->sc_bios_allocated_flush_page = 0;
1571                 error = agp_i915_chipset_flush_alloc_page(dev, 0, ~0);
1572                 if (error != 0)
1573                         return (error);
1574                 temp = rman_get_start(sc->sc_flush_page_res);
1575                 pci_write_config(sc->bdev, AGP_I965_IFPADDR + 4,
1576                     (temp >> 32) & UINT32_MAX, 4);
1577                 pci_write_config(sc->bdev, AGP_I965_IFPADDR,
1578                     (temp & UINT32_MAX) | 1, 4);
1579         }
1580         return (0);
1581 }
1582
1583 static void
1584 agp_i965_chipset_flush_teardown(device_t dev)
1585 {
1586         struct agp_i810_softc *sc;
1587         uint32_t temp_lo;
1588
1589         sc = device_get_softc(dev);
1590         if (sc->sc_flush_page_res == NULL)
1591                 return;
1592         if (!sc->sc_bios_allocated_flush_page) {
1593                 temp_lo = pci_read_config(sc->bdev, AGP_I965_IFPADDR, 4);
1594                 temp_lo &= ~1;
1595                 pci_write_config(sc->bdev, AGP_I965_IFPADDR, temp_lo, 4);
1596         }
1597         agp_i915_chipset_flush_free_page(dev);
1598 }
1599
1600 static void
1601 agp_i915_chipset_flush(device_t dev)
1602 {
1603         struct agp_i810_softc *sc;
1604
1605         sc = device_get_softc(dev);
1606         *(uint32_t *)sc->sc_flush_page_vaddr = 1;
1607 }
1608
1609 int
1610 agp_intel_gtt_chipset_flush(device_t dev)
1611 {
1612         struct agp_i810_softc *sc;
1613
1614         sc = device_get_softc(dev);
1615         sc->match->driver->chipset_flush(dev);
1616         return (0);
1617 }
1618
1619 void
1620 intel_gtt_clear_range(u_int first_entry, u_int num_entries)
1621 {
1622
1623         agp_intel_gtt_clear_range(intel_agp, first_entry, num_entries);
1624 }
1625
1626 void intel_gtt_get(u64 *gtt_total, size_t *stolen_size,
1627                    phys_addr_t *mappable_base, u64 *mappable_end)
1628 {
1629         struct agp_info ainfo;
1630
1631         intel_private.base = agp_intel_gtt_get(intel_agp);
1632
1633         *gtt_total = intel_private.base.gtt_total_entries << PAGE_SHIFT;
1634         *stolen_size = intel_private.base.stolen_size;
1635         agp_get_info(intel_agp, &ainfo);
1636         *mappable_base = ainfo.ai_aperture_base;
1637         *mappable_end = intel_private.base.gtt_mappable_entries << PAGE_SHIFT;
1638 }
1639
1640 int
1641 intel_gtt_chipset_flush(void)
1642 {
1643
1644         return (agp_intel_gtt_chipset_flush(intel_agp));
1645 }
1646
1647 /*
1648  * Only used by gen6
1649  */
1650 void
1651 intel_gtt_sync_pte(u_int entry)
1652 {
1653         struct agp_i810_softc *sc;
1654
1655         sc = device_get_softc(intel_agp);
1656         sc->match->driver->sync_gtt_pte(intel_agp, entry);
1657 }
1658
1659 /*
1660  * Only used by gen6
1661  */
1662 void
1663 intel_gtt_write(u_int entry, uint32_t val)
1664 {
1665         struct agp_i810_softc *sc;
1666
1667         sc = device_get_softc(intel_agp);
1668         sc->match->driver->write_gtt(intel_agp, entry, val);
1669 }
1670
1671 #define GFX_FLSH_CNTL   0x2170
1672
1673 bool
1674 intel_enable_gtt(void)
1675 {
1676         struct agp_i810_softc *sc = device_get_softc(intel_agp);
1677
1678         /* Some chipsets such as Pineview can't report if the GTT
1679          * has been enabled or not.
1680          * Assume everything is fine. */
1681
1682         /* Flush all chipset write buffers nevertheless */
1683         bus_write_4(sc->sc_res[0], GFX_FLSH_CNTL, 1);
1684         bus_write_4(sc->sc_res[0], GFX_FLSH_CNTL, 0);
1685
1686         return true;
1687 }