Remove old binutils225 manual pages via 'make upgrade'.
[dragonfly.git] / sys / dev / agp / intel-gtt.c
1 /*
2  * Copyright (c) 2000 Doug Rabson
3  * Copyright (c) 2000 Ruslan Ermilov
4  * Copyright (c) 2011 The FreeBSD Foundation
5  * Copyright (c) 2017-2019 Fran├žois Tigeot <ftigeot@wolfpond.org>
6  * All rights reserved.
7  *
8  * Portions of this software were developed by Konstantin Belousov
9  * under sponsorship from the FreeBSD Foundation.
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions
13  * are met:
14  * 1. Redistributions of source code must retain the above copyright
15  *    notice, this list of conditions and the following disclaimer.
16  * 2. Redistributions in binary form must reproduce the above copyright
17  *    notice, this list of conditions and the following disclaimer in the
18  *    documentation and/or other materials provided with the distribution.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
24  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30  * SUCH DAMAGE.
31  */
32
33 /*
34  * Fixes for 830/845G support: David Dawes <dawes@xfree86.org>
35  * 852GM/855GM/865G support added by David Dawes <dawes@xfree86.org>
36  *
37  * This is generic Intel GTT handling code, morphed from the AGP
38  * bridge code.
39  */
40
41 #if 0
42 #define KTR_AGP_I810    KTR_DEV
43 #else
44 #define KTR_AGP_I810    0
45 #endif
46
47 #include <sys/param.h>
48 #include <sys/systm.h>
49 #include <sys/malloc.h>
50 #include <sys/kernel.h>
51 #include <sys/bus.h>
52 #include <sys/lock.h>
53 #include <sys/rman.h>
54
55 #include "pcidevs.h"
56 #include <bus/pci/pcivar.h>
57 #include <bus/pci/pcireg.h>
58 #include "agppriv.h"
59 #include "agpreg.h"
60
61 #include <vm/vm.h>
62 #include <vm/vm_object.h>
63 #include <vm/vm_page.h>
64 #include <vm/vm_param.h>
65 #include <vm/vm_pageout.h>
66 #include <vm/pmap.h>
67
68 #include <vm/vm_page2.h>
69
70 #include <machine/md_var.h>
71
72 #include <linux/slab.h>
73 #include <linux/scatterlist.h>
74 #include <drm/intel-gtt.h>
75
76 struct agp_i810_match;
77
78 static int agp_i915_check_active(device_t bridge_dev);
79
80 static void agp_i810_set_desc(device_t dev, const struct agp_i810_match *match);
81
82 static void agp_i915_dump_regs(device_t dev);
83 static void agp_i965_dump_regs(device_t dev);
84
85 static int agp_i915_get_stolen_size(device_t dev);
86
87 static int agp_i915_get_gtt_mappable_entries(device_t dev);
88
89 static int agp_i810_get_gtt_total_entries(device_t dev);
90 static int agp_i965_get_gtt_total_entries(device_t dev);
91 static int agp_gen5_get_gtt_total_entries(device_t dev);
92
93 static int agp_i830_install_gatt(device_t dev);
94
95 static void agp_i830_deinstall_gatt(device_t dev);
96
97 static void agp_i915_install_gtt_pte(device_t dev, u_int index,
98     vm_offset_t physical, int flags);
99 static void agp_i965_install_gtt_pte(device_t dev, u_int index,
100     vm_offset_t physical, int flags);
101 static void agp_g4x_install_gtt_pte(device_t dev, u_int index,
102     vm_offset_t physical, int flags);
103
104 static void agp_i915_write_gtt(device_t dev, u_int index, uint32_t pte);
105 static void agp_i965_write_gtt(device_t dev, u_int index, uint32_t pte);
106 static void agp_g4x_write_gtt(device_t dev, u_int index, uint32_t pte);
107
108 static void agp_i915_sync_gtt_pte(device_t dev, u_int index);
109 static void agp_i965_sync_gtt_pte(device_t dev, u_int index);
110 static void agp_g4x_sync_gtt_pte(device_t dev, u_int index);
111
112 static int agp_i915_set_aperture(device_t dev, u_int32_t aperture);
113
114 static int agp_i915_chipset_flush_setup(device_t dev);
115 static int agp_i965_chipset_flush_setup(device_t dev);
116
117 static void agp_i915_chipset_flush_teardown(device_t dev);
118 static void agp_i965_chipset_flush_teardown(device_t dev);
119
120 static void agp_i915_chipset_flush(device_t dev);
121
122 enum {
123         CHIP_I810,      /* i810/i815 */
124         CHIP_I830,      /* 830M/845G */
125         CHIP_I855,      /* 852GM/855GM/865G */
126         CHIP_I915,      /* 915G/915GM */
127         CHIP_I965,      /* G965 */
128         CHIP_G33,       /* G33/Q33/Q35 */
129         CHIP_IGD,       /* Pineview */
130         CHIP_G4X,       /* G45/Q45 */
131 };
132
133 /* The i810 through i855 have the registers at BAR 1, and the GATT gets
134  * allocated by us.  The i915 has registers in BAR 0 and the GATT is at the
135  * start of the stolen memory, and should only be accessed by the OS through
136  * BAR 3.  The G965 has registers and GATT in the same BAR (0) -- first 512KB
137  * is registers, second 512KB is GATT.
138  */
139 static struct resource_spec agp_i915_res_spec[] = {
140         { SYS_RES_MEMORY, AGP_I915_MMADR, RF_ACTIVE | RF_SHAREABLE },
141         { SYS_RES_MEMORY, AGP_I915_GTTADR, RF_ACTIVE | RF_SHAREABLE },
142         { -1, 0 }
143 };
144
145 static struct resource_spec agp_i965_res_spec[] = {
146         { SYS_RES_MEMORY, AGP_I965_GTTMMADR, RF_ACTIVE | RF_SHAREABLE },
147         { -1, 0 }
148 };
149
150 struct agp_i810_softc {
151         struct agp_softc agp;
152         u_int32_t initial_aperture;     /* aperture size at startup */
153         struct agp_gatt *gatt;
154         u_int32_t dcache_size;          /* i810 only */
155         u_int32_t stolen;               /* number of i830/845 gtt
156                                            entries for stolen memory */
157         u_int stolen_size;              /* BIOS-reserved graphics memory */
158         u_int gtt_total_entries;        /* Total number of gtt ptes */
159         u_int gtt_mappable_entries;     /* Number of gtt ptes mappable by CPU */
160         device_t bdev;                  /* bridge device */
161         void *argb_cursor;              /* contigmalloc area for ARGB cursor */
162         struct resource *sc_res[3];
163         const struct agp_i810_match *match;
164         int sc_flush_page_rid;
165         struct resource *sc_flush_page_res;
166         void *sc_flush_page_vaddr;
167         int sc_bios_allocated_flush_page;
168 };
169
170 static device_t intel_agp;
171
172 struct agp_i810_driver {
173         int chiptype;
174         int gen;
175         int busdma_addr_mask_sz;
176         struct resource_spec *res_spec;
177         int (*check_active)(device_t);
178         void (*set_desc)(device_t, const struct agp_i810_match *);
179         void (*dump_regs)(device_t);
180         int (*get_stolen_size)(device_t);
181         int (*get_gtt_total_entries)(device_t);
182         int (*get_gtt_mappable_entries)(device_t);
183         int (*install_gatt)(device_t);
184         void (*deinstall_gatt)(device_t);
185         void (*write_gtt)(device_t, u_int, uint32_t);
186         void (*install_gtt_pte)(device_t, u_int, vm_offset_t, int);
187         void (*sync_gtt_pte)(device_t, u_int);
188         int (*set_aperture)(device_t, u_int32_t);
189         int (*chipset_flush_setup)(device_t);
190         void (*chipset_flush_teardown)(device_t);
191         void (*chipset_flush)(device_t);
192 };
193
194 static struct {
195         struct intel_gtt base;
196 } intel_private;
197
198 static const struct agp_i810_driver agp_i810_i915_driver = {
199         .chiptype = CHIP_I915,
200         .gen = 3,
201         .busdma_addr_mask_sz = 32,
202         .res_spec = agp_i915_res_spec,
203         .check_active = agp_i915_check_active,
204         .set_desc = agp_i810_set_desc,
205         .dump_regs = agp_i915_dump_regs,
206         .get_stolen_size = agp_i915_get_stolen_size,
207         .get_gtt_mappable_entries = agp_i915_get_gtt_mappable_entries,
208         .get_gtt_total_entries = agp_i810_get_gtt_total_entries,
209         .install_gatt = agp_i830_install_gatt,
210         .deinstall_gatt = agp_i830_deinstall_gatt,
211         .write_gtt = agp_i915_write_gtt,
212         .install_gtt_pte = agp_i915_install_gtt_pte,
213         .sync_gtt_pte = agp_i915_sync_gtt_pte,
214         .set_aperture = agp_i915_set_aperture,
215         .chipset_flush_setup = agp_i915_chipset_flush_setup,
216         .chipset_flush_teardown = agp_i915_chipset_flush_teardown,
217         .chipset_flush = agp_i915_chipset_flush,
218 };
219
220 static const struct agp_i810_driver agp_i810_g965_driver = {
221         .chiptype = CHIP_I965,
222         .gen = 4,
223         .busdma_addr_mask_sz = 36,
224         .res_spec = agp_i965_res_spec,
225         .check_active = agp_i915_check_active,
226         .set_desc = agp_i810_set_desc,
227         .dump_regs = agp_i965_dump_regs,
228         .get_stolen_size = agp_i915_get_stolen_size,
229         .get_gtt_mappable_entries = agp_i915_get_gtt_mappable_entries,
230         .get_gtt_total_entries = agp_i965_get_gtt_total_entries,
231         .install_gatt = agp_i830_install_gatt,
232         .deinstall_gatt = agp_i830_deinstall_gatt,
233         .write_gtt = agp_i965_write_gtt,
234         .install_gtt_pte = agp_i965_install_gtt_pte,
235         .sync_gtt_pte = agp_i965_sync_gtt_pte,
236         .set_aperture = agp_i915_set_aperture,
237         .chipset_flush_setup = agp_i965_chipset_flush_setup,
238         .chipset_flush_teardown = agp_i965_chipset_flush_teardown,
239         .chipset_flush = agp_i915_chipset_flush,
240 };
241
242 static const struct agp_i810_driver agp_i810_g33_driver = {
243         .chiptype = CHIP_G33,
244         .gen = 3,
245         .busdma_addr_mask_sz = 36,
246         .res_spec = agp_i915_res_spec,
247         .check_active = agp_i915_check_active,
248         .set_desc = agp_i810_set_desc,
249         .dump_regs = agp_i965_dump_regs,
250         .get_stolen_size = agp_i915_get_stolen_size,
251         .get_gtt_mappable_entries = agp_i915_get_gtt_mappable_entries,
252         .get_gtt_total_entries = agp_i965_get_gtt_total_entries,
253         .install_gatt = agp_i830_install_gatt,
254         .deinstall_gatt = agp_i830_deinstall_gatt,
255         .write_gtt = agp_i915_write_gtt,
256         .install_gtt_pte = agp_i965_install_gtt_pte,
257         .sync_gtt_pte = agp_i915_sync_gtt_pte,
258         .set_aperture = agp_i915_set_aperture,
259         .chipset_flush_setup = agp_i965_chipset_flush_setup,
260         .chipset_flush_teardown = agp_i965_chipset_flush_teardown,
261         .chipset_flush = agp_i915_chipset_flush,
262 };
263
264 static const struct agp_i810_driver pineview_gtt_driver = {
265         .chiptype = CHIP_IGD,
266         .gen = 3,
267         .busdma_addr_mask_sz = 36,
268         .res_spec = agp_i915_res_spec,
269         .check_active = agp_i915_check_active,
270         .set_desc = agp_i810_set_desc,
271         .dump_regs = agp_i915_dump_regs,
272         .get_stolen_size = agp_i915_get_stolen_size,
273         .get_gtt_mappable_entries = agp_i915_get_gtt_mappable_entries,
274         .get_gtt_total_entries = agp_i965_get_gtt_total_entries,
275         .install_gatt = agp_i830_install_gatt,
276         .deinstall_gatt = agp_i830_deinstall_gatt,
277         .write_gtt = agp_i915_write_gtt,
278         .install_gtt_pte = agp_i965_install_gtt_pte,
279         .sync_gtt_pte = agp_i915_sync_gtt_pte,
280         .set_aperture = agp_i915_set_aperture,
281         .chipset_flush_setup = agp_i965_chipset_flush_setup,
282         .chipset_flush_teardown = agp_i965_chipset_flush_teardown,
283         .chipset_flush = agp_i915_chipset_flush,
284 };
285
286 static const struct agp_i810_driver agp_i810_g4x_driver = {
287         .chiptype = CHIP_G4X,
288         .gen = 5,
289         .busdma_addr_mask_sz = 36,
290         .res_spec = agp_i965_res_spec,
291         .check_active = agp_i915_check_active,
292         .set_desc = agp_i810_set_desc,
293         .dump_regs = agp_i965_dump_regs,
294         .get_stolen_size = agp_i915_get_stolen_size,
295         .get_gtt_mappable_entries = agp_i915_get_gtt_mappable_entries,
296         .get_gtt_total_entries = agp_gen5_get_gtt_total_entries,
297         .install_gatt = agp_i830_install_gatt,
298         .deinstall_gatt = agp_i830_deinstall_gatt,
299         .write_gtt = agp_g4x_write_gtt,
300         .install_gtt_pte = agp_g4x_install_gtt_pte,
301         .sync_gtt_pte = agp_g4x_sync_gtt_pte,
302         .set_aperture = agp_i915_set_aperture,
303         .chipset_flush_setup = agp_i965_chipset_flush_setup,
304         .chipset_flush_teardown = agp_i965_chipset_flush_teardown,
305         .chipset_flush = agp_i915_chipset_flush,
306 };
307
308 /* For adding new devices, devid is the id of the graphics controller
309  * (pci:0:2:0, for example).  The placeholder (usually at pci:0:2:1) for the
310  * second head should never be added.  The bridge_offset is the offset to
311  * subtract from devid to get the id of the hostb that the device is on.
312  */
313 static const struct agp_i810_match {
314         uint16_t devid;
315         char *name;
316         const struct agp_i810_driver *driver;
317 } agp_i810_matches[] = {
318         {
319                 .devid = 0x2582,
320                 .name = "Intel 82915G (915G GMCH) SVGA controller",
321                 .driver = &agp_i810_i915_driver
322         },
323         {
324                 .devid = 0x258A,
325                 .name = "Intel E7221 SVGA controller",
326                 .driver = &agp_i810_i915_driver
327         },
328         {
329                 .devid = 0x2592,
330                 .name = "Intel 82915GM (915GM GMCH) SVGA controller",
331                 .driver = &agp_i810_i915_driver
332         },
333         {
334                 .devid = 0x2772,
335                 .name = "Intel 82945G (945G GMCH) SVGA controller",
336                 .driver = &agp_i810_i915_driver
337         },
338         {
339                 .devid = 0x27A2,
340                 .name = "Intel 82945GM (945GM GMCH) SVGA controller",
341                 .driver = &agp_i810_i915_driver
342         },
343         {
344                 .devid = 0x27AE,
345                 .name = "Intel 945GME SVGA controller",
346                 .driver = &agp_i810_i915_driver
347         },
348         {
349                 .devid = 0x2972,
350                 .name = "Intel 946GZ SVGA controller",
351                 .driver = &agp_i810_g965_driver
352         },
353         {
354                 .devid = 0x2982,
355                 .name = "Intel G965 SVGA controller",
356                 .driver = &agp_i810_g965_driver
357         },
358         {
359                 .devid = 0x2992,
360                 .name = "Intel Q965 SVGA controller",
361                 .driver = &agp_i810_g965_driver
362         },
363         {
364                 .devid = 0x29A2,
365                 .name = "Intel G965 SVGA controller",
366                 .driver = &agp_i810_g965_driver
367         },
368         {
369                 .devid = 0x29B2,
370                 .name = "Intel Q35 SVGA controller",
371                 .driver = &agp_i810_g33_driver
372         },
373         {
374                 .devid = 0x29C2,
375                 .name = "Intel G33 SVGA controller",
376                 .driver = &agp_i810_g33_driver
377         },
378         {
379                 .devid = 0x29D2,
380                 .name = "Intel Q33 SVGA controller",
381                 .driver = &agp_i810_g33_driver
382         },
383         {
384                 .devid = 0xA001,
385                 .name = "Intel Pineview SVGA controller",
386                 .driver = &pineview_gtt_driver
387         },
388         {
389                 .devid = 0xA011,
390                 .name = "Intel Pineview (M) SVGA controller",
391                 .driver = &pineview_gtt_driver
392         },
393         {
394                 .devid = 0x2A02,
395                 .name = "Intel GM965 SVGA controller",
396                 .driver = &agp_i810_g965_driver
397         },
398         {
399                 .devid = 0x2A12,
400                 .name = "Intel GME965 SVGA controller",
401                 .driver = &agp_i810_g965_driver
402         },
403         {
404                 .devid = 0x2A42,
405                 .name = "Intel GM45 SVGA controller",
406                 .driver = &agp_i810_g4x_driver
407         },
408         {
409                 .devid = 0x2E02,
410                 .name = "Intel Eaglelake SVGA controller",
411                 .driver = &agp_i810_g4x_driver
412         },
413         {
414                 .devid = 0x2E12,
415                 .name = "Intel Q45 SVGA controller",
416                 .driver = &agp_i810_g4x_driver
417         },
418         {
419                 .devid = 0x2E22,
420                 .name = "Intel G45 SVGA controller",
421                 .driver = &agp_i810_g4x_driver
422         },
423         {
424                 .devid = 0x2E32,
425                 .name = "Intel G41 SVGA controller",
426                 .driver = &agp_i810_g4x_driver
427         },
428         {
429                 .devid = 0x0042,
430                 .name = "Intel Ironlake (D) SVGA controller",
431                 .driver = &agp_i810_g4x_driver
432         },
433         {
434                 .devid = 0x0046,
435                 .name = "Intel Ironlake (M) SVGA controller",
436                 .driver = &agp_i810_g4x_driver
437         },
438
439         {
440                 .devid = 0,
441         }
442 };
443
444 static const struct agp_i810_match*
445 agp_i810_match(device_t dev)
446 {
447         int i, devid;
448
449         if (pci_get_vendor(dev) != PCI_VENDOR_INTEL)
450                 return (NULL);
451
452         devid = pci_get_device(dev);
453         for (i = 0; agp_i810_matches[i].devid != 0; i++) {
454                 if (agp_i810_matches[i].devid == devid)
455                         break;
456         }
457         if (agp_i810_matches[i].devid == 0)
458                 return (NULL);
459         else
460                 return (&agp_i810_matches[i]);
461 }
462
463 /*
464  * Find bridge device.
465  */
466 static device_t
467 agp_i810_find_bridge(device_t dev)
468 {
469
470         return (pci_find_dbsf(0, 0, 0, 0));
471 }
472
473 static void
474 agp_i810_identify(driver_t *driver, device_t parent)
475 {
476
477         if (device_find_child(parent, "agp", -1) == NULL &&
478             agp_i810_match(parent))
479                 device_add_child(parent, "agp", -1);
480 }
481
482 static int
483 agp_i915_check_active(device_t bridge_dev)
484 {
485         int deven;
486
487         deven = pci_read_config(bridge_dev, AGP_I915_DEVEN, 4);
488         if ((deven & AGP_I915_DEVEN_D2F0) == AGP_I915_DEVEN_D2F0_DISABLED)
489                 return (ENXIO);
490         return (0);
491 }
492
493 static void
494 agp_i810_set_desc(device_t dev, const struct agp_i810_match *match)
495 {
496
497         device_set_desc(dev, match->name);
498 }
499
500 static int
501 agp_i810_probe(device_t dev)
502 {
503         device_t bdev;
504         const struct agp_i810_match *match;
505         int err = 0;
506
507         if (resource_disabled("agp", device_get_unit(dev)))
508                 return (ENXIO);
509         match = agp_i810_match(dev);
510         if (match == NULL)
511                 return (ENXIO);
512
513         bdev = agp_i810_find_bridge(dev);
514         if (bdev == NULL) {
515                 if (bootverbose)
516                         kprintf("I810: can't find bridge device\n");
517                 return (ENXIO);
518         }
519
520         /*
521          * checking whether internal graphics device has been activated.
522          */
523         if (match->driver->check_active != NULL) {
524                 err = match->driver->check_active(bdev);
525                 if (err != 0) {
526                         if (bootverbose)
527                                 kprintf("i810: disabled, not probing\n");
528                         return (err);
529                 }
530         }
531
532         match->driver->set_desc(dev, match);
533         return (BUS_PROBE_DEFAULT);
534 }
535
536 static void
537 agp_i915_dump_regs(device_t dev)
538 {
539         struct agp_i810_softc *sc = device_get_softc(dev);
540
541         device_printf(dev, "AGP_I810_PGTBL_CTL: %08x\n",
542             bus_read_4(sc->sc_res[0], AGP_I810_PGTBL_CTL));
543         device_printf(dev, "AGP_I855_GCC1: 0x%02x\n",
544             pci_read_config(sc->bdev, AGP_I855_GCC1, 1));
545         device_printf(dev, "AGP_I915_MSAC: 0x%02x\n",
546             pci_read_config(sc->bdev, AGP_I915_MSAC, 1));
547 }
548
549 static void
550 agp_i965_dump_regs(device_t dev)
551 {
552         struct agp_i810_softc *sc = device_get_softc(dev);
553
554         device_printf(dev, "AGP_I965_PGTBL_CTL2: %08x\n",
555             bus_read_4(sc->sc_res[0], AGP_I965_PGTBL_CTL2));
556         device_printf(dev, "AGP_I855_GCC1: 0x%02x\n",
557             pci_read_config(dev, AGP_I855_GCC1, 1));
558         device_printf(dev, "AGP_I965_MSAC: 0x%02x\n",
559             pci_read_config(dev, AGP_I965_MSAC, 1));
560 }
561
562 static int
563 agp_i915_get_stolen_size(device_t dev)
564 {
565         struct agp_i810_softc *sc;
566         unsigned int gcc1, stolen, gtt_size;
567
568         sc = device_get_softc(dev);
569
570         /*
571          * Stolen memory is set up at the beginning of the aperture by
572          * the BIOS, consisting of the GATT followed by 4kb for the
573          * BIOS display.
574          */
575         switch (sc->match->driver->chiptype) {
576         case CHIP_I855:
577                 gtt_size = 128;
578                 break;
579         case CHIP_I915:
580                 gtt_size = 256;
581                 break;
582         case CHIP_I965:
583                 switch (bus_read_4(sc->sc_res[0], AGP_I810_PGTBL_CTL) &
584                         AGP_I810_PGTBL_SIZE_MASK) {
585                 case AGP_I810_PGTBL_SIZE_128KB:
586                         gtt_size = 128;
587                         break;
588                 case AGP_I810_PGTBL_SIZE_256KB:
589                         gtt_size = 256;
590                         break;
591                 case AGP_I810_PGTBL_SIZE_512KB:
592                         gtt_size = 512;
593                         break;
594                 case AGP_I965_PGTBL_SIZE_1MB:
595                         gtt_size = 1024;
596                         break;
597                 case AGP_I965_PGTBL_SIZE_2MB:
598                         gtt_size = 2048;
599                         break;
600                 case AGP_I965_PGTBL_SIZE_1_5MB:
601                         gtt_size = 1024 + 512;
602                         break;
603                 default:
604                         device_printf(dev, "Bad PGTBL size\n");
605                         return (EINVAL);
606                 }
607                 break;
608         case CHIP_G33:
609                 gcc1 = pci_read_config(sc->bdev, AGP_I855_GCC1, 2);
610                 switch (gcc1 & AGP_G33_MGGC_GGMS_MASK) {
611                 case AGP_G33_MGGC_GGMS_SIZE_1M:
612                         gtt_size = 1024;
613                         break;
614                 case AGP_G33_MGGC_GGMS_SIZE_2M:
615                         gtt_size = 2048;
616                         break;
617                 default:
618                         device_printf(dev, "Bad PGTBL size\n");
619                         return (EINVAL);
620                 }
621                 break;
622         case CHIP_IGD:
623         case CHIP_G4X:
624                 gtt_size = 0;
625                 break;
626         default:
627                 device_printf(dev, "Bad chiptype\n");
628                 return (EINVAL);
629         }
630
631         /* GCC1 is called MGGC on i915+ */
632         gcc1 = pci_read_config(sc->bdev, AGP_I855_GCC1, 1);
633         switch (gcc1 & AGP_I855_GCC1_GMS) {
634         case AGP_I855_GCC1_GMS_STOLEN_1M:
635                 stolen = 1024;
636                 break;
637         case AGP_I855_GCC1_GMS_STOLEN_4M:
638                 stolen = 4 * 1024;
639                 break;
640         case AGP_I855_GCC1_GMS_STOLEN_8M:
641                 stolen = 8 * 1024;
642                 break;
643         case AGP_I855_GCC1_GMS_STOLEN_16M:
644                 stolen = 16 * 1024;
645                 break;
646         case AGP_I855_GCC1_GMS_STOLEN_32M:
647                 stolen = 32 * 1024;
648                 break;
649         case AGP_I915_GCC1_GMS_STOLEN_48M:
650                 stolen = sc->match->driver->gen > 2 ? 48 * 1024 : 0;
651                 break;
652         case AGP_I915_GCC1_GMS_STOLEN_64M:
653                 stolen = sc->match->driver->gen > 2 ? 64 * 1024 : 0;
654                 break;
655         case AGP_G33_GCC1_GMS_STOLEN_128M:
656                 stolen = sc->match->driver->gen > 2 ? 128 * 1024 : 0;
657                 break;
658         case AGP_G33_GCC1_GMS_STOLEN_256M:
659                 stolen = sc->match->driver->gen > 2 ? 256 * 1024 : 0;
660                 break;
661         case AGP_G4X_GCC1_GMS_STOLEN_96M:
662                 if (sc->match->driver->chiptype == CHIP_I965 ||
663                     sc->match->driver->chiptype == CHIP_G4X)
664                         stolen = 96 * 1024;
665                 else
666                         stolen = 0;
667                 break;
668         case AGP_G4X_GCC1_GMS_STOLEN_160M:
669                 if (sc->match->driver->chiptype == CHIP_I965 ||
670                     sc->match->driver->chiptype == CHIP_G4X)
671                         stolen = 160 * 1024;
672                 else
673                         stolen = 0;
674                 break;
675         case AGP_G4X_GCC1_GMS_STOLEN_224M:
676                 if (sc->match->driver->chiptype == CHIP_I965 ||
677                     sc->match->driver->chiptype == CHIP_G4X)
678                         stolen = 224 * 1024;
679                 else
680                         stolen = 0;
681                 break;
682         case AGP_G4X_GCC1_GMS_STOLEN_352M:
683                 if (sc->match->driver->chiptype == CHIP_I965 ||
684                     sc->match->driver->chiptype == CHIP_G4X)
685                         stolen = 352 * 1024;
686                 else
687                         stolen = 0;
688                 break;
689         default:
690                 device_printf(dev,
691                     "unknown memory configuration, disabling (GCC1 %x)\n",
692                     gcc1);
693                 return (EINVAL);
694         }
695
696         gtt_size += 4;
697         sc->stolen_size = stolen * 1024;
698         sc->stolen = (stolen - gtt_size) * 1024 / 4096;
699
700         return (0);
701 }
702
703 static int
704 agp_i915_get_gtt_mappable_entries(device_t dev)
705 {
706         struct agp_i810_softc *sc;
707         uint32_t ap;
708
709         sc = device_get_softc(dev);
710         ap = AGP_GET_APERTURE(dev);
711         sc->gtt_mappable_entries = ap >> AGP_PAGE_SHIFT;
712         return (0);
713 }
714
715 static int
716 agp_i810_get_gtt_total_entries(device_t dev)
717 {
718         struct agp_i810_softc *sc;
719
720         sc = device_get_softc(dev);
721         sc->gtt_total_entries = sc->gtt_mappable_entries;
722         return (0);
723 }
724
725 static int
726 agp_i965_get_gtt_total_entries(device_t dev)
727 {
728         struct agp_i810_softc *sc;
729         uint32_t pgetbl_ctl;
730         int error;
731
732         sc = device_get_softc(dev);
733         error = 0;
734         pgetbl_ctl = bus_read_4(sc->sc_res[0], AGP_I810_PGTBL_CTL);
735         switch (pgetbl_ctl & AGP_I810_PGTBL_SIZE_MASK) {
736         case AGP_I810_PGTBL_SIZE_128KB:
737                 sc->gtt_total_entries = 128 * 1024 / 4;
738                 break;
739         case AGP_I810_PGTBL_SIZE_256KB:
740                 sc->gtt_total_entries = 256 * 1024 / 4;
741                 break;
742         case AGP_I810_PGTBL_SIZE_512KB:
743                 sc->gtt_total_entries = 512 * 1024 / 4;
744                 break;
745         /* GTT pagetable sizes bigger than 512KB are not possible on G33! */
746         case AGP_I810_PGTBL_SIZE_1MB:
747                 sc->gtt_total_entries = 1024 * 1024 / 4;
748                 break;
749         case AGP_I810_PGTBL_SIZE_2MB:
750                 sc->gtt_total_entries = 2 * 1024 * 1024 / 4;
751                 break;
752         case AGP_I810_PGTBL_SIZE_1_5MB:
753                 sc->gtt_total_entries = (1024 + 512) * 1024 / 4;
754                 break;
755         default:
756                 device_printf(dev, "Unknown page table size\n");
757                 error = ENXIO;
758         }
759         return (error);
760 }
761
762 static void
763 agp_gen5_adjust_pgtbl_size(device_t dev, uint32_t sz)
764 {
765         struct agp_i810_softc *sc;
766         uint32_t pgetbl_ctl, pgetbl_ctl2;
767
768         sc = device_get_softc(dev);
769
770         /* Disable per-process page table. */
771         pgetbl_ctl2 = bus_read_4(sc->sc_res[0], AGP_I965_PGTBL_CTL2);
772         pgetbl_ctl2 &= ~AGP_I810_PGTBL_ENABLED;
773         bus_write_4(sc->sc_res[0], AGP_I965_PGTBL_CTL2, pgetbl_ctl2);
774
775         /* Write the new ggtt size. */
776         pgetbl_ctl = bus_read_4(sc->sc_res[0], AGP_I810_PGTBL_CTL);
777         pgetbl_ctl &= ~AGP_I810_PGTBL_SIZE_MASK;
778         pgetbl_ctl |= sz;
779         bus_write_4(sc->sc_res[0], AGP_I810_PGTBL_CTL, pgetbl_ctl);
780 }
781
782 static int
783 agp_gen5_get_gtt_total_entries(device_t dev)
784 {
785         struct agp_i810_softc *sc;
786         uint16_t gcc1;
787
788         sc = device_get_softc(dev);
789
790         gcc1 = pci_read_config(sc->bdev, AGP_I830_GCC1, 2);
791         switch (gcc1 & AGP_G4x_GCC1_SIZE_MASK) {
792         case AGP_G4x_GCC1_SIZE_1M:
793         case AGP_G4x_GCC1_SIZE_VT_1M:
794                 agp_gen5_adjust_pgtbl_size(dev, AGP_I810_PGTBL_SIZE_1MB);
795                 break;
796         case AGP_G4x_GCC1_SIZE_VT_1_5M:
797                 agp_gen5_adjust_pgtbl_size(dev, AGP_I810_PGTBL_SIZE_1_5MB);
798                 break;
799         case AGP_G4x_GCC1_SIZE_2M:
800         case AGP_G4x_GCC1_SIZE_VT_2M:
801                 agp_gen5_adjust_pgtbl_size(dev, AGP_I810_PGTBL_SIZE_2MB);
802                 break;
803         default:
804                 device_printf(dev, "Unknown page table size\n");
805                 return (ENXIO);
806         }
807
808         return (agp_i965_get_gtt_total_entries(dev));
809 }
810
811 static int
812 agp_i830_install_gatt(device_t dev)
813 {
814         struct agp_i810_softc *sc;
815         uint32_t pgtblctl;
816
817         sc = device_get_softc(dev);
818
819         /*
820          * The i830 automatically initializes the 128k gatt on boot.
821          * GATT address is already in there, make sure it's enabled.
822          */
823         pgtblctl = bus_read_4(sc->sc_res[0], AGP_I810_PGTBL_CTL);
824         pgtblctl |= 1;
825         bus_write_4(sc->sc_res[0], AGP_I810_PGTBL_CTL, pgtblctl);
826
827         sc->gatt->ag_physical = pgtblctl & ~1;
828         return (0);
829 }
830
831 static int
832 agp_i810_attach(device_t dev)
833 {
834         struct agp_i810_softc *sc;
835         int error;
836
837         sc = device_get_softc(dev);
838         sc->bdev = agp_i810_find_bridge(dev);
839         if (sc->bdev == NULL)
840                 return (ENOENT);
841
842         sc->match = agp_i810_match(dev);
843
844         agp_set_aperture_resource(dev, sc->match->driver->gen <= 2 ?
845             AGP_APBASE : AGP_I915_GMADR);
846         error = agp_generic_attach(dev);
847         if (error)
848                 return (error);
849
850         if (ptoa((vm_paddr_t)Maxmem) >
851             (1ULL << sc->match->driver->busdma_addr_mask_sz) - 1) {
852                 device_printf(dev, "agp_i810 does not support physical "
853                     "memory above %ju.\n", (uintmax_t)(1ULL <<
854                     sc->match->driver->busdma_addr_mask_sz) - 1);
855                 return (ENOENT);
856         }
857
858         if (bus_alloc_resources(dev, sc->match->driver->res_spec, sc->sc_res)) {
859                 agp_generic_detach(dev);
860                 return (ENODEV);
861         }
862
863         sc->initial_aperture = AGP_GET_APERTURE(dev);
864         sc->gatt = kmalloc(sizeof(struct agp_gatt), M_DRM, M_WAITOK);
865         sc->gatt->ag_entries = AGP_GET_APERTURE(dev) >> AGP_PAGE_SHIFT;
866
867         if ((error = sc->match->driver->get_stolen_size(dev)) != 0 ||
868             (error = sc->match->driver->install_gatt(dev)) != 0 ||
869             (error = sc->match->driver->get_gtt_mappable_entries(dev)) != 0 ||
870             (error = sc->match->driver->get_gtt_total_entries(dev)) != 0 ||
871             (error = sc->match->driver->chipset_flush_setup(dev)) != 0) {
872                 bus_release_resources(dev, sc->match->driver->res_spec,
873                     sc->sc_res);
874                 kfree(sc->gatt);
875                 agp_generic_detach(dev);
876                 return (error);
877         }
878
879         intel_agp = dev;
880         device_printf(dev, "aperture size is %dM",
881             sc->initial_aperture / 1024 / 1024);
882         if (sc->stolen > 0)
883                 kprintf(", detected %dk stolen memory\n", sc->stolen * 4);
884         else
885                 kprintf("\n");
886         if (bootverbose) {
887                 sc->match->driver->dump_regs(dev);
888                 device_printf(dev, "Mappable GTT entries: %d\n",
889                     sc->gtt_mappable_entries);
890                 device_printf(dev, "Total GTT entries: %d\n",
891                     sc->gtt_total_entries);
892         }
893         return (0);
894 }
895
896 static void
897 agp_i830_deinstall_gatt(device_t dev)
898 {
899         struct agp_i810_softc *sc;
900         unsigned int pgtblctl;
901
902         sc = device_get_softc(dev);
903         pgtblctl = bus_read_4(sc->sc_res[0], AGP_I810_PGTBL_CTL);
904         pgtblctl &= ~1;
905         bus_write_4(sc->sc_res[0], AGP_I810_PGTBL_CTL, pgtblctl);
906 }
907
908 static int
909 agp_i810_detach(device_t dev)
910 {
911         struct agp_i810_softc *sc;
912
913         sc = device_get_softc(dev);
914         agp_free_cdev(dev);
915
916         /* Clear the GATT base. */
917         sc->match->driver->deinstall_gatt(dev);
918
919         sc->match->driver->chipset_flush_teardown(dev);
920
921         /* Put the aperture back the way it started. */
922         AGP_SET_APERTURE(dev, sc->initial_aperture);
923
924         kfree(sc->gatt);
925         bus_release_resources(dev, sc->match->driver->res_spec, sc->sc_res);
926         agp_free_res(dev);
927
928         return (0);
929 }
930
931 static int
932 agp_i810_resume(device_t dev)
933 {
934         struct agp_i810_softc *sc;
935         sc = device_get_softc(dev);
936
937         AGP_SET_APERTURE(dev, sc->initial_aperture);
938
939         /* Install the GATT. */
940         bus_write_4(sc->sc_res[0], AGP_I810_PGTBL_CTL,
941         sc->gatt->ag_physical | 1);
942
943         return (bus_generic_resume(dev));
944 }
945
946 /**
947  * Sets the PCI resource size of the aperture on i830-class and below chipsets,
948  * while returning failure on later chipsets when an actual change is
949  * requested.
950  *
951  * This whole function is likely bogus, as the kernel would probably need to
952  * reconfigure the placement of the AGP aperture if a larger size is requested,
953  * which doesn't happen currently.
954  */
955
956 static int
957 agp_i915_set_aperture(device_t dev, u_int32_t aperture)
958 {
959
960         return (agp_generic_set_aperture(dev, aperture));
961 }
962
963 static int
964 agp_i810_method_set_aperture(device_t dev, u_int32_t aperture)
965 {
966         struct agp_i810_softc *sc;
967
968         sc = device_get_softc(dev);
969         return (sc->match->driver->set_aperture(dev, aperture));
970 }
971
972 /**
973  * Writes a GTT entry mapping the page at the given offset from the
974  * beginning of the aperture to the given physical address.  Setup the
975  * caching mode according to flags.
976  *
977  * For gen 1, 2 and 3, GTT start is located at AGP_I810_GTT offset
978  * from corresponding BAR start. For gen 4, offset is 512KB +
979  * AGP_I810_GTT, for gen 5 and 6 it is 2MB + AGP_I810_GTT.
980  *
981  * Also, the bits of the physical page address above 4GB needs to be
982  * placed into bits 40-32 of PTE.
983  */
984 static void
985 agp_i915_install_gtt_pte(device_t dev, u_int index, vm_offset_t physical,
986     int flags)
987 {
988         uint32_t pte;
989
990         pte = (u_int32_t)physical | I810_PTE_VALID;
991         if (flags == AGP_USER_CACHED_MEMORY)
992                 pte |= I830_PTE_SYSTEM_CACHED;
993
994         agp_i915_write_gtt(dev, index, pte);
995 }
996
997 static void
998 agp_i915_write_gtt(device_t dev, u_int index, uint32_t pte)
999 {
1000         struct agp_i810_softc *sc;
1001
1002         sc = device_get_softc(dev);
1003         bus_write_4(sc->sc_res[0], index * 4, pte);
1004 }
1005
1006 static void
1007 agp_i965_install_gtt_pte(device_t dev, u_int index, vm_offset_t physical,
1008     int flags)
1009 {
1010         uint32_t pte;
1011
1012         pte = (u_int32_t)physical | I810_PTE_VALID;
1013         if (flags == AGP_USER_CACHED_MEMORY)
1014                 pte |= I830_PTE_SYSTEM_CACHED;
1015
1016         pte |= (physical >> 28) & 0xf0;
1017         agp_i965_write_gtt(dev, index, pte);
1018 }
1019
1020 static void
1021 agp_i965_write_gtt(device_t dev, u_int index, uint32_t pte)
1022 {
1023         struct agp_i810_softc *sc;
1024
1025         sc = device_get_softc(dev);
1026         bus_write_4(sc->sc_res[0], index * 4 + (512 * 1024), pte);
1027 }
1028
1029 static void
1030 agp_g4x_install_gtt_pte(device_t dev, u_int index, vm_offset_t physical,
1031     int flags)
1032 {
1033         uint32_t pte;
1034
1035         pte = (u_int32_t)physical | I810_PTE_VALID;
1036         if (flags == AGP_USER_CACHED_MEMORY)
1037                 pte |= I830_PTE_SYSTEM_CACHED;
1038
1039         pte |= (physical >> 28) & 0xf0;
1040         agp_g4x_write_gtt(dev, index, pte);
1041 }
1042
1043 static void
1044 agp_g4x_write_gtt(device_t dev, u_int index, uint32_t pte)
1045 {
1046         struct agp_i810_softc *sc;
1047
1048         sc = device_get_softc(dev);
1049         bus_write_4(sc->sc_res[0], index * 4 + (2 * 1024 * 1024), pte);
1050 }
1051
1052 static int
1053 agp_i810_bind_page(device_t dev, vm_offset_t offset, vm_offset_t physical)
1054 {
1055         struct agp_i810_softc *sc = device_get_softc(dev);
1056         u_int index;
1057
1058         if (offset >= (sc->gatt->ag_entries << AGP_PAGE_SHIFT)) {
1059                 device_printf(dev, "failed: offset is 0x%08jx, "
1060                     "shift is %d, entries is %d\n", (intmax_t)offset,
1061                     AGP_PAGE_SHIFT, sc->gatt->ag_entries);
1062                 return (EINVAL);
1063         }
1064         index = offset >> AGP_PAGE_SHIFT;
1065         if (sc->stolen != 0 && index < sc->stolen) {
1066                 device_printf(dev, "trying to bind into stolen memory\n");
1067                 return (EINVAL);
1068         }
1069         sc->match->driver->install_gtt_pte(dev, index, physical, 0);
1070         return (0);
1071 }
1072
1073 static int
1074 agp_i810_unbind_page(device_t dev, vm_offset_t offset)
1075 {
1076         struct agp_i810_softc *sc;
1077         u_int index;
1078
1079         sc = device_get_softc(dev);
1080         if (offset >= (sc->gatt->ag_entries << AGP_PAGE_SHIFT))
1081                 return (EINVAL);
1082         index = offset >> AGP_PAGE_SHIFT;
1083         if (sc->stolen != 0 && index < sc->stolen) {
1084                 device_printf(dev, "trying to unbind from stolen memory\n");
1085                 return (EINVAL);
1086         }
1087         sc->match->driver->install_gtt_pte(dev, index, 0, 0);
1088         return (0);
1089 }
1090
1091 static void
1092 agp_i915_sync_gtt_pte(device_t dev, u_int index)
1093 {
1094         struct agp_i810_softc *sc;
1095
1096         sc = device_get_softc(dev);
1097         bus_read_4(sc->sc_res[1], index * 4);
1098 }
1099
1100 static void
1101 agp_i965_sync_gtt_pte(device_t dev, u_int index)
1102 {
1103         struct agp_i810_softc *sc;
1104
1105         sc = device_get_softc(dev);
1106         bus_read_4(sc->sc_res[0], index * 4 + (512 * 1024));
1107 }
1108
1109 static void
1110 agp_g4x_sync_gtt_pte(device_t dev, u_int index)
1111 {
1112         struct agp_i810_softc *sc;
1113
1114         sc = device_get_softc(dev);
1115         bus_read_4(sc->sc_res[0], index * 4 + (2 * 1024 * 1024));
1116 }
1117
1118 /*
1119  * Writing via memory mapped registers already flushes all TLBs.
1120  */
1121 static void
1122 agp_i810_flush_tlb(device_t dev)
1123 {
1124 }
1125
1126 static int
1127 agp_i810_enable(device_t dev, u_int32_t mode)
1128 {
1129
1130         return (0);
1131 }
1132
1133 static struct agp_memory *
1134 agp_i810_alloc_memory(device_t dev, int type, vm_size_t size)
1135 {
1136         struct agp_i810_softc *sc;
1137         struct agp_memory *mem;
1138         vm_page_t m;
1139
1140         sc = device_get_softc(dev);
1141
1142         if ((size & (AGP_PAGE_SIZE - 1)) != 0 ||
1143             sc->agp.as_allocated + size > sc->agp.as_maxmem)
1144                 return (0);
1145
1146         if (type == 1) {
1147                 /*
1148                  * Mapping local DRAM into GATT.
1149                  */
1150                 if (sc->match->driver->chiptype != CHIP_I810)
1151                         return (0);
1152                 if (size != sc->dcache_size)
1153                         return (0);
1154         } else if (type == 2) {
1155                 /*
1156                  * Type 2 is the contiguous physical memory type, that hands
1157                  * back a physical address.  This is used for cursors on i810.
1158                  * Hand back as many single pages with physical as the user
1159                  * wants, but only allow one larger allocation (ARGB cursor)
1160                  * for simplicity.
1161                  */
1162                 if (size != AGP_PAGE_SIZE) {
1163                         if (sc->argb_cursor != NULL)
1164                                 return (0);
1165
1166                         /* Allocate memory for ARGB cursor, if we can. */
1167                         sc->argb_cursor = contigmalloc(size, M_DRM,
1168                            0, 0, ~0, PAGE_SIZE, 0);
1169                         if (sc->argb_cursor == NULL)
1170                                 return (0);
1171                 }
1172         }
1173
1174         mem = kmalloc(sizeof *mem, M_DRM, M_INTWAIT);
1175         mem->am_id = sc->agp.as_nextid++;
1176         mem->am_size = size;
1177         mem->am_type = type;
1178         if (type != 1 && (type != 2 || size == AGP_PAGE_SIZE))
1179                 mem->am_obj = vm_object_allocate(OBJT_DEFAULT,
1180                     atop(round_page(size)));
1181         else
1182                 mem->am_obj = 0;
1183
1184         if (type == 2) {
1185                 if (size == AGP_PAGE_SIZE) {
1186                         /*
1187                          * Allocate and wire down the page now so that we can
1188                          * get its physical address.
1189                          */
1190                         VM_OBJECT_LOCK(mem->am_obj);
1191                         m = vm_page_grab(mem->am_obj, 0, VM_ALLOC_NORMAL |
1192                                                          VM_ALLOC_ZERO |
1193                                                          VM_ALLOC_RETRY);
1194                         vm_page_wire(m);
1195                         VM_OBJECT_UNLOCK(mem->am_obj);
1196                         mem->am_physical = VM_PAGE_TO_PHYS(m);
1197                         vm_page_wakeup(m);
1198                 } else {
1199                         /* Our allocation is already nicely wired down for us.
1200                          * Just grab the physical address.
1201                          */
1202                         mem->am_physical = vtophys(sc->argb_cursor);
1203                 }
1204         } else
1205                 mem->am_physical = 0;
1206
1207         mem->am_offset = 0;
1208         mem->am_is_bound = 0;
1209         TAILQ_INSERT_TAIL(&sc->agp.as_memory, mem, am_link);
1210         sc->agp.as_allocated += size;
1211
1212         return (mem);
1213 }
1214
1215 static int
1216 agp_i810_free_memory(device_t dev, struct agp_memory *mem)
1217 {
1218         struct agp_i810_softc *sc;
1219
1220         if (mem->am_is_bound)
1221                 return (EBUSY);
1222
1223         sc = device_get_softc(dev);
1224
1225         if (mem->am_type == 2) {
1226                 if (mem->am_size == AGP_PAGE_SIZE) {
1227                         /*
1228                          * Unwire the page which we wired in alloc_memory.
1229                          */
1230                         vm_page_t m;
1231
1232                         vm_object_hold(mem->am_obj);
1233                         m = vm_page_lookup_busy_wait(mem->am_obj, 0,
1234                                                      FALSE, "agppg");
1235                         vm_object_drop(mem->am_obj);
1236                         vm_page_unwire(m, 0);
1237                         vm_page_wakeup(m);
1238                 } else {
1239                         contigfree(sc->argb_cursor, mem->am_size, M_DRM);
1240                         sc->argb_cursor = NULL;
1241                 }
1242         }
1243
1244         sc->agp.as_allocated -= mem->am_size;
1245         TAILQ_REMOVE(&sc->agp.as_memory, mem, am_link);
1246         if (mem->am_obj)
1247                 vm_object_deallocate(mem->am_obj);
1248         kfree(mem);
1249         return (0);
1250 }
1251
1252 static int
1253 agp_i810_bind_memory(device_t dev, struct agp_memory *mem, vm_offset_t offset)
1254 {
1255         struct agp_i810_softc *sc;
1256         vm_offset_t i;
1257
1258         /* Do some sanity checks first. */
1259         if ((offset & (AGP_PAGE_SIZE - 1)) != 0 ||
1260             offset + mem->am_size > AGP_GET_APERTURE(dev)) {
1261                 device_printf(dev, "binding memory at bad offset %#x\n",
1262                     (int)offset);
1263                 return (EINVAL);
1264         }
1265
1266         sc = device_get_softc(dev);
1267         if (mem->am_type == 2 && mem->am_size != AGP_PAGE_SIZE) {
1268                 lockmgr(&sc->agp.as_lock, LK_EXCLUSIVE);
1269                 if (mem->am_is_bound) {
1270                         lockmgr(&sc->agp.as_lock, LK_RELEASE);
1271                         return EINVAL;
1272                 }
1273                 /* The memory's already wired down, just stick it in the GTT. */
1274                 for (i = 0; i < mem->am_size; i += AGP_PAGE_SIZE) {
1275                         sc->match->driver->install_gtt_pte(dev, (offset + i) >>
1276                             AGP_PAGE_SHIFT, mem->am_physical + i, 0);
1277                 }
1278                 agp_flush_cache();
1279                 mem->am_offset = offset;
1280                 mem->am_is_bound = 1;
1281                 lockmgr(&sc->agp.as_lock, LK_RELEASE);
1282                 return (0);
1283         }
1284
1285         if (mem->am_type != 1)
1286                 return (agp_generic_bind_memory(dev, mem, offset));
1287
1288         /*
1289          * Mapping local DRAM into GATT.
1290          */
1291         if (sc->match->driver->chiptype != CHIP_I810)
1292                 return (EINVAL);
1293         for (i = 0; i < mem->am_size; i += AGP_PAGE_SIZE)
1294                 bus_write_4(sc->sc_res[0],
1295                     AGP_I810_GTT + (i >> AGP_PAGE_SHIFT) * 4, i | 3);
1296
1297         return (0);
1298 }
1299
1300 static int
1301 agp_i810_unbind_memory(device_t dev, struct agp_memory *mem)
1302 {
1303         struct agp_i810_softc *sc;
1304         vm_offset_t i;
1305
1306         sc = device_get_softc(dev);
1307
1308         if (mem->am_type == 2 && mem->am_size != AGP_PAGE_SIZE) {
1309                 lockmgr(&sc->agp.as_lock, LK_EXCLUSIVE);
1310                 if (!mem->am_is_bound) {
1311                         lockmgr(&sc->agp.as_lock, LK_RELEASE);
1312                         return (EINVAL);
1313                 }
1314
1315                 for (i = 0; i < mem->am_size; i += AGP_PAGE_SIZE) {
1316                         sc->match->driver->install_gtt_pte(dev,
1317                             (mem->am_offset + i) >> AGP_PAGE_SHIFT, 0, 0);
1318                 }
1319                 agp_flush_cache();
1320                 mem->am_is_bound = 0;
1321                 lockmgr(&sc->agp.as_lock, LK_RELEASE);
1322                 return (0);
1323         }
1324
1325         if (mem->am_type != 1)
1326                 return (agp_generic_unbind_memory(dev, mem));
1327
1328         if (sc->match->driver->chiptype != CHIP_I810)
1329                 return (EINVAL);
1330         for (i = 0; i < mem->am_size; i += AGP_PAGE_SIZE) {
1331                 sc->match->driver->install_gtt_pte(dev, i >> AGP_PAGE_SHIFT,
1332                     0, 0);
1333         }
1334         return (0);
1335 }
1336
1337 static device_method_t agp_i810_methods[] = {
1338         /* Device interface */
1339         DEVMETHOD(device_identify,      agp_i810_identify),
1340         DEVMETHOD(device_probe,         agp_i810_probe),
1341         DEVMETHOD(device_attach,        agp_i810_attach),
1342         DEVMETHOD(device_detach,        agp_i810_detach),
1343         DEVMETHOD(device_suspend,       bus_generic_suspend),
1344         DEVMETHOD(device_resume,        agp_i810_resume),
1345
1346         /* AGP interface */
1347         DEVMETHOD(agp_get_aperture,     agp_generic_get_aperture),
1348         DEVMETHOD(agp_set_aperture,     agp_i810_method_set_aperture),
1349         DEVMETHOD(agp_bind_page,        agp_i810_bind_page),
1350         DEVMETHOD(agp_unbind_page,      agp_i810_unbind_page),
1351         DEVMETHOD(agp_flush_tlb,        agp_i810_flush_tlb),
1352         DEVMETHOD(agp_enable,           agp_i810_enable),
1353         DEVMETHOD(agp_alloc_memory,     agp_i810_alloc_memory),
1354         DEVMETHOD(agp_free_memory,      agp_i810_free_memory),
1355         DEVMETHOD(agp_bind_memory,      agp_i810_bind_memory),
1356         DEVMETHOD(agp_unbind_memory,    agp_i810_unbind_memory),
1357         DEVMETHOD(agp_chipset_flush,    agp_intel_gtt_chipset_flush),
1358
1359         DEVMETHOD_END
1360 };
1361
1362 static driver_t agp_i810_driver = {
1363         "agp",
1364         agp_i810_methods,
1365         sizeof(struct agp_i810_softc),
1366 };
1367
1368 static devclass_t agp_devclass;
1369
1370 DRIVER_MODULE(agp_i810, vgapci, agp_i810_driver, agp_devclass, NULL, NULL);
1371 MODULE_DEPEND(agp_i810, agp, 1, 1, 1);
1372 MODULE_DEPEND(agp_i810, pci, 1, 1, 1);
1373
1374 extern vm_page_t bogus_page;
1375
1376 void
1377 agp_intel_gtt_clear_range(device_t dev, u_int first_entry, u_int num_entries)
1378 {
1379         struct agp_i810_softc *sc;
1380         u_int i;
1381
1382         sc = device_get_softc(dev);
1383         for (i = 0; i < num_entries; i++)
1384                 sc->match->driver->install_gtt_pte(dev, first_entry + i,
1385                     VM_PAGE_TO_PHYS(bogus_page), 0);
1386         sc->match->driver->sync_gtt_pte(dev, first_entry + num_entries - 1);
1387 }
1388
1389 void
1390 agp_intel_gtt_insert_pages(device_t dev, u_int first_entry, u_int num_entries,
1391     vm_page_t *pages, u_int flags)
1392 {
1393         struct agp_i810_softc *sc;
1394         u_int i;
1395
1396         sc = device_get_softc(dev);
1397         for (i = 0; i < num_entries; i++) {
1398                 KKASSERT(pages[i]->valid == VM_PAGE_BITS_ALL);
1399                 KKASSERT(pages[i]->wire_count > 0);
1400                 sc->match->driver->install_gtt_pte(dev, first_entry + i,
1401                     VM_PAGE_TO_PHYS(pages[i]), flags);
1402         }
1403         sc->match->driver->sync_gtt_pte(dev, first_entry + num_entries - 1);
1404 }
1405
1406 void
1407 intel_gtt_insert_page(dma_addr_t addr, unsigned int pg, unsigned int flags)
1408 {
1409         struct agp_i810_softc *sc = device_get_softc(intel_agp);
1410
1411         sc->match->driver->install_gtt_pte(intel_agp, addr, pg, flags);
1412 }
1413
1414 void
1415 intel_gtt_insert_sg_entries(struct sg_table *st,
1416                             unsigned int pg_start,
1417                             unsigned int flags)
1418 {
1419         struct agp_i810_softc *sc = device_get_softc(intel_agp);
1420         struct scatterlist *sg;
1421         dma_addr_t page;
1422         int i, j, npages, subpage;
1423
1424         i = 0;
1425         for_each_sg(st->sgl, sg, st->nents, j) {
1426                 npages = sg_dma_len(sg) / PAGE_SIZE;
1427                 for (subpage = 0; subpage < npages; subpage++) {
1428                         page = sg_dma_address(sg) + subpage * PAGE_SIZE;
1429                         sc->match->driver->install_gtt_pte(intel_agp,
1430                                 pg_start + i, page, flags);
1431                         i++;
1432                 }
1433         }
1434         sc->match->driver->sync_gtt_pte(intel_agp, pg_start + i - 1);
1435 }
1436
1437
1438 struct intel_gtt
1439 agp_intel_gtt_get(device_t dev)
1440 {
1441         struct agp_i810_softc *sc;
1442         struct intel_gtt res;
1443
1444         sc = device_get_softc(dev);
1445         res.stolen_size = sc->stolen_size;
1446         res.gtt_total_entries = sc->gtt_total_entries;
1447         res.gtt_mappable_entries = sc->gtt_mappable_entries;
1448         res.do_idle_maps = 0;
1449         res.scratch_page_dma = VM_PAGE_TO_PHYS(bogus_page);
1450         return (res);
1451 }
1452
1453 static int
1454 agp_i915_chipset_flush_alloc_page(device_t dev, uint64_t start, uint64_t end)
1455 {
1456         struct agp_i810_softc *sc;
1457         device_t vga;
1458
1459         sc = device_get_softc(dev);
1460         vga = device_get_parent(dev);
1461         sc->sc_flush_page_rid = 100;
1462         sc->sc_flush_page_res = BUS_ALLOC_RESOURCE(device_get_parent(vga), dev,
1463             SYS_RES_MEMORY, &sc->sc_flush_page_rid, start, end, PAGE_SIZE,
1464             RF_ACTIVE, -1);
1465         if (sc->sc_flush_page_res == NULL) {
1466                 device_printf(dev, "Failed to allocate flush page at 0x%jx\n",
1467                     (uintmax_t)start);
1468                 return (EINVAL);
1469         }
1470         sc->sc_flush_page_vaddr = rman_get_virtual(sc->sc_flush_page_res);
1471         if (bootverbose) {
1472                 device_printf(dev, "Allocated flush page phys 0x%jx virt %p\n",
1473                     (uintmax_t)rman_get_start(sc->sc_flush_page_res),
1474                     sc->sc_flush_page_vaddr);
1475         }
1476         return (0);
1477 }
1478
1479 static void
1480 agp_i915_chipset_flush_free_page(device_t dev)
1481 {
1482         struct agp_i810_softc *sc;
1483         device_t vga;
1484
1485         sc = device_get_softc(dev);
1486         vga = device_get_parent(dev);
1487         if (sc->sc_flush_page_res == NULL)
1488                 return;
1489         BUS_DEACTIVATE_RESOURCE(device_get_parent(vga), dev, SYS_RES_MEMORY,
1490             sc->sc_flush_page_rid, sc->sc_flush_page_res);
1491         BUS_RELEASE_RESOURCE(device_get_parent(vga), dev, SYS_RES_MEMORY,
1492             sc->sc_flush_page_rid, sc->sc_flush_page_res);
1493 }
1494
1495 static int
1496 agp_i915_chipset_flush_setup(device_t dev)
1497 {
1498         struct agp_i810_softc *sc;
1499         uint32_t temp;
1500         int error;
1501
1502         sc = device_get_softc(dev);
1503         temp = pci_read_config(sc->bdev, AGP_I915_IFPADDR, 4);
1504         if ((temp & 1) != 0) {
1505                 temp &= ~1;
1506                 if (bootverbose)
1507                         device_printf(dev,
1508                             "Found already configured flush page at 0x%jx\n",
1509                             (uintmax_t)temp);
1510                 sc->sc_bios_allocated_flush_page = 1;
1511                 /*
1512                  * In the case BIOS initialized the flush pointer (?)
1513                  * register, expect that BIOS also set up the resource
1514                  * for the page.
1515                  */
1516                 error = agp_i915_chipset_flush_alloc_page(dev, temp,
1517                     temp + PAGE_SIZE - 1);
1518                 if (error != 0)
1519                         return (error);
1520         } else {
1521                 sc->sc_bios_allocated_flush_page = 0;
1522                 error = agp_i915_chipset_flush_alloc_page(dev, 0, 0xffffffff);
1523                 if (error != 0)
1524                         return (error);
1525                 temp = rman_get_start(sc->sc_flush_page_res);
1526                 pci_write_config(sc->bdev, AGP_I915_IFPADDR, temp | 1, 4);
1527         }
1528         return (0);
1529 }
1530
1531 static void
1532 agp_i915_chipset_flush_teardown(device_t dev)
1533 {
1534         struct agp_i810_softc *sc;
1535         uint32_t temp;
1536
1537         sc = device_get_softc(dev);
1538         if (sc->sc_flush_page_res == NULL)
1539                 return;
1540         if (!sc->sc_bios_allocated_flush_page) {
1541                 temp = pci_read_config(sc->bdev, AGP_I915_IFPADDR, 4);
1542                 temp &= ~1;
1543                 pci_write_config(sc->bdev, AGP_I915_IFPADDR, temp, 4);
1544         }
1545         agp_i915_chipset_flush_free_page(dev);
1546 }
1547
1548 static int
1549 agp_i965_chipset_flush_setup(device_t dev)
1550 {
1551         struct agp_i810_softc *sc;
1552         uint64_t temp;
1553         uint32_t temp_hi, temp_lo;
1554         int error;
1555
1556         sc = device_get_softc(dev);
1557
1558         temp_hi = pci_read_config(sc->bdev, AGP_I965_IFPADDR + 4, 4);
1559         temp_lo = pci_read_config(sc->bdev, AGP_I965_IFPADDR, 4);
1560
1561         if ((temp_lo & 1) != 0) {
1562                 temp = ((uint64_t)temp_hi << 32) | (temp_lo & ~1);
1563                 if (bootverbose)
1564                         device_printf(dev,
1565                             "Found already configured flush page at 0x%jx\n",
1566                             (uintmax_t)temp);
1567                 sc->sc_bios_allocated_flush_page = 1;
1568                 /*
1569                  * In the case BIOS initialized the flush pointer (?)
1570                  * register, expect that BIOS also set up the resource
1571                  * for the page.
1572                  */
1573                 error = agp_i915_chipset_flush_alloc_page(dev, temp,
1574                     temp + PAGE_SIZE - 1);
1575                 if (error != 0)
1576                         return (error);
1577         } else {
1578                 sc->sc_bios_allocated_flush_page = 0;
1579                 error = agp_i915_chipset_flush_alloc_page(dev, 0, ~0);
1580                 if (error != 0)
1581                         return (error);
1582                 temp = rman_get_start(sc->sc_flush_page_res);
1583                 pci_write_config(sc->bdev, AGP_I965_IFPADDR + 4,
1584                     (temp >> 32) & UINT32_MAX, 4);
1585                 pci_write_config(sc->bdev, AGP_I965_IFPADDR,
1586                     (temp & UINT32_MAX) | 1, 4);
1587         }
1588         return (0);
1589 }
1590
1591 static void
1592 agp_i965_chipset_flush_teardown(device_t dev)
1593 {
1594         struct agp_i810_softc *sc;
1595         uint32_t temp_lo;
1596
1597         sc = device_get_softc(dev);
1598         if (sc->sc_flush_page_res == NULL)
1599                 return;
1600         if (!sc->sc_bios_allocated_flush_page) {
1601                 temp_lo = pci_read_config(sc->bdev, AGP_I965_IFPADDR, 4);
1602                 temp_lo &= ~1;
1603                 pci_write_config(sc->bdev, AGP_I965_IFPADDR, temp_lo, 4);
1604         }
1605         agp_i915_chipset_flush_free_page(dev);
1606 }
1607
1608 static void
1609 agp_i915_chipset_flush(device_t dev)
1610 {
1611         struct agp_i810_softc *sc;
1612
1613         sc = device_get_softc(dev);
1614         *(uint32_t *)sc->sc_flush_page_vaddr = 1;
1615 }
1616
1617 int
1618 agp_intel_gtt_chipset_flush(device_t dev)
1619 {
1620         struct agp_i810_softc *sc;
1621
1622         sc = device_get_softc(dev);
1623         sc->match->driver->chipset_flush(dev);
1624         return (0);
1625 }
1626
1627 void
1628 intel_gtt_clear_range(u_int first_entry, u_int num_entries)
1629 {
1630
1631         agp_intel_gtt_clear_range(intel_agp, first_entry, num_entries);
1632 }
1633
1634 void intel_gtt_get(u64 *gtt_total, size_t *stolen_size,
1635                    phys_addr_t *mappable_base, u64 *mappable_end)
1636 {
1637         struct agp_info ainfo;
1638
1639         intel_private.base = agp_intel_gtt_get(intel_agp);
1640
1641         *gtt_total = intel_private.base.gtt_total_entries << PAGE_SHIFT;
1642         *stolen_size = intel_private.base.stolen_size;
1643         agp_get_info(intel_agp, &ainfo);
1644         *mappable_base = ainfo.ai_aperture_base;
1645         *mappable_end = intel_private.base.gtt_mappable_entries << PAGE_SHIFT;
1646 }
1647
1648 int
1649 intel_gtt_chipset_flush(void)
1650 {
1651
1652         return (agp_intel_gtt_chipset_flush(intel_agp));
1653 }
1654
1655 /*
1656  * Only used by gen6
1657  */
1658 void
1659 intel_gtt_sync_pte(u_int entry)
1660 {
1661         struct agp_i810_softc *sc;
1662
1663         sc = device_get_softc(intel_agp);
1664         sc->match->driver->sync_gtt_pte(intel_agp, entry);
1665 }
1666
1667 /*
1668  * Only used by gen6
1669  */
1670 void
1671 intel_gtt_write(u_int entry, uint32_t val)
1672 {
1673         struct agp_i810_softc *sc;
1674
1675         sc = device_get_softc(intel_agp);
1676         sc->match->driver->write_gtt(intel_agp, entry, val);
1677 }
1678
1679 #define GFX_FLSH_CNTL   0x2170
1680
1681 bool
1682 intel_enable_gtt(void)
1683 {
1684         struct agp_i810_softc *sc = device_get_softc(intel_agp);
1685
1686         /* Some chipsets such as Pineview can't report if the GTT
1687          * has been enabled or not.
1688          * Assume everything is fine. */
1689
1690         /* Flush all chipset write buffers nevertheless */
1691         bus_write_4(sc->sc_res[0], GFX_FLSH_CNTL, 1);
1692         bus_write_4(sc->sc_res[0], GFX_FLSH_CNTL, 0);
1693
1694         return true;
1695 }