drm/radeon: Sync to Linux 3.11
[dragonfly.git] / sys / dev / drm / radeon / ni.c
1 /*
2  * Copyright 2010 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Alex Deucher
23  * $FreeBSD: head/sys/dev/drm2/radeon/ni.c 254885 2013-08-25 19:37:15Z dumbbell $
24  */
25
26 #include <drm/drmP.h>
27 #include <linux/firmware.h>
28 #include "radeon.h"
29 #include "radeon_asic.h"
30 #include <uapi_drm/radeon_drm.h>
31 #include "nid.h"
32 #include "atom.h"
33 #include "ni_reg.h"
34 #include "cayman_blit_shaders.h"
35 #include "radeon_ucode.h"
36 #include "clearstate_cayman.h"
37
38 static u32 tn_rlc_save_restore_register_list[] =
39 {
40         0x98fc,
41         0x98f0,
42         0x9834,
43         0x9838,
44         0x9870,
45         0x9874,
46         0x8a14,
47         0x8b24,
48         0x8bcc,
49         0x8b10,
50         0x8c30,
51         0x8d00,
52         0x8d04,
53         0x8c00,
54         0x8c04,
55         0x8c10,
56         0x8c14,
57         0x8d8c,
58         0x8cf0,
59         0x8e38,
60         0x9508,
61         0x9688,
62         0x9608,
63         0x960c,
64         0x9610,
65         0x9614,
66         0x88c4,
67         0x8978,
68         0x88d4,
69         0x900c,
70         0x9100,
71         0x913c,
72         0x90e8,
73         0x9354,
74         0xa008,
75         0x98f8,
76         0x9148,
77         0x914c,
78         0x3f94,
79         0x98f4,
80         0x9b7c,
81         0x3f8c,
82         0x8950,
83         0x8954,
84         0x8a18,
85         0x8b28,
86         0x9144,
87         0x3f90,
88         0x915c,
89         0x9160,
90         0x9178,
91         0x917c,
92         0x9180,
93         0x918c,
94         0x9190,
95         0x9194,
96         0x9198,
97         0x919c,
98         0x91a8,
99         0x91ac,
100         0x91b0,
101         0x91b4,
102         0x91b8,
103         0x91c4,
104         0x91c8,
105         0x91cc,
106         0x91d0,
107         0x91d4,
108         0x91e0,
109         0x91e4,
110         0x91ec,
111         0x91f0,
112         0x91f4,
113         0x9200,
114         0x9204,
115         0x929c,
116         0x8030,
117         0x9150,
118         0x9a60,
119         0x920c,
120         0x9210,
121         0x9228,
122         0x922c,
123         0x9244,
124         0x9248,
125         0x91e8,
126         0x9294,
127         0x9208,
128         0x9224,
129         0x9240,
130         0x9220,
131         0x923c,
132         0x9258,
133         0x9744,
134         0xa200,
135         0xa204,
136         0xa208,
137         0xa20c,
138         0x8d58,
139         0x9030,
140         0x9034,
141         0x9038,
142         0x903c,
143         0x9040,
144         0x9654,
145         0x897c,
146         0xa210,
147         0xa214,
148         0x9868,
149         0xa02c,
150         0x9664,
151         0x9698,
152         0x949c,
153         0x8e10,
154         0x8e18,
155         0x8c50,
156         0x8c58,
157         0x8c60,
158         0x8c68,
159         0x89b4,
160         0x9830,
161         0x802c,
162 };
163 static u32 tn_rlc_save_restore_register_list_size = ARRAY_SIZE(tn_rlc_save_restore_register_list);
164
165 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
166 extern void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
167 extern void evergreen_program_aspm(struct radeon_device *rdev);
168
169 /* Firmware Names */
170 MODULE_FIRMWARE("radeon/BARTS_pfp.bin");
171 MODULE_FIRMWARE("radeon/BARTS_me.bin");
172 MODULE_FIRMWARE("radeon/BARTS_mc.bin");
173 MODULE_FIRMWARE("radeon/BARTS_smc.bin");
174 MODULE_FIRMWARE("radeon/BTC_rlc.bin");
175 MODULE_FIRMWARE("radeon/TURKS_pfp.bin");
176 MODULE_FIRMWARE("radeon/TURKS_me.bin");
177 MODULE_FIRMWARE("radeon/TURKS_mc.bin");
178 MODULE_FIRMWARE("radeon/TURKS_smc.bin");
179 MODULE_FIRMWARE("radeon/CAICOS_pfp.bin");
180 MODULE_FIRMWARE("radeon/CAICOS_me.bin");
181 MODULE_FIRMWARE("radeon/CAICOS_mc.bin");
182 MODULE_FIRMWARE("radeon/CAICOS_smc.bin");
183 MODULE_FIRMWARE("radeon/CAYMAN_pfp.bin");
184 MODULE_FIRMWARE("radeon/CAYMAN_me.bin");
185 MODULE_FIRMWARE("radeon/CAYMAN_mc.bin");
186 MODULE_FIRMWARE("radeon/CAYMAN_rlc.bin");
187 MODULE_FIRMWARE("radeon/CAYMAN_smc.bin");
188 MODULE_FIRMWARE("radeon/ARUBA_pfp.bin");
189 MODULE_FIRMWARE("radeon/ARUBA_me.bin");
190 MODULE_FIRMWARE("radeon/ARUBA_rlc.bin");
191
192 static const u32 cayman_golden_registers2[] =
193 {
194         0x3e5c, 0xffffffff, 0x00000000,
195         0x3e48, 0xffffffff, 0x00000000,
196         0x3e4c, 0xffffffff, 0x00000000,
197         0x3e64, 0xffffffff, 0x00000000,
198         0x3e50, 0xffffffff, 0x00000000,
199         0x3e60, 0xffffffff, 0x00000000
200 };
201
202 static const u32 cayman_golden_registers[] =
203 {
204         0x5eb4, 0xffffffff, 0x00000002,
205         0x5e78, 0x8f311ff1, 0x001000f0,
206         0x3f90, 0xffff0000, 0xff000000,
207         0x9148, 0xffff0000, 0xff000000,
208         0x3f94, 0xffff0000, 0xff000000,
209         0x914c, 0xffff0000, 0xff000000,
210         0xc78, 0x00000080, 0x00000080,
211         0xbd4, 0x70073777, 0x00011003,
212         0xd02c, 0xbfffff1f, 0x08421000,
213         0xd0b8, 0x73773777, 0x02011003,
214         0x5bc0, 0x00200000, 0x50100000,
215         0x98f8, 0x33773777, 0x02011003,
216         0x98fc, 0xffffffff, 0x76541032,
217         0x7030, 0x31000311, 0x00000011,
218         0x2f48, 0x33773777, 0x42010001,
219         0x6b28, 0x00000010, 0x00000012,
220         0x7728, 0x00000010, 0x00000012,
221         0x10328, 0x00000010, 0x00000012,
222         0x10f28, 0x00000010, 0x00000012,
223         0x11b28, 0x00000010, 0x00000012,
224         0x12728, 0x00000010, 0x00000012,
225         0x240c, 0x000007ff, 0x00000000,
226         0x8a14, 0xf000001f, 0x00000007,
227         0x8b24, 0x3fff3fff, 0x00ff0fff,
228         0x8b10, 0x0000ff0f, 0x00000000,
229         0x28a4c, 0x07ffffff, 0x06000000,
230         0x10c, 0x00000001, 0x00010003,
231         0xa02c, 0xffffffff, 0x0000009b,
232         0x913c, 0x0000010f, 0x01000100,
233         0x8c04, 0xf8ff00ff, 0x40600060,
234         0x28350, 0x00000f01, 0x00000000,
235         0x9508, 0x3700001f, 0x00000002,
236         0x960c, 0xffffffff, 0x54763210,
237         0x88c4, 0x001f3ae3, 0x00000082,
238         0x88d0, 0xffffffff, 0x0f40df40,
239         0x88d4, 0x0000001f, 0x00000010,
240         0x8974, 0xffffffff, 0x00000000
241 };
242
243 static const u32 dvst_golden_registers2[] =
244 {
245         0x8f8, 0xffffffff, 0,
246         0x8fc, 0x00380000, 0,
247         0x8f8, 0xffffffff, 1,
248         0x8fc, 0x0e000000, 0
249 };
250
251 static const u32 dvst_golden_registers[] =
252 {
253         0x690, 0x3fff3fff, 0x20c00033,
254         0x918c, 0x0fff0fff, 0x00010006,
255         0x91a8, 0x0fff0fff, 0x00010006,
256         0x9150, 0xffffdfff, 0x6e944040,
257         0x917c, 0x0fff0fff, 0x00030002,
258         0x9198, 0x0fff0fff, 0x00030002,
259         0x915c, 0x0fff0fff, 0x00010000,
260         0x3f90, 0xffff0001, 0xff000000,
261         0x9178, 0x0fff0fff, 0x00070000,
262         0x9194, 0x0fff0fff, 0x00070000,
263         0x9148, 0xffff0001, 0xff000000,
264         0x9190, 0x0fff0fff, 0x00090008,
265         0x91ac, 0x0fff0fff, 0x00090008,
266         0x3f94, 0xffff0000, 0xff000000,
267         0x914c, 0xffff0000, 0xff000000,
268         0x929c, 0x00000fff, 0x00000001,
269         0x55e4, 0xff607fff, 0xfc000100,
270         0x8a18, 0xff000fff, 0x00000100,
271         0x8b28, 0xff000fff, 0x00000100,
272         0x9144, 0xfffc0fff, 0x00000100,
273         0x6ed8, 0x00010101, 0x00010000,
274         0x9830, 0xffffffff, 0x00000000,
275         0x9834, 0xf00fffff, 0x00000400,
276         0x9838, 0xfffffffe, 0x00000000,
277         0xd0c0, 0xff000fff, 0x00000100,
278         0xd02c, 0xbfffff1f, 0x08421000,
279         0xd0b8, 0x73773777, 0x12010001,
280         0x5bb0, 0x000000f0, 0x00000070,
281         0x98f8, 0x73773777, 0x12010001,
282         0x98fc, 0xffffffff, 0x00000010,
283         0x9b7c, 0x00ff0000, 0x00fc0000,
284         0x8030, 0x00001f0f, 0x0000100a,
285         0x2f48, 0x73773777, 0x12010001,
286         0x2408, 0x00030000, 0x000c007f,
287         0x8a14, 0xf000003f, 0x00000007,
288         0x8b24, 0x3fff3fff, 0x00ff0fff,
289         0x8b10, 0x0000ff0f, 0x00000000,
290         0x28a4c, 0x07ffffff, 0x06000000,
291         0x4d8, 0x00000fff, 0x00000100,
292         0xa008, 0xffffffff, 0x00010000,
293         0x913c, 0xffff03ff, 0x01000100,
294         0x8c00, 0x000000ff, 0x00000003,
295         0x8c04, 0xf8ff00ff, 0x40600060,
296         0x8cf0, 0x1fff1fff, 0x08e00410,
297         0x28350, 0x00000f01, 0x00000000,
298         0x9508, 0xf700071f, 0x00000002,
299         0x960c, 0xffffffff, 0x54763210,
300         0x20ef8, 0x01ff01ff, 0x00000002,
301         0x20e98, 0xfffffbff, 0x00200000,
302         0x2015c, 0xffffffff, 0x00000f40,
303         0x88c4, 0x001f3ae3, 0x00000082,
304         0x8978, 0x3fffffff, 0x04050140,
305         0x88d4, 0x0000001f, 0x00000010,
306         0x8974, 0xffffffff, 0x00000000
307 };
308
309 static const u32 scrapper_golden_registers[] =
310 {
311         0x690, 0x3fff3fff, 0x20c00033,
312         0x918c, 0x0fff0fff, 0x00010006,
313         0x918c, 0x0fff0fff, 0x00010006,
314         0x91a8, 0x0fff0fff, 0x00010006,
315         0x91a8, 0x0fff0fff, 0x00010006,
316         0x9150, 0xffffdfff, 0x6e944040,
317         0x9150, 0xffffdfff, 0x6e944040,
318         0x917c, 0x0fff0fff, 0x00030002,
319         0x917c, 0x0fff0fff, 0x00030002,
320         0x9198, 0x0fff0fff, 0x00030002,
321         0x9198, 0x0fff0fff, 0x00030002,
322         0x915c, 0x0fff0fff, 0x00010000,
323         0x915c, 0x0fff0fff, 0x00010000,
324         0x3f90, 0xffff0001, 0xff000000,
325         0x3f90, 0xffff0001, 0xff000000,
326         0x9178, 0x0fff0fff, 0x00070000,
327         0x9178, 0x0fff0fff, 0x00070000,
328         0x9194, 0x0fff0fff, 0x00070000,
329         0x9194, 0x0fff0fff, 0x00070000,
330         0x9148, 0xffff0001, 0xff000000,
331         0x9148, 0xffff0001, 0xff000000,
332         0x9190, 0x0fff0fff, 0x00090008,
333         0x9190, 0x0fff0fff, 0x00090008,
334         0x91ac, 0x0fff0fff, 0x00090008,
335         0x91ac, 0x0fff0fff, 0x00090008,
336         0x3f94, 0xffff0000, 0xff000000,
337         0x3f94, 0xffff0000, 0xff000000,
338         0x914c, 0xffff0000, 0xff000000,
339         0x914c, 0xffff0000, 0xff000000,
340         0x929c, 0x00000fff, 0x00000001,
341         0x929c, 0x00000fff, 0x00000001,
342         0x55e4, 0xff607fff, 0xfc000100,
343         0x8a18, 0xff000fff, 0x00000100,
344         0x8a18, 0xff000fff, 0x00000100,
345         0x8b28, 0xff000fff, 0x00000100,
346         0x8b28, 0xff000fff, 0x00000100,
347         0x9144, 0xfffc0fff, 0x00000100,
348         0x9144, 0xfffc0fff, 0x00000100,
349         0x6ed8, 0x00010101, 0x00010000,
350         0x9830, 0xffffffff, 0x00000000,
351         0x9830, 0xffffffff, 0x00000000,
352         0x9834, 0xf00fffff, 0x00000400,
353         0x9834, 0xf00fffff, 0x00000400,
354         0x9838, 0xfffffffe, 0x00000000,
355         0x9838, 0xfffffffe, 0x00000000,
356         0xd0c0, 0xff000fff, 0x00000100,
357         0xd02c, 0xbfffff1f, 0x08421000,
358         0xd02c, 0xbfffff1f, 0x08421000,
359         0xd0b8, 0x73773777, 0x12010001,
360         0xd0b8, 0x73773777, 0x12010001,
361         0x5bb0, 0x000000f0, 0x00000070,
362         0x98f8, 0x73773777, 0x12010001,
363         0x98f8, 0x73773777, 0x12010001,
364         0x98fc, 0xffffffff, 0x00000010,
365         0x98fc, 0xffffffff, 0x00000010,
366         0x9b7c, 0x00ff0000, 0x00fc0000,
367         0x9b7c, 0x00ff0000, 0x00fc0000,
368         0x8030, 0x00001f0f, 0x0000100a,
369         0x8030, 0x00001f0f, 0x0000100a,
370         0x2f48, 0x73773777, 0x12010001,
371         0x2f48, 0x73773777, 0x12010001,
372         0x2408, 0x00030000, 0x000c007f,
373         0x8a14, 0xf000003f, 0x00000007,
374         0x8a14, 0xf000003f, 0x00000007,
375         0x8b24, 0x3fff3fff, 0x00ff0fff,
376         0x8b24, 0x3fff3fff, 0x00ff0fff,
377         0x8b10, 0x0000ff0f, 0x00000000,
378         0x8b10, 0x0000ff0f, 0x00000000,
379         0x28a4c, 0x07ffffff, 0x06000000,
380         0x28a4c, 0x07ffffff, 0x06000000,
381         0x4d8, 0x00000fff, 0x00000100,
382         0x4d8, 0x00000fff, 0x00000100,
383         0xa008, 0xffffffff, 0x00010000,
384         0xa008, 0xffffffff, 0x00010000,
385         0x913c, 0xffff03ff, 0x01000100,
386         0x913c, 0xffff03ff, 0x01000100,
387         0x90e8, 0x001fffff, 0x010400c0,
388         0x8c00, 0x000000ff, 0x00000003,
389         0x8c00, 0x000000ff, 0x00000003,
390         0x8c04, 0xf8ff00ff, 0x40600060,
391         0x8c04, 0xf8ff00ff, 0x40600060,
392         0x8c30, 0x0000000f, 0x00040005,
393         0x8cf0, 0x1fff1fff, 0x08e00410,
394         0x8cf0, 0x1fff1fff, 0x08e00410,
395         0x900c, 0x00ffffff, 0x0017071f,
396         0x28350, 0x00000f01, 0x00000000,
397         0x28350, 0x00000f01, 0x00000000,
398         0x9508, 0xf700071f, 0x00000002,
399         0x9508, 0xf700071f, 0x00000002,
400         0x9688, 0x00300000, 0x0017000f,
401         0x960c, 0xffffffff, 0x54763210,
402         0x960c, 0xffffffff, 0x54763210,
403         0x20ef8, 0x01ff01ff, 0x00000002,
404         0x20e98, 0xfffffbff, 0x00200000,
405         0x2015c, 0xffffffff, 0x00000f40,
406         0x88c4, 0x001f3ae3, 0x00000082,
407         0x88c4, 0x001f3ae3, 0x00000082,
408         0x8978, 0x3fffffff, 0x04050140,
409         0x8978, 0x3fffffff, 0x04050140,
410         0x88d4, 0x0000001f, 0x00000010,
411         0x88d4, 0x0000001f, 0x00000010,
412         0x8974, 0xffffffff, 0x00000000,
413         0x8974, 0xffffffff, 0x00000000
414 };
415
416 static void ni_init_golden_registers(struct radeon_device *rdev)
417 {
418         switch (rdev->family) {
419         case CHIP_CAYMAN:
420                 radeon_program_register_sequence(rdev,
421                                                  cayman_golden_registers,
422                                                  (const u32)ARRAY_SIZE(cayman_golden_registers));
423                 radeon_program_register_sequence(rdev,
424                                                  cayman_golden_registers2,
425                                                  (const u32)ARRAY_SIZE(cayman_golden_registers2));
426                 break;
427         case CHIP_ARUBA:
428                 if ((rdev->ddev->pci_device == 0x9900) ||
429                     (rdev->ddev->pci_device == 0x9901) ||
430                     (rdev->ddev->pci_device == 0x9903) ||
431                     (rdev->ddev->pci_device == 0x9904) ||
432                     (rdev->ddev->pci_device == 0x9905) ||
433                     (rdev->ddev->pci_device == 0x9906) ||
434                     (rdev->ddev->pci_device == 0x9907) ||
435                     (rdev->ddev->pci_device == 0x9908) ||
436                     (rdev->ddev->pci_device == 0x9909) ||
437                     (rdev->ddev->pci_device == 0x990A) ||
438                     (rdev->ddev->pci_device == 0x990B) ||
439                     (rdev->ddev->pci_device == 0x990C) ||
440                     (rdev->ddev->pci_device == 0x990D) ||
441                     (rdev->ddev->pci_device == 0x990E) ||
442                     (rdev->ddev->pci_device == 0x990F) ||
443                     (rdev->ddev->pci_device == 0x9910) ||
444                     (rdev->ddev->pci_device == 0x9913) ||
445                     (rdev->ddev->pci_device == 0x9917) ||
446                     (rdev->ddev->pci_device == 0x9918)) {
447                         radeon_program_register_sequence(rdev,
448                                                          dvst_golden_registers,
449                                                          (const u32)ARRAY_SIZE(dvst_golden_registers));
450                         radeon_program_register_sequence(rdev,
451                                                          dvst_golden_registers2,
452                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
453                 } else {
454                         radeon_program_register_sequence(rdev,
455                                                          scrapper_golden_registers,
456                                                          (const u32)ARRAY_SIZE(scrapper_golden_registers));
457                         radeon_program_register_sequence(rdev,
458                                                          dvst_golden_registers2,
459                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
460                 }
461                 break;
462         default:
463                 break;
464         }
465 }
466
467 #define BTC_IO_MC_REGS_SIZE 29
468
469 static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
470         {0x00000077, 0xff010100},
471         {0x00000078, 0x00000000},
472         {0x00000079, 0x00001434},
473         {0x0000007a, 0xcc08ec08},
474         {0x0000007b, 0x00040000},
475         {0x0000007c, 0x000080c0},
476         {0x0000007d, 0x09000000},
477         {0x0000007e, 0x00210404},
478         {0x00000081, 0x08a8e800},
479         {0x00000082, 0x00030444},
480         {0x00000083, 0x00000000},
481         {0x00000085, 0x00000001},
482         {0x00000086, 0x00000002},
483         {0x00000087, 0x48490000},
484         {0x00000088, 0x20244647},
485         {0x00000089, 0x00000005},
486         {0x0000008b, 0x66030000},
487         {0x0000008c, 0x00006603},
488         {0x0000008d, 0x00000100},
489         {0x0000008f, 0x00001c0a},
490         {0x00000090, 0xff000001},
491         {0x00000094, 0x00101101},
492         {0x00000095, 0x00000fff},
493         {0x00000096, 0x00116fff},
494         {0x00000097, 0x60010000},
495         {0x00000098, 0x10010000},
496         {0x00000099, 0x00006000},
497         {0x0000009a, 0x00001000},
498         {0x0000009f, 0x00946a00}
499 };
500
501 static const u32 turks_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
502         {0x00000077, 0xff010100},
503         {0x00000078, 0x00000000},
504         {0x00000079, 0x00001434},
505         {0x0000007a, 0xcc08ec08},
506         {0x0000007b, 0x00040000},
507         {0x0000007c, 0x000080c0},
508         {0x0000007d, 0x09000000},
509         {0x0000007e, 0x00210404},
510         {0x00000081, 0x08a8e800},
511         {0x00000082, 0x00030444},
512         {0x00000083, 0x00000000},
513         {0x00000085, 0x00000001},
514         {0x00000086, 0x00000002},
515         {0x00000087, 0x48490000},
516         {0x00000088, 0x20244647},
517         {0x00000089, 0x00000005},
518         {0x0000008b, 0x66030000},
519         {0x0000008c, 0x00006603},
520         {0x0000008d, 0x00000100},
521         {0x0000008f, 0x00001c0a},
522         {0x00000090, 0xff000001},
523         {0x00000094, 0x00101101},
524         {0x00000095, 0x00000fff},
525         {0x00000096, 0x00116fff},
526         {0x00000097, 0x60010000},
527         {0x00000098, 0x10010000},
528         {0x00000099, 0x00006000},
529         {0x0000009a, 0x00001000},
530         {0x0000009f, 0x00936a00}
531 };
532
533 static const u32 caicos_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
534         {0x00000077, 0xff010100},
535         {0x00000078, 0x00000000},
536         {0x00000079, 0x00001434},
537         {0x0000007a, 0xcc08ec08},
538         {0x0000007b, 0x00040000},
539         {0x0000007c, 0x000080c0},
540         {0x0000007d, 0x09000000},
541         {0x0000007e, 0x00210404},
542         {0x00000081, 0x08a8e800},
543         {0x00000082, 0x00030444},
544         {0x00000083, 0x00000000},
545         {0x00000085, 0x00000001},
546         {0x00000086, 0x00000002},
547         {0x00000087, 0x48490000},
548         {0x00000088, 0x20244647},
549         {0x00000089, 0x00000005},
550         {0x0000008b, 0x66030000},
551         {0x0000008c, 0x00006603},
552         {0x0000008d, 0x00000100},
553         {0x0000008f, 0x00001c0a},
554         {0x00000090, 0xff000001},
555         {0x00000094, 0x00101101},
556         {0x00000095, 0x00000fff},
557         {0x00000096, 0x00116fff},
558         {0x00000097, 0x60010000},
559         {0x00000098, 0x10010000},
560         {0x00000099, 0x00006000},
561         {0x0000009a, 0x00001000},
562         {0x0000009f, 0x00916a00}
563 };
564
565 static const u32 cayman_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
566         {0x00000077, 0xff010100},
567         {0x00000078, 0x00000000},
568         {0x00000079, 0x00001434},
569         {0x0000007a, 0xcc08ec08},
570         {0x0000007b, 0x00040000},
571         {0x0000007c, 0x000080c0},
572         {0x0000007d, 0x09000000},
573         {0x0000007e, 0x00210404},
574         {0x00000081, 0x08a8e800},
575         {0x00000082, 0x00030444},
576         {0x00000083, 0x00000000},
577         {0x00000085, 0x00000001},
578         {0x00000086, 0x00000002},
579         {0x00000087, 0x48490000},
580         {0x00000088, 0x20244647},
581         {0x00000089, 0x00000005},
582         {0x0000008b, 0x66030000},
583         {0x0000008c, 0x00006603},
584         {0x0000008d, 0x00000100},
585         {0x0000008f, 0x00001c0a},
586         {0x00000090, 0xff000001},
587         {0x00000094, 0x00101101},
588         {0x00000095, 0x00000fff},
589         {0x00000096, 0x00116fff},
590         {0x00000097, 0x60010000},
591         {0x00000098, 0x10010000},
592         {0x00000099, 0x00006000},
593         {0x0000009a, 0x00001000},
594         {0x0000009f, 0x00976b00}
595 };
596
597 int ni_mc_load_microcode(struct radeon_device *rdev)
598 {
599         const __be32 *fw_data;
600         u32 mem_type, running, blackout = 0;
601         u32 *io_mc_regs;
602         int i, ucode_size, regs_size;
603
604         if (!rdev->mc_fw)
605                 return -EINVAL;
606
607         switch (rdev->family) {
608         case CHIP_BARTS:
609                 io_mc_regs = (u32 *)&barts_io_mc_regs;
610                 ucode_size = BTC_MC_UCODE_SIZE;
611                 regs_size = BTC_IO_MC_REGS_SIZE;
612                 break;
613         case CHIP_TURKS:
614                 io_mc_regs = (u32 *)&turks_io_mc_regs;
615                 ucode_size = BTC_MC_UCODE_SIZE;
616                 regs_size = BTC_IO_MC_REGS_SIZE;
617                 break;
618         case CHIP_CAICOS:
619         default:
620                 io_mc_regs = (u32 *)&caicos_io_mc_regs;
621                 ucode_size = BTC_MC_UCODE_SIZE;
622                 regs_size = BTC_IO_MC_REGS_SIZE;
623                 break;
624         case CHIP_CAYMAN:
625                 io_mc_regs = (u32 *)&cayman_io_mc_regs;
626                 ucode_size = CAYMAN_MC_UCODE_SIZE;
627                 regs_size = BTC_IO_MC_REGS_SIZE;
628                 break;
629         }
630
631         mem_type = (RREG32(MC_SEQ_MISC0) & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT;
632         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
633
634         if ((mem_type == MC_SEQ_MISC0_GDDR5_VALUE) && (running == 0)) {
635                 if (running) {
636                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
637                         WREG32(MC_SHARED_BLACKOUT_CNTL, 1);
638                 }
639
640                 /* reset the engine and set to writable */
641                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
642                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
643
644                 /* load mc io regs */
645                 for (i = 0; i < regs_size; i++) {
646                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
647                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
648                 }
649                 /* load the MC ucode */
650                 fw_data = (const __be32 *)rdev->mc_fw->data;
651                 for (i = 0; i < ucode_size; i++)
652                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
653
654                 /* put the engine back into the active state */
655                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
656                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
657                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
658
659                 /* wait for training to complete */
660                 for (i = 0; i < rdev->usec_timeout; i++) {
661                         if (RREG32(MC_IO_PAD_CNTL_D0) & MEM_FALL_OUT_CMD)
662                                 break;
663                         DRM_UDELAY(1);
664                 }
665
666                 if (running)
667                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
668         }
669
670         return 0;
671 }
672
673 int ni_init_microcode(struct radeon_device *rdev)
674 {
675         const char *chip_name;
676         const char *rlc_chip_name;
677         size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size;
678         size_t smc_req_size = 0;
679         char fw_name[30];
680         int err;
681
682         DRM_DEBUG("\n");
683
684         switch (rdev->family) {
685         case CHIP_BARTS:
686                 chip_name = "BARTS";
687                 rlc_chip_name = "BTC";
688                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
689                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
690                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
691                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
692                 smc_req_size = ALIGN(BARTS_SMC_UCODE_SIZE, 4);
693                 break;
694         case CHIP_TURKS:
695                 chip_name = "TURKS";
696                 rlc_chip_name = "BTC";
697                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
698                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
699                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
700                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
701                 smc_req_size = ALIGN(TURKS_SMC_UCODE_SIZE, 4);
702                 break;
703         case CHIP_CAICOS:
704                 chip_name = "CAICOS";
705                 rlc_chip_name = "BTC";
706                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
707                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
708                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
709                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
710                 smc_req_size = ALIGN(CAICOS_SMC_UCODE_SIZE, 4);
711                 break;
712         case CHIP_CAYMAN:
713                 chip_name = "CAYMAN";
714                 rlc_chip_name = "CAYMAN";
715                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
716                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
717                 rlc_req_size = CAYMAN_RLC_UCODE_SIZE * 4;
718                 mc_req_size = CAYMAN_MC_UCODE_SIZE * 4;
719                 smc_req_size = ALIGN(CAYMAN_SMC_UCODE_SIZE, 4);
720                 break;
721         case CHIP_ARUBA:
722                 chip_name = "ARUBA";
723                 rlc_chip_name = "ARUBA";
724                 /* pfp/me same size as CAYMAN */
725                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
726                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
727                 rlc_req_size = ARUBA_RLC_UCODE_SIZE * 4;
728                 mc_req_size = 0;
729                 break;
730         default: panic("%s: Unsupported family %d", __func__, rdev->family);
731         }
732
733         DRM_INFO("Loading %s Microcode\n", chip_name);
734         err = 0;
735
736         ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_pfp", chip_name);
737         rdev->pfp_fw = firmware_get(fw_name);
738         if (rdev->pfp_fw == NULL) {
739                 err = -ENOENT;
740                 goto out;
741         }
742         if (rdev->pfp_fw->datasize != pfp_req_size) {
743                 DRM_ERROR(
744                        "ni_pfp: Bogus length %zu in firmware \"%s\"\n",
745                        rdev->pfp_fw->datasize, fw_name);
746                 err = -EINVAL;
747                 goto out;
748         }
749
750         ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_me", chip_name);
751         rdev->me_fw = firmware_get(fw_name);
752         if (rdev->me_fw == NULL) {
753                 err = -ENOENT;
754                 goto out;
755         }
756         if (rdev->me_fw->datasize != me_req_size) {
757                 DRM_ERROR(
758                        "ni_me: Bogus length %zu in firmware \"%s\"\n",
759                        rdev->me_fw->datasize, fw_name);
760                 err = -EINVAL;
761         }
762
763         ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_rlc",
764                   rlc_chip_name);
765         rdev->rlc_fw = firmware_get(fw_name);
766         if (rdev->rlc_fw == NULL) {
767                 err = -ENOENT;
768                 goto out;
769         }
770         if (rdev->rlc_fw->datasize != rlc_req_size) {
771                 DRM_ERROR(
772                        "ni_rlc: Bogus length %zu in firmware \"%s\"\n",
773                        rdev->rlc_fw->datasize, fw_name);
774                 err = -EINVAL;
775         }
776
777         /* no MC ucode on TN */
778         if (!(rdev->flags & RADEON_IS_IGP)) {
779                 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mc",
780                           chip_name);
781                 rdev->mc_fw = firmware_get(fw_name);
782                 if (rdev->mc_fw == NULL) {
783                         err = -ENOENT;
784                         goto out;
785                 }
786                 if (rdev->mc_fw->datasize != mc_req_size) {
787                         DRM_ERROR(
788                                "ni_mc: Bogus length %zu in firmware \"%s\"\n",
789                                rdev->mc_fw->datasize, fw_name);
790                         err = -EINVAL;
791                 }
792         }
793
794         if ((rdev->family >= CHIP_BARTS) && (rdev->family <= CHIP_CAYMAN)) {
795                 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_smc", chip_name);
796                 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
797                 if (err) {
798                         printk(KERN_ERR
799                                "smc: error loading firmware \"%s\"\n",
800                                fw_name);
801                         release_firmware(rdev->smc_fw);
802                         rdev->smc_fw = NULL;
803                 } else if (rdev->smc_fw->datasize != smc_req_size) {
804                         printk(KERN_ERR
805                                "ni_smc: Bogus length %zu in firmware \"%s\"\n",
806                                rdev->smc_fw->datasize, fw_name);
807                         err = -EINVAL;
808                 }
809         }
810
811 out:
812         if (err) {
813                 if (err != -EINVAL)
814                         DRM_ERROR(
815                                "ni_cp: Failed to load firmware \"%s\"\n",
816                                fw_name);
817                 if (rdev->pfp_fw != NULL) {
818                         firmware_put(rdev->pfp_fw, FIRMWARE_UNLOAD);
819                         rdev->pfp_fw = NULL;
820                 }
821                 if (rdev->me_fw != NULL) {
822                         firmware_put(rdev->me_fw, FIRMWARE_UNLOAD);
823                         rdev->me_fw = NULL;
824                 }
825                 if (rdev->rlc_fw != NULL) {
826                         firmware_put(rdev->rlc_fw, FIRMWARE_UNLOAD);
827                         rdev->rlc_fw = NULL;
828                 }
829                 if (rdev->mc_fw != NULL) {
830                         firmware_put(rdev->mc_fw, FIRMWARE_UNLOAD);
831                         rdev->mc_fw = NULL;
832                 }
833                 if (rdev->smc_fw != NULL) {
834                         firmware_put(rdev->smc_fw, FIRMWARE_UNLOAD);
835                         rdev->smc_fw = NULL;
836                 }
837         }
838         return err;
839 }
840
841 /**
842  * ni_fini_microcode - drop the firmwares image references
843  *
844  * @rdev: radeon_device pointer
845  *
846  * Drop the pfp, me, mc and rlc firmwares image references.
847  * Called at driver shutdown.
848  */
849 void ni_fini_microcode(struct radeon_device *rdev)
850 {
851
852         if (rdev->pfp_fw != NULL) {
853                 firmware_put(rdev->pfp_fw, FIRMWARE_UNLOAD);
854                 rdev->pfp_fw = NULL;
855         }
856
857         if (rdev->me_fw != NULL) {
858                 firmware_put(rdev->me_fw, FIRMWARE_UNLOAD);
859                 rdev->me_fw = NULL;
860         }
861
862         if (rdev->rlc_fw != NULL) {
863                 firmware_put(rdev->rlc_fw, FIRMWARE_UNLOAD);
864                 rdev->rlc_fw = NULL;
865         }
866
867         if (rdev->mc_fw != NULL) {
868                 firmware_put(rdev->mc_fw, FIRMWARE_UNLOAD);
869                 rdev->mc_fw = NULL;
870         }
871 }
872
873
874 int tn_get_temp(struct radeon_device *rdev)
875 {
876         u32 temp = RREG32_SMC(TN_CURRENT_GNB_TEMP) & 0x7ff;
877         int actual_temp = (temp / 8) - 49;
878
879         return actual_temp * 1000;
880 }
881
882 /*
883  * Core functions
884  */
885 static void cayman_gpu_init(struct radeon_device *rdev)
886 {
887         u32 gb_addr_config = 0;
888         u32 mc_shared_chmap, mc_arb_ramcfg;
889         u32 cgts_tcc_disable;
890         u32 sx_debug_1;
891         u32 smx_dc_ctl0;
892         u32 cgts_sm_ctrl_reg;
893         u32 hdp_host_path_cntl;
894         u32 tmp;
895         u32 disabled_rb_mask;
896         int i, j;
897
898         switch (rdev->family) {
899         case CHIP_CAYMAN:
900                 rdev->config.cayman.max_shader_engines = 2;
901                 rdev->config.cayman.max_pipes_per_simd = 4;
902                 rdev->config.cayman.max_tile_pipes = 8;
903                 rdev->config.cayman.max_simds_per_se = 12;
904                 rdev->config.cayman.max_backends_per_se = 4;
905                 rdev->config.cayman.max_texture_channel_caches = 8;
906                 rdev->config.cayman.max_gprs = 256;
907                 rdev->config.cayman.max_threads = 256;
908                 rdev->config.cayman.max_gs_threads = 32;
909                 rdev->config.cayman.max_stack_entries = 512;
910                 rdev->config.cayman.sx_num_of_sets = 8;
911                 rdev->config.cayman.sx_max_export_size = 256;
912                 rdev->config.cayman.sx_max_export_pos_size = 64;
913                 rdev->config.cayman.sx_max_export_smx_size = 192;
914                 rdev->config.cayman.max_hw_contexts = 8;
915                 rdev->config.cayman.sq_num_cf_insts = 2;
916
917                 rdev->config.cayman.sc_prim_fifo_size = 0x100;
918                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
919                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
920                 gb_addr_config = CAYMAN_GB_ADDR_CONFIG_GOLDEN;
921                 break;
922         case CHIP_ARUBA:
923         default:
924                 rdev->config.cayman.max_shader_engines = 1;
925                 rdev->config.cayman.max_pipes_per_simd = 4;
926                 rdev->config.cayman.max_tile_pipes = 2;
927                 if ((rdev->ddev->pci_device == 0x9900) ||
928                     (rdev->ddev->pci_device == 0x9901) ||
929                     (rdev->ddev->pci_device == 0x9905) ||
930                     (rdev->ddev->pci_device == 0x9906) ||
931                     (rdev->ddev->pci_device == 0x9907) ||
932                     (rdev->ddev->pci_device == 0x9908) ||
933                     (rdev->ddev->pci_device == 0x9909) ||
934                     (rdev->ddev->pci_device == 0x990B) ||
935                     (rdev->ddev->pci_device == 0x990C) ||
936                     (rdev->ddev->pci_device == 0x990F) ||
937                     (rdev->ddev->pci_device == 0x9910) ||
938                     (rdev->ddev->pci_device == 0x9917) ||
939                     (rdev->ddev->pci_device == 0x9999) ||
940                     (rdev->ddev->pci_device == 0x999C)) {
941                         rdev->config.cayman.max_simds_per_se = 6;
942                         rdev->config.cayman.max_backends_per_se = 2;
943                 } else if ((rdev->ddev->pci_device == 0x9903) ||
944                            (rdev->ddev->pci_device == 0x9904) ||
945                            (rdev->ddev->pci_device == 0x990A) ||
946                            (rdev->ddev->pci_device == 0x990D) ||
947                            (rdev->ddev->pci_device == 0x990E) ||
948                            (rdev->ddev->pci_device == 0x9913) ||
949                            (rdev->ddev->pci_device == 0x9918) ||
950                            (rdev->ddev->pci_device == 0x999D)) {
951                         rdev->config.cayman.max_simds_per_se = 4;
952                         rdev->config.cayman.max_backends_per_se = 2;
953                 } else if ((rdev->ddev->pci_device == 0x9919) ||
954                            (rdev->ddev->pci_device == 0x9990) ||
955                            (rdev->ddev->pci_device == 0x9991) ||
956                            (rdev->ddev->pci_device == 0x9994) ||
957                            (rdev->ddev->pci_device == 0x9995) ||
958                            (rdev->ddev->pci_device == 0x9996) ||
959                            (rdev->ddev->pci_device == 0x999A) ||
960                            (rdev->ddev->pci_device == 0x99A0)) {
961                         rdev->config.cayman.max_simds_per_se = 3;
962                         rdev->config.cayman.max_backends_per_se = 1;
963                 } else {
964                         rdev->config.cayman.max_simds_per_se = 2;
965                         rdev->config.cayman.max_backends_per_se = 1;
966                 }
967                 rdev->config.cayman.max_texture_channel_caches = 2;
968                 rdev->config.cayman.max_gprs = 256;
969                 rdev->config.cayman.max_threads = 256;
970                 rdev->config.cayman.max_gs_threads = 32;
971                 rdev->config.cayman.max_stack_entries = 512;
972                 rdev->config.cayman.sx_num_of_sets = 8;
973                 rdev->config.cayman.sx_max_export_size = 256;
974                 rdev->config.cayman.sx_max_export_pos_size = 64;
975                 rdev->config.cayman.sx_max_export_smx_size = 192;
976                 rdev->config.cayman.max_hw_contexts = 8;
977                 rdev->config.cayman.sq_num_cf_insts = 2;
978
979                 rdev->config.cayman.sc_prim_fifo_size = 0x40;
980                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
981                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
982                 gb_addr_config = ARUBA_GB_ADDR_CONFIG_GOLDEN;
983                 break;
984         }
985
986         /* Initialize HDP */
987         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
988                 WREG32((0x2c14 + j), 0x00000000);
989                 WREG32((0x2c18 + j), 0x00000000);
990                 WREG32((0x2c1c + j), 0x00000000);
991                 WREG32((0x2c20 + j), 0x00000000);
992                 WREG32((0x2c24 + j), 0x00000000);
993         }
994
995         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
996
997         evergreen_fix_pci_max_read_req_size(rdev);
998
999         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
1000         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
1001
1002         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
1003         rdev->config.cayman.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
1004         if (rdev->config.cayman.mem_row_size_in_kb > 4)
1005                 rdev->config.cayman.mem_row_size_in_kb = 4;
1006         /* XXX use MC settings? */
1007         rdev->config.cayman.shader_engine_tile_size = 32;
1008         rdev->config.cayman.num_gpus = 1;
1009         rdev->config.cayman.multi_gpu_tile_size = 64;
1010
1011         tmp = (gb_addr_config & NUM_PIPES_MASK) >> NUM_PIPES_SHIFT;
1012         rdev->config.cayman.num_tile_pipes = (1 << tmp);
1013         tmp = (gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT;
1014         rdev->config.cayman.mem_max_burst_length_bytes = (tmp + 1) * 256;
1015         tmp = (gb_addr_config & NUM_SHADER_ENGINES_MASK) >> NUM_SHADER_ENGINES_SHIFT;
1016         rdev->config.cayman.num_shader_engines = tmp + 1;
1017         tmp = (gb_addr_config & NUM_GPUS_MASK) >> NUM_GPUS_SHIFT;
1018         rdev->config.cayman.num_gpus = tmp + 1;
1019         tmp = (gb_addr_config & MULTI_GPU_TILE_SIZE_MASK) >> MULTI_GPU_TILE_SIZE_SHIFT;
1020         rdev->config.cayman.multi_gpu_tile_size = 1 << tmp;
1021         tmp = (gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT;
1022         rdev->config.cayman.mem_row_size_in_kb = 1 << tmp;
1023
1024
1025         /* setup tiling info dword.  gb_addr_config is not adequate since it does
1026          * not have bank info, so create a custom tiling dword.
1027          * bits 3:0   num_pipes
1028          * bits 7:4   num_banks
1029          * bits 11:8  group_size
1030          * bits 15:12 row_size
1031          */
1032         rdev->config.cayman.tile_config = 0;
1033         switch (rdev->config.cayman.num_tile_pipes) {
1034         case 1:
1035         default:
1036                 rdev->config.cayman.tile_config |= (0 << 0);
1037                 break;
1038         case 2:
1039                 rdev->config.cayman.tile_config |= (1 << 0);
1040                 break;
1041         case 4:
1042                 rdev->config.cayman.tile_config |= (2 << 0);
1043                 break;
1044         case 8:
1045                 rdev->config.cayman.tile_config |= (3 << 0);
1046                 break;
1047         }
1048
1049         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1050         if (rdev->flags & RADEON_IS_IGP)
1051                 rdev->config.cayman.tile_config |= 1 << 4;
1052         else {
1053                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
1054                 case 0: /* four banks */
1055                         rdev->config.cayman.tile_config |= 0 << 4;
1056                         break;
1057                 case 1: /* eight banks */
1058                         rdev->config.cayman.tile_config |= 1 << 4;
1059                         break;
1060                 case 2: /* sixteen banks */
1061                 default:
1062                         rdev->config.cayman.tile_config |= 2 << 4;
1063                         break;
1064                 }
1065         }
1066         rdev->config.cayman.tile_config |=
1067                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
1068         rdev->config.cayman.tile_config |=
1069                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
1070
1071         tmp = 0;
1072         for (i = (rdev->config.cayman.max_shader_engines - 1); i >= 0; i--) {
1073                 u32 rb_disable_bitmap;
1074
1075                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1076                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1077                 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
1078                 tmp <<= 4;
1079                 tmp |= rb_disable_bitmap;
1080         }
1081         /* enabled rb are just the one not disabled :) */
1082         disabled_rb_mask = tmp;
1083         tmp = 0;
1084         for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1085                 tmp |= (1 << i);
1086         /* if all the backends are disabled, fix it up here */
1087         if ((disabled_rb_mask & tmp) == tmp) {
1088                 for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1089                         disabled_rb_mask &= ~(1 << i);
1090         }
1091
1092         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1093         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1094
1095         WREG32(GB_ADDR_CONFIG, gb_addr_config);
1096         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
1097         if (ASIC_IS_DCE6(rdev))
1098                 WREG32(DMIF_ADDR_CALC, gb_addr_config);
1099         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
1100         WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
1101         WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1102         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
1103         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
1104         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
1105
1106         if ((rdev->config.cayman.max_backends_per_se == 1) &&
1107             (rdev->flags & RADEON_IS_IGP)) {
1108                 if ((disabled_rb_mask & 3) == 1) {
1109                         /* RB0 disabled, RB1 enabled */
1110                         tmp = 0x11111111;
1111                 } else {
1112                         /* RB1 disabled, RB0 enabled */
1113                         tmp = 0x00000000;
1114                 }
1115         } else {
1116                 tmp = gb_addr_config & NUM_PIPES_MASK;
1117                 tmp = r6xx_remap_render_backend(rdev, tmp,
1118                                                 rdev->config.cayman.max_backends_per_se *
1119                                                 rdev->config.cayman.max_shader_engines,
1120                                                 CAYMAN_MAX_BACKENDS, disabled_rb_mask);
1121         }
1122         WREG32(GB_BACKEND_MAP, tmp);
1123
1124         cgts_tcc_disable = 0xffff0000;
1125         for (i = 0; i < rdev->config.cayman.max_texture_channel_caches; i++)
1126                 cgts_tcc_disable &= ~(1 << (16 + i));
1127         WREG32(CGTS_TCC_DISABLE, cgts_tcc_disable);
1128         WREG32(CGTS_SYS_TCC_DISABLE, cgts_tcc_disable);
1129         WREG32(CGTS_USER_SYS_TCC_DISABLE, cgts_tcc_disable);
1130         WREG32(CGTS_USER_TCC_DISABLE, cgts_tcc_disable);
1131
1132         /* reprogram the shader complex */
1133         cgts_sm_ctrl_reg = RREG32(CGTS_SM_CTRL_REG);
1134         for (i = 0; i < 16; i++)
1135                 WREG32(CGTS_SM_CTRL_REG, OVERRIDE);
1136         WREG32(CGTS_SM_CTRL_REG, cgts_sm_ctrl_reg);
1137
1138         /* set HW defaults for 3D engine */
1139         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
1140
1141         sx_debug_1 = RREG32(SX_DEBUG_1);
1142         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1143         WREG32(SX_DEBUG_1, sx_debug_1);
1144
1145         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1146         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
1147         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.cayman.sx_num_of_sets);
1148         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1149
1150         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4) | CRC_SIMD_ID_WADDR_DISABLE);
1151
1152         /* need to be explicitly zero-ed */
1153         WREG32(VGT_OFFCHIP_LDS_BASE, 0);
1154         WREG32(SQ_LSTMP_RING_BASE, 0);
1155         WREG32(SQ_HSTMP_RING_BASE, 0);
1156         WREG32(SQ_ESTMP_RING_BASE, 0);
1157         WREG32(SQ_GSTMP_RING_BASE, 0);
1158         WREG32(SQ_VSTMP_RING_BASE, 0);
1159         WREG32(SQ_PSTMP_RING_BASE, 0);
1160
1161         WREG32(TA_CNTL_AUX, DISABLE_CUBE_ANISO);
1162
1163         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.cayman.sx_max_export_size / 4) - 1) |
1164                                         POSITION_BUFFER_SIZE((rdev->config.cayman.sx_max_export_pos_size / 4) - 1) |
1165                                         SMX_BUFFER_SIZE((rdev->config.cayman.sx_max_export_smx_size / 4) - 1)));
1166
1167         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.cayman.sc_prim_fifo_size) |
1168                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_hiz_tile_fifo_size) |
1169                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_earlyz_tile_fifo_size)));
1170
1171
1172         WREG32(VGT_NUM_INSTANCES, 1);
1173
1174         WREG32(CP_PERFMON_CNTL, 0);
1175
1176         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.cayman.sq_num_cf_insts) |
1177                                   FETCH_FIFO_HIWATER(0x4) |
1178                                   DONE_FIFO_HIWATER(0xe0) |
1179                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
1180
1181         WREG32(SQ_GPR_RESOURCE_MGMT_1, NUM_CLAUSE_TEMP_GPRS(4));
1182         WREG32(SQ_CONFIG, (VC_ENABLE |
1183                            EXPORT_SRC_C |
1184                            GFX_PRIO(0) |
1185                            CS1_PRIO(0) |
1186                            CS2_PRIO(1)));
1187         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, DYN_GPR_ENABLE);
1188
1189         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1190                                           FORCE_EOV_MAX_REZ_CNT(255)));
1191
1192         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
1193                AUTO_INVLD_EN(ES_AND_GS_AUTO));
1194
1195         WREG32(VGT_GS_VERTEX_REUSE, 16);
1196         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1197
1198         WREG32(CB_PERF_CTR0_SEL_0, 0);
1199         WREG32(CB_PERF_CTR0_SEL_1, 0);
1200         WREG32(CB_PERF_CTR1_SEL_0, 0);
1201         WREG32(CB_PERF_CTR1_SEL_1, 0);
1202         WREG32(CB_PERF_CTR2_SEL_0, 0);
1203         WREG32(CB_PERF_CTR2_SEL_1, 0);
1204         WREG32(CB_PERF_CTR3_SEL_0, 0);
1205         WREG32(CB_PERF_CTR3_SEL_1, 0);
1206
1207         tmp = RREG32(HDP_MISC_CNTL);
1208         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
1209         WREG32(HDP_MISC_CNTL, tmp);
1210
1211         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1212         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1213
1214         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
1215
1216         DRM_UDELAY(50);
1217
1218         /* set clockgating golden values on TN */
1219         if (rdev->family == CHIP_ARUBA) {
1220                 tmp = RREG32_CG(CG_CGTT_LOCAL_0);
1221                 tmp &= ~0x00380000;
1222                 WREG32_CG(CG_CGTT_LOCAL_0, tmp);
1223                 tmp = RREG32_CG(CG_CGTT_LOCAL_1);
1224                 tmp &= ~0x0e000000;
1225                 WREG32_CG(CG_CGTT_LOCAL_1, tmp);
1226         }
1227 }
1228
1229 /*
1230  * GART
1231  */
1232 void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev)
1233 {
1234         /* flush hdp cache */
1235         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
1236
1237         /* bits 0-7 are the VM contexts0-7 */
1238         WREG32(VM_INVALIDATE_REQUEST, 1);
1239 }
1240
1241 static int cayman_pcie_gart_enable(struct radeon_device *rdev)
1242 {
1243         int i, r;
1244
1245         if (rdev->gart.robj == NULL) {
1246                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
1247                 return -EINVAL;
1248         }
1249         r = radeon_gart_table_vram_pin(rdev);
1250         if (r)
1251                 return r;
1252         radeon_gart_restore(rdev);
1253         /* Setup TLB control */
1254         WREG32(MC_VM_MX_L1_TLB_CNTL,
1255                (0xA << 7) |
1256                ENABLE_L1_TLB |
1257                ENABLE_L1_FRAGMENT_PROCESSING |
1258                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1259                ENABLE_ADVANCED_DRIVER_MODEL |
1260                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1261         /* Setup L2 cache */
1262         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
1263                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1264                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1265                EFFECTIVE_L2_QUEUE_SIZE(7) |
1266                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1267         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
1268         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1269                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1270         /* setup context0 */
1271         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
1272         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
1273         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
1274         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
1275                         (u32)(rdev->dummy_page.addr >> 12));
1276         WREG32(VM_CONTEXT0_CNTL2, 0);
1277         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
1278                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
1279
1280         WREG32(0x15D4, 0);
1281         WREG32(0x15D8, 0);
1282         WREG32(0x15DC, 0);
1283
1284         /* empty context1-7 */
1285         /* Assign the pt base to something valid for now; the pts used for
1286          * the VMs are determined by the application and setup and assigned
1287          * on the fly in the vm part of radeon_gart.c
1288          */
1289         for (i = 1; i < 8; i++) {
1290                 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (i << 2), 0);
1291                 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (i << 2), rdev->vm_manager.max_pfn);
1292                 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
1293                         rdev->gart.table_addr >> 12);
1294         }
1295
1296         /* enable context1-7 */
1297         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
1298                (u32)(rdev->dummy_page.addr >> 12));
1299         WREG32(VM_CONTEXT1_CNTL2, 4);
1300         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
1301                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1302                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1303                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1304                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1305                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
1306                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
1307                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
1308                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
1309                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
1310                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
1311                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1312                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1313
1314         cayman_pcie_gart_tlb_flush(rdev);
1315         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1316                  (unsigned)(rdev->mc.gtt_size >> 20),
1317                  (unsigned long long)rdev->gart.table_addr);
1318         rdev->gart.ready = true;
1319         return 0;
1320 }
1321
1322 static void cayman_pcie_gart_disable(struct radeon_device *rdev)
1323 {
1324         /* Disable all tables */
1325         WREG32(VM_CONTEXT0_CNTL, 0);
1326         WREG32(VM_CONTEXT1_CNTL, 0);
1327         /* Setup TLB control */
1328         WREG32(MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING |
1329                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1330                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1331         /* Setup L2 cache */
1332         WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1333                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1334                EFFECTIVE_L2_QUEUE_SIZE(7) |
1335                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1336         WREG32(VM_L2_CNTL2, 0);
1337         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1338                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1339         radeon_gart_table_vram_unpin(rdev);
1340 }
1341
1342 static void cayman_pcie_gart_fini(struct radeon_device *rdev)
1343 {
1344         cayman_pcie_gart_disable(rdev);
1345         radeon_gart_table_vram_free(rdev);
1346         radeon_gart_fini(rdev);
1347 }
1348
1349 void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
1350                               int ring, u32 cp_int_cntl)
1351 {
1352         u32 srbm_gfx_cntl = RREG32(SRBM_GFX_CNTL) & ~3;
1353
1354         WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl | (ring & 3));
1355         WREG32(CP_INT_CNTL, cp_int_cntl);
1356 }
1357
1358 /*
1359  * CP.
1360  */
1361 void cayman_fence_ring_emit(struct radeon_device *rdev,
1362                             struct radeon_fence *fence)
1363 {
1364         struct radeon_ring *ring = &rdev->ring[fence->ring];
1365         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
1366
1367         /* flush read cache over gart for this vmid */
1368         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1369         radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
1370         radeon_ring_write(ring, 0);
1371         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1372         radeon_ring_write(ring, PACKET3_TC_ACTION_ENA | PACKET3_SH_ACTION_ENA);
1373         radeon_ring_write(ring, 0xFFFFFFFF);
1374         radeon_ring_write(ring, 0);
1375         radeon_ring_write(ring, 10); /* poll interval */
1376         /* EVENT_WRITE_EOP - flush caches, send int */
1377         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
1378         radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
1379         radeon_ring_write(ring, addr & 0xffffffff);
1380         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
1381         radeon_ring_write(ring, fence->seq);
1382         radeon_ring_write(ring, 0);
1383 }
1384
1385 void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1386 {
1387         struct radeon_ring *ring = &rdev->ring[ib->ring];
1388
1389         /* set to DX10/11 mode */
1390         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
1391         radeon_ring_write(ring, 1);
1392
1393         if (ring->rptr_save_reg) {
1394                 uint32_t next_rptr = ring->wptr + 3 + 4 + 8;
1395                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1396                 radeon_ring_write(ring, ((ring->rptr_save_reg - 
1397                                           PACKET3_SET_CONFIG_REG_START) >> 2));
1398                 radeon_ring_write(ring, next_rptr);
1399         }
1400
1401         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1402         radeon_ring_write(ring,
1403 #ifdef __BIG_ENDIAN
1404                           (2 << 0) |
1405 #endif
1406                           (ib->gpu_addr & 0xFFFFFFFC));
1407         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
1408         radeon_ring_write(ring, ib->length_dw | 
1409                           (ib->vm ? (ib->vm->id << 24) : 0));
1410
1411         /* flush read cache over gart for this vmid */
1412         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1413         radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
1414         radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
1415         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1416         radeon_ring_write(ring, PACKET3_TC_ACTION_ENA | PACKET3_SH_ACTION_ENA);
1417         radeon_ring_write(ring, 0xFFFFFFFF);
1418         radeon_ring_write(ring, 0);
1419         radeon_ring_write(ring, 10); /* poll interval */
1420 }
1421
1422 void cayman_uvd_semaphore_emit(struct radeon_device *rdev,
1423                                struct radeon_ring *ring,
1424                                struct radeon_semaphore *semaphore,
1425                                bool emit_wait)
1426 {
1427         uint64_t addr = semaphore->gpu_addr;
1428
1429         radeon_ring_write(ring, PACKET0(UVD_SEMA_ADDR_LOW, 0));
1430         radeon_ring_write(ring, (addr >> 3) & 0x000FFFFF);
1431
1432         radeon_ring_write(ring, PACKET0(UVD_SEMA_ADDR_HIGH, 0));
1433         radeon_ring_write(ring, (addr >> 23) & 0x000FFFFF);
1434
1435         radeon_ring_write(ring, PACKET0(UVD_SEMA_CMD, 0));
1436         radeon_ring_write(ring, 0x80 | (emit_wait ? 1 : 0));
1437 }
1438
1439 static void cayman_cp_enable(struct radeon_device *rdev, bool enable)
1440 {
1441         if (enable)
1442                 WREG32(CP_ME_CNTL, 0);
1443         else {
1444                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1445                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1446                 WREG32(SCRATCH_UMSK, 0);
1447                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1448         }
1449 }
1450
1451 static int cayman_cp_load_microcode(struct radeon_device *rdev)
1452 {
1453         const __be32 *fw_data;
1454         int i;
1455
1456         if (!rdev->me_fw || !rdev->pfp_fw)
1457                 return -EINVAL;
1458
1459         cayman_cp_enable(rdev, false);
1460
1461         fw_data = (const __be32 *)rdev->pfp_fw->data;
1462         WREG32(CP_PFP_UCODE_ADDR, 0);
1463         for (i = 0; i < CAYMAN_PFP_UCODE_SIZE; i++)
1464                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1465         WREG32(CP_PFP_UCODE_ADDR, 0);
1466
1467         fw_data = (const __be32 *)rdev->me_fw->data;
1468         WREG32(CP_ME_RAM_WADDR, 0);
1469         for (i = 0; i < CAYMAN_PM4_UCODE_SIZE; i++)
1470                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1471
1472         WREG32(CP_PFP_UCODE_ADDR, 0);
1473         WREG32(CP_ME_RAM_WADDR, 0);
1474         WREG32(CP_ME_RAM_RADDR, 0);
1475         return 0;
1476 }
1477
1478 static int cayman_cp_start(struct radeon_device *rdev)
1479 {
1480         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1481         int r, i;
1482
1483         r = radeon_ring_lock(rdev, ring, 7);
1484         if (r) {
1485                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1486                 return r;
1487         }
1488         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
1489         radeon_ring_write(ring, 0x1);
1490         radeon_ring_write(ring, 0x0);
1491         radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1);
1492         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1493         radeon_ring_write(ring, 0);
1494         radeon_ring_write(ring, 0);
1495         radeon_ring_unlock_commit(rdev, ring);
1496
1497         cayman_cp_enable(rdev, true);
1498
1499         r = radeon_ring_lock(rdev, ring, cayman_default_size + 19);
1500         if (r) {
1501                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1502                 return r;
1503         }
1504
1505         /* setup clear context state */
1506         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1507         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1508
1509         for (i = 0; i < cayman_default_size; i++)
1510                 radeon_ring_write(ring, cayman_default_state[i]);
1511
1512         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1513         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1514
1515         /* set clear context state */
1516         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
1517         radeon_ring_write(ring, 0);
1518
1519         /* SQ_VTX_BASE_VTX_LOC */
1520         radeon_ring_write(ring, 0xc0026f00);
1521         radeon_ring_write(ring, 0x00000000);
1522         radeon_ring_write(ring, 0x00000000);
1523         radeon_ring_write(ring, 0x00000000);
1524
1525         /* Clear consts */
1526         radeon_ring_write(ring, 0xc0036f00);
1527         radeon_ring_write(ring, 0x00000bc4);
1528         radeon_ring_write(ring, 0xffffffff);
1529         radeon_ring_write(ring, 0xffffffff);
1530         radeon_ring_write(ring, 0xffffffff);
1531
1532         radeon_ring_write(ring, 0xc0026900);
1533         radeon_ring_write(ring, 0x00000316);
1534         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1535         radeon_ring_write(ring, 0x00000010); /*  */
1536
1537         radeon_ring_unlock_commit(rdev, ring);
1538
1539         /* XXX init other rings */
1540
1541         return 0;
1542 }
1543
1544 static void cayman_cp_fini(struct radeon_device *rdev)
1545 {
1546         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1547         cayman_cp_enable(rdev, false);
1548         radeon_ring_fini(rdev, ring);
1549         radeon_scratch_free(rdev, ring->rptr_save_reg);
1550 }
1551
1552 static int cayman_cp_resume(struct radeon_device *rdev)
1553 {
1554         static const int ridx[] = {
1555                 RADEON_RING_TYPE_GFX_INDEX,
1556                 CAYMAN_RING_TYPE_CP1_INDEX,
1557                 CAYMAN_RING_TYPE_CP2_INDEX
1558         };
1559         static const unsigned cp_rb_cntl[] = {
1560                 CP_RB0_CNTL,
1561                 CP_RB1_CNTL,
1562                 CP_RB2_CNTL,
1563         };
1564         static const unsigned cp_rb_rptr_addr[] = {
1565                 CP_RB0_RPTR_ADDR,
1566                 CP_RB1_RPTR_ADDR,
1567                 CP_RB2_RPTR_ADDR
1568         };
1569         static const unsigned cp_rb_rptr_addr_hi[] = {
1570                 CP_RB0_RPTR_ADDR_HI,
1571                 CP_RB1_RPTR_ADDR_HI,
1572                 CP_RB2_RPTR_ADDR_HI
1573         };
1574         static const unsigned cp_rb_base[] = {
1575                 CP_RB0_BASE,
1576                 CP_RB1_BASE,
1577                 CP_RB2_BASE
1578         };
1579         struct radeon_ring *ring;
1580         int i, r;
1581
1582         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1583         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1584                                  SOFT_RESET_PA |
1585                                  SOFT_RESET_SH |
1586                                  SOFT_RESET_VGT |
1587                                  SOFT_RESET_SPI |
1588                                  SOFT_RESET_SX));
1589         RREG32(GRBM_SOFT_RESET);
1590         DRM_MDELAY(15);
1591         WREG32(GRBM_SOFT_RESET, 0);
1592         RREG32(GRBM_SOFT_RESET);
1593
1594         WREG32(CP_SEM_WAIT_TIMER, 0x0);
1595         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
1596
1597         /* Set the write pointer delay */
1598         WREG32(CP_RB_WPTR_DELAY, 0);
1599
1600         WREG32(CP_DEBUG, (1 << 27));
1601
1602         /* set the wb address whether it's enabled or not */
1603         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
1604         WREG32(SCRATCH_UMSK, 0xff);
1605
1606         for (i = 0; i < 3; ++i) {
1607                 uint32_t rb_cntl;
1608                 uint64_t addr;
1609
1610                 /* Set ring buffer size */
1611                 ring = &rdev->ring[ridx[i]];
1612                 rb_cntl = drm_order(ring->ring_size / 8);
1613                 rb_cntl |= drm_order(RADEON_GPU_PAGE_SIZE/8) << 8;
1614 #ifdef __BIG_ENDIAN
1615                 rb_cntl |= BUF_SWAP_32BIT;
1616 #endif
1617                 WREG32(cp_rb_cntl[i], rb_cntl);
1618
1619                 /* set the wb address whether it's enabled or not */
1620                 addr = rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET;
1621                 WREG32(cp_rb_rptr_addr[i], addr & 0xFFFFFFFC);
1622                 WREG32(cp_rb_rptr_addr_hi[i], upper_32_bits(addr) & 0xFF);
1623         }
1624
1625         /* set the rb base addr, this causes an internal reset of ALL rings */
1626         for (i = 0; i < 3; ++i) {
1627                 ring = &rdev->ring[ridx[i]];
1628                 WREG32(cp_rb_base[i], ring->gpu_addr >> 8);
1629         }
1630
1631         for (i = 0; i < 3; ++i) {
1632                 /* Initialize the ring buffer's read and write pointers */
1633                 ring = &rdev->ring[ridx[i]];
1634                 WREG32_P(cp_rb_cntl[i], RB_RPTR_WR_ENA, ~RB_RPTR_WR_ENA);
1635
1636                 ring->rptr = ring->wptr = 0;
1637                 WREG32(ring->rptr_reg, ring->rptr);
1638                 WREG32(ring->wptr_reg, ring->wptr);
1639
1640                 DRM_MDELAY(1);
1641                 WREG32_P(cp_rb_cntl[i], 0, ~RB_RPTR_WR_ENA);
1642         }
1643
1644         /* start the rings */
1645         cayman_cp_start(rdev);
1646         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
1647         rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1648         rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1649         /* this only test cp0 */
1650         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
1651         if (r) {
1652                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1653                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1654                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1655                 return r;
1656         }
1657
1658         return 0;
1659 }
1660
1661 /*
1662  * DMA
1663  * Starting with R600, the GPU has an asynchronous
1664  * DMA engine.  The programming model is very similar
1665  * to the 3D engine (ring buffer, IBs, etc.), but the
1666  * DMA controller has it's own packet format that is
1667  * different form the PM4 format used by the 3D engine.
1668  * It supports copying data, writing embedded data,
1669  * solid fills, and a number of other things.  It also
1670  * has support for tiling/detiling of buffers.
1671  * Cayman and newer support two asynchronous DMA engines.
1672  */
1673 /**
1674  * cayman_dma_ring_ib_execute - Schedule an IB on the DMA engine
1675  *
1676  * @rdev: radeon_device pointer
1677  * @ib: IB object to schedule
1678  *
1679  * Schedule an IB in the DMA ring (cayman-SI).
1680  */
1681 void cayman_dma_ring_ib_execute(struct radeon_device *rdev,
1682                                 struct radeon_ib *ib)
1683 {
1684         struct radeon_ring *ring = &rdev->ring[ib->ring];
1685
1686         if (rdev->wb.enabled) {
1687                 u32 next_rptr = ring->wptr + 4;
1688                 while ((next_rptr & 7) != 5)
1689                         next_rptr++;
1690                 next_rptr += 3;
1691                 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1));
1692                 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
1693                 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);
1694                 radeon_ring_write(ring, next_rptr);
1695         }
1696
1697         /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
1698          * Pad as necessary with NOPs.
1699          */
1700         while ((ring->wptr & 7) != 5)
1701                 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
1702         radeon_ring_write(ring, DMA_IB_PACKET(DMA_PACKET_INDIRECT_BUFFER, ib->vm ? ib->vm->id : 0, 0));
1703         radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
1704         radeon_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF));
1705
1706 }
1707
1708 /**
1709  * cayman_dma_stop - stop the async dma engines
1710  *
1711  * @rdev: radeon_device pointer
1712  *
1713  * Stop the async dma engines (cayman-SI).
1714  */
1715 void cayman_dma_stop(struct radeon_device *rdev)
1716 {
1717         u32 rb_cntl;
1718
1719         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1720
1721         /* dma0 */
1722         rb_cntl = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
1723         rb_cntl &= ~DMA_RB_ENABLE;
1724         WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, rb_cntl);
1725
1726         /* dma1 */
1727         rb_cntl = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
1728         rb_cntl &= ~DMA_RB_ENABLE;
1729         WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, rb_cntl);
1730
1731         rdev->ring[R600_RING_TYPE_DMA_INDEX].ready = false;
1732         rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX].ready = false;
1733 }
1734
1735 /**
1736  * cayman_dma_resume - setup and start the async dma engines
1737  *
1738  * @rdev: radeon_device pointer
1739  *
1740  * Set up the DMA ring buffers and enable them. (cayman-SI).
1741  * Returns 0 for success, error for failure.
1742  */
1743 int cayman_dma_resume(struct radeon_device *rdev)
1744 {
1745         struct radeon_ring *ring;
1746         u32 rb_cntl, dma_cntl, ib_cntl;
1747         u32 rb_bufsz;
1748         u32 reg_offset, wb_offset;
1749         int i, r;
1750
1751         /* Reset dma */
1752         WREG32(SRBM_SOFT_RESET, SOFT_RESET_DMA | SOFT_RESET_DMA1);
1753         RREG32(SRBM_SOFT_RESET);
1754         DRM_UDELAY(50);
1755         WREG32(SRBM_SOFT_RESET, 0);
1756
1757         for (i = 0; i < 2; i++) {
1758                 if (i == 0) {
1759                         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
1760                         reg_offset = DMA0_REGISTER_OFFSET;
1761                         wb_offset = R600_WB_DMA_RPTR_OFFSET;
1762                 } else {
1763                         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
1764                         reg_offset = DMA1_REGISTER_OFFSET;
1765                         wb_offset = CAYMAN_WB_DMA1_RPTR_OFFSET;
1766                 }
1767
1768                 WREG32(DMA_SEM_INCOMPLETE_TIMER_CNTL + reg_offset, 0);
1769                 WREG32(DMA_SEM_WAIT_FAIL_TIMER_CNTL + reg_offset, 0);
1770
1771                 /* Set ring buffer size in dwords */
1772                 rb_bufsz = drm_order(ring->ring_size / 4);
1773                 rb_cntl = rb_bufsz << 1;
1774 #ifdef __BIG_ENDIAN
1775                 rb_cntl |= DMA_RB_SWAP_ENABLE | DMA_RPTR_WRITEBACK_SWAP_ENABLE;
1776 #endif
1777                 WREG32(DMA_RB_CNTL + reg_offset, rb_cntl);
1778
1779                 /* Initialize the ring buffer's read and write pointers */
1780                 WREG32(DMA_RB_RPTR + reg_offset, 0);
1781                 WREG32(DMA_RB_WPTR + reg_offset, 0);
1782
1783                 /* set the wb address whether it's enabled or not */
1784                 WREG32(DMA_RB_RPTR_ADDR_HI + reg_offset,
1785                        upper_32_bits(rdev->wb.gpu_addr + wb_offset) & 0xFF);
1786                 WREG32(DMA_RB_RPTR_ADDR_LO + reg_offset,
1787                        ((rdev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC));
1788
1789                 if (rdev->wb.enabled)
1790                         rb_cntl |= DMA_RPTR_WRITEBACK_ENABLE;
1791
1792                 WREG32(DMA_RB_BASE + reg_offset, ring->gpu_addr >> 8);
1793
1794                 /* enable DMA IBs */
1795                 ib_cntl = DMA_IB_ENABLE | CMD_VMID_FORCE;
1796 #ifdef __BIG_ENDIAN
1797                 ib_cntl |= DMA_IB_SWAP_ENABLE;
1798 #endif
1799                 WREG32(DMA_IB_CNTL + reg_offset, ib_cntl);
1800
1801                 dma_cntl = RREG32(DMA_CNTL + reg_offset);
1802                 dma_cntl &= ~CTXEMPTY_INT_ENABLE;
1803                 WREG32(DMA_CNTL + reg_offset, dma_cntl);
1804
1805                 ring->wptr = 0;
1806                 WREG32(DMA_RB_WPTR + reg_offset, ring->wptr << 2);
1807
1808                 ring->rptr = RREG32(DMA_RB_RPTR + reg_offset) >> 2;
1809
1810                 WREG32(DMA_RB_CNTL + reg_offset, rb_cntl | DMA_RB_ENABLE);
1811
1812                 ring->ready = true;
1813
1814                 r = radeon_ring_test(rdev, ring->idx, ring);
1815                 if (r) {
1816                         ring->ready = false;
1817                         return r;
1818                 }
1819         }
1820
1821         radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
1822
1823         return 0;
1824 }
1825
1826 /**
1827  * cayman_dma_fini - tear down the async dma engines
1828  *
1829  * @rdev: radeon_device pointer
1830  *
1831  * Stop the async dma engines and free the rings (cayman-SI).
1832  */
1833 void cayman_dma_fini(struct radeon_device *rdev)
1834 {
1835         cayman_dma_stop(rdev);
1836         radeon_ring_fini(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX]);
1837         radeon_ring_fini(rdev, &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]);
1838 }
1839
1840 static u32 cayman_gpu_check_soft_reset(struct radeon_device *rdev)
1841 {
1842         u32 reset_mask = 0;
1843         u32 tmp;
1844
1845         /* GRBM_STATUS */
1846         tmp = RREG32(GRBM_STATUS);
1847         if (tmp & (PA_BUSY | SC_BUSY |
1848                    SH_BUSY | SX_BUSY |
1849                    TA_BUSY | VGT_BUSY |
1850                    DB_BUSY | CB_BUSY |
1851                    GDS_BUSY | SPI_BUSY |
1852                    IA_BUSY | IA_BUSY_NO_DMA))
1853                 reset_mask |= RADEON_RESET_GFX;
1854
1855         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
1856                    CP_BUSY | CP_COHERENCY_BUSY))
1857                 reset_mask |= RADEON_RESET_CP;
1858
1859         if (tmp & GRBM_EE_BUSY)
1860                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
1861
1862         /* DMA_STATUS_REG 0 */
1863         tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
1864         if (!(tmp & DMA_IDLE))
1865                 reset_mask |= RADEON_RESET_DMA;
1866
1867         /* DMA_STATUS_REG 1 */
1868         tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
1869         if (!(tmp & DMA_IDLE))
1870                 reset_mask |= RADEON_RESET_DMA1;
1871
1872         /* SRBM_STATUS2 */
1873         tmp = RREG32(SRBM_STATUS2);
1874         if (tmp & DMA_BUSY)
1875                 reset_mask |= RADEON_RESET_DMA;
1876
1877         if (tmp & DMA1_BUSY)
1878                 reset_mask |= RADEON_RESET_DMA1;
1879
1880         /* SRBM_STATUS */
1881         tmp = RREG32(SRBM_STATUS);
1882         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
1883                 reset_mask |= RADEON_RESET_RLC;
1884
1885         if (tmp & IH_BUSY)
1886                 reset_mask |= RADEON_RESET_IH;
1887
1888         if (tmp & SEM_BUSY)
1889                 reset_mask |= RADEON_RESET_SEM;
1890
1891         if (tmp & GRBM_RQ_PENDING)
1892                 reset_mask |= RADEON_RESET_GRBM;
1893
1894         if (tmp & VMC_BUSY)
1895                 reset_mask |= RADEON_RESET_VMC;
1896
1897         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
1898                    MCC_BUSY | MCD_BUSY))
1899                 reset_mask |= RADEON_RESET_MC;
1900
1901         if (evergreen_is_display_hung(rdev))
1902                 reset_mask |= RADEON_RESET_DISPLAY;
1903
1904         /* VM_L2_STATUS */
1905         tmp = RREG32(VM_L2_STATUS);
1906         if (tmp & L2_BUSY)
1907                 reset_mask |= RADEON_RESET_VMC;
1908
1909         /* Skip MC reset as it's mostly likely not hung, just busy */
1910         if (reset_mask & RADEON_RESET_MC) {
1911                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1912                 reset_mask &= ~RADEON_RESET_MC;
1913         }
1914
1915         return reset_mask;
1916 }
1917
1918 static void cayman_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1919 {
1920         struct evergreen_mc_save save;
1921         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1922         u32 tmp;
1923
1924         if (reset_mask == 0)
1925                 return;
1926
1927         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1928
1929         evergreen_print_gpu_status_regs(rdev);
1930         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_ADDR   0x%08X\n",
1931                  RREG32(0x14F8));
1932         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n",
1933                  RREG32(0x14D8));
1934         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
1935                  RREG32(0x14FC));
1936         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1937                  RREG32(0x14DC));
1938
1939         /* Disable CP parsing/prefetching */
1940         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
1941
1942         if (reset_mask & RADEON_RESET_DMA) {
1943                 /* dma0 */
1944                 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
1945                 tmp &= ~DMA_RB_ENABLE;
1946                 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
1947         }
1948
1949         if (reset_mask & RADEON_RESET_DMA1) {
1950                 /* dma1 */
1951                 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
1952                 tmp &= ~DMA_RB_ENABLE;
1953                 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
1954         }
1955
1956         DRM_UDELAY(50);
1957
1958         evergreen_mc_stop(rdev, &save);
1959         if (evergreen_mc_wait_for_idle(rdev)) {
1960                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1961         }
1962
1963         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
1964                 grbm_soft_reset = SOFT_RESET_CB |
1965                         SOFT_RESET_DB |
1966                         SOFT_RESET_GDS |
1967                         SOFT_RESET_PA |
1968                         SOFT_RESET_SC |
1969                         SOFT_RESET_SPI |
1970                         SOFT_RESET_SH |
1971                         SOFT_RESET_SX |
1972                         SOFT_RESET_TC |
1973                         SOFT_RESET_TA |
1974                         SOFT_RESET_VGT |
1975                         SOFT_RESET_IA;
1976         }
1977
1978         if (reset_mask & RADEON_RESET_CP) {
1979                 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
1980
1981                 srbm_soft_reset |= SOFT_RESET_GRBM;
1982         }
1983
1984         if (reset_mask & RADEON_RESET_DMA)
1985                 srbm_soft_reset |= SOFT_RESET_DMA;
1986
1987         if (reset_mask & RADEON_RESET_DMA1)
1988                 srbm_soft_reset |= SOFT_RESET_DMA1;
1989
1990         if (reset_mask & RADEON_RESET_DISPLAY)
1991                 srbm_soft_reset |= SOFT_RESET_DC;
1992
1993         if (reset_mask & RADEON_RESET_RLC)
1994                 srbm_soft_reset |= SOFT_RESET_RLC;
1995
1996         if (reset_mask & RADEON_RESET_SEM)
1997                 srbm_soft_reset |= SOFT_RESET_SEM;
1998
1999         if (reset_mask & RADEON_RESET_IH)
2000                 srbm_soft_reset |= SOFT_RESET_IH;
2001
2002         if (reset_mask & RADEON_RESET_GRBM)
2003                 srbm_soft_reset |= SOFT_RESET_GRBM;
2004
2005         if (reset_mask & RADEON_RESET_VMC)
2006                 srbm_soft_reset |= SOFT_RESET_VMC;
2007
2008         if (!(rdev->flags & RADEON_IS_IGP)) {
2009                 if (reset_mask & RADEON_RESET_MC)
2010                         srbm_soft_reset |= SOFT_RESET_MC;
2011         }
2012
2013         if (grbm_soft_reset) {
2014                 tmp = RREG32(GRBM_SOFT_RESET);
2015                 tmp |= grbm_soft_reset;
2016                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
2017                 WREG32(GRBM_SOFT_RESET, tmp);
2018                 tmp = RREG32(GRBM_SOFT_RESET);
2019
2020                 DRM_UDELAY(50);
2021
2022                 tmp &= ~grbm_soft_reset;
2023                 WREG32(GRBM_SOFT_RESET, tmp);
2024                 tmp = RREG32(GRBM_SOFT_RESET);
2025         }
2026
2027         if (srbm_soft_reset) {
2028                 tmp = RREG32(SRBM_SOFT_RESET);
2029                 tmp |= srbm_soft_reset;
2030                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
2031                 WREG32(SRBM_SOFT_RESET, tmp);
2032                 tmp = RREG32(SRBM_SOFT_RESET);
2033
2034                 DRM_UDELAY(50);
2035
2036                 tmp &= ~srbm_soft_reset;
2037                 WREG32(SRBM_SOFT_RESET, tmp);
2038                 tmp = RREG32(SRBM_SOFT_RESET);
2039         }
2040
2041         /* Wait a little for things to settle down */
2042         DRM_UDELAY(50);
2043
2044         evergreen_mc_resume(rdev, &save);
2045         DRM_UDELAY(50);
2046
2047         evergreen_print_gpu_status_regs(rdev);
2048 }
2049
2050 int cayman_asic_reset(struct radeon_device *rdev)
2051 {
2052         u32 reset_mask;
2053
2054         reset_mask = cayman_gpu_check_soft_reset(rdev);
2055
2056         if (reset_mask)
2057                 r600_set_bios_scratch_engine_hung(rdev, true);
2058
2059         cayman_gpu_soft_reset(rdev, reset_mask);
2060
2061         reset_mask = cayman_gpu_check_soft_reset(rdev);
2062
2063         if (!reset_mask)
2064                 r600_set_bios_scratch_engine_hung(rdev, false);
2065
2066         return 0;
2067 }
2068
2069 /**
2070  * cayman_gfx_is_lockup - Check if the GFX engine is locked up
2071  *
2072  * @rdev: radeon_device pointer
2073  * @ring: radeon_ring structure holding ring information
2074  *
2075  * Check if the GFX engine is locked up.
2076  * Returns true if the engine appears to be locked up, false if not.
2077  */
2078 bool cayman_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
2079 {
2080         u32 reset_mask = cayman_gpu_check_soft_reset(rdev);
2081
2082         if (!(reset_mask & (RADEON_RESET_GFX |
2083                             RADEON_RESET_COMPUTE |
2084                             RADEON_RESET_CP))) {
2085                 radeon_ring_lockup_update(ring);
2086                 return false;
2087         }
2088         /* force CP activities */
2089         radeon_ring_force_activity(rdev, ring);
2090         return radeon_ring_test_lockup(rdev, ring);
2091 }
2092
2093 /**
2094  * cayman_dma_is_lockup - Check if the DMA engine is locked up
2095  *
2096  * @rdev: radeon_device pointer
2097  * @ring: radeon_ring structure holding ring information
2098  *
2099  * Check if the async DMA engine is locked up.
2100  * Returns true if the engine appears to be locked up, false if not.
2101  */
2102 bool cayman_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
2103 {
2104         u32 reset_mask = cayman_gpu_check_soft_reset(rdev);
2105         u32 mask;
2106
2107         if (ring->idx == R600_RING_TYPE_DMA_INDEX)
2108                 mask = RADEON_RESET_DMA;
2109         else
2110                 mask = RADEON_RESET_DMA1;
2111
2112         if (!(reset_mask & mask)) {
2113                 radeon_ring_lockup_update(ring);
2114                 return false;
2115         }
2116         /* force ring activities */
2117         radeon_ring_force_activity(rdev, ring);
2118         return radeon_ring_test_lockup(rdev, ring);
2119 }
2120
2121 static int cayman_startup(struct radeon_device *rdev)
2122 {
2123         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2124         int r;
2125
2126         /* enable pcie gen2 link */
2127         evergreen_pcie_gen2_enable(rdev);
2128         /* enable aspm */
2129         evergreen_program_aspm(rdev);
2130
2131         evergreen_mc_program(rdev);
2132
2133         if (rdev->flags & RADEON_IS_IGP) {
2134                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
2135                         r = ni_init_microcode(rdev);
2136                         if (r) {
2137                                 DRM_ERROR("Failed to load firmware!\n");
2138                                 return r;
2139                         }
2140                 }
2141         } else {
2142                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
2143                         r = ni_init_microcode(rdev);
2144                         if (r) {
2145                                 DRM_ERROR("Failed to load firmware!\n");
2146                                 return r;
2147                         }
2148                 }
2149
2150                 r = ni_mc_load_microcode(rdev);
2151                 if (r) {
2152                         DRM_ERROR("Failed to load MC firmware!\n");
2153                         return r;
2154                 }
2155         }
2156
2157         r = r600_vram_scratch_init(rdev);
2158         if (r)
2159                 return r;
2160
2161         r = cayman_pcie_gart_enable(rdev);
2162         if (r)
2163                 return r;
2164         cayman_gpu_init(rdev);
2165
2166         r = evergreen_blit_init(rdev);
2167         if (r) {
2168                 r600_blit_fini(rdev);
2169                 rdev->asic->copy.copy = NULL;
2170                 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
2171         }
2172
2173         /* allocate rlc buffers */
2174         if (rdev->flags & RADEON_IS_IGP) {
2175                 rdev->rlc.reg_list = tn_rlc_save_restore_register_list;
2176                 rdev->rlc.reg_list_size = tn_rlc_save_restore_register_list_size;
2177                 rdev->rlc.cs_data = cayman_cs_data;
2178                 r = sumo_rlc_init(rdev);
2179                 if (r) {
2180                         DRM_ERROR("Failed to init rlc BOs!\n");
2181                         return r;
2182                 }
2183         }
2184
2185         /* allocate wb buffer */
2186         r = radeon_wb_init(rdev);
2187         if (r)
2188                 return r;
2189
2190         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
2191         if (r) {
2192                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2193                 return r;
2194         }
2195
2196         r = rv770_uvd_resume(rdev);
2197         if (!r) {
2198                 r = radeon_fence_driver_start_ring(rdev,
2199                                                    R600_RING_TYPE_UVD_INDEX);
2200                 if (r)
2201                         dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
2202         }
2203         if (r)
2204                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
2205
2206         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
2207         if (r) {
2208                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2209                 return r;
2210         }
2211
2212         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
2213         if (r) {
2214                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2215                 return r;
2216         }
2217
2218         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
2219         if (r) {
2220                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2221                 return r;
2222         }
2223
2224         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
2225         if (r) {
2226                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2227                 return r;
2228         }
2229
2230         /* Enable IRQ */
2231         if (!rdev->irq.installed) {
2232                 r = radeon_irq_kms_init(rdev);
2233                 if (r)
2234                         return r;
2235         }
2236
2237         r = r600_irq_init(rdev);
2238         if (r) {
2239                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
2240                 radeon_irq_kms_fini(rdev);
2241                 return r;
2242         }
2243         evergreen_irq_set(rdev);
2244
2245         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2246                              CP_RB0_RPTR, CP_RB0_WPTR,
2247                              0, 0xfffff, RADEON_CP_PACKET2);
2248         if (r)
2249                 return r;
2250
2251         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2252         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2253                              DMA_RB_RPTR + DMA0_REGISTER_OFFSET,
2254                              DMA_RB_WPTR + DMA0_REGISTER_OFFSET,
2255                              2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2256         if (r)
2257                 return r;
2258
2259         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2260         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2261                              DMA_RB_RPTR + DMA1_REGISTER_OFFSET,
2262                              DMA_RB_WPTR + DMA1_REGISTER_OFFSET,
2263                              2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2264         if (r)
2265                 return r;
2266
2267         r = cayman_cp_load_microcode(rdev);
2268         if (r)
2269                 return r;
2270         r = cayman_cp_resume(rdev);
2271         if (r)
2272                 return r;
2273
2274         r = cayman_dma_resume(rdev);
2275         if (r)
2276                 return r;
2277
2278         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2279         if (ring->ring_size) {
2280                 r = radeon_ring_init(rdev, ring, ring->ring_size,
2281                                      R600_WB_UVD_RPTR_OFFSET,
2282                                      UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
2283                                      0, 0xfffff, RADEON_CP_PACKET2);
2284                 if (!r)
2285                         r = r600_uvd_init(rdev);
2286                 if (r)
2287                         DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
2288         }
2289
2290         r = radeon_ib_pool_init(rdev);
2291         if (r) {
2292                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2293                 return r;
2294         }
2295
2296         r = radeon_vm_manager_init(rdev);
2297         if (r) {
2298                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
2299                 return r;
2300         }
2301
2302         r = r600_audio_init(rdev);
2303         if (r)
2304                 return r;
2305
2306         return 0;
2307 }
2308
2309 int cayman_resume(struct radeon_device *rdev)
2310 {
2311         int r;
2312
2313         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
2314          * posting will perform necessary task to bring back GPU into good
2315          * shape.
2316          */
2317         /* post card */
2318         atom_asic_init(rdev->mode_info.atom_context);
2319
2320         /* init golden registers */
2321         ni_init_golden_registers(rdev);
2322
2323         rdev->accel_working = true;
2324         r = cayman_startup(rdev);
2325         if (r) {
2326                 DRM_ERROR("cayman startup failed on resume\n");
2327                 rdev->accel_working = false;
2328                 return r;
2329         }
2330         return r;
2331 }
2332
2333 int cayman_suspend(struct radeon_device *rdev)
2334 {
2335         r600_audio_fini(rdev);
2336         radeon_vm_manager_fini(rdev);
2337         cayman_cp_enable(rdev, false);
2338         cayman_dma_stop(rdev);
2339         r600_uvd_stop(rdev);
2340         radeon_uvd_suspend(rdev);
2341         evergreen_irq_suspend(rdev);
2342         radeon_wb_disable(rdev);
2343         cayman_pcie_gart_disable(rdev);
2344         return 0;
2345 }
2346
2347 /* Plan is to move initialization in that function and use
2348  * helper function so that radeon_device_init pretty much
2349  * do nothing more than calling asic specific function. This
2350  * should also allow to remove a bunch of callback function
2351  * like vram_info.
2352  */
2353 int cayman_init(struct radeon_device *rdev)
2354 {
2355         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2356         int r;
2357
2358         /* Read BIOS */
2359         if (!radeon_get_bios(rdev)) {
2360                 if (ASIC_IS_AVIVO(rdev))
2361                         return -EINVAL;
2362         }
2363         /* Must be an ATOMBIOS */
2364         if (!rdev->is_atom_bios) {
2365                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
2366                 return -EINVAL;
2367         }
2368         r = radeon_atombios_init(rdev);
2369         if (r)
2370                 return r;
2371
2372         /* Post card if necessary */
2373         if (!radeon_card_posted(rdev)) {
2374                 if (!rdev->bios) {
2375                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
2376                         return -EINVAL;
2377                 }
2378                 DRM_INFO("GPU not posted. posting now...\n");
2379                 atom_asic_init(rdev->mode_info.atom_context);
2380         }
2381         /* init golden registers */
2382         ni_init_golden_registers(rdev);
2383         /* Initialize scratch registers */
2384         r600_scratch_init(rdev);
2385         /* Initialize surface registers */
2386         radeon_surface_init(rdev);
2387         /* Initialize clocks */
2388         radeon_get_clock_info(rdev->ddev);
2389         /* Fence driver */
2390         r = radeon_fence_driver_init(rdev);
2391         if (r)
2392                 return r;
2393         /* initialize memory controller */
2394         r = evergreen_mc_init(rdev);
2395         if (r)
2396                 return r;
2397         /* Memory manager */
2398         r = radeon_bo_init(rdev);
2399         if (r)
2400                 return r;
2401
2402         ring->ring_obj = NULL;
2403         r600_ring_init(rdev, ring, 1024 * 1024);
2404
2405         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2406         ring->ring_obj = NULL;
2407         r600_ring_init(rdev, ring, 64 * 1024);
2408
2409         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2410         ring->ring_obj = NULL;
2411         r600_ring_init(rdev, ring, 64 * 1024);
2412
2413         r = radeon_uvd_init(rdev);
2414         if (!r) {
2415                 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2416                 ring->ring_obj = NULL;
2417                 r600_ring_init(rdev, ring, 4096);
2418         }
2419
2420         rdev->ih.ring_obj = NULL;
2421         r600_ih_ring_init(rdev, 64 * 1024);
2422
2423         r = r600_pcie_gart_init(rdev);
2424         if (r)
2425                 return r;
2426
2427         rdev->accel_working = true;
2428         r = cayman_startup(rdev);
2429         if (r) {
2430                 dev_err(rdev->dev, "disabling GPU acceleration\n");
2431                 cayman_cp_fini(rdev);
2432                 cayman_dma_fini(rdev);
2433                 r600_irq_fini(rdev);
2434                 if (rdev->flags & RADEON_IS_IGP)
2435                         sumo_rlc_fini(rdev);
2436                 radeon_wb_fini(rdev);
2437                 radeon_ib_pool_fini(rdev);
2438                 radeon_vm_manager_fini(rdev);
2439                 radeon_irq_kms_fini(rdev);
2440                 cayman_pcie_gart_fini(rdev);
2441                 rdev->accel_working = false;
2442         }
2443
2444         /* Don't start up if the MC ucode is missing.
2445          * The default clocks and voltages before the MC ucode
2446          * is loaded are not suffient for advanced operations.
2447          *
2448          * We can skip this check for TN, because there is no MC
2449          * ucode.
2450          */
2451         if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
2452                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
2453                 return -EINVAL;
2454         }
2455
2456         return 0;
2457 }
2458
2459 void cayman_fini(struct radeon_device *rdev)
2460 {
2461         r600_blit_fini(rdev);
2462         cayman_cp_fini(rdev);
2463         cayman_dma_fini(rdev);
2464         r600_irq_fini(rdev);
2465         if (rdev->flags & RADEON_IS_IGP)
2466                 sumo_rlc_fini(rdev);
2467         radeon_wb_fini(rdev);
2468         radeon_vm_manager_fini(rdev);
2469         radeon_ib_pool_fini(rdev);
2470         radeon_irq_kms_fini(rdev);
2471         r600_uvd_stop(rdev);
2472         radeon_uvd_fini(rdev);
2473         cayman_pcie_gart_fini(rdev);
2474         r600_vram_scratch_fini(rdev);
2475         radeon_gem_fini(rdev);
2476         radeon_fence_driver_fini(rdev);
2477         radeon_bo_fini(rdev);
2478         radeon_atombios_fini(rdev);
2479         ni_fini_microcode(rdev);
2480         drm_free(rdev->bios, M_DRM);
2481         rdev->bios = NULL;
2482 }
2483
2484 /*
2485  * vm
2486  */
2487 int cayman_vm_init(struct radeon_device *rdev)
2488 {
2489         /* number of VMs */
2490         rdev->vm_manager.nvm = 8;
2491         /* base offset of vram pages */
2492         if (rdev->flags & RADEON_IS_IGP) {
2493                 u64 tmp = RREG32(FUS_MC_VM_FB_OFFSET);
2494                 tmp <<= 22;
2495                 rdev->vm_manager.vram_base_offset = tmp;
2496         } else
2497                 rdev->vm_manager.vram_base_offset = 0;
2498         return 0;
2499 }
2500
2501 void cayman_vm_fini(struct radeon_device *rdev)
2502 {
2503 }
2504
2505 /**
2506  * cayman_vm_decode_fault - print human readable fault info
2507  *
2508  * @rdev: radeon_device pointer
2509  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
2510  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
2511  *
2512  * Print human readable fault information (cayman/TN).
2513  */
2514 void cayman_vm_decode_fault(struct radeon_device *rdev,
2515                             u32 status, u32 addr)
2516 {
2517         u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
2518         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
2519         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
2520         char *block;
2521
2522         switch (mc_id) {
2523         case 32:
2524         case 16:
2525         case 96:
2526         case 80:
2527         case 160:
2528         case 144:
2529         case 224:
2530         case 208:
2531                 block = "CB";
2532                 break;
2533         case 33:
2534         case 17:
2535         case 97:
2536         case 81:
2537         case 161:
2538         case 145:
2539         case 225:
2540         case 209:
2541                 block = "CB_FMASK";
2542                 break;
2543         case 34:
2544         case 18:
2545         case 98:
2546         case 82:
2547         case 162:
2548         case 146:
2549         case 226:
2550         case 210:
2551                 block = "CB_CMASK";
2552                 break;
2553         case 35:
2554         case 19:
2555         case 99:
2556         case 83:
2557         case 163:
2558         case 147:
2559         case 227:
2560         case 211:
2561                 block = "CB_IMMED";
2562                 break;
2563         case 36:
2564         case 20:
2565         case 100:
2566         case 84:
2567         case 164:
2568         case 148:
2569         case 228:
2570         case 212:
2571                 block = "DB";
2572                 break;
2573         case 37:
2574         case 21:
2575         case 101:
2576         case 85:
2577         case 165:
2578         case 149:
2579         case 229:
2580         case 213:
2581                 block = "DB_HTILE";
2582                 break;
2583         case 38:
2584         case 22:
2585         case 102:
2586         case 86:
2587         case 166:
2588         case 150:
2589         case 230:
2590         case 214:
2591                 block = "SX";
2592                 break;
2593         case 39:
2594         case 23:
2595         case 103:
2596         case 87:
2597         case 167:
2598         case 151:
2599         case 231:
2600         case 215:
2601                 block = "DB_STEN";
2602                 break;
2603         case 40:
2604         case 24:
2605         case 104:
2606         case 88:
2607         case 232:
2608         case 216:
2609         case 168:
2610         case 152:
2611                 block = "TC_TFETCH";
2612                 break;
2613         case 41:
2614         case 25:
2615         case 105:
2616         case 89:
2617         case 233:
2618         case 217:
2619         case 169:
2620         case 153:
2621                 block = "TC_VFETCH";
2622                 break;
2623         case 42:
2624         case 26:
2625         case 106:
2626         case 90:
2627         case 234:
2628         case 218:
2629         case 170:
2630         case 154:
2631                 block = "VC";
2632                 break;
2633         case 112:
2634                 block = "CP";
2635                 break;
2636         case 113:
2637         case 114:
2638                 block = "SH";
2639                 break;
2640         case 115:
2641                 block = "VGT";
2642                 break;
2643         case 178:
2644                 block = "IH";
2645                 break;
2646         case 51:
2647                 block = "RLC";
2648                 break;
2649         case 55:
2650                 block = "DMA";
2651                 break;
2652         case 56:
2653                 block = "HDP";
2654                 break;
2655         default:
2656                 block = "unknown";
2657                 break;
2658         }
2659
2660         printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
2661                protections, vmid, addr,
2662                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
2663                block, mc_id);
2664 }
2665
2666 #define R600_ENTRY_VALID   (1 << 0)
2667 #define R600_PTE_SYSTEM    (1 << 1)
2668 #define R600_PTE_SNOOPED   (1 << 2)
2669 #define R600_PTE_READABLE  (1 << 5)
2670 #define R600_PTE_WRITEABLE (1 << 6)
2671
2672 uint32_t cayman_vm_page_flags(struct radeon_device *rdev, uint32_t flags)
2673 {
2674         uint32_t r600_flags = 0;
2675         r600_flags |= (flags & RADEON_VM_PAGE_VALID) ? R600_ENTRY_VALID : 0;
2676         r600_flags |= (flags & RADEON_VM_PAGE_READABLE) ? R600_PTE_READABLE : 0;
2677         r600_flags |= (flags & RADEON_VM_PAGE_WRITEABLE) ? R600_PTE_WRITEABLE : 0;
2678         if (flags & RADEON_VM_PAGE_SYSTEM) {
2679                 r600_flags |= R600_PTE_SYSTEM;
2680                 r600_flags |= (flags & RADEON_VM_PAGE_SNOOPED) ? R600_PTE_SNOOPED : 0;
2681         }
2682         return r600_flags;
2683 }
2684
2685 /**
2686  * cayman_vm_set_page - update the page tables using the CP
2687  *
2688  * @rdev: radeon_device pointer
2689  * @ib: indirect buffer to fill with commands
2690  * @pe: addr of the page entry
2691  * @addr: dst addr to write into pe
2692  * @count: number of page entries to update
2693  * @incr: increase next addr by incr bytes
2694  * @flags: access flags
2695  *
2696  * Update the page tables using the CP (cayman/TN).
2697  */
2698 void cayman_vm_set_page(struct radeon_device *rdev,
2699                         struct radeon_ib *ib,
2700                         uint64_t pe,
2701                         uint64_t addr, unsigned count,
2702                         uint32_t incr, uint32_t flags)
2703 {
2704         uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
2705         uint64_t value;
2706         unsigned ndw;
2707
2708         if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
2709                 while (count) {
2710                         ndw = 1 + count * 2;
2711                         if (ndw > 0x3FFF)
2712                                 ndw = 0x3FFF;
2713
2714                         ib->ptr[ib->length_dw++] = PACKET3(PACKET3_ME_WRITE, ndw);
2715                         ib->ptr[ib->length_dw++] = pe;
2716                         ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
2717                         for (; ndw > 1; ndw -= 2, --count, pe += 8) {
2718                                 if (flags & RADEON_VM_PAGE_SYSTEM) {
2719                                         value = radeon_vm_map_gart(rdev, addr);
2720                                         value &= 0xFFFFFFFFFFFFF000ULL;
2721                                 } else if (flags & RADEON_VM_PAGE_VALID) {
2722                                         value = addr;
2723                                 } else {
2724                                         value = 0;
2725                                 }
2726                                 addr += incr;
2727                                 value |= r600_flags;
2728                                 ib->ptr[ib->length_dw++] = value;
2729                                 ib->ptr[ib->length_dw++] = upper_32_bits(value);
2730                         }
2731                 }
2732         } else {
2733                 if ((flags & RADEON_VM_PAGE_SYSTEM) ||
2734                     (count == 1)) {
2735                         while (count) {
2736                                 ndw = count * 2;
2737                                 if (ndw > 0xFFFFE)
2738                                         ndw = 0xFFFFE;
2739
2740                                 /* for non-physically contiguous pages (system) */
2741                                 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, ndw);
2742                                 ib->ptr[ib->length_dw++] = pe;
2743                                 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
2744                                 for (; ndw > 0; ndw -= 2, --count, pe += 8) {
2745                                         if (flags & RADEON_VM_PAGE_SYSTEM) {
2746                                                 value = radeon_vm_map_gart(rdev, addr);
2747                                                 value &= 0xFFFFFFFFFFFFF000ULL;
2748                                         } else if (flags & RADEON_VM_PAGE_VALID) {
2749                                                 value = addr;
2750                                         } else {
2751                                                 value = 0;
2752                                         }
2753                                         addr += incr;
2754                                         value |= r600_flags;
2755                                         ib->ptr[ib->length_dw++] = value;
2756                                         ib->ptr[ib->length_dw++] = upper_32_bits(value);
2757                                 }
2758                         }
2759                         while (ib->length_dw & 0x7)
2760                                 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0);
2761                 } else {
2762                         while (count) {
2763                                 ndw = count * 2;
2764                                 if (ndw > 0xFFFFE)
2765                                         ndw = 0xFFFFE;
2766
2767                                 if (flags & RADEON_VM_PAGE_VALID)
2768                                         value = addr;
2769                                 else
2770                                         value = 0;
2771                                 /* for physically contiguous pages (vram) */
2772                                 ib->ptr[ib->length_dw++] = DMA_PTE_PDE_PACKET(ndw);
2773                                 ib->ptr[ib->length_dw++] = pe; /* dst addr */
2774                                 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
2775                                 ib->ptr[ib->length_dw++] = r600_flags; /* mask */
2776                                 ib->ptr[ib->length_dw++] = 0;
2777                                 ib->ptr[ib->length_dw++] = value; /* value */
2778                                 ib->ptr[ib->length_dw++] = upper_32_bits(value);
2779                                 ib->ptr[ib->length_dw++] = incr; /* increment size */
2780                                 ib->ptr[ib->length_dw++] = 0;
2781                                 pe += ndw * 4;
2782                                 addr += (ndw / 2) * incr;
2783                                 count -= ndw / 2;
2784                         }
2785                 }
2786                 while (ib->length_dw & 0x7)
2787                         ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0);
2788         }
2789 }
2790
2791 /**
2792  * cayman_vm_flush - vm flush using the CP
2793  *
2794  * @rdev: radeon_device pointer
2795  *
2796  * Update the page table base and flush the VM TLB
2797  * using the CP (cayman-si).
2798  */
2799 void cayman_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
2800 {
2801         struct radeon_ring *ring = &rdev->ring[ridx];
2802
2803         if (vm == NULL)
2804                 return;
2805
2806         radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2), 0));
2807         radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
2808
2809         /* flush hdp cache */
2810         radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
2811         radeon_ring_write(ring, 0x1);
2812
2813         /* bits 0-7 are the VM contexts0-7 */
2814         radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0));
2815         radeon_ring_write(ring, 1 << vm->id);
2816
2817         /* sync PFP to ME, otherwise we might get invalid PFP reads */
2818         radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
2819         radeon_ring_write(ring, 0x0);
2820 }
2821
2822 void cayman_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
2823 {
2824         struct radeon_ring *ring = &rdev->ring[ridx];
2825
2826         if (vm == NULL)
2827                 return;
2828
2829         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0));
2830         radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2));
2831         radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
2832
2833         /* flush hdp cache */
2834         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0));
2835         radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
2836         radeon_ring_write(ring, 1);
2837
2838         /* bits 0-7 are the VM contexts0-7 */
2839         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0));
2840         radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2));
2841         radeon_ring_write(ring, 1 << vm->id);
2842 }
2843