drm/linux: Improve request_firmware() compatibility
[dragonfly.git] / sys / dev / drm / radeon / si.c
1 /*
2  * Copyright 2011 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Alex Deucher
23  */
24 #include <linux/firmware.h>
25 #include <linux/module.h>
26 #include <drm/drmP.h>
27 #include "radeon.h"
28 #include "radeon_asic.h"
29 #include <uapi_drm/radeon_drm.h>
30 #include "sid.h"
31 #include "atom.h"
32 #include "si_blit_shaders.h"
33 #include "clearstate_si.h"
34 #include "radeon_ucode.h"
35
36
37 MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
38 MODULE_FIRMWARE("radeon/TAHITI_me.bin");
39 MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
40 MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
41 MODULE_FIRMWARE("radeon/TAHITI_mc2.bin");
42 MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
43 MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
44
45 MODULE_FIRMWARE("radeon/tahiti_pfp.bin");
46 MODULE_FIRMWARE("radeon/tahiti_me.bin");
47 MODULE_FIRMWARE("radeon/tahiti_ce.bin");
48 MODULE_FIRMWARE("radeon/tahiti_mc.bin");
49 MODULE_FIRMWARE("radeon/tahiti_rlc.bin");
50 MODULE_FIRMWARE("radeon/tahiti_smc.bin");
51
52 MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
53 MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
54 MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
55 MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
56 MODULE_FIRMWARE("radeon/PITCAIRN_mc2.bin");
57 MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
58 MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
59
60 MODULE_FIRMWARE("radeon/pitcairn_pfp.bin");
61 MODULE_FIRMWARE("radeon/pitcairn_me.bin");
62 MODULE_FIRMWARE("radeon/pitcairn_ce.bin");
63 MODULE_FIRMWARE("radeon/pitcairn_mc.bin");
64 MODULE_FIRMWARE("radeon/pitcairn_rlc.bin");
65 MODULE_FIRMWARE("radeon/pitcairn_smc.bin");
66
67 MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
68 MODULE_FIRMWARE("radeon/VERDE_me.bin");
69 MODULE_FIRMWARE("radeon/VERDE_ce.bin");
70 MODULE_FIRMWARE("radeon/VERDE_mc.bin");
71 MODULE_FIRMWARE("radeon/VERDE_mc2.bin");
72 MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
73 MODULE_FIRMWARE("radeon/VERDE_smc.bin");
74
75 MODULE_FIRMWARE("radeon/verde_pfp.bin");
76 MODULE_FIRMWARE("radeon/verde_me.bin");
77 MODULE_FIRMWARE("radeon/verde_ce.bin");
78 MODULE_FIRMWARE("radeon/verde_mc.bin");
79 MODULE_FIRMWARE("radeon/verde_rlc.bin");
80 MODULE_FIRMWARE("radeon/verde_smc.bin");
81
82 MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
83 MODULE_FIRMWARE("radeon/OLAND_me.bin");
84 MODULE_FIRMWARE("radeon/OLAND_ce.bin");
85 MODULE_FIRMWARE("radeon/OLAND_mc.bin");
86 MODULE_FIRMWARE("radeon/OLAND_mc2.bin");
87 MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
88 MODULE_FIRMWARE("radeon/OLAND_smc.bin");
89
90 MODULE_FIRMWARE("radeon/oland_pfp.bin");
91 MODULE_FIRMWARE("radeon/oland_me.bin");
92 MODULE_FIRMWARE("radeon/oland_ce.bin");
93 MODULE_FIRMWARE("radeon/oland_mc.bin");
94 MODULE_FIRMWARE("radeon/oland_rlc.bin");
95 MODULE_FIRMWARE("radeon/oland_smc.bin");
96
97 MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
98 MODULE_FIRMWARE("radeon/HAINAN_me.bin");
99 MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
100 MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
101 MODULE_FIRMWARE("radeon/HAINAN_mc2.bin");
102 MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
103 MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
104
105 MODULE_FIRMWARE("radeon/hainan_pfp.bin");
106 MODULE_FIRMWARE("radeon/hainan_me.bin");
107 MODULE_FIRMWARE("radeon/hainan_ce.bin");
108 MODULE_FIRMWARE("radeon/hainan_mc.bin");
109 MODULE_FIRMWARE("radeon/hainan_rlc.bin");
110 MODULE_FIRMWARE("radeon/hainan_smc.bin");
111
112 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
113 static void si_pcie_gen3_enable(struct radeon_device *rdev);
114 static void si_program_aspm(struct radeon_device *rdev);
115 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
116                                          bool enable);
117 static void si_init_pg(struct radeon_device *rdev);
118 static void si_init_cg(struct radeon_device *rdev);
119 static void si_fini_pg(struct radeon_device *rdev);
120 static void si_fini_cg(struct radeon_device *rdev);
121 static void si_rlc_stop(struct radeon_device *rdev);
122
123 static const u32 verde_rlc_save_restore_register_list[] =
124 {
125         (0x8000 << 16) | (0x98f4 >> 2),
126         0x00000000,
127         (0x8040 << 16) | (0x98f4 >> 2),
128         0x00000000,
129         (0x8000 << 16) | (0xe80 >> 2),
130         0x00000000,
131         (0x8040 << 16) | (0xe80 >> 2),
132         0x00000000,
133         (0x8000 << 16) | (0x89bc >> 2),
134         0x00000000,
135         (0x8040 << 16) | (0x89bc >> 2),
136         0x00000000,
137         (0x8000 << 16) | (0x8c1c >> 2),
138         0x00000000,
139         (0x8040 << 16) | (0x8c1c >> 2),
140         0x00000000,
141         (0x9c00 << 16) | (0x98f0 >> 2),
142         0x00000000,
143         (0x9c00 << 16) | (0xe7c >> 2),
144         0x00000000,
145         (0x8000 << 16) | (0x9148 >> 2),
146         0x00000000,
147         (0x8040 << 16) | (0x9148 >> 2),
148         0x00000000,
149         (0x9c00 << 16) | (0x9150 >> 2),
150         0x00000000,
151         (0x9c00 << 16) | (0x897c >> 2),
152         0x00000000,
153         (0x9c00 << 16) | (0x8d8c >> 2),
154         0x00000000,
155         (0x9c00 << 16) | (0xac54 >> 2),
156         0X00000000,
157         0x3,
158         (0x9c00 << 16) | (0x98f8 >> 2),
159         0x00000000,
160         (0x9c00 << 16) | (0x9910 >> 2),
161         0x00000000,
162         (0x9c00 << 16) | (0x9914 >> 2),
163         0x00000000,
164         (0x9c00 << 16) | (0x9918 >> 2),
165         0x00000000,
166         (0x9c00 << 16) | (0x991c >> 2),
167         0x00000000,
168         (0x9c00 << 16) | (0x9920 >> 2),
169         0x00000000,
170         (0x9c00 << 16) | (0x9924 >> 2),
171         0x00000000,
172         (0x9c00 << 16) | (0x9928 >> 2),
173         0x00000000,
174         (0x9c00 << 16) | (0x992c >> 2),
175         0x00000000,
176         (0x9c00 << 16) | (0x9930 >> 2),
177         0x00000000,
178         (0x9c00 << 16) | (0x9934 >> 2),
179         0x00000000,
180         (0x9c00 << 16) | (0x9938 >> 2),
181         0x00000000,
182         (0x9c00 << 16) | (0x993c >> 2),
183         0x00000000,
184         (0x9c00 << 16) | (0x9940 >> 2),
185         0x00000000,
186         (0x9c00 << 16) | (0x9944 >> 2),
187         0x00000000,
188         (0x9c00 << 16) | (0x9948 >> 2),
189         0x00000000,
190         (0x9c00 << 16) | (0x994c >> 2),
191         0x00000000,
192         (0x9c00 << 16) | (0x9950 >> 2),
193         0x00000000,
194         (0x9c00 << 16) | (0x9954 >> 2),
195         0x00000000,
196         (0x9c00 << 16) | (0x9958 >> 2),
197         0x00000000,
198         (0x9c00 << 16) | (0x995c >> 2),
199         0x00000000,
200         (0x9c00 << 16) | (0x9960 >> 2),
201         0x00000000,
202         (0x9c00 << 16) | (0x9964 >> 2),
203         0x00000000,
204         (0x9c00 << 16) | (0x9968 >> 2),
205         0x00000000,
206         (0x9c00 << 16) | (0x996c >> 2),
207         0x00000000,
208         (0x9c00 << 16) | (0x9970 >> 2),
209         0x00000000,
210         (0x9c00 << 16) | (0x9974 >> 2),
211         0x00000000,
212         (0x9c00 << 16) | (0x9978 >> 2),
213         0x00000000,
214         (0x9c00 << 16) | (0x997c >> 2),
215         0x00000000,
216         (0x9c00 << 16) | (0x9980 >> 2),
217         0x00000000,
218         (0x9c00 << 16) | (0x9984 >> 2),
219         0x00000000,
220         (0x9c00 << 16) | (0x9988 >> 2),
221         0x00000000,
222         (0x9c00 << 16) | (0x998c >> 2),
223         0x00000000,
224         (0x9c00 << 16) | (0x8c00 >> 2),
225         0x00000000,
226         (0x9c00 << 16) | (0x8c14 >> 2),
227         0x00000000,
228         (0x9c00 << 16) | (0x8c04 >> 2),
229         0x00000000,
230         (0x9c00 << 16) | (0x8c08 >> 2),
231         0x00000000,
232         (0x8000 << 16) | (0x9b7c >> 2),
233         0x00000000,
234         (0x8040 << 16) | (0x9b7c >> 2),
235         0x00000000,
236         (0x8000 << 16) | (0xe84 >> 2),
237         0x00000000,
238         (0x8040 << 16) | (0xe84 >> 2),
239         0x00000000,
240         (0x8000 << 16) | (0x89c0 >> 2),
241         0x00000000,
242         (0x8040 << 16) | (0x89c0 >> 2),
243         0x00000000,
244         (0x8000 << 16) | (0x914c >> 2),
245         0x00000000,
246         (0x8040 << 16) | (0x914c >> 2),
247         0x00000000,
248         (0x8000 << 16) | (0x8c20 >> 2),
249         0x00000000,
250         (0x8040 << 16) | (0x8c20 >> 2),
251         0x00000000,
252         (0x8000 << 16) | (0x9354 >> 2),
253         0x00000000,
254         (0x8040 << 16) | (0x9354 >> 2),
255         0x00000000,
256         (0x9c00 << 16) | (0x9060 >> 2),
257         0x00000000,
258         (0x9c00 << 16) | (0x9364 >> 2),
259         0x00000000,
260         (0x9c00 << 16) | (0x9100 >> 2),
261         0x00000000,
262         (0x9c00 << 16) | (0x913c >> 2),
263         0x00000000,
264         (0x8000 << 16) | (0x90e0 >> 2),
265         0x00000000,
266         (0x8000 << 16) | (0x90e4 >> 2),
267         0x00000000,
268         (0x8000 << 16) | (0x90e8 >> 2),
269         0x00000000,
270         (0x8040 << 16) | (0x90e0 >> 2),
271         0x00000000,
272         (0x8040 << 16) | (0x90e4 >> 2),
273         0x00000000,
274         (0x8040 << 16) | (0x90e8 >> 2),
275         0x00000000,
276         (0x9c00 << 16) | (0x8bcc >> 2),
277         0x00000000,
278         (0x9c00 << 16) | (0x8b24 >> 2),
279         0x00000000,
280         (0x9c00 << 16) | (0x88c4 >> 2),
281         0x00000000,
282         (0x9c00 << 16) | (0x8e50 >> 2),
283         0x00000000,
284         (0x9c00 << 16) | (0x8c0c >> 2),
285         0x00000000,
286         (0x9c00 << 16) | (0x8e58 >> 2),
287         0x00000000,
288         (0x9c00 << 16) | (0x8e5c >> 2),
289         0x00000000,
290         (0x9c00 << 16) | (0x9508 >> 2),
291         0x00000000,
292         (0x9c00 << 16) | (0x950c >> 2),
293         0x00000000,
294         (0x9c00 << 16) | (0x9494 >> 2),
295         0x00000000,
296         (0x9c00 << 16) | (0xac0c >> 2),
297         0x00000000,
298         (0x9c00 << 16) | (0xac10 >> 2),
299         0x00000000,
300         (0x9c00 << 16) | (0xac14 >> 2),
301         0x00000000,
302         (0x9c00 << 16) | (0xae00 >> 2),
303         0x00000000,
304         (0x9c00 << 16) | (0xac08 >> 2),
305         0x00000000,
306         (0x9c00 << 16) | (0x88d4 >> 2),
307         0x00000000,
308         (0x9c00 << 16) | (0x88c8 >> 2),
309         0x00000000,
310         (0x9c00 << 16) | (0x88cc >> 2),
311         0x00000000,
312         (0x9c00 << 16) | (0x89b0 >> 2),
313         0x00000000,
314         (0x9c00 << 16) | (0x8b10 >> 2),
315         0x00000000,
316         (0x9c00 << 16) | (0x8a14 >> 2),
317         0x00000000,
318         (0x9c00 << 16) | (0x9830 >> 2),
319         0x00000000,
320         (0x9c00 << 16) | (0x9834 >> 2),
321         0x00000000,
322         (0x9c00 << 16) | (0x9838 >> 2),
323         0x00000000,
324         (0x9c00 << 16) | (0x9a10 >> 2),
325         0x00000000,
326         (0x8000 << 16) | (0x9870 >> 2),
327         0x00000000,
328         (0x8000 << 16) | (0x9874 >> 2),
329         0x00000000,
330         (0x8001 << 16) | (0x9870 >> 2),
331         0x00000000,
332         (0x8001 << 16) | (0x9874 >> 2),
333         0x00000000,
334         (0x8040 << 16) | (0x9870 >> 2),
335         0x00000000,
336         (0x8040 << 16) | (0x9874 >> 2),
337         0x00000000,
338         (0x8041 << 16) | (0x9870 >> 2),
339         0x00000000,
340         (0x8041 << 16) | (0x9874 >> 2),
341         0x00000000,
342         0x00000000
343 };
344
345 static const u32 tahiti_golden_rlc_registers[] =
346 {
347         0xc424, 0xffffffff, 0x00601005,
348         0xc47c, 0xffffffff, 0x10104040,
349         0xc488, 0xffffffff, 0x0100000a,
350         0xc314, 0xffffffff, 0x00000800,
351         0xc30c, 0xffffffff, 0x800000f4,
352         0xf4a8, 0xffffffff, 0x00000000
353 };
354
355 static const u32 tahiti_golden_registers[] =
356 {
357         0x9a10, 0x00010000, 0x00018208,
358         0x9830, 0xffffffff, 0x00000000,
359         0x9834, 0xf00fffff, 0x00000400,
360         0x9838, 0x0002021c, 0x00020200,
361         0xc78, 0x00000080, 0x00000000,
362         0xd030, 0x000300c0, 0x00800040,
363         0xd830, 0x000300c0, 0x00800040,
364         0x5bb0, 0x000000f0, 0x00000070,
365         0x5bc0, 0x00200000, 0x50100000,
366         0x7030, 0x31000311, 0x00000011,
367         0x277c, 0x00000003, 0x000007ff,
368         0x240c, 0x000007ff, 0x00000000,
369         0x8a14, 0xf000001f, 0x00000007,
370         0x8b24, 0xffffffff, 0x00ffffff,
371         0x8b10, 0x0000ff0f, 0x00000000,
372         0x28a4c, 0x07ffffff, 0x4e000000,
373         0x28350, 0x3f3f3fff, 0x2a00126a,
374         0x30, 0x000000ff, 0x0040,
375         0x34, 0x00000040, 0x00004040,
376         0x9100, 0x07ffffff, 0x03000000,
377         0x8e88, 0x01ff1f3f, 0x00000000,
378         0x8e84, 0x01ff1f3f, 0x00000000,
379         0x9060, 0x0000007f, 0x00000020,
380         0x9508, 0x00010000, 0x00010000,
381         0xac14, 0x00000200, 0x000002fb,
382         0xac10, 0xffffffff, 0x0000543b,
383         0xac0c, 0xffffffff, 0xa9210876,
384         0x88d0, 0xffffffff, 0x000fff40,
385         0x88d4, 0x0000001f, 0x00000010,
386         0x1410, 0x20000000, 0x20fffed8,
387         0x15c0, 0x000c0fc0, 0x000c0400
388 };
389
390 static const u32 tahiti_golden_registers2[] =
391 {
392         0xc64, 0x00000001, 0x00000001
393 };
394
395 static const u32 pitcairn_golden_rlc_registers[] =
396 {
397         0xc424, 0xffffffff, 0x00601004,
398         0xc47c, 0xffffffff, 0x10102020,
399         0xc488, 0xffffffff, 0x01000020,
400         0xc314, 0xffffffff, 0x00000800,
401         0xc30c, 0xffffffff, 0x800000a4
402 };
403
404 static const u32 pitcairn_golden_registers[] =
405 {
406         0x9a10, 0x00010000, 0x00018208,
407         0x9830, 0xffffffff, 0x00000000,
408         0x9834, 0xf00fffff, 0x00000400,
409         0x9838, 0x0002021c, 0x00020200,
410         0xc78, 0x00000080, 0x00000000,
411         0xd030, 0x000300c0, 0x00800040,
412         0xd830, 0x000300c0, 0x00800040,
413         0x5bb0, 0x000000f0, 0x00000070,
414         0x5bc0, 0x00200000, 0x50100000,
415         0x7030, 0x31000311, 0x00000011,
416         0x2ae4, 0x00073ffe, 0x000022a2,
417         0x240c, 0x000007ff, 0x00000000,
418         0x8a14, 0xf000001f, 0x00000007,
419         0x8b24, 0xffffffff, 0x00ffffff,
420         0x8b10, 0x0000ff0f, 0x00000000,
421         0x28a4c, 0x07ffffff, 0x4e000000,
422         0x28350, 0x3f3f3fff, 0x2a00126a,
423         0x30, 0x000000ff, 0x0040,
424         0x34, 0x00000040, 0x00004040,
425         0x9100, 0x07ffffff, 0x03000000,
426         0x9060, 0x0000007f, 0x00000020,
427         0x9508, 0x00010000, 0x00010000,
428         0xac14, 0x000003ff, 0x000000f7,
429         0xac10, 0xffffffff, 0x00000000,
430         0xac0c, 0xffffffff, 0x32761054,
431         0x88d4, 0x0000001f, 0x00000010,
432         0x15c0, 0x000c0fc0, 0x000c0400
433 };
434
435 static const u32 verde_golden_rlc_registers[] =
436 {
437         0xc424, 0xffffffff, 0x033f1005,
438         0xc47c, 0xffffffff, 0x10808020,
439         0xc488, 0xffffffff, 0x00800008,
440         0xc314, 0xffffffff, 0x00001000,
441         0xc30c, 0xffffffff, 0x80010014
442 };
443
444 static const u32 verde_golden_registers[] =
445 {
446         0x9a10, 0x00010000, 0x00018208,
447         0x9830, 0xffffffff, 0x00000000,
448         0x9834, 0xf00fffff, 0x00000400,
449         0x9838, 0x0002021c, 0x00020200,
450         0xc78, 0x00000080, 0x00000000,
451         0xd030, 0x000300c0, 0x00800040,
452         0xd030, 0x000300c0, 0x00800040,
453         0xd830, 0x000300c0, 0x00800040,
454         0xd830, 0x000300c0, 0x00800040,
455         0x5bb0, 0x000000f0, 0x00000070,
456         0x5bc0, 0x00200000, 0x50100000,
457         0x7030, 0x31000311, 0x00000011,
458         0x2ae4, 0x00073ffe, 0x000022a2,
459         0x2ae4, 0x00073ffe, 0x000022a2,
460         0x2ae4, 0x00073ffe, 0x000022a2,
461         0x240c, 0x000007ff, 0x00000000,
462         0x240c, 0x000007ff, 0x00000000,
463         0x240c, 0x000007ff, 0x00000000,
464         0x8a14, 0xf000001f, 0x00000007,
465         0x8a14, 0xf000001f, 0x00000007,
466         0x8a14, 0xf000001f, 0x00000007,
467         0x8b24, 0xffffffff, 0x00ffffff,
468         0x8b10, 0x0000ff0f, 0x00000000,
469         0x28a4c, 0x07ffffff, 0x4e000000,
470         0x28350, 0x3f3f3fff, 0x0000124a,
471         0x28350, 0x3f3f3fff, 0x0000124a,
472         0x28350, 0x3f3f3fff, 0x0000124a,
473         0x30, 0x000000ff, 0x0040,
474         0x34, 0x00000040, 0x00004040,
475         0x9100, 0x07ffffff, 0x03000000,
476         0x9100, 0x07ffffff, 0x03000000,
477         0x8e88, 0x01ff1f3f, 0x00000000,
478         0x8e88, 0x01ff1f3f, 0x00000000,
479         0x8e88, 0x01ff1f3f, 0x00000000,
480         0x8e84, 0x01ff1f3f, 0x00000000,
481         0x8e84, 0x01ff1f3f, 0x00000000,
482         0x8e84, 0x01ff1f3f, 0x00000000,
483         0x9060, 0x0000007f, 0x00000020,
484         0x9508, 0x00010000, 0x00010000,
485         0xac14, 0x000003ff, 0x00000003,
486         0xac14, 0x000003ff, 0x00000003,
487         0xac14, 0x000003ff, 0x00000003,
488         0xac10, 0xffffffff, 0x00000000,
489         0xac10, 0xffffffff, 0x00000000,
490         0xac10, 0xffffffff, 0x00000000,
491         0xac0c, 0xffffffff, 0x00001032,
492         0xac0c, 0xffffffff, 0x00001032,
493         0xac0c, 0xffffffff, 0x00001032,
494         0x88d4, 0x0000001f, 0x00000010,
495         0x88d4, 0x0000001f, 0x00000010,
496         0x88d4, 0x0000001f, 0x00000010,
497         0x15c0, 0x000c0fc0, 0x000c0400
498 };
499
500 static const u32 oland_golden_rlc_registers[] =
501 {
502         0xc424, 0xffffffff, 0x00601005,
503         0xc47c, 0xffffffff, 0x10104040,
504         0xc488, 0xffffffff, 0x0100000a,
505         0xc314, 0xffffffff, 0x00000800,
506         0xc30c, 0xffffffff, 0x800000f4
507 };
508
509 static const u32 oland_golden_registers[] =
510 {
511         0x9a10, 0x00010000, 0x00018208,
512         0x9830, 0xffffffff, 0x00000000,
513         0x9834, 0xf00fffff, 0x00000400,
514         0x9838, 0x0002021c, 0x00020200,
515         0xc78, 0x00000080, 0x00000000,
516         0xd030, 0x000300c0, 0x00800040,
517         0xd830, 0x000300c0, 0x00800040,
518         0x5bb0, 0x000000f0, 0x00000070,
519         0x5bc0, 0x00200000, 0x50100000,
520         0x7030, 0x31000311, 0x00000011,
521         0x2ae4, 0x00073ffe, 0x000022a2,
522         0x240c, 0x000007ff, 0x00000000,
523         0x8a14, 0xf000001f, 0x00000007,
524         0x8b24, 0xffffffff, 0x00ffffff,
525         0x8b10, 0x0000ff0f, 0x00000000,
526         0x28a4c, 0x07ffffff, 0x4e000000,
527         0x28350, 0x3f3f3fff, 0x00000082,
528         0x30, 0x000000ff, 0x0040,
529         0x34, 0x00000040, 0x00004040,
530         0x9100, 0x07ffffff, 0x03000000,
531         0x9060, 0x0000007f, 0x00000020,
532         0x9508, 0x00010000, 0x00010000,
533         0xac14, 0x000003ff, 0x000000f3,
534         0xac10, 0xffffffff, 0x00000000,
535         0xac0c, 0xffffffff, 0x00003210,
536         0x88d4, 0x0000001f, 0x00000010,
537         0x15c0, 0x000c0fc0, 0x000c0400
538 };
539
540 static const u32 hainan_golden_registers[] =
541 {
542         0x9a10, 0x00010000, 0x00018208,
543         0x9830, 0xffffffff, 0x00000000,
544         0x9834, 0xf00fffff, 0x00000400,
545         0x9838, 0x0002021c, 0x00020200,
546         0xd0c0, 0xff000fff, 0x00000100,
547         0xd030, 0x000300c0, 0x00800040,
548         0xd8c0, 0xff000fff, 0x00000100,
549         0xd830, 0x000300c0, 0x00800040,
550         0x2ae4, 0x00073ffe, 0x000022a2,
551         0x240c, 0x000007ff, 0x00000000,
552         0x8a14, 0xf000001f, 0x00000007,
553         0x8b24, 0xffffffff, 0x00ffffff,
554         0x8b10, 0x0000ff0f, 0x00000000,
555         0x28a4c, 0x07ffffff, 0x4e000000,
556         0x28350, 0x3f3f3fff, 0x00000000,
557         0x30, 0x000000ff, 0x0040,
558         0x34, 0x00000040, 0x00004040,
559         0x9100, 0x03e00000, 0x03600000,
560         0x9060, 0x0000007f, 0x00000020,
561         0x9508, 0x00010000, 0x00010000,
562         0xac14, 0x000003ff, 0x000000f1,
563         0xac10, 0xffffffff, 0x00000000,
564         0xac0c, 0xffffffff, 0x00003210,
565         0x88d4, 0x0000001f, 0x00000010,
566         0x15c0, 0x000c0fc0, 0x000c0400
567 };
568
569 static const u32 hainan_golden_registers2[] =
570 {
571         0x98f8, 0xffffffff, 0x02010001
572 };
573
574 static const u32 tahiti_mgcg_cgcg_init[] =
575 {
576         0xc400, 0xffffffff, 0xfffffffc,
577         0x802c, 0xffffffff, 0xe0000000,
578         0x9a60, 0xffffffff, 0x00000100,
579         0x92a4, 0xffffffff, 0x00000100,
580         0xc164, 0xffffffff, 0x00000100,
581         0x9774, 0xffffffff, 0x00000100,
582         0x8984, 0xffffffff, 0x06000100,
583         0x8a18, 0xffffffff, 0x00000100,
584         0x92a0, 0xffffffff, 0x00000100,
585         0xc380, 0xffffffff, 0x00000100,
586         0x8b28, 0xffffffff, 0x00000100,
587         0x9144, 0xffffffff, 0x00000100,
588         0x8d88, 0xffffffff, 0x00000100,
589         0x8d8c, 0xffffffff, 0x00000100,
590         0x9030, 0xffffffff, 0x00000100,
591         0x9034, 0xffffffff, 0x00000100,
592         0x9038, 0xffffffff, 0x00000100,
593         0x903c, 0xffffffff, 0x00000100,
594         0xad80, 0xffffffff, 0x00000100,
595         0xac54, 0xffffffff, 0x00000100,
596         0x897c, 0xffffffff, 0x06000100,
597         0x9868, 0xffffffff, 0x00000100,
598         0x9510, 0xffffffff, 0x00000100,
599         0xaf04, 0xffffffff, 0x00000100,
600         0xae04, 0xffffffff, 0x00000100,
601         0x949c, 0xffffffff, 0x00000100,
602         0x802c, 0xffffffff, 0xe0000000,
603         0x9160, 0xffffffff, 0x00010000,
604         0x9164, 0xffffffff, 0x00030002,
605         0x9168, 0xffffffff, 0x00040007,
606         0x916c, 0xffffffff, 0x00060005,
607         0x9170, 0xffffffff, 0x00090008,
608         0x9174, 0xffffffff, 0x00020001,
609         0x9178, 0xffffffff, 0x00040003,
610         0x917c, 0xffffffff, 0x00000007,
611         0x9180, 0xffffffff, 0x00060005,
612         0x9184, 0xffffffff, 0x00090008,
613         0x9188, 0xffffffff, 0x00030002,
614         0x918c, 0xffffffff, 0x00050004,
615         0x9190, 0xffffffff, 0x00000008,
616         0x9194, 0xffffffff, 0x00070006,
617         0x9198, 0xffffffff, 0x000a0009,
618         0x919c, 0xffffffff, 0x00040003,
619         0x91a0, 0xffffffff, 0x00060005,
620         0x91a4, 0xffffffff, 0x00000009,
621         0x91a8, 0xffffffff, 0x00080007,
622         0x91ac, 0xffffffff, 0x000b000a,
623         0x91b0, 0xffffffff, 0x00050004,
624         0x91b4, 0xffffffff, 0x00070006,
625         0x91b8, 0xffffffff, 0x0008000b,
626         0x91bc, 0xffffffff, 0x000a0009,
627         0x91c0, 0xffffffff, 0x000d000c,
628         0x91c4, 0xffffffff, 0x00060005,
629         0x91c8, 0xffffffff, 0x00080007,
630         0x91cc, 0xffffffff, 0x0000000b,
631         0x91d0, 0xffffffff, 0x000a0009,
632         0x91d4, 0xffffffff, 0x000d000c,
633         0x91d8, 0xffffffff, 0x00070006,
634         0x91dc, 0xffffffff, 0x00090008,
635         0x91e0, 0xffffffff, 0x0000000c,
636         0x91e4, 0xffffffff, 0x000b000a,
637         0x91e8, 0xffffffff, 0x000e000d,
638         0x91ec, 0xffffffff, 0x00080007,
639         0x91f0, 0xffffffff, 0x000a0009,
640         0x91f4, 0xffffffff, 0x0000000d,
641         0x91f8, 0xffffffff, 0x000c000b,
642         0x91fc, 0xffffffff, 0x000f000e,
643         0x9200, 0xffffffff, 0x00090008,
644         0x9204, 0xffffffff, 0x000b000a,
645         0x9208, 0xffffffff, 0x000c000f,
646         0x920c, 0xffffffff, 0x000e000d,
647         0x9210, 0xffffffff, 0x00110010,
648         0x9214, 0xffffffff, 0x000a0009,
649         0x9218, 0xffffffff, 0x000c000b,
650         0x921c, 0xffffffff, 0x0000000f,
651         0x9220, 0xffffffff, 0x000e000d,
652         0x9224, 0xffffffff, 0x00110010,
653         0x9228, 0xffffffff, 0x000b000a,
654         0x922c, 0xffffffff, 0x000d000c,
655         0x9230, 0xffffffff, 0x00000010,
656         0x9234, 0xffffffff, 0x000f000e,
657         0x9238, 0xffffffff, 0x00120011,
658         0x923c, 0xffffffff, 0x000c000b,
659         0x9240, 0xffffffff, 0x000e000d,
660         0x9244, 0xffffffff, 0x00000011,
661         0x9248, 0xffffffff, 0x0010000f,
662         0x924c, 0xffffffff, 0x00130012,
663         0x9250, 0xffffffff, 0x000d000c,
664         0x9254, 0xffffffff, 0x000f000e,
665         0x9258, 0xffffffff, 0x00100013,
666         0x925c, 0xffffffff, 0x00120011,
667         0x9260, 0xffffffff, 0x00150014,
668         0x9264, 0xffffffff, 0x000e000d,
669         0x9268, 0xffffffff, 0x0010000f,
670         0x926c, 0xffffffff, 0x00000013,
671         0x9270, 0xffffffff, 0x00120011,
672         0x9274, 0xffffffff, 0x00150014,
673         0x9278, 0xffffffff, 0x000f000e,
674         0x927c, 0xffffffff, 0x00110010,
675         0x9280, 0xffffffff, 0x00000014,
676         0x9284, 0xffffffff, 0x00130012,
677         0x9288, 0xffffffff, 0x00160015,
678         0x928c, 0xffffffff, 0x0010000f,
679         0x9290, 0xffffffff, 0x00120011,
680         0x9294, 0xffffffff, 0x00000015,
681         0x9298, 0xffffffff, 0x00140013,
682         0x929c, 0xffffffff, 0x00170016,
683         0x9150, 0xffffffff, 0x96940200,
684         0x8708, 0xffffffff, 0x00900100,
685         0xc478, 0xffffffff, 0x00000080,
686         0xc404, 0xffffffff, 0x0020003f,
687         0x30, 0xffffffff, 0x0000001c,
688         0x34, 0x000f0000, 0x000f0000,
689         0x160c, 0xffffffff, 0x00000100,
690         0x1024, 0xffffffff, 0x00000100,
691         0x102c, 0x00000101, 0x00000000,
692         0x20a8, 0xffffffff, 0x00000104,
693         0x264c, 0x000c0000, 0x000c0000,
694         0x2648, 0x000c0000, 0x000c0000,
695         0x55e4, 0xff000fff, 0x00000100,
696         0x55e8, 0x00000001, 0x00000001,
697         0x2f50, 0x00000001, 0x00000001,
698         0x30cc, 0xc0000fff, 0x00000104,
699         0xc1e4, 0x00000001, 0x00000001,
700         0xd0c0, 0xfffffff0, 0x00000100,
701         0xd8c0, 0xfffffff0, 0x00000100
702 };
703
704 static const u32 pitcairn_mgcg_cgcg_init[] =
705 {
706         0xc400, 0xffffffff, 0xfffffffc,
707         0x802c, 0xffffffff, 0xe0000000,
708         0x9a60, 0xffffffff, 0x00000100,
709         0x92a4, 0xffffffff, 0x00000100,
710         0xc164, 0xffffffff, 0x00000100,
711         0x9774, 0xffffffff, 0x00000100,
712         0x8984, 0xffffffff, 0x06000100,
713         0x8a18, 0xffffffff, 0x00000100,
714         0x92a0, 0xffffffff, 0x00000100,
715         0xc380, 0xffffffff, 0x00000100,
716         0x8b28, 0xffffffff, 0x00000100,
717         0x9144, 0xffffffff, 0x00000100,
718         0x8d88, 0xffffffff, 0x00000100,
719         0x8d8c, 0xffffffff, 0x00000100,
720         0x9030, 0xffffffff, 0x00000100,
721         0x9034, 0xffffffff, 0x00000100,
722         0x9038, 0xffffffff, 0x00000100,
723         0x903c, 0xffffffff, 0x00000100,
724         0xad80, 0xffffffff, 0x00000100,
725         0xac54, 0xffffffff, 0x00000100,
726         0x897c, 0xffffffff, 0x06000100,
727         0x9868, 0xffffffff, 0x00000100,
728         0x9510, 0xffffffff, 0x00000100,
729         0xaf04, 0xffffffff, 0x00000100,
730         0xae04, 0xffffffff, 0x00000100,
731         0x949c, 0xffffffff, 0x00000100,
732         0x802c, 0xffffffff, 0xe0000000,
733         0x9160, 0xffffffff, 0x00010000,
734         0x9164, 0xffffffff, 0x00030002,
735         0x9168, 0xffffffff, 0x00040007,
736         0x916c, 0xffffffff, 0x00060005,
737         0x9170, 0xffffffff, 0x00090008,
738         0x9174, 0xffffffff, 0x00020001,
739         0x9178, 0xffffffff, 0x00040003,
740         0x917c, 0xffffffff, 0x00000007,
741         0x9180, 0xffffffff, 0x00060005,
742         0x9184, 0xffffffff, 0x00090008,
743         0x9188, 0xffffffff, 0x00030002,
744         0x918c, 0xffffffff, 0x00050004,
745         0x9190, 0xffffffff, 0x00000008,
746         0x9194, 0xffffffff, 0x00070006,
747         0x9198, 0xffffffff, 0x000a0009,
748         0x919c, 0xffffffff, 0x00040003,
749         0x91a0, 0xffffffff, 0x00060005,
750         0x91a4, 0xffffffff, 0x00000009,
751         0x91a8, 0xffffffff, 0x00080007,
752         0x91ac, 0xffffffff, 0x000b000a,
753         0x91b0, 0xffffffff, 0x00050004,
754         0x91b4, 0xffffffff, 0x00070006,
755         0x91b8, 0xffffffff, 0x0008000b,
756         0x91bc, 0xffffffff, 0x000a0009,
757         0x91c0, 0xffffffff, 0x000d000c,
758         0x9200, 0xffffffff, 0x00090008,
759         0x9204, 0xffffffff, 0x000b000a,
760         0x9208, 0xffffffff, 0x000c000f,
761         0x920c, 0xffffffff, 0x000e000d,
762         0x9210, 0xffffffff, 0x00110010,
763         0x9214, 0xffffffff, 0x000a0009,
764         0x9218, 0xffffffff, 0x000c000b,
765         0x921c, 0xffffffff, 0x0000000f,
766         0x9220, 0xffffffff, 0x000e000d,
767         0x9224, 0xffffffff, 0x00110010,
768         0x9228, 0xffffffff, 0x000b000a,
769         0x922c, 0xffffffff, 0x000d000c,
770         0x9230, 0xffffffff, 0x00000010,
771         0x9234, 0xffffffff, 0x000f000e,
772         0x9238, 0xffffffff, 0x00120011,
773         0x923c, 0xffffffff, 0x000c000b,
774         0x9240, 0xffffffff, 0x000e000d,
775         0x9244, 0xffffffff, 0x00000011,
776         0x9248, 0xffffffff, 0x0010000f,
777         0x924c, 0xffffffff, 0x00130012,
778         0x9250, 0xffffffff, 0x000d000c,
779         0x9254, 0xffffffff, 0x000f000e,
780         0x9258, 0xffffffff, 0x00100013,
781         0x925c, 0xffffffff, 0x00120011,
782         0x9260, 0xffffffff, 0x00150014,
783         0x9150, 0xffffffff, 0x96940200,
784         0x8708, 0xffffffff, 0x00900100,
785         0xc478, 0xffffffff, 0x00000080,
786         0xc404, 0xffffffff, 0x0020003f,
787         0x30, 0xffffffff, 0x0000001c,
788         0x34, 0x000f0000, 0x000f0000,
789         0x160c, 0xffffffff, 0x00000100,
790         0x1024, 0xffffffff, 0x00000100,
791         0x102c, 0x00000101, 0x00000000,
792         0x20a8, 0xffffffff, 0x00000104,
793         0x55e4, 0xff000fff, 0x00000100,
794         0x55e8, 0x00000001, 0x00000001,
795         0x2f50, 0x00000001, 0x00000001,
796         0x30cc, 0xc0000fff, 0x00000104,
797         0xc1e4, 0x00000001, 0x00000001,
798         0xd0c0, 0xfffffff0, 0x00000100,
799         0xd8c0, 0xfffffff0, 0x00000100
800 };
801
802 static const u32 verde_mgcg_cgcg_init[] =
803 {
804         0xc400, 0xffffffff, 0xfffffffc,
805         0x802c, 0xffffffff, 0xe0000000,
806         0x9a60, 0xffffffff, 0x00000100,
807         0x92a4, 0xffffffff, 0x00000100,
808         0xc164, 0xffffffff, 0x00000100,
809         0x9774, 0xffffffff, 0x00000100,
810         0x8984, 0xffffffff, 0x06000100,
811         0x8a18, 0xffffffff, 0x00000100,
812         0x92a0, 0xffffffff, 0x00000100,
813         0xc380, 0xffffffff, 0x00000100,
814         0x8b28, 0xffffffff, 0x00000100,
815         0x9144, 0xffffffff, 0x00000100,
816         0x8d88, 0xffffffff, 0x00000100,
817         0x8d8c, 0xffffffff, 0x00000100,
818         0x9030, 0xffffffff, 0x00000100,
819         0x9034, 0xffffffff, 0x00000100,
820         0x9038, 0xffffffff, 0x00000100,
821         0x903c, 0xffffffff, 0x00000100,
822         0xad80, 0xffffffff, 0x00000100,
823         0xac54, 0xffffffff, 0x00000100,
824         0x897c, 0xffffffff, 0x06000100,
825         0x9868, 0xffffffff, 0x00000100,
826         0x9510, 0xffffffff, 0x00000100,
827         0xaf04, 0xffffffff, 0x00000100,
828         0xae04, 0xffffffff, 0x00000100,
829         0x949c, 0xffffffff, 0x00000100,
830         0x802c, 0xffffffff, 0xe0000000,
831         0x9160, 0xffffffff, 0x00010000,
832         0x9164, 0xffffffff, 0x00030002,
833         0x9168, 0xffffffff, 0x00040007,
834         0x916c, 0xffffffff, 0x00060005,
835         0x9170, 0xffffffff, 0x00090008,
836         0x9174, 0xffffffff, 0x00020001,
837         0x9178, 0xffffffff, 0x00040003,
838         0x917c, 0xffffffff, 0x00000007,
839         0x9180, 0xffffffff, 0x00060005,
840         0x9184, 0xffffffff, 0x00090008,
841         0x9188, 0xffffffff, 0x00030002,
842         0x918c, 0xffffffff, 0x00050004,
843         0x9190, 0xffffffff, 0x00000008,
844         0x9194, 0xffffffff, 0x00070006,
845         0x9198, 0xffffffff, 0x000a0009,
846         0x919c, 0xffffffff, 0x00040003,
847         0x91a0, 0xffffffff, 0x00060005,
848         0x91a4, 0xffffffff, 0x00000009,
849         0x91a8, 0xffffffff, 0x00080007,
850         0x91ac, 0xffffffff, 0x000b000a,
851         0x91b0, 0xffffffff, 0x00050004,
852         0x91b4, 0xffffffff, 0x00070006,
853         0x91b8, 0xffffffff, 0x0008000b,
854         0x91bc, 0xffffffff, 0x000a0009,
855         0x91c0, 0xffffffff, 0x000d000c,
856         0x9200, 0xffffffff, 0x00090008,
857         0x9204, 0xffffffff, 0x000b000a,
858         0x9208, 0xffffffff, 0x000c000f,
859         0x920c, 0xffffffff, 0x000e000d,
860         0x9210, 0xffffffff, 0x00110010,
861         0x9214, 0xffffffff, 0x000a0009,
862         0x9218, 0xffffffff, 0x000c000b,
863         0x921c, 0xffffffff, 0x0000000f,
864         0x9220, 0xffffffff, 0x000e000d,
865         0x9224, 0xffffffff, 0x00110010,
866         0x9228, 0xffffffff, 0x000b000a,
867         0x922c, 0xffffffff, 0x000d000c,
868         0x9230, 0xffffffff, 0x00000010,
869         0x9234, 0xffffffff, 0x000f000e,
870         0x9238, 0xffffffff, 0x00120011,
871         0x923c, 0xffffffff, 0x000c000b,
872         0x9240, 0xffffffff, 0x000e000d,
873         0x9244, 0xffffffff, 0x00000011,
874         0x9248, 0xffffffff, 0x0010000f,
875         0x924c, 0xffffffff, 0x00130012,
876         0x9250, 0xffffffff, 0x000d000c,
877         0x9254, 0xffffffff, 0x000f000e,
878         0x9258, 0xffffffff, 0x00100013,
879         0x925c, 0xffffffff, 0x00120011,
880         0x9260, 0xffffffff, 0x00150014,
881         0x9150, 0xffffffff, 0x96940200,
882         0x8708, 0xffffffff, 0x00900100,
883         0xc478, 0xffffffff, 0x00000080,
884         0xc404, 0xffffffff, 0x0020003f,
885         0x30, 0xffffffff, 0x0000001c,
886         0x34, 0x000f0000, 0x000f0000,
887         0x160c, 0xffffffff, 0x00000100,
888         0x1024, 0xffffffff, 0x00000100,
889         0x102c, 0x00000101, 0x00000000,
890         0x20a8, 0xffffffff, 0x00000104,
891         0x264c, 0x000c0000, 0x000c0000,
892         0x2648, 0x000c0000, 0x000c0000,
893         0x55e4, 0xff000fff, 0x00000100,
894         0x55e8, 0x00000001, 0x00000001,
895         0x2f50, 0x00000001, 0x00000001,
896         0x30cc, 0xc0000fff, 0x00000104,
897         0xc1e4, 0x00000001, 0x00000001,
898         0xd0c0, 0xfffffff0, 0x00000100,
899         0xd8c0, 0xfffffff0, 0x00000100
900 };
901
902 static const u32 oland_mgcg_cgcg_init[] =
903 {
904         0xc400, 0xffffffff, 0xfffffffc,
905         0x802c, 0xffffffff, 0xe0000000,
906         0x9a60, 0xffffffff, 0x00000100,
907         0x92a4, 0xffffffff, 0x00000100,
908         0xc164, 0xffffffff, 0x00000100,
909         0x9774, 0xffffffff, 0x00000100,
910         0x8984, 0xffffffff, 0x06000100,
911         0x8a18, 0xffffffff, 0x00000100,
912         0x92a0, 0xffffffff, 0x00000100,
913         0xc380, 0xffffffff, 0x00000100,
914         0x8b28, 0xffffffff, 0x00000100,
915         0x9144, 0xffffffff, 0x00000100,
916         0x8d88, 0xffffffff, 0x00000100,
917         0x8d8c, 0xffffffff, 0x00000100,
918         0x9030, 0xffffffff, 0x00000100,
919         0x9034, 0xffffffff, 0x00000100,
920         0x9038, 0xffffffff, 0x00000100,
921         0x903c, 0xffffffff, 0x00000100,
922         0xad80, 0xffffffff, 0x00000100,
923         0xac54, 0xffffffff, 0x00000100,
924         0x897c, 0xffffffff, 0x06000100,
925         0x9868, 0xffffffff, 0x00000100,
926         0x9510, 0xffffffff, 0x00000100,
927         0xaf04, 0xffffffff, 0x00000100,
928         0xae04, 0xffffffff, 0x00000100,
929         0x949c, 0xffffffff, 0x00000100,
930         0x802c, 0xffffffff, 0xe0000000,
931         0x9160, 0xffffffff, 0x00010000,
932         0x9164, 0xffffffff, 0x00030002,
933         0x9168, 0xffffffff, 0x00040007,
934         0x916c, 0xffffffff, 0x00060005,
935         0x9170, 0xffffffff, 0x00090008,
936         0x9174, 0xffffffff, 0x00020001,
937         0x9178, 0xffffffff, 0x00040003,
938         0x917c, 0xffffffff, 0x00000007,
939         0x9180, 0xffffffff, 0x00060005,
940         0x9184, 0xffffffff, 0x00090008,
941         0x9188, 0xffffffff, 0x00030002,
942         0x918c, 0xffffffff, 0x00050004,
943         0x9190, 0xffffffff, 0x00000008,
944         0x9194, 0xffffffff, 0x00070006,
945         0x9198, 0xffffffff, 0x000a0009,
946         0x919c, 0xffffffff, 0x00040003,
947         0x91a0, 0xffffffff, 0x00060005,
948         0x91a4, 0xffffffff, 0x00000009,
949         0x91a8, 0xffffffff, 0x00080007,
950         0x91ac, 0xffffffff, 0x000b000a,
951         0x91b0, 0xffffffff, 0x00050004,
952         0x91b4, 0xffffffff, 0x00070006,
953         0x91b8, 0xffffffff, 0x0008000b,
954         0x91bc, 0xffffffff, 0x000a0009,
955         0x91c0, 0xffffffff, 0x000d000c,
956         0x91c4, 0xffffffff, 0x00060005,
957         0x91c8, 0xffffffff, 0x00080007,
958         0x91cc, 0xffffffff, 0x0000000b,
959         0x91d0, 0xffffffff, 0x000a0009,
960         0x91d4, 0xffffffff, 0x000d000c,
961         0x9150, 0xffffffff, 0x96940200,
962         0x8708, 0xffffffff, 0x00900100,
963         0xc478, 0xffffffff, 0x00000080,
964         0xc404, 0xffffffff, 0x0020003f,
965         0x30, 0xffffffff, 0x0000001c,
966         0x34, 0x000f0000, 0x000f0000,
967         0x160c, 0xffffffff, 0x00000100,
968         0x1024, 0xffffffff, 0x00000100,
969         0x102c, 0x00000101, 0x00000000,
970         0x20a8, 0xffffffff, 0x00000104,
971         0x264c, 0x000c0000, 0x000c0000,
972         0x2648, 0x000c0000, 0x000c0000,
973         0x55e4, 0xff000fff, 0x00000100,
974         0x55e8, 0x00000001, 0x00000001,
975         0x2f50, 0x00000001, 0x00000001,
976         0x30cc, 0xc0000fff, 0x00000104,
977         0xc1e4, 0x00000001, 0x00000001,
978         0xd0c0, 0xfffffff0, 0x00000100,
979         0xd8c0, 0xfffffff0, 0x00000100
980 };
981
982 static const u32 hainan_mgcg_cgcg_init[] =
983 {
984         0xc400, 0xffffffff, 0xfffffffc,
985         0x802c, 0xffffffff, 0xe0000000,
986         0x9a60, 0xffffffff, 0x00000100,
987         0x92a4, 0xffffffff, 0x00000100,
988         0xc164, 0xffffffff, 0x00000100,
989         0x9774, 0xffffffff, 0x00000100,
990         0x8984, 0xffffffff, 0x06000100,
991         0x8a18, 0xffffffff, 0x00000100,
992         0x92a0, 0xffffffff, 0x00000100,
993         0xc380, 0xffffffff, 0x00000100,
994         0x8b28, 0xffffffff, 0x00000100,
995         0x9144, 0xffffffff, 0x00000100,
996         0x8d88, 0xffffffff, 0x00000100,
997         0x8d8c, 0xffffffff, 0x00000100,
998         0x9030, 0xffffffff, 0x00000100,
999         0x9034, 0xffffffff, 0x00000100,
1000         0x9038, 0xffffffff, 0x00000100,
1001         0x903c, 0xffffffff, 0x00000100,
1002         0xad80, 0xffffffff, 0x00000100,
1003         0xac54, 0xffffffff, 0x00000100,
1004         0x897c, 0xffffffff, 0x06000100,
1005         0x9868, 0xffffffff, 0x00000100,
1006         0x9510, 0xffffffff, 0x00000100,
1007         0xaf04, 0xffffffff, 0x00000100,
1008         0xae04, 0xffffffff, 0x00000100,
1009         0x949c, 0xffffffff, 0x00000100,
1010         0x802c, 0xffffffff, 0xe0000000,
1011         0x9160, 0xffffffff, 0x00010000,
1012         0x9164, 0xffffffff, 0x00030002,
1013         0x9168, 0xffffffff, 0x00040007,
1014         0x916c, 0xffffffff, 0x00060005,
1015         0x9170, 0xffffffff, 0x00090008,
1016         0x9174, 0xffffffff, 0x00020001,
1017         0x9178, 0xffffffff, 0x00040003,
1018         0x917c, 0xffffffff, 0x00000007,
1019         0x9180, 0xffffffff, 0x00060005,
1020         0x9184, 0xffffffff, 0x00090008,
1021         0x9188, 0xffffffff, 0x00030002,
1022         0x918c, 0xffffffff, 0x00050004,
1023         0x9190, 0xffffffff, 0x00000008,
1024         0x9194, 0xffffffff, 0x00070006,
1025         0x9198, 0xffffffff, 0x000a0009,
1026         0x919c, 0xffffffff, 0x00040003,
1027         0x91a0, 0xffffffff, 0x00060005,
1028         0x91a4, 0xffffffff, 0x00000009,
1029         0x91a8, 0xffffffff, 0x00080007,
1030         0x91ac, 0xffffffff, 0x000b000a,
1031         0x91b0, 0xffffffff, 0x00050004,
1032         0x91b4, 0xffffffff, 0x00070006,
1033         0x91b8, 0xffffffff, 0x0008000b,
1034         0x91bc, 0xffffffff, 0x000a0009,
1035         0x91c0, 0xffffffff, 0x000d000c,
1036         0x91c4, 0xffffffff, 0x00060005,
1037         0x91c8, 0xffffffff, 0x00080007,
1038         0x91cc, 0xffffffff, 0x0000000b,
1039         0x91d0, 0xffffffff, 0x000a0009,
1040         0x91d4, 0xffffffff, 0x000d000c,
1041         0x9150, 0xffffffff, 0x96940200,
1042         0x8708, 0xffffffff, 0x00900100,
1043         0xc478, 0xffffffff, 0x00000080,
1044         0xc404, 0xffffffff, 0x0020003f,
1045         0x30, 0xffffffff, 0x0000001c,
1046         0x34, 0x000f0000, 0x000f0000,
1047         0x160c, 0xffffffff, 0x00000100,
1048         0x1024, 0xffffffff, 0x00000100,
1049         0x20a8, 0xffffffff, 0x00000104,
1050         0x264c, 0x000c0000, 0x000c0000,
1051         0x2648, 0x000c0000, 0x000c0000,
1052         0x2f50, 0x00000001, 0x00000001,
1053         0x30cc, 0xc0000fff, 0x00000104,
1054         0xc1e4, 0x00000001, 0x00000001,
1055         0xd0c0, 0xfffffff0, 0x00000100,
1056         0xd8c0, 0xfffffff0, 0x00000100
1057 };
1058
1059 static u32 verde_pg_init[] =
1060 {
1061         0x353c, 0xffffffff, 0x40000,
1062         0x3538, 0xffffffff, 0x200010ff,
1063         0x353c, 0xffffffff, 0x0,
1064         0x353c, 0xffffffff, 0x0,
1065         0x353c, 0xffffffff, 0x0,
1066         0x353c, 0xffffffff, 0x0,
1067         0x353c, 0xffffffff, 0x0,
1068         0x353c, 0xffffffff, 0x7007,
1069         0x3538, 0xffffffff, 0x300010ff,
1070         0x353c, 0xffffffff, 0x0,
1071         0x353c, 0xffffffff, 0x0,
1072         0x353c, 0xffffffff, 0x0,
1073         0x353c, 0xffffffff, 0x0,
1074         0x353c, 0xffffffff, 0x0,
1075         0x353c, 0xffffffff, 0x400000,
1076         0x3538, 0xffffffff, 0x100010ff,
1077         0x353c, 0xffffffff, 0x0,
1078         0x353c, 0xffffffff, 0x0,
1079         0x353c, 0xffffffff, 0x0,
1080         0x353c, 0xffffffff, 0x0,
1081         0x353c, 0xffffffff, 0x0,
1082         0x353c, 0xffffffff, 0x120200,
1083         0x3538, 0xffffffff, 0x500010ff,
1084         0x353c, 0xffffffff, 0x0,
1085         0x353c, 0xffffffff, 0x0,
1086         0x353c, 0xffffffff, 0x0,
1087         0x353c, 0xffffffff, 0x0,
1088         0x353c, 0xffffffff, 0x0,
1089         0x353c, 0xffffffff, 0x1e1e16,
1090         0x3538, 0xffffffff, 0x600010ff,
1091         0x353c, 0xffffffff, 0x0,
1092         0x353c, 0xffffffff, 0x0,
1093         0x353c, 0xffffffff, 0x0,
1094         0x353c, 0xffffffff, 0x0,
1095         0x353c, 0xffffffff, 0x0,
1096         0x353c, 0xffffffff, 0x171f1e,
1097         0x3538, 0xffffffff, 0x700010ff,
1098         0x353c, 0xffffffff, 0x0,
1099         0x353c, 0xffffffff, 0x0,
1100         0x353c, 0xffffffff, 0x0,
1101         0x353c, 0xffffffff, 0x0,
1102         0x353c, 0xffffffff, 0x0,
1103         0x353c, 0xffffffff, 0x0,
1104         0x3538, 0xffffffff, 0x9ff,
1105         0x3500, 0xffffffff, 0x0,
1106         0x3504, 0xffffffff, 0x10000800,
1107         0x3504, 0xffffffff, 0xf,
1108         0x3504, 0xffffffff, 0xf,
1109         0x3500, 0xffffffff, 0x4,
1110         0x3504, 0xffffffff, 0x1000051e,
1111         0x3504, 0xffffffff, 0xffff,
1112         0x3504, 0xffffffff, 0xffff,
1113         0x3500, 0xffffffff, 0x8,
1114         0x3504, 0xffffffff, 0x80500,
1115         0x3500, 0xffffffff, 0x12,
1116         0x3504, 0xffffffff, 0x9050c,
1117         0x3500, 0xffffffff, 0x1d,
1118         0x3504, 0xffffffff, 0xb052c,
1119         0x3500, 0xffffffff, 0x2a,
1120         0x3504, 0xffffffff, 0x1053e,
1121         0x3500, 0xffffffff, 0x2d,
1122         0x3504, 0xffffffff, 0x10546,
1123         0x3500, 0xffffffff, 0x30,
1124         0x3504, 0xffffffff, 0xa054e,
1125         0x3500, 0xffffffff, 0x3c,
1126         0x3504, 0xffffffff, 0x1055f,
1127         0x3500, 0xffffffff, 0x3f,
1128         0x3504, 0xffffffff, 0x10567,
1129         0x3500, 0xffffffff, 0x42,
1130         0x3504, 0xffffffff, 0x1056f,
1131         0x3500, 0xffffffff, 0x45,
1132         0x3504, 0xffffffff, 0x10572,
1133         0x3500, 0xffffffff, 0x48,
1134         0x3504, 0xffffffff, 0x20575,
1135         0x3500, 0xffffffff, 0x4c,
1136         0x3504, 0xffffffff, 0x190801,
1137         0x3500, 0xffffffff, 0x67,
1138         0x3504, 0xffffffff, 0x1082a,
1139         0x3500, 0xffffffff, 0x6a,
1140         0x3504, 0xffffffff, 0x1b082d,
1141         0x3500, 0xffffffff, 0x87,
1142         0x3504, 0xffffffff, 0x310851,
1143         0x3500, 0xffffffff, 0xba,
1144         0x3504, 0xffffffff, 0x891,
1145         0x3500, 0xffffffff, 0xbc,
1146         0x3504, 0xffffffff, 0x893,
1147         0x3500, 0xffffffff, 0xbe,
1148         0x3504, 0xffffffff, 0x20895,
1149         0x3500, 0xffffffff, 0xc2,
1150         0x3504, 0xffffffff, 0x20899,
1151         0x3500, 0xffffffff, 0xc6,
1152         0x3504, 0xffffffff, 0x2089d,
1153         0x3500, 0xffffffff, 0xca,
1154         0x3504, 0xffffffff, 0x8a1,
1155         0x3500, 0xffffffff, 0xcc,
1156         0x3504, 0xffffffff, 0x8a3,
1157         0x3500, 0xffffffff, 0xce,
1158         0x3504, 0xffffffff, 0x308a5,
1159         0x3500, 0xffffffff, 0xd3,
1160         0x3504, 0xffffffff, 0x6d08cd,
1161         0x3500, 0xffffffff, 0x142,
1162         0x3504, 0xffffffff, 0x2000095a,
1163         0x3504, 0xffffffff, 0x1,
1164         0x3500, 0xffffffff, 0x144,
1165         0x3504, 0xffffffff, 0x301f095b,
1166         0x3500, 0xffffffff, 0x165,
1167         0x3504, 0xffffffff, 0xc094d,
1168         0x3500, 0xffffffff, 0x173,
1169         0x3504, 0xffffffff, 0xf096d,
1170         0x3500, 0xffffffff, 0x184,
1171         0x3504, 0xffffffff, 0x15097f,
1172         0x3500, 0xffffffff, 0x19b,
1173         0x3504, 0xffffffff, 0xc0998,
1174         0x3500, 0xffffffff, 0x1a9,
1175         0x3504, 0xffffffff, 0x409a7,
1176         0x3500, 0xffffffff, 0x1af,
1177         0x3504, 0xffffffff, 0xcdc,
1178         0x3500, 0xffffffff, 0x1b1,
1179         0x3504, 0xffffffff, 0x800,
1180         0x3508, 0xffffffff, 0x6c9b2000,
1181         0x3510, 0xfc00, 0x2000,
1182         0x3544, 0xffffffff, 0xfc0,
1183         0x28d4, 0x00000100, 0x100
1184 };
1185
1186 static void si_init_golden_registers(struct radeon_device *rdev)
1187 {
1188         switch (rdev->family) {
1189         case CHIP_TAHITI:
1190                 radeon_program_register_sequence(rdev,
1191                                                  tahiti_golden_registers,
1192                                                  (const u32)ARRAY_SIZE(tahiti_golden_registers));
1193                 radeon_program_register_sequence(rdev,
1194                                                  tahiti_golden_rlc_registers,
1195                                                  (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1196                 radeon_program_register_sequence(rdev,
1197                                                  tahiti_mgcg_cgcg_init,
1198                                                  (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1199                 radeon_program_register_sequence(rdev,
1200                                                  tahiti_golden_registers2,
1201                                                  (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1202                 break;
1203         case CHIP_PITCAIRN:
1204                 radeon_program_register_sequence(rdev,
1205                                                  pitcairn_golden_registers,
1206                                                  (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1207                 radeon_program_register_sequence(rdev,
1208                                                  pitcairn_golden_rlc_registers,
1209                                                  (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1210                 radeon_program_register_sequence(rdev,
1211                                                  pitcairn_mgcg_cgcg_init,
1212                                                  (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1213                 break;
1214         case CHIP_VERDE:
1215                 radeon_program_register_sequence(rdev,
1216                                                  verde_golden_registers,
1217                                                  (const u32)ARRAY_SIZE(verde_golden_registers));
1218                 radeon_program_register_sequence(rdev,
1219                                                  verde_golden_rlc_registers,
1220                                                  (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1221                 radeon_program_register_sequence(rdev,
1222                                                  verde_mgcg_cgcg_init,
1223                                                  (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1224                 radeon_program_register_sequence(rdev,
1225                                                  verde_pg_init,
1226                                                  (const u32)ARRAY_SIZE(verde_pg_init));
1227                 break;
1228         case CHIP_OLAND:
1229                 radeon_program_register_sequence(rdev,
1230                                                  oland_golden_registers,
1231                                                  (const u32)ARRAY_SIZE(oland_golden_registers));
1232                 radeon_program_register_sequence(rdev,
1233                                                  oland_golden_rlc_registers,
1234                                                  (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1235                 radeon_program_register_sequence(rdev,
1236                                                  oland_mgcg_cgcg_init,
1237                                                  (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1238                 break;
1239         case CHIP_HAINAN:
1240                 radeon_program_register_sequence(rdev,
1241                                                  hainan_golden_registers,
1242                                                  (const u32)ARRAY_SIZE(hainan_golden_registers));
1243                 radeon_program_register_sequence(rdev,
1244                                                  hainan_golden_registers2,
1245                                                  (const u32)ARRAY_SIZE(hainan_golden_registers2));
1246                 radeon_program_register_sequence(rdev,
1247                                                  hainan_mgcg_cgcg_init,
1248                                                  (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1249                 break;
1250         default:
1251                 break;
1252         }
1253 }
1254
1255 #define PCIE_BUS_CLK                10000
1256 #define TCLK                        (PCIE_BUS_CLK / 10)
1257
1258 /**
1259  * si_get_xclk - get the xclk
1260  *
1261  * @rdev: radeon_device pointer
1262  *
1263  * Returns the reference clock used by the gfx engine
1264  * (SI).
1265  */
1266 u32 si_get_xclk(struct radeon_device *rdev)
1267 {
1268         u32 reference_clock = rdev->clock.spll.reference_freq;
1269         u32 tmp;
1270
1271         tmp = RREG32(CG_CLKPIN_CNTL_2);
1272         if (tmp & MUX_TCLK_TO_XCLK)
1273                 return TCLK;
1274
1275         tmp = RREG32(CG_CLKPIN_CNTL);
1276         if (tmp & XTALIN_DIVIDE)
1277                 return reference_clock / 4;
1278
1279         return reference_clock;
1280 }
1281
1282 /* get temperature in millidegrees */
1283 int si_get_temp(struct radeon_device *rdev)
1284 {
1285         u32 temp;
1286         int actual_temp = 0;
1287
1288         temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1289                 CTF_TEMP_SHIFT;
1290
1291         if (temp & 0x200)
1292                 actual_temp = 255;
1293         else
1294                 actual_temp = temp & 0x1ff;
1295
1296         actual_temp = (actual_temp * 1000);
1297
1298         return actual_temp;
1299 }
1300
1301 #define TAHITI_IO_MC_REGS_SIZE 36
1302
1303 static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1304         {0x0000006f, 0x03044000},
1305         {0x00000070, 0x0480c018},
1306         {0x00000071, 0x00000040},
1307         {0x00000072, 0x01000000},
1308         {0x00000074, 0x000000ff},
1309         {0x00000075, 0x00143400},
1310         {0x00000076, 0x08ec0800},
1311         {0x00000077, 0x040000cc},
1312         {0x00000079, 0x00000000},
1313         {0x0000007a, 0x21000409},
1314         {0x0000007c, 0x00000000},
1315         {0x0000007d, 0xe8000000},
1316         {0x0000007e, 0x044408a8},
1317         {0x0000007f, 0x00000003},
1318         {0x00000080, 0x00000000},
1319         {0x00000081, 0x01000000},
1320         {0x00000082, 0x02000000},
1321         {0x00000083, 0x00000000},
1322         {0x00000084, 0xe3f3e4f4},
1323         {0x00000085, 0x00052024},
1324         {0x00000087, 0x00000000},
1325         {0x00000088, 0x66036603},
1326         {0x00000089, 0x01000000},
1327         {0x0000008b, 0x1c0a0000},
1328         {0x0000008c, 0xff010000},
1329         {0x0000008e, 0xffffefff},
1330         {0x0000008f, 0xfff3efff},
1331         {0x00000090, 0xfff3efbf},
1332         {0x00000094, 0x00101101},
1333         {0x00000095, 0x00000fff},
1334         {0x00000096, 0x00116fff},
1335         {0x00000097, 0x60010000},
1336         {0x00000098, 0x10010000},
1337         {0x00000099, 0x00006000},
1338         {0x0000009a, 0x00001000},
1339         {0x0000009f, 0x00a77400}
1340 };
1341
1342 static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1343         {0x0000006f, 0x03044000},
1344         {0x00000070, 0x0480c018},
1345         {0x00000071, 0x00000040},
1346         {0x00000072, 0x01000000},
1347         {0x00000074, 0x000000ff},
1348         {0x00000075, 0x00143400},
1349         {0x00000076, 0x08ec0800},
1350         {0x00000077, 0x040000cc},
1351         {0x00000079, 0x00000000},
1352         {0x0000007a, 0x21000409},
1353         {0x0000007c, 0x00000000},
1354         {0x0000007d, 0xe8000000},
1355         {0x0000007e, 0x044408a8},
1356         {0x0000007f, 0x00000003},
1357         {0x00000080, 0x00000000},
1358         {0x00000081, 0x01000000},
1359         {0x00000082, 0x02000000},
1360         {0x00000083, 0x00000000},
1361         {0x00000084, 0xe3f3e4f4},
1362         {0x00000085, 0x00052024},
1363         {0x00000087, 0x00000000},
1364         {0x00000088, 0x66036603},
1365         {0x00000089, 0x01000000},
1366         {0x0000008b, 0x1c0a0000},
1367         {0x0000008c, 0xff010000},
1368         {0x0000008e, 0xffffefff},
1369         {0x0000008f, 0xfff3efff},
1370         {0x00000090, 0xfff3efbf},
1371         {0x00000094, 0x00101101},
1372         {0x00000095, 0x00000fff},
1373         {0x00000096, 0x00116fff},
1374         {0x00000097, 0x60010000},
1375         {0x00000098, 0x10010000},
1376         {0x00000099, 0x00006000},
1377         {0x0000009a, 0x00001000},
1378         {0x0000009f, 0x00a47400}
1379 };
1380
1381 static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1382         {0x0000006f, 0x03044000},
1383         {0x00000070, 0x0480c018},
1384         {0x00000071, 0x00000040},
1385         {0x00000072, 0x01000000},
1386         {0x00000074, 0x000000ff},
1387         {0x00000075, 0x00143400},
1388         {0x00000076, 0x08ec0800},
1389         {0x00000077, 0x040000cc},
1390         {0x00000079, 0x00000000},
1391         {0x0000007a, 0x21000409},
1392         {0x0000007c, 0x00000000},
1393         {0x0000007d, 0xe8000000},
1394         {0x0000007e, 0x044408a8},
1395         {0x0000007f, 0x00000003},
1396         {0x00000080, 0x00000000},
1397         {0x00000081, 0x01000000},
1398         {0x00000082, 0x02000000},
1399         {0x00000083, 0x00000000},
1400         {0x00000084, 0xe3f3e4f4},
1401         {0x00000085, 0x00052024},
1402         {0x00000087, 0x00000000},
1403         {0x00000088, 0x66036603},
1404         {0x00000089, 0x01000000},
1405         {0x0000008b, 0x1c0a0000},
1406         {0x0000008c, 0xff010000},
1407         {0x0000008e, 0xffffefff},
1408         {0x0000008f, 0xfff3efff},
1409         {0x00000090, 0xfff3efbf},
1410         {0x00000094, 0x00101101},
1411         {0x00000095, 0x00000fff},
1412         {0x00000096, 0x00116fff},
1413         {0x00000097, 0x60010000},
1414         {0x00000098, 0x10010000},
1415         {0x00000099, 0x00006000},
1416         {0x0000009a, 0x00001000},
1417         {0x0000009f, 0x00a37400}
1418 };
1419
1420 static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1421         {0x0000006f, 0x03044000},
1422         {0x00000070, 0x0480c018},
1423         {0x00000071, 0x00000040},
1424         {0x00000072, 0x01000000},
1425         {0x00000074, 0x000000ff},
1426         {0x00000075, 0x00143400},
1427         {0x00000076, 0x08ec0800},
1428         {0x00000077, 0x040000cc},
1429         {0x00000079, 0x00000000},
1430         {0x0000007a, 0x21000409},
1431         {0x0000007c, 0x00000000},
1432         {0x0000007d, 0xe8000000},
1433         {0x0000007e, 0x044408a8},
1434         {0x0000007f, 0x00000003},
1435         {0x00000080, 0x00000000},
1436         {0x00000081, 0x01000000},
1437         {0x00000082, 0x02000000},
1438         {0x00000083, 0x00000000},
1439         {0x00000084, 0xe3f3e4f4},
1440         {0x00000085, 0x00052024},
1441         {0x00000087, 0x00000000},
1442         {0x00000088, 0x66036603},
1443         {0x00000089, 0x01000000},
1444         {0x0000008b, 0x1c0a0000},
1445         {0x0000008c, 0xff010000},
1446         {0x0000008e, 0xffffefff},
1447         {0x0000008f, 0xfff3efff},
1448         {0x00000090, 0xfff3efbf},
1449         {0x00000094, 0x00101101},
1450         {0x00000095, 0x00000fff},
1451         {0x00000096, 0x00116fff},
1452         {0x00000097, 0x60010000},
1453         {0x00000098, 0x10010000},
1454         {0x00000099, 0x00006000},
1455         {0x0000009a, 0x00001000},
1456         {0x0000009f, 0x00a17730}
1457 };
1458
1459 static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1460         {0x0000006f, 0x03044000},
1461         {0x00000070, 0x0480c018},
1462         {0x00000071, 0x00000040},
1463         {0x00000072, 0x01000000},
1464         {0x00000074, 0x000000ff},
1465         {0x00000075, 0x00143400},
1466         {0x00000076, 0x08ec0800},
1467         {0x00000077, 0x040000cc},
1468         {0x00000079, 0x00000000},
1469         {0x0000007a, 0x21000409},
1470         {0x0000007c, 0x00000000},
1471         {0x0000007d, 0xe8000000},
1472         {0x0000007e, 0x044408a8},
1473         {0x0000007f, 0x00000003},
1474         {0x00000080, 0x00000000},
1475         {0x00000081, 0x01000000},
1476         {0x00000082, 0x02000000},
1477         {0x00000083, 0x00000000},
1478         {0x00000084, 0xe3f3e4f4},
1479         {0x00000085, 0x00052024},
1480         {0x00000087, 0x00000000},
1481         {0x00000088, 0x66036603},
1482         {0x00000089, 0x01000000},
1483         {0x0000008b, 0x1c0a0000},
1484         {0x0000008c, 0xff010000},
1485         {0x0000008e, 0xffffefff},
1486         {0x0000008f, 0xfff3efff},
1487         {0x00000090, 0xfff3efbf},
1488         {0x00000094, 0x00101101},
1489         {0x00000095, 0x00000fff},
1490         {0x00000096, 0x00116fff},
1491         {0x00000097, 0x60010000},
1492         {0x00000098, 0x10010000},
1493         {0x00000099, 0x00006000},
1494         {0x0000009a, 0x00001000},
1495         {0x0000009f, 0x00a07730}
1496 };
1497
1498 /* ucode loading */
1499 int si_mc_load_microcode(struct radeon_device *rdev)
1500 {
1501         const __be32 *fw_data = NULL;
1502         const __le32 *new_fw_data = NULL;
1503         u32 running, blackout = 0;
1504         u32 *io_mc_regs = NULL;
1505         const __le32 *new_io_mc_regs = NULL;
1506         int i, regs_size, ucode_size;
1507
1508         if (!rdev->mc_fw)
1509                 return -EINVAL;
1510
1511         if (rdev->new_fw) {
1512                 const struct mc_firmware_header_v1_0 *hdr =
1513                         (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1514
1515                 radeon_ucode_print_mc_hdr(&hdr->header);
1516                 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1517                 new_io_mc_regs = (const __le32 *)
1518                         ((const char *)rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1519                 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1520                 new_fw_data = (const __le32 *)
1521                         ((const char *)rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1522         } else {
1523                 ucode_size = rdev->mc_fw->datasize / 4;
1524
1525                 switch (rdev->family) {
1526                 case CHIP_TAHITI:
1527                         io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1528                         regs_size = TAHITI_IO_MC_REGS_SIZE;
1529                         break;
1530                 case CHIP_PITCAIRN:
1531                         io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1532                         regs_size = TAHITI_IO_MC_REGS_SIZE;
1533                         break;
1534                 case CHIP_VERDE:
1535                 default:
1536                         io_mc_regs = (u32 *)&verde_io_mc_regs;
1537                         regs_size = TAHITI_IO_MC_REGS_SIZE;
1538                         break;
1539                 case CHIP_OLAND:
1540                         io_mc_regs = (u32 *)&oland_io_mc_regs;
1541                         regs_size = TAHITI_IO_MC_REGS_SIZE;
1542                         break;
1543                 case CHIP_HAINAN:
1544                         io_mc_regs = (u32 *)&hainan_io_mc_regs;
1545                         regs_size = TAHITI_IO_MC_REGS_SIZE;
1546                         break;
1547                 }
1548                 fw_data = (const __be32 *)rdev->mc_fw->data;
1549         }
1550
1551         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1552
1553         if (running == 0) {
1554                 if (running) {
1555                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1556                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1557                 }
1558
1559                 /* reset the engine and set to writable */
1560                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1561                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1562
1563                 /* load mc io regs */
1564                 for (i = 0; i < regs_size; i++) {
1565                         if (rdev->new_fw) {
1566                                 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1567                                 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1568                         } else {
1569                                 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1570                                 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1571                         }
1572                 }
1573                 /* load the MC ucode */
1574                 for (i = 0; i < ucode_size; i++) {
1575                         if (rdev->new_fw)
1576                                 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1577                         else
1578                                 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1579                 }
1580
1581                 /* put the engine back into the active state */
1582                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1583                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1584                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1585
1586                 /* wait for training to complete */
1587                 for (i = 0; i < rdev->usec_timeout; i++) {
1588                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1589                                 break;
1590                         udelay(1);
1591                 }
1592                 for (i = 0; i < rdev->usec_timeout; i++) {
1593                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1594                                 break;
1595                         udelay(1);
1596                 }
1597
1598                 if (running)
1599                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1600         }
1601
1602         return 0;
1603 }
1604
1605 static int si_init_microcode(struct radeon_device *rdev)
1606 {
1607         const char *chip_name;
1608         const char *new_chip_name;
1609         size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1610         size_t smc_req_size, mc2_req_size;
1611         char fw_name[30];
1612         int err;
1613         int new_fw = 0;
1614
1615         DRM_DEBUG("\n");
1616
1617         switch (rdev->family) {
1618         case CHIP_TAHITI:
1619                 chip_name = "TAHITI";
1620                 new_chip_name = "tahiti";
1621                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1622                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1623                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1624                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1625                 mc_req_size = SI_MC_UCODE_SIZE * 4;
1626                 mc2_req_size = TAHITI_MC_UCODE_SIZE * 4;
1627                 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
1628                 break;
1629         case CHIP_PITCAIRN:
1630                 chip_name = "PITCAIRN";
1631                 new_chip_name = "pitcairn";
1632                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1633                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1634                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1635                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1636                 mc_req_size = SI_MC_UCODE_SIZE * 4;
1637                 mc2_req_size = PITCAIRN_MC_UCODE_SIZE * 4;
1638                 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
1639                 break;
1640         case CHIP_VERDE:
1641                 chip_name = "VERDE";
1642                 new_chip_name = "verde";
1643                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1644                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1645                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1646                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1647                 mc_req_size = SI_MC_UCODE_SIZE * 4;
1648                 mc2_req_size = VERDE_MC_UCODE_SIZE * 4;
1649                 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
1650                 break;
1651         case CHIP_OLAND:
1652                 chip_name = "OLAND";
1653                 new_chip_name = "oland";
1654                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1655                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1656                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1657                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1658                 mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1659                 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
1660                 break;
1661         case CHIP_HAINAN:
1662                 chip_name = "HAINAN";
1663                 new_chip_name = "hainan";
1664                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1665                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1666                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1667                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1668                 mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1669                 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
1670                 break;
1671         default: BUG();
1672         }
1673
1674         DRM_INFO("Loading %s Microcode\n", new_chip_name);
1675
1676         ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_pfp", new_chip_name);
1677         err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1678         if (err) {
1679                 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_pfp", chip_name);
1680                 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1681                 if (err)
1682                         goto out;
1683                 if (rdev->pfp_fw->datasize != pfp_req_size) {
1684                         printk(KERN_ERR
1685                                "si_cp: Bogus length %zu in firmware \"%s\"\n",
1686                                rdev->pfp_fw->datasize, fw_name);
1687                         err = -EINVAL;
1688                         goto out;
1689                 }
1690         } else {
1691                 err = radeon_ucode_validate(rdev->pfp_fw);
1692                 if (err) {
1693                         printk(KERN_ERR
1694                                "si_cp: validation failed for firmware \"%s\"\n",
1695                                fw_name);
1696                         goto out;
1697                 } else {
1698                         new_fw++;
1699                 }
1700         }
1701
1702         ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_me", new_chip_name);
1703         err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1704         if (err) {
1705                 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_me", chip_name);
1706                 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1707                 if (err)
1708                         goto out;
1709                 if (rdev->me_fw->datasize != me_req_size) {
1710                         printk(KERN_ERR
1711                                "si_cp: Bogus length %zu in firmware \"%s\"\n",
1712                                rdev->me_fw->datasize, fw_name);
1713                         err = -EINVAL;
1714                 }
1715         } else {
1716                 err = radeon_ucode_validate(rdev->me_fw);
1717                 if (err) {
1718                         printk(KERN_ERR
1719                                "si_cp: validation failed for firmware \"%s\"\n",
1720                                fw_name);
1721                         goto out;
1722                 } else {
1723                         new_fw++;
1724                 }
1725         }
1726
1727         ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_ce", new_chip_name);
1728         err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1729         if (err) {
1730                 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_ce", chip_name);
1731                 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1732                 if (err)
1733                         goto out;
1734                 if (rdev->ce_fw->datasize != ce_req_size) {
1735                         printk(KERN_ERR
1736                                "si_cp: Bogus length %zu in firmware \"%s\"\n",
1737                                rdev->ce_fw->datasize, fw_name);
1738                         err = -EINVAL;
1739                 }
1740         } else {
1741                 err = radeon_ucode_validate(rdev->ce_fw);
1742                 if (err) {
1743                         printk(KERN_ERR
1744                                "si_cp: validation failed for firmware \"%s\"\n",
1745                                fw_name);
1746                         goto out;
1747                 } else {
1748                         new_fw++;
1749                 }
1750         }
1751
1752         ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_rlc", new_chip_name);
1753         err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1754         if (err) {
1755                 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_rlc", chip_name);
1756                 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1757                 if (err)
1758                         goto out;
1759                 if (rdev->rlc_fw->datasize != rlc_req_size) {
1760                         printk(KERN_ERR
1761                                "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1762                                rdev->rlc_fw->datasize, fw_name);
1763                         err = -EINVAL;
1764                 }
1765         } else {
1766                 err = radeon_ucode_validate(rdev->rlc_fw);
1767                 if (err) {
1768                         printk(KERN_ERR
1769                                "si_cp: validation failed for firmware \"%s\"\n",
1770                                fw_name);
1771                         goto out;
1772                 } else {
1773                         new_fw++;
1774                 }
1775         }
1776
1777         ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mc", new_chip_name);
1778         err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1779         if (err) {
1780                 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mc2", chip_name);
1781                 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1782                 if (err) {
1783                         ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mc", chip_name);
1784                         err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1785                         if (err)
1786                                 goto out;
1787                 }
1788                 if ((rdev->mc_fw->datasize != mc_req_size) &&
1789                     (rdev->mc_fw->datasize != mc2_req_size)) {
1790                         printk(KERN_ERR
1791                                "si_mc: Bogus length %zu in firmware \"%s\"\n",
1792                                rdev->mc_fw->datasize, fw_name);
1793                         err = -EINVAL;
1794                 }
1795                 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->datasize);
1796         } else {
1797                 err = radeon_ucode_validate(rdev->mc_fw);
1798                 if (err) {
1799                         printk(KERN_ERR
1800                                "si_cp: validation failed for firmware \"%s\"\n",
1801                                fw_name);
1802                         goto out;
1803                 } else {
1804                         new_fw++;
1805                 }
1806         }
1807
1808         ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_smc", new_chip_name);
1809         err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1810         if (err) {
1811                 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_smc", chip_name);
1812                 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1813                 if (err) {
1814                         printk(KERN_ERR
1815                                "smc: error loading firmware \"%s\"\n",
1816                                fw_name);
1817                         release_firmware(rdev->smc_fw);
1818                         rdev->smc_fw = NULL;
1819                         err = 0;
1820                 } else if (rdev->smc_fw->datasize != smc_req_size) {
1821                         printk(KERN_ERR
1822                                "si_smc: Bogus length %zu in firmware \"%s\"\n",
1823                                rdev->smc_fw->datasize, fw_name);
1824                         err = -EINVAL;
1825                 }
1826         } else {
1827                 err = radeon_ucode_validate(rdev->smc_fw);
1828                 if (err) {
1829                         printk(KERN_ERR
1830                                "si_cp: validation failed for firmware \"%s\"\n",
1831                                fw_name);
1832                         goto out;
1833                 } else {
1834                         new_fw++;
1835                 }
1836         }
1837
1838         if (new_fw == 0) {
1839                 rdev->new_fw = false;
1840         } else if (new_fw < 6) {
1841                 printk(KERN_ERR "si_fw: mixing new and old firmware!\n");
1842                 err = -EINVAL;
1843         } else {
1844                 rdev->new_fw = true;
1845         }
1846 out:
1847         if (err) {
1848                 if (err != -EINVAL)
1849                         printk(KERN_ERR
1850                                "si_cp: Failed to load firmware \"%s\"\n",
1851                                fw_name);
1852                 release_firmware(rdev->pfp_fw);
1853                 rdev->pfp_fw = NULL;
1854                 release_firmware(rdev->me_fw);
1855                 rdev->me_fw = NULL;
1856                 release_firmware(rdev->ce_fw);
1857                 rdev->ce_fw = NULL;
1858                 release_firmware(rdev->rlc_fw);
1859                 rdev->rlc_fw = NULL;
1860                 release_firmware(rdev->mc_fw);
1861                 rdev->mc_fw = NULL;
1862                 release_firmware(rdev->smc_fw);
1863                 rdev->smc_fw = NULL;
1864         }
1865         return err;
1866 }
1867
1868 /**
1869  * si_fini_microcode - drop the firmwares image references
1870  *
1871  * @rdev: radeon_device pointer
1872  *
1873  * Drop the pfp, me, rlc, mc and ce firmware image references.
1874  * Called at driver shutdown.
1875  */
1876 static void si_fini_microcode(struct radeon_device *rdev)
1877 {
1878         release_firmware(rdev->pfp_fw);
1879         rdev->pfp_fw = NULL;
1880         release_firmware(rdev->me_fw);
1881         rdev->me_fw = NULL;
1882         release_firmware(rdev->rlc_fw);
1883         rdev->rlc_fw = NULL;
1884         release_firmware(rdev->mc_fw);
1885         rdev->mc_fw = NULL;
1886         release_firmware(rdev->smc_fw);
1887         rdev->smc_fw = NULL;
1888         release_firmware(rdev->ce_fw);
1889         rdev->ce_fw = NULL;
1890 }
1891
1892 /* watermark setup */
1893 static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1894                                    struct radeon_crtc *radeon_crtc,
1895                                    struct drm_display_mode *mode,
1896                                    struct drm_display_mode *other_mode)
1897 {
1898         u32 tmp, buffer_alloc, i;
1899         u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1900         /*
1901          * Line Buffer Setup
1902          * There are 3 line buffers, each one shared by 2 display controllers.
1903          * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1904          * the display controllers.  The paritioning is done via one of four
1905          * preset allocations specified in bits 21:20:
1906          *  0 - half lb
1907          *  2 - whole lb, other crtc must be disabled
1908          */
1909         /* this can get tricky if we have two large displays on a paired group
1910          * of crtcs.  Ideally for multiple large displays we'd assign them to
1911          * non-linked crtcs for maximum line buffer allocation.
1912          */
1913         if (radeon_crtc->base.enabled && mode) {
1914                 if (other_mode) {
1915                         tmp = 0; /* 1/2 */
1916                         buffer_alloc = 1;
1917                 } else {
1918                         tmp = 2; /* whole */
1919                         buffer_alloc = 2;
1920                 }
1921         } else {
1922                 tmp = 0;
1923                 buffer_alloc = 0;
1924         }
1925
1926         WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1927                DC_LB_MEMORY_CONFIG(tmp));
1928
1929         WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1930                DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1931         for (i = 0; i < rdev->usec_timeout; i++) {
1932                 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1933                     DMIF_BUFFERS_ALLOCATED_COMPLETED)
1934                         break;
1935                 udelay(1);
1936         }
1937
1938         if (radeon_crtc->base.enabled && mode) {
1939                 switch (tmp) {
1940                 case 0:
1941                 default:
1942                         return 4096 * 2;
1943                 case 2:
1944                         return 8192 * 2;
1945                 }
1946         }
1947
1948         /* controller not enabled, so no lb used */
1949         return 0;
1950 }
1951
1952 static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
1953 {
1954         u32 tmp = RREG32(MC_SHARED_CHMAP);
1955
1956         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1957         case 0:
1958         default:
1959                 return 1;
1960         case 1:
1961                 return 2;
1962         case 2:
1963                 return 4;
1964         case 3:
1965                 return 8;
1966         case 4:
1967                 return 3;
1968         case 5:
1969                 return 6;
1970         case 6:
1971                 return 10;
1972         case 7:
1973                 return 12;
1974         case 8:
1975                 return 16;
1976         }
1977 }
1978
1979 struct dce6_wm_params {
1980         u32 dram_channels; /* number of dram channels */
1981         u32 yclk;          /* bandwidth per dram data pin in kHz */
1982         u32 sclk;          /* engine clock in kHz */
1983         u32 disp_clk;      /* display clock in kHz */
1984         u32 src_width;     /* viewport width */
1985         u32 active_time;   /* active display time in ns */
1986         u32 blank_time;    /* blank time in ns */
1987         bool interlaced;    /* mode is interlaced */
1988         fixed20_12 vsc;    /* vertical scale ratio */
1989         u32 num_heads;     /* number of active crtcs */
1990         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1991         u32 lb_size;       /* line buffer allocated to pipe */
1992         u32 vtaps;         /* vertical scaler taps */
1993 };
1994
1995 static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1996 {
1997         /* Calculate raw DRAM Bandwidth */
1998         fixed20_12 dram_efficiency; /* 0.7 */
1999         fixed20_12 yclk, dram_channels, bandwidth;
2000         fixed20_12 a;
2001
2002         a.full = dfixed_const(1000);
2003         yclk.full = dfixed_const(wm->yclk);
2004         yclk.full = dfixed_div(yclk, a);
2005         dram_channels.full = dfixed_const(wm->dram_channels * 4);
2006         a.full = dfixed_const(10);
2007         dram_efficiency.full = dfixed_const(7);
2008         dram_efficiency.full = dfixed_div(dram_efficiency, a);
2009         bandwidth.full = dfixed_mul(dram_channels, yclk);
2010         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
2011
2012         return dfixed_trunc(bandwidth);
2013 }
2014
2015 static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2016 {
2017         /* Calculate DRAM Bandwidth and the part allocated to display. */
2018         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2019         fixed20_12 yclk, dram_channels, bandwidth;
2020         fixed20_12 a;
2021
2022         a.full = dfixed_const(1000);
2023         yclk.full = dfixed_const(wm->yclk);
2024         yclk.full = dfixed_div(yclk, a);
2025         dram_channels.full = dfixed_const(wm->dram_channels * 4);
2026         a.full = dfixed_const(10);
2027         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2028         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2029         bandwidth.full = dfixed_mul(dram_channels, yclk);
2030         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2031
2032         return dfixed_trunc(bandwidth);
2033 }
2034
2035 static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
2036 {
2037         /* Calculate the display Data return Bandwidth */
2038         fixed20_12 return_efficiency; /* 0.8 */
2039         fixed20_12 sclk, bandwidth;
2040         fixed20_12 a;
2041
2042         a.full = dfixed_const(1000);
2043         sclk.full = dfixed_const(wm->sclk);
2044         sclk.full = dfixed_div(sclk, a);
2045         a.full = dfixed_const(10);
2046         return_efficiency.full = dfixed_const(8);
2047         return_efficiency.full = dfixed_div(return_efficiency, a);
2048         a.full = dfixed_const(32);
2049         bandwidth.full = dfixed_mul(a, sclk);
2050         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2051
2052         return dfixed_trunc(bandwidth);
2053 }
2054
2055 static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
2056 {
2057         return 32;
2058 }
2059
2060 static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
2061 {
2062         /* Calculate the DMIF Request Bandwidth */
2063         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2064         fixed20_12 disp_clk, sclk, bandwidth;
2065         fixed20_12 a, b1, b2;
2066         u32 min_bandwidth;
2067
2068         a.full = dfixed_const(1000);
2069         disp_clk.full = dfixed_const(wm->disp_clk);
2070         disp_clk.full = dfixed_div(disp_clk, a);
2071         a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
2072         b1.full = dfixed_mul(a, disp_clk);
2073
2074         a.full = dfixed_const(1000);
2075         sclk.full = dfixed_const(wm->sclk);
2076         sclk.full = dfixed_div(sclk, a);
2077         a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
2078         b2.full = dfixed_mul(a, sclk);
2079
2080         a.full = dfixed_const(10);
2081         disp_clk_request_efficiency.full = dfixed_const(8);
2082         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2083
2084         min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
2085
2086         a.full = dfixed_const(min_bandwidth);
2087         bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
2088
2089         return dfixed_trunc(bandwidth);
2090 }
2091
2092 static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
2093 {
2094         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2095         u32 dram_bandwidth = dce6_dram_bandwidth(wm);
2096         u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
2097         u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
2098
2099         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2100 }
2101
2102 static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
2103 {
2104         /* Calculate the display mode Average Bandwidth
2105          * DisplayMode should contain the source and destination dimensions,
2106          * timing, etc.
2107          */
2108         fixed20_12 bpp;
2109         fixed20_12 line_time;
2110         fixed20_12 src_width;
2111         fixed20_12 bandwidth;
2112         fixed20_12 a;
2113
2114         a.full = dfixed_const(1000);
2115         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2116         line_time.full = dfixed_div(line_time, a);
2117         bpp.full = dfixed_const(wm->bytes_per_pixel);
2118         src_width.full = dfixed_const(wm->src_width);
2119         bandwidth.full = dfixed_mul(src_width, bpp);
2120         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2121         bandwidth.full = dfixed_div(bandwidth, line_time);
2122
2123         return dfixed_trunc(bandwidth);
2124 }
2125
2126 static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
2127 {
2128         /* First calcualte the latency in ns */
2129         u32 mc_latency = 2000; /* 2000 ns. */
2130         u32 available_bandwidth = dce6_available_bandwidth(wm);
2131         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2132         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2133         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2134         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2135                 (wm->num_heads * cursor_line_pair_return_time);
2136         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2137         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2138         u32 tmp, dmif_size = 12288;
2139         fixed20_12 a, b, c;
2140
2141         if (wm->num_heads == 0)
2142                 return 0;
2143
2144         a.full = dfixed_const(2);
2145         b.full = dfixed_const(1);
2146         if ((wm->vsc.full > a.full) ||
2147             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2148             (wm->vtaps >= 5) ||
2149             ((wm->vsc.full >= a.full) && wm->interlaced))
2150                 max_src_lines_per_dst_line = 4;
2151         else
2152                 max_src_lines_per_dst_line = 2;
2153
2154         a.full = dfixed_const(available_bandwidth);
2155         b.full = dfixed_const(wm->num_heads);
2156         a.full = dfixed_div(a, b);
2157
2158         b.full = dfixed_const(mc_latency + 512);
2159         c.full = dfixed_const(wm->disp_clk);
2160         b.full = dfixed_div(b, c);
2161
2162         c.full = dfixed_const(dmif_size);
2163         b.full = dfixed_div(c, b);
2164
2165         tmp = min(dfixed_trunc(a), dfixed_trunc(b));
2166
2167         b.full = dfixed_const(1000);
2168         c.full = dfixed_const(wm->disp_clk);
2169         b.full = dfixed_div(c, b);
2170         c.full = dfixed_const(wm->bytes_per_pixel);
2171         b.full = dfixed_mul(b, c);
2172
2173         lb_fill_bw = min(tmp, dfixed_trunc(b));
2174
2175         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2176         b.full = dfixed_const(1000);
2177         c.full = dfixed_const(lb_fill_bw);
2178         b.full = dfixed_div(c, b);
2179         a.full = dfixed_div(a, b);
2180         line_fill_time = dfixed_trunc(a);
2181
2182         if (line_fill_time < wm->active_time)
2183                 return latency;
2184         else
2185                 return latency + (line_fill_time - wm->active_time);
2186
2187 }
2188
2189 static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2190 {
2191         if (dce6_average_bandwidth(wm) <=
2192             (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
2193                 return true;
2194         else
2195                 return false;
2196 };
2197
2198 static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2199 {
2200         if (dce6_average_bandwidth(wm) <=
2201             (dce6_available_bandwidth(wm) / wm->num_heads))
2202                 return true;
2203         else
2204                 return false;
2205 };
2206
2207 static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2208 {
2209         u32 lb_partitions = wm->lb_size / wm->src_width;
2210         u32 line_time = wm->active_time + wm->blank_time;
2211         u32 latency_tolerant_lines;
2212         u32 latency_hiding;
2213         fixed20_12 a;
2214
2215         a.full = dfixed_const(1);
2216         if (wm->vsc.full > a.full)
2217                 latency_tolerant_lines = 1;
2218         else {
2219                 if (lb_partitions <= (wm->vtaps + 1))
2220                         latency_tolerant_lines = 1;
2221                 else
2222                         latency_tolerant_lines = 2;
2223         }
2224
2225         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2226
2227         if (dce6_latency_watermark(wm) <= latency_hiding)
2228                 return true;
2229         else
2230                 return false;
2231 }
2232
2233 static void dce6_program_watermarks(struct radeon_device *rdev,
2234                                          struct radeon_crtc *radeon_crtc,
2235                                          u32 lb_size, u32 num_heads)
2236 {
2237         struct drm_display_mode *mode = &radeon_crtc->base.mode;
2238         struct dce6_wm_params wm_low, wm_high;
2239         u32 dram_channels;
2240         u32 pixel_period;
2241         u32 line_time = 0;
2242         u32 latency_watermark_a = 0, latency_watermark_b = 0;
2243         u32 priority_a_mark = 0, priority_b_mark = 0;
2244         u32 priority_a_cnt = PRIORITY_OFF;
2245         u32 priority_b_cnt = PRIORITY_OFF;
2246         u32 tmp, arb_control3;
2247         fixed20_12 a, b, c;
2248
2249         if (radeon_crtc->base.enabled && num_heads && mode) {
2250                 pixel_period = 1000000 / (u32)mode->clock;
2251                 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2252                 priority_a_cnt = 0;
2253                 priority_b_cnt = 0;
2254
2255                 if (rdev->family == CHIP_ARUBA)
2256                         dram_channels = evergreen_get_number_of_dram_channels(rdev);
2257                 else
2258                         dram_channels = si_get_number_of_dram_channels(rdev);
2259
2260                 /* watermark for high clocks */
2261                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2262                         wm_high.yclk =
2263                                 radeon_dpm_get_mclk(rdev, false) * 10;
2264                         wm_high.sclk =
2265                                 radeon_dpm_get_sclk(rdev, false) * 10;
2266                 } else {
2267                         wm_high.yclk = rdev->pm.current_mclk * 10;
2268                         wm_high.sclk = rdev->pm.current_sclk * 10;
2269                 }
2270
2271                 wm_high.disp_clk = mode->clock;
2272                 wm_high.src_width = mode->crtc_hdisplay;
2273                 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2274                 wm_high.blank_time = line_time - wm_high.active_time;
2275                 wm_high.interlaced = false;
2276                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2277                         wm_high.interlaced = true;
2278                 wm_high.vsc = radeon_crtc->vsc;
2279                 wm_high.vtaps = 1;
2280                 if (radeon_crtc->rmx_type != RMX_OFF)
2281                         wm_high.vtaps = 2;
2282                 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2283                 wm_high.lb_size = lb_size;
2284                 wm_high.dram_channels = dram_channels;
2285                 wm_high.num_heads = num_heads;
2286
2287                 /* watermark for low clocks */
2288                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2289                         wm_low.yclk =
2290                                 radeon_dpm_get_mclk(rdev, true) * 10;
2291                         wm_low.sclk =
2292                                 radeon_dpm_get_sclk(rdev, true) * 10;
2293                 } else {
2294                         wm_low.yclk = rdev->pm.current_mclk * 10;
2295                         wm_low.sclk = rdev->pm.current_sclk * 10;
2296                 }
2297
2298                 wm_low.disp_clk = mode->clock;
2299                 wm_low.src_width = mode->crtc_hdisplay;
2300                 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2301                 wm_low.blank_time = line_time - wm_low.active_time;
2302                 wm_low.interlaced = false;
2303                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2304                         wm_low.interlaced = true;
2305                 wm_low.vsc = radeon_crtc->vsc;
2306                 wm_low.vtaps = 1;
2307                 if (radeon_crtc->rmx_type != RMX_OFF)
2308                         wm_low.vtaps = 2;
2309                 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2310                 wm_low.lb_size = lb_size;
2311                 wm_low.dram_channels = dram_channels;
2312                 wm_low.num_heads = num_heads;
2313
2314                 /* set for high clocks */
2315                 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
2316                 /* set for low clocks */
2317                 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
2318
2319                 /* possibly force display priority to high */
2320                 /* should really do this at mode validation time... */
2321                 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2322                     !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2323                     !dce6_check_latency_hiding(&wm_high) ||
2324                     (rdev->disp_priority == 2)) {
2325                         DRM_DEBUG_KMS("force priority to high\n");
2326                         priority_a_cnt |= PRIORITY_ALWAYS_ON;
2327                         priority_b_cnt |= PRIORITY_ALWAYS_ON;
2328                 }
2329                 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2330                     !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2331                     !dce6_check_latency_hiding(&wm_low) ||
2332                     (rdev->disp_priority == 2)) {
2333                         DRM_DEBUG_KMS("force priority to high\n");
2334                         priority_a_cnt |= PRIORITY_ALWAYS_ON;
2335                         priority_b_cnt |= PRIORITY_ALWAYS_ON;
2336                 }
2337
2338                 a.full = dfixed_const(1000);
2339                 b.full = dfixed_const(mode->clock);
2340                 b.full = dfixed_div(b, a);
2341                 c.full = dfixed_const(latency_watermark_a);
2342                 c.full = dfixed_mul(c, b);
2343                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2344                 c.full = dfixed_div(c, a);
2345                 a.full = dfixed_const(16);
2346                 c.full = dfixed_div(c, a);
2347                 priority_a_mark = dfixed_trunc(c);
2348                 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2349
2350                 a.full = dfixed_const(1000);
2351                 b.full = dfixed_const(mode->clock);
2352                 b.full = dfixed_div(b, a);
2353                 c.full = dfixed_const(latency_watermark_b);
2354                 c.full = dfixed_mul(c, b);
2355                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2356                 c.full = dfixed_div(c, a);
2357                 a.full = dfixed_const(16);
2358                 c.full = dfixed_div(c, a);
2359                 priority_b_mark = dfixed_trunc(c);
2360                 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2361         }
2362
2363         /* select wm A */
2364         arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2365         tmp = arb_control3;
2366         tmp &= ~LATENCY_WATERMARK_MASK(3);
2367         tmp |= LATENCY_WATERMARK_MASK(1);
2368         WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2369         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2370                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2371                 LATENCY_HIGH_WATERMARK(line_time)));
2372         /* select wm B */
2373         tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2374         tmp &= ~LATENCY_WATERMARK_MASK(3);
2375         tmp |= LATENCY_WATERMARK_MASK(2);
2376         WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2377         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2378                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2379                 LATENCY_HIGH_WATERMARK(line_time)));
2380         /* restore original selection */
2381         WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2382
2383         /* write the priority marks */
2384         WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2385         WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2386
2387         /* save values for DPM */
2388         radeon_crtc->line_time = line_time;
2389         radeon_crtc->wm_high = latency_watermark_a;
2390         radeon_crtc->wm_low = latency_watermark_b;
2391 }
2392
2393 void dce6_bandwidth_update(struct radeon_device *rdev)
2394 {
2395         struct drm_display_mode *mode0 = NULL;
2396         struct drm_display_mode *mode1 = NULL;
2397         u32 num_heads = 0, lb_size;
2398         int i;
2399
2400         if (!rdev->mode_info.mode_config_initialized)
2401                 return;
2402
2403         radeon_update_display_priority(rdev);
2404
2405         for (i = 0; i < rdev->num_crtc; i++) {
2406                 if (rdev->mode_info.crtcs[i]->base.enabled)
2407                         num_heads++;
2408         }
2409         for (i = 0; i < rdev->num_crtc; i += 2) {
2410                 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2411                 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2412                 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2413                 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2414                 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2415                 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2416         }
2417 }
2418
2419 /*
2420  * Core functions
2421  */
2422 static void si_tiling_mode_table_init(struct radeon_device *rdev)
2423 {
2424         const u32 num_tile_mode_states = 32;
2425         u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2426
2427         switch (rdev->config.si.mem_row_size_in_kb) {
2428         case 1:
2429                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2430                 break;
2431         case 2:
2432         default:
2433                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2434                 break;
2435         case 4:
2436                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2437                 break;
2438         }
2439
2440         if ((rdev->family == CHIP_TAHITI) ||
2441             (rdev->family == CHIP_PITCAIRN)) {
2442                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2443                         switch (reg_offset) {
2444                         case 0:  /* non-AA compressed depth or any compressed stencil */
2445                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2446                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2447                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2448                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2449                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2450                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2451                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2452                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2453                                 break;
2454                         case 1:  /* 2xAA/4xAA compressed depth only */
2455                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2456                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2457                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2458                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2459                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2460                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2461                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2462                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2463                                 break;
2464                         case 2:  /* 8xAA compressed depth only */
2465                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2466                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2467                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2468                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2469                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2470                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2471                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2472                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2473                                 break;
2474                         case 3:  /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2475                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2476                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2477                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2478                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2479                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2480                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2481                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2482                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2483                                 break;
2484                         case 4:  /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2485                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2486                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2487                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2488                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2489                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2490                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2491                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2492                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2493                                 break;
2494                         case 5:  /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2495                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2496                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2497                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2498                                                  TILE_SPLIT(split_equal_to_row_size) |
2499                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2500                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2501                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2502                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2503                                 break;
2504                         case 6:  /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2505                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2506                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2507                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2508                                                  TILE_SPLIT(split_equal_to_row_size) |
2509                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2510                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2511                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2512                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2513                                 break;
2514                         case 7:  /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2515                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2516                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2517                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2518                                                  TILE_SPLIT(split_equal_to_row_size) |
2519                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2520                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2521                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2522                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2523                                 break;
2524                         case 8:  /* 1D and 1D Array Surfaces */
2525                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2526                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2527                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2528                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2529                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2530                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2531                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2532                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2533                                 break;
2534                         case 9:  /* Displayable maps. */
2535                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2536                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2537                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2538                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2539                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2540                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2541                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2542                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2543                                 break;
2544                         case 10:  /* Display 8bpp. */
2545                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2546                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2547                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2548                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2549                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2550                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2551                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2552                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2553                                 break;
2554                         case 11:  /* Display 16bpp. */
2555                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2556                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2557                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2558                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2559                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2560                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2561                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2562                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2563                                 break;
2564                         case 12:  /* Display 32bpp. */
2565                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2566                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2567                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2568                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2569                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2570                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2571                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2572                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2573                                 break;
2574                         case 13:  /* Thin. */
2575                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2576                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2577                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2578                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2579                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2580                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2581                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2582                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2583                                 break;
2584                         case 14:  /* Thin 8 bpp. */
2585                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2586                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2587                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2588                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2589                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2590                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2591                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2592                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2593                                 break;
2594                         case 15:  /* Thin 16 bpp. */
2595                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2596                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2597                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2598                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2599                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2600                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2601                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2602                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2603                                 break;
2604                         case 16:  /* Thin 32 bpp. */
2605                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2606                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2607                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2608                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2609                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2610                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2611                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2612                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2613                                 break;
2614                         case 17:  /* Thin 64 bpp. */
2615                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2616                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2617                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2618                                                  TILE_SPLIT(split_equal_to_row_size) |
2619                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2620                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2621                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2622                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2623                                 break;
2624                         case 21:  /* 8 bpp PRT. */
2625                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2626                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2627                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2628                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2629                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2630                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2631                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2632                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2633                                 break;
2634                         case 22:  /* 16 bpp PRT */
2635                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2636                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2637                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2638                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2639                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2640                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2641                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2642                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2643                                 break;
2644                         case 23:  /* 32 bpp PRT */
2645                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2646                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2647                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2648                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2649                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2650                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2651                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2652                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2653                                 break;
2654                         case 24:  /* 64 bpp PRT */
2655                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2656                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2657                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2658                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2659                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2660                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2661                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2662                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2663                                 break;
2664                         case 25:  /* 128 bpp PRT */
2665                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2666                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2667                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2668                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2669                                                  NUM_BANKS(ADDR_SURF_8_BANK) |
2670                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2671                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2672                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2673                                 break;
2674                         default:
2675                                 gb_tile_moden = 0;
2676                                 break;
2677                         }
2678                         rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2679                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2680                 }
2681         } else if ((rdev->family == CHIP_VERDE) ||
2682                    (rdev->family == CHIP_OLAND) ||
2683                    (rdev->family == CHIP_HAINAN)) {
2684                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2685                         switch (reg_offset) {
2686                         case 0:  /* non-AA compressed depth or any compressed stencil */
2687                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2688                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2689                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2690                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2691                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2692                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2693                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2694                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2695                                 break;
2696                         case 1:  /* 2xAA/4xAA compressed depth only */
2697                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2698                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2699                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2700                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2701                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2702                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2703                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2704                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2705                                 break;
2706                         case 2:  /* 8xAA compressed depth only */
2707                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2708                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2709                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2710                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2711                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2712                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2713                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2714                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2715                                 break;
2716                         case 3:  /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2717                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2718                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2719                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2720                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2721                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2722                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2723                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2724                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2725                                 break;
2726                         case 4:  /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2727                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2728                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2729                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2730                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2731                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2732                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2733                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2734                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2735                                 break;
2736                         case 5:  /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2737                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2738                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2739                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2740                                                  TILE_SPLIT(split_equal_to_row_size) |
2741                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2742                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2743                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2744                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2745                                 break;
2746                         case 6:  /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2747                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2748                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2749                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2750                                                  TILE_SPLIT(split_equal_to_row_size) |
2751                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2752                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2753                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2754                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2755                                 break;
2756                         case 7:  /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2757                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2758                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2759                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2760                                                  TILE_SPLIT(split_equal_to_row_size) |
2761                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2762                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2763                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2764                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2765                                 break;
2766                         case 8:  /* 1D and 1D Array Surfaces */
2767                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2768                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2769                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2770                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2771                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2772                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2773                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2774                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2775                                 break;
2776                         case 9:  /* Displayable maps. */
2777                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2778                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2779                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2780                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2781                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2782                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2783                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2784                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2785                                 break;
2786                         case 10:  /* Display 8bpp. */
2787                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2788                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2789                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2790                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2791                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2792                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2793                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2794                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2795                                 break;
2796                         case 11:  /* Display 16bpp. */
2797                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2798                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2799                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2800                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2801                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2802                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2803                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2804                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2805                                 break;
2806                         case 12:  /* Display 32bpp. */
2807                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2808                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2809                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2810                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2811                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2812                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2813                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2814                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2815                                 break;
2816                         case 13:  /* Thin. */
2817                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2818                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2819                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2820                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2821                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2822                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2823                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2824                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2825                                 break;
2826                         case 14:  /* Thin 8 bpp. */
2827                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2828                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2829                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2830                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2831                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2832                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2833                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2834                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2835                                 break;
2836                         case 15:  /* Thin 16 bpp. */
2837                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2838                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2839                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2840                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2841                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2842                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2843                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2844                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2845                                 break;
2846                         case 16:  /* Thin 32 bpp. */
2847                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2848                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2849                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2850                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2851                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2852                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2853                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2854                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2855                                 break;
2856                         case 17:  /* Thin 64 bpp. */
2857                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2858                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2859                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2860                                                  TILE_SPLIT(split_equal_to_row_size) |
2861                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2862                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2863                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2864                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2865                                 break;
2866                         case 21:  /* 8 bpp PRT. */
2867                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2868                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2869                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2870                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2871                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2872                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2873                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2874                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2875                                 break;
2876                         case 22:  /* 16 bpp PRT */
2877                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2878                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2879                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2880                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2881                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2882                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2883                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2884                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2885                                 break;
2886                         case 23:  /* 32 bpp PRT */
2887                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2888                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2889                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2890                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2891                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2892                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2893                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2894                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2895                                 break;
2896                         case 24:  /* 64 bpp PRT */
2897                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2898                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2899                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2900                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2901                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2902                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2903                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2904                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2905                                 break;
2906                         case 25:  /* 128 bpp PRT */
2907                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2908                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2909                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2910                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2911                                                  NUM_BANKS(ADDR_SURF_8_BANK) |
2912                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2913                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2914                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2915                                 break;
2916                         default:
2917                                 gb_tile_moden = 0;
2918                                 break;
2919                         }
2920                         rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2921                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2922                 }
2923         } else
2924                 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2925 }
2926
2927 static void si_select_se_sh(struct radeon_device *rdev,
2928                             u32 se_num, u32 sh_num)
2929 {
2930         u32 data = INSTANCE_BROADCAST_WRITES;
2931
2932         if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
2933                 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
2934         else if (se_num == 0xffffffff)
2935                 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2936         else if (sh_num == 0xffffffff)
2937                 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2938         else
2939                 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2940         WREG32(GRBM_GFX_INDEX, data);
2941 }
2942
2943 static u32 si_create_bitmask(u32 bit_width)
2944 {
2945         u32 i, mask = 0;
2946
2947         for (i = 0; i < bit_width; i++) {
2948                 mask <<= 1;
2949                 mask |= 1;
2950         }
2951         return mask;
2952 }
2953
2954 static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2955 {
2956         u32 data, mask;
2957
2958         data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2959         if (data & 1)
2960                 data &= INACTIVE_CUS_MASK;
2961         else
2962                 data = 0;
2963         data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2964
2965         data >>= INACTIVE_CUS_SHIFT;
2966
2967         mask = si_create_bitmask(cu_per_sh);
2968
2969         return ~data & mask;
2970 }
2971
2972 static void si_setup_spi(struct radeon_device *rdev,
2973                          u32 se_num, u32 sh_per_se,
2974                          u32 cu_per_sh)
2975 {
2976         int i, j, k;
2977         u32 data, mask, active_cu;
2978
2979         for (i = 0; i < se_num; i++) {
2980                 for (j = 0; j < sh_per_se; j++) {
2981                         si_select_se_sh(rdev, i, j);
2982                         data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2983                         active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2984
2985                         mask = 1;
2986                         for (k = 0; k < 16; k++) {
2987                                 mask <<= k;
2988                                 if (active_cu & mask) {
2989                                         data &= ~mask;
2990                                         WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2991                                         break;
2992                                 }
2993                         }
2994                 }
2995         }
2996         si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2997 }
2998
2999 static u32 si_get_rb_disabled(struct radeon_device *rdev,
3000                               u32 max_rb_num_per_se,
3001                               u32 sh_per_se)
3002 {
3003         u32 data, mask;
3004
3005         data = RREG32(CC_RB_BACKEND_DISABLE);
3006         if (data & 1)
3007                 data &= BACKEND_DISABLE_MASK;
3008         else
3009                 data = 0;
3010         data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3011
3012         data >>= BACKEND_DISABLE_SHIFT;
3013
3014         mask = si_create_bitmask(max_rb_num_per_se / sh_per_se);
3015
3016         return data & mask;
3017 }
3018
3019 static void si_setup_rb(struct radeon_device *rdev,
3020                         u32 se_num, u32 sh_per_se,
3021                         u32 max_rb_num_per_se)
3022 {
3023         int i, j;
3024         u32 data, mask;
3025         u32 disabled_rbs = 0;
3026         u32 enabled_rbs = 0;
3027
3028         for (i = 0; i < se_num; i++) {
3029                 for (j = 0; j < sh_per_se; j++) {
3030                         si_select_se_sh(rdev, i, j);
3031                         data = si_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3032                         disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
3033                 }
3034         }
3035         si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3036
3037         mask = 1;
3038         for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3039                 if (!(disabled_rbs & mask))
3040                         enabled_rbs |= mask;
3041                 mask <<= 1;
3042         }
3043
3044         rdev->config.si.backend_enable_mask = enabled_rbs;
3045
3046         for (i = 0; i < se_num; i++) {
3047                 si_select_se_sh(rdev, i, 0xffffffff);
3048                 data = 0;
3049                 for (j = 0; j < sh_per_se; j++) {
3050                         switch (enabled_rbs & 3) {
3051                         case 1:
3052                                 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3053                                 break;
3054                         case 2:
3055                                 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3056                                 break;
3057                         case 3:
3058                         default:
3059                                 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3060                                 break;
3061                         }
3062                         enabled_rbs >>= 2;
3063                 }
3064                 WREG32(PA_SC_RASTER_CONFIG, data);
3065         }
3066         si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3067 }
3068
3069 static void si_gpu_init(struct radeon_device *rdev)
3070 {
3071         u32 gb_addr_config = 0;
3072         u32 mc_shared_chmap, mc_arb_ramcfg;
3073         u32 sx_debug_1;
3074         u32 hdp_host_path_cntl;
3075         u32 tmp;
3076         int i, j;
3077
3078         switch (rdev->family) {
3079         case CHIP_TAHITI:
3080                 rdev->config.si.max_shader_engines = 2;
3081                 rdev->config.si.max_tile_pipes = 12;
3082                 rdev->config.si.max_cu_per_sh = 8;
3083                 rdev->config.si.max_sh_per_se = 2;
3084                 rdev->config.si.max_backends_per_se = 4;
3085                 rdev->config.si.max_texture_channel_caches = 12;
3086                 rdev->config.si.max_gprs = 256;
3087                 rdev->config.si.max_gs_threads = 32;
3088                 rdev->config.si.max_hw_contexts = 8;
3089
3090                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3091                 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3092                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3093                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3094                 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
3095                 break;
3096         case CHIP_PITCAIRN:
3097                 rdev->config.si.max_shader_engines = 2;
3098                 rdev->config.si.max_tile_pipes = 8;
3099                 rdev->config.si.max_cu_per_sh = 5;
3100                 rdev->config.si.max_sh_per_se = 2;
3101                 rdev->config.si.max_backends_per_se = 4;
3102                 rdev->config.si.max_texture_channel_caches = 8;
3103                 rdev->config.si.max_gprs = 256;
3104                 rdev->config.si.max_gs_threads = 32;
3105                 rdev->config.si.max_hw_contexts = 8;
3106
3107                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3108                 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3109                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3110                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3111                 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
3112                 break;
3113         case CHIP_VERDE:
3114         default:
3115                 rdev->config.si.max_shader_engines = 1;
3116                 rdev->config.si.max_tile_pipes = 4;
3117                 rdev->config.si.max_cu_per_sh = 5;
3118                 rdev->config.si.max_sh_per_se = 2;
3119                 rdev->config.si.max_backends_per_se = 4;
3120                 rdev->config.si.max_texture_channel_caches = 4;
3121                 rdev->config.si.max_gprs = 256;
3122                 rdev->config.si.max_gs_threads = 32;
3123                 rdev->config.si.max_hw_contexts = 8;
3124
3125                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3126                 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3127                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3128                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3129                 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
3130                 break;
3131         case CHIP_OLAND:
3132                 rdev->config.si.max_shader_engines = 1;
3133                 rdev->config.si.max_tile_pipes = 4;
3134                 rdev->config.si.max_cu_per_sh = 6;
3135                 rdev->config.si.max_sh_per_se = 1;
3136                 rdev->config.si.max_backends_per_se = 2;
3137                 rdev->config.si.max_texture_channel_caches = 4;
3138                 rdev->config.si.max_gprs = 256;
3139                 rdev->config.si.max_gs_threads = 16;
3140                 rdev->config.si.max_hw_contexts = 8;
3141
3142                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3143                 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3144                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3145                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3146                 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
3147                 break;
3148         case CHIP_HAINAN:
3149                 rdev->config.si.max_shader_engines = 1;
3150                 rdev->config.si.max_tile_pipes = 4;
3151                 rdev->config.si.max_cu_per_sh = 5;
3152                 rdev->config.si.max_sh_per_se = 1;
3153                 rdev->config.si.max_backends_per_se = 1;
3154                 rdev->config.si.max_texture_channel_caches = 2;
3155                 rdev->config.si.max_gprs = 256;
3156                 rdev->config.si.max_gs_threads = 16;
3157                 rdev->config.si.max_hw_contexts = 8;
3158
3159                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3160                 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3161                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3162                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3163                 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
3164                 break;
3165         }
3166
3167         /* Initialize HDP */
3168         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3169                 WREG32((0x2c14 + j), 0x00000000);
3170                 WREG32((0x2c18 + j), 0x00000000);
3171                 WREG32((0x2c1c + j), 0x00000000);
3172                 WREG32((0x2c20 + j), 0x00000000);
3173                 WREG32((0x2c24 + j), 0x00000000);
3174         }
3175
3176         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3177
3178         evergreen_fix_pci_max_read_req_size(rdev);
3179
3180         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3181
3182         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3183         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3184
3185         rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
3186         rdev->config.si.mem_max_burst_length_bytes = 256;
3187         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3188         rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3189         if (rdev->config.si.mem_row_size_in_kb > 4)
3190                 rdev->config.si.mem_row_size_in_kb = 4;
3191         /* XXX use MC settings? */
3192         rdev->config.si.shader_engine_tile_size = 32;
3193         rdev->config.si.num_gpus = 1;
3194         rdev->config.si.multi_gpu_tile_size = 64;
3195
3196         /* fix up row size */
3197         gb_addr_config &= ~ROW_SIZE_MASK;
3198         switch (rdev->config.si.mem_row_size_in_kb) {
3199         case 1:
3200         default:
3201                 gb_addr_config |= ROW_SIZE(0);
3202                 break;
3203         case 2:
3204                 gb_addr_config |= ROW_SIZE(1);
3205                 break;
3206         case 4:
3207                 gb_addr_config |= ROW_SIZE(2);
3208                 break;
3209         }
3210
3211         /* setup tiling info dword.  gb_addr_config is not adequate since it does
3212          * not have bank info, so create a custom tiling dword.
3213          * bits 3:0   num_pipes
3214          * bits 7:4   num_banks
3215          * bits 11:8  group_size
3216          * bits 15:12 row_size
3217          */
3218         rdev->config.si.tile_config = 0;
3219         switch (rdev->config.si.num_tile_pipes) {
3220         case 1:
3221                 rdev->config.si.tile_config |= (0 << 0);
3222                 break;
3223         case 2:
3224                 rdev->config.si.tile_config |= (1 << 0);
3225                 break;
3226         case 4:
3227                 rdev->config.si.tile_config |= (2 << 0);
3228                 break;
3229         case 8:
3230         default:
3231                 /* XXX what about 12? */
3232                 rdev->config.si.tile_config |= (3 << 0);
3233                 break;
3234         }       
3235         switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3236         case 0: /* four banks */
3237                 rdev->config.si.tile_config |= 0 << 4;
3238                 break;
3239         case 1: /* eight banks */
3240                 rdev->config.si.tile_config |= 1 << 4;
3241                 break;
3242         case 2: /* sixteen banks */
3243         default:
3244                 rdev->config.si.tile_config |= 2 << 4;
3245                 break;
3246         }
3247         rdev->config.si.tile_config |=
3248                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3249         rdev->config.si.tile_config |=
3250                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3251
3252         WREG32(GB_ADDR_CONFIG, gb_addr_config);
3253         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3254         WREG32(DMIF_ADDR_CALC, gb_addr_config);
3255         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3256         WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3257         WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
3258         if (rdev->has_uvd) {
3259                 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3260                 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3261                 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3262         }
3263
3264         si_tiling_mode_table_init(rdev);
3265
3266         si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3267                     rdev->config.si.max_sh_per_se,
3268                     rdev->config.si.max_backends_per_se);
3269
3270         si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3271                      rdev->config.si.max_sh_per_se,
3272                      rdev->config.si.max_cu_per_sh);
3273
3274         rdev->config.si.active_cus = 0;
3275         for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
3276                 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
3277                         rdev->config.si.active_cus +=
3278                                 hweight32(si_get_cu_active_bitmap(rdev, i, j));
3279                 }
3280         }
3281
3282         /* set HW defaults for 3D engine */
3283         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3284                                      ROQ_IB2_START(0x2b)));
3285         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3286
3287         sx_debug_1 = RREG32(SX_DEBUG_1);
3288         WREG32(SX_DEBUG_1, sx_debug_1);
3289
3290         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3291
3292         WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3293                                  SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3294                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3295                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3296
3297         WREG32(VGT_NUM_INSTANCES, 1);
3298
3299         WREG32(CP_PERFMON_CNTL, 0);
3300
3301         WREG32(SQ_CONFIG, 0);
3302
3303         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3304                                           FORCE_EOV_MAX_REZ_CNT(255)));
3305
3306         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3307                AUTO_INVLD_EN(ES_AND_GS_AUTO));
3308
3309         WREG32(VGT_GS_VERTEX_REUSE, 16);
3310         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3311
3312         WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3313         WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3314         WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3315         WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3316         WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3317         WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3318         WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3319         WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3320
3321         tmp = RREG32(HDP_MISC_CNTL);
3322         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3323         WREG32(HDP_MISC_CNTL, tmp);
3324
3325         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3326         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3327
3328         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3329
3330         udelay(50);
3331 }
3332
3333 /*
3334  * GPU scratch registers helpers function.
3335  */
3336 static void si_scratch_init(struct radeon_device *rdev)
3337 {
3338         int i;
3339
3340         rdev->scratch.num_reg = 7;
3341         rdev->scratch.reg_base = SCRATCH_REG0;
3342         for (i = 0; i < rdev->scratch.num_reg; i++) {
3343                 rdev->scratch.free[i] = true;
3344                 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3345         }
3346 }
3347
3348 void si_fence_ring_emit(struct radeon_device *rdev,
3349                         struct radeon_fence *fence)
3350 {
3351         struct radeon_ring *ring = &rdev->ring[fence->ring];
3352         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3353
3354         /* flush read cache over gart */
3355         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3356         radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3357         radeon_ring_write(ring, 0);
3358         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3359         radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3360                           PACKET3_TC_ACTION_ENA |
3361                           PACKET3_SH_KCACHE_ACTION_ENA |
3362                           PACKET3_SH_ICACHE_ACTION_ENA);
3363         radeon_ring_write(ring, 0xFFFFFFFF);
3364         radeon_ring_write(ring, 0);
3365         radeon_ring_write(ring, 10); /* poll interval */
3366         /* EVENT_WRITE_EOP - flush caches, send int */
3367         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3368         radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3369         radeon_ring_write(ring, lower_32_bits(addr));
3370         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3371         radeon_ring_write(ring, fence->seq);
3372         radeon_ring_write(ring, 0);
3373 }
3374
3375 /*
3376  * IB stuff
3377  */
3378 void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3379 {
3380         struct radeon_ring *ring = &rdev->ring[ib->ring];
3381         u32 header;
3382
3383         if (ib->is_const_ib) {
3384                 /* set switch buffer packet before const IB */
3385                 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3386                 radeon_ring_write(ring, 0);
3387
3388                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3389         } else {
3390                 u32 next_rptr;
3391                 if (ring->rptr_save_reg) {
3392                         next_rptr = ring->wptr + 3 + 4 + 8;
3393                         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3394                         radeon_ring_write(ring, ((ring->rptr_save_reg -
3395                                                   PACKET3_SET_CONFIG_REG_START) >> 2));
3396                         radeon_ring_write(ring, next_rptr);
3397                 } else if (rdev->wb.enabled) {
3398                         next_rptr = ring->wptr + 5 + 4 + 8;
3399                         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3400                         radeon_ring_write(ring, (1 << 8));
3401                         radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3402                         radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
3403                         radeon_ring_write(ring, next_rptr);
3404                 }
3405
3406                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3407         }
3408
3409         radeon_ring_write(ring, header);
3410         radeon_ring_write(ring,
3411 #ifdef __BIG_ENDIAN
3412                           (2 << 0) |
3413 #endif
3414                           (ib->gpu_addr & 0xFFFFFFFC));
3415         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3416         radeon_ring_write(ring, ib->length_dw |
3417                           (ib->vm ? (ib->vm->id << 24) : 0));
3418
3419         if (!ib->is_const_ib) {
3420                 /* flush read cache over gart for this vmid */
3421                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3422                 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3423                 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
3424                 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3425                 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3426                                   PACKET3_TC_ACTION_ENA |
3427                                   PACKET3_SH_KCACHE_ACTION_ENA |
3428                                   PACKET3_SH_ICACHE_ACTION_ENA);
3429                 radeon_ring_write(ring, 0xFFFFFFFF);
3430                 radeon_ring_write(ring, 0);
3431                 radeon_ring_write(ring, 10); /* poll interval */
3432         }
3433 }
3434
3435 /*
3436  * CP.
3437  */
3438 static void si_cp_enable(struct radeon_device *rdev, bool enable)
3439 {
3440         if (enable)
3441                 WREG32(CP_ME_CNTL, 0);
3442         else {
3443                 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3444                         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3445                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3446                 WREG32(SCRATCH_UMSK, 0);
3447                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3448                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3449                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3450         }
3451         udelay(50);
3452 }
3453
3454 static int si_cp_load_microcode(struct radeon_device *rdev)
3455 {
3456         int i;
3457
3458         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3459                 return -EINVAL;
3460
3461         si_cp_enable(rdev, false);
3462
3463         if (rdev->new_fw) {
3464                 const struct gfx_firmware_header_v1_0 *pfp_hdr =
3465                         (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
3466                 const struct gfx_firmware_header_v1_0 *ce_hdr =
3467                         (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
3468                 const struct gfx_firmware_header_v1_0 *me_hdr =
3469                         (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
3470                 const __le32 *fw_data;
3471                 u32 fw_size;
3472
3473                 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
3474                 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
3475                 radeon_ucode_print_gfx_hdr(&me_hdr->header);
3476
3477                 /* PFP */
3478                 fw_data = (const __le32 *)
3479                         ((const char *)rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3480                 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3481                 WREG32(CP_PFP_UCODE_ADDR, 0);
3482                 for (i = 0; i < fw_size; i++)
3483                         WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3484                 WREG32(CP_PFP_UCODE_ADDR, 0);
3485
3486                 /* CE */
3487                 fw_data = (const __le32 *)
3488                         ((const char *)rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3489                 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3490                 WREG32(CP_CE_UCODE_ADDR, 0);
3491                 for (i = 0; i < fw_size; i++)
3492                         WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3493                 WREG32(CP_CE_UCODE_ADDR, 0);
3494
3495                 /* ME */
3496                 fw_data = (const __be32 *)
3497                         ((const char *)rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3498                 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3499                 WREG32(CP_ME_RAM_WADDR, 0);
3500                 for (i = 0; i < fw_size; i++)
3501                         WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3502                 WREG32(CP_ME_RAM_WADDR, 0);
3503         } else {
3504                 const __be32 *fw_data;
3505
3506                 /* PFP */
3507                 fw_data = (const __be32 *)rdev->pfp_fw->data;
3508                 WREG32(CP_PFP_UCODE_ADDR, 0);
3509                 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3510                         WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3511                 WREG32(CP_PFP_UCODE_ADDR, 0);
3512
3513                 /* CE */
3514                 fw_data = (const __be32 *)rdev->ce_fw->data;
3515                 WREG32(CP_CE_UCODE_ADDR, 0);
3516                 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3517                         WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3518                 WREG32(CP_CE_UCODE_ADDR, 0);
3519
3520                 /* ME */
3521                 fw_data = (const __be32 *)rdev->me_fw->data;
3522                 WREG32(CP_ME_RAM_WADDR, 0);
3523                 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3524                         WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3525                 WREG32(CP_ME_RAM_WADDR, 0);
3526         }
3527
3528         WREG32(CP_PFP_UCODE_ADDR, 0);
3529         WREG32(CP_CE_UCODE_ADDR, 0);
3530         WREG32(CP_ME_RAM_WADDR, 0);
3531         WREG32(CP_ME_RAM_RADDR, 0);
3532         return 0;
3533 }
3534
3535 static int si_cp_start(struct radeon_device *rdev)
3536 {
3537         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3538         int r, i;
3539
3540         r = radeon_ring_lock(rdev, ring, 7 + 4);
3541         if (r) {
3542                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3543                 return r;
3544         }
3545         /* init the CP */
3546         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3547         radeon_ring_write(ring, 0x1);
3548         radeon_ring_write(ring, 0x0);
3549         radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3550         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3551         radeon_ring_write(ring, 0);
3552         radeon_ring_write(ring, 0);
3553
3554         /* init the CE partitions */
3555         radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3556         radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3557         radeon_ring_write(ring, 0xc000);
3558         radeon_ring_write(ring, 0xe000);
3559         radeon_ring_unlock_commit(rdev, ring, false);
3560
3561         si_cp_enable(rdev, true);
3562
3563         r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3564         if (r) {
3565                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3566                 return r;
3567         }
3568
3569         /* setup clear context state */
3570         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3571         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3572
3573         for (i = 0; i < si_default_size; i++)
3574                 radeon_ring_write(ring, si_default_state[i]);
3575
3576         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3577         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3578
3579         /* set clear context state */
3580         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3581         radeon_ring_write(ring, 0);
3582
3583         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3584         radeon_ring_write(ring, 0x00000316);
3585         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3586         radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3587
3588         radeon_ring_unlock_commit(rdev, ring, false);
3589
3590         for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3591                 ring = &rdev->ring[i];
3592                 r = radeon_ring_lock(rdev, ring, 2);
3593
3594                 /* clear the compute context state */
3595                 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3596                 radeon_ring_write(ring, 0);
3597
3598                 radeon_ring_unlock_commit(rdev, ring, false);
3599         }
3600
3601         return 0;
3602 }
3603
3604 static void si_cp_fini(struct radeon_device *rdev)
3605 {
3606         struct radeon_ring *ring;
3607         si_cp_enable(rdev, false);
3608
3609         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3610         radeon_ring_fini(rdev, ring);
3611         radeon_scratch_free(rdev, ring->rptr_save_reg);
3612
3613         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3614         radeon_ring_fini(rdev, ring);
3615         radeon_scratch_free(rdev, ring->rptr_save_reg);
3616
3617         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3618         radeon_ring_fini(rdev, ring);
3619         radeon_scratch_free(rdev, ring->rptr_save_reg);
3620 }
3621
3622 static int si_cp_resume(struct radeon_device *rdev)
3623 {
3624         struct radeon_ring *ring;
3625         u32 tmp;
3626         u32 rb_bufsz;
3627         int r;
3628
3629         si_enable_gui_idle_interrupt(rdev, false);
3630
3631         WREG32(CP_SEM_WAIT_TIMER, 0x0);
3632         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3633
3634         /* Set the write pointer delay */
3635         WREG32(CP_RB_WPTR_DELAY, 0);
3636
3637         WREG32(CP_DEBUG, 0);
3638         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3639
3640         /* ring 0 - compute and gfx */
3641         /* Set ring buffer size */
3642         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3643         rb_bufsz = order_base_2(ring->ring_size / 8);
3644         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3645 #ifdef __BIG_ENDIAN
3646         tmp |= BUF_SWAP_32BIT;
3647 #endif
3648         WREG32(CP_RB0_CNTL, tmp);
3649
3650         /* Initialize the ring buffer's read and write pointers */
3651         WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3652         ring->wptr = 0;
3653         WREG32(CP_RB0_WPTR, ring->wptr);
3654
3655         /* set the wb address whether it's enabled or not */
3656         WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3657         WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3658
3659         if (rdev->wb.enabled)
3660                 WREG32(SCRATCH_UMSK, 0xff);
3661         else {
3662                 tmp |= RB_NO_UPDATE;
3663                 WREG32(SCRATCH_UMSK, 0);
3664         }
3665
3666         mdelay(1);
3667         WREG32(CP_RB0_CNTL, tmp);
3668
3669         WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3670
3671         /* ring1  - compute only */
3672         /* Set ring buffer size */
3673         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3674         rb_bufsz = order_base_2(ring->ring_size / 8);
3675         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3676 #ifdef __BIG_ENDIAN
3677         tmp |= BUF_SWAP_32BIT;