2 * Copyright 2011 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
24 #include <linux/firmware.h>
25 #include <linux/module.h>
28 #include "radeon_asic.h"
29 #include <uapi_drm/radeon_drm.h>
32 #include "si_blit_shaders.h"
33 #include "clearstate_si.h"
34 #include "radeon_ucode.h"
37 MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
38 MODULE_FIRMWARE("radeon/TAHITI_me.bin");
39 MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
40 MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
41 MODULE_FIRMWARE("radeon/TAHITI_mc2.bin");
42 MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
43 MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
44 MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
45 MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
46 MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
47 MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
48 MODULE_FIRMWARE("radeon/PITCAIRN_mc2.bin");
49 MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
50 MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
51 MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
52 MODULE_FIRMWARE("radeon/VERDE_me.bin");
53 MODULE_FIRMWARE("radeon/VERDE_ce.bin");
54 MODULE_FIRMWARE("radeon/VERDE_mc.bin");
55 MODULE_FIRMWARE("radeon/VERDE_mc2.bin");
56 MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
57 MODULE_FIRMWARE("radeon/VERDE_smc.bin");
58 MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
59 MODULE_FIRMWARE("radeon/OLAND_me.bin");
60 MODULE_FIRMWARE("radeon/OLAND_ce.bin");
61 MODULE_FIRMWARE("radeon/OLAND_mc.bin");
62 MODULE_FIRMWARE("radeon/OLAND_mc2.bin");
63 MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
64 MODULE_FIRMWARE("radeon/OLAND_smc.bin");
65 MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
66 MODULE_FIRMWARE("radeon/HAINAN_me.bin");
67 MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
68 MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
69 MODULE_FIRMWARE("radeon/HAINAN_mc2.bin");
70 MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
71 MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
73 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
74 static void si_pcie_gen3_enable(struct radeon_device *rdev);
75 static void si_program_aspm(struct radeon_device *rdev);
76 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
78 static void si_init_pg(struct radeon_device *rdev);
79 static void si_init_cg(struct radeon_device *rdev);
80 static void si_fini_pg(struct radeon_device *rdev);
81 static void si_fini_cg(struct radeon_device *rdev);
82 static void si_rlc_stop(struct radeon_device *rdev);
84 static const u32 verde_rlc_save_restore_register_list[] =
86 (0x8000 << 16) | (0x98f4 >> 2),
88 (0x8040 << 16) | (0x98f4 >> 2),
90 (0x8000 << 16) | (0xe80 >> 2),
92 (0x8040 << 16) | (0xe80 >> 2),
94 (0x8000 << 16) | (0x89bc >> 2),
96 (0x8040 << 16) | (0x89bc >> 2),
98 (0x8000 << 16) | (0x8c1c >> 2),
100 (0x8040 << 16) | (0x8c1c >> 2),
102 (0x9c00 << 16) | (0x98f0 >> 2),
104 (0x9c00 << 16) | (0xe7c >> 2),
106 (0x8000 << 16) | (0x9148 >> 2),
108 (0x8040 << 16) | (0x9148 >> 2),
110 (0x9c00 << 16) | (0x9150 >> 2),
112 (0x9c00 << 16) | (0x897c >> 2),
114 (0x9c00 << 16) | (0x8d8c >> 2),
116 (0x9c00 << 16) | (0xac54 >> 2),
119 (0x9c00 << 16) | (0x98f8 >> 2),
121 (0x9c00 << 16) | (0x9910 >> 2),
123 (0x9c00 << 16) | (0x9914 >> 2),
125 (0x9c00 << 16) | (0x9918 >> 2),
127 (0x9c00 << 16) | (0x991c >> 2),
129 (0x9c00 << 16) | (0x9920 >> 2),
131 (0x9c00 << 16) | (0x9924 >> 2),
133 (0x9c00 << 16) | (0x9928 >> 2),
135 (0x9c00 << 16) | (0x992c >> 2),
137 (0x9c00 << 16) | (0x9930 >> 2),
139 (0x9c00 << 16) | (0x9934 >> 2),
141 (0x9c00 << 16) | (0x9938 >> 2),
143 (0x9c00 << 16) | (0x993c >> 2),
145 (0x9c00 << 16) | (0x9940 >> 2),
147 (0x9c00 << 16) | (0x9944 >> 2),
149 (0x9c00 << 16) | (0x9948 >> 2),
151 (0x9c00 << 16) | (0x994c >> 2),
153 (0x9c00 << 16) | (0x9950 >> 2),
155 (0x9c00 << 16) | (0x9954 >> 2),
157 (0x9c00 << 16) | (0x9958 >> 2),
159 (0x9c00 << 16) | (0x995c >> 2),
161 (0x9c00 << 16) | (0x9960 >> 2),
163 (0x9c00 << 16) | (0x9964 >> 2),
165 (0x9c00 << 16) | (0x9968 >> 2),
167 (0x9c00 << 16) | (0x996c >> 2),
169 (0x9c00 << 16) | (0x9970 >> 2),
171 (0x9c00 << 16) | (0x9974 >> 2),
173 (0x9c00 << 16) | (0x9978 >> 2),
175 (0x9c00 << 16) | (0x997c >> 2),
177 (0x9c00 << 16) | (0x9980 >> 2),
179 (0x9c00 << 16) | (0x9984 >> 2),
181 (0x9c00 << 16) | (0x9988 >> 2),
183 (0x9c00 << 16) | (0x998c >> 2),
185 (0x9c00 << 16) | (0x8c00 >> 2),
187 (0x9c00 << 16) | (0x8c14 >> 2),
189 (0x9c00 << 16) | (0x8c04 >> 2),
191 (0x9c00 << 16) | (0x8c08 >> 2),
193 (0x8000 << 16) | (0x9b7c >> 2),
195 (0x8040 << 16) | (0x9b7c >> 2),
197 (0x8000 << 16) | (0xe84 >> 2),
199 (0x8040 << 16) | (0xe84 >> 2),
201 (0x8000 << 16) | (0x89c0 >> 2),
203 (0x8040 << 16) | (0x89c0 >> 2),
205 (0x8000 << 16) | (0x914c >> 2),
207 (0x8040 << 16) | (0x914c >> 2),
209 (0x8000 << 16) | (0x8c20 >> 2),
211 (0x8040 << 16) | (0x8c20 >> 2),
213 (0x8000 << 16) | (0x9354 >> 2),
215 (0x8040 << 16) | (0x9354 >> 2),
217 (0x9c00 << 16) | (0x9060 >> 2),
219 (0x9c00 << 16) | (0x9364 >> 2),
221 (0x9c00 << 16) | (0x9100 >> 2),
223 (0x9c00 << 16) | (0x913c >> 2),
225 (0x8000 << 16) | (0x90e0 >> 2),
227 (0x8000 << 16) | (0x90e4 >> 2),
229 (0x8000 << 16) | (0x90e8 >> 2),
231 (0x8040 << 16) | (0x90e0 >> 2),
233 (0x8040 << 16) | (0x90e4 >> 2),
235 (0x8040 << 16) | (0x90e8 >> 2),
237 (0x9c00 << 16) | (0x8bcc >> 2),
239 (0x9c00 << 16) | (0x8b24 >> 2),
241 (0x9c00 << 16) | (0x88c4 >> 2),
243 (0x9c00 << 16) | (0x8e50 >> 2),
245 (0x9c00 << 16) | (0x8c0c >> 2),
247 (0x9c00 << 16) | (0x8e58 >> 2),
249 (0x9c00 << 16) | (0x8e5c >> 2),
251 (0x9c00 << 16) | (0x9508 >> 2),
253 (0x9c00 << 16) | (0x950c >> 2),
255 (0x9c00 << 16) | (0x9494 >> 2),
257 (0x9c00 << 16) | (0xac0c >> 2),
259 (0x9c00 << 16) | (0xac10 >> 2),
261 (0x9c00 << 16) | (0xac14 >> 2),
263 (0x9c00 << 16) | (0xae00 >> 2),
265 (0x9c00 << 16) | (0xac08 >> 2),
267 (0x9c00 << 16) | (0x88d4 >> 2),
269 (0x9c00 << 16) | (0x88c8 >> 2),
271 (0x9c00 << 16) | (0x88cc >> 2),
273 (0x9c00 << 16) | (0x89b0 >> 2),
275 (0x9c00 << 16) | (0x8b10 >> 2),
277 (0x9c00 << 16) | (0x8a14 >> 2),
279 (0x9c00 << 16) | (0x9830 >> 2),
281 (0x9c00 << 16) | (0x9834 >> 2),
283 (0x9c00 << 16) | (0x9838 >> 2),
285 (0x9c00 << 16) | (0x9a10 >> 2),
287 (0x8000 << 16) | (0x9870 >> 2),
289 (0x8000 << 16) | (0x9874 >> 2),
291 (0x8001 << 16) | (0x9870 >> 2),
293 (0x8001 << 16) | (0x9874 >> 2),
295 (0x8040 << 16) | (0x9870 >> 2),
297 (0x8040 << 16) | (0x9874 >> 2),
299 (0x8041 << 16) | (0x9870 >> 2),
301 (0x8041 << 16) | (0x9874 >> 2),
306 static const u32 tahiti_golden_rlc_registers[] =
308 0xc424, 0xffffffff, 0x00601005,
309 0xc47c, 0xffffffff, 0x10104040,
310 0xc488, 0xffffffff, 0x0100000a,
311 0xc314, 0xffffffff, 0x00000800,
312 0xc30c, 0xffffffff, 0x800000f4,
313 0xf4a8, 0xffffffff, 0x00000000
316 static const u32 tahiti_golden_registers[] =
318 0x9a10, 0x00010000, 0x00018208,
319 0x9830, 0xffffffff, 0x00000000,
320 0x9834, 0xf00fffff, 0x00000400,
321 0x9838, 0x0002021c, 0x00020200,
322 0xc78, 0x00000080, 0x00000000,
323 0xd030, 0x000300c0, 0x00800040,
324 0xd830, 0x000300c0, 0x00800040,
325 0x5bb0, 0x000000f0, 0x00000070,
326 0x5bc0, 0x00200000, 0x50100000,
327 0x7030, 0x31000311, 0x00000011,
328 0x277c, 0x00000003, 0x000007ff,
329 0x240c, 0x000007ff, 0x00000000,
330 0x8a14, 0xf000001f, 0x00000007,
331 0x8b24, 0xffffffff, 0x00ffffff,
332 0x8b10, 0x0000ff0f, 0x00000000,
333 0x28a4c, 0x07ffffff, 0x4e000000,
334 0x28350, 0x3f3f3fff, 0x2a00126a,
335 0x30, 0x000000ff, 0x0040,
336 0x34, 0x00000040, 0x00004040,
337 0x9100, 0x07ffffff, 0x03000000,
338 0x8e88, 0x01ff1f3f, 0x00000000,
339 0x8e84, 0x01ff1f3f, 0x00000000,
340 0x9060, 0x0000007f, 0x00000020,
341 0x9508, 0x00010000, 0x00010000,
342 0xac14, 0x00000200, 0x000002fb,
343 0xac10, 0xffffffff, 0x0000543b,
344 0xac0c, 0xffffffff, 0xa9210876,
345 0x88d0, 0xffffffff, 0x000fff40,
346 0x88d4, 0x0000001f, 0x00000010,
347 0x1410, 0x20000000, 0x20fffed8,
348 0x15c0, 0x000c0fc0, 0x000c0400
351 static const u32 tahiti_golden_registers2[] =
353 0xc64, 0x00000001, 0x00000001
356 static const u32 pitcairn_golden_rlc_registers[] =
358 0xc424, 0xffffffff, 0x00601004,
359 0xc47c, 0xffffffff, 0x10102020,
360 0xc488, 0xffffffff, 0x01000020,
361 0xc314, 0xffffffff, 0x00000800,
362 0xc30c, 0xffffffff, 0x800000a4
365 static const u32 pitcairn_golden_registers[] =
367 0x9a10, 0x00010000, 0x00018208,
368 0x9830, 0xffffffff, 0x00000000,
369 0x9834, 0xf00fffff, 0x00000400,
370 0x9838, 0x0002021c, 0x00020200,
371 0xc78, 0x00000080, 0x00000000,
372 0xd030, 0x000300c0, 0x00800040,
373 0xd830, 0x000300c0, 0x00800040,
374 0x5bb0, 0x000000f0, 0x00000070,
375 0x5bc0, 0x00200000, 0x50100000,
376 0x7030, 0x31000311, 0x00000011,
377 0x2ae4, 0x00073ffe, 0x000022a2,
378 0x240c, 0x000007ff, 0x00000000,
379 0x8a14, 0xf000001f, 0x00000007,
380 0x8b24, 0xffffffff, 0x00ffffff,
381 0x8b10, 0x0000ff0f, 0x00000000,
382 0x28a4c, 0x07ffffff, 0x4e000000,
383 0x28350, 0x3f3f3fff, 0x2a00126a,
384 0x30, 0x000000ff, 0x0040,
385 0x34, 0x00000040, 0x00004040,
386 0x9100, 0x07ffffff, 0x03000000,
387 0x9060, 0x0000007f, 0x00000020,
388 0x9508, 0x00010000, 0x00010000,
389 0xac14, 0x000003ff, 0x000000f7,
390 0xac10, 0xffffffff, 0x00000000,
391 0xac0c, 0xffffffff, 0x32761054,
392 0x88d4, 0x0000001f, 0x00000010,
393 0x15c0, 0x000c0fc0, 0x000c0400
396 static const u32 verde_golden_rlc_registers[] =
398 0xc424, 0xffffffff, 0x033f1005,
399 0xc47c, 0xffffffff, 0x10808020,
400 0xc488, 0xffffffff, 0x00800008,
401 0xc314, 0xffffffff, 0x00001000,
402 0xc30c, 0xffffffff, 0x80010014
405 static const u32 verde_golden_registers[] =
407 0x9a10, 0x00010000, 0x00018208,
408 0x9830, 0xffffffff, 0x00000000,
409 0x9834, 0xf00fffff, 0x00000400,
410 0x9838, 0x0002021c, 0x00020200,
411 0xc78, 0x00000080, 0x00000000,
412 0xd030, 0x000300c0, 0x00800040,
413 0xd030, 0x000300c0, 0x00800040,
414 0xd830, 0x000300c0, 0x00800040,
415 0xd830, 0x000300c0, 0x00800040,
416 0x5bb0, 0x000000f0, 0x00000070,
417 0x5bc0, 0x00200000, 0x50100000,
418 0x7030, 0x31000311, 0x00000011,
419 0x2ae4, 0x00073ffe, 0x000022a2,
420 0x2ae4, 0x00073ffe, 0x000022a2,
421 0x2ae4, 0x00073ffe, 0x000022a2,
422 0x240c, 0x000007ff, 0x00000000,
423 0x240c, 0x000007ff, 0x00000000,
424 0x240c, 0x000007ff, 0x00000000,
425 0x8a14, 0xf000001f, 0x00000007,
426 0x8a14, 0xf000001f, 0x00000007,
427 0x8a14, 0xf000001f, 0x00000007,
428 0x8b24, 0xffffffff, 0x00ffffff,
429 0x8b10, 0x0000ff0f, 0x00000000,
430 0x28a4c, 0x07ffffff, 0x4e000000,
431 0x28350, 0x3f3f3fff, 0x0000124a,
432 0x28350, 0x3f3f3fff, 0x0000124a,
433 0x28350, 0x3f3f3fff, 0x0000124a,
434 0x30, 0x000000ff, 0x0040,
435 0x34, 0x00000040, 0x00004040,
436 0x9100, 0x07ffffff, 0x03000000,
437 0x9100, 0x07ffffff, 0x03000000,
438 0x8e88, 0x01ff1f3f, 0x00000000,
439 0x8e88, 0x01ff1f3f, 0x00000000,
440 0x8e88, 0x01ff1f3f, 0x00000000,
441 0x8e84, 0x01ff1f3f, 0x00000000,
442 0x8e84, 0x01ff1f3f, 0x00000000,
443 0x8e84, 0x01ff1f3f, 0x00000000,
444 0x9060, 0x0000007f, 0x00000020,
445 0x9508, 0x00010000, 0x00010000,
446 0xac14, 0x000003ff, 0x00000003,
447 0xac14, 0x000003ff, 0x00000003,
448 0xac14, 0x000003ff, 0x00000003,
449 0xac10, 0xffffffff, 0x00000000,
450 0xac10, 0xffffffff, 0x00000000,
451 0xac10, 0xffffffff, 0x00000000,
452 0xac0c, 0xffffffff, 0x00001032,
453 0xac0c, 0xffffffff, 0x00001032,
454 0xac0c, 0xffffffff, 0x00001032,
455 0x88d4, 0x0000001f, 0x00000010,
456 0x88d4, 0x0000001f, 0x00000010,
457 0x88d4, 0x0000001f, 0x00000010,
458 0x15c0, 0x000c0fc0, 0x000c0400
461 static const u32 oland_golden_rlc_registers[] =
463 0xc424, 0xffffffff, 0x00601005,
464 0xc47c, 0xffffffff, 0x10104040,
465 0xc488, 0xffffffff, 0x0100000a,
466 0xc314, 0xffffffff, 0x00000800,
467 0xc30c, 0xffffffff, 0x800000f4
470 static const u32 oland_golden_registers[] =
472 0x9a10, 0x00010000, 0x00018208,
473 0x9830, 0xffffffff, 0x00000000,
474 0x9834, 0xf00fffff, 0x00000400,
475 0x9838, 0x0002021c, 0x00020200,
476 0xc78, 0x00000080, 0x00000000,
477 0xd030, 0x000300c0, 0x00800040,
478 0xd830, 0x000300c0, 0x00800040,
479 0x5bb0, 0x000000f0, 0x00000070,
480 0x5bc0, 0x00200000, 0x50100000,
481 0x7030, 0x31000311, 0x00000011,
482 0x2ae4, 0x00073ffe, 0x000022a2,
483 0x240c, 0x000007ff, 0x00000000,
484 0x8a14, 0xf000001f, 0x00000007,
485 0x8b24, 0xffffffff, 0x00ffffff,
486 0x8b10, 0x0000ff0f, 0x00000000,
487 0x28a4c, 0x07ffffff, 0x4e000000,
488 0x28350, 0x3f3f3fff, 0x00000082,
489 0x30, 0x000000ff, 0x0040,
490 0x34, 0x00000040, 0x00004040,
491 0x9100, 0x07ffffff, 0x03000000,
492 0x9060, 0x0000007f, 0x00000020,
493 0x9508, 0x00010000, 0x00010000,
494 0xac14, 0x000003ff, 0x000000f3,
495 0xac10, 0xffffffff, 0x00000000,
496 0xac0c, 0xffffffff, 0x00003210,
497 0x88d4, 0x0000001f, 0x00000010,
498 0x15c0, 0x000c0fc0, 0x000c0400
501 static const u32 hainan_golden_registers[] =
503 0x9a10, 0x00010000, 0x00018208,
504 0x9830, 0xffffffff, 0x00000000,
505 0x9834, 0xf00fffff, 0x00000400,
506 0x9838, 0x0002021c, 0x00020200,
507 0xd0c0, 0xff000fff, 0x00000100,
508 0xd030, 0x000300c0, 0x00800040,
509 0xd8c0, 0xff000fff, 0x00000100,
510 0xd830, 0x000300c0, 0x00800040,
511 0x2ae4, 0x00073ffe, 0x000022a2,
512 0x240c, 0x000007ff, 0x00000000,
513 0x8a14, 0xf000001f, 0x00000007,
514 0x8b24, 0xffffffff, 0x00ffffff,
515 0x8b10, 0x0000ff0f, 0x00000000,
516 0x28a4c, 0x07ffffff, 0x4e000000,
517 0x28350, 0x3f3f3fff, 0x00000000,
518 0x30, 0x000000ff, 0x0040,
519 0x34, 0x00000040, 0x00004040,
520 0x9100, 0x03e00000, 0x03600000,
521 0x9060, 0x0000007f, 0x00000020,
522 0x9508, 0x00010000, 0x00010000,
523 0xac14, 0x000003ff, 0x000000f1,
524 0xac10, 0xffffffff, 0x00000000,
525 0xac0c, 0xffffffff, 0x00003210,
526 0x88d4, 0x0000001f, 0x00000010,
527 0x15c0, 0x000c0fc0, 0x000c0400
530 static const u32 hainan_golden_registers2[] =
532 0x98f8, 0xffffffff, 0x02010001
535 static const u32 tahiti_mgcg_cgcg_init[] =
537 0xc400, 0xffffffff, 0xfffffffc,
538 0x802c, 0xffffffff, 0xe0000000,
539 0x9a60, 0xffffffff, 0x00000100,
540 0x92a4, 0xffffffff, 0x00000100,
541 0xc164, 0xffffffff, 0x00000100,
542 0x9774, 0xffffffff, 0x00000100,
543 0x8984, 0xffffffff, 0x06000100,
544 0x8a18, 0xffffffff, 0x00000100,
545 0x92a0, 0xffffffff, 0x00000100,
546 0xc380, 0xffffffff, 0x00000100,
547 0x8b28, 0xffffffff, 0x00000100,
548 0x9144, 0xffffffff, 0x00000100,
549 0x8d88, 0xffffffff, 0x00000100,
550 0x8d8c, 0xffffffff, 0x00000100,
551 0x9030, 0xffffffff, 0x00000100,
552 0x9034, 0xffffffff, 0x00000100,
553 0x9038, 0xffffffff, 0x00000100,
554 0x903c, 0xffffffff, 0x00000100,
555 0xad80, 0xffffffff, 0x00000100,
556 0xac54, 0xffffffff, 0x00000100,
557 0x897c, 0xffffffff, 0x06000100,
558 0x9868, 0xffffffff, 0x00000100,
559 0x9510, 0xffffffff, 0x00000100,
560 0xaf04, 0xffffffff, 0x00000100,
561 0xae04, 0xffffffff, 0x00000100,
562 0x949c, 0xffffffff, 0x00000100,
563 0x802c, 0xffffffff, 0xe0000000,
564 0x9160, 0xffffffff, 0x00010000,
565 0x9164, 0xffffffff, 0x00030002,
566 0x9168, 0xffffffff, 0x00040007,
567 0x916c, 0xffffffff, 0x00060005,
568 0x9170, 0xffffffff, 0x00090008,
569 0x9174, 0xffffffff, 0x00020001,
570 0x9178, 0xffffffff, 0x00040003,
571 0x917c, 0xffffffff, 0x00000007,
572 0x9180, 0xffffffff, 0x00060005,
573 0x9184, 0xffffffff, 0x00090008,
574 0x9188, 0xffffffff, 0x00030002,
575 0x918c, 0xffffffff, 0x00050004,
576 0x9190, 0xffffffff, 0x00000008,
577 0x9194, 0xffffffff, 0x00070006,
578 0x9198, 0xffffffff, 0x000a0009,
579 0x919c, 0xffffffff, 0x00040003,
580 0x91a0, 0xffffffff, 0x00060005,
581 0x91a4, 0xffffffff, 0x00000009,
582 0x91a8, 0xffffffff, 0x00080007,
583 0x91ac, 0xffffffff, 0x000b000a,
584 0x91b0, 0xffffffff, 0x00050004,
585 0x91b4, 0xffffffff, 0x00070006,
586 0x91b8, 0xffffffff, 0x0008000b,
587 0x91bc, 0xffffffff, 0x000a0009,
588 0x91c0, 0xffffffff, 0x000d000c,
589 0x91c4, 0xffffffff, 0x00060005,
590 0x91c8, 0xffffffff, 0x00080007,
591 0x91cc, 0xffffffff, 0x0000000b,
592 0x91d0, 0xffffffff, 0x000a0009,
593 0x91d4, 0xffffffff, 0x000d000c,
594 0x91d8, 0xffffffff, 0x00070006,
595 0x91dc, 0xffffffff, 0x00090008,
596 0x91e0, 0xffffffff, 0x0000000c,
597 0x91e4, 0xffffffff, 0x000b000a,
598 0x91e8, 0xffffffff, 0x000e000d,
599 0x91ec, 0xffffffff, 0x00080007,
600 0x91f0, 0xffffffff, 0x000a0009,
601 0x91f4, 0xffffffff, 0x0000000d,
602 0x91f8, 0xffffffff, 0x000c000b,
603 0x91fc, 0xffffffff, 0x000f000e,
604 0x9200, 0xffffffff, 0x00090008,
605 0x9204, 0xffffffff, 0x000b000a,
606 0x9208, 0xffffffff, 0x000c000f,
607 0x920c, 0xffffffff, 0x000e000d,
608 0x9210, 0xffffffff, 0x00110010,
609 0x9214, 0xffffffff, 0x000a0009,
610 0x9218, 0xffffffff, 0x000c000b,
611 0x921c, 0xffffffff, 0x0000000f,
612 0x9220, 0xffffffff, 0x000e000d,
613 0x9224, 0xffffffff, 0x00110010,
614 0x9228, 0xffffffff, 0x000b000a,
615 0x922c, 0xffffffff, 0x000d000c,
616 0x9230, 0xffffffff, 0x00000010,
617 0x9234, 0xffffffff, 0x000f000e,
618 0x9238, 0xffffffff, 0x00120011,
619 0x923c, 0xffffffff, 0x000c000b,
620 0x9240, 0xffffffff, 0x000e000d,
621 0x9244, 0xffffffff, 0x00000011,
622 0x9248, 0xffffffff, 0x0010000f,
623 0x924c, 0xffffffff, 0x00130012,
624 0x9250, 0xffffffff, 0x000d000c,
625 0x9254, 0xffffffff, 0x000f000e,
626 0x9258, 0xffffffff, 0x00100013,
627 0x925c, 0xffffffff, 0x00120011,
628 0x9260, 0xffffffff, 0x00150014,
629 0x9264, 0xffffffff, 0x000e000d,
630 0x9268, 0xffffffff, 0x0010000f,
631 0x926c, 0xffffffff, 0x00000013,
632 0x9270, 0xffffffff, 0x00120011,
633 0x9274, 0xffffffff, 0x00150014,
634 0x9278, 0xffffffff, 0x000f000e,
635 0x927c, 0xffffffff, 0x00110010,
636 0x9280, 0xffffffff, 0x00000014,
637 0x9284, 0xffffffff, 0x00130012,
638 0x9288, 0xffffffff, 0x00160015,
639 0x928c, 0xffffffff, 0x0010000f,
640 0x9290, 0xffffffff, 0x00120011,
641 0x9294, 0xffffffff, 0x00000015,
642 0x9298, 0xffffffff, 0x00140013,
643 0x929c, 0xffffffff, 0x00170016,
644 0x9150, 0xffffffff, 0x96940200,
645 0x8708, 0xffffffff, 0x00900100,
646 0xc478, 0xffffffff, 0x00000080,
647 0xc404, 0xffffffff, 0x0020003f,
648 0x30, 0xffffffff, 0x0000001c,
649 0x34, 0x000f0000, 0x000f0000,
650 0x160c, 0xffffffff, 0x00000100,
651 0x1024, 0xffffffff, 0x00000100,
652 0x102c, 0x00000101, 0x00000000,
653 0x20a8, 0xffffffff, 0x00000104,
654 0x264c, 0x000c0000, 0x000c0000,
655 0x2648, 0x000c0000, 0x000c0000,
656 0x55e4, 0xff000fff, 0x00000100,
657 0x55e8, 0x00000001, 0x00000001,
658 0x2f50, 0x00000001, 0x00000001,
659 0x30cc, 0xc0000fff, 0x00000104,
660 0xc1e4, 0x00000001, 0x00000001,
661 0xd0c0, 0xfffffff0, 0x00000100,
662 0xd8c0, 0xfffffff0, 0x00000100
665 static const u32 pitcairn_mgcg_cgcg_init[] =
667 0xc400, 0xffffffff, 0xfffffffc,
668 0x802c, 0xffffffff, 0xe0000000,
669 0x9a60, 0xffffffff, 0x00000100,
670 0x92a4, 0xffffffff, 0x00000100,
671 0xc164, 0xffffffff, 0x00000100,
672 0x9774, 0xffffffff, 0x00000100,
673 0x8984, 0xffffffff, 0x06000100,
674 0x8a18, 0xffffffff, 0x00000100,
675 0x92a0, 0xffffffff, 0x00000100,
676 0xc380, 0xffffffff, 0x00000100,
677 0x8b28, 0xffffffff, 0x00000100,
678 0x9144, 0xffffffff, 0x00000100,
679 0x8d88, 0xffffffff, 0x00000100,
680 0x8d8c, 0xffffffff, 0x00000100,
681 0x9030, 0xffffffff, 0x00000100,
682 0x9034, 0xffffffff, 0x00000100,
683 0x9038, 0xffffffff, 0x00000100,
684 0x903c, 0xffffffff, 0x00000100,
685 0xad80, 0xffffffff, 0x00000100,
686 0xac54, 0xffffffff, 0x00000100,
687 0x897c, 0xffffffff, 0x06000100,
688 0x9868, 0xffffffff, 0x00000100,
689 0x9510, 0xffffffff, 0x00000100,
690 0xaf04, 0xffffffff, 0x00000100,
691 0xae04, 0xffffffff, 0x00000100,
692 0x949c, 0xffffffff, 0x00000100,
693 0x802c, 0xffffffff, 0xe0000000,
694 0x9160, 0xffffffff, 0x00010000,
695 0x9164, 0xffffffff, 0x00030002,
696 0x9168, 0xffffffff, 0x00040007,
697 0x916c, 0xffffffff, 0x00060005,
698 0x9170, 0xffffffff, 0x00090008,
699 0x9174, 0xffffffff, 0x00020001,
700 0x9178, 0xffffffff, 0x00040003,
701 0x917c, 0xffffffff, 0x00000007,
702 0x9180, 0xffffffff, 0x00060005,
703 0x9184, 0xffffffff, 0x00090008,
704 0x9188, 0xffffffff, 0x00030002,
705 0x918c, 0xffffffff, 0x00050004,
706 0x9190, 0xffffffff, 0x00000008,
707 0x9194, 0xffffffff, 0x00070006,
708 0x9198, 0xffffffff, 0x000a0009,
709 0x919c, 0xffffffff, 0x00040003,
710 0x91a0, 0xffffffff, 0x00060005,
711 0x91a4, 0xffffffff, 0x00000009,
712 0x91a8, 0xffffffff, 0x00080007,
713 0x91ac, 0xffffffff, 0x000b000a,
714 0x91b0, 0xffffffff, 0x00050004,
715 0x91b4, 0xffffffff, 0x00070006,
716 0x91b8, 0xffffffff, 0x0008000b,
717 0x91bc, 0xffffffff, 0x000a0009,
718 0x91c0, 0xffffffff, 0x000d000c,
719 0x9200, 0xffffffff, 0x00090008,
720 0x9204, 0xffffffff, 0x000b000a,
721 0x9208, 0xffffffff, 0x000c000f,
722 0x920c, 0xffffffff, 0x000e000d,
723 0x9210, 0xffffffff, 0x00110010,
724 0x9214, 0xffffffff, 0x000a0009,
725 0x9218, 0xffffffff, 0x000c000b,
726 0x921c, 0xffffffff, 0x0000000f,
727 0x9220, 0xffffffff, 0x000e000d,
728 0x9224, 0xffffffff, 0x00110010,
729 0x9228, 0xffffffff, 0x000b000a,
730 0x922c, 0xffffffff, 0x000d000c,
731 0x9230, 0xffffffff, 0x00000010,
732 0x9234, 0xffffffff, 0x000f000e,
733 0x9238, 0xffffffff, 0x00120011,
734 0x923c, 0xffffffff, 0x000c000b,
735 0x9240, 0xffffffff, 0x000e000d,
736 0x9244, 0xffffffff, 0x00000011,
737 0x9248, 0xffffffff, 0x0010000f,
738 0x924c, 0xffffffff, 0x00130012,
739 0x9250, 0xffffffff, 0x000d000c,
740 0x9254, 0xffffffff, 0x000f000e,
741 0x9258, 0xffffffff, 0x00100013,
742 0x925c, 0xffffffff, 0x00120011,
743 0x9260, 0xffffffff, 0x00150014,
744 0x9150, 0xffffffff, 0x96940200,
745 0x8708, 0xffffffff, 0x00900100,
746 0xc478, 0xffffffff, 0x00000080,
747 0xc404, 0xffffffff, 0x0020003f,
748 0x30, 0xffffffff, 0x0000001c,
749 0x34, 0x000f0000, 0x000f0000,
750 0x160c, 0xffffffff, 0x00000100,
751 0x1024, 0xffffffff, 0x00000100,
752 0x102c, 0x00000101, 0x00000000,
753 0x20a8, 0xffffffff, 0x00000104,
754 0x55e4, 0xff000fff, 0x00000100,
755 0x55e8, 0x00000001, 0x00000001,
756 0x2f50, 0x00000001, 0x00000001,
757 0x30cc, 0xc0000fff, 0x00000104,
758 0xc1e4, 0x00000001, 0x00000001,
759 0xd0c0, 0xfffffff0, 0x00000100,
760 0xd8c0, 0xfffffff0, 0x00000100
763 static const u32 verde_mgcg_cgcg_init[] =
765 0xc400, 0xffffffff, 0xfffffffc,
766 0x802c, 0xffffffff, 0xe0000000,
767 0x9a60, 0xffffffff, 0x00000100,
768 0x92a4, 0xffffffff, 0x00000100,
769 0xc164, 0xffffffff, 0x00000100,
770 0x9774, 0xffffffff, 0x00000100,
771 0x8984, 0xffffffff, 0x06000100,
772 0x8a18, 0xffffffff, 0x00000100,
773 0x92a0, 0xffffffff, 0x00000100,
774 0xc380, 0xffffffff, 0x00000100,
775 0x8b28, 0xffffffff, 0x00000100,
776 0x9144, 0xffffffff, 0x00000100,
777 0x8d88, 0xffffffff, 0x00000100,
778 0x8d8c, 0xffffffff, 0x00000100,
779 0x9030, 0xffffffff, 0x00000100,
780 0x9034, 0xffffffff, 0x00000100,
781 0x9038, 0xffffffff, 0x00000100,
782 0x903c, 0xffffffff, 0x00000100,
783 0xad80, 0xffffffff, 0x00000100,
784 0xac54, 0xffffffff, 0x00000100,
785 0x897c, 0xffffffff, 0x06000100,
786 0x9868, 0xffffffff, 0x00000100,
787 0x9510, 0xffffffff, 0x00000100,
788 0xaf04, 0xffffffff, 0x00000100,
789 0xae04, 0xffffffff, 0x00000100,
790 0x949c, 0xffffffff, 0x00000100,
791 0x802c, 0xffffffff, 0xe0000000,
792 0x9160, 0xffffffff, 0x00010000,
793 0x9164, 0xffffffff, 0x00030002,
794 0x9168, 0xffffffff, 0x00040007,
795 0x916c, 0xffffffff, 0x00060005,
796 0x9170, 0xffffffff, 0x00090008,
797 0x9174, 0xffffffff, 0x00020001,
798 0x9178, 0xffffffff, 0x00040003,
799 0x917c, 0xffffffff, 0x00000007,
800 0x9180, 0xffffffff, 0x00060005,
801 0x9184, 0xffffffff, 0x00090008,
802 0x9188, 0xffffffff, 0x00030002,
803 0x918c, 0xffffffff, 0x00050004,
804 0x9190, 0xffffffff, 0x00000008,
805 0x9194, 0xffffffff, 0x00070006,
806 0x9198, 0xffffffff, 0x000a0009,
807 0x919c, 0xffffffff, 0x00040003,
808 0x91a0, 0xffffffff, 0x00060005,
809 0x91a4, 0xffffffff, 0x00000009,
810 0x91a8, 0xffffffff, 0x00080007,
811 0x91ac, 0xffffffff, 0x000b000a,
812 0x91b0, 0xffffffff, 0x00050004,
813 0x91b4, 0xffffffff, 0x00070006,
814 0x91b8, 0xffffffff, 0x0008000b,
815 0x91bc, 0xffffffff, 0x000a0009,
816 0x91c0, 0xffffffff, 0x000d000c,
817 0x9200, 0xffffffff, 0x00090008,
818 0x9204, 0xffffffff, 0x000b000a,
819 0x9208, 0xffffffff, 0x000c000f,
820 0x920c, 0xffffffff, 0x000e000d,
821 0x9210, 0xffffffff, 0x00110010,
822 0x9214, 0xffffffff, 0x000a0009,
823 0x9218, 0xffffffff, 0x000c000b,
824 0x921c, 0xffffffff, 0x0000000f,
825 0x9220, 0xffffffff, 0x000e000d,
826 0x9224, 0xffffffff, 0x00110010,
827 0x9228, 0xffffffff, 0x000b000a,
828 0x922c, 0xffffffff, 0x000d000c,
829 0x9230, 0xffffffff, 0x00000010,
830 0x9234, 0xffffffff, 0x000f000e,
831 0x9238, 0xffffffff, 0x00120011,
832 0x923c, 0xffffffff, 0x000c000b,
833 0x9240, 0xffffffff, 0x000e000d,
834 0x9244, 0xffffffff, 0x00000011,
835 0x9248, 0xffffffff, 0x0010000f,
836 0x924c, 0xffffffff, 0x00130012,
837 0x9250, 0xffffffff, 0x000d000c,
838 0x9254, 0xffffffff, 0x000f000e,
839 0x9258, 0xffffffff, 0x00100013,
840 0x925c, 0xffffffff, 0x00120011,
841 0x9260, 0xffffffff, 0x00150014,
842 0x9150, 0xffffffff, 0x96940200,
843 0x8708, 0xffffffff, 0x00900100,
844 0xc478, 0xffffffff, 0x00000080,
845 0xc404, 0xffffffff, 0x0020003f,
846 0x30, 0xffffffff, 0x0000001c,
847 0x34, 0x000f0000, 0x000f0000,
848 0x160c, 0xffffffff, 0x00000100,
849 0x1024, 0xffffffff, 0x00000100,
850 0x102c, 0x00000101, 0x00000000,
851 0x20a8, 0xffffffff, 0x00000104,
852 0x264c, 0x000c0000, 0x000c0000,
853 0x2648, 0x000c0000, 0x000c0000,
854 0x55e4, 0xff000fff, 0x00000100,
855 0x55e8, 0x00000001, 0x00000001,
856 0x2f50, 0x00000001, 0x00000001,
857 0x30cc, 0xc0000fff, 0x00000104,
858 0xc1e4, 0x00000001, 0x00000001,
859 0xd0c0, 0xfffffff0, 0x00000100,
860 0xd8c0, 0xfffffff0, 0x00000100
863 static const u32 oland_mgcg_cgcg_init[] =
865 0xc400, 0xffffffff, 0xfffffffc,
866 0x802c, 0xffffffff, 0xe0000000,
867 0x9a60, 0xffffffff, 0x00000100,
868 0x92a4, 0xffffffff, 0x00000100,
869 0xc164, 0xffffffff, 0x00000100,
870 0x9774, 0xffffffff, 0x00000100,
871 0x8984, 0xffffffff, 0x06000100,
872 0x8a18, 0xffffffff, 0x00000100,
873 0x92a0, 0xffffffff, 0x00000100,
874 0xc380, 0xffffffff, 0x00000100,
875 0x8b28, 0xffffffff, 0x00000100,
876 0x9144, 0xffffffff, 0x00000100,
877 0x8d88, 0xffffffff, 0x00000100,
878 0x8d8c, 0xffffffff, 0x00000100,
879 0x9030, 0xffffffff, 0x00000100,
880 0x9034, 0xffffffff, 0x00000100,
881 0x9038, 0xffffffff, 0x00000100,
882 0x903c, 0xffffffff, 0x00000100,
883 0xad80, 0xffffffff, 0x00000100,
884 0xac54, 0xffffffff, 0x00000100,
885 0x897c, 0xffffffff, 0x06000100,
886 0x9868, 0xffffffff, 0x00000100,
887 0x9510, 0xffffffff, 0x00000100,
888 0xaf04, 0xffffffff, 0x00000100,
889 0xae04, 0xffffffff, 0x00000100,
890 0x949c, 0xffffffff, 0x00000100,
891 0x802c, 0xffffffff, 0xe0000000,
892 0x9160, 0xffffffff, 0x00010000,
893 0x9164, 0xffffffff, 0x00030002,
894 0x9168, 0xffffffff, 0x00040007,
895 0x916c, 0xffffffff, 0x00060005,
896 0x9170, 0xffffffff, 0x00090008,
897 0x9174, 0xffffffff, 0x00020001,
898 0x9178, 0xffffffff, 0x00040003,
899 0x917c, 0xffffffff, 0x00000007,
900 0x9180, 0xffffffff, 0x00060005,
901 0x9184, 0xffffffff, 0x00090008,
902 0x9188, 0xffffffff, 0x00030002,
903 0x918c, 0xffffffff, 0x00050004,
904 0x9190, 0xffffffff, 0x00000008,
905 0x9194, 0xffffffff, 0x00070006,
906 0x9198, 0xffffffff, 0x000a0009,
907 0x919c, 0xffffffff, 0x00040003,
908 0x91a0, 0xffffffff, 0x00060005,
909 0x91a4, 0xffffffff, 0x00000009,
910 0x91a8, 0xffffffff, 0x00080007,
911 0x91ac, 0xffffffff, 0x000b000a,
912 0x91b0, 0xffffffff, 0x00050004,
913 0x91b4, 0xffffffff, 0x00070006,
914 0x91b8, 0xffffffff, 0x0008000b,
915 0x91bc, 0xffffffff, 0x000a0009,
916 0x91c0, 0xffffffff, 0x000d000c,
917 0x91c4, 0xffffffff, 0x00060005,
918 0x91c8, 0xffffffff, 0x00080007,
919 0x91cc, 0xffffffff, 0x0000000b,
920 0x91d0, 0xffffffff, 0x000a0009,
921 0x91d4, 0xffffffff, 0x000d000c,
922 0x9150, 0xffffffff, 0x96940200,
923 0x8708, 0xffffffff, 0x00900100,
924 0xc478, 0xffffffff, 0x00000080,
925 0xc404, 0xffffffff, 0x0020003f,
926 0x30, 0xffffffff, 0x0000001c,
927 0x34, 0x000f0000, 0x000f0000,
928 0x160c, 0xffffffff, 0x00000100,
929 0x1024, 0xffffffff, 0x00000100,
930 0x102c, 0x00000101, 0x00000000,
931 0x20a8, 0xffffffff, 0x00000104,
932 0x264c, 0x000c0000, 0x000c0000,
933 0x2648, 0x000c0000, 0x000c0000,
934 0x55e4, 0xff000fff, 0x00000100,
935 0x55e8, 0x00000001, 0x00000001,
936 0x2f50, 0x00000001, 0x00000001,
937 0x30cc, 0xc0000fff, 0x00000104,
938 0xc1e4, 0x00000001, 0x00000001,
939 0xd0c0, 0xfffffff0, 0x00000100,
940 0xd8c0, 0xfffffff0, 0x00000100
943 static const u32 hainan_mgcg_cgcg_init[] =
945 0xc400, 0xffffffff, 0xfffffffc,
946 0x802c, 0xffffffff, 0xe0000000,
947 0x9a60, 0xffffffff, 0x00000100,
948 0x92a4, 0xffffffff, 0x00000100,
949 0xc164, 0xffffffff, 0x00000100,
950 0x9774, 0xffffffff, 0x00000100,
951 0x8984, 0xffffffff, 0x06000100,
952 0x8a18, 0xffffffff, 0x00000100,
953 0x92a0, 0xffffffff, 0x00000100,
954 0xc380, 0xffffffff, 0x00000100,
955 0x8b28, 0xffffffff, 0x00000100,
956 0x9144, 0xffffffff, 0x00000100,
957 0x8d88, 0xffffffff, 0x00000100,
958 0x8d8c, 0xffffffff, 0x00000100,
959 0x9030, 0xffffffff, 0x00000100,
960 0x9034, 0xffffffff, 0x00000100,
961 0x9038, 0xffffffff, 0x00000100,
962 0x903c, 0xffffffff, 0x00000100,
963 0xad80, 0xffffffff, 0x00000100,
964 0xac54, 0xffffffff, 0x00000100,
965 0x897c, 0xffffffff, 0x06000100,
966 0x9868, 0xffffffff, 0x00000100,
967 0x9510, 0xffffffff, 0x00000100,
968 0xaf04, 0xffffffff, 0x00000100,
969 0xae04, 0xffffffff, 0x00000100,
970 0x949c, 0xffffffff, 0x00000100,
971 0x802c, 0xffffffff, 0xe0000000,
972 0x9160, 0xffffffff, 0x00010000,
973 0x9164, 0xffffffff, 0x00030002,
974 0x9168, 0xffffffff, 0x00040007,
975 0x916c, 0xffffffff, 0x00060005,
976 0x9170, 0xffffffff, 0x00090008,
977 0x9174, 0xffffffff, 0x00020001,
978 0x9178, 0xffffffff, 0x00040003,
979 0x917c, 0xffffffff, 0x00000007,
980 0x9180, 0xffffffff, 0x00060005,
981 0x9184, 0xffffffff, 0x00090008,
982 0x9188, 0xffffffff, 0x00030002,
983 0x918c, 0xffffffff, 0x00050004,
984 0x9190, 0xffffffff, 0x00000008,
985 0x9194, 0xffffffff, 0x00070006,
986 0x9198, 0xffffffff, 0x000a0009,
987 0x919c, 0xffffffff, 0x00040003,
988 0x91a0, 0xffffffff, 0x00060005,
989 0x91a4, 0xffffffff, 0x00000009,
990 0x91a8, 0xffffffff, 0x00080007,
991 0x91ac, 0xffffffff, 0x000b000a,
992 0x91b0, 0xffffffff, 0x00050004,
993 0x91b4, 0xffffffff, 0x00070006,
994 0x91b8, 0xffffffff, 0x0008000b,
995 0x91bc, 0xffffffff, 0x000a0009,
996 0x91c0, 0xffffffff, 0x000d000c,
997 0x91c4, 0xffffffff, 0x00060005,
998 0x91c8, 0xffffffff, 0x00080007,
999 0x91cc, 0xffffffff, 0x0000000b,
1000 0x91d0, 0xffffffff, 0x000a0009,
1001 0x91d4, 0xffffffff, 0x000d000c,
1002 0x9150, 0xffffffff, 0x96940200,
1003 0x8708, 0xffffffff, 0x00900100,
1004 0xc478, 0xffffffff, 0x00000080,
1005 0xc404, 0xffffffff, 0x0020003f,
1006 0x30, 0xffffffff, 0x0000001c,
1007 0x34, 0x000f0000, 0x000f0000,
1008 0x160c, 0xffffffff, 0x00000100,
1009 0x1024, 0xffffffff, 0x00000100,
1010 0x20a8, 0xffffffff, 0x00000104,
1011 0x264c, 0x000c0000, 0x000c0000,
1012 0x2648, 0x000c0000, 0x000c0000,
1013 0x2f50, 0x00000001, 0x00000001,
1014 0x30cc, 0xc0000fff, 0x00000104,
1015 0xc1e4, 0x00000001, 0x00000001,
1016 0xd0c0, 0xfffffff0, 0x00000100,
1017 0xd8c0, 0xfffffff0, 0x00000100
1020 static u32 verde_pg_init[] =
1022 0x353c, 0xffffffff, 0x40000,
1023 0x3538, 0xffffffff, 0x200010ff,
1024 0x353c, 0xffffffff, 0x0,
1025 0x353c, 0xffffffff, 0x0,
1026 0x353c, 0xffffffff, 0x0,
1027 0x353c, 0xffffffff, 0x0,
1028 0x353c, 0xffffffff, 0x0,
1029 0x353c, 0xffffffff, 0x7007,
1030 0x3538, 0xffffffff, 0x300010ff,
1031 0x353c, 0xffffffff, 0x0,
1032 0x353c, 0xffffffff, 0x0,
1033 0x353c, 0xffffffff, 0x0,
1034 0x353c, 0xffffffff, 0x0,
1035 0x353c, 0xffffffff, 0x0,
1036 0x353c, 0xffffffff, 0x400000,
1037 0x3538, 0xffffffff, 0x100010ff,
1038 0x353c, 0xffffffff, 0x0,
1039 0x353c, 0xffffffff, 0x0,
1040 0x353c, 0xffffffff, 0x0,
1041 0x353c, 0xffffffff, 0x0,
1042 0x353c, 0xffffffff, 0x0,
1043 0x353c, 0xffffffff, 0x120200,
1044 0x3538, 0xffffffff, 0x500010ff,
1045 0x353c, 0xffffffff, 0x0,
1046 0x353c, 0xffffffff, 0x0,
1047 0x353c, 0xffffffff, 0x0,
1048 0x353c, 0xffffffff, 0x0,
1049 0x353c, 0xffffffff, 0x0,
1050 0x353c, 0xffffffff, 0x1e1e16,
1051 0x3538, 0xffffffff, 0x600010ff,
1052 0x353c, 0xffffffff, 0x0,
1053 0x353c, 0xffffffff, 0x0,
1054 0x353c, 0xffffffff, 0x0,
1055 0x353c, 0xffffffff, 0x0,
1056 0x353c, 0xffffffff, 0x0,
1057 0x353c, 0xffffffff, 0x171f1e,
1058 0x3538, 0xffffffff, 0x700010ff,
1059 0x353c, 0xffffffff, 0x0,
1060 0x353c, 0xffffffff, 0x0,
1061 0x353c, 0xffffffff, 0x0,
1062 0x353c, 0xffffffff, 0x0,
1063 0x353c, 0xffffffff, 0x0,
1064 0x353c, 0xffffffff, 0x0,
1065 0x3538, 0xffffffff, 0x9ff,
1066 0x3500, 0xffffffff, 0x0,
1067 0x3504, 0xffffffff, 0x10000800,
1068 0x3504, 0xffffffff, 0xf,
1069 0x3504, 0xffffffff, 0xf,
1070 0x3500, 0xffffffff, 0x4,
1071 0x3504, 0xffffffff, 0x1000051e,
1072 0x3504, 0xffffffff, 0xffff,
1073 0x3504, 0xffffffff, 0xffff,
1074 0x3500, 0xffffffff, 0x8,
1075 0x3504, 0xffffffff, 0x80500,
1076 0x3500, 0xffffffff, 0x12,
1077 0x3504, 0xffffffff, 0x9050c,
1078 0x3500, 0xffffffff, 0x1d,
1079 0x3504, 0xffffffff, 0xb052c,
1080 0x3500, 0xffffffff, 0x2a,
1081 0x3504, 0xffffffff, 0x1053e,
1082 0x3500, 0xffffffff, 0x2d,
1083 0x3504, 0xffffffff, 0x10546,
1084 0x3500, 0xffffffff, 0x30,
1085 0x3504, 0xffffffff, 0xa054e,
1086 0x3500, 0xffffffff, 0x3c,
1087 0x3504, 0xffffffff, 0x1055f,
1088 0x3500, 0xffffffff, 0x3f,
1089 0x3504, 0xffffffff, 0x10567,
1090 0x3500, 0xffffffff, 0x42,
1091 0x3504, 0xffffffff, 0x1056f,
1092 0x3500, 0xffffffff, 0x45,
1093 0x3504, 0xffffffff, 0x10572,
1094 0x3500, 0xffffffff, 0x48,
1095 0x3504, 0xffffffff, 0x20575,
1096 0x3500, 0xffffffff, 0x4c,
1097 0x3504, 0xffffffff, 0x190801,
1098 0x3500, 0xffffffff, 0x67,
1099 0x3504, 0xffffffff, 0x1082a,
1100 0x3500, 0xffffffff, 0x6a,
1101 0x3504, 0xffffffff, 0x1b082d,
1102 0x3500, 0xffffffff, 0x87,
1103 0x3504, 0xffffffff, 0x310851,
1104 0x3500, 0xffffffff, 0xba,
1105 0x3504, 0xffffffff, 0x891,
1106 0x3500, 0xffffffff, 0xbc,
1107 0x3504, 0xffffffff, 0x893,
1108 0x3500, 0xffffffff, 0xbe,
1109 0x3504, 0xffffffff, 0x20895,
1110 0x3500, 0xffffffff, 0xc2,
1111 0x3504, 0xffffffff, 0x20899,
1112 0x3500, 0xffffffff, 0xc6,
1113 0x3504, 0xffffffff, 0x2089d,
1114 0x3500, 0xffffffff, 0xca,
1115 0x3504, 0xffffffff, 0x8a1,
1116 0x3500, 0xffffffff, 0xcc,
1117 0x3504, 0xffffffff, 0x8a3,
1118 0x3500, 0xffffffff, 0xce,
1119 0x3504, 0xffffffff, 0x308a5,
1120 0x3500, 0xffffffff, 0xd3,
1121 0x3504, 0xffffffff, 0x6d08cd,
1122 0x3500, 0xffffffff, 0x142,
1123 0x3504, 0xffffffff, 0x2000095a,
1124 0x3504, 0xffffffff, 0x1,
1125 0x3500, 0xffffffff, 0x144,
1126 0x3504, 0xffffffff, 0x301f095b,
1127 0x3500, 0xffffffff, 0x165,
1128 0x3504, 0xffffffff, 0xc094d,
1129 0x3500, 0xffffffff, 0x173,
1130 0x3504, 0xffffffff, 0xf096d,
1131 0x3500, 0xffffffff, 0x184,
1132 0x3504, 0xffffffff, 0x15097f,
1133 0x3500, 0xffffffff, 0x19b,
1134 0x3504, 0xffffffff, 0xc0998,
1135 0x3500, 0xffffffff, 0x1a9,
1136 0x3504, 0xffffffff, 0x409a7,
1137 0x3500, 0xffffffff, 0x1af,
1138 0x3504, 0xffffffff, 0xcdc,
1139 0x3500, 0xffffffff, 0x1b1,
1140 0x3504, 0xffffffff, 0x800,
1141 0x3508, 0xffffffff, 0x6c9b2000,
1142 0x3510, 0xfc00, 0x2000,
1143 0x3544, 0xffffffff, 0xfc0,
1144 0x28d4, 0x00000100, 0x100
1147 static void si_init_golden_registers(struct radeon_device *rdev)
1149 switch (rdev->family) {
1151 radeon_program_register_sequence(rdev,
1152 tahiti_golden_registers,
1153 (const u32)ARRAY_SIZE(tahiti_golden_registers));
1154 radeon_program_register_sequence(rdev,
1155 tahiti_golden_rlc_registers,
1156 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1157 radeon_program_register_sequence(rdev,
1158 tahiti_mgcg_cgcg_init,
1159 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1160 radeon_program_register_sequence(rdev,
1161 tahiti_golden_registers2,
1162 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1165 radeon_program_register_sequence(rdev,
1166 pitcairn_golden_registers,
1167 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1168 radeon_program_register_sequence(rdev,
1169 pitcairn_golden_rlc_registers,
1170 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1171 radeon_program_register_sequence(rdev,
1172 pitcairn_mgcg_cgcg_init,
1173 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1176 radeon_program_register_sequence(rdev,
1177 verde_golden_registers,
1178 (const u32)ARRAY_SIZE(verde_golden_registers));
1179 radeon_program_register_sequence(rdev,
1180 verde_golden_rlc_registers,
1181 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1182 radeon_program_register_sequence(rdev,
1183 verde_mgcg_cgcg_init,
1184 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1185 radeon_program_register_sequence(rdev,
1187 (const u32)ARRAY_SIZE(verde_pg_init));
1190 radeon_program_register_sequence(rdev,
1191 oland_golden_registers,
1192 (const u32)ARRAY_SIZE(oland_golden_registers));
1193 radeon_program_register_sequence(rdev,
1194 oland_golden_rlc_registers,
1195 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1196 radeon_program_register_sequence(rdev,
1197 oland_mgcg_cgcg_init,
1198 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1201 radeon_program_register_sequence(rdev,
1202 hainan_golden_registers,
1203 (const u32)ARRAY_SIZE(hainan_golden_registers));
1204 radeon_program_register_sequence(rdev,
1205 hainan_golden_registers2,
1206 (const u32)ARRAY_SIZE(hainan_golden_registers2));
1207 radeon_program_register_sequence(rdev,
1208 hainan_mgcg_cgcg_init,
1209 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1216 #define PCIE_BUS_CLK 10000
1217 #define TCLK (PCIE_BUS_CLK / 10)
1220 * si_get_xclk - get the xclk
1222 * @rdev: radeon_device pointer
1224 * Returns the reference clock used by the gfx engine
1227 u32 si_get_xclk(struct radeon_device *rdev)
1229 u32 reference_clock = rdev->clock.spll.reference_freq;
1232 tmp = RREG32(CG_CLKPIN_CNTL_2);
1233 if (tmp & MUX_TCLK_TO_XCLK)
1236 tmp = RREG32(CG_CLKPIN_CNTL);
1237 if (tmp & XTALIN_DIVIDE)
1238 return reference_clock / 4;
1240 return reference_clock;
1243 /* get temperature in millidegrees */
1244 int si_get_temp(struct radeon_device *rdev)
1247 int actual_temp = 0;
1249 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1255 actual_temp = temp & 0x1ff;
1257 actual_temp = (actual_temp * 1000);
1262 #define TAHITI_IO_MC_REGS_SIZE 36
1264 static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1265 {0x0000006f, 0x03044000},
1266 {0x00000070, 0x0480c018},
1267 {0x00000071, 0x00000040},
1268 {0x00000072, 0x01000000},
1269 {0x00000074, 0x000000ff},
1270 {0x00000075, 0x00143400},
1271 {0x00000076, 0x08ec0800},
1272 {0x00000077, 0x040000cc},
1273 {0x00000079, 0x00000000},
1274 {0x0000007a, 0x21000409},
1275 {0x0000007c, 0x00000000},
1276 {0x0000007d, 0xe8000000},
1277 {0x0000007e, 0x044408a8},
1278 {0x0000007f, 0x00000003},
1279 {0x00000080, 0x00000000},
1280 {0x00000081, 0x01000000},
1281 {0x00000082, 0x02000000},
1282 {0x00000083, 0x00000000},
1283 {0x00000084, 0xe3f3e4f4},
1284 {0x00000085, 0x00052024},
1285 {0x00000087, 0x00000000},
1286 {0x00000088, 0x66036603},
1287 {0x00000089, 0x01000000},
1288 {0x0000008b, 0x1c0a0000},
1289 {0x0000008c, 0xff010000},
1290 {0x0000008e, 0xffffefff},
1291 {0x0000008f, 0xfff3efff},
1292 {0x00000090, 0xfff3efbf},
1293 {0x00000094, 0x00101101},
1294 {0x00000095, 0x00000fff},
1295 {0x00000096, 0x00116fff},
1296 {0x00000097, 0x60010000},
1297 {0x00000098, 0x10010000},
1298 {0x00000099, 0x00006000},
1299 {0x0000009a, 0x00001000},
1300 {0x0000009f, 0x00a77400}
1303 static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1304 {0x0000006f, 0x03044000},
1305 {0x00000070, 0x0480c018},
1306 {0x00000071, 0x00000040},
1307 {0x00000072, 0x01000000},
1308 {0x00000074, 0x000000ff},
1309 {0x00000075, 0x00143400},
1310 {0x00000076, 0x08ec0800},
1311 {0x00000077, 0x040000cc},
1312 {0x00000079, 0x00000000},
1313 {0x0000007a, 0x21000409},
1314 {0x0000007c, 0x00000000},
1315 {0x0000007d, 0xe8000000},
1316 {0x0000007e, 0x044408a8},
1317 {0x0000007f, 0x00000003},
1318 {0x00000080, 0x00000000},
1319 {0x00000081, 0x01000000},
1320 {0x00000082, 0x02000000},
1321 {0x00000083, 0x00000000},
1322 {0x00000084, 0xe3f3e4f4},
1323 {0x00000085, 0x00052024},
1324 {0x00000087, 0x00000000},
1325 {0x00000088, 0x66036603},
1326 {0x00000089, 0x01000000},
1327 {0x0000008b, 0x1c0a0000},
1328 {0x0000008c, 0xff010000},
1329 {0x0000008e, 0xffffefff},
1330 {0x0000008f, 0xfff3efff},
1331 {0x00000090, 0xfff3efbf},
1332 {0x00000094, 0x00101101},
1333 {0x00000095, 0x00000fff},
1334 {0x00000096, 0x00116fff},
1335 {0x00000097, 0x60010000},
1336 {0x00000098, 0x10010000},
1337 {0x00000099, 0x00006000},
1338 {0x0000009a, 0x00001000},
1339 {0x0000009f, 0x00a47400}
1342 static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1343 {0x0000006f, 0x03044000},
1344 {0x00000070, 0x0480c018},
1345 {0x00000071, 0x00000040},
1346 {0x00000072, 0x01000000},
1347 {0x00000074, 0x000000ff},
1348 {0x00000075, 0x00143400},
1349 {0x00000076, 0x08ec0800},
1350 {0x00000077, 0x040000cc},
1351 {0x00000079, 0x00000000},
1352 {0x0000007a, 0x21000409},
1353 {0x0000007c, 0x00000000},
1354 {0x0000007d, 0xe8000000},
1355 {0x0000007e, 0x044408a8},
1356 {0x0000007f, 0x00000003},
1357 {0x00000080, 0x00000000},
1358 {0x00000081, 0x01000000},
1359 {0x00000082, 0x02000000},
1360 {0x00000083, 0x00000000},
1361 {0x00000084, 0xe3f3e4f4},
1362 {0x00000085, 0x00052024},
1363 {0x00000087, 0x00000000},
1364 {0x00000088, 0x66036603},
1365 {0x00000089, 0x01000000},
1366 {0x0000008b, 0x1c0a0000},
1367 {0x0000008c, 0xff010000},
1368 {0x0000008e, 0xffffefff},
1369 {0x0000008f, 0xfff3efff},
1370 {0x00000090, 0xfff3efbf},
1371 {0x00000094, 0x00101101},
1372 {0x00000095, 0x00000fff},
1373 {0x00000096, 0x00116fff},
1374 {0x00000097, 0x60010000},
1375 {0x00000098, 0x10010000},
1376 {0x00000099, 0x00006000},
1377 {0x0000009a, 0x00001000},
1378 {0x0000009f, 0x00a37400}
1381 static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1382 {0x0000006f, 0x03044000},
1383 {0x00000070, 0x0480c018},
1384 {0x00000071, 0x00000040},
1385 {0x00000072, 0x01000000},
1386 {0x00000074, 0x000000ff},
1387 {0x00000075, 0x00143400},
1388 {0x00000076, 0x08ec0800},
1389 {0x00000077, 0x040000cc},
1390 {0x00000079, 0x00000000},
1391 {0x0000007a, 0x21000409},
1392 {0x0000007c, 0x00000000},
1393 {0x0000007d, 0xe8000000},
1394 {0x0000007e, 0x044408a8},
1395 {0x0000007f, 0x00000003},
1396 {0x00000080, 0x00000000},
1397 {0x00000081, 0x01000000},
1398 {0x00000082, 0x02000000},
1399 {0x00000083, 0x00000000},
1400 {0x00000084, 0xe3f3e4f4},
1401 {0x00000085, 0x00052024},
1402 {0x00000087, 0x00000000},
1403 {0x00000088, 0x66036603},
1404 {0x00000089, 0x01000000},
1405 {0x0000008b, 0x1c0a0000},
1406 {0x0000008c, 0xff010000},
1407 {0x0000008e, 0xffffefff},
1408 {0x0000008f, 0xfff3efff},
1409 {0x00000090, 0xfff3efbf},
1410 {0x00000094, 0x00101101},
1411 {0x00000095, 0x00000fff},
1412 {0x00000096, 0x00116fff},
1413 {0x00000097, 0x60010000},
1414 {0x00000098, 0x10010000},
1415 {0x00000099, 0x00006000},
1416 {0x0000009a, 0x00001000},
1417 {0x0000009f, 0x00a17730}
1420 static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1421 {0x0000006f, 0x03044000},
1422 {0x00000070, 0x0480c018},
1423 {0x00000071, 0x00000040},
1424 {0x00000072, 0x01000000},
1425 {0x00000074, 0x000000ff},
1426 {0x00000075, 0x00143400},
1427 {0x00000076, 0x08ec0800},
1428 {0x00000077, 0x040000cc},
1429 {0x00000079, 0x00000000},
1430 {0x0000007a, 0x21000409},
1431 {0x0000007c, 0x00000000},
1432 {0x0000007d, 0xe8000000},
1433 {0x0000007e, 0x044408a8},
1434 {0x0000007f, 0x00000003},
1435 {0x00000080, 0x00000000},
1436 {0x00000081, 0x01000000},
1437 {0x00000082, 0x02000000},
1438 {0x00000083, 0x00000000},
1439 {0x00000084, 0xe3f3e4f4},
1440 {0x00000085, 0x00052024},
1441 {0x00000087, 0x00000000},
1442 {0x00000088, 0x66036603},
1443 {0x00000089, 0x01000000},
1444 {0x0000008b, 0x1c0a0000},
1445 {0x0000008c, 0xff010000},
1446 {0x0000008e, 0xffffefff},
1447 {0x0000008f, 0xfff3efff},
1448 {0x00000090, 0xfff3efbf},
1449 {0x00000094, 0x00101101},
1450 {0x00000095, 0x00000fff},
1451 {0x00000096, 0x00116fff},
1452 {0x00000097, 0x60010000},
1453 {0x00000098, 0x10010000},
1454 {0x00000099, 0x00006000},
1455 {0x0000009a, 0x00001000},
1456 {0x0000009f, 0x00a07730}
1460 int si_mc_load_microcode(struct radeon_device *rdev)
1462 const __be32 *fw_data;
1463 u32 running, blackout = 0;
1465 int i, regs_size, ucode_size;
1470 ucode_size = rdev->mc_fw->datasize / 4;
1472 switch (rdev->family) {
1474 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1475 regs_size = TAHITI_IO_MC_REGS_SIZE;
1478 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1479 regs_size = TAHITI_IO_MC_REGS_SIZE;
1483 io_mc_regs = (u32 *)&verde_io_mc_regs;
1484 regs_size = TAHITI_IO_MC_REGS_SIZE;
1487 io_mc_regs = (u32 *)&oland_io_mc_regs;
1488 regs_size = TAHITI_IO_MC_REGS_SIZE;
1491 io_mc_regs = (u32 *)&hainan_io_mc_regs;
1492 regs_size = TAHITI_IO_MC_REGS_SIZE;
1496 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1500 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1501 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1504 /* reset the engine and set to writable */
1505 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1506 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1508 /* load mc io regs */
1509 for (i = 0; i < regs_size; i++) {
1510 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1511 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1513 /* load the MC ucode */
1514 fw_data = (const __be32 *)rdev->mc_fw->data;
1515 for (i = 0; i < ucode_size; i++)
1516 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1518 /* put the engine back into the active state */
1519 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1520 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1521 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1523 /* wait for training to complete */
1524 for (i = 0; i < rdev->usec_timeout; i++) {
1525 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1529 for (i = 0; i < rdev->usec_timeout; i++) {
1530 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1536 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1542 static int si_init_microcode(struct radeon_device *rdev)
1544 const char *chip_name;
1545 const char *rlc_chip_name;
1546 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1547 size_t smc_req_size, mc2_req_size;
1553 switch (rdev->family) {
1555 chip_name = "TAHITI";
1556 rlc_chip_name = "TAHITI";
1557 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1558 me_req_size = SI_PM4_UCODE_SIZE * 4;
1559 ce_req_size = SI_CE_UCODE_SIZE * 4;
1560 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1561 mc_req_size = SI_MC_UCODE_SIZE * 4;
1562 mc2_req_size = TAHITI_MC_UCODE_SIZE * 4;
1563 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
1566 chip_name = "PITCAIRN";
1567 rlc_chip_name = "PITCAIRN";
1568 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1569 me_req_size = SI_PM4_UCODE_SIZE * 4;
1570 ce_req_size = SI_CE_UCODE_SIZE * 4;
1571 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1572 mc_req_size = SI_MC_UCODE_SIZE * 4;
1573 mc2_req_size = PITCAIRN_MC_UCODE_SIZE * 4;
1574 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
1577 chip_name = "VERDE";
1578 rlc_chip_name = "VERDE";
1579 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1580 me_req_size = SI_PM4_UCODE_SIZE * 4;
1581 ce_req_size = SI_CE_UCODE_SIZE * 4;
1582 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1583 mc_req_size = SI_MC_UCODE_SIZE * 4;
1584 mc2_req_size = VERDE_MC_UCODE_SIZE * 4;
1585 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
1588 chip_name = "OLAND";
1589 rlc_chip_name = "OLAND";
1590 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1591 me_req_size = SI_PM4_UCODE_SIZE * 4;
1592 ce_req_size = SI_CE_UCODE_SIZE * 4;
1593 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1594 mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1595 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
1598 chip_name = "HAINAN";
1599 rlc_chip_name = "HAINAN";
1600 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1601 me_req_size = SI_PM4_UCODE_SIZE * 4;
1602 ce_req_size = SI_CE_UCODE_SIZE * 4;
1603 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1604 mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1605 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
1610 DRM_INFO("Loading %s Microcode\n", chip_name);
1612 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_pfp", chip_name);
1613 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1616 if (rdev->pfp_fw->datasize != pfp_req_size) {
1618 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1619 rdev->pfp_fw->datasize, fw_name);
1624 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_me", chip_name);
1625 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1628 if (rdev->me_fw->datasize != me_req_size) {
1630 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1631 rdev->me_fw->datasize, fw_name);
1635 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_ce", chip_name);
1636 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1639 if (rdev->ce_fw->datasize != ce_req_size) {
1641 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1642 rdev->ce_fw->datasize, fw_name);
1646 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_rlc", rlc_chip_name);
1647 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1650 if (rdev->rlc_fw->datasize != rlc_req_size) {
1652 "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1653 rdev->rlc_fw->datasize, fw_name);
1657 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mc2", chip_name);
1658 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1660 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mc", chip_name);
1661 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1665 if ((rdev->mc_fw->datasize != mc_req_size) &&
1666 (rdev->mc_fw->datasize != mc2_req_size)) {
1668 "si_mc: Bogus length %zu in firmware \"%s\"\n",
1669 rdev->mc_fw->datasize, fw_name);
1672 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->datasize);
1674 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_smc", chip_name);
1675 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1678 "smc: error loading firmware \"%s\"\n",
1680 release_firmware(rdev->smc_fw);
1681 rdev->smc_fw = NULL;
1683 } else if (rdev->smc_fw->datasize != smc_req_size) {
1685 "si_smc: Bogus length %zu in firmware \"%s\"\n",
1686 rdev->smc_fw->datasize, fw_name);
1694 "si_cp: Failed to load firmware \"%s\"\n",
1696 release_firmware(rdev->pfp_fw);
1697 rdev->pfp_fw = NULL;
1698 release_firmware(rdev->me_fw);
1700 release_firmware(rdev->ce_fw);
1702 release_firmware(rdev->rlc_fw);
1703 rdev->rlc_fw = NULL;
1704 release_firmware(rdev->mc_fw);
1706 release_firmware(rdev->smc_fw);
1707 rdev->smc_fw = NULL;
1713 * si_fini_microcode - drop the firmwares image references
1715 * @rdev: radeon_device pointer
1717 * Drop the pfp, me, rlc, mc and ce firmware image references.
1718 * Called at driver shutdown.
1720 static void si_fini_microcode(struct radeon_device *rdev)
1722 release_firmware(rdev->pfp_fw);
1723 rdev->pfp_fw = NULL;
1724 release_firmware(rdev->me_fw);
1726 release_firmware(rdev->rlc_fw);
1727 rdev->rlc_fw = NULL;
1728 release_firmware(rdev->mc_fw);
1730 release_firmware(rdev->smc_fw);
1731 rdev->smc_fw = NULL;
1732 release_firmware(rdev->ce_fw);
1736 /* watermark setup */
1737 static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1738 struct radeon_crtc *radeon_crtc,
1739 struct drm_display_mode *mode,
1740 struct drm_display_mode *other_mode)
1742 u32 tmp, buffer_alloc, i;
1743 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1746 * There are 3 line buffers, each one shared by 2 display controllers.
1747 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1748 * the display controllers. The paritioning is done via one of four
1749 * preset allocations specified in bits 21:20:
1751 * 2 - whole lb, other crtc must be disabled
1753 /* this can get tricky if we have two large displays on a paired group
1754 * of crtcs. Ideally for multiple large displays we'd assign them to
1755 * non-linked crtcs for maximum line buffer allocation.
1757 if (radeon_crtc->base.enabled && mode) {
1762 tmp = 2; /* whole */
1770 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1771 DC_LB_MEMORY_CONFIG(tmp));
1773 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1774 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1775 for (i = 0; i < rdev->usec_timeout; i++) {
1776 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1777 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1782 if (radeon_crtc->base.enabled && mode) {
1792 /* controller not enabled, so no lb used */
1796 static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
1798 u32 tmp = RREG32(MC_SHARED_CHMAP);
1800 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1823 struct dce6_wm_params {
1824 u32 dram_channels; /* number of dram channels */
1825 u32 yclk; /* bandwidth per dram data pin in kHz */
1826 u32 sclk; /* engine clock in kHz */
1827 u32 disp_clk; /* display clock in kHz */
1828 u32 src_width; /* viewport width */
1829 u32 active_time; /* active display time in ns */
1830 u32 blank_time; /* blank time in ns */
1831 bool interlaced; /* mode is interlaced */
1832 fixed20_12 vsc; /* vertical scale ratio */
1833 u32 num_heads; /* number of active crtcs */
1834 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1835 u32 lb_size; /* line buffer allocated to pipe */
1836 u32 vtaps; /* vertical scaler taps */
1839 static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1841 /* Calculate raw DRAM Bandwidth */
1842 fixed20_12 dram_efficiency; /* 0.7 */
1843 fixed20_12 yclk, dram_channels, bandwidth;
1846 a.full = dfixed_const(1000);
1847 yclk.full = dfixed_const(wm->yclk);
1848 yclk.full = dfixed_div(yclk, a);
1849 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1850 a.full = dfixed_const(10);
1851 dram_efficiency.full = dfixed_const(7);
1852 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1853 bandwidth.full = dfixed_mul(dram_channels, yclk);
1854 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1856 return dfixed_trunc(bandwidth);
1859 static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1861 /* Calculate DRAM Bandwidth and the part allocated to display. */
1862 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1863 fixed20_12 yclk, dram_channels, bandwidth;
1866 a.full = dfixed_const(1000);
1867 yclk.full = dfixed_const(wm->yclk);
1868 yclk.full = dfixed_div(yclk, a);
1869 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1870 a.full = dfixed_const(10);
1871 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1872 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1873 bandwidth.full = dfixed_mul(dram_channels, yclk);
1874 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1876 return dfixed_trunc(bandwidth);
1879 static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
1881 /* Calculate the display Data return Bandwidth */
1882 fixed20_12 return_efficiency; /* 0.8 */
1883 fixed20_12 sclk, bandwidth;
1886 a.full = dfixed_const(1000);
1887 sclk.full = dfixed_const(wm->sclk);
1888 sclk.full = dfixed_div(sclk, a);
1889 a.full = dfixed_const(10);
1890 return_efficiency.full = dfixed_const(8);
1891 return_efficiency.full = dfixed_div(return_efficiency, a);
1892 a.full = dfixed_const(32);
1893 bandwidth.full = dfixed_mul(a, sclk);
1894 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1896 return dfixed_trunc(bandwidth);
1899 static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
1904 static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
1906 /* Calculate the DMIF Request Bandwidth */
1907 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1908 fixed20_12 disp_clk, sclk, bandwidth;
1909 fixed20_12 a, b1, b2;
1912 a.full = dfixed_const(1000);
1913 disp_clk.full = dfixed_const(wm->disp_clk);
1914 disp_clk.full = dfixed_div(disp_clk, a);
1915 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
1916 b1.full = dfixed_mul(a, disp_clk);
1918 a.full = dfixed_const(1000);
1919 sclk.full = dfixed_const(wm->sclk);
1920 sclk.full = dfixed_div(sclk, a);
1921 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
1922 b2.full = dfixed_mul(a, sclk);
1924 a.full = dfixed_const(10);
1925 disp_clk_request_efficiency.full = dfixed_const(8);
1926 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1928 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
1930 a.full = dfixed_const(min_bandwidth);
1931 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
1933 return dfixed_trunc(bandwidth);
1936 static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
1938 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1939 u32 dram_bandwidth = dce6_dram_bandwidth(wm);
1940 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
1941 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
1943 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1946 static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
1948 /* Calculate the display mode Average Bandwidth
1949 * DisplayMode should contain the source and destination dimensions,
1953 fixed20_12 line_time;
1954 fixed20_12 src_width;
1955 fixed20_12 bandwidth;
1958 a.full = dfixed_const(1000);
1959 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
1960 line_time.full = dfixed_div(line_time, a);
1961 bpp.full = dfixed_const(wm->bytes_per_pixel);
1962 src_width.full = dfixed_const(wm->src_width);
1963 bandwidth.full = dfixed_mul(src_width, bpp);
1964 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
1965 bandwidth.full = dfixed_div(bandwidth, line_time);
1967 return dfixed_trunc(bandwidth);
1970 static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
1972 /* First calcualte the latency in ns */
1973 u32 mc_latency = 2000; /* 2000 ns. */
1974 u32 available_bandwidth = dce6_available_bandwidth(wm);
1975 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
1976 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
1977 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
1978 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
1979 (wm->num_heads * cursor_line_pair_return_time);
1980 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
1981 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
1982 u32 tmp, dmif_size = 12288;
1985 if (wm->num_heads == 0)
1988 a.full = dfixed_const(2);
1989 b.full = dfixed_const(1);
1990 if ((wm->vsc.full > a.full) ||
1991 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
1993 ((wm->vsc.full >= a.full) && wm->interlaced))
1994 max_src_lines_per_dst_line = 4;
1996 max_src_lines_per_dst_line = 2;
1998 a.full = dfixed_const(available_bandwidth);
1999 b.full = dfixed_const(wm->num_heads);
2000 a.full = dfixed_div(a, b);
2002 b.full = dfixed_const(mc_latency + 512);
2003 c.full = dfixed_const(wm->disp_clk);
2004 b.full = dfixed_div(b, c);
2006 c.full = dfixed_const(dmif_size);
2007 b.full = dfixed_div(c, b);
2009 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
2011 b.full = dfixed_const(1000);
2012 c.full = dfixed_const(wm->disp_clk);
2013 b.full = dfixed_div(c, b);
2014 c.full = dfixed_const(wm->bytes_per_pixel);
2015 b.full = dfixed_mul(b, c);
2017 lb_fill_bw = min(tmp, dfixed_trunc(b));
2019 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2020 b.full = dfixed_const(1000);
2021 c.full = dfixed_const(lb_fill_bw);
2022 b.full = dfixed_div(c, b);
2023 a.full = dfixed_div(a, b);
2024 line_fill_time = dfixed_trunc(a);
2026 if (line_fill_time < wm->active_time)
2029 return latency + (line_fill_time - wm->active_time);
2033 static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2035 if (dce6_average_bandwidth(wm) <=
2036 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
2042 static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2044 if (dce6_average_bandwidth(wm) <=
2045 (dce6_available_bandwidth(wm) / wm->num_heads))
2051 static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2053 u32 lb_partitions = wm->lb_size / wm->src_width;
2054 u32 line_time = wm->active_time + wm->blank_time;
2055 u32 latency_tolerant_lines;
2059 a.full = dfixed_const(1);
2060 if (wm->vsc.full > a.full)
2061 latency_tolerant_lines = 1;
2063 if (lb_partitions <= (wm->vtaps + 1))
2064 latency_tolerant_lines = 1;
2066 latency_tolerant_lines = 2;
2069 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2071 if (dce6_latency_watermark(wm) <= latency_hiding)
2077 static void dce6_program_watermarks(struct radeon_device *rdev,
2078 struct radeon_crtc *radeon_crtc,
2079 u32 lb_size, u32 num_heads)
2081 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2082 struct dce6_wm_params wm_low, wm_high;
2086 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2087 u32 priority_a_mark = 0, priority_b_mark = 0;
2088 u32 priority_a_cnt = PRIORITY_OFF;
2089 u32 priority_b_cnt = PRIORITY_OFF;
2090 u32 tmp, arb_control3;
2093 if (radeon_crtc->base.enabled && num_heads && mode) {
2094 pixel_period = 1000000 / (u32)mode->clock;
2095 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2099 if (rdev->family == CHIP_ARUBA)
2100 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2102 dram_channels = si_get_number_of_dram_channels(rdev);
2104 /* watermark for high clocks */
2105 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2107 radeon_dpm_get_mclk(rdev, false) * 10;
2109 radeon_dpm_get_sclk(rdev, false) * 10;
2111 wm_high.yclk = rdev->pm.current_mclk * 10;
2112 wm_high.sclk = rdev->pm.current_sclk * 10;
2115 wm_high.disp_clk = mode->clock;
2116 wm_high.src_width = mode->crtc_hdisplay;
2117 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2118 wm_high.blank_time = line_time - wm_high.active_time;
2119 wm_high.interlaced = false;
2120 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2121 wm_high.interlaced = true;
2122 wm_high.vsc = radeon_crtc->vsc;
2124 if (radeon_crtc->rmx_type != RMX_OFF)
2126 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2127 wm_high.lb_size = lb_size;
2128 wm_high.dram_channels = dram_channels;
2129 wm_high.num_heads = num_heads;
2131 /* watermark for low clocks */
2132 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2134 radeon_dpm_get_mclk(rdev, true) * 10;
2136 radeon_dpm_get_sclk(rdev, true) * 10;
2138 wm_low.yclk = rdev->pm.current_mclk * 10;
2139 wm_low.sclk = rdev->pm.current_sclk * 10;
2142 wm_low.disp_clk = mode->clock;
2143 wm_low.src_width = mode->crtc_hdisplay;
2144 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2145 wm_low.blank_time = line_time - wm_low.active_time;
2146 wm_low.interlaced = false;
2147 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2148 wm_low.interlaced = true;
2149 wm_low.vsc = radeon_crtc->vsc;
2151 if (radeon_crtc->rmx_type != RMX_OFF)
2153 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2154 wm_low.lb_size = lb_size;
2155 wm_low.dram_channels = dram_channels;
2156 wm_low.num_heads = num_heads;
2158 /* set for high clocks */
2159 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
2160 /* set for low clocks */
2161 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
2163 /* possibly force display priority to high */
2164 /* should really do this at mode validation time... */
2165 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2166 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2167 !dce6_check_latency_hiding(&wm_high) ||
2168 (rdev->disp_priority == 2)) {
2169 DRM_DEBUG_KMS("force priority to high\n");
2170 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2171 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2173 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2174 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2175 !dce6_check_latency_hiding(&wm_low) ||
2176 (rdev->disp_priority == 2)) {
2177 DRM_DEBUG_KMS("force priority to high\n");
2178 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2179 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2182 a.full = dfixed_const(1000);
2183 b.full = dfixed_const(mode->clock);
2184 b.full = dfixed_div(b, a);
2185 c.full = dfixed_const(latency_watermark_a);
2186 c.full = dfixed_mul(c, b);
2187 c.full = dfixed_mul(c, radeon_crtc->hsc);
2188 c.full = dfixed_div(c, a);
2189 a.full = dfixed_const(16);
2190 c.full = dfixed_div(c, a);
2191 priority_a_mark = dfixed_trunc(c);
2192 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2194 a.full = dfixed_const(1000);
2195 b.full = dfixed_const(mode->clock);
2196 b.full = dfixed_div(b, a);
2197 c.full = dfixed_const(latency_watermark_b);
2198 c.full = dfixed_mul(c, b);
2199 c.full = dfixed_mul(c, radeon_crtc->hsc);
2200 c.full = dfixed_div(c, a);
2201 a.full = dfixed_const(16);
2202 c.full = dfixed_div(c, a);
2203 priority_b_mark = dfixed_trunc(c);
2204 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2208 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2210 tmp &= ~LATENCY_WATERMARK_MASK(3);
2211 tmp |= LATENCY_WATERMARK_MASK(1);
2212 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2213 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2214 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2215 LATENCY_HIGH_WATERMARK(line_time)));
2217 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2218 tmp &= ~LATENCY_WATERMARK_MASK(3);
2219 tmp |= LATENCY_WATERMARK_MASK(2);
2220 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2221 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2222 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2223 LATENCY_HIGH_WATERMARK(line_time)));
2224 /* restore original selection */
2225 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2227 /* write the priority marks */
2228 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2229 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2231 /* save values for DPM */
2232 radeon_crtc->line_time = line_time;
2233 radeon_crtc->wm_high = latency_watermark_a;
2234 radeon_crtc->wm_low = latency_watermark_b;
2237 void dce6_bandwidth_update(struct radeon_device *rdev)
2239 struct drm_display_mode *mode0 = NULL;
2240 struct drm_display_mode *mode1 = NULL;
2241 u32 num_heads = 0, lb_size;
2244 if (!rdev->mode_info.mode_config_initialized)
2247 radeon_update_display_priority(rdev);
2249 for (i = 0; i < rdev->num_crtc; i++) {
2250 if (rdev->mode_info.crtcs[i]->base.enabled)
2253 for (i = 0; i < rdev->num_crtc; i += 2) {
2254 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2255 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2256 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2257 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2258 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2259 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2266 static void si_tiling_mode_table_init(struct radeon_device *rdev)
2268 const u32 num_tile_mode_states = 32;
2269 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2271 switch (rdev->config.si.mem_row_size_in_kb) {
2273 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2277 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2280 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2284 if ((rdev->family == CHIP_TAHITI) ||
2285 (rdev->family == CHIP_PITCAIRN)) {
2286 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2287 switch (reg_offset) {
2288 case 0: /* non-AA compressed depth or any compressed stencil */
2289 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2290 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2291 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2292 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2293 NUM_BANKS(ADDR_SURF_16_BANK) |
2294 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2295 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2296 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2298 case 1: /* 2xAA/4xAA compressed depth only */
2299 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2300 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2301 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2302 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2303 NUM_BANKS(ADDR_SURF_16_BANK) |
2304 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2305 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2306 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2308 case 2: /* 8xAA compressed depth only */
2309 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2310 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2311 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2312 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2313 NUM_BANKS(ADDR_SURF_16_BANK) |
2314 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2315 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2316 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2318 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2319 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2320 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2321 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2322 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2323 NUM_BANKS(ADDR_SURF_16_BANK) |
2324 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2325 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2326 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2328 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2329 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2330 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2331 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2332 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2333 NUM_BANKS(ADDR_SURF_16_BANK) |
2334 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2335 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2336 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2338 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2339 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2340 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2341 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2342 TILE_SPLIT(split_equal_to_row_size) |
2343 NUM_BANKS(ADDR_SURF_16_BANK) |
2344 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2345 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2346 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2348 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2349 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2350 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2351 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2352 TILE_SPLIT(split_equal_to_row_size) |
2353 NUM_BANKS(ADDR_SURF_16_BANK) |
2354 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2355 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2356 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2358 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2359 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2360 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2361 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2362 TILE_SPLIT(split_equal_to_row_size) |
2363 NUM_BANKS(ADDR_SURF_16_BANK) |
2364 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2365 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2366 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2368 case 8: /* 1D and 1D Array Surfaces */
2369 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2370 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2371 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2372 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2373 NUM_BANKS(ADDR_SURF_16_BANK) |
2374 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2375 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2376 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2378 case 9: /* Displayable maps. */
2379 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2380 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2381 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2382 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2383 NUM_BANKS(ADDR_SURF_16_BANK) |
2384 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2385 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2386 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2388 case 10: /* Display 8bpp. */
2389 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2390 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2391 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2392 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2393 NUM_BANKS(ADDR_SURF_16_BANK) |
2394 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2395 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2396 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2398 case 11: /* Display 16bpp. */
2399 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2400 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2401 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2402 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2403 NUM_BANKS(ADDR_SURF_16_BANK) |
2404 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2405 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2406 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2408 case 12: /* Display 32bpp. */
2409 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2410 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2411 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2412 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2413 NUM_BANKS(ADDR_SURF_16_BANK) |
2414 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2415 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2416 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2418 case 13: /* Thin. */
2419 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2420 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2421 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2422 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2423 NUM_BANKS(ADDR_SURF_16_BANK) |
2424 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2425 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2426 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2428 case 14: /* Thin 8 bpp. */
2429 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2430 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2431 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2432 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2433 NUM_BANKS(ADDR_SURF_16_BANK) |
2434 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2435 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2436 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2438 case 15: /* Thin 16 bpp. */
2439 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2440 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2441 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2442 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2443 NUM_BANKS(ADDR_SURF_16_BANK) |
2444 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2445 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2446 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2448 case 16: /* Thin 32 bpp. */
2449 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2450 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2451 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2452 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2453 NUM_BANKS(ADDR_SURF_16_BANK) |
2454 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2455 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2456 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2458 case 17: /* Thin 64 bpp. */
2459 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2460 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2461 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2462 TILE_SPLIT(split_equal_to_row_size) |
2463 NUM_BANKS(ADDR_SURF_16_BANK) |
2464 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2465 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2466 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2468 case 21: /* 8 bpp PRT. */
2469 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2470 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2471 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2472 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2473 NUM_BANKS(ADDR_SURF_16_BANK) |
2474 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2475 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2476 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2478 case 22: /* 16 bpp PRT */
2479 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2480 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2481 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2482 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2483 NUM_BANKS(ADDR_SURF_16_BANK) |
2484 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2485 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2486 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2488 case 23: /* 32 bpp PRT */
2489 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2490 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2491 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2492 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2493 NUM_BANKS(ADDR_SURF_16_BANK) |
2494 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2495 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2496 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2498 case 24: /* 64 bpp PRT */
2499 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2500 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2501 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2502 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2503 NUM_BANKS(ADDR_SURF_16_BANK) |
2504 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2505 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2506 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2508 case 25: /* 128 bpp PRT */
2509 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2510 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2511 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2512 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2513 NUM_BANKS(ADDR_SURF_8_BANK) |
2514 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2515 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2516 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2522 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2523 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2525 } else if ((rdev->family == CHIP_VERDE) ||
2526 (rdev->family == CHIP_OLAND) ||
2527 (rdev->family == CHIP_HAINAN)) {
2528 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2529 switch (reg_offset) {
2530 case 0: /* non-AA compressed depth or any compressed stencil */
2531 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2532 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2533 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2534 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2535 NUM_BANKS(ADDR_SURF_16_BANK) |
2536 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2537 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2538 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2540 case 1: /* 2xAA/4xAA compressed depth only */
2541 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2542 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2543 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2544 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2545 NUM_BANKS(ADDR_SURF_16_BANK) |
2546 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2547 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2548 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2550 case 2: /* 8xAA compressed depth only */
2551 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2552 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2553 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2554 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2555 NUM_BANKS(ADDR_SURF_16_BANK) |
2556 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2557 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2558 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2560 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2561 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2562 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2563 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2564 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2565 NUM_BANKS(ADDR_SURF_16_BANK) |
2566 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2567 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2568 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2570 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2571 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2572 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2573 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2574 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2575 NUM_BANKS(ADDR_SURF_16_BANK) |
2576 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2577 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2578 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2580 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2581 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2582 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2583 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2584 TILE_SPLIT(split_equal_to_row_size) |
2585 NUM_BANKS(ADDR_SURF_16_BANK) |
2586 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2587 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2588 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2590 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2591 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2592 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2593 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2594 TILE_SPLIT(split_equal_to_row_size) |
2595 NUM_BANKS(ADDR_SURF_16_BANK) |
2596 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2597 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2598 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2600 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2601 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2602 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2603 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2604 TILE_SPLIT(split_equal_to_row_size) |
2605 NUM_BANKS(ADDR_SURF_16_BANK) |
2606 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2607 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2608 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2610 case 8: /* 1D and 1D Array Surfaces */
2611 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2612 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2613 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2614 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2615 NUM_BANKS(ADDR_SURF_16_BANK) |
2616 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2617 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2618 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2620 case 9: /* Displayable maps. */
2621 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2622 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2623 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2624 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2625 NUM_BANKS(ADDR_SURF_16_BANK) |
2626 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2627 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2628 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2630 case 10: /* Display 8bpp. */
2631 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2632 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2633 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2634 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2635 NUM_BANKS(ADDR_SURF_16_BANK) |
2636 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2637 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2638 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2640 case 11: /* Display 16bpp. */
2641 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2642 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2643 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2644 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2645 NUM_BANKS(ADDR_SURF_16_BANK) |
2646 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2647 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2648 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2650 case 12: /* Display 32bpp. */
2651 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2652 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2653 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2654 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2655 NUM_BANKS(ADDR_SURF_16_BANK) |
2656 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2657 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2658 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2660 case 13: /* Thin. */
2661 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2662 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2663 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2664 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2665 NUM_BANKS(ADDR_SURF_16_BANK) |
2666 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2667 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2668 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2670 case 14: /* Thin 8 bpp. */
2671 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2672 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2673 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2674 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2675 NUM_BANKS(ADDR_SURF_16_BANK) |
2676 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2677 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2678 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2680 case 15: /* Thin 16 bpp. */
2681 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2682 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2683 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2684 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2685 NUM_BANKS(ADDR_SURF_16_BANK) |
2686 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2687 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2688 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2690 case 16: /* Thin 32 bpp. */
2691 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2692 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2693 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2694 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2695 NUM_BANKS(ADDR_SURF_16_BANK) |
2696 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2697 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2698 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2700 case 17: /* Thin 64 bpp. */
2701 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2702 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2703 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2704 TILE_SPLIT(split_equal_to_row_size) |
2705 NUM_BANKS(ADDR_SURF_16_BANK) |
2706 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2707 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2708 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2710 case 21: /* 8 bpp PRT. */
2711 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2712 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2713 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2714 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2715 NUM_BANKS(ADDR_SURF_16_BANK) |
2716 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2717 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2718 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2720 case 22: /* 16 bpp PRT */
2721 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2722 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2723 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2724 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2725 NUM_BANKS(ADDR_SURF_16_BANK) |
2726 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2727 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2728 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2730 case 23: /* 32 bpp PRT */
2731 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2732 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2733 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2734 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2735 NUM_BANKS(ADDR_SURF_16_BANK) |
2736 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2737 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2738 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2740 case 24: /* 64 bpp PRT */
2741 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2742 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2743 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2744 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2745 NUM_BANKS(ADDR_SURF_16_BANK) |
2746 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2747 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2748 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2750 case 25: /* 128 bpp PRT */
2751 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2752 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2753 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2754 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2755 NUM_BANKS(ADDR_SURF_8_BANK) |
2756 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2757 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2758 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2764 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2765 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2768 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2771 static void si_select_se_sh(struct radeon_device *rdev,
2772 u32 se_num, u32 sh_num)
2774 u32 data = INSTANCE_BROADCAST_WRITES;
2776 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
2777 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
2778 else if (se_num == 0xffffffff)
2779 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2780 else if (sh_num == 0xffffffff)
2781 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2783 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2784 WREG32(GRBM_GFX_INDEX, data);
2787 static u32 si_create_bitmask(u32 bit_width)
2791 for (i = 0; i < bit_width; i++) {
2798 static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2802 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2804 data &= INACTIVE_CUS_MASK;
2807 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2809 data >>= INACTIVE_CUS_SHIFT;
2811 mask = si_create_bitmask(cu_per_sh);
2813 return ~data & mask;
2816 static void si_setup_spi(struct radeon_device *rdev,
2817 u32 se_num, u32 sh_per_se,
2821 u32 data, mask, active_cu;
2823 for (i = 0; i < se_num; i++) {
2824 for (j = 0; j < sh_per_se; j++) {
2825 si_select_se_sh(rdev, i, j);
2826 data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2827 active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2830 for (k = 0; k < 16; k++) {
2832 if (active_cu & mask) {
2834 WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2840 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2843 static u32 si_get_rb_disabled(struct radeon_device *rdev,
2844 u32 max_rb_num_per_se,
2849 data = RREG32(CC_RB_BACKEND_DISABLE);
2851 data &= BACKEND_DISABLE_MASK;
2854 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2856 data >>= BACKEND_DISABLE_SHIFT;
2858 mask = si_create_bitmask(max_rb_num_per_se / sh_per_se);
2863 static void si_setup_rb(struct radeon_device *rdev,
2864 u32 se_num, u32 sh_per_se,
2865 u32 max_rb_num_per_se)
2869 u32 disabled_rbs = 0;
2870 u32 enabled_rbs = 0;
2872 for (i = 0; i < se_num; i++) {
2873 for (j = 0; j < sh_per_se; j++) {
2874 si_select_se_sh(rdev, i, j);
2875 data = si_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
2876 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
2879 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2882 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
2883 if (!(disabled_rbs & mask))
2884 enabled_rbs |= mask;
2888 rdev->config.si.backend_enable_mask = enabled_rbs;
2890 for (i = 0; i < se_num; i++) {
2891 si_select_se_sh(rdev, i, 0xffffffff);
2893 for (j = 0; j < sh_per_se; j++) {
2894 switch (enabled_rbs & 3) {
2896 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2899 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2903 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2908 WREG32(PA_SC_RASTER_CONFIG, data);
2910 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2913 static void si_gpu_init(struct radeon_device *rdev)
2915 u32 gb_addr_config = 0;
2916 u32 mc_shared_chmap, mc_arb_ramcfg;
2918 u32 hdp_host_path_cntl;
2922 switch (rdev->family) {
2924 rdev->config.si.max_shader_engines = 2;
2925 rdev->config.si.max_tile_pipes = 12;
2926 rdev->config.si.max_cu_per_sh = 8;
2927 rdev->config.si.max_sh_per_se = 2;
2928 rdev->config.si.max_backends_per_se = 4;
2929 rdev->config.si.max_texture_channel_caches = 12;
2930 rdev->config.si.max_gprs = 256;
2931 rdev->config.si.max_gs_threads = 32;
2932 rdev->config.si.max_hw_contexts = 8;
2934 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2935 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2936 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2937 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2938 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
2941 rdev->config.si.max_shader_engines = 2;
2942 rdev->config.si.max_tile_pipes = 8;
2943 rdev->config.si.max_cu_per_sh = 5;
2944 rdev->config.si.max_sh_per_se = 2;
2945 rdev->config.si.max_backends_per_se = 4;
2946 rdev->config.si.max_texture_channel_caches = 8;
2947 rdev->config.si.max_gprs = 256;
2948 rdev->config.si.max_gs_threads = 32;
2949 rdev->config.si.max_hw_contexts = 8;
2951 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2952 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2953 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2954 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2955 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
2959 rdev->config.si.max_shader_engines = 1;
2960 rdev->config.si.max_tile_pipes = 4;
2961 rdev->config.si.max_cu_per_sh = 5;
2962 rdev->config.si.max_sh_per_se = 2;
2963 rdev->config.si.max_backends_per_se = 4;
2964 rdev->config.si.max_texture_channel_caches = 4;
2965 rdev->config.si.max_gprs = 256;
2966 rdev->config.si.max_gs_threads = 32;
2967 rdev->config.si.max_hw_contexts = 8;
2969 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2970 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2971 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2972 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2973 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2976 rdev->config.si.max_shader_engines = 1;
2977 rdev->config.si.max_tile_pipes = 4;
2978 rdev->config.si.max_cu_per_sh = 6;
2979 rdev->config.si.max_sh_per_se = 1;
2980 rdev->config.si.max_backends_per_se = 2;
2981 rdev->config.si.max_texture_channel_caches = 4;
2982 rdev->config.si.max_gprs = 256;
2983 rdev->config.si.max_gs_threads = 16;
2984 rdev->config.si.max_hw_contexts = 8;
2986 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2987 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2988 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2989 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2990 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2993 rdev->config.si.max_shader_engines = 1;
2994 rdev->config.si.max_tile_pipes = 4;
2995 rdev->config.si.max_cu_per_sh = 5;
2996 rdev->config.si.max_sh_per_se = 1;
2997 rdev->config.si.max_backends_per_se = 1;
2998 rdev->config.si.max_texture_channel_caches = 2;
2999 rdev->config.si.max_gprs = 256;
3000 rdev->config.si.max_gs_threads = 16;
3001 rdev->config.si.max_hw_contexts = 8;
3003 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3004 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3005 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3006 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3007 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
3011 /* Initialize HDP */
3012 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3013 WREG32((0x2c14 + j), 0x00000000);
3014 WREG32((0x2c18 + j), 0x00000000);
3015 WREG32((0x2c1c + j), 0x00000000);
3016 WREG32((0x2c20 + j), 0x00000000);
3017 WREG32((0x2c24 + j), 0x00000000);
3020 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3022 evergreen_fix_pci_max_read_req_size(rdev);
3024 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3026 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3027 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3029 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
3030 rdev->config.si.mem_max_burst_length_bytes = 256;
3031 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3032 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3033 if (rdev->config.si.mem_row_size_in_kb > 4)
3034 rdev->config.si.mem_row_size_in_kb = 4;
3035 /* XXX use MC settings? */
3036 rdev->config.si.shader_engine_tile_size = 32;
3037 rdev->config.si.num_gpus = 1;
3038 rdev->config.si.multi_gpu_tile_size = 64;
3040 /* fix up row size */
3041 gb_addr_config &= ~ROW_SIZE_MASK;
3042 switch (rdev->config.si.mem_row_size_in_kb) {
3045 gb_addr_config |= ROW_SIZE(0);
3048 gb_addr_config |= ROW_SIZE(1);
3051 gb_addr_config |= ROW_SIZE(2);
3055 /* setup tiling info dword. gb_addr_config is not adequate since it does
3056 * not have bank info, so create a custom tiling dword.
3057 * bits 3:0 num_pipes
3058 * bits 7:4 num_banks
3059 * bits 11:8 group_size
3060 * bits 15:12 row_size
3062 rdev->config.si.tile_config = 0;
3063 switch (rdev->config.si.num_tile_pipes) {
3065 rdev->config.si.tile_config |= (0 << 0);
3068 rdev->config.si.tile_config |= (1 << 0);
3071 rdev->config.si.tile_config |= (2 << 0);
3075 /* XXX what about 12? */
3076 rdev->config.si.tile_config |= (3 << 0);
3079 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3080 case 0: /* four banks */
3081 rdev->config.si.tile_config |= 0 << 4;
3083 case 1: /* eight banks */
3084 rdev->config.si.tile_config |= 1 << 4;
3086 case 2: /* sixteen banks */
3088 rdev->config.si.tile_config |= 2 << 4;
3091 rdev->config.si.tile_config |=
3092 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3093 rdev->config.si.tile_config |=
3094 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3096 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3097 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3098 WREG32(DMIF_ADDR_CALC, gb_addr_config);
3099 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3100 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3101 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
3102 if (rdev->has_uvd) {
3103 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3104 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3105 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3108 si_tiling_mode_table_init(rdev);
3110 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3111 rdev->config.si.max_sh_per_se,
3112 rdev->config.si.max_backends_per_se);
3114 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3115 rdev->config.si.max_sh_per_se,
3116 rdev->config.si.max_cu_per_sh);
3118 rdev->config.si.active_cus = 0;
3119 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
3120 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
3121 rdev->config.si.active_cus +=
3122 hweight32(si_get_cu_active_bitmap(rdev, i, j));
3126 /* set HW defaults for 3D engine */
3127 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3128 ROQ_IB2_START(0x2b)));
3129 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3131 sx_debug_1 = RREG32(SX_DEBUG_1);
3132 WREG32(SX_DEBUG_1, sx_debug_1);
3134 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3136 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3137 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3138 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3139 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3141 WREG32(VGT_NUM_INSTANCES, 1);
3143 WREG32(CP_PERFMON_CNTL, 0);
3145 WREG32(SQ_CONFIG, 0);
3147 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3148 FORCE_EOV_MAX_REZ_CNT(255)));
3150 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3151 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3153 WREG32(VGT_GS_VERTEX_REUSE, 16);
3154 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3156 WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3157 WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3158 WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3159 WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3160 WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3161 WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3162 WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3163 WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3165 tmp = RREG32(HDP_MISC_CNTL);
3166 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3167 WREG32(HDP_MISC_CNTL, tmp);
3169 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3170 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3172 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3178 * GPU scratch registers helpers function.
3180 static void si_scratch_init(struct radeon_device *rdev)
3184 rdev->scratch.num_reg = 7;
3185 rdev->scratch.reg_base = SCRATCH_REG0;
3186 for (i = 0; i < rdev->scratch.num_reg; i++) {
3187 rdev->scratch.free[i] = true;
3188 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3192 void si_fence_ring_emit(struct radeon_device *rdev,
3193 struct radeon_fence *fence)
3195 struct radeon_ring *ring = &rdev->ring[fence->ring];
3196 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3198 /* flush read cache over gart */
3199 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3200 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3201 radeon_ring_write(ring, 0);
3202 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3203 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3204 PACKET3_TC_ACTION_ENA |
3205 PACKET3_SH_KCACHE_ACTION_ENA |
3206 PACKET3_SH_ICACHE_ACTION_ENA);
3207 radeon_ring_write(ring, 0xFFFFFFFF);
3208 radeon_ring_write(ring, 0);
3209 radeon_ring_write(ring, 10); /* poll interval */
3210 /* EVENT_WRITE_EOP - flush caches, send int */
3211 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3212 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3213 radeon_ring_write(ring, lower_32_bits(addr));
3214 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3215 radeon_ring_write(ring, fence->seq);
3216 radeon_ring_write(ring, 0);
3222 void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3224 struct radeon_ring *ring = &rdev->ring[ib->ring];
3227 if (ib->is_const_ib) {
3228 /* set switch buffer packet before const IB */
3229 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3230 radeon_ring_write(ring, 0);
3232 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3235 if (ring->rptr_save_reg) {
3236 next_rptr = ring->wptr + 3 + 4 + 8;
3237 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3238 radeon_ring_write(ring, ((ring->rptr_save_reg -
3239 PACKET3_SET_CONFIG_REG_START) >> 2));
3240 radeon_ring_write(ring, next_rptr);
3241 } else if (rdev->wb.enabled) {
3242 next_rptr = ring->wptr + 5 + 4 + 8;
3243 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3244 radeon_ring_write(ring, (1 << 8));
3245 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3246 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
3247 radeon_ring_write(ring, next_rptr);
3250 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3253 radeon_ring_write(ring, header);
3254 radeon_ring_write(ring,
3258 (ib->gpu_addr & 0xFFFFFFFC));
3259 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3260 radeon_ring_write(ring, ib->length_dw |
3261 (ib->vm ? (ib->vm->id << 24) : 0));
3263 if (!ib->is_const_ib) {
3264 /* flush read cache over gart for this vmid */
3265 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3266 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3267 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
3268 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3269 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3270 PACKET3_TC_ACTION_ENA |
3271 PACKET3_SH_KCACHE_ACTION_ENA |
3272 PACKET3_SH_ICACHE_ACTION_ENA);
3273 radeon_ring_write(ring, 0xFFFFFFFF);
3274 radeon_ring_write(ring, 0);
3275 radeon_ring_write(ring, 10); /* poll interval */
3282 static void si_cp_enable(struct radeon_device *rdev, bool enable)
3285 WREG32(CP_ME_CNTL, 0);
3287 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3288 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3289 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3290 WREG32(SCRATCH_UMSK, 0);
3291 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3292 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3293 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3298 static int si_cp_load_microcode(struct radeon_device *rdev)
3300 const __be32 *fw_data;
3303 if (!rdev->me_fw || !rdev->pfp_fw)
3306 si_cp_enable(rdev, false);
3309 fw_data = (const __be32 *)rdev->pfp_fw->data;
3310 WREG32(CP_PFP_UCODE_ADDR, 0);
3311 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3312 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3313 WREG32(CP_PFP_UCODE_ADDR, 0);
3316 fw_data = (const __be32 *)rdev->ce_fw->data;
3317 WREG32(CP_CE_UCODE_ADDR, 0);
3318 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3319 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3320 WREG32(CP_CE_UCODE_ADDR, 0);
3323 fw_data = (const __be32 *)rdev->me_fw->data;
3324 WREG32(CP_ME_RAM_WADDR, 0);
3325 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3326 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3327 WREG32(CP_ME_RAM_WADDR, 0);
3329 WREG32(CP_PFP_UCODE_ADDR, 0);
3330 WREG32(CP_CE_UCODE_ADDR, 0);
3331 WREG32(CP_ME_RAM_WADDR, 0);
3332 WREG32(CP_ME_RAM_RADDR, 0);
3336 static int si_cp_start(struct radeon_device *rdev)
3338 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3341 r = radeon_ring_lock(rdev, ring, 7 + 4);
3343 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3347 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3348 radeon_ring_write(ring, 0x1);
3349 radeon_ring_write(ring, 0x0);
3350 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3351 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3352 radeon_ring_write(ring, 0);
3353 radeon_ring_write(ring, 0);
3355 /* init the CE partitions */
3356 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3357 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3358 radeon_ring_write(ring, 0xc000);
3359 radeon_ring_write(ring, 0xe000);
3360 radeon_ring_unlock_commit(rdev, ring, false);
3362 si_cp_enable(rdev, true);
3364 r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3366 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3370 /* setup clear context state */
3371 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3372 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3374 for (i = 0; i < si_default_size; i++)
3375 radeon_ring_write(ring, si_default_state[i]);
3377 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3378 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3380 /* set clear context state */
3381 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3382 radeon_ring_write(ring, 0);
3384 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3385 radeon_ring_write(ring, 0x00000316);
3386 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3387 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3389 radeon_ring_unlock_commit(rdev, ring, false);
3391 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3392 ring = &rdev->ring[i];
3393 r = radeon_ring_lock(rdev, ring, 2);
3395 /* clear the compute context state */
3396 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3397 radeon_ring_write(ring, 0);
3399 radeon_ring_unlock_commit(rdev, ring, false);
3405 static void si_cp_fini(struct radeon_device *rdev)
3407 struct radeon_ring *ring;
3408 si_cp_enable(rdev, false);
3410 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3411 radeon_ring_fini(rdev, ring);
3412 radeon_scratch_free(rdev, ring->rptr_save_reg);
3414 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3415 radeon_ring_fini(rdev, ring);
3416 radeon_scratch_free(rdev, ring->rptr_save_reg);
3418 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3419 radeon_ring_fini(rdev, ring);
3420 radeon_scratch_free(rdev, ring->rptr_save_reg);
3423 static int si_cp_resume(struct radeon_device *rdev)
3425 struct radeon_ring *ring;
3430 si_enable_gui_idle_interrupt(rdev, false);
3432 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3433 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3435 /* Set the write pointer delay */
3436 WREG32(CP_RB_WPTR_DELAY, 0);
3438 WREG32(CP_DEBUG, 0);
3439 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3441 /* ring 0 - compute and gfx */
3442 /* Set ring buffer size */
3443 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3444 rb_bufsz = order_base_2(ring->ring_size / 8);
3445 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3447 tmp |= BUF_SWAP_32BIT;
3449 WREG32(CP_RB0_CNTL, tmp);
3451 /* Initialize the ring buffer's read and write pointers */
3452 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3454 WREG32(CP_RB0_WPTR, ring->wptr);
3456 /* set the wb address whether it's enabled or not */
3457 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3458 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3460 if (rdev->wb.enabled)
3461 WREG32(SCRATCH_UMSK, 0xff);
3463 tmp |= RB_NO_UPDATE;
3464 WREG32(SCRATCH_UMSK, 0);
3468 WREG32(CP_RB0_CNTL, tmp);
3470 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3472 /* ring1 - compute only */
3473 /* Set ring buffer size */
3474 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3475 rb_bufsz = order_base_2(ring->ring_size / 8);
3476 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3478 tmp |= BUF_SWAP_32BIT;
3480 WREG32(CP_RB1_CNTL, tmp);
3482 /* Initialize the ring buffer's read and write pointers */
3483 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3485 WREG32(CP_RB1_WPTR, ring->wptr);
3487 /* set the wb address whether it's enabled or not */
3488 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3489 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3492 WREG32(CP_RB1_CNTL, tmp);
3494 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3496 /* ring2 - compute only */
3497 /* Set ring buffer size */
3498 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3499 rb_bufsz = order_base_2(ring->ring_size / 8);
3500 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3502 tmp |= BUF_SWAP_32BIT;
3504 WREG32(CP_RB2_CNTL, tmp);
3506 /* Initialize the ring buffer's read and write pointers */
3507 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3509 WREG32(CP_RB2_WPTR, ring->wptr);
3511 /* set the wb address whether it's enabled or not */
3512 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3513 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3516 WREG32(CP_RB2_CNTL, tmp);
3518 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3520 /* start the rings */
3522 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3523 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3524 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3525 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3527 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3528 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3529 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3532 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3534 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3536 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3538 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3541 si_enable_gui_idle_interrupt(rdev, true);
3543 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3544 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
3549 u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
3555 tmp = RREG32(GRBM_STATUS);
3556 if (tmp & (PA_BUSY | SC_BUSY |
3557 BCI_BUSY | SX_BUSY |
3558 TA_BUSY | VGT_BUSY |
3560 GDS_BUSY | SPI_BUSY |
3561 IA_BUSY | IA_BUSY_NO_DMA))
3562 reset_mask |= RADEON_RESET_GFX;
3564 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3565 CP_BUSY | CP_COHERENCY_BUSY))
3566 reset_mask |= RADEON_RESET_CP;
3568 if (tmp & GRBM_EE_BUSY)
3569 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3572 tmp = RREG32(GRBM_STATUS2);
3573 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3574 reset_mask |= RADEON_RESET_RLC;
3576 /* DMA_STATUS_REG 0 */
3577 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3578 if (!(tmp & DMA_IDLE))
3579 reset_mask |= RADEON_RESET_DMA;
3581 /* DMA_STATUS_REG 1 */
3582 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3583 if (!(tmp & DMA_IDLE))
3584 reset_mask |= RADEON_RESET_DMA1;
3587 tmp = RREG32(SRBM_STATUS2);
3589 reset_mask |= RADEON_RESET_DMA;
3591 if (tmp & DMA1_BUSY)
3592 reset_mask |= RADEON_RESET_DMA1;
3595 tmp = RREG32(SRBM_STATUS);
3598 reset_mask |= RADEON_RESET_IH;
3601 reset_mask |= RADEON_RESET_SEM;
3603 if (tmp & GRBM_RQ_PENDING)
3604 reset_mask |= RADEON_RESET_GRBM;
3607 reset_mask |= RADEON_RESET_VMC;
3609 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3610 MCC_BUSY | MCD_BUSY))
3611 reset_mask |= RADEON_RESET_MC;
3613 if (evergreen_is_display_hung(rdev))
3614 reset_mask |= RADEON_RESET_DISPLAY;
3617 tmp = RREG32(VM_L2_STATUS);
3619 reset_mask |= RADEON_RESET_VMC;
3621 /* Skip MC reset as it's mostly likely not hung, just busy */
3622 if (reset_mask & RADEON_RESET_MC) {
3623 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3624 reset_mask &= ~RADEON_RESET_MC;
3630 static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3632 struct evergreen_mc_save save;
3633 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3636 if (reset_mask == 0)
3639 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3641 evergreen_print_gpu_status_regs(rdev);
3642 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3643 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3644 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3645 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3654 /* Disable CP parsing/prefetching */
3655 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3657 if (reset_mask & RADEON_RESET_DMA) {
3659 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3660 tmp &= ~DMA_RB_ENABLE;
3661 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3663 if (reset_mask & RADEON_RESET_DMA1) {
3665 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3666 tmp &= ~DMA_RB_ENABLE;
3667 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3672 evergreen_mc_stop(rdev, &save);
3673 if (evergreen_mc_wait_for_idle(rdev)) {
3674 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3677 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3678 grbm_soft_reset = SOFT_RESET_CB |
3692 if (reset_mask & RADEON_RESET_CP) {
3693 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3695 srbm_soft_reset |= SOFT_RESET_GRBM;
3698 if (reset_mask & RADEON_RESET_DMA)
3699 srbm_soft_reset |= SOFT_RESET_DMA;
3701 if (reset_mask & RADEON_RESET_DMA1)
3702 srbm_soft_reset |= SOFT_RESET_DMA1;
3704 if (reset_mask & RADEON_RESET_DISPLAY)
3705 srbm_soft_reset |= SOFT_RESET_DC;
3707 if (reset_mask & RADEON_RESET_RLC)
3708 grbm_soft_reset |= SOFT_RESET_RLC;
3710 if (reset_mask & RADEON_RESET_SEM)
3711 srbm_soft_reset |= SOFT_RESET_SEM;
3713 if (reset_mask & RADEON_RESET_IH)
3714 srbm_soft_reset |= SOFT_RESET_IH;
3716 if (reset_mask & RADEON_RESET_GRBM)
3717 srbm_soft_reset |= SOFT_RESET_GRBM;
3719 if (reset_mask & RADEON_RESET_VMC)
3720 srbm_soft_reset |= SOFT_RESET_VMC;
3722 if (reset_mask & RADEON_RESET_MC)
3723 srbm_soft_reset |= SOFT_RESET_MC;
3725 if (grbm_soft_reset) {
3726 tmp = RREG32(GRBM_SOFT_RESET);
3727 tmp |= grbm_soft_reset;
3728 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3729 WREG32(GRBM_SOFT_RESET, tmp);
3730 tmp = RREG32(GRBM_SOFT_RESET);
3734 tmp &= ~grbm_soft_reset;
3735 WREG32(GRBM_SOFT_RESET, tmp);
3736 tmp = RREG32(GRBM_SOFT_RESET);
3739 if (srbm_soft_reset) {
3740 tmp = RREG32(SRBM_SOFT_RESET);
3741 tmp |= srbm_soft_reset;
3742 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3743 WREG32(SRBM_SOFT_RESET, tmp);
3744 tmp = RREG32(SRBM_SOFT_RESET);
3748 tmp &= ~srbm_soft_reset;
3749 WREG32(SRBM_SOFT_RESET, tmp);
3750 tmp = RREG32(SRBM_SOFT_RESET);
3753 /* Wait a little for things to settle down */
3756 evergreen_mc_resume(rdev, &save);
3759 evergreen_print_gpu_status_regs(rdev);
3762 static void si_set_clk_bypass_mode(struct radeon_device *rdev)
3766 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3767 tmp |= SPLL_BYPASS_EN;
3768 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3770 tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3771 tmp |= SPLL_CTLREQ_CHG;
3772 WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3774 for (i = 0; i < rdev->usec_timeout; i++) {
3775 if (RREG32(SPLL_STATUS) & SPLL_CHG_STATUS)
3780 tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3781 tmp &= ~(SPLL_CTLREQ_CHG | SCLK_MUX_UPDATE);
3782 WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3784 tmp = RREG32(MPLL_CNTL_MODE);
3785 tmp &= ~MPLL_MCLK_SEL;
3786 WREG32(MPLL_CNTL_MODE, tmp);
3789 static void si_spll_powerdown(struct radeon_device *rdev)
3793 tmp = RREG32(SPLL_CNTL_MODE);
3794 tmp |= SPLL_SW_DIR_CONTROL;
3795 WREG32(SPLL_CNTL_MODE, tmp);
3797 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3799 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3801 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3803 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3805 tmp = RREG32(SPLL_CNTL_MODE);
3806 tmp &= ~SPLL_SW_DIR_CONTROL;
3807 WREG32(SPLL_CNTL_MODE, tmp);
3810 static void si_gpu_pci_config_reset(struct radeon_device *rdev)
3812 struct evergreen_mc_save save;
3815 dev_info(rdev->dev, "GPU pci config reset\n");
3823 /* Disable CP parsing/prefetching */
3824 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3826 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3827 tmp &= ~DMA_RB_ENABLE;
3828 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3830 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3831 tmp &= ~DMA_RB_ENABLE;
3832 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3833 /* XXX other engines? */
3835 /* halt the rlc, disable cp internal ints */
3840 /* disable mem access */
3841 evergreen_mc_stop(rdev, &save);
3842 if (evergreen_mc_wait_for_idle(rdev)) {
3843 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
3846 /* set mclk/sclk to bypass */
3847 si_set_clk_bypass_mode(rdev);
3848 /* powerdown spll */
3849 si_spll_powerdown(rdev);
3851 pci_disable_busmaster(rdev->pdev->dev);
3853 radeon_pci_config_reset(rdev);
3854 /* wait for asic to come out of reset */
3855 for (i = 0; i < rdev->usec_timeout; i++) {
3856 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
3862 int si_asic_reset(struct radeon_device *rdev)
3866 reset_mask = si_gpu_check_soft_reset(rdev);
3869 r600_set_bios_scratch_engine_hung(rdev, true);
3871 /* try soft reset */
3872 si_gpu_soft_reset(rdev, reset_mask);
3874 reset_mask = si_gpu_check_soft_reset(rdev);
3876 /* try pci config reset */
3877 if (reset_mask && radeon_hard_reset)
3878 si_gpu_pci_config_reset(rdev);
3880 reset_mask = si_gpu_check_soft_reset(rdev);
3883 r600_set_bios_scratch_engine_hung(rdev, false);
3889 * si_gfx_is_lockup - Check if the GFX engine is locked up
3891 * @rdev: radeon_device pointer
3892 * @ring: radeon_ring structure holding ring information
3894 * Check if the GFX engine is locked up.
3895 * Returns true if the engine appears to be locked up, false if not.
3897 bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3899 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3901 if (!(reset_mask & (RADEON_RESET_GFX |
3902 RADEON_RESET_COMPUTE |
3903 RADEON_RESET_CP))) {
3904 radeon_ring_lockup_update(rdev, ring);
3907 return radeon_ring_test_lockup(rdev, ring);
3911 static void si_mc_program(struct radeon_device *rdev)
3913 struct evergreen_mc_save save;
3917 /* Initialize HDP */
3918 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3919 WREG32((0x2c14 + j), 0x00000000);
3920 WREG32((0x2c18 + j), 0x00000000);
3921 WREG32((0x2c1c + j), 0x00000000);
3922 WREG32((0x2c20 + j), 0x00000000);
3923 WREG32((0x2c24 + j), 0x00000000);
3925 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
3927 evergreen_mc_stop(rdev, &save);
3928 if (radeon_mc_wait_for_idle(rdev)) {
3929 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3931 if (!ASIC_IS_NODCE(rdev))
3932 /* Lockout access through VGA aperture*/
3933 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
3934 /* Update configuration */
3935 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
3936 rdev->mc.vram_start >> 12);
3937 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
3938 rdev->mc.vram_end >> 12);
3939 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
3940 rdev->vram_scratch.gpu_addr >> 12);
3941 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
3942 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
3943 WREG32(MC_VM_FB_LOCATION, tmp);
3944 /* XXX double check these! */
3945 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
3946 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
3947 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
3948 WREG32(MC_VM_AGP_BASE, 0);
3949 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
3950 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
3951 if (radeon_mc_wait_for_idle(rdev)) {
3952 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3954 evergreen_mc_resume(rdev, &save);
3955 if (!ASIC_IS_NODCE(rdev)) {
3956 /* we need to own VRAM, so turn off the VGA renderer here
3957 * to stop it overwriting our objects */
3958 rv515_vga_render_disable(rdev);
3962 void si_vram_gtt_location(struct radeon_device *rdev,
3963 struct radeon_mc *mc)
3965 if (mc->mc_vram_size > 0xFFC0000000ULL) {
3966 /* leave room for at least 1024M GTT */
3967 dev_warn(rdev->dev, "limiting VRAM\n");
3968 mc->real_vram_size = 0xFFC0000000ULL;
3969 mc->mc_vram_size = 0xFFC0000000ULL;
3971 radeon_vram_location(rdev, &rdev->mc, 0);
3972 rdev->mc.gtt_base_align = 0;
3973 radeon_gtt_location(rdev, mc);
3976 static int si_mc_init(struct radeon_device *rdev)
3979 int chansize, numchan;
3981 /* Get VRAM informations */
3982 rdev->mc.vram_is_ddr = true;
3983 tmp = RREG32(MC_ARB_RAMCFG);
3984 if (tmp & CHANSIZE_OVERRIDE) {
3986 } else if (tmp & CHANSIZE_MASK) {
3991 tmp = RREG32(MC_SHARED_CHMAP);
3992 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4022 rdev->mc.vram_width = numchan * chansize;
4023 /* Could aper size report 0 ? */
4024 rdev->mc.aper_base = drm_get_resource_start(rdev->ddev, 0);
4025 rdev->mc.aper_size = drm_get_resource_len(rdev->ddev, 0);
4026 /* size in MB on si */
4027 tmp = RREG32(CONFIG_MEMSIZE);
4028 /* some boards may have garbage in the upper 16 bits */
4029 if (tmp & 0xffff0000) {
4030 DRM_INFO("Probable bad vram size: 0x%08x\n", tmp);
4034 rdev->mc.mc_vram_size = tmp * 1024ULL * 1024ULL;
4035 rdev->mc.real_vram_size = rdev->mc.mc_vram_size;
4036 rdev->mc.visible_vram_size = rdev->mc.aper_size;
4037 si_vram_gtt_location(rdev, &rdev->mc);
4038 radeon_update_bandwidth_info(rdev);
4046 void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
4048 /* flush hdp cache */
4049 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
4051 /* bits 0-15 are the VM contexts0-15 */
4052 WREG32(VM_INVALIDATE_REQUEST, 1);
4055 static int si_pcie_gart_enable(struct radeon_device *rdev)
4059 if (rdev->gart.robj == NULL) {
4060 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
4063 r = radeon_gart_table_vram_pin(rdev);
4066 /* Setup TLB control */
4067 WREG32(MC_VM_MX_L1_TLB_CNTL,
4070 ENABLE_L1_FRAGMENT_PROCESSING |
4071 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4072 ENABLE_ADVANCED_DRIVER_MODEL |
4073 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4074 /* Setup L2 cache */
4075 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
4076 ENABLE_L2_FRAGMENT_PROCESSING |
4077 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4078 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4079 EFFECTIVE_L2_QUEUE_SIZE(7) |
4080 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4081 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
4082 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4084 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
4085 /* setup context0 */
4086 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
4087 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
4088 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
4089 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
4090 (u32)(rdev->dummy_page.addr >> 12));
4091 WREG32(VM_CONTEXT0_CNTL2, 0);
4092 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
4093 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
4099 /* empty context1-15 */
4100 /* set vm size, must be a multiple of 4 */
4101 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
4102 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
4103 /* Assign the pt base to something valid for now; the pts used for
4104 * the VMs are determined by the application and setup and assigned
4105 * on the fly in the vm part of radeon_gart.c
4107 for (i = 1; i < 16; i++) {
4109 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
4110 rdev->vm_manager.saved_table_addr[i]);
4112 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
4113 rdev->vm_manager.saved_table_addr[i]);
4116 /* enable context1-15 */
4117 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
4118 (u32)(rdev->dummy_page.addr >> 12));
4119 WREG32(VM_CONTEXT1_CNTL2, 4);
4120 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4121 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
4122 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4123 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4124 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4125 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4126 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
4127 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
4128 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
4129 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
4130 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
4131 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
4132 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4133 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
4135 si_pcie_gart_tlb_flush(rdev);
4136 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
4137 (unsigned)(rdev->mc.gtt_size >> 20),
4138 (unsigned long long)rdev->gart.table_addr);
4139 rdev->gart.ready = true;
4143 static void si_pcie_gart_disable(struct radeon_device *rdev)
4147 for (i = 1; i < 16; ++i) {
4150 reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
4152 reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
4153 rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
4156 /* Disable all tables */
4157 WREG32(VM_CONTEXT0_CNTL, 0);
4158 WREG32(VM_CONTEXT1_CNTL, 0);
4159 /* Setup TLB control */
4160 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4161 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4162 /* Setup L2 cache */
4163 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4164 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4165 EFFECTIVE_L2_QUEUE_SIZE(7) |
4166 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4167 WREG32(VM_L2_CNTL2, 0);
4168 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4169 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4170 radeon_gart_table_vram_unpin(rdev);
4173 static void si_pcie_gart_fini(struct radeon_device *rdev)
4175 si_pcie_gart_disable(rdev);
4176 radeon_gart_table_vram_free(rdev);
4177 radeon_gart_fini(rdev);
4181 static bool si_vm_reg_valid(u32 reg)
4183 /* context regs are fine */
4187 /* check config regs */
4189 case GRBM_GFX_INDEX:
4190 case CP_STRMOUT_CNTL:
4191 case VGT_VTX_VECT_EJECT_REG:
4192 case VGT_CACHE_INVALIDATION:
4193 case VGT_ESGS_RING_SIZE:
4194 case VGT_GSVS_RING_SIZE:
4195 case VGT_GS_VERTEX_REUSE:
4196 case VGT_PRIMITIVE_TYPE:
4197 case VGT_INDEX_TYPE:
4198 case VGT_NUM_INDICES:
4199 case VGT_NUM_INSTANCES:
4200 case VGT_TF_RING_SIZE:
4201 case VGT_HS_OFFCHIP_PARAM:
4202 case VGT_TF_MEMORY_BASE:
4204 case PA_SU_LINE_STIPPLE_VALUE:
4205 case PA_SC_LINE_STIPPLE_STATE:
4208 case SPI_STATIC_THREAD_MGMT_1:
4209 case SPI_STATIC_THREAD_MGMT_2:
4210 case SPI_STATIC_THREAD_MGMT_3:
4211 case SPI_PS_MAX_WAVE_ID:
4212 case SPI_CONFIG_CNTL:
4213 case SPI_CONFIG_CNTL_1:
4217 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4222 static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4223 u32 *ib, struct radeon_cs_packet *pkt)
4225 switch (pkt->opcode) {
4227 case PACKET3_SET_BASE:
4228 case PACKET3_SET_CE_DE_COUNTERS:
4229 case PACKET3_LOAD_CONST_RAM:
4230 case PACKET3_WRITE_CONST_RAM:
4231 case PACKET3_WRITE_CONST_RAM_OFFSET:
4232 case PACKET3_DUMP_CONST_RAM:
4233 case PACKET3_INCREMENT_CE_COUNTER:
4234 case PACKET3_WAIT_ON_DE_COUNTER:
4235 case PACKET3_CE_WRITE:
4238 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4244 static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
4246 u32 start_reg, reg, i;
4247 u32 command = ib[idx + 4];
4248 u32 info = ib[idx + 1];
4249 u32 idx_value = ib[idx];
4250 if (command & PACKET3_CP_DMA_CMD_SAS) {
4251 /* src address space is register */
4252 if (((info & 0x60000000) >> 29) == 0) {
4253 start_reg = idx_value << 2;
4254 if (command & PACKET3_CP_DMA_CMD_SAIC) {
4256 if (!si_vm_reg_valid(reg)) {
4257 DRM_ERROR("CP DMA Bad SRC register\n");
4261 for (i = 0; i < (command & 0x1fffff); i++) {
4262 reg = start_reg + (4 * i);
4263 if (!si_vm_reg_valid(reg)) {
4264 DRM_ERROR("CP DMA Bad SRC register\n");
4271 if (command & PACKET3_CP_DMA_CMD_DAS) {
4272 /* dst address space is register */
4273 if (((info & 0x00300000) >> 20) == 0) {
4274 start_reg = ib[idx + 2];
4275 if (command & PACKET3_CP_DMA_CMD_DAIC) {
4277 if (!si_vm_reg_valid(reg)) {
4278 DRM_ERROR("CP DMA Bad DST register\n");
4282 for (i = 0; i < (command & 0x1fffff); i++) {
4283 reg = start_reg + (4 * i);
4284 if (!si_vm_reg_valid(reg)) {
4285 DRM_ERROR("CP DMA Bad DST register\n");
4295 static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4296 u32 *ib, struct radeon_cs_packet *pkt)
4299 u32 idx = pkt->idx + 1;
4300 u32 idx_value = ib[idx];
4301 u32 start_reg, end_reg, reg, i;
4303 switch (pkt->opcode) {
4305 case PACKET3_SET_BASE:
4306 case PACKET3_CLEAR_STATE:
4307 case PACKET3_INDEX_BUFFER_SIZE:
4308 case PACKET3_DISPATCH_DIRECT:
4309 case PACKET3_DISPATCH_INDIRECT:
4310 case PACKET3_ALLOC_GDS:
4311 case PACKET3_WRITE_GDS_RAM:
4312 case PACKET3_ATOMIC_GDS:
4313 case PACKET3_ATOMIC:
4314 case PACKET3_OCCLUSION_QUERY:
4315 case PACKET3_SET_PREDICATION:
4316 case PACKET3_COND_EXEC:
4317 case PACKET3_PRED_EXEC:
4318 case PACKET3_DRAW_INDIRECT:
4319 case PACKET3_DRAW_INDEX_INDIRECT:
4320 case PACKET3_INDEX_BASE:
4321 case PACKET3_DRAW_INDEX_2:
4322 case PACKET3_CONTEXT_CONTROL:
4323 case PACKET3_INDEX_TYPE:
4324 case PACKET3_DRAW_INDIRECT_MULTI:
4325 case PACKET3_DRAW_INDEX_AUTO:
4326 case PACKET3_DRAW_INDEX_IMMD:
4327 case PACKET3_NUM_INSTANCES:
4328 case PACKET3_DRAW_INDEX_MULTI_AUTO:
4329 case PACKET3_STRMOUT_BUFFER_UPDATE:
4330 case PACKET3_DRAW_INDEX_OFFSET_2:
4331 case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4332 case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4333 case PACKET3_MPEG_INDEX:
4334 case PACKET3_WAIT_REG_MEM:
4335 case PACKET3_MEM_WRITE:
4336 case PACKET3_PFP_SYNC_ME:
4337 case PACKET3_SURFACE_SYNC:
4338 case PACKET3_EVENT_WRITE:
4339 case PACKET3_EVENT_WRITE_EOP:
4340 case PACKET3_EVENT_WRITE_EOS:
4341 case PACKET3_SET_CONTEXT_REG:
4342 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4343 case PACKET3_SET_SH_REG:
4344 case PACKET3_SET_SH_REG_OFFSET:
4345 case PACKET3_INCREMENT_DE_COUNTER:
4346 case PACKET3_WAIT_ON_CE_COUNTER:
4347 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4348 case PACKET3_ME_WRITE:
4350 case PACKET3_COPY_DATA:
4351 if ((idx_value & 0xf00) == 0) {
4352 reg = ib[idx + 3] * 4;
4353 if (!si_vm_reg_valid(reg))
4357 case PACKET3_WRITE_DATA:
4358 if ((idx_value & 0xf00) == 0) {
4359 start_reg = ib[idx + 1] * 4;
4360 if (idx_value & 0x10000) {
4361 if (!si_vm_reg_valid(start_reg))
4364 for (i = 0; i < (pkt->count - 2); i++) {
4365 reg = start_reg + (4 * i);
4366 if (!si_vm_reg_valid(reg))
4372 case PACKET3_COND_WRITE:
4373 if (idx_value & 0x100) {
4374 reg = ib[idx + 5] * 4;
4375 if (!si_vm_reg_valid(reg))
4379 case PACKET3_COPY_DW:
4380 if (idx_value & 0x2) {
4381 reg = ib[idx + 3] * 4;
4382 if (!si_vm_reg_valid(reg))
4386 case PACKET3_SET_CONFIG_REG:
4387 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4388 end_reg = 4 * pkt->count + start_reg - 4;
4389 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4390 (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4391 (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4392 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4395 for (i = 0; i < pkt->count; i++) {
4396 reg = start_reg + (4 * i);
4397 if (!si_vm_reg_valid(reg))
4401 case PACKET3_CP_DMA:
4402 r = si_vm_packet3_cp_dma_check(ib, idx);
4407 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4413 static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4414 u32 *ib, struct radeon_cs_packet *pkt)
4417 u32 idx = pkt->idx + 1;
4418 u32 idx_value = ib[idx];
4419 u32 start_reg, reg, i;
4421 switch (pkt->opcode) {
4423 case PACKET3_SET_BASE:
4424 case PACKET3_CLEAR_STATE:
4425 case PACKET3_DISPATCH_DIRECT:
4426 case PACKET3_DISPATCH_INDIRECT:
4427 case PACKET3_ALLOC_GDS:
4428 case PACKET3_WRITE_GDS_RAM:
4429 case PACKET3_ATOMIC_GDS:
4430 case PACKET3_ATOMIC:
4431 case PACKET3_OCCLUSION_QUERY:
4432 case PACKET3_SET_PREDICATION:
4433 case PACKET3_COND_EXEC:
4434 case PACKET3_PRED_EXEC:
4435 case PACKET3_CONTEXT_CONTROL:
4436 case PACKET3_STRMOUT_BUFFER_UPDATE:
4437 case PACKET3_WAIT_REG_MEM:
4438 case PACKET3_MEM_WRITE:
4439 case PACKET3_PFP_SYNC_ME:
4440 case PACKET3_SURFACE_SYNC:
4441 case PACKET3_EVENT_WRITE:
4442 case PACKET3_EVENT_WRITE_EOP:
4443 case PACKET3_EVENT_WRITE_EOS:
4444 case PACKET3_SET_CONTEXT_REG:
4445 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4446 case PACKET3_SET_SH_REG:
4447 case PACKET3_SET_SH_REG_OFFSET:
4448 case PACKET3_INCREMENT_DE_COUNTER:
4449 case PACKET3_WAIT_ON_CE_COUNTER:
4450 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4451 case PACKET3_ME_WRITE:
4453 case PACKET3_COPY_DATA:
4454 if ((idx_value & 0xf00) == 0) {
4455 reg = ib[idx + 3] * 4;
4456 if (!si_vm_reg_valid(reg))
4460 case PACKET3_WRITE_DATA:
4461 if ((idx_value & 0xf00) == 0) {
4462 start_reg = ib[idx + 1] * 4;
4463 if (idx_value & 0x10000) {
4464 if (!si_vm_reg_valid(start_reg))
4467 for (i = 0; i < (pkt->count - 2); i++) {
4468 reg = start_reg + (4 * i);
4469 if (!si_vm_reg_valid(reg))
4475 case PACKET3_COND_WRITE:
4476 if (idx_value & 0x100) {
4477 reg = ib[idx + 5] * 4;
4478 if (!si_vm_reg_valid(reg))
4482 case PACKET3_COPY_DW:
4483 if (idx_value & 0x2) {
4484 reg = ib[idx + 3] * 4;
4485 if (!si_vm_reg_valid(reg))
4489 case PACKET3_CP_DMA:
4490 r = si_vm_packet3_cp_dma_check(ib, idx);
4495 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4501 int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4505 struct radeon_cs_packet pkt;
4509 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4510 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
4513 case RADEON_PACKET_TYPE0:
4514 dev_err(rdev->dev, "Packet0 not allowed!\n");
4515 for (i = 0; i < ib->length_dw; i++) {
4517 printk("\t0x%08x <---\n", ib->ptr[i]);
4519 printk("\t0x%08x\n", ib->ptr[i]);
4523 case RADEON_PACKET_TYPE2:
4526 case RADEON_PACKET_TYPE3:
4527 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
4528 if (ib->is_const_ib)
4529 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4532 case RADEON_RING_TYPE_GFX_INDEX:
4533 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4535 case CAYMAN_RING_TYPE_CP1_INDEX:
4536 case CAYMAN_RING_TYPE_CP2_INDEX:
4537 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4540 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
4545 idx += pkt.count + 2;
4548 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4554 } while (idx < ib->length_dw);
4562 int si_vm_init(struct radeon_device *rdev)
4565 rdev->vm_manager.nvm = 16;
4566 /* base offset of vram pages */
4567 rdev->vm_manager.vram_base_offset = 0;
4572 void si_vm_fini(struct radeon_device *rdev)
4577 * si_vm_decode_fault - print human readable fault info
4579 * @rdev: radeon_device pointer
4580 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4581 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4583 * Print human readable fault information (SI).
4585 static void si_vm_decode_fault(struct radeon_device *rdev,
4586 u32 status, u32 addr)
4588 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4589 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4590 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4593 if (rdev->family == CHIP_TAHITI) {
4834 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
4835 protections, vmid, addr,
4836 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
4840 void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4842 struct radeon_ring *ring = &rdev->ring[ridx];
4847 /* write new base address */
4848 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4849 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
4850 WRITE_DATA_DST_SEL(0)));
4853 radeon_ring_write(ring,
4854 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
4856 radeon_ring_write(ring,
4857 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
4859 radeon_ring_write(ring, 0);
4860 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4862 /* flush hdp cache */
4863 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4864 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
4865 WRITE_DATA_DST_SEL(0)));
4866 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4867 radeon_ring_write(ring, 0);
4868 radeon_ring_write(ring, 0x1);
4870 /* bits 0-15 are the VM contexts0-15 */
4871 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4872 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
4873 WRITE_DATA_DST_SEL(0)));
4874 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4875 radeon_ring_write(ring, 0);
4876 radeon_ring_write(ring, 1 << vm->id);
4878 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4879 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4880 radeon_ring_write(ring, 0x0);
4884 * Power and clock gating
4886 static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
4890 for (i = 0; i < rdev->usec_timeout; i++) {
4891 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
4896 for (i = 0; i < rdev->usec_timeout; i++) {
4897 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
4903 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
4906 u32 tmp = RREG32(CP_INT_CNTL_RING0);
4911 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4913 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4914 WREG32(CP_INT_CNTL_RING0, tmp);
4917 /* read a gfx register */
4918 tmp = RREG32(DB_DEPTH_INFO);
4920 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
4921 for (i = 0; i < rdev->usec_timeout; i++) {
4922 if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
4929 static void si_set_uvd_dcm(struct radeon_device *rdev,
4934 tmp = RREG32(UVD_CGC_CTRL);
4935 tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
4936 tmp |= DCM | CG_DT(1) | CLK_OD(4);
4940 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
4946 WREG32(UVD_CGC_CTRL, tmp);
4947 WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
4950 void si_init_uvd_internal_cg(struct radeon_device *rdev)
4952 bool hw_mode = true;
4955 si_set_uvd_dcm(rdev, false);
4957 u32 tmp = RREG32(UVD_CGC_CTRL);
4959 WREG32(UVD_CGC_CTRL, tmp);
4963 static u32 si_halt_rlc(struct radeon_device *rdev)
4967 orig = data = RREG32(RLC_CNTL);
4969 if (data & RLC_ENABLE) {
4970 data &= ~RLC_ENABLE;
4971 WREG32(RLC_CNTL, data);
4973 si_wait_for_rlc_serdes(rdev);
4979 static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
4983 tmp = RREG32(RLC_CNTL);
4985 WREG32(RLC_CNTL, rlc);
4988 static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
4992 orig = data = RREG32(DMA_PG);
4993 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
4994 data |= PG_CNTL_ENABLE;
4996 data &= ~PG_CNTL_ENABLE;
4998 WREG32(DMA_PG, data);
5001 static void si_init_dma_pg(struct radeon_device *rdev)
5005 WREG32(DMA_PGFSM_WRITE, 0x00002000);
5006 WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
5008 for (tmp = 0; tmp < 5; tmp++)
5009 WREG32(DMA_PGFSM_WRITE, 0);
5012 static void si_enable_gfx_cgpg(struct radeon_device *rdev,
5017 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
5018 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
5019 WREG32(RLC_TTOP_D, tmp);
5021 tmp = RREG32(RLC_PG_CNTL);
5022 tmp |= GFX_PG_ENABLE;
5023 WREG32(RLC_PG_CNTL, tmp);
5025 tmp = RREG32(RLC_AUTO_PG_CTRL);
5027 WREG32(RLC_AUTO_PG_CTRL, tmp);
5029 tmp = RREG32(RLC_AUTO_PG_CTRL);
5031 WREG32(RLC_AUTO_PG_CTRL, tmp);
5033 tmp = RREG32(DB_RENDER_CONTROL);
5037 static void si_init_gfx_cgpg(struct radeon_device *rdev)
5041 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5043 tmp = RREG32(RLC_PG_CNTL);
5045 WREG32(RLC_PG_CNTL, tmp);
5047 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5049 tmp = RREG32(RLC_AUTO_PG_CTRL);
5051 tmp &= ~GRBM_REG_SGIT_MASK;
5052 tmp |= GRBM_REG_SGIT(0x700);
5053 tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
5054 WREG32(RLC_AUTO_PG_CTRL, tmp);
5057 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
5059 u32 mask = 0, tmp, tmp1;
5062 si_select_se_sh(rdev, se, sh);
5063 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
5064 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
5065 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5072 for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
5077 return (~tmp) & mask;
5080 static void si_init_ao_cu_mask(struct radeon_device *rdev)
5082 u32 i, j, k, active_cu_number = 0;
5083 u32 mask, counter, cu_bitmap;
5086 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
5087 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
5091 for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
5092 if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
5100 active_cu_number += counter;
5101 tmp |= (cu_bitmap << (i * 16 + j * 8));
5105 WREG32(RLC_PG_AO_CU_MASK, tmp);
5107 tmp = RREG32(RLC_MAX_PG_CU);
5108 tmp &= ~MAX_PU_CU_MASK;
5109 tmp |= MAX_PU_CU(active_cu_number);
5110 WREG32(RLC_MAX_PG_CU, tmp);
5113 static void si_enable_cgcg(struct radeon_device *rdev,
5116 u32 data, orig, tmp;
5118 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5120 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5121 si_enable_gui_idle_interrupt(rdev, true);
5123 WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
5125 tmp = si_halt_rlc(rdev);
5127 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5128 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5129 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
5131 si_wait_for_rlc_serdes(rdev);
5133 si_update_rlc(rdev, tmp);
5135 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
5137 data |= CGCG_EN | CGLS_EN;
5139 si_enable_gui_idle_interrupt(rdev, false);
5141 RREG32(CB_CGTT_SCLK_CTRL);
5142 RREG32(CB_CGTT_SCLK_CTRL);
5143 RREG32(CB_CGTT_SCLK_CTRL);
5144 RREG32(CB_CGTT_SCLK_CTRL);
5146 data &= ~(CGCG_EN | CGLS_EN);
5150 WREG32(RLC_CGCG_CGLS_CTRL, data);
5153 static void si_enable_mgcg(struct radeon_device *rdev,
5156 u32 data, orig, tmp = 0;
5158 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5159 orig = data = RREG32(CGTS_SM_CTRL_REG);
5162 WREG32(CGTS_SM_CTRL_REG, data);
5164 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5165 orig = data = RREG32(CP_MEM_SLP_CNTL);
5166 data |= CP_MEM_LS_EN;
5168 WREG32(CP_MEM_SLP_CNTL, data);
5171 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5174 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5176 tmp = si_halt_rlc(rdev);
5178 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5179 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5180 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5182 si_update_rlc(rdev, tmp);
5184 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5187 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5189 data = RREG32(CP_MEM_SLP_CNTL);
5190 if (data & CP_MEM_LS_EN) {
5191 data &= ~CP_MEM_LS_EN;
5192 WREG32(CP_MEM_SLP_CNTL, data);
5194 orig = data = RREG32(CGTS_SM_CTRL_REG);
5195 data |= LS_OVERRIDE | OVERRIDE;
5197 WREG32(CGTS_SM_CTRL_REG, data);
5199 tmp = si_halt_rlc(rdev);
5201 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5202 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5203 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5205 si_update_rlc(rdev, tmp);
5209 static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5212 u32 orig, data, tmp;
5214 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
5215 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5217 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5219 orig = data = RREG32(UVD_CGC_CTRL);
5222 WREG32(UVD_CGC_CTRL, data);
5224 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5225 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5227 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5229 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5231 orig = data = RREG32(UVD_CGC_CTRL);
5234 WREG32(UVD_CGC_CTRL, data);
5236 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5237 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5241 static const u32 mc_cg_registers[] =
5254 static void si_enable_mc_ls(struct radeon_device *rdev,
5260 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5261 orig = data = RREG32(mc_cg_registers[i]);
5262 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5263 data |= MC_LS_ENABLE;
5265 data &= ~MC_LS_ENABLE;
5267 WREG32(mc_cg_registers[i], data);
5271 static void si_enable_mc_mgcg(struct radeon_device *rdev,
5277 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5278 orig = data = RREG32(mc_cg_registers[i]);
5279 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5280 data |= MC_CG_ENABLE;
5282 data &= ~MC_CG_ENABLE;
5284 WREG32(mc_cg_registers[i], data);
5288 static void si_enable_dma_mgcg(struct radeon_device *rdev,
5291 u32 orig, data, offset;
5294 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5295 for (i = 0; i < 2; i++) {
5297 offset = DMA0_REGISTER_OFFSET;
5299 offset = DMA1_REGISTER_OFFSET;
5300 orig = data = RREG32(DMA_POWER_CNTL + offset);
5301 data &= ~MEM_POWER_OVERRIDE;
5303 WREG32(DMA_POWER_CNTL + offset, data);
5304 WREG32(DMA_CLK_CTRL + offset, 0x00000100);
5307 for (i = 0; i < 2; i++) {
5309 offset = DMA0_REGISTER_OFFSET;
5311 offset = DMA1_REGISTER_OFFSET;
5312 orig = data = RREG32(DMA_POWER_CNTL + offset);
5313 data |= MEM_POWER_OVERRIDE;
5315 WREG32(DMA_POWER_CNTL + offset, data);
5317 orig = data = RREG32(DMA_CLK_CTRL + offset);
5320 WREG32(DMA_CLK_CTRL + offset, data);
5325 static void si_enable_bif_mgls(struct radeon_device *rdev,
5330 orig = data = RREG32_PCIE(PCIE_CNTL2);
5332 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5333 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5334 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5336 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5337 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5340 WREG32_PCIE(PCIE_CNTL2, data);
5343 static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5348 orig = data = RREG32(HDP_HOST_PATH_CNTL);
5350 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5351 data &= ~CLOCK_GATING_DIS;
5353 data |= CLOCK_GATING_DIS;
5356 WREG32(HDP_HOST_PATH_CNTL, data);
5359 static void si_enable_hdp_ls(struct radeon_device *rdev,
5364 orig = data = RREG32(HDP_MEM_POWER_LS);
5366 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5367 data |= HDP_LS_ENABLE;
5369 data &= ~HDP_LS_ENABLE;
5372 WREG32(HDP_MEM_POWER_LS, data);
5375 static void si_update_cg(struct radeon_device *rdev,
5376 u32 block, bool enable)
5378 if (block & RADEON_CG_BLOCK_GFX) {
5379 si_enable_gui_idle_interrupt(rdev, false);
5380 /* order matters! */
5382 si_enable_mgcg(rdev, true);
5383 si_enable_cgcg(rdev, true);
5385 si_enable_cgcg(rdev, false);
5386 si_enable_mgcg(rdev, false);
5388 si_enable_gui_idle_interrupt(rdev, true);
5391 if (block & RADEON_CG_BLOCK_MC) {
5392 si_enable_mc_mgcg(rdev, enable);
5393 si_enable_mc_ls(rdev, enable);
5396 if (block & RADEON_CG_BLOCK_SDMA) {
5397 si_enable_dma_mgcg(rdev, enable);
5400 if (block & RADEON_CG_BLOCK_BIF) {
5401 si_enable_bif_mgls(rdev, enable);
5404 if (block & RADEON_CG_BLOCK_UVD) {
5405 if (rdev->has_uvd) {
5406 si_enable_uvd_mgcg(rdev, enable);
5410 if (block & RADEON_CG_BLOCK_HDP) {
5411 si_enable_hdp_mgcg(rdev, enable);
5412 si_enable_hdp_ls(rdev, enable);
5416 static void si_init_cg(struct radeon_device *rdev)
5418 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5419 RADEON_CG_BLOCK_MC |
5420 RADEON_CG_BLOCK_SDMA |
5421 RADEON_CG_BLOCK_BIF |
5422 RADEON_CG_BLOCK_HDP), true);
5423 if (rdev->has_uvd) {
5424 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
5425 si_init_uvd_internal_cg(rdev);
5429 static void si_fini_cg(struct radeon_device *rdev)
5431 if (rdev->has_uvd) {
5432 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
5434 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5435 RADEON_CG_BLOCK_MC |
5436 RADEON_CG_BLOCK_SDMA |
5437 RADEON_CG_BLOCK_BIF |
5438 RADEON_CG_BLOCK_HDP), false);
5441 u32 si_get_csb_size(struct radeon_device *rdev)
5444 const struct cs_section_def *sect = NULL;
5445 const struct cs_extent_def *ext = NULL;
5447 if (rdev->rlc.cs_data == NULL)
5450 /* begin clear state */
5452 /* context control state */
5455 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5456 for (ext = sect->section; ext->extent != NULL; ++ext) {
5457 if (sect->id == SECT_CONTEXT)
5458 count += 2 + ext->reg_count;
5463 /* pa_sc_raster_config */
5465 /* end clear state */
5473 void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5476 const struct cs_section_def *sect = NULL;
5477 const struct cs_extent_def *ext = NULL;
5479 if (rdev->rlc.cs_data == NULL)
5484 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5485 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
5487 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5488 buffer[count++] = cpu_to_le32(0x80000000);
5489 buffer[count++] = cpu_to_le32(0x80000000);
5491 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5492 for (ext = sect->section; ext->extent != NULL; ++ext) {
5493 if (sect->id == SECT_CONTEXT) {
5495 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
5496 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
5497 for (i = 0; i < ext->reg_count; i++)
5498 buffer[count++] = cpu_to_le32(ext->extent[i]);
5505 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
5506 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
5507 switch (rdev->family) {
5510 buffer[count++] = cpu_to_le32(0x2a00126a);
5513 buffer[count++] = cpu_to_le32(0x0000124a);
5516 buffer[count++] = cpu_to_le32(0x00000082);
5519 buffer[count++] = cpu_to_le32(0x00000000);
5522 buffer[count++] = cpu_to_le32(0x00000000);
5526 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5527 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
5529 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
5530 buffer[count++] = cpu_to_le32(0);
5533 static void si_init_pg(struct radeon_device *rdev)
5535 if (rdev->pg_flags) {
5536 if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5537 si_init_dma_pg(rdev);
5539 si_init_ao_cu_mask(rdev);
5540 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
5541 si_init_gfx_cgpg(rdev);
5543 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5544 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5546 si_enable_dma_pg(rdev, true);
5547 si_enable_gfx_cgpg(rdev, true);
5549 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5550 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5554 static void si_fini_pg(struct radeon_device *rdev)
5556 if (rdev->pg_flags) {
5557 si_enable_dma_pg(rdev, false);
5558 si_enable_gfx_cgpg(rdev, false);
5565 void si_rlc_reset(struct radeon_device *rdev)
5567 u32 tmp = RREG32(GRBM_SOFT_RESET);
5569 tmp |= SOFT_RESET_RLC;
5570 WREG32(GRBM_SOFT_RESET, tmp);
5572 tmp &= ~SOFT_RESET_RLC;
5573 WREG32(GRBM_SOFT_RESET, tmp);
5577 static void si_rlc_stop(struct radeon_device *rdev)
5579 WREG32(RLC_CNTL, 0);
5581 si_enable_gui_idle_interrupt(rdev, false);
5583 si_wait_for_rlc_serdes(rdev);
5586 static void si_rlc_start(struct radeon_device *rdev)
5588 WREG32(RLC_CNTL, RLC_ENABLE);
5590 si_enable_gui_idle_interrupt(rdev, true);
5595 static bool si_lbpw_supported(struct radeon_device *rdev)
5599 /* Enable LBPW only for DDR3 */
5600 tmp = RREG32(MC_SEQ_MISC0);
5601 if ((tmp & 0xF0000000) == 0xB0000000)
5606 static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5610 tmp = RREG32(RLC_LB_CNTL);
5612 tmp |= LOAD_BALANCE_ENABLE;
5614 tmp &= ~LOAD_BALANCE_ENABLE;
5615 WREG32(RLC_LB_CNTL, tmp);
5618 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5619 WREG32(SPI_LB_CU_MASK, 0x00ff);
5623 static int si_rlc_resume(struct radeon_device *rdev)
5626 const __be32 *fw_data;
5639 WREG32(RLC_RL_BASE, 0);
5640 WREG32(RLC_RL_SIZE, 0);
5641 WREG32(RLC_LB_CNTL, 0);
5642 WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5643 WREG32(RLC_LB_CNTR_INIT, 0);
5644 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5646 WREG32(RLC_MC_CNTL, 0);
5647 WREG32(RLC_UCODE_CNTL, 0);
5649 fw_data = (const __be32 *)rdev->rlc_fw->data;
5650 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5651 WREG32(RLC_UCODE_ADDR, i);
5652 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5654 WREG32(RLC_UCODE_ADDR, 0);
5656 si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5663 static void si_enable_interrupts(struct radeon_device *rdev)
5665 u32 ih_cntl = RREG32(IH_CNTL);
5666 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5668 ih_cntl |= ENABLE_INTR;
5669 ih_rb_cntl |= IH_RB_ENABLE;
5670 WREG32(IH_CNTL, ih_cntl);
5671 WREG32(IH_RB_CNTL, ih_rb_cntl);
5672 rdev->ih.enabled = true;
5675 static void si_disable_interrupts(struct radeon_device *rdev)
5677 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5678 u32 ih_cntl = RREG32(IH_CNTL);
5680 ih_rb_cntl &= ~IH_RB_ENABLE;
5681 ih_cntl &= ~ENABLE_INTR;
5682 WREG32(IH_RB_CNTL, ih_rb_cntl);
5683 WREG32(IH_CNTL, ih_cntl);
5684 /* set rptr, wptr to 0 */
5685 WREG32(IH_RB_RPTR, 0);
5686 WREG32(IH_RB_WPTR, 0);
5687 rdev->ih.enabled = false;
5691 static void si_disable_interrupt_state(struct radeon_device *rdev)
5695 tmp = RREG32(CP_INT_CNTL_RING0) &
5696 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5697 WREG32(CP_INT_CNTL_RING0, tmp);
5698 WREG32(CP_INT_CNTL_RING1, 0);
5699 WREG32(CP_INT_CNTL_RING2, 0);
5700 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5701 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5702 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5703 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
5704 WREG32(GRBM_INT_CNTL, 0);
5705 if (rdev->num_crtc >= 2) {
5706 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5707 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5709 if (rdev->num_crtc >= 4) {
5710 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5711 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5713 if (rdev->num_crtc >= 6) {
5714 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5715 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5718 if (rdev->num_crtc >= 2) {
5719 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5720 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5722 if (rdev->num_crtc >= 4) {
5723 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5724 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5726 if (rdev->num_crtc >= 6) {
5727 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5728 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5731 if (!ASIC_IS_NODCE(rdev)) {
5732 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
5734 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5735 WREG32(DC_HPD1_INT_CONTROL, tmp);
5736 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5737 WREG32(DC_HPD2_INT_CONTROL, tmp);
5738 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5739 WREG32(DC_HPD3_INT_CONTROL, tmp);
5740 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5741 WREG32(DC_HPD4_INT_CONTROL, tmp);
5742 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5743 WREG32(DC_HPD5_INT_CONTROL, tmp);
5744 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5745 WREG32(DC_HPD6_INT_CONTROL, tmp);
5749 static int si_irq_init(struct radeon_device *rdev)
5753 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5756 ret = r600_ih_ring_alloc(rdev);
5761 si_disable_interrupts(rdev);
5764 ret = si_rlc_resume(rdev);
5766 r600_ih_ring_fini(rdev);
5770 /* setup interrupt control */
5771 /* set dummy read address to ring address */
5772 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
5773 interrupt_cntl = RREG32(INTERRUPT_CNTL);
5774 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5775 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5777 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5778 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5779 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5780 WREG32(INTERRUPT_CNTL, interrupt_cntl);
5782 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
5783 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
5785 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5786 IH_WPTR_OVERFLOW_CLEAR |
5789 if (rdev->wb.enabled)
5790 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
5792 /* set the writeback address whether it's enabled or not */
5793 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
5794 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
5796 WREG32(IH_RB_CNTL, ih_rb_cntl);
5798 /* set rptr, wptr to 0 */
5799 WREG32(IH_RB_RPTR, 0);
5800 WREG32(IH_RB_WPTR, 0);
5802 /* Default settings for IH_CNTL (disabled at first) */
5803 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
5804 /* RPTR_REARM only works if msi's are enabled */
5805 if (rdev->msi_enabled)
5806 ih_cntl |= RPTR_REARM;
5807 WREG32(IH_CNTL, ih_cntl);
5809 /* force the active interrupt state to all disabled */
5810 si_disable_interrupt_state(rdev);
5812 pci_enable_busmaster(rdev->dev);
5815 si_enable_interrupts(rdev);
5820 int si_irq_set(struct radeon_device *rdev)
5823 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
5824 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
5825 u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
5826 u32 grbm_int_cntl = 0;
5827 u32 dma_cntl, dma_cntl1;
5828 u32 thermal_int = 0;
5830 if (!rdev->irq.installed) {
5831 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
5834 /* don't enable anything if the ih is disabled */
5835 if (!rdev->ih.enabled) {
5836 si_disable_interrupts(rdev);
5837 /* force the active interrupt state to all disabled */
5838 si_disable_interrupt_state(rdev);
5842 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
5843 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5845 if (!ASIC_IS_NODCE(rdev)) {
5846 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
5847 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
5848 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
5849 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
5850 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
5851 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
5854 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5855 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5857 thermal_int = RREG32(CG_THERMAL_INT) &
5858 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
5860 /* enable CP interrupts on all rings */
5861 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
5862 DRM_DEBUG("si_irq_set: sw int gfx\n");
5863 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
5865 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
5866 DRM_DEBUG("si_irq_set: sw int cp1\n");
5867 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
5869 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
5870 DRM_DEBUG("si_irq_set: sw int cp2\n");
5871 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
5873 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
5874 DRM_DEBUG("si_irq_set: sw int dma\n");
5875 dma_cntl |= TRAP_ENABLE;
5878 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
5879 DRM_DEBUG("si_irq_set: sw int dma1\n");
5880 dma_cntl1 |= TRAP_ENABLE;
5882 if (rdev->irq.crtc_vblank_int[0] ||
5883 atomic_read(&rdev->irq.pflip[0])) {
5884 DRM_DEBUG("si_irq_set: vblank 0\n");
5885 crtc1 |= VBLANK_INT_MASK;
5887 if (rdev->irq.crtc_vblank_int[1] ||
5888 atomic_read(&rdev->irq.pflip[1])) {
5889 DRM_DEBUG("si_irq_set: vblank 1\n");
5890 crtc2 |= VBLANK_INT_MASK;
5892 if (rdev->irq.crtc_vblank_int[2] ||
5893 atomic_read(&rdev->irq.pflip[2])) {
5894 DRM_DEBUG("si_irq_set: vblank 2\n");
5895 crtc3 |= VBLANK_INT_MASK;
5897 if (rdev->irq.crtc_vblank_int[3] ||
5898 atomic_read(&rdev->irq.pflip[3])) {
5899 DRM_DEBUG("si_irq_set: vblank 3\n");
5900 crtc4 |= VBLANK_INT_MASK;
5902 if (rdev->irq.crtc_vblank_int[4] ||
5903 atomic_read(&rdev->irq.pflip[4])) {
5904 DRM_DEBUG("si_irq_set: vblank 4\n");
5905 crtc5 |= VBLANK_INT_MASK;
5907 if (rdev->irq.crtc_vblank_int[5] ||
5908 atomic_read(&rdev->irq.pflip[5])) {
5909 DRM_DEBUG("si_irq_set: vblank 5\n");
5910 crtc6 |= VBLANK_INT_MASK;
5912 if (rdev->irq.hpd[0]) {
5913 DRM_DEBUG("si_irq_set: hpd 1\n");
5914 hpd1 |= DC_HPDx_INT_EN;
5916 if (rdev->irq.hpd[1]) {
5917 DRM_DEBUG("si_irq_set: hpd 2\n");
5918 hpd2 |= DC_HPDx_INT_EN;
5920 if (rdev->irq.hpd[2]) {
5921 DRM_DEBUG("si_irq_set: hpd 3\n");
5922 hpd3 |= DC_HPDx_INT_EN;
5924 if (rdev->irq.hpd[3]) {
5925 DRM_DEBUG("si_irq_set: hpd 4\n");
5926 hpd4 |= DC_HPDx_INT_EN;
5928 if (rdev->irq.hpd[4]) {
5929 DRM_DEBUG("si_irq_set: hpd 5\n");
5930 hpd5 |= DC_HPDx_INT_EN;
5932 if (rdev->irq.hpd[5]) {
5933 DRM_DEBUG("si_irq_set: hpd 6\n");
5934 hpd6 |= DC_HPDx_INT_EN;
5937 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
5938 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
5939 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
5941 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
5942 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
5944 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
5946 if (rdev->irq.dpm_thermal) {
5947 DRM_DEBUG("dpm thermal\n");
5948 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
5951 if (rdev->num_crtc >= 2) {
5952 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
5953 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
5955 if (rdev->num_crtc >= 4) {
5956 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
5957 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
5959 if (rdev->num_crtc >= 6) {
5960 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
5961 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
5964 if (rdev->num_crtc >= 2) {
5965 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
5966 GRPH_PFLIP_INT_MASK);
5967 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
5968 GRPH_PFLIP_INT_MASK);
5970 if (rdev->num_crtc >= 4) {
5971 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
5972 GRPH_PFLIP_INT_MASK);
5973 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
5974 GRPH_PFLIP_INT_MASK);
5976 if (rdev->num_crtc >= 6) {
5977 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
5978 GRPH_PFLIP_INT_MASK);
5979 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
5980 GRPH_PFLIP_INT_MASK);
5983 if (!ASIC_IS_NODCE(rdev)) {
5984 WREG32(DC_HPD1_INT_CONTROL, hpd1);
5985 WREG32(DC_HPD2_INT_CONTROL, hpd2);
5986 WREG32(DC_HPD3_INT_CONTROL, hpd3);
5987 WREG32(DC_HPD4_INT_CONTROL, hpd4);
5988 WREG32(DC_HPD5_INT_CONTROL, hpd5);
5989 WREG32(DC_HPD6_INT_CONTROL, hpd6);
5992 WREG32(CG_THERMAL_INT, thermal_int);
5997 static inline void si_irq_ack(struct radeon_device *rdev)
6001 if (ASIC_IS_NODCE(rdev))
6004 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
6005 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
6006 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
6007 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
6008 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
6009 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
6010 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
6011 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
6012 if (rdev->num_crtc >= 4) {
6013 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
6014 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
6016 if (rdev->num_crtc >= 6) {
6017 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
6018 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
6021 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
6022 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6023 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
6024 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6025 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
6026 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6027 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
6028 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6029 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
6030 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6031 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
6032 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
6034 if (rdev->num_crtc >= 4) {
6035 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
6036 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6037 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
6038 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6039 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
6040 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
6041 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
6042 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
6043 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
6044 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
6045 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
6046 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
6049 if (rdev->num_crtc >= 6) {
6050 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
6051 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6052 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
6053 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6054 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
6055 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
6056 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
6057 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
6058 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
6059 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
6060 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
6061 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
6064 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6065 tmp = RREG32(DC_HPD1_INT_CONTROL);
6066 tmp |= DC_HPDx_INT_ACK;
6067 WREG32(DC_HPD1_INT_CONTROL, tmp);
6069 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6070 tmp = RREG32(DC_HPD2_INT_CONTROL);
6071 tmp |= DC_HPDx_INT_ACK;
6072 WREG32(DC_HPD2_INT_CONTROL, tmp);
6074 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6075 tmp = RREG32(DC_HPD3_INT_CONTROL);
6076 tmp |= DC_HPDx_INT_ACK;
6077 WREG32(DC_HPD3_INT_CONTROL, tmp);
6079 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6080 tmp = RREG32(DC_HPD4_INT_CONTROL);
6081 tmp |= DC_HPDx_INT_ACK;
6082 WREG32(DC_HPD4_INT_CONTROL, tmp);
6084 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6085 tmp = RREG32(DC_HPD5_INT_CONTROL);
6086 tmp |= DC_HPDx_INT_ACK;
6087 WREG32(DC_HPD5_INT_CONTROL, tmp);
6089 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6090 tmp = RREG32(DC_HPD5_INT_CONTROL);
6091 tmp |= DC_HPDx_INT_ACK;
6092 WREG32(DC_HPD6_INT_CONTROL, tmp);
6096 static void si_irq_disable(struct radeon_device *rdev)
6098 si_disable_interrupts(rdev);
6099 /* Wait and acknowledge irq */
6102 si_disable_interrupt_state(rdev);
6105 static void si_irq_suspend(struct radeon_device *rdev)
6107 si_irq_disable(rdev);
6111 static void si_irq_fini(struct radeon_device *rdev)
6113 si_irq_suspend(rdev);
6114 r600_ih_ring_fini(rdev);
6117 static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
6121 if (rdev->wb.enabled)
6122 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6124 wptr = RREG32(IH_RB_WPTR);
6126 if (wptr & RB_OVERFLOW) {
6127 wptr &= ~RB_OVERFLOW;
6128 /* When a ring buffer overflow happen start parsing interrupt
6129 * from the last not overwritten vector (wptr + 16). Hopefully
6130 * this should allow us to catchup.
6132 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
6133 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
6134 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6135 tmp = RREG32(IH_RB_CNTL);
6136 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6137 WREG32(IH_RB_CNTL, tmp);
6139 return (wptr & rdev->ih.ptr_mask);
6143 * Each IV ring entry is 128 bits:
6144 * [7:0] - interrupt source id
6146 * [59:32] - interrupt source data
6147 * [63:60] - reserved
6150 * [127:80] - reserved
6152 irqreturn_t si_irq_process(struct radeon_device *rdev)
6156 u32 src_id, src_data, ring_id;
6158 bool queue_hotplug = false;
6159 bool queue_thermal = false;
6162 if (!rdev->ih.enabled || rdev->shutdown)
6165 wptr = si_get_ih_wptr(rdev);
6168 /* is somebody else already processing irqs? */
6169 if (atomic_xchg(&rdev->ih.lock, 1))
6172 rptr = rdev->ih.rptr;
6173 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6175 /* Order reading of wptr vs. reading of IH ring data */
6178 /* display interrupts */
6181 while (rptr != wptr) {
6182 /* wptr/rptr are in bytes! */
6183 ring_index = rptr / 4;
6184 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6185 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6186 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6189 case 1: /* D1 vblank/vline */
6191 case 0: /* D1 vblank */
6192 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6193 if (rdev->irq.crtc_vblank_int[0]) {
6194 drm_handle_vblank(rdev->ddev, 0);
6195 rdev->pm.vblank_sync = true;
6196 wake_up(&rdev->irq.vblank_queue);
6198 if (atomic_read(&rdev->irq.pflip[0]))
6199 radeon_crtc_handle_vblank(rdev, 0);
6200 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
6201 DRM_DEBUG("IH: D1 vblank\n");
6204 case 1: /* D1 vline */
6205 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
6206 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
6207 DRM_DEBUG("IH: D1 vline\n");
6211 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6215 case 2: /* D2 vblank/vline */
6217 case 0: /* D2 vblank */
6218 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6219 if (rdev->irq.crtc_vblank_int[1]) {
6220 drm_handle_vblank(rdev->ddev, 1);
6221 rdev->pm.vblank_sync = true;
6222 wake_up(&rdev->irq.vblank_queue);
6224 if (atomic_read(&rdev->irq.pflip[1]))
6225 radeon_crtc_handle_vblank(rdev, 1);
6226 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
6227 DRM_DEBUG("IH: D2 vblank\n");
6230 case 1: /* D2 vline */
6231 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
6232 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
6233 DRM_DEBUG("IH: D2 vline\n");
6237 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6241 case 3: /* D3 vblank/vline */
6243 case 0: /* D3 vblank */
6244 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
6245 if (rdev->irq.crtc_vblank_int[2]) {
6246 drm_handle_vblank(rdev->ddev, 2);
6247 rdev->pm.vblank_sync = true;
6248 wake_up(&rdev->irq.vblank_queue);
6250 if (atomic_read(&rdev->irq.pflip[2]))
6251 radeon_crtc_handle_vblank(rdev, 2);
6252 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
6253 DRM_DEBUG("IH: D3 vblank\n");
6256 case 1: /* D3 vline */
6257 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
6258 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
6259 DRM_DEBUG("IH: D3 vline\n");
6263 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6267 case 4: /* D4 vblank/vline */
6269 case 0: /* D4 vblank */
6270 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
6271 if (rdev->irq.crtc_vblank_int[3]) {
6272 drm_handle_vblank(rdev->ddev, 3);
6273 rdev->pm.vblank_sync = true;
6274 wake_up(&rdev->irq.vblank_queue);
6276 if (atomic_read(&rdev->irq.pflip[3]))
6277 radeon_crtc_handle_vblank(rdev, 3);
6278 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
6279 DRM_DEBUG("IH: D4 vblank\n");
6282 case 1: /* D4 vline */
6283 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
6284 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
6285 DRM_DEBUG("IH: D4 vline\n");
6289 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6293 case 5: /* D5 vblank/vline */
6295 case 0: /* D5 vblank */
6296 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
6297 if (rdev->irq.crtc_vblank_int[4]) {
6298 drm_handle_vblank(rdev->ddev, 4);
6299 rdev->pm.vblank_sync = true;
6300 wake_up(&rdev->irq.vblank_queue);
6302 if (atomic_read(&rdev->irq.pflip[4]))
6303 radeon_crtc_handle_vblank(rdev, 4);
6304 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6305 DRM_DEBUG("IH: D5 vblank\n");
6308 case 1: /* D5 vline */
6309 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
6310 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6311 DRM_DEBUG("IH: D5 vline\n");
6315 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6319 case 6: /* D6 vblank/vline */
6321 case 0: /* D6 vblank */
6322 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
6323 if (rdev->irq.crtc_vblank_int[5]) {
6324 drm_handle_vblank(rdev->ddev, 5);
6325 rdev->pm.vblank_sync = true;
6326 wake_up(&rdev->irq.vblank_queue);
6328 if (atomic_read(&rdev->irq.pflip[5]))
6329 radeon_crtc_handle_vblank(rdev, 5);
6330 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6331 DRM_DEBUG("IH: D6 vblank\n");
6334 case 1: /* D6 vline */
6335 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
6336 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6337 DRM_DEBUG("IH: D6 vline\n");
6341 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6345 case 8: /* D1 page flip */
6346 case 10: /* D2 page flip */
6347 case 12: /* D3 page flip */
6348 case 14: /* D4 page flip */
6349 case 16: /* D5 page flip */
6350 case 18: /* D6 page flip */
6351 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
6352 if (radeon_use_pflipirq > 0)
6353 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
6355 case 42: /* HPD hotplug */
6358 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6359 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
6360 queue_hotplug = true;
6361 DRM_DEBUG("IH: HPD1\n");
6365 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6366 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6367 queue_hotplug = true;
6368 DRM_DEBUG("IH: HPD2\n");
6372 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6373 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6374 queue_hotplug = true;
6375 DRM_DEBUG("IH: HPD3\n");
6379 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6380 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6381 queue_hotplug = true;
6382 DRM_DEBUG("IH: HPD4\n");
6386 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6387 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6388 queue_hotplug = true;
6389 DRM_DEBUG("IH: HPD5\n");
6393 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6394 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6395 queue_hotplug = true;
6396 DRM_DEBUG("IH: HPD6\n");
6400 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6405 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
6406 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
6410 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6411 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
6412 /* reset addr and status */
6413 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6414 if (addr == 0x0 && status == 0x0)
6416 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6417 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
6419 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
6421 si_vm_decode_fault(rdev, status, addr);
6423 case 176: /* RINGID0 CP_INT */
6424 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6426 case 177: /* RINGID1 CP_INT */
6427 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6429 case 178: /* RINGID2 CP_INT */
6430 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6432 case 181: /* CP EOP event */
6433 DRM_DEBUG("IH: CP EOP\n");
6436 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6439 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6442 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6446 case 224: /* DMA trap event */
6447 DRM_DEBUG("IH: DMA trap\n");
6448 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6450 case 230: /* thermal low to high */
6451 DRM_DEBUG("IH: thermal low to high\n");
6452 rdev->pm.dpm.thermal.high_to_low = false;
6453 queue_thermal = true;
6455 case 231: /* thermal high to low */
6456 DRM_DEBUG("IH: thermal high to low\n");
6457 rdev->pm.dpm.thermal.high_to_low = true;
6458 queue_thermal = true;
6460 case 233: /* GUI IDLE */
6461 DRM_DEBUG("IH: GUI idle\n");
6463 case 244: /* DMA trap event */
6464 DRM_DEBUG("IH: DMA1 trap\n");
6465 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6468 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6472 /* wptr/rptr are in bytes! */
6474 rptr &= rdev->ih.ptr_mask;
6475 WREG32(IH_RB_RPTR, rptr);
6478 taskqueue_enqueue(rdev->tq, &rdev->hotplug_work);
6479 if (queue_thermal && rdev->pm.dpm_enabled)
6480 taskqueue_enqueue(rdev->tq, &rdev->pm.dpm.thermal.work);
6481 rdev->ih.rptr = rptr;
6482 atomic_set(&rdev->ih.lock, 0);
6484 /* make sure wptr hasn't changed while processing */
6485 wptr = si_get_ih_wptr(rdev);
6493 * startup/shutdown callbacks
6495 static int si_startup(struct radeon_device *rdev)
6497 struct radeon_ring *ring;
6500 /* enable pcie gen2/3 link */
6501 si_pcie_gen3_enable(rdev);
6503 si_program_aspm(rdev);
6505 /* scratch needs to be initialized before MC */
6506 r = r600_vram_scratch_init(rdev);
6510 si_mc_program(rdev);
6512 if (!rdev->pm.dpm_enabled) {
6513 r = si_mc_load_microcode(rdev);
6515 DRM_ERROR("Failed to load MC firmware!\n");
6520 r = si_pcie_gart_enable(rdev);
6525 /* allocate rlc buffers */
6526 if (rdev->family == CHIP_VERDE) {
6527 rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6528 rdev->rlc.reg_list_size =
6529 (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
6531 rdev->rlc.cs_data = si_cs_data;
6532 r = sumo_rlc_init(rdev);
6534 DRM_ERROR("Failed to init rlc BOs!\n");
6538 /* allocate wb buffer */
6539 r = radeon_wb_init(rdev);
6543 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6545 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6549 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6551 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6555 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6557 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6561 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6563 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6567 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6569 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6573 if (rdev->has_uvd) {
6574 r = uvd_v2_2_resume(rdev);
6576 r = radeon_fence_driver_start_ring(rdev,
6577 R600_RING_TYPE_UVD_INDEX);
6579 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
6582 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
6586 if (!rdev->irq.installed) {
6587 r = radeon_irq_kms_init(rdev);
6592 r = si_irq_init(rdev);
6594 DRM_ERROR("radeon: IH init failed (%d).\n", r);
6595 radeon_irq_kms_fini(rdev);
6600 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6601 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
6606 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6607 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
6612 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6613 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
6618 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6619 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
6620 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6624 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6625 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
6626 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6630 r = si_cp_load_microcode(rdev);
6633 r = si_cp_resume(rdev);
6637 r = cayman_dma_resume(rdev);
6641 if (rdev->has_uvd) {
6642 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6643 if (ring->ring_size) {
6644 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
6647 r = uvd_v1_0_init(rdev);
6649 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
6653 r = radeon_ib_pool_init(rdev);
6655 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
6659 r = radeon_vm_manager_init(rdev);
6661 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
6665 r = dce6_audio_init(rdev);
6672 int si_resume(struct radeon_device *rdev)
6676 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
6677 * posting will perform necessary task to bring back GPU into good
6681 atom_asic_init(rdev->mode_info.atom_context);
6683 /* init golden registers */
6684 si_init_golden_registers(rdev);
6686 if (rdev->pm.pm_method == PM_METHOD_DPM)
6687 radeon_pm_resume(rdev);
6689 rdev->accel_working = true;
6690 r = si_startup(rdev);
6692 DRM_ERROR("si startup failed on resume\n");
6693 rdev->accel_working = false;
6701 int si_suspend(struct radeon_device *rdev)
6703 radeon_pm_suspend(rdev);
6704 dce6_audio_fini(rdev);
6705 radeon_vm_manager_fini(rdev);
6706 si_cp_enable(rdev, false);
6707 cayman_dma_stop(rdev);
6708 if (rdev->has_uvd) {
6709 uvd_v1_0_fini(rdev);
6710 radeon_uvd_suspend(rdev);
6714 si_irq_suspend(rdev);
6715 radeon_wb_disable(rdev);
6716 si_pcie_gart_disable(rdev);
6720 /* Plan is to move initialization in that function and use
6721 * helper function so that radeon_device_init pretty much
6722 * do nothing more than calling asic specific function. This
6723 * should also allow to remove a bunch of callback function
6726 int si_init(struct radeon_device *rdev)
6728 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6732 if (!radeon_get_bios(rdev)) {
6733 if (ASIC_IS_AVIVO(rdev))
6736 /* Must be an ATOMBIOS */
6737 if (!rdev->is_atom_bios) {
6738 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
6741 r = radeon_atombios_init(rdev);
6745 /* Post card if necessary */
6746 if (!radeon_card_posted(rdev)) {
6748 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
6751 DRM_INFO("GPU not posted. posting now...\n");
6752 atom_asic_init(rdev->mode_info.atom_context);
6754 /* init golden registers */
6755 si_init_golden_registers(rdev);
6756 /* Initialize scratch registers */
6757 si_scratch_init(rdev);
6758 /* Initialize surface registers */
6759 radeon_surface_init(rdev);
6760 /* Initialize clocks */
6761 radeon_get_clock_info(rdev->ddev);
6764 r = radeon_fence_driver_init(rdev);
6768 /* initialize memory controller */
6769 r = si_mc_init(rdev);
6772 /* Memory manager */
6773 r = radeon_bo_init(rdev);
6777 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6778 !rdev->rlc_fw || !rdev->mc_fw) {
6779 r = si_init_microcode(rdev);
6781 DRM_ERROR("Failed to load firmware!\n");
6786 /* Initialize power management */
6787 radeon_pm_init(rdev);
6789 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6790 ring->ring_obj = NULL;
6791 r600_ring_init(rdev, ring, 1024 * 1024);
6793 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6794 ring->ring_obj = NULL;
6795 r600_ring_init(rdev, ring, 1024 * 1024);
6797 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6798 ring->ring_obj = NULL;
6799 r600_ring_init(rdev, ring, 1024 * 1024);
6801 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6802 ring->ring_obj = NULL;
6803 r600_ring_init(rdev, ring, 64 * 1024);
6805 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6806 ring->ring_obj = NULL;
6807 r600_ring_init(rdev, ring, 64 * 1024);
6809 if (rdev->has_uvd) {
6810 r = radeon_uvd_init(rdev);
6812 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6813 ring->ring_obj = NULL;
6814 r600_ring_init(rdev, ring, 4096);
6818 rdev->ih.ring_obj = NULL;
6819 r600_ih_ring_init(rdev, 64 * 1024);
6821 r = r600_pcie_gart_init(rdev);
6825 rdev->accel_working = true;
6826 r = si_startup(rdev);
6828 dev_err(rdev->dev, "disabling GPU acceleration\n");
6830 cayman_dma_fini(rdev);
6832 sumo_rlc_fini(rdev);
6833 radeon_wb_fini(rdev);
6834 radeon_ib_pool_fini(rdev);
6835 radeon_vm_manager_fini(rdev);
6836 radeon_irq_kms_fini(rdev);
6837 si_pcie_gart_fini(rdev);
6838 rdev->accel_working = false;
6841 /* Don't start up if the MC ucode is missing.
6842 * The default clocks and voltages before the MC ucode
6843 * is loaded are not suffient for advanced operations.
6846 DRM_ERROR("radeon: MC ucode required for NI+.\n");
6853 void si_fini(struct radeon_device *rdev)
6855 radeon_pm_fini(rdev);
6857 cayman_dma_fini(rdev);
6861 sumo_rlc_fini(rdev);
6862 radeon_wb_fini(rdev);
6863 radeon_vm_manager_fini(rdev);
6864 radeon_ib_pool_fini(rdev);
6865 radeon_irq_kms_fini(rdev);
6866 if (rdev->has_uvd) {
6867 uvd_v1_0_fini(rdev);
6868 radeon_uvd_fini(rdev);
6870 si_pcie_gart_fini(rdev);
6871 r600_vram_scratch_fini(rdev);
6872 radeon_gem_fini(rdev);
6873 radeon_fence_driver_fini(rdev);
6874 radeon_bo_fini(rdev);
6875 radeon_atombios_fini(rdev);
6876 si_fini_microcode(rdev);
6882 * si_get_gpu_clock_counter - return GPU clock counter snapshot
6884 * @rdev: radeon_device pointer
6886 * Fetches a GPU clock counter snapshot (SI).
6887 * Returns the 64 bit clock counter snapshot.
6889 uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
6893 spin_lock(&rdev->gpu_clock_mutex);
6894 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
6895 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
6896 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
6897 spin_unlock(&rdev->gpu_clock_mutex);
6901 int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
6903 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
6906 /* bypass vclk and dclk with bclk */
6907 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6908 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
6909 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6911 /* put PLL in bypass mode */
6912 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
6914 if (!vclk || !dclk) {
6915 /* keep the Bypass mode, put PLL to sleep */
6916 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6920 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
6921 16384, 0x03FFFFFF, 0, 128, 5,
6922 &fb_div, &vclk_div, &dclk_div);
6926 /* set RESET_ANTI_MUX to 0 */
6927 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
6929 /* set VCO_MODE to 1 */
6930 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
6932 /* toggle UPLL_SLEEP to 1 then back to 0 */
6933 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6934 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
6936 /* deassert UPLL_RESET */
6937 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6941 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
6945 /* assert UPLL_RESET again */
6946 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
6948 /* disable spread spectrum. */
6949 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
6951 /* set feedback divider */
6952 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
6954 /* set ref divider to 0 */
6955 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
6957 if (fb_div < 307200)
6958 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
6960 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
6962 /* set PDIV_A and PDIV_B */
6963 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6964 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
6965 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
6967 /* give the PLL some time to settle */
6970 /* deassert PLL_RESET */
6971 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6975 /* switch from bypass mode to normal mode */
6976 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
6978 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
6982 /* switch VCLK and DCLK selection */
6983 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6984 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
6985 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6992 static void si_pcie_gen3_enable(struct radeon_device *rdev)
6994 struct pci_dev *root = rdev->pdev->bus->self;
6995 int bridge_pos, gpu_pos;
6996 u32 speed_cntl, mask, current_data_rate;
7000 if (radeon_pcie_gen2 == 0)
7003 if (rdev->flags & RADEON_IS_IGP)
7006 if (!(rdev->flags & RADEON_IS_PCIE))
7009 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
7013 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
7016 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7017 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
7018 LC_CURRENT_DATA_RATE_SHIFT;
7019 if (mask & DRM_PCIE_SPEED_80) {
7020 if (current_data_rate == 2) {
7021 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
7024 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
7025 } else if (mask & DRM_PCIE_SPEED_50) {
7026 if (current_data_rate == 1) {
7027 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
7030 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
7033 bridge_pos = pci_get_pciecap_ptr(root->dev);
7037 gpu_pos = pci_get_pciecap_ptr(rdev->pdev->dev);
7041 if (mask & DRM_PCIE_SPEED_80) {
7042 /* re-try equalization if gen3 is not already enabled */
7043 if (current_data_rate != 2) {
7044 u16 bridge_cfg, gpu_cfg;
7045 u16 bridge_cfg2, gpu_cfg2;
7046 u32 max_lw, current_lw, tmp;
7048 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
7049 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
7051 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
7052 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
7054 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
7055 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
7057 tmp = RREG32_PCIE(PCIE_LC_STATUS1);
7058 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
7059 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
7061 if (current_lw < max_lw) {
7062 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7063 if (tmp & LC_RENEGOTIATION_SUPPORT) {
7064 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
7065 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
7066 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
7067 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
7071 for (i = 0; i < 10; i++) {
7073 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
7074 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
7077 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
7078 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
7080 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
7081 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
7083 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7084 tmp |= LC_SET_QUIESCE;
7085 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7087 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7089 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7094 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
7095 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7096 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
7097 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
7099 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
7100 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7101 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
7102 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
7105 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
7106 tmp16 &= ~((1 << 4) | (7 << 9));
7107 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
7108 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
7110 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7111 tmp16 &= ~((1 << 4) | (7 << 9));
7112 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
7113 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7115 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7116 tmp &= ~LC_SET_QUIESCE;
7117 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7122 /* set the link speed */
7123 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
7124 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
7125 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7127 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7129 if (mask & DRM_PCIE_SPEED_80)
7130 tmp16 |= 3; /* gen3 */
7131 else if (mask & DRM_PCIE_SPEED_50)
7132 tmp16 |= 2; /* gen2 */
7134 tmp16 |= 1; /* gen1 */
7135 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7137 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7138 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
7139 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7141 for (i = 0; i < rdev->usec_timeout; i++) {
7142 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7143 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
7149 static void si_program_aspm(struct radeon_device *rdev)
7152 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
7153 bool disable_clkreq = false;
7155 if (radeon_aspm == 0)
7158 if (!(rdev->flags & RADEON_IS_PCIE))
7161 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7162 data &= ~LC_XMIT_N_FTS_MASK;
7163 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
7165 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
7167 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
7168 data |= LC_GO_TO_RECOVERY;
7170 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
7172 orig = data = RREG32_PCIE(PCIE_P_CNTL);
7173 data |= P_IGNORE_EDB_ERR;
7175 WREG32_PCIE(PCIE_P_CNTL, data);
7177 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7178 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
7179 data |= LC_PMI_TO_L1_DIS;
7181 data |= LC_L0S_INACTIVITY(7);
7184 data |= LC_L1_INACTIVITY(7);
7185 data &= ~LC_PMI_TO_L1_DIS;
7187 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7189 if (!disable_plloff_in_l1) {
7190 bool clk_req_support;
7192 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7193 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7194 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7196 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7198 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7199 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7200 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7202 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7204 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7205 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7206 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7208 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7210 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7211 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7212 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7214 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7216 if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7217 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7218 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7220 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7222 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7223 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7225 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7227 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
7228 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7230 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
7232 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
7233 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7235 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
7237 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7238 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7240 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7242 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7243 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7245 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7247 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7248 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7250 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7252 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7253 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7255 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7257 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7258 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7259 data |= LC_DYN_LANES_PWR_STATE(3);
7261 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7263 orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7264 data &= ~LS2_EXIT_TIME_MASK;
7265 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7266 data |= LS2_EXIT_TIME(5);
7268 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7270 orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7271 data &= ~LS2_EXIT_TIME_MASK;
7272 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7273 data |= LS2_EXIT_TIME(5);
7275 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7277 if (!disable_clkreq) {
7279 struct pci_dev *root = rdev->pdev->bus->self;
7282 clk_req_support = false;
7283 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7284 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7285 clk_req_support = true;
7287 clk_req_support = false;
7290 clk_req_support = false;
7293 if (clk_req_support) {
7294 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7295 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7297 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7299 orig = data = RREG32(THM_CLK_CNTL);
7300 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7301 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7303 WREG32(THM_CLK_CNTL, data);
7305 orig = data = RREG32(MISC_CLK_CNTL);
7306 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7307 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7309 WREG32(MISC_CLK_CNTL, data);
7311 orig = data = RREG32(CG_CLKPIN_CNTL);
7312 data &= ~BCLK_AS_XCLK;
7314 WREG32(CG_CLKPIN_CNTL, data);
7316 orig = data = RREG32(CG_CLKPIN_CNTL_2);
7317 data &= ~FORCE_BIF_REFCLK_EN;
7319 WREG32(CG_CLKPIN_CNTL_2, data);
7321 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7322 data &= ~MPLL_CLKOUT_SEL_MASK;
7323 data |= MPLL_CLKOUT_SEL(4);
7325 WREG32(MPLL_BYPASSCLK_SEL, data);
7327 orig = data = RREG32(SPLL_CNTL_MODE);
7328 data &= ~SPLL_REFCLK_SEL_MASK;
7330 WREG32(SPLL_CNTL_MODE, data);
7335 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7338 orig = data = RREG32_PCIE(PCIE_CNTL2);
7339 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7341 WREG32_PCIE(PCIE_CNTL2, data);
7344 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7345 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7346 data = RREG32_PCIE(PCIE_LC_STATUS1);
7347 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7348 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7349 data &= ~LC_L0S_INACTIVITY_MASK;
7351 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
tuxillo's projects / dragonfly.git / blob