2 * Copyright 2011 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
24 #include <linux/firmware.h>
25 #include <linux/module.h>
28 #include "radeon_asic.h"
29 #include <uapi_drm/radeon_drm.h>
32 #include "si_blit_shaders.h"
33 #include "clearstate_si.h"
34 #include "radeon_ucode.h"
37 MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
38 MODULE_FIRMWARE("radeon/TAHITI_me.bin");
39 MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
40 MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
41 MODULE_FIRMWARE("radeon/TAHITI_mc2.bin");
42 MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
43 MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
44 MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
45 MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
46 MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
47 MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
48 MODULE_FIRMWARE("radeon/PITCAIRN_mc2.bin");
49 MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
50 MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
51 MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
52 MODULE_FIRMWARE("radeon/VERDE_me.bin");
53 MODULE_FIRMWARE("radeon/VERDE_ce.bin");
54 MODULE_FIRMWARE("radeon/VERDE_mc.bin");
55 MODULE_FIRMWARE("radeon/VERDE_mc2.bin");
56 MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
57 MODULE_FIRMWARE("radeon/VERDE_smc.bin");
58 MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
59 MODULE_FIRMWARE("radeon/OLAND_me.bin");
60 MODULE_FIRMWARE("radeon/OLAND_ce.bin");
61 MODULE_FIRMWARE("radeon/OLAND_mc.bin");
62 MODULE_FIRMWARE("radeon/OLAND_mc2.bin");
63 MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
64 MODULE_FIRMWARE("radeon/OLAND_smc.bin");
65 MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
66 MODULE_FIRMWARE("radeon/HAINAN_me.bin");
67 MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
68 MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
69 MODULE_FIRMWARE("radeon/HAINAN_mc2.bin");
70 MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
71 MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
73 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
74 static void si_pcie_gen3_enable(struct radeon_device *rdev);
75 static void si_program_aspm(struct radeon_device *rdev);
76 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
78 static void si_init_pg(struct radeon_device *rdev);
79 static void si_init_cg(struct radeon_device *rdev);
80 static void si_fini_pg(struct radeon_device *rdev);
81 static void si_fini_cg(struct radeon_device *rdev);
82 static void si_rlc_stop(struct radeon_device *rdev);
84 static const u32 verde_rlc_save_restore_register_list[] =
86 (0x8000 << 16) | (0x98f4 >> 2),
88 (0x8040 << 16) | (0x98f4 >> 2),
90 (0x8000 << 16) | (0xe80 >> 2),
92 (0x8040 << 16) | (0xe80 >> 2),
94 (0x8000 << 16) | (0x89bc >> 2),
96 (0x8040 << 16) | (0x89bc >> 2),
98 (0x8000 << 16) | (0x8c1c >> 2),
100 (0x8040 << 16) | (0x8c1c >> 2),
102 (0x9c00 << 16) | (0x98f0 >> 2),
104 (0x9c00 << 16) | (0xe7c >> 2),
106 (0x8000 << 16) | (0x9148 >> 2),
108 (0x8040 << 16) | (0x9148 >> 2),
110 (0x9c00 << 16) | (0x9150 >> 2),
112 (0x9c00 << 16) | (0x897c >> 2),
114 (0x9c00 << 16) | (0x8d8c >> 2),
116 (0x9c00 << 16) | (0xac54 >> 2),
119 (0x9c00 << 16) | (0x98f8 >> 2),
121 (0x9c00 << 16) | (0x9910 >> 2),
123 (0x9c00 << 16) | (0x9914 >> 2),
125 (0x9c00 << 16) | (0x9918 >> 2),
127 (0x9c00 << 16) | (0x991c >> 2),
129 (0x9c00 << 16) | (0x9920 >> 2),
131 (0x9c00 << 16) | (0x9924 >> 2),
133 (0x9c00 << 16) | (0x9928 >> 2),
135 (0x9c00 << 16) | (0x992c >> 2),
137 (0x9c00 << 16) | (0x9930 >> 2),
139 (0x9c00 << 16) | (0x9934 >> 2),
141 (0x9c00 << 16) | (0x9938 >> 2),
143 (0x9c00 << 16) | (0x993c >> 2),
145 (0x9c00 << 16) | (0x9940 >> 2),
147 (0x9c00 << 16) | (0x9944 >> 2),
149 (0x9c00 << 16) | (0x9948 >> 2),
151 (0x9c00 << 16) | (0x994c >> 2),
153 (0x9c00 << 16) | (0x9950 >> 2),
155 (0x9c00 << 16) | (0x9954 >> 2),
157 (0x9c00 << 16) | (0x9958 >> 2),
159 (0x9c00 << 16) | (0x995c >> 2),
161 (0x9c00 << 16) | (0x9960 >> 2),
163 (0x9c00 << 16) | (0x9964 >> 2),
165 (0x9c00 << 16) | (0x9968 >> 2),
167 (0x9c00 << 16) | (0x996c >> 2),
169 (0x9c00 << 16) | (0x9970 >> 2),
171 (0x9c00 << 16) | (0x9974 >> 2),
173 (0x9c00 << 16) | (0x9978 >> 2),
175 (0x9c00 << 16) | (0x997c >> 2),
177 (0x9c00 << 16) | (0x9980 >> 2),
179 (0x9c00 << 16) | (0x9984 >> 2),
181 (0x9c00 << 16) | (0x9988 >> 2),
183 (0x9c00 << 16) | (0x998c >> 2),
185 (0x9c00 << 16) | (0x8c00 >> 2),
187 (0x9c00 << 16) | (0x8c14 >> 2),
189 (0x9c00 << 16) | (0x8c04 >> 2),
191 (0x9c00 << 16) | (0x8c08 >> 2),
193 (0x8000 << 16) | (0x9b7c >> 2),
195 (0x8040 << 16) | (0x9b7c >> 2),
197 (0x8000 << 16) | (0xe84 >> 2),
199 (0x8040 << 16) | (0xe84 >> 2),
201 (0x8000 << 16) | (0x89c0 >> 2),
203 (0x8040 << 16) | (0x89c0 >> 2),
205 (0x8000 << 16) | (0x914c >> 2),
207 (0x8040 << 16) | (0x914c >> 2),
209 (0x8000 << 16) | (0x8c20 >> 2),
211 (0x8040 << 16) | (0x8c20 >> 2),
213 (0x8000 << 16) | (0x9354 >> 2),
215 (0x8040 << 16) | (0x9354 >> 2),
217 (0x9c00 << 16) | (0x9060 >> 2),
219 (0x9c00 << 16) | (0x9364 >> 2),
221 (0x9c00 << 16) | (0x9100 >> 2),
223 (0x9c00 << 16) | (0x913c >> 2),
225 (0x8000 << 16) | (0x90e0 >> 2),
227 (0x8000 << 16) | (0x90e4 >> 2),
229 (0x8000 << 16) | (0x90e8 >> 2),
231 (0x8040 << 16) | (0x90e0 >> 2),
233 (0x8040 << 16) | (0x90e4 >> 2),
235 (0x8040 << 16) | (0x90e8 >> 2),
237 (0x9c00 << 16) | (0x8bcc >> 2),
239 (0x9c00 << 16) | (0x8b24 >> 2),
241 (0x9c00 << 16) | (0x88c4 >> 2),
243 (0x9c00 << 16) | (0x8e50 >> 2),
245 (0x9c00 << 16) | (0x8c0c >> 2),
247 (0x9c00 << 16) | (0x8e58 >> 2),
249 (0x9c00 << 16) | (0x8e5c >> 2),
251 (0x9c00 << 16) | (0x9508 >> 2),
253 (0x9c00 << 16) | (0x950c >> 2),
255 (0x9c00 << 16) | (0x9494 >> 2),
257 (0x9c00 << 16) | (0xac0c >> 2),
259 (0x9c00 << 16) | (0xac10 >> 2),
261 (0x9c00 << 16) | (0xac14 >> 2),
263 (0x9c00 << 16) | (0xae00 >> 2),
265 (0x9c00 << 16) | (0xac08 >> 2),
267 (0x9c00 << 16) | (0x88d4 >> 2),
269 (0x9c00 << 16) | (0x88c8 >> 2),
271 (0x9c00 << 16) | (0x88cc >> 2),
273 (0x9c00 << 16) | (0x89b0 >> 2),
275 (0x9c00 << 16) | (0x8b10 >> 2),
277 (0x9c00 << 16) | (0x8a14 >> 2),
279 (0x9c00 << 16) | (0x9830 >> 2),
281 (0x9c00 << 16) | (0x9834 >> 2),
283 (0x9c00 << 16) | (0x9838 >> 2),
285 (0x9c00 << 16) | (0x9a10 >> 2),
287 (0x8000 << 16) | (0x9870 >> 2),
289 (0x8000 << 16) | (0x9874 >> 2),
291 (0x8001 << 16) | (0x9870 >> 2),
293 (0x8001 << 16) | (0x9874 >> 2),
295 (0x8040 << 16) | (0x9870 >> 2),
297 (0x8040 << 16) | (0x9874 >> 2),
299 (0x8041 << 16) | (0x9870 >> 2),
301 (0x8041 << 16) | (0x9874 >> 2),
306 static const u32 tahiti_golden_rlc_registers[] =
308 0xc424, 0xffffffff, 0x00601005,
309 0xc47c, 0xffffffff, 0x10104040,
310 0xc488, 0xffffffff, 0x0100000a,
311 0xc314, 0xffffffff, 0x00000800,
312 0xc30c, 0xffffffff, 0x800000f4,
313 0xf4a8, 0xffffffff, 0x00000000
316 static const u32 tahiti_golden_registers[] =
318 0x9a10, 0x00010000, 0x00018208,
319 0x9830, 0xffffffff, 0x00000000,
320 0x9834, 0xf00fffff, 0x00000400,
321 0x9838, 0x0002021c, 0x00020200,
322 0xc78, 0x00000080, 0x00000000,
323 0xd030, 0x000300c0, 0x00800040,
324 0xd830, 0x000300c0, 0x00800040,
325 0x5bb0, 0x000000f0, 0x00000070,
326 0x5bc0, 0x00200000, 0x50100000,
327 0x7030, 0x31000311, 0x00000011,
328 0x277c, 0x00000003, 0x000007ff,
329 0x240c, 0x000007ff, 0x00000000,
330 0x8a14, 0xf000001f, 0x00000007,
331 0x8b24, 0xffffffff, 0x00ffffff,
332 0x8b10, 0x0000ff0f, 0x00000000,
333 0x28a4c, 0x07ffffff, 0x4e000000,
334 0x28350, 0x3f3f3fff, 0x2a00126a,
335 0x30, 0x000000ff, 0x0040,
336 0x34, 0x00000040, 0x00004040,
337 0x9100, 0x07ffffff, 0x03000000,
338 0x8e88, 0x01ff1f3f, 0x00000000,
339 0x8e84, 0x01ff1f3f, 0x00000000,
340 0x9060, 0x0000007f, 0x00000020,
341 0x9508, 0x00010000, 0x00010000,
342 0xac14, 0x00000200, 0x000002fb,
343 0xac10, 0xffffffff, 0x0000543b,
344 0xac0c, 0xffffffff, 0xa9210876,
345 0x88d0, 0xffffffff, 0x000fff40,
346 0x88d4, 0x0000001f, 0x00000010,
347 0x1410, 0x20000000, 0x20fffed8,
348 0x15c0, 0x000c0fc0, 0x000c0400
351 static const u32 tahiti_golden_registers2[] =
353 0xc64, 0x00000001, 0x00000001
356 static const u32 pitcairn_golden_rlc_registers[] =
358 0xc424, 0xffffffff, 0x00601004,
359 0xc47c, 0xffffffff, 0x10102020,
360 0xc488, 0xffffffff, 0x01000020,
361 0xc314, 0xffffffff, 0x00000800,
362 0xc30c, 0xffffffff, 0x800000a4
365 static const u32 pitcairn_golden_registers[] =
367 0x9a10, 0x00010000, 0x00018208,
368 0x9830, 0xffffffff, 0x00000000,
369 0x9834, 0xf00fffff, 0x00000400,
370 0x9838, 0x0002021c, 0x00020200,
371 0xc78, 0x00000080, 0x00000000,
372 0xd030, 0x000300c0, 0x00800040,
373 0xd830, 0x000300c0, 0x00800040,
374 0x5bb0, 0x000000f0, 0x00000070,
375 0x5bc0, 0x00200000, 0x50100000,
376 0x7030, 0x31000311, 0x00000011,
377 0x2ae4, 0x00073ffe, 0x000022a2,
378 0x240c, 0x000007ff, 0x00000000,
379 0x8a14, 0xf000001f, 0x00000007,
380 0x8b24, 0xffffffff, 0x00ffffff,
381 0x8b10, 0x0000ff0f, 0x00000000,
382 0x28a4c, 0x07ffffff, 0x4e000000,
383 0x28350, 0x3f3f3fff, 0x2a00126a,
384 0x30, 0x000000ff, 0x0040,
385 0x34, 0x00000040, 0x00004040,
386 0x9100, 0x07ffffff, 0x03000000,
387 0x9060, 0x0000007f, 0x00000020,
388 0x9508, 0x00010000, 0x00010000,
389 0xac14, 0x000003ff, 0x000000f7,
390 0xac10, 0xffffffff, 0x00000000,
391 0xac0c, 0xffffffff, 0x32761054,
392 0x88d4, 0x0000001f, 0x00000010,
393 0x15c0, 0x000c0fc0, 0x000c0400
396 static const u32 verde_golden_rlc_registers[] =
398 0xc424, 0xffffffff, 0x033f1005,
399 0xc47c, 0xffffffff, 0x10808020,
400 0xc488, 0xffffffff, 0x00800008,
401 0xc314, 0xffffffff, 0x00001000,
402 0xc30c, 0xffffffff, 0x80010014
405 static const u32 verde_golden_registers[] =
407 0x9a10, 0x00010000, 0x00018208,
408 0x9830, 0xffffffff, 0x00000000,
409 0x9834, 0xf00fffff, 0x00000400,
410 0x9838, 0x0002021c, 0x00020200,
411 0xc78, 0x00000080, 0x00000000,
412 0xd030, 0x000300c0, 0x00800040,
413 0xd030, 0x000300c0, 0x00800040,
414 0xd830, 0x000300c0, 0x00800040,
415 0xd830, 0x000300c0, 0x00800040,
416 0x5bb0, 0x000000f0, 0x00000070,
417 0x5bc0, 0x00200000, 0x50100000,
418 0x7030, 0x31000311, 0x00000011,
419 0x2ae4, 0x00073ffe, 0x000022a2,
420 0x2ae4, 0x00073ffe, 0x000022a2,
421 0x2ae4, 0x00073ffe, 0x000022a2,
422 0x240c, 0x000007ff, 0x00000000,
423 0x240c, 0x000007ff, 0x00000000,
424 0x240c, 0x000007ff, 0x00000000,
425 0x8a14, 0xf000001f, 0x00000007,
426 0x8a14, 0xf000001f, 0x00000007,
427 0x8a14, 0xf000001f, 0x00000007,
428 0x8b24, 0xffffffff, 0x00ffffff,
429 0x8b10, 0x0000ff0f, 0x00000000,
430 0x28a4c, 0x07ffffff, 0x4e000000,
431 0x28350, 0x3f3f3fff, 0x0000124a,
432 0x28350, 0x3f3f3fff, 0x0000124a,
433 0x28350, 0x3f3f3fff, 0x0000124a,
434 0x30, 0x000000ff, 0x0040,
435 0x34, 0x00000040, 0x00004040,
436 0x9100, 0x07ffffff, 0x03000000,
437 0x9100, 0x07ffffff, 0x03000000,
438 0x8e88, 0x01ff1f3f, 0x00000000,
439 0x8e88, 0x01ff1f3f, 0x00000000,
440 0x8e88, 0x01ff1f3f, 0x00000000,
441 0x8e84, 0x01ff1f3f, 0x00000000,
442 0x8e84, 0x01ff1f3f, 0x00000000,
443 0x8e84, 0x01ff1f3f, 0x00000000,
444 0x9060, 0x0000007f, 0x00000020,
445 0x9508, 0x00010000, 0x00010000,
446 0xac14, 0x000003ff, 0x00000003,
447 0xac14, 0x000003ff, 0x00000003,
448 0xac14, 0x000003ff, 0x00000003,
449 0xac10, 0xffffffff, 0x00000000,
450 0xac10, 0xffffffff, 0x00000000,
451 0xac10, 0xffffffff, 0x00000000,
452 0xac0c, 0xffffffff, 0x00001032,
453 0xac0c, 0xffffffff, 0x00001032,
454 0xac0c, 0xffffffff, 0x00001032,
455 0x88d4, 0x0000001f, 0x00000010,
456 0x88d4, 0x0000001f, 0x00000010,
457 0x88d4, 0x0000001f, 0x00000010,
458 0x15c0, 0x000c0fc0, 0x000c0400
461 static const u32 oland_golden_rlc_registers[] =
463 0xc424, 0xffffffff, 0x00601005,
464 0xc47c, 0xffffffff, 0x10104040,
465 0xc488, 0xffffffff, 0x0100000a,
466 0xc314, 0xffffffff, 0x00000800,
467 0xc30c, 0xffffffff, 0x800000f4
470 static const u32 oland_golden_registers[] =
472 0x9a10, 0x00010000, 0x00018208,
473 0x9830, 0xffffffff, 0x00000000,
474 0x9834, 0xf00fffff, 0x00000400,
475 0x9838, 0x0002021c, 0x00020200,
476 0xc78, 0x00000080, 0x00000000,
477 0xd030, 0x000300c0, 0x00800040,
478 0xd830, 0x000300c0, 0x00800040,
479 0x5bb0, 0x000000f0, 0x00000070,
480 0x5bc0, 0x00200000, 0x50100000,
481 0x7030, 0x31000311, 0x00000011,
482 0x2ae4, 0x00073ffe, 0x000022a2,
483 0x240c, 0x000007ff, 0x00000000,
484 0x8a14, 0xf000001f, 0x00000007,
485 0x8b24, 0xffffffff, 0x00ffffff,
486 0x8b10, 0x0000ff0f, 0x00000000,
487 0x28a4c, 0x07ffffff, 0x4e000000,
488 0x28350, 0x3f3f3fff, 0x00000082,
489 0x30, 0x000000ff, 0x0040,
490 0x34, 0x00000040, 0x00004040,
491 0x9100, 0x07ffffff, 0x03000000,
492 0x9060, 0x0000007f, 0x00000020,
493 0x9508, 0x00010000, 0x00010000,
494 0xac14, 0x000003ff, 0x000000f3,
495 0xac10, 0xffffffff, 0x00000000,
496 0xac0c, 0xffffffff, 0x00003210,
497 0x88d4, 0x0000001f, 0x00000010,
498 0x15c0, 0x000c0fc0, 0x000c0400
501 static const u32 hainan_golden_registers[] =
503 0x9a10, 0x00010000, 0x00018208,
504 0x9830, 0xffffffff, 0x00000000,
505 0x9834, 0xf00fffff, 0x00000400,
506 0x9838, 0x0002021c, 0x00020200,
507 0xd0c0, 0xff000fff, 0x00000100,
508 0xd030, 0x000300c0, 0x00800040,
509 0xd8c0, 0xff000fff, 0x00000100,
510 0xd830, 0x000300c0, 0x00800040,
511 0x2ae4, 0x00073ffe, 0x000022a2,
512 0x240c, 0x000007ff, 0x00000000,
513 0x8a14, 0xf000001f, 0x00000007,
514 0x8b24, 0xffffffff, 0x00ffffff,
515 0x8b10, 0x0000ff0f, 0x00000000,
516 0x28a4c, 0x07ffffff, 0x4e000000,
517 0x28350, 0x3f3f3fff, 0x00000000,
518 0x30, 0x000000ff, 0x0040,
519 0x34, 0x00000040, 0x00004040,
520 0x9100, 0x03e00000, 0x03600000,
521 0x9060, 0x0000007f, 0x00000020,
522 0x9508, 0x00010000, 0x00010000,
523 0xac14, 0x000003ff, 0x000000f1,
524 0xac10, 0xffffffff, 0x00000000,
525 0xac0c, 0xffffffff, 0x00003210,
526 0x88d4, 0x0000001f, 0x00000010,
527 0x15c0, 0x000c0fc0, 0x000c0400
530 static const u32 hainan_golden_registers2[] =
532 0x98f8, 0xffffffff, 0x02010001
535 static const u32 tahiti_mgcg_cgcg_init[] =
537 0xc400, 0xffffffff, 0xfffffffc,
538 0x802c, 0xffffffff, 0xe0000000,
539 0x9a60, 0xffffffff, 0x00000100,
540 0x92a4, 0xffffffff, 0x00000100,
541 0xc164, 0xffffffff, 0x00000100,
542 0x9774, 0xffffffff, 0x00000100,
543 0x8984, 0xffffffff, 0x06000100,
544 0x8a18, 0xffffffff, 0x00000100,
545 0x92a0, 0xffffffff, 0x00000100,
546 0xc380, 0xffffffff, 0x00000100,
547 0x8b28, 0xffffffff, 0x00000100,
548 0x9144, 0xffffffff, 0x00000100,
549 0x8d88, 0xffffffff, 0x00000100,
550 0x8d8c, 0xffffffff, 0x00000100,
551 0x9030, 0xffffffff, 0x00000100,
552 0x9034, 0xffffffff, 0x00000100,
553 0x9038, 0xffffffff, 0x00000100,
554 0x903c, 0xffffffff, 0x00000100,
555 0xad80, 0xffffffff, 0x00000100,
556 0xac54, 0xffffffff, 0x00000100,
557 0x897c, 0xffffffff, 0x06000100,
558 0x9868, 0xffffffff, 0x00000100,
559 0x9510, 0xffffffff, 0x00000100,
560 0xaf04, 0xffffffff, 0x00000100,
561 0xae04, 0xffffffff, 0x00000100,
562 0x949c, 0xffffffff, 0x00000100,
563 0x802c, 0xffffffff, 0xe0000000,
564 0x9160, 0xffffffff, 0x00010000,
565 0x9164, 0xffffffff, 0x00030002,
566 0x9168, 0xffffffff, 0x00040007,
567 0x916c, 0xffffffff, 0x00060005,
568 0x9170, 0xffffffff, 0x00090008,
569 0x9174, 0xffffffff, 0x00020001,
570 0x9178, 0xffffffff, 0x00040003,
571 0x917c, 0xffffffff, 0x00000007,
572 0x9180, 0xffffffff, 0x00060005,
573 0x9184, 0xffffffff, 0x00090008,
574 0x9188, 0xffffffff, 0x00030002,
575 0x918c, 0xffffffff, 0x00050004,
576 0x9190, 0xffffffff, 0x00000008,
577 0x9194, 0xffffffff, 0x00070006,
578 0x9198, 0xffffffff, 0x000a0009,
579 0x919c, 0xffffffff, 0x00040003,
580 0x91a0, 0xffffffff, 0x00060005,
581 0x91a4, 0xffffffff, 0x00000009,
582 0x91a8, 0xffffffff, 0x00080007,
583 0x91ac, 0xffffffff, 0x000b000a,
584 0x91b0, 0xffffffff, 0x00050004,
585 0x91b4, 0xffffffff, 0x00070006,
586 0x91b8, 0xffffffff, 0x0008000b,
587 0x91bc, 0xffffffff, 0x000a0009,
588 0x91c0, 0xffffffff, 0x000d000c,
589 0x91c4, 0xffffffff, 0x00060005,
590 0x91c8, 0xffffffff, 0x00080007,
591 0x91cc, 0xffffffff, 0x0000000b,
592 0x91d0, 0xffffffff, 0x000a0009,
593 0x91d4, 0xffffffff, 0x000d000c,
594 0x91d8, 0xffffffff, 0x00070006,
595 0x91dc, 0xffffffff, 0x00090008,
596 0x91e0, 0xffffffff, 0x0000000c,
597 0x91e4, 0xffffffff, 0x000b000a,
598 0x91e8, 0xffffffff, 0x000e000d,
599 0x91ec, 0xffffffff, 0x00080007,
600 0x91f0, 0xffffffff, 0x000a0009,
601 0x91f4, 0xffffffff, 0x0000000d,
602 0x91f8, 0xffffffff, 0x000c000b,
603 0x91fc, 0xffffffff, 0x000f000e,
604 0x9200, 0xffffffff, 0x00090008,
605 0x9204, 0xffffffff, 0x000b000a,
606 0x9208, 0xffffffff, 0x000c000f,
607 0x920c, 0xffffffff, 0x000e000d,
608 0x9210, 0xffffffff, 0x00110010,
609 0x9214, 0xffffffff, 0x000a0009,
610 0x9218, 0xffffffff, 0x000c000b,
611 0x921c, 0xffffffff, 0x0000000f,
612 0x9220, 0xffffffff, 0x000e000d,
613 0x9224, 0xffffffff, 0x00110010,
614 0x9228, 0xffffffff, 0x000b000a,
615 0x922c, 0xffffffff, 0x000d000c,
616 0x9230, 0xffffffff, 0x00000010,
617 0x9234, 0xffffffff, 0x000f000e,
618 0x9238, 0xffffffff, 0x00120011,
619 0x923c, 0xffffffff, 0x000c000b,
620 0x9240, 0xffffffff, 0x000e000d,
621 0x9244, 0xffffffff, 0x00000011,
622 0x9248, 0xffffffff, 0x0010000f,
623 0x924c, 0xffffffff, 0x00130012,
624 0x9250, 0xffffffff, 0x000d000c,
625 0x9254, 0xffffffff, 0x000f000e,
626 0x9258, 0xffffffff, 0x00100013,
627 0x925c, 0xffffffff, 0x00120011,
628 0x9260, 0xffffffff, 0x00150014,
629 0x9264, 0xffffffff, 0x000e000d,
630 0x9268, 0xffffffff, 0x0010000f,
631 0x926c, 0xffffffff, 0x00000013,
632 0x9270, 0xffffffff, 0x00120011,
633 0x9274, 0xffffffff, 0x00150014,
634 0x9278, 0xffffffff, 0x000f000e,
635 0x927c, 0xffffffff, 0x00110010,
636 0x9280, 0xffffffff, 0x00000014,
637 0x9284, 0xffffffff, 0x00130012,
638 0x9288, 0xffffffff, 0x00160015,
639 0x928c, 0xffffffff, 0x0010000f,
640 0x9290, 0xffffffff, 0x00120011,
641 0x9294, 0xffffffff, 0x00000015,
642 0x9298, 0xffffffff, 0x00140013,
643 0x929c, 0xffffffff, 0x00170016,
644 0x9150, 0xffffffff, 0x96940200,
645 0x8708, 0xffffffff, 0x00900100,
646 0xc478, 0xffffffff, 0x00000080,
647 0xc404, 0xffffffff, 0x0020003f,
648 0x30, 0xffffffff, 0x0000001c,
649 0x34, 0x000f0000, 0x000f0000,
650 0x160c, 0xffffffff, 0x00000100,
651 0x1024, 0xffffffff, 0x00000100,
652 0x102c, 0x00000101, 0x00000000,
653 0x20a8, 0xffffffff, 0x00000104,
654 0x264c, 0x000c0000, 0x000c0000,
655 0x2648, 0x000c0000, 0x000c0000,
656 0x55e4, 0xff000fff, 0x00000100,
657 0x55e8, 0x00000001, 0x00000001,
658 0x2f50, 0x00000001, 0x00000001,
659 0x30cc, 0xc0000fff, 0x00000104,
660 0xc1e4, 0x00000001, 0x00000001,
661 0xd0c0, 0xfffffff0, 0x00000100,
662 0xd8c0, 0xfffffff0, 0x00000100
665 static const u32 pitcairn_mgcg_cgcg_init[] =
667 0xc400, 0xffffffff, 0xfffffffc,
668 0x802c, 0xffffffff, 0xe0000000,
669 0x9a60, 0xffffffff, 0x00000100,
670 0x92a4, 0xffffffff, 0x00000100,
671 0xc164, 0xffffffff, 0x00000100,
672 0x9774, 0xffffffff, 0x00000100,
673 0x8984, 0xffffffff, 0x06000100,
674 0x8a18, 0xffffffff, 0x00000100,
675 0x92a0, 0xffffffff, 0x00000100,
676 0xc380, 0xffffffff, 0x00000100,
677 0x8b28, 0xffffffff, 0x00000100,
678 0x9144, 0xffffffff, 0x00000100,
679 0x8d88, 0xffffffff, 0x00000100,
680 0x8d8c, 0xffffffff, 0x00000100,
681 0x9030, 0xffffffff, 0x00000100,
682 0x9034, 0xffffffff, 0x00000100,
683 0x9038, 0xffffffff, 0x00000100,
684 0x903c, 0xffffffff, 0x00000100,
685 0xad80, 0xffffffff, 0x00000100,
686 0xac54, 0xffffffff, 0x00000100,
687 0x897c, 0xffffffff, 0x06000100,
688 0x9868, 0xffffffff, 0x00000100,
689 0x9510, 0xffffffff, 0x00000100,
690 0xaf04, 0xffffffff, 0x00000100,
691 0xae04, 0xffffffff, 0x00000100,
692 0x949c, 0xffffffff, 0x00000100,
693 0x802c, 0xffffffff, 0xe0000000,
694 0x9160, 0xffffffff, 0x00010000,
695 0x9164, 0xffffffff, 0x00030002,
696 0x9168, 0xffffffff, 0x00040007,
697 0x916c, 0xffffffff, 0x00060005,
698 0x9170, 0xffffffff, 0x00090008,
699 0x9174, 0xffffffff, 0x00020001,
700 0x9178, 0xffffffff, 0x00040003,
701 0x917c, 0xffffffff, 0x00000007,
702 0x9180, 0xffffffff, 0x00060005,
703 0x9184, 0xffffffff, 0x00090008,
704 0x9188, 0xffffffff, 0x00030002,
705 0x918c, 0xffffffff, 0x00050004,
706 0x9190, 0xffffffff, 0x00000008,
707 0x9194, 0xffffffff, 0x00070006,
708 0x9198, 0xffffffff, 0x000a0009,
709 0x919c, 0xffffffff, 0x00040003,
710 0x91a0, 0xffffffff, 0x00060005,
711 0x91a4, 0xffffffff, 0x00000009,
712 0x91a8, 0xffffffff, 0x00080007,
713 0x91ac, 0xffffffff, 0x000b000a,
714 0x91b0, 0xffffffff, 0x00050004,
715 0x91b4, 0xffffffff, 0x00070006,
716 0x91b8, 0xffffffff, 0x0008000b,
717 0x91bc, 0xffffffff, 0x000a0009,
718 0x91c0, 0xffffffff, 0x000d000c,
719 0x9200, 0xffffffff, 0x00090008,
720 0x9204, 0xffffffff, 0x000b000a,
721 0x9208, 0xffffffff, 0x000c000f,
722 0x920c, 0xffffffff, 0x000e000d,
723 0x9210, 0xffffffff, 0x00110010,
724 0x9214, 0xffffffff, 0x000a0009,
725 0x9218, 0xffffffff, 0x000c000b,
726 0x921c, 0xffffffff, 0x0000000f,
727 0x9220, 0xffffffff, 0x000e000d,
728 0x9224, 0xffffffff, 0x00110010,
729 0x9228, 0xffffffff, 0x000b000a,
730 0x922c, 0xffffffff, 0x000d000c,
731 0x9230, 0xffffffff, 0x00000010,
732 0x9234, 0xffffffff, 0x000f000e,
733 0x9238, 0xffffffff, 0x00120011,
734 0x923c, 0xffffffff, 0x000c000b,
735 0x9240, 0xffffffff, 0x000e000d,
736 0x9244, 0xffffffff, 0x00000011,
737 0x9248, 0xffffffff, 0x0010000f,
738 0x924c, 0xffffffff, 0x00130012,
739 0x9250, 0xffffffff, 0x000d000c,
740 0x9254, 0xffffffff, 0x000f000e,
741 0x9258, 0xffffffff, 0x00100013,
742 0x925c, 0xffffffff, 0x00120011,
743 0x9260, 0xffffffff, 0x00150014,
744 0x9150, 0xffffffff, 0x96940200,
745 0x8708, 0xffffffff, 0x00900100,
746 0xc478, 0xffffffff, 0x00000080,
747 0xc404, 0xffffffff, 0x0020003f,
748 0x30, 0xffffffff, 0x0000001c,
749 0x34, 0x000f0000, 0x000f0000,
750 0x160c, 0xffffffff, 0x00000100,
751 0x1024, 0xffffffff, 0x00000100,
752 0x102c, 0x00000101, 0x00000000,
753 0x20a8, 0xffffffff, 0x00000104,
754 0x55e4, 0xff000fff, 0x00000100,
755 0x55e8, 0x00000001, 0x00000001,
756 0x2f50, 0x00000001, 0x00000001,
757 0x30cc, 0xc0000fff, 0x00000104,
758 0xc1e4, 0x00000001, 0x00000001,
759 0xd0c0, 0xfffffff0, 0x00000100,
760 0xd8c0, 0xfffffff0, 0x00000100
763 static const u32 verde_mgcg_cgcg_init[] =
765 0xc400, 0xffffffff, 0xfffffffc,
766 0x802c, 0xffffffff, 0xe0000000,
767 0x9a60, 0xffffffff, 0x00000100,
768 0x92a4, 0xffffffff, 0x00000100,
769 0xc164, 0xffffffff, 0x00000100,
770 0x9774, 0xffffffff, 0x00000100,
771 0x8984, 0xffffffff, 0x06000100,
772 0x8a18, 0xffffffff, 0x00000100,
773 0x92a0, 0xffffffff, 0x00000100,
774 0xc380, 0xffffffff, 0x00000100,
775 0x8b28, 0xffffffff, 0x00000100,
776 0x9144, 0xffffffff, 0x00000100,
777 0x8d88, 0xffffffff, 0x00000100,
778 0x8d8c, 0xffffffff, 0x00000100,
779 0x9030, 0xffffffff, 0x00000100,
780 0x9034, 0xffffffff, 0x00000100,
781 0x9038, 0xffffffff, 0x00000100,
782 0x903c, 0xffffffff, 0x00000100,
783 0xad80, 0xffffffff, 0x00000100,
784 0xac54, 0xffffffff, 0x00000100,
785 0x897c, 0xffffffff, 0x06000100,
786 0x9868, 0xffffffff, 0x00000100,
787 0x9510, 0xffffffff, 0x00000100,
788 0xaf04, 0xffffffff, 0x00000100,
789 0xae04, 0xffffffff, 0x00000100,
790 0x949c, 0xffffffff, 0x00000100,
791 0x802c, 0xffffffff, 0xe0000000,
792 0x9160, 0xffffffff, 0x00010000,
793 0x9164, 0xffffffff, 0x00030002,
794 0x9168, 0xffffffff, 0x00040007,
795 0x916c, 0xffffffff, 0x00060005,
796 0x9170, 0xffffffff, 0x00090008,
797 0x9174, 0xffffffff, 0x00020001,
798 0x9178, 0xffffffff, 0x00040003,
799 0x917c, 0xffffffff, 0x00000007,
800 0x9180, 0xffffffff, 0x00060005,
801 0x9184, 0xffffffff, 0x00090008,
802 0x9188, 0xffffffff, 0x00030002,
803 0x918c, 0xffffffff, 0x00050004,
804 0x9190, 0xffffffff, 0x00000008,
805 0x9194, 0xffffffff, 0x00070006,
806 0x9198, 0xffffffff, 0x000a0009,
807 0x919c, 0xffffffff, 0x00040003,
808 0x91a0, 0xffffffff, 0x00060005,
809 0x91a4, 0xffffffff, 0x00000009,
810 0x91a8, 0xffffffff, 0x00080007,
811 0x91ac, 0xffffffff, 0x000b000a,
812 0x91b0, 0xffffffff, 0x00050004,
813 0x91b4, 0xffffffff, 0x00070006,
814 0x91b8, 0xffffffff, 0x0008000b,
815 0x91bc, 0xffffffff, 0x000a0009,
816 0x91c0, 0xffffffff, 0x000d000c,
817 0x9200, 0xffffffff, 0x00090008,
818 0x9204, 0xffffffff, 0x000b000a,
819 0x9208, 0xffffffff, 0x000c000f,
820 0x920c, 0xffffffff, 0x000e000d,
821 0x9210, 0xffffffff, 0x00110010,
822 0x9214, 0xffffffff, 0x000a0009,
823 0x9218, 0xffffffff, 0x000c000b,
824 0x921c, 0xffffffff, 0x0000000f,
825 0x9220, 0xffffffff, 0x000e000d,
826 0x9224, 0xffffffff, 0x00110010,
827 0x9228, 0xffffffff, 0x000b000a,
828 0x922c, 0xffffffff, 0x000d000c,
829 0x9230, 0xffffffff, 0x00000010,
830 0x9234, 0xffffffff, 0x000f000e,
831 0x9238, 0xffffffff, 0x00120011,
832 0x923c, 0xffffffff, 0x000c000b,
833 0x9240, 0xffffffff, 0x000e000d,
834 0x9244, 0xffffffff, 0x00000011,
835 0x9248, 0xffffffff, 0x0010000f,
836 0x924c, 0xffffffff, 0x00130012,
837 0x9250, 0xffffffff, 0x000d000c,
838 0x9254, 0xffffffff, 0x000f000e,
839 0x9258, 0xffffffff, 0x00100013,
840 0x925c, 0xffffffff, 0x00120011,
841 0x9260, 0xffffffff, 0x00150014,
842 0x9150, 0xffffffff, 0x96940200,
843 0x8708, 0xffffffff, 0x00900100,
844 0xc478, 0xffffffff, 0x00000080,
845 0xc404, 0xffffffff, 0x0020003f,
846 0x30, 0xffffffff, 0x0000001c,
847 0x34, 0x000f0000, 0x000f0000,
848 0x160c, 0xffffffff, 0x00000100,
849 0x1024, 0xffffffff, 0x00000100,
850 0x102c, 0x00000101, 0x00000000,
851 0x20a8, 0xffffffff, 0x00000104,
852 0x264c, 0x000c0000, 0x000c0000,
853 0x2648, 0x000c0000, 0x000c0000,
854 0x55e4, 0xff000fff, 0x00000100,
855 0x55e8, 0x00000001, 0x00000001,
856 0x2f50, 0x00000001, 0x00000001,
857 0x30cc, 0xc0000fff, 0x00000104,
858 0xc1e4, 0x00000001, 0x00000001,
859 0xd0c0, 0xfffffff0, 0x00000100,
860 0xd8c0, 0xfffffff0, 0x00000100
863 static const u32 oland_mgcg_cgcg_init[] =
865 0xc400, 0xffffffff, 0xfffffffc,
866 0x802c, 0xffffffff, 0xe0000000,
867 0x9a60, 0xffffffff, 0x00000100,
868 0x92a4, 0xffffffff, 0x00000100,
869 0xc164, 0xffffffff, 0x00000100,
870 0x9774, 0xffffffff, 0x00000100,
871 0x8984, 0xffffffff, 0x06000100,
872 0x8a18, 0xffffffff, 0x00000100,
873 0x92a0, 0xffffffff, 0x00000100,
874 0xc380, 0xffffffff, 0x00000100,
875 0x8b28, 0xffffffff, 0x00000100,
876 0x9144, 0xffffffff, 0x00000100,
877 0x8d88, 0xffffffff, 0x00000100,
878 0x8d8c, 0xffffffff, 0x00000100,
879 0x9030, 0xffffffff, 0x00000100,
880 0x9034, 0xffffffff, 0x00000100,
881 0x9038, 0xffffffff, 0x00000100,
882 0x903c, 0xffffffff, 0x00000100,
883 0xad80, 0xffffffff, 0x00000100,
884 0xac54, 0xffffffff, 0x00000100,
885 0x897c, 0xffffffff, 0x06000100,
886 0x9868, 0xffffffff, 0x00000100,
887 0x9510, 0xffffffff, 0x00000100,
888 0xaf04, 0xffffffff, 0x00000100,
889 0xae04, 0xffffffff, 0x00000100,
890 0x949c, 0xffffffff, 0x00000100,
891 0x802c, 0xffffffff, 0xe0000000,
892 0x9160, 0xffffffff, 0x00010000,
893 0x9164, 0xffffffff, 0x00030002,
894 0x9168, 0xffffffff, 0x00040007,
895 0x916c, 0xffffffff, 0x00060005,
896 0x9170, 0xffffffff, 0x00090008,
897 0x9174, 0xffffffff, 0x00020001,
898 0x9178, 0xffffffff, 0x00040003,
899 0x917c, 0xffffffff, 0x00000007,
900 0x9180, 0xffffffff, 0x00060005,
901 0x9184, 0xffffffff, 0x00090008,
902 0x9188, 0xffffffff, 0x00030002,
903 0x918c, 0xffffffff, 0x00050004,
904 0x9190, 0xffffffff, 0x00000008,
905 0x9194, 0xffffffff, 0x00070006,
906 0x9198, 0xffffffff, 0x000a0009,
907 0x919c, 0xffffffff, 0x00040003,
908 0x91a0, 0xffffffff, 0x00060005,
909 0x91a4, 0xffffffff, 0x00000009,
910 0x91a8, 0xffffffff, 0x00080007,
911 0x91ac, 0xffffffff, 0x000b000a,
912 0x91b0, 0xffffffff, 0x00050004,
913 0x91b4, 0xffffffff, 0x00070006,
914 0x91b8, 0xffffffff, 0x0008000b,
915 0x91bc, 0xffffffff, 0x000a0009,
916 0x91c0, 0xffffffff, 0x000d000c,
917 0x91c4, 0xffffffff, 0x00060005,
918 0x91c8, 0xffffffff, 0x00080007,
919 0x91cc, 0xffffffff, 0x0000000b,
920 0x91d0, 0xffffffff, 0x000a0009,
921 0x91d4, 0xffffffff, 0x000d000c,
922 0x9150, 0xffffffff, 0x96940200,
923 0x8708, 0xffffffff, 0x00900100,
924 0xc478, 0xffffffff, 0x00000080,
925 0xc404, 0xffffffff, 0x0020003f,
926 0x30, 0xffffffff, 0x0000001c,
927 0x34, 0x000f0000, 0x000f0000,
928 0x160c, 0xffffffff, 0x00000100,
929 0x1024, 0xffffffff, 0x00000100,
930 0x102c, 0x00000101, 0x00000000,
931 0x20a8, 0xffffffff, 0x00000104,
932 0x264c, 0x000c0000, 0x000c0000,
933 0x2648, 0x000c0000, 0x000c0000,
934 0x55e4, 0xff000fff, 0x00000100,
935 0x55e8, 0x00000001, 0x00000001,
936 0x2f50, 0x00000001, 0x00000001,
937 0x30cc, 0xc0000fff, 0x00000104,
938 0xc1e4, 0x00000001, 0x00000001,
939 0xd0c0, 0xfffffff0, 0x00000100,
940 0xd8c0, 0xfffffff0, 0x00000100
943 static const u32 hainan_mgcg_cgcg_init[] =
945 0xc400, 0xffffffff, 0xfffffffc,
946 0x802c, 0xffffffff, 0xe0000000,
947 0x9a60, 0xffffffff, 0x00000100,
948 0x92a4, 0xffffffff, 0x00000100,
949 0xc164, 0xffffffff, 0x00000100,
950 0x9774, 0xffffffff, 0x00000100,
951 0x8984, 0xffffffff, 0x06000100,
952 0x8a18, 0xffffffff, 0x00000100,
953 0x92a0, 0xffffffff, 0x00000100,
954 0xc380, 0xffffffff, 0x00000100,
955 0x8b28, 0xffffffff, 0x00000100,
956 0x9144, 0xffffffff, 0x00000100,
957 0x8d88, 0xffffffff, 0x00000100,
958 0x8d8c, 0xffffffff, 0x00000100,
959 0x9030, 0xffffffff, 0x00000100,
960 0x9034, 0xffffffff, 0x00000100,
961 0x9038, 0xffffffff, 0x00000100,
962 0x903c, 0xffffffff, 0x00000100,
963 0xad80, 0xffffffff, 0x00000100,
964 0xac54, 0xffffffff, 0x00000100,
965 0x897c, 0xffffffff, 0x06000100,
966 0x9868, 0xffffffff, 0x00000100,
967 0x9510, 0xffffffff, 0x00000100,
968 0xaf04, 0xffffffff, 0x00000100,
969 0xae04, 0xffffffff, 0x00000100,
970 0x949c, 0xffffffff, 0x00000100,
971 0x802c, 0xffffffff, 0xe0000000,
972 0x9160, 0xffffffff, 0x00010000,
973 0x9164, 0xffffffff, 0x00030002,
974 0x9168, 0xffffffff, 0x00040007,
975 0x916c, 0xffffffff, 0x00060005,
976 0x9170, 0xffffffff, 0x00090008,
977 0x9174, 0xffffffff, 0x00020001,
978 0x9178, 0xffffffff, 0x00040003,
979 0x917c, 0xffffffff, 0x00000007,
980 0x9180, 0xffffffff, 0x00060005,
981 0x9184, 0xffffffff, 0x00090008,
982 0x9188, 0xffffffff, 0x00030002,
983 0x918c, 0xffffffff, 0x00050004,
984 0x9190, 0xffffffff, 0x00000008,
985 0x9194, 0xffffffff, 0x00070006,
986 0x9198, 0xffffffff, 0x000a0009,
987 0x919c, 0xffffffff, 0x00040003,
988 0x91a0, 0xffffffff, 0x00060005,
989 0x91a4, 0xffffffff, 0x00000009,
990 0x91a8, 0xffffffff, 0x00080007,
991 0x91ac, 0xffffffff, 0x000b000a,
992 0x91b0, 0xffffffff, 0x00050004,
993 0x91b4, 0xffffffff, 0x00070006,
994 0x91b8, 0xffffffff, 0x0008000b,
995 0x91bc, 0xffffffff, 0x000a0009,
996 0x91c0, 0xffffffff, 0x000d000c,
997 0x91c4, 0xffffffff, 0x00060005,
998 0x91c8, 0xffffffff, 0x00080007,
999 0x91cc, 0xffffffff, 0x0000000b,
1000 0x91d0, 0xffffffff, 0x000a0009,
1001 0x91d4, 0xffffffff, 0x000d000c,
1002 0x9150, 0xffffffff, 0x96940200,
1003 0x8708, 0xffffffff, 0x00900100,
1004 0xc478, 0xffffffff, 0x00000080,
1005 0xc404, 0xffffffff, 0x0020003f,
1006 0x30, 0xffffffff, 0x0000001c,
1007 0x34, 0x000f0000, 0x000f0000,
1008 0x160c, 0xffffffff, 0x00000100,
1009 0x1024, 0xffffffff, 0x00000100,
1010 0x20a8, 0xffffffff, 0x00000104,
1011 0x264c, 0x000c0000, 0x000c0000,
1012 0x2648, 0x000c0000, 0x000c0000,
1013 0x2f50, 0x00000001, 0x00000001,
1014 0x30cc, 0xc0000fff, 0x00000104,
1015 0xc1e4, 0x00000001, 0x00000001,
1016 0xd0c0, 0xfffffff0, 0x00000100,
1017 0xd8c0, 0xfffffff0, 0x00000100
1020 static u32 verde_pg_init[] =
1022 0x353c, 0xffffffff, 0x40000,
1023 0x3538, 0xffffffff, 0x200010ff,
1024 0x353c, 0xffffffff, 0x0,
1025 0x353c, 0xffffffff, 0x0,
1026 0x353c, 0xffffffff, 0x0,
1027 0x353c, 0xffffffff, 0x0,
1028 0x353c, 0xffffffff, 0x0,
1029 0x353c, 0xffffffff, 0x7007,
1030 0x3538, 0xffffffff, 0x300010ff,
1031 0x353c, 0xffffffff, 0x0,
1032 0x353c, 0xffffffff, 0x0,
1033 0x353c, 0xffffffff, 0x0,
1034 0x353c, 0xffffffff, 0x0,
1035 0x353c, 0xffffffff, 0x0,
1036 0x353c, 0xffffffff, 0x400000,
1037 0x3538, 0xffffffff, 0x100010ff,
1038 0x353c, 0xffffffff, 0x0,
1039 0x353c, 0xffffffff, 0x0,
1040 0x353c, 0xffffffff, 0x0,
1041 0x353c, 0xffffffff, 0x0,
1042 0x353c, 0xffffffff, 0x0,
1043 0x353c, 0xffffffff, 0x120200,
1044 0x3538, 0xffffffff, 0x500010ff,
1045 0x353c, 0xffffffff, 0x0,
1046 0x353c, 0xffffffff, 0x0,
1047 0x353c, 0xffffffff, 0x0,
1048 0x353c, 0xffffffff, 0x0,
1049 0x353c, 0xffffffff, 0x0,
1050 0x353c, 0xffffffff, 0x1e1e16,
1051 0x3538, 0xffffffff, 0x600010ff,
1052 0x353c, 0xffffffff, 0x0,
1053 0x353c, 0xffffffff, 0x0,
1054 0x353c, 0xffffffff, 0x0,
1055 0x353c, 0xffffffff, 0x0,
1056 0x353c, 0xffffffff, 0x0,
1057 0x353c, 0xffffffff, 0x171f1e,
1058 0x3538, 0xffffffff, 0x700010ff,
1059 0x353c, 0xffffffff, 0x0,
1060 0x353c, 0xffffffff, 0x0,
1061 0x353c, 0xffffffff, 0x0,
1062 0x353c, 0xffffffff, 0x0,
1063 0x353c, 0xffffffff, 0x0,
1064 0x353c, 0xffffffff, 0x0,
1065 0x3538, 0xffffffff, 0x9ff,
1066 0x3500, 0xffffffff, 0x0,
1067 0x3504, 0xffffffff, 0x10000800,
1068 0x3504, 0xffffffff, 0xf,
1069 0x3504, 0xffffffff, 0xf,
1070 0x3500, 0xffffffff, 0x4,
1071 0x3504, 0xffffffff, 0x1000051e,
1072 0x3504, 0xffffffff, 0xffff,
1073 0x3504, 0xffffffff, 0xffff,
1074 0x3500, 0xffffffff, 0x8,
1075 0x3504, 0xffffffff, 0x80500,
1076 0x3500, 0xffffffff, 0x12,
1077 0x3504, 0xffffffff, 0x9050c,
1078 0x3500, 0xffffffff, 0x1d,
1079 0x3504, 0xffffffff, 0xb052c,
1080 0x3500, 0xffffffff, 0x2a,
1081 0x3504, 0xffffffff, 0x1053e,
1082 0x3500, 0xffffffff, 0x2d,
1083 0x3504, 0xffffffff, 0x10546,
1084 0x3500, 0xffffffff, 0x30,
1085 0x3504, 0xffffffff, 0xa054e,
1086 0x3500, 0xffffffff, 0x3c,
1087 0x3504, 0xffffffff, 0x1055f,
1088 0x3500, 0xffffffff, 0x3f,
1089 0x3504, 0xffffffff, 0x10567,
1090 0x3500, 0xffffffff, 0x42,
1091 0x3504, 0xffffffff, 0x1056f,
1092 0x3500, 0xffffffff, 0x45,
1093 0x3504, 0xffffffff, 0x10572,
1094 0x3500, 0xffffffff, 0x48,
1095 0x3504, 0xffffffff, 0x20575,
1096 0x3500, 0xffffffff, 0x4c,
1097 0x3504, 0xffffffff, 0x190801,
1098 0x3500, 0xffffffff, 0x67,
1099 0x3504, 0xffffffff, 0x1082a,
1100 0x3500, 0xffffffff, 0x6a,
1101 0x3504, 0xffffffff, 0x1b082d,
1102 0x3500, 0xffffffff, 0x87,
1103 0x3504, 0xffffffff, 0x310851,
1104 0x3500, 0xffffffff, 0xba,
1105 0x3504, 0xffffffff, 0x891,
1106 0x3500, 0xffffffff, 0xbc,
1107 0x3504, 0xffffffff, 0x893,
1108 0x3500, 0xffffffff, 0xbe,
1109 0x3504, 0xffffffff, 0x20895,
1110 0x3500, 0xffffffff, 0xc2,
1111 0x3504, 0xffffffff, 0x20899,
1112 0x3500, 0xffffffff, 0xc6,
1113 0x3504, 0xffffffff, 0x2089d,
1114 0x3500, 0xffffffff, 0xca,
1115 0x3504, 0xffffffff, 0x8a1,
1116 0x3500, 0xffffffff, 0xcc,
1117 0x3504, 0xffffffff, 0x8a3,
1118 0x3500, 0xffffffff, 0xce,
1119 0x3504, 0xffffffff, 0x308a5,
1120 0x3500, 0xffffffff, 0xd3,
1121 0x3504, 0xffffffff, 0x6d08cd,
1122 0x3500, 0xffffffff, 0x142,
1123 0x3504, 0xffffffff, 0x2000095a,
1124 0x3504, 0xffffffff, 0x1,
1125 0x3500, 0xffffffff, 0x144,
1126 0x3504, 0xffffffff, 0x301f095b,
1127 0x3500, 0xffffffff, 0x165,
1128 0x3504, 0xffffffff, 0xc094d,
1129 0x3500, 0xffffffff, 0x173,
1130 0x3504, 0xffffffff, 0xf096d,
1131 0x3500, 0xffffffff, 0x184,
1132 0x3504, 0xffffffff, 0x15097f,
1133 0x3500, 0xffffffff, 0x19b,
1134 0x3504, 0xffffffff, 0xc0998,
1135 0x3500, 0xffffffff, 0x1a9,
1136 0x3504, 0xffffffff, 0x409a7,
1137 0x3500, 0xffffffff, 0x1af,
1138 0x3504, 0xffffffff, 0xcdc,
1139 0x3500, 0xffffffff, 0x1b1,
1140 0x3504, 0xffffffff, 0x800,
1141 0x3508, 0xffffffff, 0x6c9b2000,
1142 0x3510, 0xfc00, 0x2000,
1143 0x3544, 0xffffffff, 0xfc0,
1144 0x28d4, 0x00000100, 0x100
1147 static void si_init_golden_registers(struct radeon_device *rdev)
1149 switch (rdev->family) {
1151 radeon_program_register_sequence(rdev,
1152 tahiti_golden_registers,
1153 (const u32)ARRAY_SIZE(tahiti_golden_registers));
1154 radeon_program_register_sequence(rdev,
1155 tahiti_golden_rlc_registers,
1156 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1157 radeon_program_register_sequence(rdev,
1158 tahiti_mgcg_cgcg_init,
1159 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1160 radeon_program_register_sequence(rdev,
1161 tahiti_golden_registers2,
1162 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1165 radeon_program_register_sequence(rdev,
1166 pitcairn_golden_registers,
1167 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1168 radeon_program_register_sequence(rdev,
1169 pitcairn_golden_rlc_registers,
1170 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1171 radeon_program_register_sequence(rdev,
1172 pitcairn_mgcg_cgcg_init,
1173 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1176 radeon_program_register_sequence(rdev,
1177 verde_golden_registers,
1178 (const u32)ARRAY_SIZE(verde_golden_registers));
1179 radeon_program_register_sequence(rdev,
1180 verde_golden_rlc_registers,
1181 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1182 radeon_program_register_sequence(rdev,
1183 verde_mgcg_cgcg_init,
1184 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1185 radeon_program_register_sequence(rdev,
1187 (const u32)ARRAY_SIZE(verde_pg_init));
1190 radeon_program_register_sequence(rdev,
1191 oland_golden_registers,
1192 (const u32)ARRAY_SIZE(oland_golden_registers));
1193 radeon_program_register_sequence(rdev,
1194 oland_golden_rlc_registers,
1195 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1196 radeon_program_register_sequence(rdev,
1197 oland_mgcg_cgcg_init,
1198 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1201 radeon_program_register_sequence(rdev,
1202 hainan_golden_registers,
1203 (const u32)ARRAY_SIZE(hainan_golden_registers));
1204 radeon_program_register_sequence(rdev,
1205 hainan_golden_registers2,
1206 (const u32)ARRAY_SIZE(hainan_golden_registers2));
1207 radeon_program_register_sequence(rdev,
1208 hainan_mgcg_cgcg_init,
1209 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1216 #define PCIE_BUS_CLK 10000
1217 #define TCLK (PCIE_BUS_CLK / 10)
1220 * si_get_xclk - get the xclk
1222 * @rdev: radeon_device pointer
1224 * Returns the reference clock used by the gfx engine
1227 u32 si_get_xclk(struct radeon_device *rdev)
1229 u32 reference_clock = rdev->clock.spll.reference_freq;
1232 tmp = RREG32(CG_CLKPIN_CNTL_2);
1233 if (tmp & MUX_TCLK_TO_XCLK)
1236 tmp = RREG32(CG_CLKPIN_CNTL);
1237 if (tmp & XTALIN_DIVIDE)
1238 return reference_clock / 4;
1240 return reference_clock;
1243 /* get temperature in millidegrees */
1244 int si_get_temp(struct radeon_device *rdev)
1247 int actual_temp = 0;
1249 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1255 actual_temp = temp & 0x1ff;
1257 actual_temp = (actual_temp * 1000);
1262 #define TAHITI_IO_MC_REGS_SIZE 36
1264 static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1265 {0x0000006f, 0x03044000},
1266 {0x00000070, 0x0480c018},
1267 {0x00000071, 0x00000040},
1268 {0x00000072, 0x01000000},
1269 {0x00000074, 0x000000ff},
1270 {0x00000075, 0x00143400},
1271 {0x00000076, 0x08ec0800},
1272 {0x00000077, 0x040000cc},
1273 {0x00000079, 0x00000000},
1274 {0x0000007a, 0x21000409},
1275 {0x0000007c, 0x00000000},
1276 {0x0000007d, 0xe8000000},
1277 {0x0000007e, 0x044408a8},
1278 {0x0000007f, 0x00000003},
1279 {0x00000080, 0x00000000},
1280 {0x00000081, 0x01000000},
1281 {0x00000082, 0x02000000},
1282 {0x00000083, 0x00000000},
1283 {0x00000084, 0xe3f3e4f4},
1284 {0x00000085, 0x00052024},
1285 {0x00000087, 0x00000000},
1286 {0x00000088, 0x66036603},
1287 {0x00000089, 0x01000000},
1288 {0x0000008b, 0x1c0a0000},
1289 {0x0000008c, 0xff010000},
1290 {0x0000008e, 0xffffefff},
1291 {0x0000008f, 0xfff3efff},
1292 {0x00000090, 0xfff3efbf},
1293 {0x00000094, 0x00101101},
1294 {0x00000095, 0x00000fff},
1295 {0x00000096, 0x00116fff},
1296 {0x00000097, 0x60010000},
1297 {0x00000098, 0x10010000},
1298 {0x00000099, 0x00006000},
1299 {0x0000009a, 0x00001000},
1300 {0x0000009f, 0x00a77400}
1303 static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1304 {0x0000006f, 0x03044000},
1305 {0x00000070, 0x0480c018},
1306 {0x00000071, 0x00000040},
1307 {0x00000072, 0x01000000},
1308 {0x00000074, 0x000000ff},
1309 {0x00000075, 0x00143400},
1310 {0x00000076, 0x08ec0800},
1311 {0x00000077, 0x040000cc},
1312 {0x00000079, 0x00000000},
1313 {0x0000007a, 0x21000409},
1314 {0x0000007c, 0x00000000},
1315 {0x0000007d, 0xe8000000},
1316 {0x0000007e, 0x044408a8},
1317 {0x0000007f, 0x00000003},
1318 {0x00000080, 0x00000000},
1319 {0x00000081, 0x01000000},
1320 {0x00000082, 0x02000000},
1321 {0x00000083, 0x00000000},
1322 {0x00000084, 0xe3f3e4f4},
1323 {0x00000085, 0x00052024},
1324 {0x00000087, 0x00000000},
1325 {0x00000088, 0x66036603},
1326 {0x00000089, 0x01000000},
1327 {0x0000008b, 0x1c0a0000},
1328 {0x0000008c, 0xff010000},
1329 {0x0000008e, 0xffffefff},
1330 {0x0000008f, 0xfff3efff},
1331 {0x00000090, 0xfff3efbf},
1332 {0x00000094, 0x00101101},
1333 {0x00000095, 0x00000fff},
1334 {0x00000096, 0x00116fff},
1335 {0x00000097, 0x60010000},
1336 {0x00000098, 0x10010000},
1337 {0x00000099, 0x00006000},
1338 {0x0000009a, 0x00001000},
1339 {0x0000009f, 0x00a47400}
1342 static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1343 {0x0000006f, 0x03044000},
1344 {0x00000070, 0x0480c018},
1345 {0x00000071, 0x00000040},
1346 {0x00000072, 0x01000000},
1347 {0x00000074, 0x000000ff},
1348 {0x00000075, 0x00143400},
1349 {0x00000076, 0x08ec0800},
1350 {0x00000077, 0x040000cc},
1351 {0x00000079, 0x00000000},
1352 {0x0000007a, 0x21000409},
1353 {0x0000007c, 0x00000000},
1354 {0x0000007d, 0xe8000000},
1355 {0x0000007e, 0x044408a8},
1356 {0x0000007f, 0x00000003},
1357 {0x00000080, 0x00000000},
1358 {0x00000081, 0x01000000},
1359 {0x00000082, 0x02000000},
1360 {0x00000083, 0x00000000},
1361 {0x00000084, 0xe3f3e4f4},
1362 {0x00000085, 0x00052024},
1363 {0x00000087, 0x00000000},
1364 {0x00000088, 0x66036603},
1365 {0x00000089, 0x01000000},
1366 {0x0000008b, 0x1c0a0000},
1367 {0x0000008c, 0xff010000},
1368 {0x0000008e, 0xffffefff},
1369 {0x0000008f, 0xfff3efff},
1370 {0x00000090, 0xfff3efbf},
1371 {0x00000094, 0x00101101},
1372 {0x00000095, 0x00000fff},
1373 {0x00000096, 0x00116fff},
1374 {0x00000097, 0x60010000},
1375 {0x00000098, 0x10010000},
1376 {0x00000099, 0x00006000},
1377 {0x0000009a, 0x00001000},
1378 {0x0000009f, 0x00a37400}
1381 static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1382 {0x0000006f, 0x03044000},
1383 {0x00000070, 0x0480c018},
1384 {0x00000071, 0x00000040},
1385 {0x00000072, 0x01000000},
1386 {0x00000074, 0x000000ff},
1387 {0x00000075, 0x00143400},
1388 {0x00000076, 0x08ec0800},
1389 {0x00000077, 0x040000cc},
1390 {0x00000079, 0x00000000},
1391 {0x0000007a, 0x21000409},
1392 {0x0000007c, 0x00000000},
1393 {0x0000007d, 0xe8000000},
1394 {0x0000007e, 0x044408a8},
1395 {0x0000007f, 0x00000003},
1396 {0x00000080, 0x00000000},
1397 {0x00000081, 0x01000000},
1398 {0x00000082, 0x02000000},
1399 {0x00000083, 0x00000000},
1400 {0x00000084, 0xe3f3e4f4},
1401 {0x00000085, 0x00052024},
1402 {0x00000087, 0x00000000},
1403 {0x00000088, 0x66036603},
1404 {0x00000089, 0x01000000},
1405 {0x0000008b, 0x1c0a0000},
1406 {0x0000008c, 0xff010000},
1407 {0x0000008e, 0xffffefff},
1408 {0x0000008f, 0xfff3efff},
1409 {0x00000090, 0xfff3efbf},
1410 {0x00000094, 0x00101101},
1411 {0x00000095, 0x00000fff},
1412 {0x00000096, 0x00116fff},
1413 {0x00000097, 0x60010000},
1414 {0x00000098, 0x10010000},
1415 {0x00000099, 0x00006000},
1416 {0x0000009a, 0x00001000},
1417 {0x0000009f, 0x00a17730}
1420 static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1421 {0x0000006f, 0x03044000},
1422 {0x00000070, 0x0480c018},
1423 {0x00000071, 0x00000040},
1424 {0x00000072, 0x01000000},
1425 {0x00000074, 0x000000ff},
1426 {0x00000075, 0x00143400},
1427 {0x00000076, 0x08ec0800},
1428 {0x00000077, 0x040000cc},
1429 {0x00000079, 0x00000000},
1430 {0x0000007a, 0x21000409},
1431 {0x0000007c, 0x00000000},
1432 {0x0000007d, 0xe8000000},
1433 {0x0000007e, 0x044408a8},
1434 {0x0000007f, 0x00000003},
1435 {0x00000080, 0x00000000},
1436 {0x00000081, 0x01000000},
1437 {0x00000082, 0x02000000},
1438 {0x00000083, 0x00000000},
1439 {0x00000084, 0xe3f3e4f4},
1440 {0x00000085, 0x00052024},
1441 {0x00000087, 0x00000000},
1442 {0x00000088, 0x66036603},
1443 {0x00000089, 0x01000000},
1444 {0x0000008b, 0x1c0a0000},
1445 {0x0000008c, 0xff010000},
1446 {0x0000008e, 0xffffefff},
1447 {0x0000008f, 0xfff3efff},
1448 {0x00000090, 0xfff3efbf},
1449 {0x00000094, 0x00101101},
1450 {0x00000095, 0x00000fff},
1451 {0x00000096, 0x00116fff},
1452 {0x00000097, 0x60010000},
1453 {0x00000098, 0x10010000},
1454 {0x00000099, 0x00006000},
1455 {0x0000009a, 0x00001000},
1456 {0x0000009f, 0x00a07730}
1460 int si_mc_load_microcode(struct radeon_device *rdev)
1462 const __be32 *fw_data;
1463 u32 running, blackout = 0;
1465 int i, regs_size, ucode_size;
1470 ucode_size = rdev->mc_fw->datasize / 4;
1472 switch (rdev->family) {
1474 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1475 regs_size = TAHITI_IO_MC_REGS_SIZE;
1478 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1479 regs_size = TAHITI_IO_MC_REGS_SIZE;
1483 io_mc_regs = (u32 *)&verde_io_mc_regs;
1484 regs_size = TAHITI_IO_MC_REGS_SIZE;
1487 io_mc_regs = (u32 *)&oland_io_mc_regs;
1488 regs_size = TAHITI_IO_MC_REGS_SIZE;
1491 io_mc_regs = (u32 *)&hainan_io_mc_regs;
1492 regs_size = TAHITI_IO_MC_REGS_SIZE;
1496 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1500 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1501 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1504 /* reset the engine and set to writable */
1505 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1506 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1508 /* load mc io regs */
1509 for (i = 0; i < regs_size; i++) {
1510 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1511 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1513 /* load the MC ucode */
1514 fw_data = (const __be32 *)rdev->mc_fw->data;
1515 for (i = 0; i < ucode_size; i++)
1516 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1518 /* put the engine back into the active state */
1519 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1520 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1521 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1523 /* wait for training to complete */
1524 for (i = 0; i < rdev->usec_timeout; i++) {
1525 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1529 for (i = 0; i < rdev->usec_timeout; i++) {
1530 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1536 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1542 static int si_init_microcode(struct radeon_device *rdev)
1544 const char *chip_name;
1545 const char *rlc_chip_name;
1546 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1547 size_t smc_req_size, mc2_req_size;
1553 switch (rdev->family) {
1555 chip_name = "TAHITI";
1556 rlc_chip_name = "TAHITI";
1557 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1558 me_req_size = SI_PM4_UCODE_SIZE * 4;
1559 ce_req_size = SI_CE_UCODE_SIZE * 4;
1560 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1561 mc_req_size = SI_MC_UCODE_SIZE * 4;
1562 mc2_req_size = TAHITI_MC_UCODE_SIZE * 4;
1563 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
1566 chip_name = "PITCAIRN";
1567 rlc_chip_name = "PITCAIRN";
1568 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1569 me_req_size = SI_PM4_UCODE_SIZE * 4;
1570 ce_req_size = SI_CE_UCODE_SIZE * 4;
1571 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1572 mc_req_size = SI_MC_UCODE_SIZE * 4;
1573 mc2_req_size = PITCAIRN_MC_UCODE_SIZE * 4;
1574 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
1577 chip_name = "VERDE";
1578 rlc_chip_name = "VERDE";
1579 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1580 me_req_size = SI_PM4_UCODE_SIZE * 4;
1581 ce_req_size = SI_CE_UCODE_SIZE * 4;
1582 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1583 mc_req_size = SI_MC_UCODE_SIZE * 4;
1584 mc2_req_size = VERDE_MC_UCODE_SIZE * 4;
1585 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
1588 chip_name = "OLAND";
1589 rlc_chip_name = "OLAND";
1590 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1591 me_req_size = SI_PM4_UCODE_SIZE * 4;
1592 ce_req_size = SI_CE_UCODE_SIZE * 4;
1593 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1594 mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1595 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
1598 chip_name = "HAINAN";
1599 rlc_chip_name = "HAINAN";
1600 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1601 me_req_size = SI_PM4_UCODE_SIZE * 4;
1602 ce_req_size = SI_CE_UCODE_SIZE * 4;
1603 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1604 mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1605 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
1610 DRM_INFO("Loading %s Microcode\n", chip_name);
1612 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_pfp", chip_name);
1613 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1616 if (rdev->pfp_fw->datasize != pfp_req_size) {
1618 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1619 rdev->pfp_fw->datasize, fw_name);
1624 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_me", chip_name);
1625 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1628 if (rdev->me_fw->datasize != me_req_size) {
1630 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1631 rdev->me_fw->datasize, fw_name);
1635 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_ce", chip_name);
1636 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1639 if (rdev->ce_fw->datasize != ce_req_size) {
1641 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1642 rdev->ce_fw->datasize, fw_name);
1646 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_rlc", rlc_chip_name);
1647 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1650 if (rdev->rlc_fw->datasize != rlc_req_size) {
1652 "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1653 rdev->rlc_fw->datasize, fw_name);
1657 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mc2", chip_name);
1658 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1660 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mc", chip_name);
1661 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1665 if ((rdev->mc_fw->datasize != mc_req_size) &&
1666 (rdev->mc_fw->datasize != mc2_req_size)) {
1668 "si_mc: Bogus length %zu in firmware \"%s\"\n",
1669 rdev->mc_fw->datasize, fw_name);
1672 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->datasize);
1674 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_smc", chip_name);
1675 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1678 "smc: error loading firmware \"%s\"\n",
1680 release_firmware(rdev->smc_fw);
1681 rdev->smc_fw = NULL;
1683 } else if (rdev->smc_fw->datasize != smc_req_size) {
1685 "si_smc: Bogus length %zu in firmware \"%s\"\n",
1686 rdev->smc_fw->datasize, fw_name);
1694 "si_cp: Failed to load firmware \"%s\"\n",
1696 release_firmware(rdev->pfp_fw);
1697 rdev->pfp_fw = NULL;
1698 release_firmware(rdev->me_fw);
1700 release_firmware(rdev->ce_fw);
1702 release_firmware(rdev->rlc_fw);
1703 rdev->rlc_fw = NULL;
1704 release_firmware(rdev->mc_fw);
1706 release_firmware(rdev->smc_fw);
1707 rdev->smc_fw = NULL;
1713 * si_fini_microcode - drop the firmwares image references
1715 * @rdev: radeon_device pointer
1717 * Drop the pfp, me, rlc, mc and ce firmware image references.
1718 * Called at driver shutdown.
1720 static void si_fini_microcode(struct radeon_device *rdev)
1722 release_firmware(rdev->pfp_fw);
1723 rdev->pfp_fw = NULL;
1724 release_firmware(rdev->me_fw);
1726 release_firmware(rdev->rlc_fw);
1727 rdev->rlc_fw = NULL;
1728 release_firmware(rdev->mc_fw);
1730 release_firmware(rdev->smc_fw);
1731 rdev->smc_fw = NULL;
1732 release_firmware(rdev->ce_fw);
1736 /* watermark setup */
1737 static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1738 struct radeon_crtc *radeon_crtc,
1739 struct drm_display_mode *mode,
1740 struct drm_display_mode *other_mode)
1742 u32 tmp, buffer_alloc, i;
1743 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1746 * There are 3 line buffers, each one shared by 2 display controllers.
1747 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1748 * the display controllers. The paritioning is done via one of four
1749 * preset allocations specified in bits 21:20:
1751 * 2 - whole lb, other crtc must be disabled
1753 /* this can get tricky if we have two large displays on a paired group
1754 * of crtcs. Ideally for multiple large displays we'd assign them to
1755 * non-linked crtcs for maximum line buffer allocation.
1757 if (radeon_crtc->base.enabled && mode) {
1762 tmp = 2; /* whole */
1770 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1771 DC_LB_MEMORY_CONFIG(tmp));
1773 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1774 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1775 for (i = 0; i < rdev->usec_timeout; i++) {
1776 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1777 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1782 if (radeon_crtc->base.enabled && mode) {
1792 /* controller not enabled, so no lb used */
1796 static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
1798 u32 tmp = RREG32(MC_SHARED_CHMAP);
1800 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1823 struct dce6_wm_params {
1824 u32 dram_channels; /* number of dram channels */
1825 u32 yclk; /* bandwidth per dram data pin in kHz */
1826 u32 sclk; /* engine clock in kHz */
1827 u32 disp_clk; /* display clock in kHz */
1828 u32 src_width; /* viewport width */
1829 u32 active_time; /* active display time in ns */
1830 u32 blank_time; /* blank time in ns */
1831 bool interlaced; /* mode is interlaced */
1832 fixed20_12 vsc; /* vertical scale ratio */
1833 u32 num_heads; /* number of active crtcs */
1834 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1835 u32 lb_size; /* line buffer allocated to pipe */
1836 u32 vtaps; /* vertical scaler taps */
1839 static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1841 /* Calculate raw DRAM Bandwidth */
1842 fixed20_12 dram_efficiency; /* 0.7 */
1843 fixed20_12 yclk, dram_channels, bandwidth;
1846 a.full = dfixed_const(1000);
1847 yclk.full = dfixed_const(wm->yclk);
1848 yclk.full = dfixed_div(yclk, a);
1849 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1850 a.full = dfixed_const(10);
1851 dram_efficiency.full = dfixed_const(7);
1852 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1853 bandwidth.full = dfixed_mul(dram_channels, yclk);
1854 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1856 return dfixed_trunc(bandwidth);
1859 static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1861 /* Calculate DRAM Bandwidth and the part allocated to display. */
1862 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1863 fixed20_12 yclk, dram_channels, bandwidth;
1866 a.full = dfixed_const(1000);
1867 yclk.full = dfixed_const(wm->yclk);
1868 yclk.full = dfixed_div(yclk, a);
1869 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1870 a.full = dfixed_const(10);
1871 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1872 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1873 bandwidth.full = dfixed_mul(dram_channels, yclk);
1874 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1876 return dfixed_trunc(bandwidth);
1879 static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
1881 /* Calculate the display Data return Bandwidth */
1882 fixed20_12 return_efficiency; /* 0.8 */
1883 fixed20_12 sclk, bandwidth;
1886 a.full = dfixed_const(1000);
1887 sclk.full = dfixed_const(wm->sclk);
1888 sclk.full = dfixed_div(sclk, a);
1889 a.full = dfixed_const(10);
1890 return_efficiency.full = dfixed_const(8);
1891 return_efficiency.full = dfixed_div(return_efficiency, a);
1892 a.full = dfixed_const(32);
1893 bandwidth.full = dfixed_mul(a, sclk);
1894 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1896 return dfixed_trunc(bandwidth);
1899 static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
1904 static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
1906 /* Calculate the DMIF Request Bandwidth */
1907 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1908 fixed20_12 disp_clk, sclk, bandwidth;
1909 fixed20_12 a, b1, b2;
1912 a.full = dfixed_const(1000);
1913 disp_clk.full = dfixed_const(wm->disp_clk);
1914 disp_clk.full = dfixed_div(disp_clk, a);
1915 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
1916 b1.full = dfixed_mul(a, disp_clk);
1918 a.full = dfixed_const(1000);
1919 sclk.full = dfixed_const(wm->sclk);
1920 sclk.full = dfixed_div(sclk, a);
1921 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
1922 b2.full = dfixed_mul(a, sclk);
1924 a.full = dfixed_const(10);
1925 disp_clk_request_efficiency.full = dfixed_const(8);
1926 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1928 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
1930 a.full = dfixed_const(min_bandwidth);
1931 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
1933 return dfixed_trunc(bandwidth);
1936 static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
1938 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1939 u32 dram_bandwidth = dce6_dram_bandwidth(wm);
1940 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
1941 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
1943 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1946 static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
1948 /* Calculate the display mode Average Bandwidth
1949 * DisplayMode should contain the source and destination dimensions,
1953 fixed20_12 line_time;
1954 fixed20_12 src_width;
1955 fixed20_12 bandwidth;
1958 a.full = dfixed_const(1000);
1959 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
1960 line_time.full = dfixed_div(line_time, a);
1961 bpp.full = dfixed_const(wm->bytes_per_pixel);
1962 src_width.full = dfixed_const(wm->src_width);
1963 bandwidth.full = dfixed_mul(src_width, bpp);
1964 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
1965 bandwidth.full = dfixed_div(bandwidth, line_time);
1967 return dfixed_trunc(bandwidth);
1970 static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
1972 /* First calcualte the latency in ns */
1973 u32 mc_latency = 2000; /* 2000 ns. */
1974 u32 available_bandwidth = dce6_available_bandwidth(wm);
1975 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
1976 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
1977 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
1978 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
1979 (wm->num_heads * cursor_line_pair_return_time);
1980 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
1981 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
1982 u32 tmp, dmif_size = 12288;
1985 if (wm->num_heads == 0)
1988 a.full = dfixed_const(2);
1989 b.full = dfixed_const(1);
1990 if ((wm->vsc.full > a.full) ||
1991 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
1993 ((wm->vsc.full >= a.full) && wm->interlaced))
1994 max_src_lines_per_dst_line = 4;
1996 max_src_lines_per_dst_line = 2;
1998 a.full = dfixed_const(available_bandwidth);
1999 b.full = dfixed_const(wm->num_heads);
2000 a.full = dfixed_div(a, b);
2002 b.full = dfixed_const(mc_latency + 512);
2003 c.full = dfixed_const(wm->disp_clk);
2004 b.full = dfixed_div(b, c);
2006 c.full = dfixed_const(dmif_size);
2007 b.full = dfixed_div(c, b);
2009 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
2011 b.full = dfixed_const(1000);
2012 c.full = dfixed_const(wm->disp_clk);
2013 b.full = dfixed_div(c, b);
2014 c.full = dfixed_const(wm->bytes_per_pixel);
2015 b.full = dfixed_mul(b, c);
2017 lb_fill_bw = min(tmp, dfixed_trunc(b));
2019 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2020 b.full = dfixed_const(1000);
2021 c.full = dfixed_const(lb_fill_bw);
2022 b.full = dfixed_div(c, b);
2023 a.full = dfixed_div(a, b);
2024 line_fill_time = dfixed_trunc(a);
2026 if (line_fill_time < wm->active_time)
2029 return latency + (line_fill_time - wm->active_time);
2033 static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2035 if (dce6_average_bandwidth(wm) <=
2036 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
2042 static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2044 if (dce6_average_bandwidth(wm) <=
2045 (dce6_available_bandwidth(wm) / wm->num_heads))
2051 static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2053 u32 lb_partitions = wm->lb_size / wm->src_width;
2054 u32 line_time = wm->active_time + wm->blank_time;
2055 u32 latency_tolerant_lines;
2059 a.full = dfixed_const(1);
2060 if (wm->vsc.full > a.full)
2061 latency_tolerant_lines = 1;
2063 if (lb_partitions <= (wm->vtaps + 1))
2064 latency_tolerant_lines = 1;
2066 latency_tolerant_lines = 2;
2069 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2071 if (dce6_latency_watermark(wm) <= latency_hiding)
2077 static void dce6_program_watermarks(struct radeon_device *rdev,
2078 struct radeon_crtc *radeon_crtc,
2079 u32 lb_size, u32 num_heads)
2081 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2082 struct dce6_wm_params wm_low, wm_high;
2086 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2087 u32 priority_a_mark = 0, priority_b_mark = 0;
2088 u32 priority_a_cnt = PRIORITY_OFF;
2089 u32 priority_b_cnt = PRIORITY_OFF;
2090 u32 tmp, arb_control3;
2093 if (radeon_crtc->base.enabled && num_heads && mode) {
2094 pixel_period = 1000000 / (u32)mode->clock;
2095 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2099 if (rdev->family == CHIP_ARUBA)
2100 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2102 dram_channels = si_get_number_of_dram_channels(rdev);
2104 /* watermark for high clocks */
2105 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2107 radeon_dpm_get_mclk(rdev, false) * 10;
2109 radeon_dpm_get_sclk(rdev, false) * 10;
2111 wm_high.yclk = rdev->pm.current_mclk * 10;
2112 wm_high.sclk = rdev->pm.current_sclk * 10;
2115 wm_high.disp_clk = mode->clock;
2116 wm_high.src_width = mode->crtc_hdisplay;
2117 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2118 wm_high.blank_time = line_time - wm_high.active_time;
2119 wm_high.interlaced = false;
2120 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2121 wm_high.interlaced = true;
2122 wm_high.vsc = radeon_crtc->vsc;
2124 if (radeon_crtc->rmx_type != RMX_OFF)
2126 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2127 wm_high.lb_size = lb_size;
2128 wm_high.dram_channels = dram_channels;
2129 wm_high.num_heads = num_heads;
2131 /* watermark for low clocks */
2132 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2134 radeon_dpm_get_mclk(rdev, true) * 10;
2136 radeon_dpm_get_sclk(rdev, true) * 10;
2138 wm_low.yclk = rdev->pm.current_mclk * 10;
2139 wm_low.sclk = rdev->pm.current_sclk * 10;
2142 wm_low.disp_clk = mode->clock;
2143 wm_low.src_width = mode->crtc_hdisplay;
2144 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2145 wm_low.blank_time = line_time - wm_low.active_time;
2146 wm_low.interlaced = false;
2147 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2148 wm_low.interlaced = true;
2149 wm_low.vsc = radeon_crtc->vsc;
2151 if (radeon_crtc->rmx_type != RMX_OFF)
2153 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2154 wm_low.lb_size = lb_size;
2155 wm_low.dram_channels = dram_channels;
2156 wm_low.num_heads = num_heads;
2158 /* set for high clocks */
2159 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
2160 /* set for low clocks */
2161 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
2163 /* possibly force display priority to high */
2164 /* should really do this at mode validation time... */
2165 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2166 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2167 !dce6_check_latency_hiding(&wm_high) ||
2168 (rdev->disp_priority == 2)) {
2169 DRM_DEBUG_KMS("force priority to high\n");
2170 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2171 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2173 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2174 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2175 !dce6_check_latency_hiding(&wm_low) ||
2176 (rdev->disp_priority == 2)) {
2177 DRM_DEBUG_KMS("force priority to high\n");
2178 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2179 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2182 a.full = dfixed_const(1000);
2183 b.full = dfixed_const(mode->clock);
2184 b.full = dfixed_div(b, a);
2185 c.full = dfixed_const(latency_watermark_a);
2186 c.full = dfixed_mul(c, b);
2187 c.full = dfixed_mul(c, radeon_crtc->hsc);
2188 c.full = dfixed_div(c, a);
2189 a.full = dfixed_const(16);
2190 c.full = dfixed_div(c, a);
2191 priority_a_mark = dfixed_trunc(c);
2192 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2194 a.full = dfixed_const(1000);
2195 b.full = dfixed_const(mode->clock);
2196 b.full = dfixed_div(b, a);
2197 c.full = dfixed_const(latency_watermark_b);
2198 c.full = dfixed_mul(c, b);
2199 c.full = dfixed_mul(c, radeon_crtc->hsc);
2200 c.full = dfixed_div(c, a);
2201 a.full = dfixed_const(16);
2202 c.full = dfixed_div(c, a);
2203 priority_b_mark = dfixed_trunc(c);
2204 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2208 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2210 tmp &= ~LATENCY_WATERMARK_MASK(3);
2211 tmp |= LATENCY_WATERMARK_MASK(1);
2212 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2213 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2214 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2215 LATENCY_HIGH_WATERMARK(line_time)));
2217 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2218 tmp &= ~LATENCY_WATERMARK_MASK(3);
2219 tmp |= LATENCY_WATERMARK_MASK(2);
2220 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2221 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2222 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2223 LATENCY_HIGH_WATERMARK(line_time)));
2224 /* restore original selection */
2225 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2227 /* write the priority marks */
2228 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2229 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2231 /* save values for DPM */
2232 radeon_crtc->line_time = line_time;
2233 radeon_crtc->wm_high = latency_watermark_a;
2234 radeon_crtc->wm_low = latency_watermark_b;
2237 void dce6_bandwidth_update(struct radeon_device *rdev)
2239 struct drm_display_mode *mode0 = NULL;
2240 struct drm_display_mode *mode1 = NULL;
2241 u32 num_heads = 0, lb_size;
2244 radeon_update_display_priority(rdev);
2246 for (i = 0; i < rdev->num_crtc; i++) {
2247 if (rdev->mode_info.crtcs[i]->base.enabled)
2250 for (i = 0; i < rdev->num_crtc; i += 2) {
2251 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2252 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2253 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2254 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2255 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2256 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2263 static void si_tiling_mode_table_init(struct radeon_device *rdev)
2265 const u32 num_tile_mode_states = 32;
2266 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2268 switch (rdev->config.si.mem_row_size_in_kb) {
2270 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2274 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2277 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2281 if ((rdev->family == CHIP_TAHITI) ||
2282 (rdev->family == CHIP_PITCAIRN)) {
2283 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2284 switch (reg_offset) {
2285 case 0: /* non-AA compressed depth or any compressed stencil */
2286 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2287 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2288 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2289 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2290 NUM_BANKS(ADDR_SURF_16_BANK) |
2291 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2292 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2293 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2295 case 1: /* 2xAA/4xAA compressed depth only */
2296 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2297 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2298 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2299 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2300 NUM_BANKS(ADDR_SURF_16_BANK) |
2301 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2302 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2303 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2305 case 2: /* 8xAA compressed depth only */
2306 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2307 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2308 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2309 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2310 NUM_BANKS(ADDR_SURF_16_BANK) |
2311 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2312 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2313 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2315 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2316 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2317 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2318 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2319 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2320 NUM_BANKS(ADDR_SURF_16_BANK) |
2321 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2322 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2323 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2325 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2326 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2327 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2328 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2329 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2330 NUM_BANKS(ADDR_SURF_16_BANK) |
2331 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2332 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2333 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2335 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2336 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2337 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2338 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2339 TILE_SPLIT(split_equal_to_row_size) |
2340 NUM_BANKS(ADDR_SURF_16_BANK) |
2341 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2342 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2343 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2345 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2346 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2347 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2348 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2349 TILE_SPLIT(split_equal_to_row_size) |
2350 NUM_BANKS(ADDR_SURF_16_BANK) |
2351 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2352 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2353 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2355 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2356 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2357 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2358 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2359 TILE_SPLIT(split_equal_to_row_size) |
2360 NUM_BANKS(ADDR_SURF_16_BANK) |
2361 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2362 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2363 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2365 case 8: /* 1D and 1D Array Surfaces */
2366 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2367 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2368 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2369 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2370 NUM_BANKS(ADDR_SURF_16_BANK) |
2371 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2372 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2373 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2375 case 9: /* Displayable maps. */
2376 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2377 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2378 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2379 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2380 NUM_BANKS(ADDR_SURF_16_BANK) |
2381 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2382 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2383 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2385 case 10: /* Display 8bpp. */
2386 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2387 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2388 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2389 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2390 NUM_BANKS(ADDR_SURF_16_BANK) |
2391 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2392 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2393 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2395 case 11: /* Display 16bpp. */
2396 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2397 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2398 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2399 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2400 NUM_BANKS(ADDR_SURF_16_BANK) |
2401 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2402 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2403 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2405 case 12: /* Display 32bpp. */
2406 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2407 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2408 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2409 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2410 NUM_BANKS(ADDR_SURF_16_BANK) |
2411 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2412 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2413 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2415 case 13: /* Thin. */
2416 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2417 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2418 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2419 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2420 NUM_BANKS(ADDR_SURF_16_BANK) |
2421 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2422 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2423 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2425 case 14: /* Thin 8 bpp. */
2426 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2427 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2428 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2429 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2430 NUM_BANKS(ADDR_SURF_16_BANK) |
2431 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2432 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2433 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2435 case 15: /* Thin 16 bpp. */
2436 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2437 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2438 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2439 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2440 NUM_BANKS(ADDR_SURF_16_BANK) |
2441 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2442 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2443 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2445 case 16: /* Thin 32 bpp. */
2446 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2447 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2448 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2449 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2450 NUM_BANKS(ADDR_SURF_16_BANK) |
2451 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2452 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2453 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2455 case 17: /* Thin 64 bpp. */
2456 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2457 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2458 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2459 TILE_SPLIT(split_equal_to_row_size) |
2460 NUM_BANKS(ADDR_SURF_16_BANK) |
2461 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2462 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2463 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2465 case 21: /* 8 bpp PRT. */
2466 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2467 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2468 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2469 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2470 NUM_BANKS(ADDR_SURF_16_BANK) |
2471 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2472 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2473 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2475 case 22: /* 16 bpp PRT */
2476 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2477 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2478 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2479 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2480 NUM_BANKS(ADDR_SURF_16_BANK) |
2481 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2482 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2483 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2485 case 23: /* 32 bpp PRT */
2486 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2487 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2488 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2489 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2490 NUM_BANKS(ADDR_SURF_16_BANK) |
2491 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2492 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2493 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2495 case 24: /* 64 bpp PRT */
2496 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2497 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2498 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2499 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2500 NUM_BANKS(ADDR_SURF_16_BANK) |
2501 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2502 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2503 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2505 case 25: /* 128 bpp PRT */
2506 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2507 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2508 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2509 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2510 NUM_BANKS(ADDR_SURF_8_BANK) |
2511 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2512 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2513 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2519 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2520 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2522 } else if ((rdev->family == CHIP_VERDE) ||
2523 (rdev->family == CHIP_OLAND) ||
2524 (rdev->family == CHIP_HAINAN)) {
2525 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2526 switch (reg_offset) {
2527 case 0: /* non-AA compressed depth or any compressed stencil */
2528 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2529 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2530 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2531 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2532 NUM_BANKS(ADDR_SURF_16_BANK) |
2533 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2534 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2535 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2537 case 1: /* 2xAA/4xAA compressed depth only */
2538 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2539 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2540 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2541 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2542 NUM_BANKS(ADDR_SURF_16_BANK) |
2543 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2544 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2545 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2547 case 2: /* 8xAA compressed depth only */
2548 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2549 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2550 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2551 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2552 NUM_BANKS(ADDR_SURF_16_BANK) |
2553 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2554 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2555 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2557 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2558 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2559 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2560 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2561 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2562 NUM_BANKS(ADDR_SURF_16_BANK) |
2563 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2564 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2565 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2567 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2568 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2569 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2570 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2571 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2572 NUM_BANKS(ADDR_SURF_16_BANK) |
2573 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2574 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2575 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2577 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2578 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2579 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2580 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2581 TILE_SPLIT(split_equal_to_row_size) |
2582 NUM_BANKS(ADDR_SURF_16_BANK) |
2583 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2584 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2585 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2587 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2588 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2589 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2590 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2591 TILE_SPLIT(split_equal_to_row_size) |
2592 NUM_BANKS(ADDR_SURF_16_BANK) |
2593 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2594 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2595 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2597 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2598 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2599 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2600 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2601 TILE_SPLIT(split_equal_to_row_size) |
2602 NUM_BANKS(ADDR_SURF_16_BANK) |
2603 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2604 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2605 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2607 case 8: /* 1D and 1D Array Surfaces */
2608 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2609 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2610 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2611 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2612 NUM_BANKS(ADDR_SURF_16_BANK) |
2613 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2614 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2615 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2617 case 9: /* Displayable maps. */
2618 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2619 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2620 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2621 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2622 NUM_BANKS(ADDR_SURF_16_BANK) |
2623 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2624 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2625 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2627 case 10: /* Display 8bpp. */
2628 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2629 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2630 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2631 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2632 NUM_BANKS(ADDR_SURF_16_BANK) |
2633 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2634 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2635 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2637 case 11: /* Display 16bpp. */
2638 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2639 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2640 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2641 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2642 NUM_BANKS(ADDR_SURF_16_BANK) |
2643 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2644 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2645 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2647 case 12: /* Display 32bpp. */
2648 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2649 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2650 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2651 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2652 NUM_BANKS(ADDR_SURF_16_BANK) |
2653 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2654 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2655 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2657 case 13: /* Thin. */
2658 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2659 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2660 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2661 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2662 NUM_BANKS(ADDR_SURF_16_BANK) |
2663 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2664 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2665 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2667 case 14: /* Thin 8 bpp. */
2668 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2669 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2670 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2671 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2672 NUM_BANKS(ADDR_SURF_16_BANK) |
2673 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2674 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2675 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2677 case 15: /* Thin 16 bpp. */
2678 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2679 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2680 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2681 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2682 NUM_BANKS(ADDR_SURF_16_BANK) |
2683 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2684 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2685 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2687 case 16: /* Thin 32 bpp. */
2688 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2689 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2690 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2691 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2692 NUM_BANKS(ADDR_SURF_16_BANK) |
2693 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2694 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2695 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2697 case 17: /* Thin 64 bpp. */
2698 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2699 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2700 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2701 TILE_SPLIT(split_equal_to_row_size) |
2702 NUM_BANKS(ADDR_SURF_16_BANK) |
2703 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2704 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2705 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2707 case 21: /* 8 bpp PRT. */
2708 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2709 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2710 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2711 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2712 NUM_BANKS(ADDR_SURF_16_BANK) |
2713 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2714 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2715 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2717 case 22: /* 16 bpp PRT */
2718 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2719 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2720 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2721 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2722 NUM_BANKS(ADDR_SURF_16_BANK) |
2723 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2724 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2725 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2727 case 23: /* 32 bpp PRT */
2728 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2729 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2730 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2731 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2732 NUM_BANKS(ADDR_SURF_16_BANK) |
2733 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2734 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2735 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2737 case 24: /* 64 bpp PRT */
2738 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2739 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2740 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2741 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2742 NUM_BANKS(ADDR_SURF_16_BANK) |
2743 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2744 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2745 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2747 case 25: /* 128 bpp PRT */
2748 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2749 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2750 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2751 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2752 NUM_BANKS(ADDR_SURF_8_BANK) |
2753 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2754 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2755 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2761 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2762 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2765 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2768 static void si_select_se_sh(struct radeon_device *rdev,
2769 u32 se_num, u32 sh_num)
2771 u32 data = INSTANCE_BROADCAST_WRITES;
2773 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
2774 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
2775 else if (se_num == 0xffffffff)
2776 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2777 else if (sh_num == 0xffffffff)
2778 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2780 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2781 WREG32(GRBM_GFX_INDEX, data);
2784 static u32 si_create_bitmask(u32 bit_width)
2788 for (i = 0; i < bit_width; i++) {
2795 static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2799 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2801 data &= INACTIVE_CUS_MASK;
2804 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2806 data >>= INACTIVE_CUS_SHIFT;
2808 mask = si_create_bitmask(cu_per_sh);
2810 return ~data & mask;
2813 static void si_setup_spi(struct radeon_device *rdev,
2814 u32 se_num, u32 sh_per_se,
2818 u32 data, mask, active_cu;
2820 for (i = 0; i < se_num; i++) {
2821 for (j = 0; j < sh_per_se; j++) {
2822 si_select_se_sh(rdev, i, j);
2823 data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2824 active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2827 for (k = 0; k < 16; k++) {
2829 if (active_cu & mask) {
2831 WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2837 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2840 static u32 si_get_rb_disabled(struct radeon_device *rdev,
2841 u32 max_rb_num_per_se,
2846 data = RREG32(CC_RB_BACKEND_DISABLE);
2848 data &= BACKEND_DISABLE_MASK;
2851 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2853 data >>= BACKEND_DISABLE_SHIFT;
2855 mask = si_create_bitmask(max_rb_num_per_se / sh_per_se);
2860 static void si_setup_rb(struct radeon_device *rdev,
2861 u32 se_num, u32 sh_per_se,
2862 u32 max_rb_num_per_se)
2866 u32 disabled_rbs = 0;
2867 u32 enabled_rbs = 0;
2869 for (i = 0; i < se_num; i++) {
2870 for (j = 0; j < sh_per_se; j++) {
2871 si_select_se_sh(rdev, i, j);
2872 data = si_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
2873 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
2876 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2879 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
2880 if (!(disabled_rbs & mask))
2881 enabled_rbs |= mask;
2885 rdev->config.si.backend_enable_mask = enabled_rbs;
2887 for (i = 0; i < se_num; i++) {
2888 si_select_se_sh(rdev, i, 0xffffffff);
2890 for (j = 0; j < sh_per_se; j++) {
2891 switch (enabled_rbs & 3) {
2893 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2896 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2900 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2905 WREG32(PA_SC_RASTER_CONFIG, data);
2907 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2910 static void si_gpu_init(struct radeon_device *rdev)
2912 u32 gb_addr_config = 0;
2913 u32 mc_shared_chmap, mc_arb_ramcfg;
2915 u32 hdp_host_path_cntl;
2919 switch (rdev->family) {
2921 rdev->config.si.max_shader_engines = 2;
2922 rdev->config.si.max_tile_pipes = 12;
2923 rdev->config.si.max_cu_per_sh = 8;
2924 rdev->config.si.max_sh_per_se = 2;
2925 rdev->config.si.max_backends_per_se = 4;
2926 rdev->config.si.max_texture_channel_caches = 12;
2927 rdev->config.si.max_gprs = 256;
2928 rdev->config.si.max_gs_threads = 32;
2929 rdev->config.si.max_hw_contexts = 8;
2931 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2932 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2933 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2934 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2935 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
2938 rdev->config.si.max_shader_engines = 2;
2939 rdev->config.si.max_tile_pipes = 8;
2940 rdev->config.si.max_cu_per_sh = 5;
2941 rdev->config.si.max_sh_per_se = 2;
2942 rdev->config.si.max_backends_per_se = 4;
2943 rdev->config.si.max_texture_channel_caches = 8;
2944 rdev->config.si.max_gprs = 256;
2945 rdev->config.si.max_gs_threads = 32;
2946 rdev->config.si.max_hw_contexts = 8;
2948 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2949 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2950 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2951 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2952 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
2956 rdev->config.si.max_shader_engines = 1;
2957 rdev->config.si.max_tile_pipes = 4;
2958 rdev->config.si.max_cu_per_sh = 5;
2959 rdev->config.si.max_sh_per_se = 2;
2960 rdev->config.si.max_backends_per_se = 4;
2961 rdev->config.si.max_texture_channel_caches = 4;
2962 rdev->config.si.max_gprs = 256;
2963 rdev->config.si.max_gs_threads = 32;
2964 rdev->config.si.max_hw_contexts = 8;
2966 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2967 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2968 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2969 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2970 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2973 rdev->config.si.max_shader_engines = 1;
2974 rdev->config.si.max_tile_pipes = 4;
2975 rdev->config.si.max_cu_per_sh = 6;
2976 rdev->config.si.max_sh_per_se = 1;
2977 rdev->config.si.max_backends_per_se = 2;
2978 rdev->config.si.max_texture_channel_caches = 4;
2979 rdev->config.si.max_gprs = 256;
2980 rdev->config.si.max_gs_threads = 16;
2981 rdev->config.si.max_hw_contexts = 8;
2983 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2984 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2985 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2986 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2987 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2990 rdev->config.si.max_shader_engines = 1;
2991 rdev->config.si.max_tile_pipes = 4;
2992 rdev->config.si.max_cu_per_sh = 5;
2993 rdev->config.si.max_sh_per_se = 1;
2994 rdev->config.si.max_backends_per_se = 1;
2995 rdev->config.si.max_texture_channel_caches = 2;
2996 rdev->config.si.max_gprs = 256;
2997 rdev->config.si.max_gs_threads = 16;
2998 rdev->config.si.max_hw_contexts = 8;
3000 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3001 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3002 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3003 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3004 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
3008 /* Initialize HDP */
3009 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3010 WREG32((0x2c14 + j), 0x00000000);
3011 WREG32((0x2c18 + j), 0x00000000);
3012 WREG32((0x2c1c + j), 0x00000000);
3013 WREG32((0x2c20 + j), 0x00000000);
3014 WREG32((0x2c24 + j), 0x00000000);
3017 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3019 evergreen_fix_pci_max_read_req_size(rdev);
3021 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3023 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3024 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3026 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
3027 rdev->config.si.mem_max_burst_length_bytes = 256;
3028 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3029 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3030 if (rdev->config.si.mem_row_size_in_kb > 4)
3031 rdev->config.si.mem_row_size_in_kb = 4;
3032 /* XXX use MC settings? */
3033 rdev->config.si.shader_engine_tile_size = 32;
3034 rdev->config.si.num_gpus = 1;
3035 rdev->config.si.multi_gpu_tile_size = 64;
3037 /* fix up row size */
3038 gb_addr_config &= ~ROW_SIZE_MASK;
3039 switch (rdev->config.si.mem_row_size_in_kb) {
3042 gb_addr_config |= ROW_SIZE(0);
3045 gb_addr_config |= ROW_SIZE(1);
3048 gb_addr_config |= ROW_SIZE(2);
3052 /* setup tiling info dword. gb_addr_config is not adequate since it does
3053 * not have bank info, so create a custom tiling dword.
3054 * bits 3:0 num_pipes
3055 * bits 7:4 num_banks
3056 * bits 11:8 group_size
3057 * bits 15:12 row_size
3059 rdev->config.si.tile_config = 0;
3060 switch (rdev->config.si.num_tile_pipes) {
3062 rdev->config.si.tile_config |= (0 << 0);
3065 rdev->config.si.tile_config |= (1 << 0);
3068 rdev->config.si.tile_config |= (2 << 0);
3072 /* XXX what about 12? */
3073 rdev->config.si.tile_config |= (3 << 0);
3076 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3077 case 0: /* four banks */
3078 rdev->config.si.tile_config |= 0 << 4;
3080 case 1: /* eight banks */
3081 rdev->config.si.tile_config |= 1 << 4;
3083 case 2: /* sixteen banks */
3085 rdev->config.si.tile_config |= 2 << 4;
3088 rdev->config.si.tile_config |=
3089 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3090 rdev->config.si.tile_config |=
3091 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3093 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3094 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3095 WREG32(DMIF_ADDR_CALC, gb_addr_config);
3096 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3097 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3098 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
3099 if (rdev->has_uvd) {
3100 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3101 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3102 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3105 si_tiling_mode_table_init(rdev);
3107 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3108 rdev->config.si.max_sh_per_se,
3109 rdev->config.si.max_backends_per_se);
3111 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3112 rdev->config.si.max_sh_per_se,
3113 rdev->config.si.max_cu_per_sh);
3115 rdev->config.si.active_cus = 0;
3116 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
3117 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
3118 rdev->config.si.active_cus +=
3119 hweight32(si_get_cu_active_bitmap(rdev, i, j));
3123 /* set HW defaults for 3D engine */
3124 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3125 ROQ_IB2_START(0x2b)));
3126 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3128 sx_debug_1 = RREG32(SX_DEBUG_1);
3129 WREG32(SX_DEBUG_1, sx_debug_1);
3131 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3133 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3134 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3135 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3136 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3138 WREG32(VGT_NUM_INSTANCES, 1);
3140 WREG32(CP_PERFMON_CNTL, 0);
3142 WREG32(SQ_CONFIG, 0);
3144 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3145 FORCE_EOV_MAX_REZ_CNT(255)));
3147 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3148 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3150 WREG32(VGT_GS_VERTEX_REUSE, 16);
3151 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3153 WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3154 WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3155 WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3156 WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3157 WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3158 WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3159 WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3160 WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3162 tmp = RREG32(HDP_MISC_CNTL);
3163 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3164 WREG32(HDP_MISC_CNTL, tmp);
3166 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3167 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3169 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3175 * GPU scratch registers helpers function.
3177 static void si_scratch_init(struct radeon_device *rdev)
3181 rdev->scratch.num_reg = 7;
3182 rdev->scratch.reg_base = SCRATCH_REG0;
3183 for (i = 0; i < rdev->scratch.num_reg; i++) {
3184 rdev->scratch.free[i] = true;
3185 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3189 void si_fence_ring_emit(struct radeon_device *rdev,
3190 struct radeon_fence *fence)
3192 struct radeon_ring *ring = &rdev->ring[fence->ring];
3193 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3195 /* flush read cache over gart */
3196 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3197 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3198 radeon_ring_write(ring, 0);
3199 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3200 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3201 PACKET3_TC_ACTION_ENA |
3202 PACKET3_SH_KCACHE_ACTION_ENA |
3203 PACKET3_SH_ICACHE_ACTION_ENA);
3204 radeon_ring_write(ring, 0xFFFFFFFF);
3205 radeon_ring_write(ring, 0);
3206 radeon_ring_write(ring, 10); /* poll interval */
3207 /* EVENT_WRITE_EOP - flush caches, send int */
3208 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3209 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3210 radeon_ring_write(ring, lower_32_bits(addr));
3211 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3212 radeon_ring_write(ring, fence->seq);
3213 radeon_ring_write(ring, 0);
3219 void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3221 struct radeon_ring *ring = &rdev->ring[ib->ring];
3224 if (ib->is_const_ib) {
3225 /* set switch buffer packet before const IB */
3226 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3227 radeon_ring_write(ring, 0);
3229 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3232 if (ring->rptr_save_reg) {
3233 next_rptr = ring->wptr + 3 + 4 + 8;
3234 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3235 radeon_ring_write(ring, ((ring->rptr_save_reg -
3236 PACKET3_SET_CONFIG_REG_START) >> 2));
3237 radeon_ring_write(ring, next_rptr);
3238 } else if (rdev->wb.enabled) {
3239 next_rptr = ring->wptr + 5 + 4 + 8;
3240 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3241 radeon_ring_write(ring, (1 << 8));
3242 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3243 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
3244 radeon_ring_write(ring, next_rptr);
3247 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3250 radeon_ring_write(ring, header);
3251 radeon_ring_write(ring,
3255 (ib->gpu_addr & 0xFFFFFFFC));
3256 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3257 radeon_ring_write(ring, ib->length_dw |
3258 (ib->vm ? (ib->vm->id << 24) : 0));
3260 if (!ib->is_const_ib) {
3261 /* flush read cache over gart for this vmid */
3262 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3263 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3264 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
3265 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3266 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3267 PACKET3_TC_ACTION_ENA |
3268 PACKET3_SH_KCACHE_ACTION_ENA |
3269 PACKET3_SH_ICACHE_ACTION_ENA);
3270 radeon_ring_write(ring, 0xFFFFFFFF);
3271 radeon_ring_write(ring, 0);
3272 radeon_ring_write(ring, 10); /* poll interval */
3279 static void si_cp_enable(struct radeon_device *rdev, bool enable)
3282 WREG32(CP_ME_CNTL, 0);
3284 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3285 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3286 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3287 WREG32(SCRATCH_UMSK, 0);
3288 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3289 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3290 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3295 static int si_cp_load_microcode(struct radeon_device *rdev)
3297 const __be32 *fw_data;
3300 if (!rdev->me_fw || !rdev->pfp_fw)
3303 si_cp_enable(rdev, false);
3306 fw_data = (const __be32 *)rdev->pfp_fw->data;
3307 WREG32(CP_PFP_UCODE_ADDR, 0);
3308 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3309 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3310 WREG32(CP_PFP_UCODE_ADDR, 0);
3313 fw_data = (const __be32 *)rdev->ce_fw->data;
3314 WREG32(CP_CE_UCODE_ADDR, 0);
3315 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3316 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3317 WREG32(CP_CE_UCODE_ADDR, 0);
3320 fw_data = (const __be32 *)rdev->me_fw->data;
3321 WREG32(CP_ME_RAM_WADDR, 0);
3322 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3323 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3324 WREG32(CP_ME_RAM_WADDR, 0);
3326 WREG32(CP_PFP_UCODE_ADDR, 0);
3327 WREG32(CP_CE_UCODE_ADDR, 0);
3328 WREG32(CP_ME_RAM_WADDR, 0);
3329 WREG32(CP_ME_RAM_RADDR, 0);
3333 static int si_cp_start(struct radeon_device *rdev)
3335 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3338 r = radeon_ring_lock(rdev, ring, 7 + 4);
3340 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3344 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3345 radeon_ring_write(ring, 0x1);
3346 radeon_ring_write(ring, 0x0);
3347 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3348 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3349 radeon_ring_write(ring, 0);
3350 radeon_ring_write(ring, 0);
3352 /* init the CE partitions */
3353 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3354 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3355 radeon_ring_write(ring, 0xc000);
3356 radeon_ring_write(ring, 0xe000);
3357 radeon_ring_unlock_commit(rdev, ring, false);
3359 si_cp_enable(rdev, true);
3361 r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3363 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3367 /* setup clear context state */
3368 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3369 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3371 for (i = 0; i < si_default_size; i++)
3372 radeon_ring_write(ring, si_default_state[i]);
3374 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3375 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3377 /* set clear context state */
3378 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3379 radeon_ring_write(ring, 0);
3381 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3382 radeon_ring_write(ring, 0x00000316);
3383 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3384 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3386 radeon_ring_unlock_commit(rdev, ring, false);
3388 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3389 ring = &rdev->ring[i];
3390 r = radeon_ring_lock(rdev, ring, 2);
3392 /* clear the compute context state */
3393 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3394 radeon_ring_write(ring, 0);
3396 radeon_ring_unlock_commit(rdev, ring, false);
3402 static void si_cp_fini(struct radeon_device *rdev)
3404 struct radeon_ring *ring;
3405 si_cp_enable(rdev, false);
3407 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3408 radeon_ring_fini(rdev, ring);
3409 radeon_scratch_free(rdev, ring->rptr_save_reg);
3411 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3412 radeon_ring_fini(rdev, ring);
3413 radeon_scratch_free(rdev, ring->rptr_save_reg);
3415 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3416 radeon_ring_fini(rdev, ring);
3417 radeon_scratch_free(rdev, ring->rptr_save_reg);
3420 static int si_cp_resume(struct radeon_device *rdev)
3422 struct radeon_ring *ring;
3427 si_enable_gui_idle_interrupt(rdev, false);
3429 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3430 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3432 /* Set the write pointer delay */
3433 WREG32(CP_RB_WPTR_DELAY, 0);
3435 WREG32(CP_DEBUG, 0);
3436 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3438 /* ring 0 - compute and gfx */
3439 /* Set ring buffer size */
3440 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3441 rb_bufsz = order_base_2(ring->ring_size / 8);
3442 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3444 tmp |= BUF_SWAP_32BIT;
3446 WREG32(CP_RB0_CNTL, tmp);
3448 /* Initialize the ring buffer's read and write pointers */
3449 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3451 WREG32(CP_RB0_WPTR, ring->wptr);
3453 /* set the wb address whether it's enabled or not */
3454 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3455 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3457 if (rdev->wb.enabled)
3458 WREG32(SCRATCH_UMSK, 0xff);
3460 tmp |= RB_NO_UPDATE;
3461 WREG32(SCRATCH_UMSK, 0);
3465 WREG32(CP_RB0_CNTL, tmp);
3467 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3469 /* ring1 - compute only */
3470 /* Set ring buffer size */
3471 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3472 rb_bufsz = order_base_2(ring->ring_size / 8);
3473 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3475 tmp |= BUF_SWAP_32BIT;
3477 WREG32(CP_RB1_CNTL, tmp);
3479 /* Initialize the ring buffer's read and write pointers */
3480 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3482 WREG32(CP_RB1_WPTR, ring->wptr);
3484 /* set the wb address whether it's enabled or not */
3485 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3486 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3489 WREG32(CP_RB1_CNTL, tmp);
3491 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3493 /* ring2 - compute only */
3494 /* Set ring buffer size */
3495 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3496 rb_bufsz = order_base_2(ring->ring_size / 8);
3497 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3499 tmp |= BUF_SWAP_32BIT;
3501 WREG32(CP_RB2_CNTL, tmp);
3503 /* Initialize the ring buffer's read and write pointers */
3504 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3506 WREG32(CP_RB2_WPTR, ring->wptr);
3508 /* set the wb address whether it's enabled or not */
3509 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3510 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3513 WREG32(CP_RB2_CNTL, tmp);
3515 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3517 /* start the rings */
3519 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3520 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3521 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3522 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3524 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3525 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3526 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3529 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3531 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3533 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3535 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3538 si_enable_gui_idle_interrupt(rdev, true);
3540 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3541 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
3546 u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
3552 tmp = RREG32(GRBM_STATUS);
3553 if (tmp & (PA_BUSY | SC_BUSY |
3554 BCI_BUSY | SX_BUSY |
3555 TA_BUSY | VGT_BUSY |
3557 GDS_BUSY | SPI_BUSY |
3558 IA_BUSY | IA_BUSY_NO_DMA))
3559 reset_mask |= RADEON_RESET_GFX;
3561 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3562 CP_BUSY | CP_COHERENCY_BUSY))
3563 reset_mask |= RADEON_RESET_CP;
3565 if (tmp & GRBM_EE_BUSY)
3566 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3569 tmp = RREG32(GRBM_STATUS2);
3570 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3571 reset_mask |= RADEON_RESET_RLC;
3573 /* DMA_STATUS_REG 0 */
3574 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3575 if (!(tmp & DMA_IDLE))
3576 reset_mask |= RADEON_RESET_DMA;
3578 /* DMA_STATUS_REG 1 */
3579 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3580 if (!(tmp & DMA_IDLE))
3581 reset_mask |= RADEON_RESET_DMA1;
3584 tmp = RREG32(SRBM_STATUS2);
3586 reset_mask |= RADEON_RESET_DMA;
3588 if (tmp & DMA1_BUSY)
3589 reset_mask |= RADEON_RESET_DMA1;
3592 tmp = RREG32(SRBM_STATUS);
3595 reset_mask |= RADEON_RESET_IH;
3598 reset_mask |= RADEON_RESET_SEM;
3600 if (tmp & GRBM_RQ_PENDING)
3601 reset_mask |= RADEON_RESET_GRBM;
3604 reset_mask |= RADEON_RESET_VMC;
3606 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3607 MCC_BUSY | MCD_BUSY))
3608 reset_mask |= RADEON_RESET_MC;
3610 if (evergreen_is_display_hung(rdev))
3611 reset_mask |= RADEON_RESET_DISPLAY;
3614 tmp = RREG32(VM_L2_STATUS);
3616 reset_mask |= RADEON_RESET_VMC;
3618 /* Skip MC reset as it's mostly likely not hung, just busy */
3619 if (reset_mask & RADEON_RESET_MC) {
3620 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3621 reset_mask &= ~RADEON_RESET_MC;
3627 static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3629 struct evergreen_mc_save save;
3630 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3633 if (reset_mask == 0)
3636 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3638 evergreen_print_gpu_status_regs(rdev);
3639 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3640 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3641 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3642 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3651 /* Disable CP parsing/prefetching */
3652 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3654 if (reset_mask & RADEON_RESET_DMA) {
3656 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3657 tmp &= ~DMA_RB_ENABLE;
3658 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3660 if (reset_mask & RADEON_RESET_DMA1) {
3662 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3663 tmp &= ~DMA_RB_ENABLE;
3664 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3669 evergreen_mc_stop(rdev, &save);
3670 if (evergreen_mc_wait_for_idle(rdev)) {
3671 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3674 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3675 grbm_soft_reset = SOFT_RESET_CB |
3689 if (reset_mask & RADEON_RESET_CP) {
3690 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3692 srbm_soft_reset |= SOFT_RESET_GRBM;
3695 if (reset_mask & RADEON_RESET_DMA)
3696 srbm_soft_reset |= SOFT_RESET_DMA;
3698 if (reset_mask & RADEON_RESET_DMA1)
3699 srbm_soft_reset |= SOFT_RESET_DMA1;
3701 if (reset_mask & RADEON_RESET_DISPLAY)
3702 srbm_soft_reset |= SOFT_RESET_DC;
3704 if (reset_mask & RADEON_RESET_RLC)
3705 grbm_soft_reset |= SOFT_RESET_RLC;
3707 if (reset_mask & RADEON_RESET_SEM)
3708 srbm_soft_reset |= SOFT_RESET_SEM;
3710 if (reset_mask & RADEON_RESET_IH)
3711 srbm_soft_reset |= SOFT_RESET_IH;
3713 if (reset_mask & RADEON_RESET_GRBM)
3714 srbm_soft_reset |= SOFT_RESET_GRBM;
3716 if (reset_mask & RADEON_RESET_VMC)
3717 srbm_soft_reset |= SOFT_RESET_VMC;
3719 if (reset_mask & RADEON_RESET_MC)
3720 srbm_soft_reset |= SOFT_RESET_MC;
3722 if (grbm_soft_reset) {
3723 tmp = RREG32(GRBM_SOFT_RESET);
3724 tmp |= grbm_soft_reset;
3725 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3726 WREG32(GRBM_SOFT_RESET, tmp);
3727 tmp = RREG32(GRBM_SOFT_RESET);
3731 tmp &= ~grbm_soft_reset;
3732 WREG32(GRBM_SOFT_RESET, tmp);
3733 tmp = RREG32(GRBM_SOFT_RESET);
3736 if (srbm_soft_reset) {
3737 tmp = RREG32(SRBM_SOFT_RESET);
3738 tmp |= srbm_soft_reset;
3739 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3740 WREG32(SRBM_SOFT_RESET, tmp);
3741 tmp = RREG32(SRBM_SOFT_RESET);
3745 tmp &= ~srbm_soft_reset;
3746 WREG32(SRBM_SOFT_RESET, tmp);
3747 tmp = RREG32(SRBM_SOFT_RESET);
3750 /* Wait a little for things to settle down */
3753 evergreen_mc_resume(rdev, &save);
3756 evergreen_print_gpu_status_regs(rdev);
3759 static void si_set_clk_bypass_mode(struct radeon_device *rdev)
3763 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3764 tmp |= SPLL_BYPASS_EN;
3765 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3767 tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3768 tmp |= SPLL_CTLREQ_CHG;
3769 WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3771 for (i = 0; i < rdev->usec_timeout; i++) {
3772 if (RREG32(SPLL_STATUS) & SPLL_CHG_STATUS)
3777 tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3778 tmp &= ~(SPLL_CTLREQ_CHG | SCLK_MUX_UPDATE);
3779 WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3781 tmp = RREG32(MPLL_CNTL_MODE);
3782 tmp &= ~MPLL_MCLK_SEL;
3783 WREG32(MPLL_CNTL_MODE, tmp);
3786 static void si_spll_powerdown(struct radeon_device *rdev)
3790 tmp = RREG32(SPLL_CNTL_MODE);
3791 tmp |= SPLL_SW_DIR_CONTROL;
3792 WREG32(SPLL_CNTL_MODE, tmp);
3794 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3796 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3798 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3800 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3802 tmp = RREG32(SPLL_CNTL_MODE);
3803 tmp &= ~SPLL_SW_DIR_CONTROL;
3804 WREG32(SPLL_CNTL_MODE, tmp);
3807 static void si_gpu_pci_config_reset(struct radeon_device *rdev)
3809 struct evergreen_mc_save save;
3812 dev_info(rdev->dev, "GPU pci config reset\n");
3820 /* Disable CP parsing/prefetching */
3821 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3823 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3824 tmp &= ~DMA_RB_ENABLE;
3825 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3827 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3828 tmp &= ~DMA_RB_ENABLE;
3829 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3830 /* XXX other engines? */
3832 /* halt the rlc, disable cp internal ints */
3837 /* disable mem access */
3838 evergreen_mc_stop(rdev, &save);
3839 if (evergreen_mc_wait_for_idle(rdev)) {
3840 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
3843 /* set mclk/sclk to bypass */
3844 si_set_clk_bypass_mode(rdev);
3845 /* powerdown spll */
3846 si_spll_powerdown(rdev);
3848 pci_disable_busmaster(rdev->pdev->dev);
3850 radeon_pci_config_reset(rdev);
3851 /* wait for asic to come out of reset */
3852 for (i = 0; i < rdev->usec_timeout; i++) {
3853 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
3859 int si_asic_reset(struct radeon_device *rdev)
3863 reset_mask = si_gpu_check_soft_reset(rdev);
3866 r600_set_bios_scratch_engine_hung(rdev, true);
3868 /* try soft reset */
3869 si_gpu_soft_reset(rdev, reset_mask);
3871 reset_mask = si_gpu_check_soft_reset(rdev);
3873 /* try pci config reset */
3874 if (reset_mask && radeon_hard_reset)
3875 si_gpu_pci_config_reset(rdev);
3877 reset_mask = si_gpu_check_soft_reset(rdev);
3880 r600_set_bios_scratch_engine_hung(rdev, false);
3886 * si_gfx_is_lockup - Check if the GFX engine is locked up
3888 * @rdev: radeon_device pointer
3889 * @ring: radeon_ring structure holding ring information
3891 * Check if the GFX engine is locked up.
3892 * Returns true if the engine appears to be locked up, false if not.
3894 bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3896 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3898 if (!(reset_mask & (RADEON_RESET_GFX |
3899 RADEON_RESET_COMPUTE |
3900 RADEON_RESET_CP))) {
3901 radeon_ring_lockup_update(rdev, ring);
3904 return radeon_ring_test_lockup(rdev, ring);
3908 static void si_mc_program(struct radeon_device *rdev)
3910 struct evergreen_mc_save save;
3914 /* Initialize HDP */
3915 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3916 WREG32((0x2c14 + j), 0x00000000);
3917 WREG32((0x2c18 + j), 0x00000000);
3918 WREG32((0x2c1c + j), 0x00000000);
3919 WREG32((0x2c20 + j), 0x00000000);
3920 WREG32((0x2c24 + j), 0x00000000);
3922 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
3924 evergreen_mc_stop(rdev, &save);
3925 if (radeon_mc_wait_for_idle(rdev)) {
3926 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3928 if (!ASIC_IS_NODCE(rdev))
3929 /* Lockout access through VGA aperture*/
3930 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
3931 /* Update configuration */
3932 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
3933 rdev->mc.vram_start >> 12);
3934 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
3935 rdev->mc.vram_end >> 12);
3936 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
3937 rdev->vram_scratch.gpu_addr >> 12);
3938 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
3939 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
3940 WREG32(MC_VM_FB_LOCATION, tmp);
3941 /* XXX double check these! */
3942 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
3943 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
3944 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
3945 WREG32(MC_VM_AGP_BASE, 0);
3946 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
3947 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
3948 if (radeon_mc_wait_for_idle(rdev)) {
3949 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3951 evergreen_mc_resume(rdev, &save);
3952 if (!ASIC_IS_NODCE(rdev)) {
3953 /* we need to own VRAM, so turn off the VGA renderer here
3954 * to stop it overwriting our objects */
3955 rv515_vga_render_disable(rdev);
3959 void si_vram_gtt_location(struct radeon_device *rdev,
3960 struct radeon_mc *mc)
3962 if (mc->mc_vram_size > 0xFFC0000000ULL) {
3963 /* leave room for at least 1024M GTT */
3964 dev_warn(rdev->dev, "limiting VRAM\n");
3965 mc->real_vram_size = 0xFFC0000000ULL;
3966 mc->mc_vram_size = 0xFFC0000000ULL;
3968 radeon_vram_location(rdev, &rdev->mc, 0);
3969 rdev->mc.gtt_base_align = 0;
3970 radeon_gtt_location(rdev, mc);
3973 static int si_mc_init(struct radeon_device *rdev)
3976 int chansize, numchan;
3978 /* Get VRAM informations */
3979 rdev->mc.vram_is_ddr = true;
3980 tmp = RREG32(MC_ARB_RAMCFG);
3981 if (tmp & CHANSIZE_OVERRIDE) {
3983 } else if (tmp & CHANSIZE_MASK) {
3988 tmp = RREG32(MC_SHARED_CHMAP);
3989 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4019 rdev->mc.vram_width = numchan * chansize;
4020 /* Could aper size report 0 ? */
4021 rdev->mc.aper_base = drm_get_resource_start(rdev->ddev, 0);
4022 rdev->mc.aper_size = drm_get_resource_len(rdev->ddev, 0);
4023 /* size in MB on si */
4024 tmp = RREG32(CONFIG_MEMSIZE);
4025 /* some boards may have garbage in the upper 16 bits */
4026 if (tmp & 0xffff0000) {
4027 DRM_INFO("Probable bad vram size: 0x%08x\n", tmp);
4031 rdev->mc.mc_vram_size = tmp * 1024ULL * 1024ULL;
4032 rdev->mc.real_vram_size = rdev->mc.mc_vram_size;
4033 rdev->mc.visible_vram_size = rdev->mc.aper_size;
4034 si_vram_gtt_location(rdev, &rdev->mc);
4035 radeon_update_bandwidth_info(rdev);
4043 void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
4045 /* flush hdp cache */
4046 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
4048 /* bits 0-15 are the VM contexts0-15 */
4049 WREG32(VM_INVALIDATE_REQUEST, 1);
4052 static int si_pcie_gart_enable(struct radeon_device *rdev)
4056 if (rdev->gart.robj == NULL) {
4057 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
4060 r = radeon_gart_table_vram_pin(rdev);
4063 /* Setup TLB control */
4064 WREG32(MC_VM_MX_L1_TLB_CNTL,
4067 ENABLE_L1_FRAGMENT_PROCESSING |
4068 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4069 ENABLE_ADVANCED_DRIVER_MODEL |
4070 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4071 /* Setup L2 cache */
4072 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
4073 ENABLE_L2_FRAGMENT_PROCESSING |
4074 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4075 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4076 EFFECTIVE_L2_QUEUE_SIZE(7) |
4077 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4078 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
4079 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4081 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
4082 /* setup context0 */
4083 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
4084 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
4085 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
4086 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
4087 (u32)(rdev->dummy_page.addr >> 12));
4088 WREG32(VM_CONTEXT0_CNTL2, 0);
4089 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
4090 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
4096 /* empty context1-15 */
4097 /* set vm size, must be a multiple of 4 */
4098 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
4099 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
4100 /* Assign the pt base to something valid for now; the pts used for
4101 * the VMs are determined by the application and setup and assigned
4102 * on the fly in the vm part of radeon_gart.c
4104 for (i = 1; i < 16; i++) {
4106 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
4107 rdev->vm_manager.saved_table_addr[i]);
4109 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
4110 rdev->vm_manager.saved_table_addr[i]);
4113 /* enable context1-15 */
4114 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
4115 (u32)(rdev->dummy_page.addr >> 12));
4116 WREG32(VM_CONTEXT1_CNTL2, 4);
4117 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4118 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
4119 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4120 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4121 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4122 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4123 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
4124 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
4125 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
4126 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
4127 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
4128 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
4129 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4130 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
4132 si_pcie_gart_tlb_flush(rdev);
4133 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
4134 (unsigned)(rdev->mc.gtt_size >> 20),
4135 (unsigned long long)rdev->gart.table_addr);
4136 rdev->gart.ready = true;
4140 static void si_pcie_gart_disable(struct radeon_device *rdev)
4144 for (i = 1; i < 16; ++i) {
4147 reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
4149 reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
4150 rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
4153 /* Disable all tables */
4154 WREG32(VM_CONTEXT0_CNTL, 0);
4155 WREG32(VM_CONTEXT1_CNTL, 0);
4156 /* Setup TLB control */
4157 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4158 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4159 /* Setup L2 cache */
4160 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4161 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4162 EFFECTIVE_L2_QUEUE_SIZE(7) |
4163 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4164 WREG32(VM_L2_CNTL2, 0);
4165 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4166 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4167 radeon_gart_table_vram_unpin(rdev);
4170 static void si_pcie_gart_fini(struct radeon_device *rdev)
4172 si_pcie_gart_disable(rdev);
4173 radeon_gart_table_vram_free(rdev);
4174 radeon_gart_fini(rdev);
4178 static bool si_vm_reg_valid(u32 reg)
4180 /* context regs are fine */
4184 /* check config regs */
4186 case GRBM_GFX_INDEX:
4187 case CP_STRMOUT_CNTL:
4188 case VGT_VTX_VECT_EJECT_REG:
4189 case VGT_CACHE_INVALIDATION:
4190 case VGT_ESGS_RING_SIZE:
4191 case VGT_GSVS_RING_SIZE:
4192 case VGT_GS_VERTEX_REUSE:
4193 case VGT_PRIMITIVE_TYPE:
4194 case VGT_INDEX_TYPE:
4195 case VGT_NUM_INDICES:
4196 case VGT_NUM_INSTANCES:
4197 case VGT_TF_RING_SIZE:
4198 case VGT_HS_OFFCHIP_PARAM:
4199 case VGT_TF_MEMORY_BASE:
4201 case PA_SU_LINE_STIPPLE_VALUE:
4202 case PA_SC_LINE_STIPPLE_STATE:
4205 case SPI_STATIC_THREAD_MGMT_1:
4206 case SPI_STATIC_THREAD_MGMT_2:
4207 case SPI_STATIC_THREAD_MGMT_3:
4208 case SPI_PS_MAX_WAVE_ID:
4209 case SPI_CONFIG_CNTL:
4210 case SPI_CONFIG_CNTL_1:
4214 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4219 static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4220 u32 *ib, struct radeon_cs_packet *pkt)
4222 switch (pkt->opcode) {
4224 case PACKET3_SET_BASE:
4225 case PACKET3_SET_CE_DE_COUNTERS:
4226 case PACKET3_LOAD_CONST_RAM:
4227 case PACKET3_WRITE_CONST_RAM:
4228 case PACKET3_WRITE_CONST_RAM_OFFSET:
4229 case PACKET3_DUMP_CONST_RAM:
4230 case PACKET3_INCREMENT_CE_COUNTER:
4231 case PACKET3_WAIT_ON_DE_COUNTER:
4232 case PACKET3_CE_WRITE:
4235 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4241 static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
4243 u32 start_reg, reg, i;
4244 u32 command = ib[idx + 4];
4245 u32 info = ib[idx + 1];
4246 u32 idx_value = ib[idx];
4247 if (command & PACKET3_CP_DMA_CMD_SAS) {
4248 /* src address space is register */
4249 if (((info & 0x60000000) >> 29) == 0) {
4250 start_reg = idx_value << 2;
4251 if (command & PACKET3_CP_DMA_CMD_SAIC) {
4253 if (!si_vm_reg_valid(reg)) {
4254 DRM_ERROR("CP DMA Bad SRC register\n");
4258 for (i = 0; i < (command & 0x1fffff); i++) {
4259 reg = start_reg + (4 * i);
4260 if (!si_vm_reg_valid(reg)) {
4261 DRM_ERROR("CP DMA Bad SRC register\n");
4268 if (command & PACKET3_CP_DMA_CMD_DAS) {
4269 /* dst address space is register */
4270 if (((info & 0x00300000) >> 20) == 0) {
4271 start_reg = ib[idx + 2];
4272 if (command & PACKET3_CP_DMA_CMD_DAIC) {
4274 if (!si_vm_reg_valid(reg)) {
4275 DRM_ERROR("CP DMA Bad DST register\n");
4279 for (i = 0; i < (command & 0x1fffff); i++) {
4280 reg = start_reg + (4 * i);
4281 if (!si_vm_reg_valid(reg)) {
4282 DRM_ERROR("CP DMA Bad DST register\n");
4292 static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4293 u32 *ib, struct radeon_cs_packet *pkt)
4296 u32 idx = pkt->idx + 1;
4297 u32 idx_value = ib[idx];
4298 u32 start_reg, end_reg, reg, i;
4300 switch (pkt->opcode) {
4302 case PACKET3_SET_BASE:
4303 case PACKET3_CLEAR_STATE:
4304 case PACKET3_INDEX_BUFFER_SIZE:
4305 case PACKET3_DISPATCH_DIRECT:
4306 case PACKET3_DISPATCH_INDIRECT:
4307 case PACKET3_ALLOC_GDS:
4308 case PACKET3_WRITE_GDS_RAM:
4309 case PACKET3_ATOMIC_GDS:
4310 case PACKET3_ATOMIC:
4311 case PACKET3_OCCLUSION_QUERY:
4312 case PACKET3_SET_PREDICATION:
4313 case PACKET3_COND_EXEC:
4314 case PACKET3_PRED_EXEC:
4315 case PACKET3_DRAW_INDIRECT:
4316 case PACKET3_DRAW_INDEX_INDIRECT:
4317 case PACKET3_INDEX_BASE:
4318 case PACKET3_DRAW_INDEX_2:
4319 case PACKET3_CONTEXT_CONTROL:
4320 case PACKET3_INDEX_TYPE:
4321 case PACKET3_DRAW_INDIRECT_MULTI:
4322 case PACKET3_DRAW_INDEX_AUTO:
4323 case PACKET3_DRAW_INDEX_IMMD:
4324 case PACKET3_NUM_INSTANCES:
4325 case PACKET3_DRAW_INDEX_MULTI_AUTO:
4326 case PACKET3_STRMOUT_BUFFER_UPDATE:
4327 case PACKET3_DRAW_INDEX_OFFSET_2:
4328 case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4329 case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4330 case PACKET3_MPEG_INDEX:
4331 case PACKET3_WAIT_REG_MEM:
4332 case PACKET3_MEM_WRITE:
4333 case PACKET3_PFP_SYNC_ME:
4334 case PACKET3_SURFACE_SYNC:
4335 case PACKET3_EVENT_WRITE:
4336 case PACKET3_EVENT_WRITE_EOP:
4337 case PACKET3_EVENT_WRITE_EOS:
4338 case PACKET3_SET_CONTEXT_REG:
4339 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4340 case PACKET3_SET_SH_REG:
4341 case PACKET3_SET_SH_REG_OFFSET:
4342 case PACKET3_INCREMENT_DE_COUNTER:
4343 case PACKET3_WAIT_ON_CE_COUNTER:
4344 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4345 case PACKET3_ME_WRITE:
4347 case PACKET3_COPY_DATA:
4348 if ((idx_value & 0xf00) == 0) {
4349 reg = ib[idx + 3] * 4;
4350 if (!si_vm_reg_valid(reg))
4354 case PACKET3_WRITE_DATA:
4355 if ((idx_value & 0xf00) == 0) {
4356 start_reg = ib[idx + 1] * 4;
4357 if (idx_value & 0x10000) {
4358 if (!si_vm_reg_valid(start_reg))
4361 for (i = 0; i < (pkt->count - 2); i++) {
4362 reg = start_reg + (4 * i);
4363 if (!si_vm_reg_valid(reg))
4369 case PACKET3_COND_WRITE:
4370 if (idx_value & 0x100) {
4371 reg = ib[idx + 5] * 4;
4372 if (!si_vm_reg_valid(reg))
4376 case PACKET3_COPY_DW:
4377 if (idx_value & 0x2) {
4378 reg = ib[idx + 3] * 4;
4379 if (!si_vm_reg_valid(reg))
4383 case PACKET3_SET_CONFIG_REG:
4384 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4385 end_reg = 4 * pkt->count + start_reg - 4;
4386 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4387 (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4388 (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4389 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4392 for (i = 0; i < pkt->count; i++) {
4393 reg = start_reg + (4 * i);
4394 if (!si_vm_reg_valid(reg))
4398 case PACKET3_CP_DMA:
4399 r = si_vm_packet3_cp_dma_check(ib, idx);
4404 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4410 static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4411 u32 *ib, struct radeon_cs_packet *pkt)
4414 u32 idx = pkt->idx + 1;
4415 u32 idx_value = ib[idx];
4416 u32 start_reg, reg, i;
4418 switch (pkt->opcode) {
4420 case PACKET3_SET_BASE:
4421 case PACKET3_CLEAR_STATE:
4422 case PACKET3_DISPATCH_DIRECT:
4423 case PACKET3_DISPATCH_INDIRECT:
4424 case PACKET3_ALLOC_GDS:
4425 case PACKET3_WRITE_GDS_RAM:
4426 case PACKET3_ATOMIC_GDS:
4427 case PACKET3_ATOMIC:
4428 case PACKET3_OCCLUSION_QUERY:
4429 case PACKET3_SET_PREDICATION:
4430 case PACKET3_COND_EXEC:
4431 case PACKET3_PRED_EXEC:
4432 case PACKET3_CONTEXT_CONTROL:
4433 case PACKET3_STRMOUT_BUFFER_UPDATE:
4434 case PACKET3_WAIT_REG_MEM:
4435 case PACKET3_MEM_WRITE:
4436 case PACKET3_PFP_SYNC_ME:
4437 case PACKET3_SURFACE_SYNC:
4438 case PACKET3_EVENT_WRITE:
4439 case PACKET3_EVENT_WRITE_EOP:
4440 case PACKET3_EVENT_WRITE_EOS:
4441 case PACKET3_SET_CONTEXT_REG:
4442 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4443 case PACKET3_SET_SH_REG:
4444 case PACKET3_SET_SH_REG_OFFSET:
4445 case PACKET3_INCREMENT_DE_COUNTER:
4446 case PACKET3_WAIT_ON_CE_COUNTER:
4447 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4448 case PACKET3_ME_WRITE:
4450 case PACKET3_COPY_DATA:
4451 if ((idx_value & 0xf00) == 0) {
4452 reg = ib[idx + 3] * 4;
4453 if (!si_vm_reg_valid(reg))
4457 case PACKET3_WRITE_DATA:
4458 if ((idx_value & 0xf00) == 0) {
4459 start_reg = ib[idx + 1] * 4;
4460 if (idx_value & 0x10000) {
4461 if (!si_vm_reg_valid(start_reg))
4464 for (i = 0; i < (pkt->count - 2); i++) {
4465 reg = start_reg + (4 * i);
4466 if (!si_vm_reg_valid(reg))
4472 case PACKET3_COND_WRITE:
4473 if (idx_value & 0x100) {
4474 reg = ib[idx + 5] * 4;
4475 if (!si_vm_reg_valid(reg))
4479 case PACKET3_COPY_DW:
4480 if (idx_value & 0x2) {
4481 reg = ib[idx + 3] * 4;
4482 if (!si_vm_reg_valid(reg))
4486 case PACKET3_CP_DMA:
4487 r = si_vm_packet3_cp_dma_check(ib, idx);
4492 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4498 int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4502 struct radeon_cs_packet pkt;
4506 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4507 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
4510 case RADEON_PACKET_TYPE0:
4511 dev_err(rdev->dev, "Packet0 not allowed!\n");
4514 case RADEON_PACKET_TYPE2:
4517 case RADEON_PACKET_TYPE3:
4518 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
4519 if (ib->is_const_ib)
4520 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4523 case RADEON_RING_TYPE_GFX_INDEX:
4524 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4526 case CAYMAN_RING_TYPE_CP1_INDEX:
4527 case CAYMAN_RING_TYPE_CP2_INDEX:
4528 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4531 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
4536 idx += pkt.count + 2;
4539 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4545 } while (idx < ib->length_dw);
4553 int si_vm_init(struct radeon_device *rdev)
4556 rdev->vm_manager.nvm = 16;
4557 /* base offset of vram pages */
4558 rdev->vm_manager.vram_base_offset = 0;
4563 void si_vm_fini(struct radeon_device *rdev)
4568 * si_vm_decode_fault - print human readable fault info
4570 * @rdev: radeon_device pointer
4571 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4572 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4574 * Print human readable fault information (SI).
4576 static void si_vm_decode_fault(struct radeon_device *rdev,
4577 u32 status, u32 addr)
4579 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4580 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4581 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4584 if (rdev->family == CHIP_TAHITI) {
4825 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
4826 protections, vmid, addr,
4827 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
4831 void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4833 struct radeon_ring *ring = &rdev->ring[ridx];
4838 /* write new base address */
4839 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4840 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
4841 WRITE_DATA_DST_SEL(0)));
4844 radeon_ring_write(ring,
4845 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
4847 radeon_ring_write(ring,
4848 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
4850 radeon_ring_write(ring, 0);
4851 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4853 /* flush hdp cache */
4854 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4855 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
4856 WRITE_DATA_DST_SEL(0)));
4857 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4858 radeon_ring_write(ring, 0);
4859 radeon_ring_write(ring, 0x1);
4861 /* bits 0-15 are the VM contexts0-15 */
4862 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4863 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
4864 WRITE_DATA_DST_SEL(0)));
4865 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4866 radeon_ring_write(ring, 0);
4867 radeon_ring_write(ring, 1 << vm->id);
4869 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4870 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4871 radeon_ring_write(ring, 0x0);
4875 * Power and clock gating
4877 static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
4881 for (i = 0; i < rdev->usec_timeout; i++) {
4882 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
4887 for (i = 0; i < rdev->usec_timeout; i++) {
4888 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
4894 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
4897 u32 tmp = RREG32(CP_INT_CNTL_RING0);
4902 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4904 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4905 WREG32(CP_INT_CNTL_RING0, tmp);
4908 /* read a gfx register */
4909 tmp = RREG32(DB_DEPTH_INFO);
4911 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
4912 for (i = 0; i < rdev->usec_timeout; i++) {
4913 if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
4920 static void si_set_uvd_dcm(struct radeon_device *rdev,
4925 tmp = RREG32(UVD_CGC_CTRL);
4926 tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
4927 tmp |= DCM | CG_DT(1) | CLK_OD(4);
4931 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
4937 WREG32(UVD_CGC_CTRL, tmp);
4938 WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
4941 void si_init_uvd_internal_cg(struct radeon_device *rdev)
4943 bool hw_mode = true;
4946 si_set_uvd_dcm(rdev, false);
4948 u32 tmp = RREG32(UVD_CGC_CTRL);
4950 WREG32(UVD_CGC_CTRL, tmp);
4954 static u32 si_halt_rlc(struct radeon_device *rdev)
4958 orig = data = RREG32(RLC_CNTL);
4960 if (data & RLC_ENABLE) {
4961 data &= ~RLC_ENABLE;
4962 WREG32(RLC_CNTL, data);
4964 si_wait_for_rlc_serdes(rdev);
4970 static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
4974 tmp = RREG32(RLC_CNTL);
4976 WREG32(RLC_CNTL, rlc);
4979 static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
4983 orig = data = RREG32(DMA_PG);
4984 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
4985 data |= PG_CNTL_ENABLE;
4987 data &= ~PG_CNTL_ENABLE;
4989 WREG32(DMA_PG, data);
4992 static void si_init_dma_pg(struct radeon_device *rdev)
4996 WREG32(DMA_PGFSM_WRITE, 0x00002000);
4997 WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
4999 for (tmp = 0; tmp < 5; tmp++)
5000 WREG32(DMA_PGFSM_WRITE, 0);
5003 static void si_enable_gfx_cgpg(struct radeon_device *rdev,
5008 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
5009 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
5010 WREG32(RLC_TTOP_D, tmp);
5012 tmp = RREG32(RLC_PG_CNTL);
5013 tmp |= GFX_PG_ENABLE;
5014 WREG32(RLC_PG_CNTL, tmp);
5016 tmp = RREG32(RLC_AUTO_PG_CTRL);
5018 WREG32(RLC_AUTO_PG_CTRL, tmp);
5020 tmp = RREG32(RLC_AUTO_PG_CTRL);
5022 WREG32(RLC_AUTO_PG_CTRL, tmp);
5024 tmp = RREG32(DB_RENDER_CONTROL);
5028 static void si_init_gfx_cgpg(struct radeon_device *rdev)
5032 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5034 tmp = RREG32(RLC_PG_CNTL);
5036 WREG32(RLC_PG_CNTL, tmp);
5038 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5040 tmp = RREG32(RLC_AUTO_PG_CTRL);
5042 tmp &= ~GRBM_REG_SGIT_MASK;
5043 tmp |= GRBM_REG_SGIT(0x700);
5044 tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
5045 WREG32(RLC_AUTO_PG_CTRL, tmp);
5048 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
5050 u32 mask = 0, tmp, tmp1;
5053 si_select_se_sh(rdev, se, sh);
5054 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
5055 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
5056 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5063 for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
5068 return (~tmp) & mask;
5071 static void si_init_ao_cu_mask(struct radeon_device *rdev)
5073 u32 i, j, k, active_cu_number = 0;
5074 u32 mask, counter, cu_bitmap;
5077 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
5078 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
5082 for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
5083 if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
5091 active_cu_number += counter;
5092 tmp |= (cu_bitmap << (i * 16 + j * 8));
5096 WREG32(RLC_PG_AO_CU_MASK, tmp);
5098 tmp = RREG32(RLC_MAX_PG_CU);
5099 tmp &= ~MAX_PU_CU_MASK;
5100 tmp |= MAX_PU_CU(active_cu_number);
5101 WREG32(RLC_MAX_PG_CU, tmp);
5104 static void si_enable_cgcg(struct radeon_device *rdev,
5107 u32 data, orig, tmp;
5109 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5111 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5112 si_enable_gui_idle_interrupt(rdev, true);
5114 WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
5116 tmp = si_halt_rlc(rdev);
5118 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5119 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5120 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
5122 si_wait_for_rlc_serdes(rdev);
5124 si_update_rlc(rdev, tmp);
5126 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
5128 data |= CGCG_EN | CGLS_EN;
5130 si_enable_gui_idle_interrupt(rdev, false);
5132 RREG32(CB_CGTT_SCLK_CTRL);
5133 RREG32(CB_CGTT_SCLK_CTRL);
5134 RREG32(CB_CGTT_SCLK_CTRL);
5135 RREG32(CB_CGTT_SCLK_CTRL);
5137 data &= ~(CGCG_EN | CGLS_EN);
5141 WREG32(RLC_CGCG_CGLS_CTRL, data);
5144 static void si_enable_mgcg(struct radeon_device *rdev,
5147 u32 data, orig, tmp = 0;
5149 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5150 orig = data = RREG32(CGTS_SM_CTRL_REG);
5153 WREG32(CGTS_SM_CTRL_REG, data);
5155 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5156 orig = data = RREG32(CP_MEM_SLP_CNTL);
5157 data |= CP_MEM_LS_EN;
5159 WREG32(CP_MEM_SLP_CNTL, data);
5162 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5165 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5167 tmp = si_halt_rlc(rdev);
5169 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5170 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5171 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5173 si_update_rlc(rdev, tmp);
5175 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5178 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5180 data = RREG32(CP_MEM_SLP_CNTL);
5181 if (data & CP_MEM_LS_EN) {
5182 data &= ~CP_MEM_LS_EN;
5183 WREG32(CP_MEM_SLP_CNTL, data);
5185 orig = data = RREG32(CGTS_SM_CTRL_REG);
5186 data |= LS_OVERRIDE | OVERRIDE;
5188 WREG32(CGTS_SM_CTRL_REG, data);
5190 tmp = si_halt_rlc(rdev);
5192 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5193 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5194 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5196 si_update_rlc(rdev, tmp);
5200 static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5203 u32 orig, data, tmp;
5205 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
5206 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5208 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5210 orig = data = RREG32(UVD_CGC_CTRL);
5213 WREG32(UVD_CGC_CTRL, data);
5215 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5216 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5218 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5220 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5222 orig = data = RREG32(UVD_CGC_CTRL);
5225 WREG32(UVD_CGC_CTRL, data);
5227 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5228 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5232 static const u32 mc_cg_registers[] =
5245 static void si_enable_mc_ls(struct radeon_device *rdev,
5251 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5252 orig = data = RREG32(mc_cg_registers[i]);
5253 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5254 data |= MC_LS_ENABLE;
5256 data &= ~MC_LS_ENABLE;
5258 WREG32(mc_cg_registers[i], data);
5262 static void si_enable_mc_mgcg(struct radeon_device *rdev,
5268 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5269 orig = data = RREG32(mc_cg_registers[i]);
5270 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5271 data |= MC_CG_ENABLE;
5273 data &= ~MC_CG_ENABLE;
5275 WREG32(mc_cg_registers[i], data);
5279 static void si_enable_dma_mgcg(struct radeon_device *rdev,
5282 u32 orig, data, offset;
5285 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5286 for (i = 0; i < 2; i++) {
5288 offset = DMA0_REGISTER_OFFSET;
5290 offset = DMA1_REGISTER_OFFSET;
5291 orig = data = RREG32(DMA_POWER_CNTL + offset);
5292 data &= ~MEM_POWER_OVERRIDE;
5294 WREG32(DMA_POWER_CNTL + offset, data);
5295 WREG32(DMA_CLK_CTRL + offset, 0x00000100);
5298 for (i = 0; i < 2; i++) {
5300 offset = DMA0_REGISTER_OFFSET;
5302 offset = DMA1_REGISTER_OFFSET;
5303 orig = data = RREG32(DMA_POWER_CNTL + offset);
5304 data |= MEM_POWER_OVERRIDE;
5306 WREG32(DMA_POWER_CNTL + offset, data);
5308 orig = data = RREG32(DMA_CLK_CTRL + offset);
5311 WREG32(DMA_CLK_CTRL + offset, data);
5316 static void si_enable_bif_mgls(struct radeon_device *rdev,
5321 orig = data = RREG32_PCIE(PCIE_CNTL2);
5323 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5324 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5325 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5327 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5328 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5331 WREG32_PCIE(PCIE_CNTL2, data);
5334 static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5339 orig = data = RREG32(HDP_HOST_PATH_CNTL);
5341 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5342 data &= ~CLOCK_GATING_DIS;
5344 data |= CLOCK_GATING_DIS;
5347 WREG32(HDP_HOST_PATH_CNTL, data);
5350 static void si_enable_hdp_ls(struct radeon_device *rdev,
5355 orig = data = RREG32(HDP_MEM_POWER_LS);
5357 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5358 data |= HDP_LS_ENABLE;
5360 data &= ~HDP_LS_ENABLE;
5363 WREG32(HDP_MEM_POWER_LS, data);
5366 static void si_update_cg(struct radeon_device *rdev,
5367 u32 block, bool enable)
5369 if (block & RADEON_CG_BLOCK_GFX) {
5370 si_enable_gui_idle_interrupt(rdev, false);
5371 /* order matters! */
5373 si_enable_mgcg(rdev, true);
5374 si_enable_cgcg(rdev, true);
5376 si_enable_cgcg(rdev, false);
5377 si_enable_mgcg(rdev, false);
5379 si_enable_gui_idle_interrupt(rdev, true);
5382 if (block & RADEON_CG_BLOCK_MC) {
5383 si_enable_mc_mgcg(rdev, enable);
5384 si_enable_mc_ls(rdev, enable);
5387 if (block & RADEON_CG_BLOCK_SDMA) {
5388 si_enable_dma_mgcg(rdev, enable);
5391 if (block & RADEON_CG_BLOCK_BIF) {
5392 si_enable_bif_mgls(rdev, enable);
5395 if (block & RADEON_CG_BLOCK_UVD) {
5396 if (rdev->has_uvd) {
5397 si_enable_uvd_mgcg(rdev, enable);
5401 if (block & RADEON_CG_BLOCK_HDP) {
5402 si_enable_hdp_mgcg(rdev, enable);
5403 si_enable_hdp_ls(rdev, enable);
5407 static void si_init_cg(struct radeon_device *rdev)
5409 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5410 RADEON_CG_BLOCK_MC |
5411 RADEON_CG_BLOCK_SDMA |
5412 RADEON_CG_BLOCK_BIF |
5413 RADEON_CG_BLOCK_HDP), true);
5414 if (rdev->has_uvd) {
5415 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
5416 si_init_uvd_internal_cg(rdev);
5420 static void si_fini_cg(struct radeon_device *rdev)
5422 if (rdev->has_uvd) {
5423 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
5425 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5426 RADEON_CG_BLOCK_MC |
5427 RADEON_CG_BLOCK_SDMA |
5428 RADEON_CG_BLOCK_BIF |
5429 RADEON_CG_BLOCK_HDP), false);
5432 u32 si_get_csb_size(struct radeon_device *rdev)
5435 const struct cs_section_def *sect = NULL;
5436 const struct cs_extent_def *ext = NULL;
5438 if (rdev->rlc.cs_data == NULL)
5441 /* begin clear state */
5443 /* context control state */
5446 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5447 for (ext = sect->section; ext->extent != NULL; ++ext) {
5448 if (sect->id == SECT_CONTEXT)
5449 count += 2 + ext->reg_count;
5454 /* pa_sc_raster_config */
5456 /* end clear state */
5464 void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5467 const struct cs_section_def *sect = NULL;
5468 const struct cs_extent_def *ext = NULL;
5470 if (rdev->rlc.cs_data == NULL)
5475 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5476 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
5478 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5479 buffer[count++] = cpu_to_le32(0x80000000);
5480 buffer[count++] = cpu_to_le32(0x80000000);
5482 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5483 for (ext = sect->section; ext->extent != NULL; ++ext) {
5484 if (sect->id == SECT_CONTEXT) {
5486 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
5487 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
5488 for (i = 0; i < ext->reg_count; i++)
5489 buffer[count++] = cpu_to_le32(ext->extent[i]);
5496 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
5497 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
5498 switch (rdev->family) {
5501 buffer[count++] = cpu_to_le32(0x2a00126a);
5504 buffer[count++] = cpu_to_le32(0x0000124a);
5507 buffer[count++] = cpu_to_le32(0x00000082);
5510 buffer[count++] = cpu_to_le32(0x00000000);
5513 buffer[count++] = cpu_to_le32(0x00000000);
5517 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5518 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
5520 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
5521 buffer[count++] = cpu_to_le32(0);
5524 static void si_init_pg(struct radeon_device *rdev)
5526 if (rdev->pg_flags) {
5527 if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5528 si_init_dma_pg(rdev);
5530 si_init_ao_cu_mask(rdev);
5531 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
5532 si_init_gfx_cgpg(rdev);
5534 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5535 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5537 si_enable_dma_pg(rdev, true);
5538 si_enable_gfx_cgpg(rdev, true);
5540 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5541 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5545 static void si_fini_pg(struct radeon_device *rdev)
5547 if (rdev->pg_flags) {
5548 si_enable_dma_pg(rdev, false);
5549 si_enable_gfx_cgpg(rdev, false);
5556 void si_rlc_reset(struct radeon_device *rdev)
5558 u32 tmp = RREG32(GRBM_SOFT_RESET);
5560 tmp |= SOFT_RESET_RLC;
5561 WREG32(GRBM_SOFT_RESET, tmp);
5563 tmp &= ~SOFT_RESET_RLC;
5564 WREG32(GRBM_SOFT_RESET, tmp);
5568 static void si_rlc_stop(struct radeon_device *rdev)
5570 WREG32(RLC_CNTL, 0);
5572 si_enable_gui_idle_interrupt(rdev, false);
5574 si_wait_for_rlc_serdes(rdev);
5577 static void si_rlc_start(struct radeon_device *rdev)
5579 WREG32(RLC_CNTL, RLC_ENABLE);
5581 si_enable_gui_idle_interrupt(rdev, true);
5586 static bool si_lbpw_supported(struct radeon_device *rdev)
5590 /* Enable LBPW only for DDR3 */
5591 tmp = RREG32(MC_SEQ_MISC0);
5592 if ((tmp & 0xF0000000) == 0xB0000000)
5597 static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5601 tmp = RREG32(RLC_LB_CNTL);
5603 tmp |= LOAD_BALANCE_ENABLE;
5605 tmp &= ~LOAD_BALANCE_ENABLE;
5606 WREG32(RLC_LB_CNTL, tmp);
5609 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5610 WREG32(SPI_LB_CU_MASK, 0x00ff);
5614 static int si_rlc_resume(struct radeon_device *rdev)
5617 const __be32 *fw_data;
5630 WREG32(RLC_RL_BASE, 0);
5631 WREG32(RLC_RL_SIZE, 0);
5632 WREG32(RLC_LB_CNTL, 0);
5633 WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5634 WREG32(RLC_LB_CNTR_INIT, 0);
5635 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5637 WREG32(RLC_MC_CNTL, 0);
5638 WREG32(RLC_UCODE_CNTL, 0);
5640 fw_data = (const __be32 *)rdev->rlc_fw->data;
5641 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5642 WREG32(RLC_UCODE_ADDR, i);
5643 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5645 WREG32(RLC_UCODE_ADDR, 0);
5647 si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5654 static void si_enable_interrupts(struct radeon_device *rdev)
5656 u32 ih_cntl = RREG32(IH_CNTL);
5657 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5659 ih_cntl |= ENABLE_INTR;
5660 ih_rb_cntl |= IH_RB_ENABLE;
5661 WREG32(IH_CNTL, ih_cntl);
5662 WREG32(IH_RB_CNTL, ih_rb_cntl);
5663 rdev->ih.enabled = true;
5666 static void si_disable_interrupts(struct radeon_device *rdev)
5668 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5669 u32 ih_cntl = RREG32(IH_CNTL);
5671 ih_rb_cntl &= ~IH_RB_ENABLE;
5672 ih_cntl &= ~ENABLE_INTR;
5673 WREG32(IH_RB_CNTL, ih_rb_cntl);
5674 WREG32(IH_CNTL, ih_cntl);
5675 /* set rptr, wptr to 0 */
5676 WREG32(IH_RB_RPTR, 0);
5677 WREG32(IH_RB_WPTR, 0);
5678 rdev->ih.enabled = false;
5682 static void si_disable_interrupt_state(struct radeon_device *rdev)
5686 tmp = RREG32(CP_INT_CNTL_RING0) &
5687 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5688 WREG32(CP_INT_CNTL_RING0, tmp);
5689 WREG32(CP_INT_CNTL_RING1, 0);
5690 WREG32(CP_INT_CNTL_RING2, 0);
5691 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5692 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5693 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5694 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
5695 WREG32(GRBM_INT_CNTL, 0);
5696 if (rdev->num_crtc >= 2) {
5697 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5698 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5700 if (rdev->num_crtc >= 4) {
5701 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5702 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5704 if (rdev->num_crtc >= 6) {
5705 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5706 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5709 if (rdev->num_crtc >= 2) {
5710 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5711 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5713 if (rdev->num_crtc >= 4) {
5714 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5715 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5717 if (rdev->num_crtc >= 6) {
5718 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5719 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5722 if (!ASIC_IS_NODCE(rdev)) {
5723 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
5725 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5726 WREG32(DC_HPD1_INT_CONTROL, tmp);
5727 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5728 WREG32(DC_HPD2_INT_CONTROL, tmp);
5729 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5730 WREG32(DC_HPD3_INT_CONTROL, tmp);
5731 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5732 WREG32(DC_HPD4_INT_CONTROL, tmp);
5733 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5734 WREG32(DC_HPD5_INT_CONTROL, tmp);
5735 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5736 WREG32(DC_HPD6_INT_CONTROL, tmp);
5740 static int si_irq_init(struct radeon_device *rdev)
5744 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5747 ret = r600_ih_ring_alloc(rdev);
5752 si_disable_interrupts(rdev);
5755 ret = si_rlc_resume(rdev);
5757 r600_ih_ring_fini(rdev);
5761 /* setup interrupt control */
5762 /* set dummy read address to ring address */
5763 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
5764 interrupt_cntl = RREG32(INTERRUPT_CNTL);
5765 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5766 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5768 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5769 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5770 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5771 WREG32(INTERRUPT_CNTL, interrupt_cntl);
5773 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
5774 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
5776 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5777 IH_WPTR_OVERFLOW_CLEAR |
5780 if (rdev->wb.enabled)
5781 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
5783 /* set the writeback address whether it's enabled or not */
5784 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
5785 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
5787 WREG32(IH_RB_CNTL, ih_rb_cntl);
5789 /* set rptr, wptr to 0 */
5790 WREG32(IH_RB_RPTR, 0);
5791 WREG32(IH_RB_WPTR, 0);
5793 /* Default settings for IH_CNTL (disabled at first) */
5794 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
5795 /* RPTR_REARM only works if msi's are enabled */
5796 if (rdev->msi_enabled)
5797 ih_cntl |= RPTR_REARM;
5798 WREG32(IH_CNTL, ih_cntl);
5800 /* force the active interrupt state to all disabled */
5801 si_disable_interrupt_state(rdev);
5803 pci_enable_busmaster(rdev->dev);
5806 si_enable_interrupts(rdev);
5811 int si_irq_set(struct radeon_device *rdev)
5814 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
5815 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
5816 u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
5817 u32 grbm_int_cntl = 0;
5818 u32 dma_cntl, dma_cntl1;
5819 u32 thermal_int = 0;
5821 if (!rdev->irq.installed) {
5822 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
5825 /* don't enable anything if the ih is disabled */
5826 if (!rdev->ih.enabled) {
5827 si_disable_interrupts(rdev);
5828 /* force the active interrupt state to all disabled */
5829 si_disable_interrupt_state(rdev);
5833 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
5834 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5836 if (!ASIC_IS_NODCE(rdev)) {
5837 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
5838 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
5839 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
5840 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
5841 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
5842 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
5845 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5846 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5848 thermal_int = RREG32(CG_THERMAL_INT) &
5849 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
5851 /* enable CP interrupts on all rings */
5852 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
5853 DRM_DEBUG("si_irq_set: sw int gfx\n");
5854 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
5856 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
5857 DRM_DEBUG("si_irq_set: sw int cp1\n");
5858 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
5860 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
5861 DRM_DEBUG("si_irq_set: sw int cp2\n");
5862 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
5864 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
5865 DRM_DEBUG("si_irq_set: sw int dma\n");
5866 dma_cntl |= TRAP_ENABLE;
5869 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
5870 DRM_DEBUG("si_irq_set: sw int dma1\n");
5871 dma_cntl1 |= TRAP_ENABLE;
5873 if (rdev->irq.crtc_vblank_int[0] ||
5874 atomic_read(&rdev->irq.pflip[0])) {
5875 DRM_DEBUG("si_irq_set: vblank 0\n");
5876 crtc1 |= VBLANK_INT_MASK;
5878 if (rdev->irq.crtc_vblank_int[1] ||
5879 atomic_read(&rdev->irq.pflip[1])) {
5880 DRM_DEBUG("si_irq_set: vblank 1\n");
5881 crtc2 |= VBLANK_INT_MASK;
5883 if (rdev->irq.crtc_vblank_int[2] ||
5884 atomic_read(&rdev->irq.pflip[2])) {
5885 DRM_DEBUG("si_irq_set: vblank 2\n");
5886 crtc3 |= VBLANK_INT_MASK;
5888 if (rdev->irq.crtc_vblank_int[3] ||
5889 atomic_read(&rdev->irq.pflip[3])) {
5890 DRM_DEBUG("si_irq_set: vblank 3\n");
5891 crtc4 |= VBLANK_INT_MASK;
5893 if (rdev->irq.crtc_vblank_int[4] ||
5894 atomic_read(&rdev->irq.pflip[4])) {
5895 DRM_DEBUG("si_irq_set: vblank 4\n");
5896 crtc5 |= VBLANK_INT_MASK;
5898 if (rdev->irq.crtc_vblank_int[5] ||
5899 atomic_read(&rdev->irq.pflip[5])) {
5900 DRM_DEBUG("si_irq_set: vblank 5\n");
5901 crtc6 |= VBLANK_INT_MASK;
5903 if (rdev->irq.hpd[0]) {
5904 DRM_DEBUG("si_irq_set: hpd 1\n");
5905 hpd1 |= DC_HPDx_INT_EN;
5907 if (rdev->irq.hpd[1]) {
5908 DRM_DEBUG("si_irq_set: hpd 2\n");
5909 hpd2 |= DC_HPDx_INT_EN;
5911 if (rdev->irq.hpd[2]) {
5912 DRM_DEBUG("si_irq_set: hpd 3\n");
5913 hpd3 |= DC_HPDx_INT_EN;
5915 if (rdev->irq.hpd[3]) {
5916 DRM_DEBUG("si_irq_set: hpd 4\n");
5917 hpd4 |= DC_HPDx_INT_EN;
5919 if (rdev->irq.hpd[4]) {
5920 DRM_DEBUG("si_irq_set: hpd 5\n");
5921 hpd5 |= DC_HPDx_INT_EN;
5923 if (rdev->irq.hpd[5]) {
5924 DRM_DEBUG("si_irq_set: hpd 6\n");
5925 hpd6 |= DC_HPDx_INT_EN;
5928 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
5929 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
5930 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
5932 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
5933 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
5935 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
5937 if (rdev->irq.dpm_thermal) {
5938 DRM_DEBUG("dpm thermal\n");
5939 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
5942 if (rdev->num_crtc >= 2) {
5943 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
5944 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
5946 if (rdev->num_crtc >= 4) {
5947 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
5948 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
5950 if (rdev->num_crtc >= 6) {
5951 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
5952 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
5955 if (rdev->num_crtc >= 2) {
5956 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
5957 GRPH_PFLIP_INT_MASK);
5958 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
5959 GRPH_PFLIP_INT_MASK);
5961 if (rdev->num_crtc >= 4) {
5962 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
5963 GRPH_PFLIP_INT_MASK);
5964 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
5965 GRPH_PFLIP_INT_MASK);
5967 if (rdev->num_crtc >= 6) {
5968 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
5969 GRPH_PFLIP_INT_MASK);
5970 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
5971 GRPH_PFLIP_INT_MASK);
5974 if (!ASIC_IS_NODCE(rdev)) {
5975 WREG32(DC_HPD1_INT_CONTROL, hpd1);
5976 WREG32(DC_HPD2_INT_CONTROL, hpd2);
5977 WREG32(DC_HPD3_INT_CONTROL, hpd3);
5978 WREG32(DC_HPD4_INT_CONTROL, hpd4);
5979 WREG32(DC_HPD5_INT_CONTROL, hpd5);
5980 WREG32(DC_HPD6_INT_CONTROL, hpd6);
5983 WREG32(CG_THERMAL_INT, thermal_int);
5988 static inline void si_irq_ack(struct radeon_device *rdev)
5992 if (ASIC_IS_NODCE(rdev))
5995 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
5996 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
5997 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
5998 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
5999 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
6000 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
6001 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
6002 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
6003 if (rdev->num_crtc >= 4) {
6004 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
6005 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
6007 if (rdev->num_crtc >= 6) {
6008 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
6009 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
6012 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
6013 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6014 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
6015 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6016 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
6017 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6018 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
6019 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6020 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
6021 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6022 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
6023 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
6025 if (rdev->num_crtc >= 4) {
6026 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
6027 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6028 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
6029 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6030 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
6031 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
6032 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
6033 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
6034 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
6035 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
6036 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
6037 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
6040 if (rdev->num_crtc >= 6) {
6041 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
6042 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6043 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
6044 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6045 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
6046 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
6047 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
6048 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
6049 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
6050 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
6051 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
6052 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
6055 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6056 tmp = RREG32(DC_HPD1_INT_CONTROL);
6057 tmp |= DC_HPDx_INT_ACK;
6058 WREG32(DC_HPD1_INT_CONTROL, tmp);
6060 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6061 tmp = RREG32(DC_HPD2_INT_CONTROL);
6062 tmp |= DC_HPDx_INT_ACK;
6063 WREG32(DC_HPD2_INT_CONTROL, tmp);
6065 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6066 tmp = RREG32(DC_HPD3_INT_CONTROL);
6067 tmp |= DC_HPDx_INT_ACK;
6068 WREG32(DC_HPD3_INT_CONTROL, tmp);
6070 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6071 tmp = RREG32(DC_HPD4_INT_CONTROL);
6072 tmp |= DC_HPDx_INT_ACK;
6073 WREG32(DC_HPD4_INT_CONTROL, tmp);
6075 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6076 tmp = RREG32(DC_HPD5_INT_CONTROL);
6077 tmp |= DC_HPDx_INT_ACK;
6078 WREG32(DC_HPD5_INT_CONTROL, tmp);
6080 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6081 tmp = RREG32(DC_HPD5_INT_CONTROL);
6082 tmp |= DC_HPDx_INT_ACK;
6083 WREG32(DC_HPD6_INT_CONTROL, tmp);
6087 static void si_irq_disable(struct radeon_device *rdev)
6089 si_disable_interrupts(rdev);
6090 /* Wait and acknowledge irq */
6093 si_disable_interrupt_state(rdev);
6096 static void si_irq_suspend(struct radeon_device *rdev)
6098 si_irq_disable(rdev);
6102 static void si_irq_fini(struct radeon_device *rdev)
6104 si_irq_suspend(rdev);
6105 r600_ih_ring_fini(rdev);
6108 static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
6112 if (rdev->wb.enabled)
6113 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6115 wptr = RREG32(IH_RB_WPTR);
6117 if (wptr & RB_OVERFLOW) {
6118 wptr &= ~RB_OVERFLOW;
6119 /* When a ring buffer overflow happen start parsing interrupt
6120 * from the last not overwritten vector (wptr + 16). Hopefully
6121 * this should allow us to catchup.
6123 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
6124 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
6125 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6126 tmp = RREG32(IH_RB_CNTL);
6127 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6128 WREG32(IH_RB_CNTL, tmp);
6130 return (wptr & rdev->ih.ptr_mask);
6134 * Each IV ring entry is 128 bits:
6135 * [7:0] - interrupt source id
6137 * [59:32] - interrupt source data
6138 * [63:60] - reserved
6141 * [127:80] - reserved
6143 irqreturn_t si_irq_process(struct radeon_device *rdev)
6147 u32 src_id, src_data, ring_id;
6149 bool queue_hotplug = false;
6150 bool queue_thermal = false;
6153 if (!rdev->ih.enabled || rdev->shutdown)
6156 wptr = si_get_ih_wptr(rdev);
6159 /* is somebody else already processing irqs? */
6160 if (atomic_xchg(&rdev->ih.lock, 1))
6163 rptr = rdev->ih.rptr;
6164 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6166 /* Order reading of wptr vs. reading of IH ring data */
6169 /* display interrupts */
6172 while (rptr != wptr) {
6173 /* wptr/rptr are in bytes! */
6174 ring_index = rptr / 4;
6175 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6176 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6177 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6180 case 1: /* D1 vblank/vline */
6182 case 0: /* D1 vblank */
6183 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6184 if (rdev->irq.crtc_vblank_int[0]) {
6185 drm_handle_vblank(rdev->ddev, 0);
6186 rdev->pm.vblank_sync = true;
6187 wake_up(&rdev->irq.vblank_queue);
6189 if (atomic_read(&rdev->irq.pflip[0]))
6190 radeon_crtc_handle_vblank(rdev, 0);
6191 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
6192 DRM_DEBUG("IH: D1 vblank\n");
6195 case 1: /* D1 vline */
6196 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
6197 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
6198 DRM_DEBUG("IH: D1 vline\n");
6202 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6206 case 2: /* D2 vblank/vline */
6208 case 0: /* D2 vblank */
6209 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6210 if (rdev->irq.crtc_vblank_int[1]) {
6211 drm_handle_vblank(rdev->ddev, 1);
6212 rdev->pm.vblank_sync = true;
6213 wake_up(&rdev->irq.vblank_queue);
6215 if (atomic_read(&rdev->irq.pflip[1]))
6216 radeon_crtc_handle_vblank(rdev, 1);
6217 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
6218 DRM_DEBUG("IH: D2 vblank\n");
6221 case 1: /* D2 vline */
6222 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
6223 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
6224 DRM_DEBUG("IH: D2 vline\n");
6228 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6232 case 3: /* D3 vblank/vline */
6234 case 0: /* D3 vblank */
6235 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
6236 if (rdev->irq.crtc_vblank_int[2]) {
6237 drm_handle_vblank(rdev->ddev, 2);
6238 rdev->pm.vblank_sync = true;
6239 wake_up(&rdev->irq.vblank_queue);
6241 if (atomic_read(&rdev->irq.pflip[2]))
6242 radeon_crtc_handle_vblank(rdev, 2);
6243 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
6244 DRM_DEBUG("IH: D3 vblank\n");
6247 case 1: /* D3 vline */
6248 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
6249 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
6250 DRM_DEBUG("IH: D3 vline\n");
6254 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6258 case 4: /* D4 vblank/vline */
6260 case 0: /* D4 vblank */
6261 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
6262 if (rdev->irq.crtc_vblank_int[3]) {
6263 drm_handle_vblank(rdev->ddev, 3);
6264 rdev->pm.vblank_sync = true;
6265 wake_up(&rdev->irq.vblank_queue);
6267 if (atomic_read(&rdev->irq.pflip[3]))
6268 radeon_crtc_handle_vblank(rdev, 3);
6269 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
6270 DRM_DEBUG("IH: D4 vblank\n");
6273 case 1: /* D4 vline */
6274 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
6275 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
6276 DRM_DEBUG("IH: D4 vline\n");
6280 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6284 case 5: /* D5 vblank/vline */
6286 case 0: /* D5 vblank */
6287 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
6288 if (rdev->irq.crtc_vblank_int[4]) {
6289 drm_handle_vblank(rdev->ddev, 4);
6290 rdev->pm.vblank_sync = true;
6291 wake_up(&rdev->irq.vblank_queue);
6293 if (atomic_read(&rdev->irq.pflip[4]))
6294 radeon_crtc_handle_vblank(rdev, 4);
6295 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6296 DRM_DEBUG("IH: D5 vblank\n");
6299 case 1: /* D5 vline */
6300 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
6301 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6302 DRM_DEBUG("IH: D5 vline\n");
6306 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6310 case 6: /* D6 vblank/vline */
6312 case 0: /* D6 vblank */
6313 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
6314 if (rdev->irq.crtc_vblank_int[5]) {
6315 drm_handle_vblank(rdev->ddev, 5);
6316 rdev->pm.vblank_sync = true;
6317 wake_up(&rdev->irq.vblank_queue);
6319 if (atomic_read(&rdev->irq.pflip[5]))
6320 radeon_crtc_handle_vblank(rdev, 5);
6321 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6322 DRM_DEBUG("IH: D6 vblank\n");
6325 case 1: /* D6 vline */
6326 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
6327 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6328 DRM_DEBUG("IH: D6 vline\n");
6332 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6336 case 8: /* D1 page flip */
6337 case 10: /* D2 page flip */
6338 case 12: /* D3 page flip */
6339 case 14: /* D4 page flip */
6340 case 16: /* D5 page flip */
6341 case 18: /* D6 page flip */
6342 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
6343 if (radeon_use_pflipirq > 0)
6344 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
6346 case 42: /* HPD hotplug */
6349 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6350 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
6351 queue_hotplug = true;
6352 DRM_DEBUG("IH: HPD1\n");
6356 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6357 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6358 queue_hotplug = true;
6359 DRM_DEBUG("IH: HPD2\n");
6363 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6364 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6365 queue_hotplug = true;
6366 DRM_DEBUG("IH: HPD3\n");
6370 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6371 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6372 queue_hotplug = true;
6373 DRM_DEBUG("IH: HPD4\n");
6377 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6378 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6379 queue_hotplug = true;
6380 DRM_DEBUG("IH: HPD5\n");
6384 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6385 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6386 queue_hotplug = true;
6387 DRM_DEBUG("IH: HPD6\n");
6391 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6396 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
6397 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
6401 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6402 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
6403 /* reset addr and status */
6404 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6405 if (addr == 0x0 && status == 0x0)
6407 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6408 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
6410 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
6412 si_vm_decode_fault(rdev, status, addr);
6414 case 176: /* RINGID0 CP_INT */
6415 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6417 case 177: /* RINGID1 CP_INT */
6418 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6420 case 178: /* RINGID2 CP_INT */
6421 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6423 case 181: /* CP EOP event */
6424 DRM_DEBUG("IH: CP EOP\n");
6427 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6430 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6433 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6437 case 224: /* DMA trap event */
6438 DRM_DEBUG("IH: DMA trap\n");
6439 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6441 case 230: /* thermal low to high */
6442 DRM_DEBUG("IH: thermal low to high\n");
6443 rdev->pm.dpm.thermal.high_to_low = false;
6444 queue_thermal = true;
6446 case 231: /* thermal high to low */
6447 DRM_DEBUG("IH: thermal high to low\n");
6448 rdev->pm.dpm.thermal.high_to_low = true;
6449 queue_thermal = true;
6451 case 233: /* GUI IDLE */
6452 DRM_DEBUG("IH: GUI idle\n");
6454 case 244: /* DMA trap event */
6455 DRM_DEBUG("IH: DMA1 trap\n");
6456 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6459 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6463 /* wptr/rptr are in bytes! */
6465 rptr &= rdev->ih.ptr_mask;
6466 WREG32(IH_RB_RPTR, rptr);
6469 taskqueue_enqueue(rdev->tq, &rdev->hotplug_work);
6470 if (queue_thermal && rdev->pm.dpm_enabled)
6471 taskqueue_enqueue(rdev->tq, &rdev->pm.dpm.thermal.work);
6472 rdev->ih.rptr = rptr;
6473 atomic_set(&rdev->ih.lock, 0);
6475 /* make sure wptr hasn't changed while processing */
6476 wptr = si_get_ih_wptr(rdev);
6484 * startup/shutdown callbacks
6486 static int si_startup(struct radeon_device *rdev)
6488 struct radeon_ring *ring;
6491 /* enable pcie gen2/3 link */
6492 si_pcie_gen3_enable(rdev);
6494 si_program_aspm(rdev);
6496 /* scratch needs to be initialized before MC */
6497 r = r600_vram_scratch_init(rdev);
6501 si_mc_program(rdev);
6503 if (!rdev->pm.dpm_enabled) {
6504 r = si_mc_load_microcode(rdev);
6506 DRM_ERROR("Failed to load MC firmware!\n");
6511 r = si_pcie_gart_enable(rdev);
6516 /* allocate rlc buffers */
6517 if (rdev->family == CHIP_VERDE) {
6518 rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6519 rdev->rlc.reg_list_size =
6520 (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
6522 rdev->rlc.cs_data = si_cs_data;
6523 r = sumo_rlc_init(rdev);
6525 DRM_ERROR("Failed to init rlc BOs!\n");
6529 /* allocate wb buffer */
6530 r = radeon_wb_init(rdev);
6534 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6536 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6540 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6542 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6546 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6548 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6552 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6554 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6558 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6560 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6564 if (rdev->has_uvd) {
6565 r = uvd_v2_2_resume(rdev);
6567 r = radeon_fence_driver_start_ring(rdev,
6568 R600_RING_TYPE_UVD_INDEX);
6570 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
6573 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
6577 if (!rdev->irq.installed) {
6578 r = radeon_irq_kms_init(rdev);
6583 r = si_irq_init(rdev);
6585 DRM_ERROR("radeon: IH init failed (%d).\n", r);
6586 radeon_irq_kms_fini(rdev);
6591 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6592 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
6597 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6598 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
6603 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6604 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
6609 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6610 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
6611 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6615 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6616 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
6617 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6621 r = si_cp_load_microcode(rdev);
6624 r = si_cp_resume(rdev);
6628 r = cayman_dma_resume(rdev);
6632 if (rdev->has_uvd) {
6633 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6634 if (ring->ring_size) {
6635 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
6638 r = uvd_v1_0_init(rdev);
6640 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
6644 r = radeon_ib_pool_init(rdev);
6646 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
6650 r = radeon_vm_manager_init(rdev);
6652 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
6656 r = dce6_audio_init(rdev);
6663 int si_resume(struct radeon_device *rdev)
6667 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
6668 * posting will perform necessary task to bring back GPU into good
6672 atom_asic_init(rdev->mode_info.atom_context);
6674 /* init golden registers */
6675 si_init_golden_registers(rdev);
6677 if (rdev->pm.pm_method == PM_METHOD_DPM)
6678 radeon_pm_resume(rdev);
6680 rdev->accel_working = true;
6681 r = si_startup(rdev);
6683 DRM_ERROR("si startup failed on resume\n");
6684 rdev->accel_working = false;
6692 int si_suspend(struct radeon_device *rdev)
6694 radeon_pm_suspend(rdev);
6695 dce6_audio_fini(rdev);
6696 radeon_vm_manager_fini(rdev);
6697 si_cp_enable(rdev, false);
6698 cayman_dma_stop(rdev);
6699 if (rdev->has_uvd) {
6700 uvd_v1_0_fini(rdev);
6701 radeon_uvd_suspend(rdev);
6705 si_irq_suspend(rdev);
6706 radeon_wb_disable(rdev);
6707 si_pcie_gart_disable(rdev);
6711 /* Plan is to move initialization in that function and use
6712 * helper function so that radeon_device_init pretty much
6713 * do nothing more than calling asic specific function. This
6714 * should also allow to remove a bunch of callback function
6717 int si_init(struct radeon_device *rdev)
6719 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6723 if (!radeon_get_bios(rdev)) {
6724 if (ASIC_IS_AVIVO(rdev))
6727 /* Must be an ATOMBIOS */
6728 if (!rdev->is_atom_bios) {
6729 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
6732 r = radeon_atombios_init(rdev);
6736 /* Post card if necessary */
6737 if (!radeon_card_posted(rdev)) {
6739 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
6742 DRM_INFO("GPU not posted. posting now...\n");
6743 atom_asic_init(rdev->mode_info.atom_context);
6745 /* init golden registers */
6746 si_init_golden_registers(rdev);
6747 /* Initialize scratch registers */
6748 si_scratch_init(rdev);
6749 /* Initialize surface registers */
6750 radeon_surface_init(rdev);
6751 /* Initialize clocks */
6752 radeon_get_clock_info(rdev->ddev);
6755 r = radeon_fence_driver_init(rdev);
6759 /* initialize memory controller */
6760 r = si_mc_init(rdev);
6763 /* Memory manager */
6764 r = radeon_bo_init(rdev);
6768 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6769 !rdev->rlc_fw || !rdev->mc_fw) {
6770 r = si_init_microcode(rdev);
6772 DRM_ERROR("Failed to load firmware!\n");
6777 /* Initialize power management */
6778 radeon_pm_init(rdev);
6780 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6781 ring->ring_obj = NULL;
6782 r600_ring_init(rdev, ring, 1024 * 1024);
6784 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6785 ring->ring_obj = NULL;
6786 r600_ring_init(rdev, ring, 1024 * 1024);
6788 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6789 ring->ring_obj = NULL;
6790 r600_ring_init(rdev, ring, 1024 * 1024);
6792 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6793 ring->ring_obj = NULL;
6794 r600_ring_init(rdev, ring, 64 * 1024);
6796 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6797 ring->ring_obj = NULL;
6798 r600_ring_init(rdev, ring, 64 * 1024);
6800 if (rdev->has_uvd) {
6801 r = radeon_uvd_init(rdev);
6803 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6804 ring->ring_obj = NULL;
6805 r600_ring_init(rdev, ring, 4096);
6809 rdev->ih.ring_obj = NULL;
6810 r600_ih_ring_init(rdev, 64 * 1024);
6812 r = r600_pcie_gart_init(rdev);
6816 rdev->accel_working = true;
6817 r = si_startup(rdev);
6819 dev_err(rdev->dev, "disabling GPU acceleration\n");
6821 cayman_dma_fini(rdev);
6823 sumo_rlc_fini(rdev);
6824 radeon_wb_fini(rdev);
6825 radeon_ib_pool_fini(rdev);
6826 radeon_vm_manager_fini(rdev);
6827 radeon_irq_kms_fini(rdev);
6828 si_pcie_gart_fini(rdev);
6829 rdev->accel_working = false;
6832 /* Don't start up if the MC ucode is missing.
6833 * The default clocks and voltages before the MC ucode
6834 * is loaded are not suffient for advanced operations.
6837 DRM_ERROR("radeon: MC ucode required for NI+.\n");
6844 void si_fini(struct radeon_device *rdev)
6846 radeon_pm_fini(rdev);
6848 cayman_dma_fini(rdev);
6852 sumo_rlc_fini(rdev);
6853 radeon_wb_fini(rdev);
6854 radeon_vm_manager_fini(rdev);
6855 radeon_ib_pool_fini(rdev);
6856 radeon_irq_kms_fini(rdev);
6857 if (rdev->has_uvd) {
6858 uvd_v1_0_fini(rdev);
6859 radeon_uvd_fini(rdev);
6861 si_pcie_gart_fini(rdev);
6862 r600_vram_scratch_fini(rdev);
6863 radeon_gem_fini(rdev);
6864 radeon_fence_driver_fini(rdev);
6865 radeon_bo_fini(rdev);
6866 radeon_atombios_fini(rdev);
6867 si_fini_microcode(rdev);
6873 * si_get_gpu_clock_counter - return GPU clock counter snapshot
6875 * @rdev: radeon_device pointer
6877 * Fetches a GPU clock counter snapshot (SI).
6878 * Returns the 64 bit clock counter snapshot.
6880 uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
6884 spin_lock(&rdev->gpu_clock_mutex);
6885 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
6886 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
6887 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
6888 spin_unlock(&rdev->gpu_clock_mutex);
6892 int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
6894 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
6897 /* bypass vclk and dclk with bclk */
6898 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6899 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
6900 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6902 /* put PLL in bypass mode */
6903 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
6905 if (!vclk || !dclk) {
6906 /* keep the Bypass mode, put PLL to sleep */
6907 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6911 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
6912 16384, 0x03FFFFFF, 0, 128, 5,
6913 &fb_div, &vclk_div, &dclk_div);
6917 /* set RESET_ANTI_MUX to 0 */
6918 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
6920 /* set VCO_MODE to 1 */
6921 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
6923 /* toggle UPLL_SLEEP to 1 then back to 0 */
6924 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6925 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
6927 /* deassert UPLL_RESET */
6928 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6932 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
6936 /* assert UPLL_RESET again */
6937 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
6939 /* disable spread spectrum. */
6940 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
6942 /* set feedback divider */
6943 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
6945 /* set ref divider to 0 */
6946 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
6948 if (fb_div < 307200)
6949 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
6951 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
6953 /* set PDIV_A and PDIV_B */
6954 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6955 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
6956 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
6958 /* give the PLL some time to settle */
6961 /* deassert PLL_RESET */
6962 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6966 /* switch from bypass mode to normal mode */
6967 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
6969 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
6973 /* switch VCLK and DCLK selection */
6974 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6975 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
6976 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6983 static void si_pcie_gen3_enable(struct radeon_device *rdev)
6985 struct pci_dev *root = rdev->pdev->bus->self;
6986 int bridge_pos, gpu_pos;
6987 u32 speed_cntl, mask, current_data_rate;
6991 if (radeon_pcie_gen2 == 0)
6994 if (rdev->flags & RADEON_IS_IGP)
6997 if (!(rdev->flags & RADEON_IS_PCIE))
7000 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
7004 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
7007 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7008 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
7009 LC_CURRENT_DATA_RATE_SHIFT;
7010 if (mask & DRM_PCIE_SPEED_80) {
7011 if (current_data_rate == 2) {
7012 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
7015 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
7016 } else if (mask & DRM_PCIE_SPEED_50) {
7017 if (current_data_rate == 1) {
7018 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
7021 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
7024 bridge_pos = pci_get_pciecap_ptr(root->dev);
7028 gpu_pos = pci_get_pciecap_ptr(rdev->pdev->dev);
7032 if (mask & DRM_PCIE_SPEED_80) {
7033 /* re-try equalization if gen3 is not already enabled */
7034 if (current_data_rate != 2) {
7035 u16 bridge_cfg, gpu_cfg;
7036 u16 bridge_cfg2, gpu_cfg2;
7037 u32 max_lw, current_lw, tmp;
7039 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
7040 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
7042 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
7043 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
7045 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
7046 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
7048 tmp = RREG32_PCIE(PCIE_LC_STATUS1);
7049 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
7050 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
7052 if (current_lw < max_lw) {
7053 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7054 if (tmp & LC_RENEGOTIATION_SUPPORT) {
7055 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
7056 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
7057 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
7058 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
7062 for (i = 0; i < 10; i++) {
7064 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
7065 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
7068 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
7069 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
7071 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
7072 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
7074 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7075 tmp |= LC_SET_QUIESCE;
7076 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7078 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7080 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7085 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
7086 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7087 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
7088 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
7090 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
7091 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7092 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
7093 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
7096 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
7097 tmp16 &= ~((1 << 4) | (7 << 9));
7098 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
7099 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
7101 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7102 tmp16 &= ~((1 << 4) | (7 << 9));
7103 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
7104 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7106 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7107 tmp &= ~LC_SET_QUIESCE;
7108 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7113 /* set the link speed */
7114 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
7115 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
7116 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7118 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7120 if (mask & DRM_PCIE_SPEED_80)
7121 tmp16 |= 3; /* gen3 */
7122 else if (mask & DRM_PCIE_SPEED_50)
7123 tmp16 |= 2; /* gen2 */
7125 tmp16 |= 1; /* gen1 */
7126 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7128 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7129 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
7130 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7132 for (i = 0; i < rdev->usec_timeout; i++) {
7133 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7134 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
7140 static void si_program_aspm(struct radeon_device *rdev)
7143 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
7144 bool disable_clkreq = false;
7146 if (radeon_aspm == 0)
7149 if (!(rdev->flags & RADEON_IS_PCIE))
7152 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7153 data &= ~LC_XMIT_N_FTS_MASK;
7154 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
7156 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
7158 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
7159 data |= LC_GO_TO_RECOVERY;
7161 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
7163 orig = data = RREG32_PCIE(PCIE_P_CNTL);
7164 data |= P_IGNORE_EDB_ERR;
7166 WREG32_PCIE(PCIE_P_CNTL, data);
7168 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7169 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
7170 data |= LC_PMI_TO_L1_DIS;
7172 data |= LC_L0S_INACTIVITY(7);
7175 data |= LC_L1_INACTIVITY(7);
7176 data &= ~LC_PMI_TO_L1_DIS;
7178 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7180 if (!disable_plloff_in_l1) {
7181 bool clk_req_support;
7183 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7184 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7185 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7187 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7189 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7190 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7191 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7193 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7195 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7196 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7197 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7199 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7201 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7202 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7203 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7205 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7207 if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7208 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7209 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7211 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7213 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7214 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7216 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7218 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
7219 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7221 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
7223 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
7224 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7226 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
7228 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7229 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7231 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7233 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7234 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7236 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7238 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7239 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7241 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7243 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7244 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7246 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7248 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7249 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7250 data |= LC_DYN_LANES_PWR_STATE(3);
7252 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7254 orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7255 data &= ~LS2_EXIT_TIME_MASK;
7256 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7257 data |= LS2_EXIT_TIME(5);
7259 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7261 orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7262 data &= ~LS2_EXIT_TIME_MASK;
7263 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7264 data |= LS2_EXIT_TIME(5);
7266 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7268 if (!disable_clkreq) {
7270 struct pci_dev *root = rdev->pdev->bus->self;
7273 clk_req_support = false;
7274 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7275 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7276 clk_req_support = true;
7278 clk_req_support = false;
7281 clk_req_support = false;
7284 if (clk_req_support) {
7285 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7286 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7288 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7290 orig = data = RREG32(THM_CLK_CNTL);
7291 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7292 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7294 WREG32(THM_CLK_CNTL, data);
7296 orig = data = RREG32(MISC_CLK_CNTL);
7297 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7298 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7300 WREG32(MISC_CLK_CNTL, data);
7302 orig = data = RREG32(CG_CLKPIN_CNTL);
7303 data &= ~BCLK_AS_XCLK;
7305 WREG32(CG_CLKPIN_CNTL, data);
7307 orig = data = RREG32(CG_CLKPIN_CNTL_2);
7308 data &= ~FORCE_BIF_REFCLK_EN;
7310 WREG32(CG_CLKPIN_CNTL_2, data);
7312 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7313 data &= ~MPLL_CLKOUT_SEL_MASK;
7314 data |= MPLL_CLKOUT_SEL(4);
7316 WREG32(MPLL_BYPASSCLK_SEL, data);
7318 orig = data = RREG32(SPLL_CNTL_MODE);
7319 data &= ~SPLL_REFCLK_SEL_MASK;
7321 WREG32(SPLL_CNTL_MODE, data);
7326 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7329 orig = data = RREG32_PCIE(PCIE_CNTL2);
7330 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7332 WREG32_PCIE(PCIE_CNTL2, data);
7335 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7336 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7337 data = RREG32_PCIE(PCIE_LC_STATUS1);
7338 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7339 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7340 data &= ~LC_L0S_INACTIVITY_MASK;
7342 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
projects / dragonfly.git / blob