2 * Copyright 2010 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
24 #include <linux/firmware.h>
25 #include <linux/module.h>
28 #include "radeon_asic.h"
29 #include <uapi_drm/radeon_drm.h>
33 #include "cayman_blit_shaders.h"
34 #include "radeon_ucode.h"
35 #include "clearstate_cayman.h"
37 static const u32 tn_rlc_save_restore_register_list[] =
164 MODULE_FIRMWARE("radeon/BARTS_pfp.bin");
165 MODULE_FIRMWARE("radeon/BARTS_me.bin");
166 MODULE_FIRMWARE("radeon/BARTS_mc.bin");
167 MODULE_FIRMWARE("radeon/BARTS_smc.bin");
168 MODULE_FIRMWARE("radeon/BTC_rlc.bin");
169 MODULE_FIRMWARE("radeon/TURKS_pfp.bin");
170 MODULE_FIRMWARE("radeon/TURKS_me.bin");
171 MODULE_FIRMWARE("radeon/TURKS_mc.bin");
172 MODULE_FIRMWARE("radeon/TURKS_smc.bin");
173 MODULE_FIRMWARE("radeon/CAICOS_pfp.bin");
174 MODULE_FIRMWARE("radeon/CAICOS_me.bin");
175 MODULE_FIRMWARE("radeon/CAICOS_mc.bin");
176 MODULE_FIRMWARE("radeon/CAICOS_smc.bin");
177 MODULE_FIRMWARE("radeon/CAYMAN_pfp.bin");
178 MODULE_FIRMWARE("radeon/CAYMAN_me.bin");
179 MODULE_FIRMWARE("radeon/CAYMAN_mc.bin");
180 MODULE_FIRMWARE("radeon/CAYMAN_rlc.bin");
181 MODULE_FIRMWARE("radeon/CAYMAN_smc.bin");
182 MODULE_FIRMWARE("radeon/ARUBA_pfp.bin");
183 MODULE_FIRMWARE("radeon/ARUBA_me.bin");
184 MODULE_FIRMWARE("radeon/ARUBA_rlc.bin");
187 static const u32 cayman_golden_registers2[] =
189 0x3e5c, 0xffffffff, 0x00000000,
190 0x3e48, 0xffffffff, 0x00000000,
191 0x3e4c, 0xffffffff, 0x00000000,
192 0x3e64, 0xffffffff, 0x00000000,
193 0x3e50, 0xffffffff, 0x00000000,
194 0x3e60, 0xffffffff, 0x00000000
197 static const u32 cayman_golden_registers[] =
199 0x5eb4, 0xffffffff, 0x00000002,
200 0x5e78, 0x8f311ff1, 0x001000f0,
201 0x3f90, 0xffff0000, 0xff000000,
202 0x9148, 0xffff0000, 0xff000000,
203 0x3f94, 0xffff0000, 0xff000000,
204 0x914c, 0xffff0000, 0xff000000,
205 0xc78, 0x00000080, 0x00000080,
206 0xbd4, 0x70073777, 0x00011003,
207 0xd02c, 0xbfffff1f, 0x08421000,
208 0xd0b8, 0x73773777, 0x02011003,
209 0x5bc0, 0x00200000, 0x50100000,
210 0x98f8, 0x33773777, 0x02011003,
211 0x98fc, 0xffffffff, 0x76541032,
212 0x7030, 0x31000311, 0x00000011,
213 0x2f48, 0x33773777, 0x42010001,
214 0x6b28, 0x00000010, 0x00000012,
215 0x7728, 0x00000010, 0x00000012,
216 0x10328, 0x00000010, 0x00000012,
217 0x10f28, 0x00000010, 0x00000012,
218 0x11b28, 0x00000010, 0x00000012,
219 0x12728, 0x00000010, 0x00000012,
220 0x240c, 0x000007ff, 0x00000000,
221 0x8a14, 0xf000001f, 0x00000007,
222 0x8b24, 0x3fff3fff, 0x00ff0fff,
223 0x8b10, 0x0000ff0f, 0x00000000,
224 0x28a4c, 0x07ffffff, 0x06000000,
225 0x10c, 0x00000001, 0x00010003,
226 0xa02c, 0xffffffff, 0x0000009b,
227 0x913c, 0x0000010f, 0x01000100,
228 0x8c04, 0xf8ff00ff, 0x40600060,
229 0x28350, 0x00000f01, 0x00000000,
230 0x9508, 0x3700001f, 0x00000002,
231 0x960c, 0xffffffff, 0x54763210,
232 0x88c4, 0x001f3ae3, 0x00000082,
233 0x88d0, 0xffffffff, 0x0f40df40,
234 0x88d4, 0x0000001f, 0x00000010,
235 0x8974, 0xffffffff, 0x00000000
238 static const u32 dvst_golden_registers2[] =
240 0x8f8, 0xffffffff, 0,
241 0x8fc, 0x00380000, 0,
242 0x8f8, 0xffffffff, 1,
246 static const u32 dvst_golden_registers[] =
248 0x690, 0x3fff3fff, 0x20c00033,
249 0x918c, 0x0fff0fff, 0x00010006,
250 0x91a8, 0x0fff0fff, 0x00010006,
251 0x9150, 0xffffdfff, 0x6e944040,
252 0x917c, 0x0fff0fff, 0x00030002,
253 0x9198, 0x0fff0fff, 0x00030002,
254 0x915c, 0x0fff0fff, 0x00010000,
255 0x3f90, 0xffff0001, 0xff000000,
256 0x9178, 0x0fff0fff, 0x00070000,
257 0x9194, 0x0fff0fff, 0x00070000,
258 0x9148, 0xffff0001, 0xff000000,
259 0x9190, 0x0fff0fff, 0x00090008,
260 0x91ac, 0x0fff0fff, 0x00090008,
261 0x3f94, 0xffff0000, 0xff000000,
262 0x914c, 0xffff0000, 0xff000000,
263 0x929c, 0x00000fff, 0x00000001,
264 0x55e4, 0xff607fff, 0xfc000100,
265 0x8a18, 0xff000fff, 0x00000100,
266 0x8b28, 0xff000fff, 0x00000100,
267 0x9144, 0xfffc0fff, 0x00000100,
268 0x6ed8, 0x00010101, 0x00010000,
269 0x9830, 0xffffffff, 0x00000000,
270 0x9834, 0xf00fffff, 0x00000400,
271 0x9838, 0xfffffffe, 0x00000000,
272 0xd0c0, 0xff000fff, 0x00000100,
273 0xd02c, 0xbfffff1f, 0x08421000,
274 0xd0b8, 0x73773777, 0x12010001,
275 0x5bb0, 0x000000f0, 0x00000070,
276 0x98f8, 0x73773777, 0x12010001,
277 0x98fc, 0xffffffff, 0x00000010,
278 0x9b7c, 0x00ff0000, 0x00fc0000,
279 0x8030, 0x00001f0f, 0x0000100a,
280 0x2f48, 0x73773777, 0x12010001,
281 0x2408, 0x00030000, 0x000c007f,
282 0x8a14, 0xf000003f, 0x00000007,
283 0x8b24, 0x3fff3fff, 0x00ff0fff,
284 0x8b10, 0x0000ff0f, 0x00000000,
285 0x28a4c, 0x07ffffff, 0x06000000,
286 0x4d8, 0x00000fff, 0x00000100,
287 0xa008, 0xffffffff, 0x00010000,
288 0x913c, 0xffff03ff, 0x01000100,
289 0x8c00, 0x000000ff, 0x00000003,
290 0x8c04, 0xf8ff00ff, 0x40600060,
291 0x8cf0, 0x1fff1fff, 0x08e00410,
292 0x28350, 0x00000f01, 0x00000000,
293 0x9508, 0xf700071f, 0x00000002,
294 0x960c, 0xffffffff, 0x54763210,
295 0x20ef8, 0x01ff01ff, 0x00000002,
296 0x20e98, 0xfffffbff, 0x00200000,
297 0x2015c, 0xffffffff, 0x00000f40,
298 0x88c4, 0x001f3ae3, 0x00000082,
299 0x8978, 0x3fffffff, 0x04050140,
300 0x88d4, 0x0000001f, 0x00000010,
301 0x8974, 0xffffffff, 0x00000000
304 static const u32 scrapper_golden_registers[] =
306 0x690, 0x3fff3fff, 0x20c00033,
307 0x918c, 0x0fff0fff, 0x00010006,
308 0x918c, 0x0fff0fff, 0x00010006,
309 0x91a8, 0x0fff0fff, 0x00010006,
310 0x91a8, 0x0fff0fff, 0x00010006,
311 0x9150, 0xffffdfff, 0x6e944040,
312 0x9150, 0xffffdfff, 0x6e944040,
313 0x917c, 0x0fff0fff, 0x00030002,
314 0x917c, 0x0fff0fff, 0x00030002,
315 0x9198, 0x0fff0fff, 0x00030002,
316 0x9198, 0x0fff0fff, 0x00030002,
317 0x915c, 0x0fff0fff, 0x00010000,
318 0x915c, 0x0fff0fff, 0x00010000,
319 0x3f90, 0xffff0001, 0xff000000,
320 0x3f90, 0xffff0001, 0xff000000,
321 0x9178, 0x0fff0fff, 0x00070000,
322 0x9178, 0x0fff0fff, 0x00070000,
323 0x9194, 0x0fff0fff, 0x00070000,
324 0x9194, 0x0fff0fff, 0x00070000,
325 0x9148, 0xffff0001, 0xff000000,
326 0x9148, 0xffff0001, 0xff000000,
327 0x9190, 0x0fff0fff, 0x00090008,
328 0x9190, 0x0fff0fff, 0x00090008,
329 0x91ac, 0x0fff0fff, 0x00090008,
330 0x91ac, 0x0fff0fff, 0x00090008,
331 0x3f94, 0xffff0000, 0xff000000,
332 0x3f94, 0xffff0000, 0xff000000,
333 0x914c, 0xffff0000, 0xff000000,
334 0x914c, 0xffff0000, 0xff000000,
335 0x929c, 0x00000fff, 0x00000001,
336 0x929c, 0x00000fff, 0x00000001,
337 0x55e4, 0xff607fff, 0xfc000100,
338 0x8a18, 0xff000fff, 0x00000100,
339 0x8a18, 0xff000fff, 0x00000100,
340 0x8b28, 0xff000fff, 0x00000100,
341 0x8b28, 0xff000fff, 0x00000100,
342 0x9144, 0xfffc0fff, 0x00000100,
343 0x9144, 0xfffc0fff, 0x00000100,
344 0x6ed8, 0x00010101, 0x00010000,
345 0x9830, 0xffffffff, 0x00000000,
346 0x9830, 0xffffffff, 0x00000000,
347 0x9834, 0xf00fffff, 0x00000400,
348 0x9834, 0xf00fffff, 0x00000400,
349 0x9838, 0xfffffffe, 0x00000000,
350 0x9838, 0xfffffffe, 0x00000000,
351 0xd0c0, 0xff000fff, 0x00000100,
352 0xd02c, 0xbfffff1f, 0x08421000,
353 0xd02c, 0xbfffff1f, 0x08421000,
354 0xd0b8, 0x73773777, 0x12010001,
355 0xd0b8, 0x73773777, 0x12010001,
356 0x5bb0, 0x000000f0, 0x00000070,
357 0x98f8, 0x73773777, 0x12010001,
358 0x98f8, 0x73773777, 0x12010001,
359 0x98fc, 0xffffffff, 0x00000010,
360 0x98fc, 0xffffffff, 0x00000010,
361 0x9b7c, 0x00ff0000, 0x00fc0000,
362 0x9b7c, 0x00ff0000, 0x00fc0000,
363 0x8030, 0x00001f0f, 0x0000100a,
364 0x8030, 0x00001f0f, 0x0000100a,
365 0x2f48, 0x73773777, 0x12010001,
366 0x2f48, 0x73773777, 0x12010001,
367 0x2408, 0x00030000, 0x000c007f,
368 0x8a14, 0xf000003f, 0x00000007,
369 0x8a14, 0xf000003f, 0x00000007,
370 0x8b24, 0x3fff3fff, 0x00ff0fff,
371 0x8b24, 0x3fff3fff, 0x00ff0fff,
372 0x8b10, 0x0000ff0f, 0x00000000,
373 0x8b10, 0x0000ff0f, 0x00000000,
374 0x28a4c, 0x07ffffff, 0x06000000,
375 0x28a4c, 0x07ffffff, 0x06000000,
376 0x4d8, 0x00000fff, 0x00000100,
377 0x4d8, 0x00000fff, 0x00000100,
378 0xa008, 0xffffffff, 0x00010000,
379 0xa008, 0xffffffff, 0x00010000,
380 0x913c, 0xffff03ff, 0x01000100,
381 0x913c, 0xffff03ff, 0x01000100,
382 0x90e8, 0x001fffff, 0x010400c0,
383 0x8c00, 0x000000ff, 0x00000003,
384 0x8c00, 0x000000ff, 0x00000003,
385 0x8c04, 0xf8ff00ff, 0x40600060,
386 0x8c04, 0xf8ff00ff, 0x40600060,
387 0x8c30, 0x0000000f, 0x00040005,
388 0x8cf0, 0x1fff1fff, 0x08e00410,
389 0x8cf0, 0x1fff1fff, 0x08e00410,
390 0x900c, 0x00ffffff, 0x0017071f,
391 0x28350, 0x00000f01, 0x00000000,
392 0x28350, 0x00000f01, 0x00000000,
393 0x9508, 0xf700071f, 0x00000002,
394 0x9508, 0xf700071f, 0x00000002,
395 0x9688, 0x00300000, 0x0017000f,
396 0x960c, 0xffffffff, 0x54763210,
397 0x960c, 0xffffffff, 0x54763210,
398 0x20ef8, 0x01ff01ff, 0x00000002,
399 0x20e98, 0xfffffbff, 0x00200000,
400 0x2015c, 0xffffffff, 0x00000f40,
401 0x88c4, 0x001f3ae3, 0x00000082,
402 0x88c4, 0x001f3ae3, 0x00000082,
403 0x8978, 0x3fffffff, 0x04050140,
404 0x8978, 0x3fffffff, 0x04050140,
405 0x88d4, 0x0000001f, 0x00000010,
406 0x88d4, 0x0000001f, 0x00000010,
407 0x8974, 0xffffffff, 0x00000000,
408 0x8974, 0xffffffff, 0x00000000
411 static void ni_init_golden_registers(struct radeon_device *rdev)
413 switch (rdev->family) {
415 radeon_program_register_sequence(rdev,
416 cayman_golden_registers,
417 (const u32)ARRAY_SIZE(cayman_golden_registers));
418 radeon_program_register_sequence(rdev,
419 cayman_golden_registers2,
420 (const u32)ARRAY_SIZE(cayman_golden_registers2));
423 if ((rdev->pdev->device == 0x9900) ||
424 (rdev->pdev->device == 0x9901) ||
425 (rdev->pdev->device == 0x9903) ||
426 (rdev->pdev->device == 0x9904) ||
427 (rdev->pdev->device == 0x9905) ||
428 (rdev->pdev->device == 0x9906) ||
429 (rdev->pdev->device == 0x9907) ||
430 (rdev->pdev->device == 0x9908) ||
431 (rdev->pdev->device == 0x9909) ||
432 (rdev->pdev->device == 0x990A) ||
433 (rdev->pdev->device == 0x990B) ||
434 (rdev->pdev->device == 0x990C) ||
435 (rdev->pdev->device == 0x990D) ||
436 (rdev->pdev->device == 0x990E) ||
437 (rdev->pdev->device == 0x990F) ||
438 (rdev->pdev->device == 0x9910) ||
439 (rdev->pdev->device == 0x9913) ||
440 (rdev->pdev->device == 0x9917) ||
441 (rdev->pdev->device == 0x9918)) {
442 radeon_program_register_sequence(rdev,
443 dvst_golden_registers,
444 (const u32)ARRAY_SIZE(dvst_golden_registers));
445 radeon_program_register_sequence(rdev,
446 dvst_golden_registers2,
447 (const u32)ARRAY_SIZE(dvst_golden_registers2));
449 radeon_program_register_sequence(rdev,
450 scrapper_golden_registers,
451 (const u32)ARRAY_SIZE(scrapper_golden_registers));
452 radeon_program_register_sequence(rdev,
453 dvst_golden_registers2,
454 (const u32)ARRAY_SIZE(dvst_golden_registers2));
462 #define BTC_IO_MC_REGS_SIZE 29
464 static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
465 {0x00000077, 0xff010100},
466 {0x00000078, 0x00000000},
467 {0x00000079, 0x00001434},
468 {0x0000007a, 0xcc08ec08},
469 {0x0000007b, 0x00040000},
470 {0x0000007c, 0x000080c0},
471 {0x0000007d, 0x09000000},
472 {0x0000007e, 0x00210404},
473 {0x00000081, 0x08a8e800},
474 {0x00000082, 0x00030444},
475 {0x00000083, 0x00000000},
476 {0x00000085, 0x00000001},
477 {0x00000086, 0x00000002},
478 {0x00000087, 0x48490000},
479 {0x00000088, 0x20244647},
480 {0x00000089, 0x00000005},
481 {0x0000008b, 0x66030000},
482 {0x0000008c, 0x00006603},
483 {0x0000008d, 0x00000100},
484 {0x0000008f, 0x00001c0a},
485 {0x00000090, 0xff000001},
486 {0x00000094, 0x00101101},
487 {0x00000095, 0x00000fff},
488 {0x00000096, 0x00116fff},
489 {0x00000097, 0x60010000},
490 {0x00000098, 0x10010000},
491 {0x00000099, 0x00006000},
492 {0x0000009a, 0x00001000},
493 {0x0000009f, 0x00946a00}
496 static const u32 turks_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
497 {0x00000077, 0xff010100},
498 {0x00000078, 0x00000000},
499 {0x00000079, 0x00001434},
500 {0x0000007a, 0xcc08ec08},
501 {0x0000007b, 0x00040000},
502 {0x0000007c, 0x000080c0},
503 {0x0000007d, 0x09000000},
504 {0x0000007e, 0x00210404},
505 {0x00000081, 0x08a8e800},
506 {0x00000082, 0x00030444},
507 {0x00000083, 0x00000000},
508 {0x00000085, 0x00000001},
509 {0x00000086, 0x00000002},
510 {0x00000087, 0x48490000},
511 {0x00000088, 0x20244647},
512 {0x00000089, 0x00000005},
513 {0x0000008b, 0x66030000},
514 {0x0000008c, 0x00006603},
515 {0x0000008d, 0x00000100},
516 {0x0000008f, 0x00001c0a},
517 {0x00000090, 0xff000001},
518 {0x00000094, 0x00101101},
519 {0x00000095, 0x00000fff},
520 {0x00000096, 0x00116fff},
521 {0x00000097, 0x60010000},
522 {0x00000098, 0x10010000},
523 {0x00000099, 0x00006000},
524 {0x0000009a, 0x00001000},
525 {0x0000009f, 0x00936a00}
528 static const u32 caicos_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
529 {0x00000077, 0xff010100},
530 {0x00000078, 0x00000000},
531 {0x00000079, 0x00001434},
532 {0x0000007a, 0xcc08ec08},
533 {0x0000007b, 0x00040000},
534 {0x0000007c, 0x000080c0},
535 {0x0000007d, 0x09000000},
536 {0x0000007e, 0x00210404},
537 {0x00000081, 0x08a8e800},
538 {0x00000082, 0x00030444},
539 {0x00000083, 0x00000000},
540 {0x00000085, 0x00000001},
541 {0x00000086, 0x00000002},
542 {0x00000087, 0x48490000},
543 {0x00000088, 0x20244647},
544 {0x00000089, 0x00000005},
545 {0x0000008b, 0x66030000},
546 {0x0000008c, 0x00006603},
547 {0x0000008d, 0x00000100},
548 {0x0000008f, 0x00001c0a},
549 {0x00000090, 0xff000001},
550 {0x00000094, 0x00101101},
551 {0x00000095, 0x00000fff},
552 {0x00000096, 0x00116fff},
553 {0x00000097, 0x60010000},
554 {0x00000098, 0x10010000},
555 {0x00000099, 0x00006000},
556 {0x0000009a, 0x00001000},
557 {0x0000009f, 0x00916a00}
560 static const u32 cayman_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
561 {0x00000077, 0xff010100},
562 {0x00000078, 0x00000000},
563 {0x00000079, 0x00001434},
564 {0x0000007a, 0xcc08ec08},
565 {0x0000007b, 0x00040000},
566 {0x0000007c, 0x000080c0},
567 {0x0000007d, 0x09000000},
568 {0x0000007e, 0x00210404},
569 {0x00000081, 0x08a8e800},
570 {0x00000082, 0x00030444},
571 {0x00000083, 0x00000000},
572 {0x00000085, 0x00000001},
573 {0x00000086, 0x00000002},
574 {0x00000087, 0x48490000},
575 {0x00000088, 0x20244647},
576 {0x00000089, 0x00000005},
577 {0x0000008b, 0x66030000},
578 {0x0000008c, 0x00006603},
579 {0x0000008d, 0x00000100},
580 {0x0000008f, 0x00001c0a},
581 {0x00000090, 0xff000001},
582 {0x00000094, 0x00101101},
583 {0x00000095, 0x00000fff},
584 {0x00000096, 0x00116fff},
585 {0x00000097, 0x60010000},
586 {0x00000098, 0x10010000},
587 {0x00000099, 0x00006000},
588 {0x0000009a, 0x00001000},
589 {0x0000009f, 0x00976b00}
592 int ni_mc_load_microcode(struct radeon_device *rdev)
594 const __be32 *fw_data;
595 u32 mem_type, running, blackout = 0;
597 int i, ucode_size, regs_size;
602 switch (rdev->family) {
604 io_mc_regs = (u32 *)&barts_io_mc_regs;
605 ucode_size = BTC_MC_UCODE_SIZE;
606 regs_size = BTC_IO_MC_REGS_SIZE;
609 io_mc_regs = (u32 *)&turks_io_mc_regs;
610 ucode_size = BTC_MC_UCODE_SIZE;
611 regs_size = BTC_IO_MC_REGS_SIZE;
615 io_mc_regs = (u32 *)&caicos_io_mc_regs;
616 ucode_size = BTC_MC_UCODE_SIZE;
617 regs_size = BTC_IO_MC_REGS_SIZE;
620 io_mc_regs = (u32 *)&cayman_io_mc_regs;
621 ucode_size = CAYMAN_MC_UCODE_SIZE;
622 regs_size = BTC_IO_MC_REGS_SIZE;
626 mem_type = (RREG32(MC_SEQ_MISC0) & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT;
627 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
629 if ((mem_type == MC_SEQ_MISC0_GDDR5_VALUE) && (running == 0)) {
631 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
632 WREG32(MC_SHARED_BLACKOUT_CNTL, 1);
635 /* reset the engine and set to writable */
636 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
637 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
639 /* load mc io regs */
640 for (i = 0; i < regs_size; i++) {
641 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
642 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
644 /* load the MC ucode */
645 fw_data = (const __be32 *)rdev->mc_fw->data;
646 for (i = 0; i < ucode_size; i++)
647 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
649 /* put the engine back into the active state */
650 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
651 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
652 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
654 /* wait for training to complete */
655 for (i = 0; i < rdev->usec_timeout; i++) {
656 if (RREG32(MC_IO_PAD_CNTL_D0) & MEM_FALL_OUT_CMD)
662 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
668 int ni_init_microcode(struct radeon_device *rdev)
670 const char *chip_name;
671 const char *rlc_chip_name;
672 size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size;
673 size_t smc_req_size = 0;
679 switch (rdev->family) {
682 rlc_chip_name = "BTC";
683 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
684 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
685 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
686 mc_req_size = BTC_MC_UCODE_SIZE * 4;
687 smc_req_size = ALIGN(BARTS_SMC_UCODE_SIZE, 4);
691 rlc_chip_name = "BTC";
692 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
693 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
694 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
695 mc_req_size = BTC_MC_UCODE_SIZE * 4;
696 smc_req_size = ALIGN(TURKS_SMC_UCODE_SIZE, 4);
699 chip_name = "CAICOS";
700 rlc_chip_name = "BTC";
701 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
702 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
703 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
704 mc_req_size = BTC_MC_UCODE_SIZE * 4;
705 smc_req_size = ALIGN(CAICOS_SMC_UCODE_SIZE, 4);
708 chip_name = "CAYMAN";
709 rlc_chip_name = "CAYMAN";
710 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
711 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
712 rlc_req_size = CAYMAN_RLC_UCODE_SIZE * 4;
713 mc_req_size = CAYMAN_MC_UCODE_SIZE * 4;
714 smc_req_size = ALIGN(CAYMAN_SMC_UCODE_SIZE, 4);
718 rlc_chip_name = "ARUBA";
719 /* pfp/me same size as CAYMAN */
720 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
721 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
722 rlc_req_size = ARUBA_RLC_UCODE_SIZE * 4;
728 DRM_INFO("Loading %s Microcode\n", chip_name);
730 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_pfp", chip_name);
731 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
734 if (rdev->pfp_fw->datasize != pfp_req_size) {
736 "ni_pfp: Bogus length %zu in firmware \"%s\"\n",
737 rdev->pfp_fw->datasize, fw_name);
742 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_me", chip_name);
743 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
746 if (rdev->me_fw->datasize != me_req_size) {
748 "ni_me: Bogus length %zu in firmware \"%s\"\n",
749 rdev->me_fw->datasize, fw_name);
753 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_rlc",
755 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
758 if (rdev->rlc_fw->datasize != rlc_req_size) {
760 "ni_rlc: Bogus length %zu in firmware \"%s\"\n",
761 rdev->rlc_fw->datasize, fw_name);
765 /* no MC ucode on TN */
766 if (!(rdev->flags & RADEON_IS_IGP)) {
767 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mc", chip_name);
768 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
771 if (rdev->mc_fw->datasize != mc_req_size) {
773 "ni_mc: Bogus length %zu in firmware \"%s\"\n",
774 rdev->mc_fw->datasize, fw_name);
779 if ((rdev->family >= CHIP_BARTS) && (rdev->family <= CHIP_CAYMAN)) {
780 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_smc", chip_name);
781 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
784 "smc: error loading firmware \"%s\"\n",
786 release_firmware(rdev->smc_fw);
789 } else if (rdev->smc_fw->datasize != smc_req_size) {
791 "ni_smc: Bogus length %zu in firmware \"%s\"\n",
792 rdev->smc_fw->datasize, fw_name);
801 "ni_cp: Failed to load firmware \"%s\"\n",
803 release_firmware(rdev->pfp_fw);
805 release_firmware(rdev->me_fw);
807 release_firmware(rdev->rlc_fw);
809 release_firmware(rdev->mc_fw);
811 release_firmware(rdev->smc_fw);
818 * ni_fini_microcode - drop the firmwares image references
820 * @rdev: radeon_device pointer
822 * Drop the pfp, me, mc and rlc firmwares image references.
823 * Called at driver shutdown.
825 void ni_fini_microcode(struct radeon_device *rdev)
827 release_firmware(rdev->pfp_fw);
829 release_firmware(rdev->me_fw);
831 release_firmware(rdev->rlc_fw);
833 release_firmware(rdev->mc_fw);
835 release_firmware(rdev->smc_fw);
839 int tn_get_temp(struct radeon_device *rdev)
841 u32 temp = RREG32_SMC(TN_CURRENT_GNB_TEMP) & 0x7ff;
842 int actual_temp = (temp / 8) - 49;
844 return actual_temp * 1000;
850 static void cayman_gpu_init(struct radeon_device *rdev)
852 u32 gb_addr_config = 0;
853 u32 mc_shared_chmap, mc_arb_ramcfg;
854 u32 cgts_tcc_disable;
857 u32 cgts_sm_ctrl_reg;
858 u32 hdp_host_path_cntl;
860 u32 disabled_rb_mask;
863 switch (rdev->family) {
865 rdev->config.cayman.max_shader_engines = 2;
866 rdev->config.cayman.max_pipes_per_simd = 4;
867 rdev->config.cayman.max_tile_pipes = 8;
868 rdev->config.cayman.max_simds_per_se = 12;
869 rdev->config.cayman.max_backends_per_se = 4;
870 rdev->config.cayman.max_texture_channel_caches = 8;
871 rdev->config.cayman.max_gprs = 256;
872 rdev->config.cayman.max_threads = 256;
873 rdev->config.cayman.max_gs_threads = 32;
874 rdev->config.cayman.max_stack_entries = 512;
875 rdev->config.cayman.sx_num_of_sets = 8;
876 rdev->config.cayman.sx_max_export_size = 256;
877 rdev->config.cayman.sx_max_export_pos_size = 64;
878 rdev->config.cayman.sx_max_export_smx_size = 192;
879 rdev->config.cayman.max_hw_contexts = 8;
880 rdev->config.cayman.sq_num_cf_insts = 2;
882 rdev->config.cayman.sc_prim_fifo_size = 0x100;
883 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
884 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
885 gb_addr_config = CAYMAN_GB_ADDR_CONFIG_GOLDEN;
889 rdev->config.cayman.max_shader_engines = 1;
890 rdev->config.cayman.max_pipes_per_simd = 4;
891 rdev->config.cayman.max_tile_pipes = 2;
892 if ((rdev->pdev->device == 0x9900) ||
893 (rdev->pdev->device == 0x9901) ||
894 (rdev->pdev->device == 0x9905) ||
895 (rdev->pdev->device == 0x9906) ||
896 (rdev->pdev->device == 0x9907) ||
897 (rdev->pdev->device == 0x9908) ||
898 (rdev->pdev->device == 0x9909) ||
899 (rdev->pdev->device == 0x990B) ||
900 (rdev->pdev->device == 0x990C) ||
901 (rdev->pdev->device == 0x990F) ||
902 (rdev->pdev->device == 0x9910) ||
903 (rdev->pdev->device == 0x9917) ||
904 (rdev->pdev->device == 0x9999) ||
905 (rdev->pdev->device == 0x999C)) {
906 rdev->config.cayman.max_simds_per_se = 6;
907 rdev->config.cayman.max_backends_per_se = 2;
908 rdev->config.cayman.max_hw_contexts = 8;
909 rdev->config.cayman.sx_max_export_size = 256;
910 rdev->config.cayman.sx_max_export_pos_size = 64;
911 rdev->config.cayman.sx_max_export_smx_size = 192;
912 } else if ((rdev->pdev->device == 0x9903) ||
913 (rdev->pdev->device == 0x9904) ||
914 (rdev->pdev->device == 0x990A) ||
915 (rdev->pdev->device == 0x990D) ||
916 (rdev->pdev->device == 0x990E) ||
917 (rdev->pdev->device == 0x9913) ||
918 (rdev->pdev->device == 0x9918) ||
919 (rdev->pdev->device == 0x999D)) {
920 rdev->config.cayman.max_simds_per_se = 4;
921 rdev->config.cayman.max_backends_per_se = 2;
922 rdev->config.cayman.max_hw_contexts = 8;
923 rdev->config.cayman.sx_max_export_size = 256;
924 rdev->config.cayman.sx_max_export_pos_size = 64;
925 rdev->config.cayman.sx_max_export_smx_size = 192;
926 } else if ((rdev->pdev->device == 0x9919) ||
927 (rdev->pdev->device == 0x9990) ||
928 (rdev->pdev->device == 0x9991) ||
929 (rdev->pdev->device == 0x9994) ||
930 (rdev->pdev->device == 0x9995) ||
931 (rdev->pdev->device == 0x9996) ||
932 (rdev->pdev->device == 0x999A) ||
933 (rdev->pdev->device == 0x99A0)) {
934 rdev->config.cayman.max_simds_per_se = 3;
935 rdev->config.cayman.max_backends_per_se = 1;
936 rdev->config.cayman.max_hw_contexts = 4;
937 rdev->config.cayman.sx_max_export_size = 128;
938 rdev->config.cayman.sx_max_export_pos_size = 32;
939 rdev->config.cayman.sx_max_export_smx_size = 96;
941 rdev->config.cayman.max_simds_per_se = 2;
942 rdev->config.cayman.max_backends_per_se = 1;
943 rdev->config.cayman.max_hw_contexts = 4;
944 rdev->config.cayman.sx_max_export_size = 128;
945 rdev->config.cayman.sx_max_export_pos_size = 32;
946 rdev->config.cayman.sx_max_export_smx_size = 96;
948 rdev->config.cayman.max_texture_channel_caches = 2;
949 rdev->config.cayman.max_gprs = 256;
950 rdev->config.cayman.max_threads = 256;
951 rdev->config.cayman.max_gs_threads = 32;
952 rdev->config.cayman.max_stack_entries = 512;
953 rdev->config.cayman.sx_num_of_sets = 8;
954 rdev->config.cayman.sq_num_cf_insts = 2;
956 rdev->config.cayman.sc_prim_fifo_size = 0x40;
957 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
958 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
959 gb_addr_config = ARUBA_GB_ADDR_CONFIG_GOLDEN;
964 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
965 WREG32((0x2c14 + j), 0x00000000);
966 WREG32((0x2c18 + j), 0x00000000);
967 WREG32((0x2c1c + j), 0x00000000);
968 WREG32((0x2c20 + j), 0x00000000);
969 WREG32((0x2c24 + j), 0x00000000);
972 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
974 evergreen_fix_pci_max_read_req_size(rdev);
976 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
977 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
979 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
980 rdev->config.cayman.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
981 if (rdev->config.cayman.mem_row_size_in_kb > 4)
982 rdev->config.cayman.mem_row_size_in_kb = 4;
983 /* XXX use MC settings? */
984 rdev->config.cayman.shader_engine_tile_size = 32;
985 rdev->config.cayman.num_gpus = 1;
986 rdev->config.cayman.multi_gpu_tile_size = 64;
988 tmp = (gb_addr_config & NUM_PIPES_MASK) >> NUM_PIPES_SHIFT;
989 rdev->config.cayman.num_tile_pipes = (1 << tmp);
990 tmp = (gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT;
991 rdev->config.cayman.mem_max_burst_length_bytes = (tmp + 1) * 256;
992 tmp = (gb_addr_config & NUM_SHADER_ENGINES_MASK) >> NUM_SHADER_ENGINES_SHIFT;
993 rdev->config.cayman.num_shader_engines = tmp + 1;
994 tmp = (gb_addr_config & NUM_GPUS_MASK) >> NUM_GPUS_SHIFT;
995 rdev->config.cayman.num_gpus = tmp + 1;
996 tmp = (gb_addr_config & MULTI_GPU_TILE_SIZE_MASK) >> MULTI_GPU_TILE_SIZE_SHIFT;
997 rdev->config.cayman.multi_gpu_tile_size = 1 << tmp;
998 tmp = (gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT;
999 rdev->config.cayman.mem_row_size_in_kb = 1 << tmp;
1002 /* setup tiling info dword. gb_addr_config is not adequate since it does
1003 * not have bank info, so create a custom tiling dword.
1004 * bits 3:0 num_pipes
1005 * bits 7:4 num_banks
1006 * bits 11:8 group_size
1007 * bits 15:12 row_size
1009 rdev->config.cayman.tile_config = 0;
1010 switch (rdev->config.cayman.num_tile_pipes) {
1013 rdev->config.cayman.tile_config |= (0 << 0);
1016 rdev->config.cayman.tile_config |= (1 << 0);
1019 rdev->config.cayman.tile_config |= (2 << 0);
1022 rdev->config.cayman.tile_config |= (3 << 0);
1026 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1027 if (rdev->flags & RADEON_IS_IGP)
1028 rdev->config.cayman.tile_config |= 1 << 4;
1030 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
1031 case 0: /* four banks */
1032 rdev->config.cayman.tile_config |= 0 << 4;
1034 case 1: /* eight banks */
1035 rdev->config.cayman.tile_config |= 1 << 4;
1037 case 2: /* sixteen banks */
1039 rdev->config.cayman.tile_config |= 2 << 4;
1043 rdev->config.cayman.tile_config |=
1044 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
1045 rdev->config.cayman.tile_config |=
1046 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
1049 for (i = (rdev->config.cayman.max_shader_engines - 1); i >= 0; i--) {
1050 u32 rb_disable_bitmap;
1052 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1053 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1054 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
1056 tmp |= rb_disable_bitmap;
1058 /* enabled rb are just the one not disabled :) */
1059 disabled_rb_mask = tmp;
1061 for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1063 /* if all the backends are disabled, fix it up here */
1064 if ((disabled_rb_mask & tmp) == tmp) {
1065 for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1066 disabled_rb_mask &= ~(1 << i);
1069 for (i = 0; i < rdev->config.cayman.max_shader_engines; i++) {
1070 u32 simd_disable_bitmap;
1072 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1073 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1074 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
1075 simd_disable_bitmap |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
1077 tmp |= simd_disable_bitmap;
1079 rdev->config.cayman.active_simds = hweight32(~tmp);
1081 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1082 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1084 WREG32(GB_ADDR_CONFIG, gb_addr_config);
1085 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
1086 if (ASIC_IS_DCE6(rdev))
1087 WREG32(DMIF_ADDR_CALC, gb_addr_config);
1088 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
1089 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
1090 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1091 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
1092 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
1093 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
1095 if ((rdev->config.cayman.max_backends_per_se == 1) &&
1096 (rdev->flags & RADEON_IS_IGP)) {
1097 if ((disabled_rb_mask & 3) == 1) {
1098 /* RB0 disabled, RB1 enabled */
1101 /* RB1 disabled, RB0 enabled */
1105 tmp = gb_addr_config & NUM_PIPES_MASK;
1106 tmp = r6xx_remap_render_backend(rdev, tmp,
1107 rdev->config.cayman.max_backends_per_se *
1108 rdev->config.cayman.max_shader_engines,
1109 CAYMAN_MAX_BACKENDS, disabled_rb_mask);
1111 WREG32(GB_BACKEND_MAP, tmp);
1113 cgts_tcc_disable = 0xffff0000;
1114 for (i = 0; i < rdev->config.cayman.max_texture_channel_caches; i++)
1115 cgts_tcc_disable &= ~(1 << (16 + i));
1116 WREG32(CGTS_TCC_DISABLE, cgts_tcc_disable);
1117 WREG32(CGTS_SYS_TCC_DISABLE, cgts_tcc_disable);
1118 WREG32(CGTS_USER_SYS_TCC_DISABLE, cgts_tcc_disable);
1119 WREG32(CGTS_USER_TCC_DISABLE, cgts_tcc_disable);
1121 /* reprogram the shader complex */
1122 cgts_sm_ctrl_reg = RREG32(CGTS_SM_CTRL_REG);
1123 for (i = 0; i < 16; i++)
1124 WREG32(CGTS_SM_CTRL_REG, OVERRIDE);
1125 WREG32(CGTS_SM_CTRL_REG, cgts_sm_ctrl_reg);
1127 /* set HW defaults for 3D engine */
1128 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
1130 sx_debug_1 = RREG32(SX_DEBUG_1);
1131 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1132 WREG32(SX_DEBUG_1, sx_debug_1);
1134 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1135 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
1136 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.cayman.sx_num_of_sets);
1137 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1139 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4) | CRC_SIMD_ID_WADDR_DISABLE);
1141 /* need to be explicitly zero-ed */
1142 WREG32(VGT_OFFCHIP_LDS_BASE, 0);
1143 WREG32(SQ_LSTMP_RING_BASE, 0);
1144 WREG32(SQ_HSTMP_RING_BASE, 0);
1145 WREG32(SQ_ESTMP_RING_BASE, 0);
1146 WREG32(SQ_GSTMP_RING_BASE, 0);
1147 WREG32(SQ_VSTMP_RING_BASE, 0);
1148 WREG32(SQ_PSTMP_RING_BASE, 0);
1150 WREG32(TA_CNTL_AUX, DISABLE_CUBE_ANISO);
1152 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.cayman.sx_max_export_size / 4) - 1) |
1153 POSITION_BUFFER_SIZE((rdev->config.cayman.sx_max_export_pos_size / 4) - 1) |
1154 SMX_BUFFER_SIZE((rdev->config.cayman.sx_max_export_smx_size / 4) - 1)));
1156 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.cayman.sc_prim_fifo_size) |
1157 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_hiz_tile_fifo_size) |
1158 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_earlyz_tile_fifo_size)));
1161 WREG32(VGT_NUM_INSTANCES, 1);
1163 WREG32(CP_PERFMON_CNTL, 0);
1165 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.cayman.sq_num_cf_insts) |
1166 FETCH_FIFO_HIWATER(0x4) |
1167 DONE_FIFO_HIWATER(0xe0) |
1168 ALU_UPDATE_FIFO_HIWATER(0x8)));
1170 WREG32(SQ_GPR_RESOURCE_MGMT_1, NUM_CLAUSE_TEMP_GPRS(4));
1171 WREG32(SQ_CONFIG, (VC_ENABLE |
1176 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, DYN_GPR_ENABLE);
1178 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1179 FORCE_EOV_MAX_REZ_CNT(255)));
1181 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
1182 AUTO_INVLD_EN(ES_AND_GS_AUTO));
1184 WREG32(VGT_GS_VERTEX_REUSE, 16);
1185 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1187 WREG32(CB_PERF_CTR0_SEL_0, 0);
1188 WREG32(CB_PERF_CTR0_SEL_1, 0);
1189 WREG32(CB_PERF_CTR1_SEL_0, 0);
1190 WREG32(CB_PERF_CTR1_SEL_1, 0);
1191 WREG32(CB_PERF_CTR2_SEL_0, 0);
1192 WREG32(CB_PERF_CTR2_SEL_1, 0);
1193 WREG32(CB_PERF_CTR3_SEL_0, 0);
1194 WREG32(CB_PERF_CTR3_SEL_1, 0);
1196 tmp = RREG32(HDP_MISC_CNTL);
1197 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
1198 WREG32(HDP_MISC_CNTL, tmp);
1200 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1201 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1203 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
1207 /* set clockgating golden values on TN */
1208 if (rdev->family == CHIP_ARUBA) {
1209 tmp = RREG32_CG(CG_CGTT_LOCAL_0);
1211 WREG32_CG(CG_CGTT_LOCAL_0, tmp);
1212 tmp = RREG32_CG(CG_CGTT_LOCAL_1);
1214 WREG32_CG(CG_CGTT_LOCAL_1, tmp);
1221 void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev)
1223 /* flush hdp cache */
1224 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
1226 /* bits 0-7 are the VM contexts0-7 */
1227 WREG32(VM_INVALIDATE_REQUEST, 1);
1230 static int cayman_pcie_gart_enable(struct radeon_device *rdev)
1234 if (rdev->gart.robj == NULL) {
1235 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
1238 r = radeon_gart_table_vram_pin(rdev);
1241 /* Setup TLB control */
1242 WREG32(MC_VM_MX_L1_TLB_CNTL,
1245 ENABLE_L1_FRAGMENT_PROCESSING |
1246 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1247 ENABLE_ADVANCED_DRIVER_MODEL |
1248 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1249 /* Setup L2 cache */
1250 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
1251 ENABLE_L2_FRAGMENT_PROCESSING |
1252 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1253 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1254 EFFECTIVE_L2_QUEUE_SIZE(7) |
1255 CONTEXT1_IDENTITY_ACCESS_MODE(1));
1256 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
1257 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1259 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1260 /* setup context0 */
1261 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
1262 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
1263 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
1264 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
1265 (u32)(rdev->dummy_page.addr >> 12));
1266 WREG32(VM_CONTEXT0_CNTL2, 0);
1267 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
1268 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
1274 /* empty context1-7 */
1275 /* Assign the pt base to something valid for now; the pts used for
1276 * the VMs are determined by the application and setup and assigned
1277 * on the fly in the vm part of radeon_gart.c
1279 for (i = 1; i < 8; i++) {
1280 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (i << 2), 0);
1281 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (i << 2), rdev->vm_manager.max_pfn);
1282 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
1283 rdev->vm_manager.saved_table_addr[i]);
1286 /* enable context1-7 */
1287 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
1288 (u32)(rdev->dummy_page.addr >> 12));
1289 WREG32(VM_CONTEXT1_CNTL2, 4);
1290 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
1291 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
1292 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1293 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1294 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1295 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1296 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
1297 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
1298 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
1299 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
1300 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
1301 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
1302 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1303 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1305 cayman_pcie_gart_tlb_flush(rdev);
1306 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1307 (unsigned)(rdev->mc.gtt_size >> 20),
1308 (unsigned long long)rdev->gart.table_addr);
1309 rdev->gart.ready = true;
1313 static void cayman_pcie_gart_disable(struct radeon_device *rdev)
1317 for (i = 1; i < 8; ++i) {
1318 rdev->vm_manager.saved_table_addr[i] = RREG32(
1319 VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2));
1322 /* Disable all tables */
1323 WREG32(VM_CONTEXT0_CNTL, 0);
1324 WREG32(VM_CONTEXT1_CNTL, 0);
1325 /* Setup TLB control */
1326 WREG32(MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING |
1327 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1328 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1329 /* Setup L2 cache */
1330 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1331 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1332 EFFECTIVE_L2_QUEUE_SIZE(7) |
1333 CONTEXT1_IDENTITY_ACCESS_MODE(1));
1334 WREG32(VM_L2_CNTL2, 0);
1335 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1336 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1337 radeon_gart_table_vram_unpin(rdev);
1340 static void cayman_pcie_gart_fini(struct radeon_device *rdev)
1342 cayman_pcie_gart_disable(rdev);
1343 radeon_gart_table_vram_free(rdev);
1344 radeon_gart_fini(rdev);
1347 void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
1348 int ring, u32 cp_int_cntl)
1350 u32 srbm_gfx_cntl = RREG32(SRBM_GFX_CNTL) & ~3;
1352 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl | (ring & 3));
1353 WREG32(CP_INT_CNTL, cp_int_cntl);
1359 void cayman_fence_ring_emit(struct radeon_device *rdev,
1360 struct radeon_fence *fence)
1362 struct radeon_ring *ring = &rdev->ring[fence->ring];
1363 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
1364 u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1365 PACKET3_SH_ACTION_ENA;
1367 /* flush read cache over gart for this vmid */
1368 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1369 radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1370 radeon_ring_write(ring, 0xFFFFFFFF);
1371 radeon_ring_write(ring, 0);
1372 radeon_ring_write(ring, 10); /* poll interval */
1373 /* EVENT_WRITE_EOP - flush caches, send int */
1374 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
1375 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
1376 radeon_ring_write(ring, lower_32_bits(addr));
1377 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
1378 radeon_ring_write(ring, fence->seq);
1379 radeon_ring_write(ring, 0);
1382 void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1384 struct radeon_ring *ring = &rdev->ring[ib->ring];
1385 u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1386 PACKET3_SH_ACTION_ENA;
1388 /* set to DX10/11 mode */
1389 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
1390 radeon_ring_write(ring, 1);
1392 if (ring->rptr_save_reg) {
1393 uint32_t next_rptr = ring->wptr + 3 + 4 + 8;
1394 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1395 radeon_ring_write(ring, ((ring->rptr_save_reg -
1396 PACKET3_SET_CONFIG_REG_START) >> 2));
1397 radeon_ring_write(ring, next_rptr);
1400 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1401 radeon_ring_write(ring,
1405 (ib->gpu_addr & 0xFFFFFFFC));
1406 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
1407 radeon_ring_write(ring, ib->length_dw |
1408 (ib->vm ? (ib->vm->id << 24) : 0));
1410 /* flush read cache over gart for this vmid */
1411 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1412 radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1413 radeon_ring_write(ring, 0xFFFFFFFF);
1414 radeon_ring_write(ring, 0);
1415 radeon_ring_write(ring, ((ib->vm ? ib->vm->id : 0) << 24) | 10); /* poll interval */
1418 static void cayman_cp_enable(struct radeon_device *rdev, bool enable)
1421 WREG32(CP_ME_CNTL, 0);
1423 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1424 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1425 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1426 WREG32(SCRATCH_UMSK, 0);
1427 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1431 u32 cayman_gfx_get_rptr(struct radeon_device *rdev,
1432 struct radeon_ring *ring)
1436 if (rdev->wb.enabled)
1437 rptr = rdev->wb.wb[ring->rptr_offs/4];
1439 if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1440 rptr = RREG32(CP_RB0_RPTR);
1441 else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1442 rptr = RREG32(CP_RB1_RPTR);
1444 rptr = RREG32(CP_RB2_RPTR);
1450 u32 cayman_gfx_get_wptr(struct radeon_device *rdev,
1451 struct radeon_ring *ring)
1455 if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1456 wptr = RREG32(CP_RB0_WPTR);
1457 else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1458 wptr = RREG32(CP_RB1_WPTR);
1460 wptr = RREG32(CP_RB2_WPTR);
1465 void cayman_gfx_set_wptr(struct radeon_device *rdev,
1466 struct radeon_ring *ring)
1468 if (ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
1469 WREG32(CP_RB0_WPTR, ring->wptr);
1470 (void)RREG32(CP_RB0_WPTR);
1471 } else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX) {
1472 WREG32(CP_RB1_WPTR, ring->wptr);
1473 (void)RREG32(CP_RB1_WPTR);
1475 WREG32(CP_RB2_WPTR, ring->wptr);
1476 (void)RREG32(CP_RB2_WPTR);
1480 static int cayman_cp_load_microcode(struct radeon_device *rdev)
1482 const __be32 *fw_data;
1485 if (!rdev->me_fw || !rdev->pfp_fw)
1488 cayman_cp_enable(rdev, false);
1490 fw_data = (const __be32 *)rdev->pfp_fw->data;
1491 WREG32(CP_PFP_UCODE_ADDR, 0);
1492 for (i = 0; i < CAYMAN_PFP_UCODE_SIZE; i++)
1493 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1494 WREG32(CP_PFP_UCODE_ADDR, 0);
1496 fw_data = (const __be32 *)rdev->me_fw->data;
1497 WREG32(CP_ME_RAM_WADDR, 0);
1498 for (i = 0; i < CAYMAN_PM4_UCODE_SIZE; i++)
1499 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1501 WREG32(CP_PFP_UCODE_ADDR, 0);
1502 WREG32(CP_ME_RAM_WADDR, 0);
1503 WREG32(CP_ME_RAM_RADDR, 0);
1507 static int cayman_cp_start(struct radeon_device *rdev)
1509 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1512 r = radeon_ring_lock(rdev, ring, 7);
1514 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1517 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
1518 radeon_ring_write(ring, 0x1);
1519 radeon_ring_write(ring, 0x0);
1520 radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1);
1521 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1522 radeon_ring_write(ring, 0);
1523 radeon_ring_write(ring, 0);
1524 radeon_ring_unlock_commit(rdev, ring, false);
1526 cayman_cp_enable(rdev, true);
1528 r = radeon_ring_lock(rdev, ring, cayman_default_size + 19);
1530 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1534 /* setup clear context state */
1535 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1536 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1538 for (i = 0; i < cayman_default_size; i++)
1539 radeon_ring_write(ring, cayman_default_state[i]);
1541 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1542 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1544 /* set clear context state */
1545 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
1546 radeon_ring_write(ring, 0);
1548 /* SQ_VTX_BASE_VTX_LOC */
1549 radeon_ring_write(ring, 0xc0026f00);
1550 radeon_ring_write(ring, 0x00000000);
1551 radeon_ring_write(ring, 0x00000000);
1552 radeon_ring_write(ring, 0x00000000);
1555 radeon_ring_write(ring, 0xc0036f00);
1556 radeon_ring_write(ring, 0x00000bc4);
1557 radeon_ring_write(ring, 0xffffffff);
1558 radeon_ring_write(ring, 0xffffffff);
1559 radeon_ring_write(ring, 0xffffffff);
1561 radeon_ring_write(ring, 0xc0026900);
1562 radeon_ring_write(ring, 0x00000316);
1563 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1564 radeon_ring_write(ring, 0x00000010); /* */
1566 radeon_ring_unlock_commit(rdev, ring, false);
1568 /* XXX init other rings */
1573 static void cayman_cp_fini(struct radeon_device *rdev)
1575 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1576 cayman_cp_enable(rdev, false);
1577 radeon_ring_fini(rdev, ring);
1578 radeon_scratch_free(rdev, ring->rptr_save_reg);
1581 static int cayman_cp_resume(struct radeon_device *rdev)
1583 static const int ridx[] = {
1584 RADEON_RING_TYPE_GFX_INDEX,
1585 CAYMAN_RING_TYPE_CP1_INDEX,
1586 CAYMAN_RING_TYPE_CP2_INDEX
1588 static const unsigned cp_rb_cntl[] = {
1593 static const unsigned cp_rb_rptr_addr[] = {
1598 static const unsigned cp_rb_rptr_addr_hi[] = {
1599 CP_RB0_RPTR_ADDR_HI,
1600 CP_RB1_RPTR_ADDR_HI,
1603 static const unsigned cp_rb_base[] = {
1608 static const unsigned cp_rb_rptr[] = {
1613 static const unsigned cp_rb_wptr[] = {
1618 struct radeon_ring *ring;
1621 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1622 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1628 RREG32(GRBM_SOFT_RESET);
1630 WREG32(GRBM_SOFT_RESET, 0);
1631 RREG32(GRBM_SOFT_RESET);
1633 WREG32(CP_SEM_WAIT_TIMER, 0x0);
1634 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
1636 /* Set the write pointer delay */
1637 WREG32(CP_RB_WPTR_DELAY, 0);
1639 WREG32(CP_DEBUG, (1 << 27));
1641 /* set the wb address whether it's enabled or not */
1642 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
1643 WREG32(SCRATCH_UMSK, 0xff);
1645 for (i = 0; i < 3; ++i) {
1649 /* Set ring buffer size */
1650 ring = &rdev->ring[ridx[i]];
1651 rb_cntl = order_base_2(ring->ring_size / 8);
1652 rb_cntl |= order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8;
1654 rb_cntl |= BUF_SWAP_32BIT;
1656 WREG32(cp_rb_cntl[i], rb_cntl);
1658 /* set the wb address whether it's enabled or not */
1659 addr = rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET;
1660 WREG32(cp_rb_rptr_addr[i], addr & 0xFFFFFFFC);
1661 WREG32(cp_rb_rptr_addr_hi[i], upper_32_bits(addr) & 0xFF);
1664 /* set the rb base addr, this causes an internal reset of ALL rings */
1665 for (i = 0; i < 3; ++i) {
1666 ring = &rdev->ring[ridx[i]];
1667 WREG32(cp_rb_base[i], ring->gpu_addr >> 8);
1670 for (i = 0; i < 3; ++i) {
1671 /* Initialize the ring buffer's read and write pointers */
1672 ring = &rdev->ring[ridx[i]];
1673 WREG32_P(cp_rb_cntl[i], RB_RPTR_WR_ENA, ~RB_RPTR_WR_ENA);
1676 WREG32(cp_rb_rptr[i], 0);
1677 WREG32(cp_rb_wptr[i], ring->wptr);
1680 WREG32_P(cp_rb_cntl[i], 0, ~RB_RPTR_WR_ENA);
1683 /* start the rings */
1684 cayman_cp_start(rdev);
1685 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
1686 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1687 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1688 /* this only test cp0 */
1689 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
1691 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1692 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1693 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1697 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1698 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
1703 u32 cayman_gpu_check_soft_reset(struct radeon_device *rdev)
1709 tmp = RREG32(GRBM_STATUS);
1710 if (tmp & (PA_BUSY | SC_BUSY |
1712 TA_BUSY | VGT_BUSY |
1714 GDS_BUSY | SPI_BUSY |
1715 IA_BUSY | IA_BUSY_NO_DMA))
1716 reset_mask |= RADEON_RESET_GFX;
1718 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
1719 CP_BUSY | CP_COHERENCY_BUSY))
1720 reset_mask |= RADEON_RESET_CP;
1722 if (tmp & GRBM_EE_BUSY)
1723 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
1725 /* DMA_STATUS_REG 0 */
1726 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
1727 if (!(tmp & DMA_IDLE))
1728 reset_mask |= RADEON_RESET_DMA;
1730 /* DMA_STATUS_REG 1 */
1731 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
1732 if (!(tmp & DMA_IDLE))
1733 reset_mask |= RADEON_RESET_DMA1;
1736 tmp = RREG32(SRBM_STATUS2);
1738 reset_mask |= RADEON_RESET_DMA;
1740 if (tmp & DMA1_BUSY)
1741 reset_mask |= RADEON_RESET_DMA1;
1744 tmp = RREG32(SRBM_STATUS);
1745 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
1746 reset_mask |= RADEON_RESET_RLC;
1749 reset_mask |= RADEON_RESET_IH;
1752 reset_mask |= RADEON_RESET_SEM;
1754 if (tmp & GRBM_RQ_PENDING)
1755 reset_mask |= RADEON_RESET_GRBM;
1758 reset_mask |= RADEON_RESET_VMC;
1760 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
1761 MCC_BUSY | MCD_BUSY))
1762 reset_mask |= RADEON_RESET_MC;
1764 if (evergreen_is_display_hung(rdev))
1765 reset_mask |= RADEON_RESET_DISPLAY;
1768 tmp = RREG32(VM_L2_STATUS);
1770 reset_mask |= RADEON_RESET_VMC;
1772 /* Skip MC reset as it's mostly likely not hung, just busy */
1773 if (reset_mask & RADEON_RESET_MC) {
1774 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1775 reset_mask &= ~RADEON_RESET_MC;
1781 static void cayman_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1783 struct evergreen_mc_save save;
1784 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1787 if (reset_mask == 0)
1790 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1792 evergreen_print_gpu_status_regs(rdev);
1793 dev_info(rdev->dev, " VM_CONTEXT0_PROTECTION_FAULT_ADDR 0x%08X\n",
1795 dev_info(rdev->dev, " VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n",
1797 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
1799 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1802 /* Disable CP parsing/prefetching */
1803 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
1805 if (reset_mask & RADEON_RESET_DMA) {
1807 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
1808 tmp &= ~DMA_RB_ENABLE;
1809 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
1812 if (reset_mask & RADEON_RESET_DMA1) {
1814 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
1815 tmp &= ~DMA_RB_ENABLE;
1816 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
1821 evergreen_mc_stop(rdev, &save);
1822 if (evergreen_mc_wait_for_idle(rdev)) {
1823 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1826 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
1827 grbm_soft_reset = SOFT_RESET_CB |
1841 if (reset_mask & RADEON_RESET_CP) {
1842 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
1844 srbm_soft_reset |= SOFT_RESET_GRBM;
1847 if (reset_mask & RADEON_RESET_DMA)
1848 srbm_soft_reset |= SOFT_RESET_DMA;
1850 if (reset_mask & RADEON_RESET_DMA1)
1851 srbm_soft_reset |= SOFT_RESET_DMA1;
1853 if (reset_mask & RADEON_RESET_DISPLAY)
1854 srbm_soft_reset |= SOFT_RESET_DC;
1856 if (reset_mask & RADEON_RESET_RLC)
1857 srbm_soft_reset |= SOFT_RESET_RLC;
1859 if (reset_mask & RADEON_RESET_SEM)
1860 srbm_soft_reset |= SOFT_RESET_SEM;
1862 if (reset_mask & RADEON_RESET_IH)
1863 srbm_soft_reset |= SOFT_RESET_IH;
1865 if (reset_mask & RADEON_RESET_GRBM)
1866 srbm_soft_reset |= SOFT_RESET_GRBM;
1868 if (reset_mask & RADEON_RESET_VMC)
1869 srbm_soft_reset |= SOFT_RESET_VMC;
1871 if (!(rdev->flags & RADEON_IS_IGP)) {
1872 if (reset_mask & RADEON_RESET_MC)
1873 srbm_soft_reset |= SOFT_RESET_MC;
1876 if (grbm_soft_reset) {
1877 tmp = RREG32(GRBM_SOFT_RESET);
1878 tmp |= grbm_soft_reset;
1879 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
1880 WREG32(GRBM_SOFT_RESET, tmp);
1881 tmp = RREG32(GRBM_SOFT_RESET);
1885 tmp &= ~grbm_soft_reset;
1886 WREG32(GRBM_SOFT_RESET, tmp);
1887 tmp = RREG32(GRBM_SOFT_RESET);
1890 if (srbm_soft_reset) {
1891 tmp = RREG32(SRBM_SOFT_RESET);
1892 tmp |= srbm_soft_reset;
1893 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1894 WREG32(SRBM_SOFT_RESET, tmp);
1895 tmp = RREG32(SRBM_SOFT_RESET);
1899 tmp &= ~srbm_soft_reset;
1900 WREG32(SRBM_SOFT_RESET, tmp);
1901 tmp = RREG32(SRBM_SOFT_RESET);
1904 /* Wait a little for things to settle down */
1907 evergreen_mc_resume(rdev, &save);
1910 evergreen_print_gpu_status_regs(rdev);
1913 int cayman_asic_reset(struct radeon_device *rdev)
1917 reset_mask = cayman_gpu_check_soft_reset(rdev);
1920 r600_set_bios_scratch_engine_hung(rdev, true);
1922 cayman_gpu_soft_reset(rdev, reset_mask);
1924 reset_mask = cayman_gpu_check_soft_reset(rdev);
1927 evergreen_gpu_pci_config_reset(rdev);
1929 r600_set_bios_scratch_engine_hung(rdev, false);
1935 * cayman_gfx_is_lockup - Check if the GFX engine is locked up
1937 * @rdev: radeon_device pointer
1938 * @ring: radeon_ring structure holding ring information
1940 * Check if the GFX engine is locked up.
1941 * Returns true if the engine appears to be locked up, false if not.
1943 bool cayman_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1945 u32 reset_mask = cayman_gpu_check_soft_reset(rdev);
1947 if (!(reset_mask & (RADEON_RESET_GFX |
1948 RADEON_RESET_COMPUTE |
1949 RADEON_RESET_CP))) {
1950 radeon_ring_lockup_update(rdev, ring);
1953 return radeon_ring_test_lockup(rdev, ring);
1956 static int cayman_startup(struct radeon_device *rdev)
1958 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1961 /* enable pcie gen2 link */
1962 evergreen_pcie_gen2_enable(rdev);
1964 evergreen_program_aspm(rdev);
1966 /* scratch needs to be initialized before MC */
1967 r = r600_vram_scratch_init(rdev);
1971 evergreen_mc_program(rdev);
1973 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
1974 r = ni_mc_load_microcode(rdev);
1976 DRM_ERROR("Failed to load MC firmware!\n");
1981 r = cayman_pcie_gart_enable(rdev);
1984 cayman_gpu_init(rdev);
1986 /* allocate rlc buffers */
1987 if (rdev->flags & RADEON_IS_IGP) {
1988 rdev->rlc.reg_list = tn_rlc_save_restore_register_list;
1989 rdev->rlc.reg_list_size =
1990 (u32)ARRAY_SIZE(tn_rlc_save_restore_register_list);
1991 rdev->rlc.cs_data = cayman_cs_data;
1992 r = sumo_rlc_init(rdev);
1994 DRM_ERROR("Failed to init rlc BOs!\n");
1999 /* allocate wb buffer */
2000 r = radeon_wb_init(rdev);
2004 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
2006 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2010 r = uvd_v2_2_resume(rdev);
2012 r = radeon_fence_driver_start_ring(rdev,
2013 R600_RING_TYPE_UVD_INDEX);
2015 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
2018 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
2020 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
2022 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2026 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
2028 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2032 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
2034 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2038 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
2040 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2045 if (!rdev->irq.installed) {
2046 r = radeon_irq_kms_init(rdev);
2051 r = r600_irq_init(rdev);
2053 DRM_ERROR("radeon: IH init failed (%d).\n", r);
2054 radeon_irq_kms_fini(rdev);
2057 evergreen_irq_set(rdev);
2059 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2064 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2065 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2066 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2070 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2071 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2072 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2076 r = cayman_cp_load_microcode(rdev);
2079 r = cayman_cp_resume(rdev);
2083 r = cayman_dma_resume(rdev);
2087 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2088 if (ring->ring_size) {
2089 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
2092 r = uvd_v1_0_init(rdev);
2094 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
2097 r = radeon_ib_pool_init(rdev);
2099 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2103 r = radeon_vm_manager_init(rdev);
2105 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
2109 if (ASIC_IS_DCE6(rdev)) {
2110 r = dce6_audio_init(rdev);
2114 r = r600_audio_init(rdev);
2122 int cayman_resume(struct radeon_device *rdev)
2126 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
2127 * posting will perform necessary task to bring back GPU into good
2131 atom_asic_init(rdev->mode_info.atom_context);
2133 /* init golden registers */
2134 ni_init_golden_registers(rdev);
2136 if (rdev->pm.pm_method == PM_METHOD_DPM)
2137 radeon_pm_resume(rdev);
2139 rdev->accel_working = true;
2140 r = cayman_startup(rdev);
2142 DRM_ERROR("cayman startup failed on resume\n");
2143 rdev->accel_working = false;
2149 int cayman_suspend(struct radeon_device *rdev)
2151 radeon_pm_suspend(rdev);
2152 if (ASIC_IS_DCE6(rdev))
2153 dce6_audio_fini(rdev);
2155 r600_audio_fini(rdev);
2156 radeon_vm_manager_fini(rdev);
2157 cayman_cp_enable(rdev, false);
2158 cayman_dma_stop(rdev);
2159 uvd_v1_0_fini(rdev);
2160 radeon_uvd_suspend(rdev);
2161 evergreen_irq_suspend(rdev);
2162 radeon_wb_disable(rdev);
2163 cayman_pcie_gart_disable(rdev);
2167 /* Plan is to move initialization in that function and use
2168 * helper function so that radeon_device_init pretty much
2169 * do nothing more than calling asic specific function. This
2170 * should also allow to remove a bunch of callback function
2173 int cayman_init(struct radeon_device *rdev)
2175 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2179 if (!radeon_get_bios(rdev)) {
2180 if (ASIC_IS_AVIVO(rdev))
2183 /* Must be an ATOMBIOS */
2184 if (!rdev->is_atom_bios) {
2185 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
2188 r = radeon_atombios_init(rdev);
2192 /* Post card if necessary */
2193 if (!radeon_card_posted(rdev)) {
2195 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
2198 DRM_INFO("GPU not posted. posting now...\n");
2199 atom_asic_init(rdev->mode_info.atom_context);
2201 /* init golden registers */
2202 ni_init_golden_registers(rdev);
2203 /* Initialize scratch registers */
2204 r600_scratch_init(rdev);
2205 /* Initialize surface registers */
2206 radeon_surface_init(rdev);
2207 /* Initialize clocks */
2208 radeon_get_clock_info(rdev->ddev);
2210 r = radeon_fence_driver_init(rdev);
2213 /* initialize memory controller */
2214 r = evergreen_mc_init(rdev);
2217 /* Memory manager */
2218 r = radeon_bo_init(rdev);
2222 if (rdev->flags & RADEON_IS_IGP) {
2223 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
2224 r = ni_init_microcode(rdev);
2226 DRM_ERROR("Failed to load firmware!\n");
2231 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
2232 r = ni_init_microcode(rdev);
2234 DRM_ERROR("Failed to load firmware!\n");
2240 /* Initialize power management */
2241 radeon_pm_init(rdev);
2243 ring->ring_obj = NULL;
2244 r600_ring_init(rdev, ring, 1024 * 1024);
2246 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2247 ring->ring_obj = NULL;
2248 r600_ring_init(rdev, ring, 64 * 1024);
2250 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2251 ring->ring_obj = NULL;
2252 r600_ring_init(rdev, ring, 64 * 1024);
2254 r = radeon_uvd_init(rdev);
2256 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2257 ring->ring_obj = NULL;
2258 r600_ring_init(rdev, ring, 4096);
2261 rdev->ih.ring_obj = NULL;
2262 r600_ih_ring_init(rdev, 64 * 1024);
2264 r = r600_pcie_gart_init(rdev);
2268 rdev->accel_working = true;
2269 r = cayman_startup(rdev);
2271 dev_err(rdev->dev, "disabling GPU acceleration\n");
2272 cayman_cp_fini(rdev);
2273 cayman_dma_fini(rdev);
2274 r600_irq_fini(rdev);
2275 if (rdev->flags & RADEON_IS_IGP)
2276 sumo_rlc_fini(rdev);
2277 radeon_wb_fini(rdev);
2278 radeon_ib_pool_fini(rdev);
2279 radeon_vm_manager_fini(rdev);
2280 radeon_irq_kms_fini(rdev);
2281 cayman_pcie_gart_fini(rdev);
2282 rdev->accel_working = false;
2285 /* Don't start up if the MC ucode is missing.
2286 * The default clocks and voltages before the MC ucode
2287 * is loaded are not suffient for advanced operations.
2289 * We can skip this check for TN, because there is no MC
2292 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
2293 DRM_ERROR("radeon: MC ucode required for NI+.\n");
2300 void cayman_fini(struct radeon_device *rdev)
2302 radeon_pm_fini(rdev);
2303 cayman_cp_fini(rdev);
2304 cayman_dma_fini(rdev);
2305 r600_irq_fini(rdev);
2306 if (rdev->flags & RADEON_IS_IGP)
2307 sumo_rlc_fini(rdev);
2308 radeon_wb_fini(rdev);
2309 radeon_vm_manager_fini(rdev);
2310 radeon_ib_pool_fini(rdev);
2311 radeon_irq_kms_fini(rdev);
2312 uvd_v1_0_fini(rdev);
2313 radeon_uvd_fini(rdev);
2314 cayman_pcie_gart_fini(rdev);
2315 r600_vram_scratch_fini(rdev);
2316 radeon_gem_fini(rdev);
2317 radeon_fence_driver_fini(rdev);
2318 radeon_bo_fini(rdev);
2319 radeon_atombios_fini(rdev);
2320 ni_fini_microcode(rdev);
2328 int cayman_vm_init(struct radeon_device *rdev)
2331 rdev->vm_manager.nvm = 8;
2332 /* base offset of vram pages */
2333 if (rdev->flags & RADEON_IS_IGP) {
2334 u64 tmp = RREG32(FUS_MC_VM_FB_OFFSET);
2336 rdev->vm_manager.vram_base_offset = tmp;
2338 rdev->vm_manager.vram_base_offset = 0;
2342 void cayman_vm_fini(struct radeon_device *rdev)
2347 * cayman_vm_decode_fault - print human readable fault info
2349 * @rdev: radeon_device pointer
2350 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
2351 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
2353 * Print human readable fault information (cayman/TN).
2355 void cayman_vm_decode_fault(struct radeon_device *rdev,
2356 u32 status, u32 addr)
2358 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
2359 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
2360 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
2452 block = "TC_TFETCH";
2462 block = "TC_VFETCH";
2501 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
2502 protections, vmid, addr,
2503 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
2508 * cayman_vm_flush - vm flush using the CP
2510 * @rdev: radeon_device pointer
2512 * Update the page table base and flush the VM TLB
2513 * using the CP (cayman-si).
2515 void cayman_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
2517 struct radeon_ring *ring = &rdev->ring[ridx];
2522 radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2), 0));
2523 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
2525 /* flush hdp cache */
2526 radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
2527 radeon_ring_write(ring, 0x1);
2529 /* bits 0-7 are the VM contexts0-7 */
2530 radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0));
2531 radeon_ring_write(ring, 1 << vm->id);
2533 /* sync PFP to ME, otherwise we might get invalid PFP reads */
2534 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
2535 radeon_ring_write(ring, 0x0);