drm/radeon: Sync to Linux 3.11
[dragonfly.git] / sys / dev / drm / radeon / evergreen.c
CommitLineData
926deccb
FT
1/*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 *
24 * $FreeBSD: head/sys/dev/drm2/radeon/evergreen.c 254885 2013-08-25 19:37:15Z dumbbell $
25 */
26
27#include <drm/drmP.h>
28#include "radeon.h"
29#include "radeon_asic.h"
30#include <uapi_drm/radeon_drm.h>
31#include "evergreend.h"
32#include "atom.h"
33#include "avivod.h"
34#include "evergreen_reg.h"
35#include "evergreen_blit_shaders.h"
57e252bf 36#include "radeon_ucode.h"
926deccb
FT
37
38static const u32 crtc_offsets[6] =
39{
40 EVERGREEN_CRTC0_REGISTER_OFFSET,
41 EVERGREEN_CRTC1_REGISTER_OFFSET,
42 EVERGREEN_CRTC2_REGISTER_OFFSET,
43 EVERGREEN_CRTC3_REGISTER_OFFSET,
44 EVERGREEN_CRTC4_REGISTER_OFFSET,
45 EVERGREEN_CRTC5_REGISTER_OFFSET
46};
47
57e252bf
MN
48#include "clearstate_evergreen.h"
49
50static u32 sumo_rlc_save_restore_register_list[] =
51{
52 0x98fc,
53 0x9830,
54 0x9834,
55 0x9838,
56 0x9870,
57 0x9874,
58 0x8a14,
59 0x8b24,
60 0x8bcc,
61 0x8b10,
62 0x8d00,
63 0x8d04,
64 0x8c00,
65 0x8c04,
66 0x8c08,
67 0x8c0c,
68 0x8d8c,
69 0x8c20,
70 0x8c24,
71 0x8c28,
72 0x8c18,
73 0x8c1c,
74 0x8cf0,
75 0x8e2c,
76 0x8e38,
77 0x8c30,
78 0x9508,
79 0x9688,
80 0x9608,
81 0x960c,
82 0x9610,
83 0x9614,
84 0x88c4,
85 0x88d4,
86 0xa008,
87 0x900c,
88 0x9100,
89 0x913c,
90 0x98f8,
91 0x98f4,
92 0x9b7c,
93 0x3f8c,
94 0x8950,
95 0x8954,
96 0x8a18,
97 0x8b28,
98 0x9144,
99 0x9148,
100 0x914c,
101 0x3f90,
102 0x3f94,
103 0x915c,
104 0x9160,
105 0x9178,
106 0x917c,
107 0x9180,
108 0x918c,
109 0x9190,
110 0x9194,
111 0x9198,
112 0x919c,
113 0x91a8,
114 0x91ac,
115 0x91b0,
116 0x91b4,
117 0x91b8,
118 0x91c4,
119 0x91c8,
120 0x91cc,
121 0x91d0,
122 0x91d4,
123 0x91e0,
124 0x91e4,
125 0x91ec,
126 0x91f0,
127 0x91f4,
128 0x9200,
129 0x9204,
130 0x929c,
131 0x9150,
132 0x802c,
133};
134static u32 sumo_rlc_save_restore_register_list_size = ARRAY_SIZE(sumo_rlc_save_restore_register_list);
135
926deccb
FT
136static void evergreen_gpu_init(struct radeon_device *rdev);
137void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
b403bed8 138void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
57e252bf 139void evergreen_program_aspm(struct radeon_device *rdev);
926deccb 140
f43cf1b1
MN
141static const u32 evergreen_golden_registers[] =
142{
143 0x3f90, 0xffff0000, 0xff000000,
144 0x9148, 0xffff0000, 0xff000000,
145 0x3f94, 0xffff0000, 0xff000000,
146 0x914c, 0xffff0000, 0xff000000,
147 0x9b7c, 0xffffffff, 0x00000000,
148 0x8a14, 0xffffffff, 0x00000007,
149 0x8b10, 0xffffffff, 0x00000000,
150 0x960c, 0xffffffff, 0x54763210,
151 0x88c4, 0xffffffff, 0x000000c2,
152 0x88d4, 0xffffffff, 0x00000010,
153 0x8974, 0xffffffff, 0x00000000,
154 0xc78, 0x00000080, 0x00000080,
155 0x5eb4, 0xffffffff, 0x00000002,
156 0x5e78, 0xffffffff, 0x001000f0,
157 0x6104, 0x01000300, 0x00000000,
158 0x5bc0, 0x00300000, 0x00000000,
159 0x7030, 0xffffffff, 0x00000011,
160 0x7c30, 0xffffffff, 0x00000011,
161 0x10830, 0xffffffff, 0x00000011,
162 0x11430, 0xffffffff, 0x00000011,
163 0x12030, 0xffffffff, 0x00000011,
164 0x12c30, 0xffffffff, 0x00000011,
165 0xd02c, 0xffffffff, 0x08421000,
166 0x240c, 0xffffffff, 0x00000380,
167 0x8b24, 0xffffffff, 0x00ff0fff,
168 0x28a4c, 0x06000000, 0x06000000,
169 0x10c, 0x00000001, 0x00000001,
170 0x8d00, 0xffffffff, 0x100e4848,
171 0x8d04, 0xffffffff, 0x00164745,
172 0x8c00, 0xffffffff, 0xe4000003,
173 0x8c04, 0xffffffff, 0x40600060,
174 0x8c08, 0xffffffff, 0x001c001c,
175 0x8cf0, 0xffffffff, 0x08e00620,
176 0x8c20, 0xffffffff, 0x00800080,
177 0x8c24, 0xffffffff, 0x00800080,
178 0x8c18, 0xffffffff, 0x20202078,
179 0x8c1c, 0xffffffff, 0x00001010,
180 0x28350, 0xffffffff, 0x00000000,
181 0xa008, 0xffffffff, 0x00010000,
182 0x5cc, 0xffffffff, 0x00000001,
183 0x9508, 0xffffffff, 0x00000002,
184 0x913c, 0x0000000f, 0x0000000a
185};
186
187static const u32 evergreen_golden_registers2[] =
188{
189 0x2f4c, 0xffffffff, 0x00000000,
190 0x54f4, 0xffffffff, 0x00000000,
191 0x54f0, 0xffffffff, 0x00000000,
192 0x5498, 0xffffffff, 0x00000000,
193 0x549c, 0xffffffff, 0x00000000,
194 0x5494, 0xffffffff, 0x00000000,
195 0x53cc, 0xffffffff, 0x00000000,
196 0x53c8, 0xffffffff, 0x00000000,
197 0x53c4, 0xffffffff, 0x00000000,
198 0x53c0, 0xffffffff, 0x00000000,
199 0x53bc, 0xffffffff, 0x00000000,
200 0x53b8, 0xffffffff, 0x00000000,
201 0x53b4, 0xffffffff, 0x00000000,
202 0x53b0, 0xffffffff, 0x00000000
203};
204
205static const u32 cypress_mgcg_init[] =
206{
207 0x802c, 0xffffffff, 0xc0000000,
208 0x5448, 0xffffffff, 0x00000100,
209 0x55e4, 0xffffffff, 0x00000100,
210 0x160c, 0xffffffff, 0x00000100,
211 0x5644, 0xffffffff, 0x00000100,
212 0xc164, 0xffffffff, 0x00000100,
213 0x8a18, 0xffffffff, 0x00000100,
214 0x897c, 0xffffffff, 0x06000100,
215 0x8b28, 0xffffffff, 0x00000100,
216 0x9144, 0xffffffff, 0x00000100,
217 0x9a60, 0xffffffff, 0x00000100,
218 0x9868, 0xffffffff, 0x00000100,
219 0x8d58, 0xffffffff, 0x00000100,
220 0x9510, 0xffffffff, 0x00000100,
221 0x949c, 0xffffffff, 0x00000100,
222 0x9654, 0xffffffff, 0x00000100,
223 0x9030, 0xffffffff, 0x00000100,
224 0x9034, 0xffffffff, 0x00000100,
225 0x9038, 0xffffffff, 0x00000100,
226 0x903c, 0xffffffff, 0x00000100,
227 0x9040, 0xffffffff, 0x00000100,
228 0xa200, 0xffffffff, 0x00000100,
229 0xa204, 0xffffffff, 0x00000100,
230 0xa208, 0xffffffff, 0x00000100,
231 0xa20c, 0xffffffff, 0x00000100,
232 0x971c, 0xffffffff, 0x00000100,
233 0x977c, 0xffffffff, 0x00000100,
234 0x3f80, 0xffffffff, 0x00000100,
235 0xa210, 0xffffffff, 0x00000100,
236 0xa214, 0xffffffff, 0x00000100,
237 0x4d8, 0xffffffff, 0x00000100,
238 0x9784, 0xffffffff, 0x00000100,
239 0x9698, 0xffffffff, 0x00000100,
240 0x4d4, 0xffffffff, 0x00000200,
241 0x30cc, 0xffffffff, 0x00000100,
242 0xd0c0, 0xffffffff, 0xff000100,
243 0x802c, 0xffffffff, 0x40000000,
244 0x915c, 0xffffffff, 0x00010000,
245 0x9160, 0xffffffff, 0x00030002,
246 0x9178, 0xffffffff, 0x00070000,
247 0x917c, 0xffffffff, 0x00030002,
248 0x9180, 0xffffffff, 0x00050004,
249 0x918c, 0xffffffff, 0x00010006,
250 0x9190, 0xffffffff, 0x00090008,
251 0x9194, 0xffffffff, 0x00070000,
252 0x9198, 0xffffffff, 0x00030002,
253 0x919c, 0xffffffff, 0x00050004,
254 0x91a8, 0xffffffff, 0x00010006,
255 0x91ac, 0xffffffff, 0x00090008,
256 0x91b0, 0xffffffff, 0x00070000,
257 0x91b4, 0xffffffff, 0x00030002,
258 0x91b8, 0xffffffff, 0x00050004,
259 0x91c4, 0xffffffff, 0x00010006,
260 0x91c8, 0xffffffff, 0x00090008,
261 0x91cc, 0xffffffff, 0x00070000,
262 0x91d0, 0xffffffff, 0x00030002,
263 0x91d4, 0xffffffff, 0x00050004,
264 0x91e0, 0xffffffff, 0x00010006,
265 0x91e4, 0xffffffff, 0x00090008,
266 0x91e8, 0xffffffff, 0x00000000,
267 0x91ec, 0xffffffff, 0x00070000,
268 0x91f0, 0xffffffff, 0x00030002,
269 0x91f4, 0xffffffff, 0x00050004,
270 0x9200, 0xffffffff, 0x00010006,
271 0x9204, 0xffffffff, 0x00090008,
272 0x9208, 0xffffffff, 0x00070000,
273 0x920c, 0xffffffff, 0x00030002,
274 0x9210, 0xffffffff, 0x00050004,
275 0x921c, 0xffffffff, 0x00010006,
276 0x9220, 0xffffffff, 0x00090008,
277 0x9224, 0xffffffff, 0x00070000,
278 0x9228, 0xffffffff, 0x00030002,
279 0x922c, 0xffffffff, 0x00050004,
280 0x9238, 0xffffffff, 0x00010006,
281 0x923c, 0xffffffff, 0x00090008,
282 0x9240, 0xffffffff, 0x00070000,
283 0x9244, 0xffffffff, 0x00030002,
284 0x9248, 0xffffffff, 0x00050004,
285 0x9254, 0xffffffff, 0x00010006,
286 0x9258, 0xffffffff, 0x00090008,
287 0x925c, 0xffffffff, 0x00070000,
288 0x9260, 0xffffffff, 0x00030002,
289 0x9264, 0xffffffff, 0x00050004,
290 0x9270, 0xffffffff, 0x00010006,
291 0x9274, 0xffffffff, 0x00090008,
292 0x9278, 0xffffffff, 0x00070000,
293 0x927c, 0xffffffff, 0x00030002,
294 0x9280, 0xffffffff, 0x00050004,
295 0x928c, 0xffffffff, 0x00010006,
296 0x9290, 0xffffffff, 0x00090008,
297 0x9294, 0xffffffff, 0x00000000,
298 0x929c, 0xffffffff, 0x00000001,
299 0x802c, 0xffffffff, 0x40010000,
300 0x915c, 0xffffffff, 0x00010000,
301 0x9160, 0xffffffff, 0x00030002,
302 0x9178, 0xffffffff, 0x00070000,
303 0x917c, 0xffffffff, 0x00030002,
304 0x9180, 0xffffffff, 0x00050004,
305 0x918c, 0xffffffff, 0x00010006,
306 0x9190, 0xffffffff, 0x00090008,
307 0x9194, 0xffffffff, 0x00070000,
308 0x9198, 0xffffffff, 0x00030002,
309 0x919c, 0xffffffff, 0x00050004,
310 0x91a8, 0xffffffff, 0x00010006,
311 0x91ac, 0xffffffff, 0x00090008,
312 0x91b0, 0xffffffff, 0x00070000,
313 0x91b4, 0xffffffff, 0x00030002,
314 0x91b8, 0xffffffff, 0x00050004,
315 0x91c4, 0xffffffff, 0x00010006,
316 0x91c8, 0xffffffff, 0x00090008,
317 0x91cc, 0xffffffff, 0x00070000,
318 0x91d0, 0xffffffff, 0x00030002,
319 0x91d4, 0xffffffff, 0x00050004,
320 0x91e0, 0xffffffff, 0x00010006,
321 0x91e4, 0xffffffff, 0x00090008,
322 0x91e8, 0xffffffff, 0x00000000,
323 0x91ec, 0xffffffff, 0x00070000,
324 0x91f0, 0xffffffff, 0x00030002,
325 0x91f4, 0xffffffff, 0x00050004,
326 0x9200, 0xffffffff, 0x00010006,
327 0x9204, 0xffffffff, 0x00090008,
328 0x9208, 0xffffffff, 0x00070000,
329 0x920c, 0xffffffff, 0x00030002,
330 0x9210, 0xffffffff, 0x00050004,
331 0x921c, 0xffffffff, 0x00010006,
332 0x9220, 0xffffffff, 0x00090008,
333 0x9224, 0xffffffff, 0x00070000,
334 0x9228, 0xffffffff, 0x00030002,
335 0x922c, 0xffffffff, 0x00050004,
336 0x9238, 0xffffffff, 0x00010006,
337 0x923c, 0xffffffff, 0x00090008,
338 0x9240, 0xffffffff, 0x00070000,
339 0x9244, 0xffffffff, 0x00030002,
340 0x9248, 0xffffffff, 0x00050004,
341 0x9254, 0xffffffff, 0x00010006,
342 0x9258, 0xffffffff, 0x00090008,
343 0x925c, 0xffffffff, 0x00070000,
344 0x9260, 0xffffffff, 0x00030002,
345 0x9264, 0xffffffff, 0x00050004,
346 0x9270, 0xffffffff, 0x00010006,
347 0x9274, 0xffffffff, 0x00090008,
348 0x9278, 0xffffffff, 0x00070000,
349 0x927c, 0xffffffff, 0x00030002,
350 0x9280, 0xffffffff, 0x00050004,
351 0x928c, 0xffffffff, 0x00010006,
352 0x9290, 0xffffffff, 0x00090008,
353 0x9294, 0xffffffff, 0x00000000,
354 0x929c, 0xffffffff, 0x00000001,
355 0x802c, 0xffffffff, 0xc0000000
356};
357
358static const u32 redwood_mgcg_init[] =
359{
360 0x802c, 0xffffffff, 0xc0000000,
361 0x5448, 0xffffffff, 0x00000100,
362 0x55e4, 0xffffffff, 0x00000100,
363 0x160c, 0xffffffff, 0x00000100,
364 0x5644, 0xffffffff, 0x00000100,
365 0xc164, 0xffffffff, 0x00000100,
366 0x8a18, 0xffffffff, 0x00000100,
367 0x897c, 0xffffffff, 0x06000100,
368 0x8b28, 0xffffffff, 0x00000100,
369 0x9144, 0xffffffff, 0x00000100,
370 0x9a60, 0xffffffff, 0x00000100,
371 0x9868, 0xffffffff, 0x00000100,
372 0x8d58, 0xffffffff, 0x00000100,
373 0x9510, 0xffffffff, 0x00000100,
374 0x949c, 0xffffffff, 0x00000100,
375 0x9654, 0xffffffff, 0x00000100,
376 0x9030, 0xffffffff, 0x00000100,
377 0x9034, 0xffffffff, 0x00000100,
378 0x9038, 0xffffffff, 0x00000100,
379 0x903c, 0xffffffff, 0x00000100,
380 0x9040, 0xffffffff, 0x00000100,
381 0xa200, 0xffffffff, 0x00000100,
382 0xa204, 0xffffffff, 0x00000100,
383 0xa208, 0xffffffff, 0x00000100,
384 0xa20c, 0xffffffff, 0x00000100,
385 0x971c, 0xffffffff, 0x00000100,
386 0x977c, 0xffffffff, 0x00000100,
387 0x3f80, 0xffffffff, 0x00000100,
388 0xa210, 0xffffffff, 0x00000100,
389 0xa214, 0xffffffff, 0x00000100,
390 0x4d8, 0xffffffff, 0x00000100,
391 0x9784, 0xffffffff, 0x00000100,
392 0x9698, 0xffffffff, 0x00000100,
393 0x4d4, 0xffffffff, 0x00000200,
394 0x30cc, 0xffffffff, 0x00000100,
395 0xd0c0, 0xffffffff, 0xff000100,
396 0x802c, 0xffffffff, 0x40000000,
397 0x915c, 0xffffffff, 0x00010000,
398 0x9160, 0xffffffff, 0x00030002,
399 0x9178, 0xffffffff, 0x00070000,
400 0x917c, 0xffffffff, 0x00030002,
401 0x9180, 0xffffffff, 0x00050004,
402 0x918c, 0xffffffff, 0x00010006,
403 0x9190, 0xffffffff, 0x00090008,
404 0x9194, 0xffffffff, 0x00070000,
405 0x9198, 0xffffffff, 0x00030002,
406 0x919c, 0xffffffff, 0x00050004,
407 0x91a8, 0xffffffff, 0x00010006,
408 0x91ac, 0xffffffff, 0x00090008,
409 0x91b0, 0xffffffff, 0x00070000,
410 0x91b4, 0xffffffff, 0x00030002,
411 0x91b8, 0xffffffff, 0x00050004,
412 0x91c4, 0xffffffff, 0x00010006,
413 0x91c8, 0xffffffff, 0x00090008,
414 0x91cc, 0xffffffff, 0x00070000,
415 0x91d0, 0xffffffff, 0x00030002,
416 0x91d4, 0xffffffff, 0x00050004,
417 0x91e0, 0xffffffff, 0x00010006,
418 0x91e4, 0xffffffff, 0x00090008,
419 0x91e8, 0xffffffff, 0x00000000,
420 0x91ec, 0xffffffff, 0x00070000,
421 0x91f0, 0xffffffff, 0x00030002,
422 0x91f4, 0xffffffff, 0x00050004,
423 0x9200, 0xffffffff, 0x00010006,
424 0x9204, 0xffffffff, 0x00090008,
425 0x9294, 0xffffffff, 0x00000000,
426 0x929c, 0xffffffff, 0x00000001,
427 0x802c, 0xffffffff, 0xc0000000
428};
429
430static const u32 cedar_golden_registers[] =
431{
432 0x3f90, 0xffff0000, 0xff000000,
433 0x9148, 0xffff0000, 0xff000000,
434 0x3f94, 0xffff0000, 0xff000000,
435 0x914c, 0xffff0000, 0xff000000,
436 0x9b7c, 0xffffffff, 0x00000000,
437 0x8a14, 0xffffffff, 0x00000007,
438 0x8b10, 0xffffffff, 0x00000000,
439 0x960c, 0xffffffff, 0x54763210,
440 0x88c4, 0xffffffff, 0x000000c2,
441 0x88d4, 0xffffffff, 0x00000000,
442 0x8974, 0xffffffff, 0x00000000,
443 0xc78, 0x00000080, 0x00000080,
444 0x5eb4, 0xffffffff, 0x00000002,
445 0x5e78, 0xffffffff, 0x001000f0,
446 0x6104, 0x01000300, 0x00000000,
447 0x5bc0, 0x00300000, 0x00000000,
448 0x7030, 0xffffffff, 0x00000011,
449 0x7c30, 0xffffffff, 0x00000011,
450 0x10830, 0xffffffff, 0x00000011,
451 0x11430, 0xffffffff, 0x00000011,
452 0xd02c, 0xffffffff, 0x08421000,
453 0x240c, 0xffffffff, 0x00000380,
454 0x8b24, 0xffffffff, 0x00ff0fff,
455 0x28a4c, 0x06000000, 0x06000000,
456 0x10c, 0x00000001, 0x00000001,
457 0x8d00, 0xffffffff, 0x100e4848,
458 0x8d04, 0xffffffff, 0x00164745,
459 0x8c00, 0xffffffff, 0xe4000003,
460 0x8c04, 0xffffffff, 0x40600060,
461 0x8c08, 0xffffffff, 0x001c001c,
462 0x8cf0, 0xffffffff, 0x08e00410,
463 0x8c20, 0xffffffff, 0x00800080,
464 0x8c24, 0xffffffff, 0x00800080,
465 0x8c18, 0xffffffff, 0x20202078,
466 0x8c1c, 0xffffffff, 0x00001010,
467 0x28350, 0xffffffff, 0x00000000,
468 0xa008, 0xffffffff, 0x00010000,
469 0x5cc, 0xffffffff, 0x00000001,
470 0x9508, 0xffffffff, 0x00000002
471};
472
473static const u32 cedar_mgcg_init[] =
474{
475 0x802c, 0xffffffff, 0xc0000000,
476 0x5448, 0xffffffff, 0x00000100,
477 0x55e4, 0xffffffff, 0x00000100,
478 0x160c, 0xffffffff, 0x00000100,
479 0x5644, 0xffffffff, 0x00000100,
480 0xc164, 0xffffffff, 0x00000100,
481 0x8a18, 0xffffffff, 0x00000100,
482 0x897c, 0xffffffff, 0x06000100,
483 0x8b28, 0xffffffff, 0x00000100,
484 0x9144, 0xffffffff, 0x00000100,
485 0x9a60, 0xffffffff, 0x00000100,
486 0x9868, 0xffffffff, 0x00000100,
487 0x8d58, 0xffffffff, 0x00000100,
488 0x9510, 0xffffffff, 0x00000100,
489 0x949c, 0xffffffff, 0x00000100,
490 0x9654, 0xffffffff, 0x00000100,
491 0x9030, 0xffffffff, 0x00000100,
492 0x9034, 0xffffffff, 0x00000100,
493 0x9038, 0xffffffff, 0x00000100,
494 0x903c, 0xffffffff, 0x00000100,
495 0x9040, 0xffffffff, 0x00000100,
496 0xa200, 0xffffffff, 0x00000100,
497 0xa204, 0xffffffff, 0x00000100,
498 0xa208, 0xffffffff, 0x00000100,
499 0xa20c, 0xffffffff, 0x00000100,
500 0x971c, 0xffffffff, 0x00000100,
501 0x977c, 0xffffffff, 0x00000100,
502 0x3f80, 0xffffffff, 0x00000100,
503 0xa210, 0xffffffff, 0x00000100,
504 0xa214, 0xffffffff, 0x00000100,
505 0x4d8, 0xffffffff, 0x00000100,
506 0x9784, 0xffffffff, 0x00000100,
507 0x9698, 0xffffffff, 0x00000100,
508 0x4d4, 0xffffffff, 0x00000200,
509 0x30cc, 0xffffffff, 0x00000100,
510 0xd0c0, 0xffffffff, 0xff000100,
511 0x802c, 0xffffffff, 0x40000000,
512 0x915c, 0xffffffff, 0x00010000,
513 0x9178, 0xffffffff, 0x00050000,
514 0x917c, 0xffffffff, 0x00030002,
515 0x918c, 0xffffffff, 0x00010004,
516 0x9190, 0xffffffff, 0x00070006,
517 0x9194, 0xffffffff, 0x00050000,
518 0x9198, 0xffffffff, 0x00030002,
519 0x91a8, 0xffffffff, 0x00010004,
520 0x91ac, 0xffffffff, 0x00070006,
521 0x91e8, 0xffffffff, 0x00000000,
522 0x9294, 0xffffffff, 0x00000000,
523 0x929c, 0xffffffff, 0x00000001,
524 0x802c, 0xffffffff, 0xc0000000
525};
526
527static const u32 juniper_mgcg_init[] =
528{
529 0x802c, 0xffffffff, 0xc0000000,
530 0x5448, 0xffffffff, 0x00000100,
531 0x55e4, 0xffffffff, 0x00000100,
532 0x160c, 0xffffffff, 0x00000100,
533 0x5644, 0xffffffff, 0x00000100,
534 0xc164, 0xffffffff, 0x00000100,
535 0x8a18, 0xffffffff, 0x00000100,
536 0x897c, 0xffffffff, 0x06000100,
537 0x8b28, 0xffffffff, 0x00000100,
538 0x9144, 0xffffffff, 0x00000100,
539 0x9a60, 0xffffffff, 0x00000100,
540 0x9868, 0xffffffff, 0x00000100,
541 0x8d58, 0xffffffff, 0x00000100,
542 0x9510, 0xffffffff, 0x00000100,
543 0x949c, 0xffffffff, 0x00000100,
544 0x9654, 0xffffffff, 0x00000100,
545 0x9030, 0xffffffff, 0x00000100,
546 0x9034, 0xffffffff, 0x00000100,
547 0x9038, 0xffffffff, 0x00000100,
548 0x903c, 0xffffffff, 0x00000100,
549 0x9040, 0xffffffff, 0x00000100,
550 0xa200, 0xffffffff, 0x00000100,
551 0xa204, 0xffffffff, 0x00000100,
552 0xa208, 0xffffffff, 0x00000100,
553 0xa20c, 0xffffffff, 0x00000100,
554 0x971c, 0xffffffff, 0x00000100,
555 0xd0c0, 0xffffffff, 0xff000100,
556 0x802c, 0xffffffff, 0x40000000,
557 0x915c, 0xffffffff, 0x00010000,
558 0x9160, 0xffffffff, 0x00030002,
559 0x9178, 0xffffffff, 0x00070000,
560 0x917c, 0xffffffff, 0x00030002,
561 0x9180, 0xffffffff, 0x00050004,
562 0x918c, 0xffffffff, 0x00010006,
563 0x9190, 0xffffffff, 0x00090008,
564 0x9194, 0xffffffff, 0x00070000,
565 0x9198, 0xffffffff, 0x00030002,
566 0x919c, 0xffffffff, 0x00050004,
567 0x91a8, 0xffffffff, 0x00010006,
568 0x91ac, 0xffffffff, 0x00090008,
569 0x91b0, 0xffffffff, 0x00070000,
570 0x91b4, 0xffffffff, 0x00030002,
571 0x91b8, 0xffffffff, 0x00050004,
572 0x91c4, 0xffffffff, 0x00010006,
573 0x91c8, 0xffffffff, 0x00090008,
574 0x91cc, 0xffffffff, 0x00070000,
575 0x91d0, 0xffffffff, 0x00030002,
576 0x91d4, 0xffffffff, 0x00050004,
577 0x91e0, 0xffffffff, 0x00010006,
578 0x91e4, 0xffffffff, 0x00090008,
579 0x91e8, 0xffffffff, 0x00000000,
580 0x91ec, 0xffffffff, 0x00070000,
581 0x91f0, 0xffffffff, 0x00030002,
582 0x91f4, 0xffffffff, 0x00050004,
583 0x9200, 0xffffffff, 0x00010006,
584 0x9204, 0xffffffff, 0x00090008,
585 0x9208, 0xffffffff, 0x00070000,
586 0x920c, 0xffffffff, 0x00030002,
587 0x9210, 0xffffffff, 0x00050004,
588 0x921c, 0xffffffff, 0x00010006,
589 0x9220, 0xffffffff, 0x00090008,
590 0x9224, 0xffffffff, 0x00070000,
591 0x9228, 0xffffffff, 0x00030002,
592 0x922c, 0xffffffff, 0x00050004,
593 0x9238, 0xffffffff, 0x00010006,
594 0x923c, 0xffffffff, 0x00090008,
595 0x9240, 0xffffffff, 0x00070000,
596 0x9244, 0xffffffff, 0x00030002,
597 0x9248, 0xffffffff, 0x00050004,
598 0x9254, 0xffffffff, 0x00010006,
599 0x9258, 0xffffffff, 0x00090008,
600 0x925c, 0xffffffff, 0x00070000,
601 0x9260, 0xffffffff, 0x00030002,
602 0x9264, 0xffffffff, 0x00050004,
603 0x9270, 0xffffffff, 0x00010006,
604 0x9274, 0xffffffff, 0x00090008,
605 0x9278, 0xffffffff, 0x00070000,
606 0x927c, 0xffffffff, 0x00030002,
607 0x9280, 0xffffffff, 0x00050004,
608 0x928c, 0xffffffff, 0x00010006,
609 0x9290, 0xffffffff, 0x00090008,
610 0x9294, 0xffffffff, 0x00000000,
611 0x929c, 0xffffffff, 0x00000001,
612 0x802c, 0xffffffff, 0xc0000000,
613 0x977c, 0xffffffff, 0x00000100,
614 0x3f80, 0xffffffff, 0x00000100,
615 0xa210, 0xffffffff, 0x00000100,
616 0xa214, 0xffffffff, 0x00000100,
617 0x4d8, 0xffffffff, 0x00000100,
618 0x9784, 0xffffffff, 0x00000100,
619 0x9698, 0xffffffff, 0x00000100,
620 0x4d4, 0xffffffff, 0x00000200,
621 0x30cc, 0xffffffff, 0x00000100,
622 0x802c, 0xffffffff, 0xc0000000
623};
624
625static const u32 supersumo_golden_registers[] =
626{
627 0x5eb4, 0xffffffff, 0x00000002,
628 0x5cc, 0xffffffff, 0x00000001,
629 0x7030, 0xffffffff, 0x00000011,
630 0x7c30, 0xffffffff, 0x00000011,
631 0x6104, 0x01000300, 0x00000000,
632 0x5bc0, 0x00300000, 0x00000000,
633 0x8c04, 0xffffffff, 0x40600060,
634 0x8c08, 0xffffffff, 0x001c001c,
635 0x8c20, 0xffffffff, 0x00800080,
636 0x8c24, 0xffffffff, 0x00800080,
637 0x8c18, 0xffffffff, 0x20202078,
638 0x8c1c, 0xffffffff, 0x00001010,
639 0x918c, 0xffffffff, 0x00010006,
640 0x91a8, 0xffffffff, 0x00010006,
641 0x91c4, 0xffffffff, 0x00010006,
642 0x91e0, 0xffffffff, 0x00010006,
643 0x9200, 0xffffffff, 0x00010006,
644 0x9150, 0xffffffff, 0x6e944040,
645 0x917c, 0xffffffff, 0x00030002,
646 0x9180, 0xffffffff, 0x00050004,
647 0x9198, 0xffffffff, 0x00030002,
648 0x919c, 0xffffffff, 0x00050004,
649 0x91b4, 0xffffffff, 0x00030002,
650 0x91b8, 0xffffffff, 0x00050004,
651 0x91d0, 0xffffffff, 0x00030002,
652 0x91d4, 0xffffffff, 0x00050004,
653 0x91f0, 0xffffffff, 0x00030002,
654 0x91f4, 0xffffffff, 0x00050004,
655 0x915c, 0xffffffff, 0x00010000,
656 0x9160, 0xffffffff, 0x00030002,
657 0x3f90, 0xffff0000, 0xff000000,
658 0x9178, 0xffffffff, 0x00070000,
659 0x9194, 0xffffffff, 0x00070000,
660 0x91b0, 0xffffffff, 0x00070000,
661 0x91cc, 0xffffffff, 0x00070000,
662 0x91ec, 0xffffffff, 0x00070000,
663 0x9148, 0xffff0000, 0xff000000,
664 0x9190, 0xffffffff, 0x00090008,
665 0x91ac, 0xffffffff, 0x00090008,
666 0x91c8, 0xffffffff, 0x00090008,
667 0x91e4, 0xffffffff, 0x00090008,
668 0x9204, 0xffffffff, 0x00090008,
669 0x3f94, 0xffff0000, 0xff000000,
670 0x914c, 0xffff0000, 0xff000000,
671 0x929c, 0xffffffff, 0x00000001,
672 0x8a18, 0xffffffff, 0x00000100,
673 0x8b28, 0xffffffff, 0x00000100,
674 0x9144, 0xffffffff, 0x00000100,
675 0x5644, 0xffffffff, 0x00000100,
676 0x9b7c, 0xffffffff, 0x00000000,
677 0x8030, 0xffffffff, 0x0000100a,
678 0x8a14, 0xffffffff, 0x00000007,
679 0x8b24, 0xffffffff, 0x00ff0fff,
680 0x8b10, 0xffffffff, 0x00000000,
681 0x28a4c, 0x06000000, 0x06000000,
682 0x4d8, 0xffffffff, 0x00000100,
683 0x913c, 0xffff000f, 0x0100000a,
684 0x960c, 0xffffffff, 0x54763210,
685 0x88c4, 0xffffffff, 0x000000c2,
686 0x88d4, 0xffffffff, 0x00000010,
687 0x8974, 0xffffffff, 0x00000000,
688 0xc78, 0x00000080, 0x00000080,
689 0x5e78, 0xffffffff, 0x001000f0,
690 0xd02c, 0xffffffff, 0x08421000,
691 0xa008, 0xffffffff, 0x00010000,
692 0x8d00, 0xffffffff, 0x100e4848,
693 0x8d04, 0xffffffff, 0x00164745,
694 0x8c00, 0xffffffff, 0xe4000003,
695 0x8cf0, 0x1fffffff, 0x08e00620,
696 0x28350, 0xffffffff, 0x00000000,
697 0x9508, 0xffffffff, 0x00000002
698};
699
700static const u32 sumo_golden_registers[] =
701{
702 0x900c, 0x00ffffff, 0x0017071f,
703 0x8c18, 0xffffffff, 0x10101060,
704 0x8c1c, 0xffffffff, 0x00001010,
705 0x8c30, 0x0000000f, 0x00000005,
706 0x9688, 0x0000000f, 0x00000007
707};
708
709static const u32 wrestler_golden_registers[] =
710{
711 0x5eb4, 0xffffffff, 0x00000002,
712 0x5cc, 0xffffffff, 0x00000001,
713 0x7030, 0xffffffff, 0x00000011,
714 0x7c30, 0xffffffff, 0x00000011,
715 0x6104, 0x01000300, 0x00000000,
716 0x5bc0, 0x00300000, 0x00000000,
717 0x918c, 0xffffffff, 0x00010006,
718 0x91a8, 0xffffffff, 0x00010006,
719 0x9150, 0xffffffff, 0x6e944040,
720 0x917c, 0xffffffff, 0x00030002,
721 0x9198, 0xffffffff, 0x00030002,
722 0x915c, 0xffffffff, 0x00010000,
723 0x3f90, 0xffff0000, 0xff000000,
724 0x9178, 0xffffffff, 0x00070000,
725 0x9194, 0xffffffff, 0x00070000,
726 0x9148, 0xffff0000, 0xff000000,
727 0x9190, 0xffffffff, 0x00090008,
728 0x91ac, 0xffffffff, 0x00090008,
729 0x3f94, 0xffff0000, 0xff000000,
730 0x914c, 0xffff0000, 0xff000000,
731 0x929c, 0xffffffff, 0x00000001,
732 0x8a18, 0xffffffff, 0x00000100,
733 0x8b28, 0xffffffff, 0x00000100,
734 0x9144, 0xffffffff, 0x00000100,
735 0x9b7c, 0xffffffff, 0x00000000,
736 0x8030, 0xffffffff, 0x0000100a,
737 0x8a14, 0xffffffff, 0x00000001,
738 0x8b24, 0xffffffff, 0x00ff0fff,
739 0x8b10, 0xffffffff, 0x00000000,
740 0x28a4c, 0x06000000, 0x06000000,
741 0x4d8, 0xffffffff, 0x00000100,
742 0x913c, 0xffff000f, 0x0100000a,
743 0x960c, 0xffffffff, 0x54763210,
744 0x88c4, 0xffffffff, 0x000000c2,
745 0x88d4, 0xffffffff, 0x00000010,
746 0x8974, 0xffffffff, 0x00000000,
747 0xc78, 0x00000080, 0x00000080,
748 0x5e78, 0xffffffff, 0x001000f0,
749 0xd02c, 0xffffffff, 0x08421000,
750 0xa008, 0xffffffff, 0x00010000,
751 0x8d00, 0xffffffff, 0x100e4848,
752 0x8d04, 0xffffffff, 0x00164745,
753 0x8c00, 0xffffffff, 0xe4000003,
754 0x8cf0, 0x1fffffff, 0x08e00410,
755 0x28350, 0xffffffff, 0x00000000,
756 0x9508, 0xffffffff, 0x00000002,
757 0x900c, 0xffffffff, 0x0017071f,
758 0x8c18, 0xffffffff, 0x10101060,
759 0x8c1c, 0xffffffff, 0x00001010
760};
761
762static const u32 barts_golden_registers[] =
763{
764 0x5eb4, 0xffffffff, 0x00000002,
765 0x5e78, 0x8f311ff1, 0x001000f0,
766 0x3f90, 0xffff0000, 0xff000000,
767 0x9148, 0xffff0000, 0xff000000,
768 0x3f94, 0xffff0000, 0xff000000,
769 0x914c, 0xffff0000, 0xff000000,
770 0xc78, 0x00000080, 0x00000080,
771 0xbd4, 0x70073777, 0x00010001,
772 0xd02c, 0xbfffff1f, 0x08421000,
773 0xd0b8, 0x03773777, 0x02011003,
774 0x5bc0, 0x00200000, 0x50100000,
775 0x98f8, 0x33773777, 0x02011003,
776 0x98fc, 0xffffffff, 0x76543210,
777 0x7030, 0x31000311, 0x00000011,
778 0x2f48, 0x00000007, 0x02011003,
779 0x6b28, 0x00000010, 0x00000012,
780 0x7728, 0x00000010, 0x00000012,
781 0x10328, 0x00000010, 0x00000012,
782 0x10f28, 0x00000010, 0x00000012,
783 0x11b28, 0x00000010, 0x00000012,
784 0x12728, 0x00000010, 0x00000012,
785 0x240c, 0x000007ff, 0x00000380,
786 0x8a14, 0xf000001f, 0x00000007,
787 0x8b24, 0x3fff3fff, 0x00ff0fff,
788 0x8b10, 0x0000ff0f, 0x00000000,
789 0x28a4c, 0x07ffffff, 0x06000000,
790 0x10c, 0x00000001, 0x00010003,
791 0xa02c, 0xffffffff, 0x0000009b,
792 0x913c, 0x0000000f, 0x0100000a,
793 0x8d00, 0xffff7f7f, 0x100e4848,
794 0x8d04, 0x00ffffff, 0x00164745,
795 0x8c00, 0xfffc0003, 0xe4000003,
796 0x8c04, 0xf8ff00ff, 0x40600060,
797 0x8c08, 0x00ff00ff, 0x001c001c,
798 0x8cf0, 0x1fff1fff, 0x08e00620,
799 0x8c20, 0x0fff0fff, 0x00800080,
800 0x8c24, 0x0fff0fff, 0x00800080,
801 0x8c18, 0xffffffff, 0x20202078,
802 0x8c1c, 0x0000ffff, 0x00001010,
803 0x28350, 0x00000f01, 0x00000000,
804 0x9508, 0x3700001f, 0x00000002,
805 0x960c, 0xffffffff, 0x54763210,
806 0x88c4, 0x001f3ae3, 0x000000c2,
807 0x88d4, 0x0000001f, 0x00000010,
808 0x8974, 0xffffffff, 0x00000000
809};
810
811static const u32 turks_golden_registers[] =
812{
813 0x5eb4, 0xffffffff, 0x00000002,
814 0x5e78, 0x8f311ff1, 0x001000f0,
815 0x8c8, 0x00003000, 0x00001070,
816 0x8cc, 0x000fffff, 0x00040035,
817 0x3f90, 0xffff0000, 0xfff00000,
818 0x9148, 0xffff0000, 0xfff00000,
819 0x3f94, 0xffff0000, 0xfff00000,
820 0x914c, 0xffff0000, 0xfff00000,
821 0xc78, 0x00000080, 0x00000080,
822 0xbd4, 0x00073007, 0x00010002,
823 0xd02c, 0xbfffff1f, 0x08421000,
824 0xd0b8, 0x03773777, 0x02010002,
825 0x5bc0, 0x00200000, 0x50100000,
826 0x98f8, 0x33773777, 0x00010002,
827 0x98fc, 0xffffffff, 0x33221100,
828 0x7030, 0x31000311, 0x00000011,
829 0x2f48, 0x33773777, 0x00010002,
830 0x6b28, 0x00000010, 0x00000012,
831 0x7728, 0x00000010, 0x00000012,
832 0x10328, 0x00000010, 0x00000012,
833 0x10f28, 0x00000010, 0x00000012,
834 0x11b28, 0x00000010, 0x00000012,
835 0x12728, 0x00000010, 0x00000012,
836 0x240c, 0x000007ff, 0x00000380,
837 0x8a14, 0xf000001f, 0x00000007,
838 0x8b24, 0x3fff3fff, 0x00ff0fff,
839 0x8b10, 0x0000ff0f, 0x00000000,
840 0x28a4c, 0x07ffffff, 0x06000000,
841 0x10c, 0x00000001, 0x00010003,
842 0xa02c, 0xffffffff, 0x0000009b,
843 0x913c, 0x0000000f, 0x0100000a,
844 0x8d00, 0xffff7f7f, 0x100e4848,
845 0x8d04, 0x00ffffff, 0x00164745,
846 0x8c00, 0xfffc0003, 0xe4000003,
847 0x8c04, 0xf8ff00ff, 0x40600060,
848 0x8c08, 0x00ff00ff, 0x001c001c,
849 0x8cf0, 0x1fff1fff, 0x08e00410,
850 0x8c20, 0x0fff0fff, 0x00800080,
851 0x8c24, 0x0fff0fff, 0x00800080,
852 0x8c18, 0xffffffff, 0x20202078,
853 0x8c1c, 0x0000ffff, 0x00001010,
854 0x28350, 0x00000f01, 0x00000000,
855 0x9508, 0x3700001f, 0x00000002,
856 0x960c, 0xffffffff, 0x54763210,
857 0x88c4, 0x001f3ae3, 0x000000c2,
858 0x88d4, 0x0000001f, 0x00000010,
859 0x8974, 0xffffffff, 0x00000000
860};
861
862static const u32 caicos_golden_registers[] =
863{
864 0x5eb4, 0xffffffff, 0x00000002,
865 0x5e78, 0x8f311ff1, 0x001000f0,
866 0x8c8, 0x00003420, 0x00001450,
867 0x8cc, 0x000fffff, 0x00040035,
868 0x3f90, 0xffff0000, 0xfffc0000,
869 0x9148, 0xffff0000, 0xfffc0000,
870 0x3f94, 0xffff0000, 0xfffc0000,
871 0x914c, 0xffff0000, 0xfffc0000,
872 0xc78, 0x00000080, 0x00000080,
873 0xbd4, 0x00073007, 0x00010001,
874 0xd02c, 0xbfffff1f, 0x08421000,
875 0xd0b8, 0x03773777, 0x02010001,
876 0x5bc0, 0x00200000, 0x50100000,
877 0x98f8, 0x33773777, 0x02010001,
878 0x98fc, 0xffffffff, 0x33221100,
879 0x7030, 0x31000311, 0x00000011,
880 0x2f48, 0x33773777, 0x02010001,
881 0x6b28, 0x00000010, 0x00000012,
882 0x7728, 0x00000010, 0x00000012,
883 0x10328, 0x00000010, 0x00000012,
884 0x10f28, 0x00000010, 0x00000012,
885 0x11b28, 0x00000010, 0x00000012,
886 0x12728, 0x00000010, 0x00000012,
887 0x240c, 0x000007ff, 0x00000380,
888 0x8a14, 0xf000001f, 0x00000001,
889 0x8b24, 0x3fff3fff, 0x00ff0fff,
890 0x8b10, 0x0000ff0f, 0x00000000,
891 0x28a4c, 0x07ffffff, 0x06000000,
892 0x10c, 0x00000001, 0x00010003,
893 0xa02c, 0xffffffff, 0x0000009b,
894 0x913c, 0x0000000f, 0x0100000a,
895 0x8d00, 0xffff7f7f, 0x100e4848,
896 0x8d04, 0x00ffffff, 0x00164745,
897 0x8c00, 0xfffc0003, 0xe4000003,
898 0x8c04, 0xf8ff00ff, 0x40600060,
899 0x8c08, 0x00ff00ff, 0x001c001c,
900 0x8cf0, 0x1fff1fff, 0x08e00410,
901 0x8c20, 0x0fff0fff, 0x00800080,
902 0x8c24, 0x0fff0fff, 0x00800080,
903 0x8c18, 0xffffffff, 0x20202078,
904 0x8c1c, 0x0000ffff, 0x00001010,
905 0x28350, 0x00000f01, 0x00000000,
906 0x9508, 0x3700001f, 0x00000002,
907 0x960c, 0xffffffff, 0x54763210,
908 0x88c4, 0x001f3ae3, 0x000000c2,
909 0x88d4, 0x0000001f, 0x00000010,
910 0x8974, 0xffffffff, 0x00000000
911};
912
913static void evergreen_init_golden_registers(struct radeon_device *rdev)
914{
915 switch (rdev->family) {
916 case CHIP_CYPRESS:
917 case CHIP_HEMLOCK:
918 radeon_program_register_sequence(rdev,
919 evergreen_golden_registers,
920 (const u32)ARRAY_SIZE(evergreen_golden_registers));
921 radeon_program_register_sequence(rdev,
922 evergreen_golden_registers2,
923 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
924 radeon_program_register_sequence(rdev,
925 cypress_mgcg_init,
926 (const u32)ARRAY_SIZE(cypress_mgcg_init));
927 break;
928 case CHIP_JUNIPER:
929 radeon_program_register_sequence(rdev,
930 evergreen_golden_registers,
931 (const u32)ARRAY_SIZE(evergreen_golden_registers));
932 radeon_program_register_sequence(rdev,
933 evergreen_golden_registers2,
934 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
935 radeon_program_register_sequence(rdev,
936 juniper_mgcg_init,
937 (const u32)ARRAY_SIZE(juniper_mgcg_init));
938 break;
939 case CHIP_REDWOOD:
940 radeon_program_register_sequence(rdev,
941 evergreen_golden_registers,
942 (const u32)ARRAY_SIZE(evergreen_golden_registers));
943 radeon_program_register_sequence(rdev,
944 evergreen_golden_registers2,
945 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
946 radeon_program_register_sequence(rdev,
947 redwood_mgcg_init,
948 (const u32)ARRAY_SIZE(redwood_mgcg_init));
949 break;
950 case CHIP_CEDAR:
951 radeon_program_register_sequence(rdev,
952 cedar_golden_registers,
953 (const u32)ARRAY_SIZE(cedar_golden_registers));
954 radeon_program_register_sequence(rdev,
955 evergreen_golden_registers2,
956 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
957 radeon_program_register_sequence(rdev,
958 cedar_mgcg_init,
959 (const u32)ARRAY_SIZE(cedar_mgcg_init));
960 break;
961 case CHIP_PALM:
962 radeon_program_register_sequence(rdev,
963 wrestler_golden_registers,
964 (const u32)ARRAY_SIZE(wrestler_golden_registers));
965 break;
966 case CHIP_SUMO:
967 radeon_program_register_sequence(rdev,
968 supersumo_golden_registers,
969 (const u32)ARRAY_SIZE(supersumo_golden_registers));
970 break;
971 case CHIP_SUMO2:
972 radeon_program_register_sequence(rdev,
973 supersumo_golden_registers,
974 (const u32)ARRAY_SIZE(supersumo_golden_registers));
975 radeon_program_register_sequence(rdev,
976 sumo_golden_registers,
977 (const u32)ARRAY_SIZE(sumo_golden_registers));
978 break;
979 case CHIP_BARTS:
980 radeon_program_register_sequence(rdev,
981 barts_golden_registers,
982 (const u32)ARRAY_SIZE(barts_golden_registers));
983 break;
984 case CHIP_TURKS:
985 radeon_program_register_sequence(rdev,
986 turks_golden_registers,
987 (const u32)ARRAY_SIZE(turks_golden_registers));
988 break;
989 case CHIP_CAICOS:
990 radeon_program_register_sequence(rdev,
991 caicos_golden_registers,
992 (const u32)ARRAY_SIZE(caicos_golden_registers));
993 break;
994 default:
995 break;
996 }
997}
998
926deccb
FT
999void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1000 unsigned *bankh, unsigned *mtaspect,
1001 unsigned *tile_split)
1002{
1003 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1004 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1005 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1006 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1007 switch (*bankw) {
1008 default:
1009 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1010 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1011 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1012 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1013 }
1014 switch (*bankh) {
1015 default:
1016 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1017 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1018 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1019 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1020 }
1021 switch (*mtaspect) {
1022 default:
1023 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1024 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1025 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1026 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1027 }
1028}
1029
f43cf1b1
MN
1030static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1031 u32 cntl_reg, u32 status_reg)
1032{
1033 int r, i;
1034 struct atom_clock_dividers dividers;
1035
1036 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1037 clock, false, &dividers);
1038 if (r)
1039 return r;
1040
1041 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1042
1043 for (i = 0; i < 100; i++) {
1044 if (RREG32(status_reg) & DCLK_STATUS)
1045 break;
1046 DRM_MDELAY(10);
1047 }
1048 if (i == 100)
1049 return -ETIMEDOUT;
1050
1051 return 0;
1052}
1053
1054int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1055{
1056 int r = 0;
1057 u32 cg_scratch = RREG32(CG_SCRATCH1);
1058
1059 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1060 if (r)
1061 goto done;
1062 cg_scratch &= 0xffff0000;
1063 cg_scratch |= vclk / 100; /* Mhz */
1064
1065 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1066 if (r)
1067 goto done;
1068 cg_scratch &= 0x0000ffff;
1069 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1070
1071done:
1072 WREG32(CG_SCRATCH1, cg_scratch);
1073
1074 return r;
1075}
1076
1077int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1078{
1079 /* start off with something large */
1080 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1081 int r;
1082
1083 /* bypass vclk and dclk with bclk */
1084 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1085 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1086 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1087
1088 /* put PLL in bypass mode */
1089 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1090
1091 if (!vclk || !dclk) {
1092 /* keep the Bypass mode, put PLL to sleep */
1093 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1094 return 0;
1095 }
1096
1097 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1098 16384, 0x03FFFFFF, 0, 128, 5,
1099 &fb_div, &vclk_div, &dclk_div);
1100 if (r)
1101 return r;
1102
1103 /* set VCO_MODE to 1 */
1104 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1105
1106 /* toggle UPLL_SLEEP to 1 then back to 0 */
1107 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1108 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1109
1110 /* deassert UPLL_RESET */
1111 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1112
1113 DRM_MDELAY(1);
1114
1115 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1116 if (r)
1117 return r;
1118
1119 /* assert UPLL_RESET again */
1120 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1121
1122 /* disable spread spectrum. */
1123 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1124
1125 /* set feedback divider */
1126 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1127
1128 /* set ref divider to 0 */
1129 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1130
1131 if (fb_div < 307200)
1132 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1133 else
1134 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1135
1136 /* set PDIV_A and PDIV_B */
1137 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1138 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1139 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1140
1141 /* give the PLL some time to settle */
1142 DRM_MDELAY(15);
1143
1144 /* deassert PLL_RESET */
1145 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1146
1147 DRM_MDELAY(15);
1148
1149 /* switch from bypass mode to normal mode */
1150 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1151
1152 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1153 if (r)
1154 return r;
1155
1156 /* switch VCLK and DCLK selection */
1157 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1158 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1159 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1160
1161 DRM_MDELAY(100);
1162
1163 return 0;
1164}
1165
926deccb
FT
1166void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1167{
1168 u16 ctl, v;
1169 int err, cap;
1170
1171 err = pci_find_extcap(rdev->dev, PCIY_EXPRESS, &cap);
1172 if (err)
1173 return;
1174
1175 cap += PCIER_DEVCTRL;
1176
1177 ctl = pci_read_config(rdev->dev, cap, 2);
1178
1179 v = (ctl & PCIEM_DEVCTL_MAX_READRQ_MASK) >> 12;
1180
1181 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1182 * to avoid hangs or perfomance issues
1183 */
1184 if ((v == 0) || (v == 6) || (v == 7)) {
1185 ctl &= ~PCIEM_DEVCTL_MAX_READRQ_MASK;
1186 ctl |= (2 << 12);
1187 pci_write_config(rdev->dev, cap, ctl, 2);
1188 }
1189}
1190
f43cf1b1
MN
1191static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1192{
1193 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1194 return true;
1195 else
1196 return false;
1197}
1198
1199static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1200{
1201 u32 pos1, pos2;
1202
1203 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1204 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1205
1206 if (pos1 != pos2)
1207 return true;
1208 else
1209 return false;
1210}
1211
926deccb
FT
1212/**
1213 * dce4_wait_for_vblank - vblank wait asic callback.
1214 *
1215 * @rdev: radeon_device pointer
1216 * @crtc: crtc to wait for vblank on
1217 *
1218 * Wait for vblank on the requested crtc (evergreen+).
1219 */
1220void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1221{
f43cf1b1 1222 unsigned i = 0;
926deccb
FT
1223
1224 if (crtc >= rdev->num_crtc)
1225 return;
1226
f43cf1b1
MN
1227 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1228 return;
1229
1230 /* depending on when we hit vblank, we may be close to active; if so,
1231 * wait for another frame.
1232 */
1233 while (dce4_is_in_vblank(rdev, crtc)) {
1234 if (i++ % 100 == 0) {
1235 if (!dce4_is_counter_moving(rdev, crtc))
926deccb 1236 break;
926deccb 1237 }
f43cf1b1
MN
1238 }
1239
1240 while (!dce4_is_in_vblank(rdev, crtc)) {
1241 if (i++ % 100 == 0) {
1242 if (!dce4_is_counter_moving(rdev, crtc))
926deccb 1243 break;
926deccb
FT
1244 }
1245 }
1246}
1247
1248/**
1249 * radeon_irq_kms_pflip_irq_get - pre-pageflip callback.
1250 *
1251 * @rdev: radeon_device pointer
1252 * @crtc: crtc to prepare for pageflip on
1253 *
1254 * Pre-pageflip callback (evergreen+).
1255 * Enables the pageflip irq (vblank irq).
1256 */
1257void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc)
1258{
1259 /* enable the pflip int */
1260 radeon_irq_kms_pflip_irq_get(rdev, crtc);
1261}
1262
1263/**
1264 * evergreen_post_page_flip - pos-pageflip callback.
1265 *
1266 * @rdev: radeon_device pointer
1267 * @crtc: crtc to cleanup pageflip on
1268 *
1269 * Post-pageflip callback (evergreen+).
1270 * Disables the pageflip irq (vblank irq).
1271 */
1272void evergreen_post_page_flip(struct radeon_device *rdev, int crtc)
1273{
1274 /* disable the pflip int */
1275 radeon_irq_kms_pflip_irq_put(rdev, crtc);
1276}
1277
1278/**
1279 * evergreen_page_flip - pageflip callback.
1280 *
1281 * @rdev: radeon_device pointer
1282 * @crtc_id: crtc to cleanup pageflip on
1283 * @crtc_base: new address of the crtc (GPU MC address)
1284 *
1285 * Does the actual pageflip (evergreen+).
1286 * During vblank we take the crtc lock and wait for the update_pending
1287 * bit to go high, when it does, we release the lock, and allow the
1288 * double buffered update to take place.
1289 * Returns the current update pending status.
1290 */
1291u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1292{
1293 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1294 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
1295 int i;
1296
1297 /* Lock the graphics update lock */
1298 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1299 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1300
1301 /* update the scanout addresses */
1302 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1303 upper_32_bits(crtc_base));
1304 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1305 (u32)crtc_base);
1306
1307 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1308 upper_32_bits(crtc_base));
1309 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1310 (u32)crtc_base);
1311
1312 /* Wait for update_pending to go high. */
1313 for (i = 0; i < rdev->usec_timeout; i++) {
1314 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1315 break;
1316 DRM_UDELAY(1);
1317 }
1318 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1319
1320 /* Unlock the lock, so double-buffering can take place inside vblank */
1321 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1322 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1323
1324 /* Return current update_pending status: */
1325 return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
1326}
1327
1328/* get temperature in millidegrees */
1329int evergreen_get_temp(struct radeon_device *rdev)
1330{
1331 u32 temp, toffset;
1332 int actual_temp = 0;
1333
1334 if (rdev->family == CHIP_JUNIPER) {
1335 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1336 TOFFSET_SHIFT;
1337 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1338 TS0_ADC_DOUT_SHIFT;
1339
1340 if (toffset & 0x100)
1341 actual_temp = temp / 2 - (0x200 - toffset);
1342 else
1343 actual_temp = temp / 2 + toffset;
1344
1345 actual_temp = actual_temp * 1000;
1346
1347 } else {
1348 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1349 ASIC_T_SHIFT;
1350
1351 if (temp & 0x400)
1352 actual_temp = -256;
1353 else if (temp & 0x200)
1354 actual_temp = 255;
1355 else if (temp & 0x100) {
1356 actual_temp = temp & 0x1ff;
1357 actual_temp |= ~0x1ff;
1358 } else
1359 actual_temp = temp & 0xff;
1360
1361 actual_temp = (actual_temp * 1000) / 2;
1362 }
1363
1364 return actual_temp;
1365}
1366
1367int sumo_get_temp(struct radeon_device *rdev)
1368{
1369 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1370 int actual_temp = temp - 49;
1371
1372 return actual_temp * 1000;
1373}
1374
1375/**
1376 * sumo_pm_init_profile - Initialize power profiles callback.
1377 *
1378 * @rdev: radeon_device pointer
1379 *
1380 * Initialize the power states used in profile mode
1381 * (sumo, trinity, SI).
1382 * Used for profile mode only.
1383 */
1384void sumo_pm_init_profile(struct radeon_device *rdev)
1385{
1386 int idx;
1387
1388 /* default */
1389 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1390 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1391 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1392 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1393
1394 /* low,mid sh/mh */
1395 if (rdev->flags & RADEON_IS_MOBILITY)
1396 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1397 else
1398 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1399
1400 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1401 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1402 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1403 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1404
1405 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1406 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1407 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1408 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1409
1410 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1411 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1412 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1413 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1414
1415 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1416 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1417 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1418 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1419
1420 /* high sh/mh */
1421 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1422 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1423 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1424 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1425 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1426 rdev->pm.power_state[idx].num_clock_modes - 1;
1427
1428 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1429 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1430 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1431 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1432 rdev->pm.power_state[idx].num_clock_modes - 1;
1433}
1434
1435/**
1436 * btc_pm_init_profile - Initialize power profiles callback.
1437 *
1438 * @rdev: radeon_device pointer
1439 *
1440 * Initialize the power states used in profile mode
1441 * (BTC, cayman).
1442 * Used for profile mode only.
1443 */
1444void btc_pm_init_profile(struct radeon_device *rdev)
1445{
1446 int idx;
1447
1448 /* default */
1449 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1450 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1451 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1452 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1453 /* starting with BTC, there is one state that is used for both
1454 * MH and SH. Difference is that we always use the high clock index for
1455 * mclk.
1456 */
1457 if (rdev->flags & RADEON_IS_MOBILITY)
1458 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1459 else
1460 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1461 /* low sh */
1462 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1463 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1464 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1465 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1466 /* mid sh */
1467 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1468 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1469 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1470 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1471 /* high sh */
1472 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1473 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1474 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1475 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1476 /* low mh */
1477 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1478 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1479 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1480 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1481 /* mid mh */
1482 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1483 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1484 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1485 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1486 /* high mh */
1487 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1488 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1489 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1490 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1491}
1492
1493/**
1494 * evergreen_pm_misc - set additional pm hw parameters callback.
1495 *
1496 * @rdev: radeon_device pointer
1497 *
1498 * Set non-clock parameters associated with a power state
1499 * (voltage, etc.) (evergreen+).
1500 */
1501void evergreen_pm_misc(struct radeon_device *rdev)
1502{
1503 int req_ps_idx = rdev->pm.requested_power_state_index;
1504 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1505 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1506 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1507
1508 if (voltage->type == VOLTAGE_SW) {
57e252bf
MN
1509 /* 0xff0x are flags rather then an actual voltage */
1510 if ((voltage->voltage & 0xff00) == 0xff00)
926deccb
FT
1511 return;
1512 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1513 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1514 rdev->pm.current_vddc = voltage->voltage;
1515 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1516 }
b403bed8
MN
1517
1518 /* starting with BTC, there is one state that is used for both
1519 * MH and SH. Difference is that we always use the high clock index for
1520 * mclk and vddci.
1521 */
1522 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1523 (rdev->family >= CHIP_BARTS) &&
1524 rdev->pm.active_crtc_count &&
1525 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1526 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1527 voltage = &rdev->pm.power_state[req_ps_idx].
1528 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1529
57e252bf
MN
1530 /* 0xff0x are flags rather then an actual voltage */
1531 if ((voltage->vddci & 0xff00) == 0xff00)
926deccb
FT
1532 return;
1533 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1534 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1535 rdev->pm.current_vddci = voltage->vddci;
1536 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1537 }
1538 }
1539}
1540
1541/**
1542 * evergreen_pm_prepare - pre-power state change callback.
1543 *
1544 * @rdev: radeon_device pointer
1545 *
1546 * Prepare for a power state change (evergreen+).
1547 */
1548void evergreen_pm_prepare(struct radeon_device *rdev)
1549{
1550 struct drm_device *ddev = rdev->ddev;
1551 struct drm_crtc *crtc;
1552 struct radeon_crtc *radeon_crtc;
1553 u32 tmp;
1554
1555 /* disable any active CRTCs */
1556 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1557 radeon_crtc = to_radeon_crtc(crtc);
1558 if (radeon_crtc->enabled) {
1559 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1560 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1561 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1562 }
1563 }
1564}
1565
1566/**
1567 * evergreen_pm_finish - post-power state change callback.
1568 *
1569 * @rdev: radeon_device pointer
1570 *
1571 * Clean up after a power state change (evergreen+).
1572 */
1573void evergreen_pm_finish(struct radeon_device *rdev)
1574{
1575 struct drm_device *ddev = rdev->ddev;
1576 struct drm_crtc *crtc;
1577 struct radeon_crtc *radeon_crtc;
1578 u32 tmp;
1579
1580 /* enable any active CRTCs */
1581 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1582 radeon_crtc = to_radeon_crtc(crtc);
1583 if (radeon_crtc->enabled) {
1584 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1585 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1586 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1587 }
1588 }
1589}
1590
1591/**
1592 * evergreen_hpd_sense - hpd sense callback.
1593 *
1594 * @rdev: radeon_device pointer
1595 * @hpd: hpd (hotplug detect) pin
1596 *
1597 * Checks if a digital monitor is connected (evergreen+).
1598 * Returns true if connected, false if not connected.
1599 */
1600bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1601{
1602 bool connected = false;
1603
1604 switch (hpd) {
1605 case RADEON_HPD_1:
1606 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1607 connected = true;
1608 break;
1609 case RADEON_HPD_2:
1610 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1611 connected = true;
1612 break;
1613 case RADEON_HPD_3:
1614 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1615 connected = true;
1616 break;
1617 case RADEON_HPD_4:
1618 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1619 connected = true;
1620 break;
1621 case RADEON_HPD_5:
1622 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1623 connected = true;
1624 break;
1625 case RADEON_HPD_6:
1626 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1627 connected = true;
1628 break;
1629 default:
1630 break;
1631 }
1632
1633 return connected;
1634}
1635
1636/**
1637 * evergreen_hpd_set_polarity - hpd set polarity callback.
1638 *
1639 * @rdev: radeon_device pointer
1640 * @hpd: hpd (hotplug detect) pin
1641 *
1642 * Set the polarity of the hpd pin (evergreen+).
1643 */
1644void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1645 enum radeon_hpd_id hpd)
1646{
1647 u32 tmp;
1648 bool connected = evergreen_hpd_sense(rdev, hpd);
1649
1650 switch (hpd) {
1651 case RADEON_HPD_1:
1652 tmp = RREG32(DC_HPD1_INT_CONTROL);
1653 if (connected)
1654 tmp &= ~DC_HPDx_INT_POLARITY;
1655 else
1656 tmp |= DC_HPDx_INT_POLARITY;
1657 WREG32(DC_HPD1_INT_CONTROL, tmp);
1658 break;
1659 case RADEON_HPD_2:
1660 tmp = RREG32(DC_HPD2_INT_CONTROL);
1661 if (connected)
1662 tmp &= ~DC_HPDx_INT_POLARITY;
1663 else
1664 tmp |= DC_HPDx_INT_POLARITY;
1665 WREG32(DC_HPD2_INT_CONTROL, tmp);
1666 break;
1667 case RADEON_HPD_3:
1668 tmp = RREG32(DC_HPD3_INT_CONTROL);
1669 if (connected)
1670 tmp &= ~DC_HPDx_INT_POLARITY;
1671 else
1672 tmp |= DC_HPDx_INT_POLARITY;
1673 WREG32(DC_HPD3_INT_CONTROL, tmp);
1674 break;
1675 case RADEON_HPD_4:
1676 tmp = RREG32(DC_HPD4_INT_CONTROL);
1677 if (connected)
1678 tmp &= ~DC_HPDx_INT_POLARITY;
1679 else
1680 tmp |= DC_HPDx_INT_POLARITY;
1681 WREG32(DC_HPD4_INT_CONTROL, tmp);
1682 break;
1683 case RADEON_HPD_5:
1684 tmp = RREG32(DC_HPD5_INT_CONTROL);
1685 if (connected)
1686 tmp &= ~DC_HPDx_INT_POLARITY;
1687 else
1688 tmp |= DC_HPDx_INT_POLARITY;
1689 WREG32(DC_HPD5_INT_CONTROL, tmp);
1690 break;
1691 case RADEON_HPD_6:
1692 tmp = RREG32(DC_HPD6_INT_CONTROL);
1693 if (connected)
1694 tmp &= ~DC_HPDx_INT_POLARITY;
1695 else
1696 tmp |= DC_HPDx_INT_POLARITY;
1697 WREG32(DC_HPD6_INT_CONTROL, tmp);
1698 break;
1699 default:
1700 break;
1701 }
1702}
1703
1704/**
1705 * evergreen_hpd_init - hpd setup callback.
1706 *
1707 * @rdev: radeon_device pointer
1708 *
1709 * Setup the hpd pins used by the card (evergreen+).
1710 * Enable the pin, set the polarity, and enable the hpd interrupts.
1711 */
1712void evergreen_hpd_init(struct radeon_device *rdev)
1713{
1714 struct drm_device *dev = rdev->ddev;
1715 struct drm_connector *connector;
1716 unsigned enabled = 0;
1717 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1718 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1719
1720 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1721 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
f43cf1b1
MN
1722
1723 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1724 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1725 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1726 * aux dp channel on imac and help (but not completely fix)
1727 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1728 * also avoid interrupt storms during dpms.
1729 */
1730 continue;
1731 }
926deccb
FT
1732 switch (radeon_connector->hpd.hpd) {
1733 case RADEON_HPD_1:
1734 WREG32(DC_HPD1_CONTROL, tmp);
1735 break;
1736 case RADEON_HPD_2:
1737 WREG32(DC_HPD2_CONTROL, tmp);
1738 break;
1739 case RADEON_HPD_3:
1740 WREG32(DC_HPD3_CONTROL, tmp);
1741 break;
1742 case RADEON_HPD_4:
1743 WREG32(DC_HPD4_CONTROL, tmp);
1744 break;
1745 case RADEON_HPD_5:
1746 WREG32(DC_HPD5_CONTROL, tmp);
1747 break;
1748 case RADEON_HPD_6:
1749 WREG32(DC_HPD6_CONTROL, tmp);
1750 break;
1751 default:
1752 break;
1753 }
1754 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
1755 enabled |= 1 << radeon_connector->hpd.hpd;
1756 }
1757 radeon_irq_kms_enable_hpd(rdev, enabled);
1758}
1759
1760/**
1761 * evergreen_hpd_fini - hpd tear down callback.
1762 *
1763 * @rdev: radeon_device pointer
1764 *
1765 * Tear down the hpd pins used by the card (evergreen+).
1766 * Disable the hpd interrupts.
1767 */
1768void evergreen_hpd_fini(struct radeon_device *rdev)
1769{
1770 struct drm_device *dev = rdev->ddev;
1771 struct drm_connector *connector;
1772 unsigned disabled = 0;
1773
1774 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1775 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1776 switch (radeon_connector->hpd.hpd) {
1777 case RADEON_HPD_1:
1778 WREG32(DC_HPD1_CONTROL, 0);
1779 break;
1780 case RADEON_HPD_2:
1781 WREG32(DC_HPD2_CONTROL, 0);
1782 break;
1783 case RADEON_HPD_3:
1784 WREG32(DC_HPD3_CONTROL, 0);
1785 break;
1786 case RADEON_HPD_4:
1787 WREG32(DC_HPD4_CONTROL, 0);
1788 break;
1789 case RADEON_HPD_5:
1790 WREG32(DC_HPD5_CONTROL, 0);
1791 break;
1792 case RADEON_HPD_6:
1793 WREG32(DC_HPD6_CONTROL, 0);
1794 break;
1795 default:
1796 break;
1797 }
1798 disabled |= 1 << radeon_connector->hpd.hpd;
1799 }
1800 radeon_irq_kms_disable_hpd(rdev, disabled);
1801}
1802
1803/* watermark setup */
1804
1805static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1806 struct radeon_crtc *radeon_crtc,
1807 struct drm_display_mode *mode,
1808 struct drm_display_mode *other_mode)
1809{
1810 u32 tmp;
1811 /*
1812 * Line Buffer Setup
1813 * There are 3 line buffers, each one shared by 2 display controllers.
1814 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1815 * the display controllers. The paritioning is done via one of four
1816 * preset allocations specified in bits 2:0:
1817 * first display controller
1818 * 0 - first half of lb (3840 * 2)
1819 * 1 - first 3/4 of lb (5760 * 2)
1820 * 2 - whole lb (7680 * 2), other crtc must be disabled
1821 * 3 - first 1/4 of lb (1920 * 2)
1822 * second display controller
1823 * 4 - second half of lb (3840 * 2)
1824 * 5 - second 3/4 of lb (5760 * 2)
1825 * 6 - whole lb (7680 * 2), other crtc must be disabled
1826 * 7 - last 1/4 of lb (1920 * 2)
1827 */
1828 /* this can get tricky if we have two large displays on a paired group
1829 * of crtcs. Ideally for multiple large displays we'd assign them to
1830 * non-linked crtcs for maximum line buffer allocation.
1831 */
1832 if (radeon_crtc->base.enabled && mode) {
1833 if (other_mode)
1834 tmp = 0; /* 1/2 */
1835 else
1836 tmp = 2; /* whole */
1837 } else
1838 tmp = 0;
1839
1840 /* second controller of the pair uses second half of the lb */
1841 if (radeon_crtc->crtc_id % 2)
1842 tmp += 4;
1843 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1844
1845 if (radeon_crtc->base.enabled && mode) {
1846 switch (tmp) {
1847 case 0:
1848 case 4:
1849 default:
1850 if (ASIC_IS_DCE5(rdev))
1851 return 4096 * 2;
1852 else
1853 return 3840 * 2;
1854 case 1:
1855 case 5:
1856 if (ASIC_IS_DCE5(rdev))
1857 return 6144 * 2;
1858 else
1859 return 5760 * 2;
1860 case 2:
1861 case 6:
1862 if (ASIC_IS_DCE5(rdev))
1863 return 8192 * 2;
1864 else
1865 return 7680 * 2;
1866 case 3:
1867 case 7:
1868 if (ASIC_IS_DCE5(rdev))
1869 return 2048 * 2;
1870 else
1871 return 1920 * 2;
1872 }
1873 }
1874
1875 /* controller not enabled, so no lb used */
1876 return 0;
1877}
1878
1879u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1880{
1881 u32 tmp = RREG32(MC_SHARED_CHMAP);
1882
1883 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1884 case 0:
1885 default:
1886 return 1;
1887 case 1:
1888 return 2;
1889 case 2:
1890 return 4;
1891 case 3:
1892 return 8;
1893 }
1894}
1895
1896struct evergreen_wm_params {
1897 u32 dram_channels; /* number of dram channels */
1898 u32 yclk; /* bandwidth per dram data pin in kHz */
1899 u32 sclk; /* engine clock in kHz */
1900 u32 disp_clk; /* display clock in kHz */
1901 u32 src_width; /* viewport width */
1902 u32 active_time; /* active display time in ns */
1903 u32 blank_time; /* blank time in ns */
1904 bool interlaced; /* mode is interlaced */
1905 fixed20_12 vsc; /* vertical scale ratio */
1906 u32 num_heads; /* number of active crtcs */
1907 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1908 u32 lb_size; /* line buffer allocated to pipe */
1909 u32 vtaps; /* vertical scaler taps */
1910};
1911
1912static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1913{
1914 /* Calculate DRAM Bandwidth and the part allocated to display. */
1915 fixed20_12 dram_efficiency; /* 0.7 */
1916 fixed20_12 yclk, dram_channels, bandwidth;
1917 fixed20_12 a;
1918
1919 a.full = dfixed_const(1000);
1920 yclk.full = dfixed_const(wm->yclk);
1921 yclk.full = dfixed_div(yclk, a);
1922 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1923 a.full = dfixed_const(10);
1924 dram_efficiency.full = dfixed_const(7);
1925 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1926 bandwidth.full = dfixed_mul(dram_channels, yclk);
1927 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1928
1929 return dfixed_trunc(bandwidth);
1930}
1931
1932static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1933{
1934 /* Calculate DRAM Bandwidth and the part allocated to display. */
1935 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1936 fixed20_12 yclk, dram_channels, bandwidth;
1937 fixed20_12 a;
1938
1939 a.full = dfixed_const(1000);
1940 yclk.full = dfixed_const(wm->yclk);
1941 yclk.full = dfixed_div(yclk, a);
1942 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1943 a.full = dfixed_const(10);
1944 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1945 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1946 bandwidth.full = dfixed_mul(dram_channels, yclk);
1947 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1948
1949 return dfixed_trunc(bandwidth);
1950}
1951
1952static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1953{
1954 /* Calculate the display Data return Bandwidth */
1955 fixed20_12 return_efficiency; /* 0.8 */
1956 fixed20_12 sclk, bandwidth;
1957 fixed20_12 a;
1958
1959 a.full = dfixed_const(1000);
1960 sclk.full = dfixed_const(wm->sclk);
1961 sclk.full = dfixed_div(sclk, a);
1962 a.full = dfixed_const(10);
1963 return_efficiency.full = dfixed_const(8);
1964 return_efficiency.full = dfixed_div(return_efficiency, a);
1965 a.full = dfixed_const(32);
1966 bandwidth.full = dfixed_mul(a, sclk);
1967 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1968
1969 return dfixed_trunc(bandwidth);
1970}
1971
1972static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
1973{
1974 /* Calculate the DMIF Request Bandwidth */
1975 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1976 fixed20_12 disp_clk, bandwidth;
1977 fixed20_12 a;
1978
1979 a.full = dfixed_const(1000);
1980 disp_clk.full = dfixed_const(wm->disp_clk);
1981 disp_clk.full = dfixed_div(disp_clk, a);
1982 a.full = dfixed_const(10);
1983 disp_clk_request_efficiency.full = dfixed_const(8);
1984 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1985 a.full = dfixed_const(32);
1986 bandwidth.full = dfixed_mul(a, disp_clk);
1987 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
1988
1989 return dfixed_trunc(bandwidth);
1990}
1991
1992static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
1993{
1994 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1995 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
1996 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
1997 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
1998
1999 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2000}
2001
2002static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2003{
2004 /* Calculate the display mode Average Bandwidth
2005 * DisplayMode should contain the source and destination dimensions,
2006 * timing, etc.
2007 */
2008 fixed20_12 bpp;
2009 fixed20_12 line_time;
2010 fixed20_12 src_width;
2011 fixed20_12 bandwidth;
2012 fixed20_12 a;
2013
2014 a.full = dfixed_const(1000);
2015 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2016 line_time.full = dfixed_div(line_time, a);
2017 bpp.full = dfixed_const(wm->bytes_per_pixel);
2018 src_width.full = dfixed_const(wm->src_width);
2019 bandwidth.full = dfixed_mul(src_width, bpp);
2020 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2021 bandwidth.full = dfixed_div(bandwidth, line_time);
2022
2023 return dfixed_trunc(bandwidth);
2024}
2025
2026static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2027{
2028 /* First calcualte the latency in ns */
2029 u32 mc_latency = 2000; /* 2000 ns. */
2030 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2031 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2032 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2033 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2034 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2035 (wm->num_heads * cursor_line_pair_return_time);
2036 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2037 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2038 fixed20_12 a, b, c;
2039
2040 if (wm->num_heads == 0)
2041 return 0;
2042
2043 a.full = dfixed_const(2);
2044 b.full = dfixed_const(1);
2045 if ((wm->vsc.full > a.full) ||
2046 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2047 (wm->vtaps >= 5) ||
2048 ((wm->vsc.full >= a.full) && wm->interlaced))
2049 max_src_lines_per_dst_line = 4;
2050 else
2051 max_src_lines_per_dst_line = 2;
2052
2053 a.full = dfixed_const(available_bandwidth);
2054 b.full = dfixed_const(wm->num_heads);
2055 a.full = dfixed_div(a, b);
2056
2057 b.full = dfixed_const(1000);
2058 c.full = dfixed_const(wm->disp_clk);
2059 b.full = dfixed_div(c, b);
2060 c.full = dfixed_const(wm->bytes_per_pixel);
2061 b.full = dfixed_mul(b, c);
2062
2063 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2064
2065 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2066 b.full = dfixed_const(1000);
2067 c.full = dfixed_const(lb_fill_bw);
2068 b.full = dfixed_div(c, b);
2069 a.full = dfixed_div(a, b);
2070 line_fill_time = dfixed_trunc(a);
2071
2072 if (line_fill_time < wm->active_time)
2073 return latency;
2074 else
2075 return latency + (line_fill_time - wm->active_time);
2076
2077}
2078
2079static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2080{
2081 if (evergreen_average_bandwidth(wm) <=
2082 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2083 return true;
2084 else
2085 return false;
2086};
2087
2088static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2089{
2090 if (evergreen_average_bandwidth(wm) <=
2091 (evergreen_available_bandwidth(wm) / wm->num_heads))
2092 return true;
2093 else
2094 return false;
2095};
2096
2097static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2098{
2099 u32 lb_partitions = wm->lb_size / wm->src_width;
2100 u32 line_time = wm->active_time + wm->blank_time;
2101 u32 latency_tolerant_lines;
2102 u32 latency_hiding;
2103 fixed20_12 a;
2104
2105 a.full = dfixed_const(1);
2106 if (wm->vsc.full > a.full)
2107 latency_tolerant_lines = 1;
2108 else {
2109 if (lb_partitions <= (wm->vtaps + 1))
2110 latency_tolerant_lines = 1;
2111 else
2112 latency_tolerant_lines = 2;
2113 }
2114
2115 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2116
2117 if (evergreen_latency_watermark(wm) <= latency_hiding)
2118 return true;
2119 else
2120 return false;
2121}
2122
2123static void evergreen_program_watermarks(struct radeon_device *rdev,
2124 struct radeon_crtc *radeon_crtc,
2125 u32 lb_size, u32 num_heads)
2126{
2127 struct drm_display_mode *mode = &radeon_crtc->base.mode;
57e252bf
MN
2128 struct evergreen_wm_params wm_low, wm_high;
2129 u32 dram_channels;
926deccb
FT
2130 u32 pixel_period;
2131 u32 line_time = 0;
2132 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2133 u32 priority_a_mark = 0, priority_b_mark = 0;
2134 u32 priority_a_cnt = PRIORITY_OFF;
2135 u32 priority_b_cnt = PRIORITY_OFF;
2136 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2137 u32 tmp, arb_control3;
2138 fixed20_12 a, b, c;
2139
2140 if (radeon_crtc->base.enabled && num_heads && mode) {
2141 pixel_period = 1000000 / (u32)mode->clock;
2142 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2143 priority_a_cnt = 0;
2144 priority_b_cnt = 0;
57e252bf
MN
2145 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2146
2147 /* watermark for high clocks */
2148 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2149 wm_high.yclk =
2150 radeon_dpm_get_mclk(rdev, false) * 10;
2151 wm_high.sclk =
2152 radeon_dpm_get_sclk(rdev, false) * 10;
2153 } else {
2154 wm_high.yclk = rdev->pm.current_mclk * 10;
2155 wm_high.sclk = rdev->pm.current_sclk * 10;
2156 }
2157
2158 wm_high.disp_clk = mode->clock;
2159 wm_high.src_width = mode->crtc_hdisplay;
2160 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2161 wm_high.blank_time = line_time - wm_high.active_time;
2162 wm_high.interlaced = false;
2163 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2164 wm_high.interlaced = true;
2165 wm_high.vsc = radeon_crtc->vsc;
2166 wm_high.vtaps = 1;
2167 if (radeon_crtc->rmx_type != RMX_OFF)
2168 wm_high.vtaps = 2;
2169 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2170 wm_high.lb_size = lb_size;
2171 wm_high.dram_channels = dram_channels;
2172 wm_high.num_heads = num_heads;
2173
2174 /* watermark for low clocks */
2175 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2176 wm_low.yclk =
2177 radeon_dpm_get_mclk(rdev, true) * 10;
2178 wm_low.sclk =
2179 radeon_dpm_get_sclk(rdev, true) * 10;
2180 } else {
2181 wm_low.yclk = rdev->pm.current_mclk * 10;
2182 wm_low.sclk = rdev->pm.current_sclk * 10;
2183 }
926deccb 2184
57e252bf
MN
2185 wm_low.disp_clk = mode->clock;
2186 wm_low.src_width = mode->crtc_hdisplay;
2187 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2188 wm_low.blank_time = line_time - wm_low.active_time;
2189 wm_low.interlaced = false;
926deccb 2190 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
57e252bf
MN
2191 wm_low.interlaced = true;
2192 wm_low.vsc = radeon_crtc->vsc;
2193 wm_low.vtaps = 1;
926deccb 2194 if (radeon_crtc->rmx_type != RMX_OFF)
57e252bf
MN
2195 wm_low.vtaps = 2;
2196 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2197 wm_low.lb_size = lb_size;
2198 wm_low.dram_channels = dram_channels;
2199 wm_low.num_heads = num_heads;
926deccb
FT
2200
2201 /* set for high clocks */
57e252bf 2202 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
926deccb 2203 /* set for low clocks */
57e252bf 2204 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
926deccb
FT
2205
2206 /* possibly force display priority to high */
2207 /* should really do this at mode validation time... */
57e252bf
MN
2208 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2209 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2210 !evergreen_check_latency_hiding(&wm_high) ||
926deccb 2211 (rdev->disp_priority == 2)) {
57e252bf 2212 DRM_DEBUG_KMS("force priority a to high\n");
926deccb 2213 priority_a_cnt |= PRIORITY_ALWAYS_ON;
57e252bf
MN
2214 }
2215 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2216 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2217 !evergreen_check_latency_hiding(&wm_low) ||
2218 (rdev->disp_priority == 2)) {
2219 DRM_DEBUG_KMS("force priority b to high\n");
926deccb
FT
2220 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2221 }
2222
2223 a.full = dfixed_const(1000);
2224 b.full = dfixed_const(mode->clock);
2225 b.full = dfixed_div(b, a);
2226 c.full = dfixed_const(latency_watermark_a);
2227 c.full = dfixed_mul(c, b);
2228 c.full = dfixed_mul(c, radeon_crtc->hsc);
2229 c.full = dfixed_div(c, a);
2230 a.full = dfixed_const(16);
2231 c.full = dfixed_div(c, a);
2232 priority_a_mark = dfixed_trunc(c);
2233 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2234
2235 a.full = dfixed_const(1000);
2236 b.full = dfixed_const(mode->clock);
2237 b.full = dfixed_div(b, a);
2238 c.full = dfixed_const(latency_watermark_b);
2239 c.full = dfixed_mul(c, b);
2240 c.full = dfixed_mul(c, radeon_crtc->hsc);
2241 c.full = dfixed_div(c, a);
2242 a.full = dfixed_const(16);
2243 c.full = dfixed_div(c, a);
2244 priority_b_mark = dfixed_trunc(c);
2245 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2246 }
2247
2248 /* select wm A */
2249 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2250 tmp = arb_control3;
2251 tmp &= ~LATENCY_WATERMARK_MASK(3);
2252 tmp |= LATENCY_WATERMARK_MASK(1);
2253 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2254 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2255 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2256 LATENCY_HIGH_WATERMARK(line_time)));
2257 /* select wm B */
2258 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2259 tmp &= ~LATENCY_WATERMARK_MASK(3);
2260 tmp |= LATENCY_WATERMARK_MASK(2);
2261 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2262 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2263 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2264 LATENCY_HIGH_WATERMARK(line_time)));
2265 /* restore original selection */
2266 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2267
2268 /* write the priority marks */
2269 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2270 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2271
57e252bf
MN
2272 /* save values for DPM */
2273 radeon_crtc->line_time = line_time;
2274 radeon_crtc->wm_high = latency_watermark_a;
2275 radeon_crtc->wm_low = latency_watermark_b;
926deccb
FT
2276}
2277
2278/**
2279 * evergreen_bandwidth_update - update display watermarks callback.
2280 *
2281 * @rdev: radeon_device pointer
2282 *
2283 * Update the display watermarks based on the requested mode(s)
2284 * (evergreen+).
2285 */
2286void evergreen_bandwidth_update(struct radeon_device *rdev)
2287{
2288 struct drm_display_mode *mode0 = NULL;
2289 struct drm_display_mode *mode1 = NULL;
2290 u32 num_heads = 0, lb_size;
2291 int i;
2292
2293 radeon_update_display_priority(rdev);
2294
2295 for (i = 0; i < rdev->num_crtc; i++) {
2296 if (rdev->mode_info.crtcs[i]->base.enabled)
2297 num_heads++;
2298 }
2299 for (i = 0; i < rdev->num_crtc; i += 2) {
2300 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2301 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2302 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2303 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2304 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2305 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2306 }
2307}
2308
2309/**
2310 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2311 *
2312 * @rdev: radeon_device pointer
2313 *
2314 * Wait for the MC (memory controller) to be idle.
2315 * (evergreen+).
2316 * Returns 0 if the MC is idle, -1 if not.
2317 */
2318int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2319{
2320 unsigned i;
2321 u32 tmp;
2322
2323 for (i = 0; i < rdev->usec_timeout; i++) {
2324 /* read MC_STATUS */
2325 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2326 if (!tmp)
2327 return 0;
2328 DRM_UDELAY(1);
2329 }
2330 return -1;
2331}
2332
2333/*
2334 * GART
2335 */
2336void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2337{
2338 unsigned i;
2339 u32 tmp;
2340
2341 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2342
2343 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2344 for (i = 0; i < rdev->usec_timeout; i++) {
2345 /* read MC_STATUS */
2346 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2347 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2348 if (tmp == 2) {
2349 DRM_ERROR("[drm] r600 flush TLB failed\n");
2350 return;
2351 }
2352 if (tmp) {
2353 return;
2354 }
2355 DRM_UDELAY(1);
2356 }
2357}
2358
2359static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2360{
2361 u32 tmp;
2362 int r;
2363
2364 if (rdev->gart.robj == NULL) {
2365 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2366 return -EINVAL;
2367 }
2368 r = radeon_gart_table_vram_pin(rdev);
2369 if (r)
2370 return r;
2371 radeon_gart_restore(rdev);
2372 /* Setup L2 cache */
2373 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2374 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2375 EFFECTIVE_L2_QUEUE_SIZE(7));
2376 WREG32(VM_L2_CNTL2, 0);
2377 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2378 /* Setup TLB control */
2379 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2380 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2381 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2382 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2383 if (rdev->flags & RADEON_IS_IGP) {
2384 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2385 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2386 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2387 } else {
2388 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2389 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2390 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2391 if ((rdev->family == CHIP_JUNIPER) ||
2392 (rdev->family == CHIP_CYPRESS) ||
2393 (rdev->family == CHIP_HEMLOCK) ||
2394 (rdev->family == CHIP_BARTS))
2395 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2396 }
2397 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2398 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2399 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2400 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2401 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2402 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2403 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2404 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2405 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2406 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2407 (u32)(rdev->dummy_page.addr >> 12));
2408 WREG32(VM_CONTEXT1_CNTL, 0);
2409
2410 evergreen_pcie_gart_tlb_flush(rdev);
2411 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2412 (unsigned)(rdev->mc.gtt_size >> 20),
2413 (unsigned long long)rdev->gart.table_addr);
2414 rdev->gart.ready = true;
2415 return 0;
2416}
2417
2418static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2419{
2420 u32 tmp;
2421
2422 /* Disable all tables */
2423 WREG32(VM_CONTEXT0_CNTL, 0);
2424 WREG32(VM_CONTEXT1_CNTL, 0);
2425
2426 /* Setup L2 cache */
2427 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2428 EFFECTIVE_L2_QUEUE_SIZE(7));
2429 WREG32(VM_L2_CNTL2, 0);
2430 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2431 /* Setup TLB control */
2432 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2433 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2434 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2435 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2436 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2437 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2438 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2439 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2440 radeon_gart_table_vram_unpin(rdev);
2441}
2442
2443static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2444{
2445 evergreen_pcie_gart_disable(rdev);
2446 radeon_gart_table_vram_free(rdev);
2447 radeon_gart_fini(rdev);
2448}
2449
2450
2451static void evergreen_agp_enable(struct radeon_device *rdev)
2452{
2453 u32 tmp;
2454
2455 /* Setup L2 cache */
2456 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2457 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2458 EFFECTIVE_L2_QUEUE_SIZE(7));
2459 WREG32(VM_L2_CNTL2, 0);
2460 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2461 /* Setup TLB control */
2462 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2463 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2464 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2465 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2466 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2467 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2468 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2469 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2470 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2471 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2472 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2473 WREG32(VM_CONTEXT0_CNTL, 0);
2474 WREG32(VM_CONTEXT1_CNTL, 0);
2475}
2476
2477void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2478{
2479 u32 crtc_enabled, tmp, frame_count, blackout;
2480 int i, j;
2481
f43cf1b1
MN
2482 if (!ASIC_IS_NODCE(rdev)) {
2483 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2484 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
926deccb 2485
f43cf1b1
MN
2486 /* disable VGA render */
2487 WREG32(VGA_RENDER_CONTROL, 0);
2488 }
926deccb
FT
2489 /* blank the display controllers */
2490 for (i = 0; i < rdev->num_crtc; i++) {
2491 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2492 if (crtc_enabled) {
2493 save->crtc_enabled[i] = true;
2494 if (ASIC_IS_DCE6(rdev)) {
2495 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2496 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2497 radeon_wait_for_vblank(rdev, i);
926deccb 2498 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
f43cf1b1 2499 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
926deccb 2500 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
926deccb
FT
2501 }
2502 } else {
2503 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2504 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2505 radeon_wait_for_vblank(rdev, i);
926deccb 2506 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
f43cf1b1 2507 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
926deccb
FT
2508 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2509 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2510 }
2511 }
2512 /* wait for the next frame */
2513 frame_count = radeon_get_vblank_counter(rdev, i);
2514 for (j = 0; j < rdev->usec_timeout; j++) {
2515 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2516 break;
2517 DRM_UDELAY(1);
2518 }
f43cf1b1
MN
2519
2520 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2521 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2522 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2523 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2524 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2525 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2526 save->crtc_enabled[i] = false;
2527 /* ***** */
926deccb
FT
2528 } else {
2529 save->crtc_enabled[i] = false;
2530 }
2531 }
2532
2533 radeon_mc_wait_for_idle(rdev);
2534
2535 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2536 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2537 /* Block CPU access */
2538 WREG32(BIF_FB_EN, 0);
2539 /* blackout the MC */
2540 blackout &= ~BLACKOUT_MODE_MASK;
2541 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2542 }
2543 /* wait for the MC to settle */
2544 DRM_UDELAY(100);
f43cf1b1
MN
2545
2546 /* lock double buffered regs */
2547 for (i = 0; i < rdev->num_crtc; i++) {
2548 if (save->crtc_enabled[i]) {
2549 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2550 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2551 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2552 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2553 }
2554 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2555 if (!(tmp & 1)) {
2556 tmp |= 1;
2557 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2558 }
2559 }
2560 }
926deccb
FT
2561}
2562
2563void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2564{
2565 u32 tmp, frame_count;
2566 int i, j;
2567
2568 /* update crtc base addresses */
2569 for (i = 0; i < rdev->num_crtc; i++) {
2570 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2571 upper_32_bits(rdev->mc.vram_start));
2572 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2573 upper_32_bits(rdev->mc.vram_start));
2574 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2575 (u32)rdev->mc.vram_start);
2576 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2577 (u32)rdev->mc.vram_start);
2578 }
f43cf1b1
MN
2579
2580 if (!ASIC_IS_NODCE(rdev)) {
2581 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2582 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2583 }
2584
2585 /* unlock regs and wait for update */
2586 for (i = 0; i < rdev->num_crtc; i++) {
2587 if (save->crtc_enabled[i]) {
2588 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2589 if ((tmp & 0x3) != 0) {
2590 tmp &= ~0x3;
2591 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2592 }
2593 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2594 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2595 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2596 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2597 }
2598 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2599 if (tmp & 1) {
2600 tmp &= ~1;
2601 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2602 }
2603 for (j = 0; j < rdev->usec_timeout; j++) {
2604 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2605 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2606 break;
2607 DRM_UDELAY(1);
2608 }
2609 }
2610 }
926deccb
FT
2611
2612 /* unblackout the MC */
2613 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2614 tmp &= ~BLACKOUT_MODE_MASK;
2615 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2616 /* allow CPU access */
2617 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2618
2619 for (i = 0; i < rdev->num_crtc; i++) {
2620 if (save->crtc_enabled[i]) {
2621 if (ASIC_IS_DCE6(rdev)) {
2622 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2623 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2624 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2625 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2626 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2627 } else {
2628 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2629 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2630 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2631 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2632 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2633 }
2634 /* wait for the next frame */
2635 frame_count = radeon_get_vblank_counter(rdev, i);
2636 for (j = 0; j < rdev->usec_timeout; j++) {
2637 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2638 break;
2639 DRM_UDELAY(1);
2640 }
2641 }
2642 }
f43cf1b1
MN
2643 if (!ASIC_IS_NODCE(rdev)) {
2644 /* Unlock vga access */
2645 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2646 DRM_MDELAY(1);
2647 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2648 }
926deccb
FT
2649}
2650
2651void evergreen_mc_program(struct radeon_device *rdev)
2652{
2653 struct evergreen_mc_save save;
2654 u32 tmp;
2655 int i, j;
2656
2657 /* Initialize HDP */
2658 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2659 WREG32((0x2c14 + j), 0x00000000);
2660 WREG32((0x2c18 + j), 0x00000000);
2661 WREG32((0x2c1c + j), 0x00000000);
2662 WREG32((0x2c20 + j), 0x00000000);
2663 WREG32((0x2c24 + j), 0x00000000);
2664 }
2665 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2666
2667 evergreen_mc_stop(rdev, &save);
2668 if (evergreen_mc_wait_for_idle(rdev)) {
2669 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2670 }
2671 /* Lockout access through VGA aperture*/
2672 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2673 /* Update configuration */
2674 if (rdev->flags & RADEON_IS_AGP) {
2675 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2676 /* VRAM before AGP */
2677 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2678 rdev->mc.vram_start >> 12);
2679 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2680 rdev->mc.gtt_end >> 12);
2681 } else {
2682 /* VRAM after AGP */
2683 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2684 rdev->mc.gtt_start >> 12);
2685 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2686 rdev->mc.vram_end >> 12);
2687 }
2688 } else {
2689 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2690 rdev->mc.vram_start >> 12);
2691 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2692 rdev->mc.vram_end >> 12);
2693 }
2694 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2695 /* llano/ontario only */
2696 if ((rdev->family == CHIP_PALM) ||
2697 (rdev->family == CHIP_SUMO) ||
2698 (rdev->family == CHIP_SUMO2)) {
2699 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2700 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2701 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2702 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2703 }
2704 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2705 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2706 WREG32(MC_VM_FB_LOCATION, tmp);
2707 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2708 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2709 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2710 if (rdev->flags & RADEON_IS_AGP) {
2711 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2712 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2713 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2714 } else {
2715 WREG32(MC_VM_AGP_BASE, 0);
2716 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2717 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2718 }
2719 if (evergreen_mc_wait_for_idle(rdev)) {
2720 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2721 }
2722 evergreen_mc_resume(rdev, &save);
2723 /* we need to own VRAM, so turn off the VGA renderer here
2724 * to stop it overwriting our objects */
2725 rv515_vga_render_disable(rdev);
2726}
2727
2728/*
2729 * CP.
2730 */
2731void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2732{
2733 struct radeon_ring *ring = &rdev->ring[ib->ring];
2734 u32 next_rptr;
2735
2736 /* set to DX10/11 mode */
2737 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2738 radeon_ring_write(ring, 1);
2739
2740 if (ring->rptr_save_reg) {
2741 next_rptr = ring->wptr + 3 + 4;
2742 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2743 radeon_ring_write(ring, ((ring->rptr_save_reg -
2744 PACKET3_SET_CONFIG_REG_START) >> 2));
2745 radeon_ring_write(ring, next_rptr);
2746 } else if (rdev->wb.enabled) {
2747 next_rptr = ring->wptr + 5 + 4;
2748 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2749 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2750 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2751 radeon_ring_write(ring, next_rptr);
2752 radeon_ring_write(ring, 0);
2753 }
2754
2755 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2756 radeon_ring_write(ring,
2757#ifdef __BIG_ENDIAN
2758 (2 << 0) |
2759#endif
2760 (ib->gpu_addr & 0xFFFFFFFC));
2761 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2762 radeon_ring_write(ring, ib->length_dw);
2763}
2764
2765
2766static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2767{
2768 const __be32 *fw_data;
2769 int i;
2770
2771 if (!rdev->me_fw || !rdev->pfp_fw)
2772 return -EINVAL;
2773
2774 r700_cp_stop(rdev);
2775 WREG32(CP_RB_CNTL,
2776#ifdef __BIG_ENDIAN
2777 BUF_SWAP_32BIT |
2778#endif
2779 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2780
2781 fw_data = (const __be32 *)rdev->pfp_fw->data;
2782 WREG32(CP_PFP_UCODE_ADDR, 0);
2783 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2784 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2785 WREG32(CP_PFP_UCODE_ADDR, 0);
2786
2787 fw_data = (const __be32 *)rdev->me_fw->data;
2788 WREG32(CP_ME_RAM_WADDR, 0);
2789 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2790 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2791
2792 WREG32(CP_PFP_UCODE_ADDR, 0);
2793 WREG32(CP_ME_RAM_WADDR, 0);
2794 WREG32(CP_ME_RAM_RADDR, 0);
2795 return 0;
2796}
2797
2798static int evergreen_cp_start(struct radeon_device *rdev)
2799{
2800 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2801 int r, i;
2802 uint32_t cp_me;
2803
2804 r = radeon_ring_lock(rdev, ring, 7);
2805 if (r) {
2806 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2807 return r;
2808 }
2809 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2810 radeon_ring_write(ring, 0x1);
2811 radeon_ring_write(ring, 0x0);
2812 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2813 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2814 radeon_ring_write(ring, 0);
2815 radeon_ring_write(ring, 0);
2816 radeon_ring_unlock_commit(rdev, ring);
2817
2818 cp_me = 0xff;
2819 WREG32(CP_ME_CNTL, cp_me);
2820
2821 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
2822 if (r) {
2823 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2824 return r;
2825 }
2826
2827 /* setup clear context state */
2828 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2829 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2830
2831 for (i = 0; i < evergreen_default_size; i++)
2832 radeon_ring_write(ring, evergreen_default_state[i]);
2833
2834 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2835 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2836
2837 /* set clear context state */
2838 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2839 radeon_ring_write(ring, 0);
2840
2841 /* SQ_VTX_BASE_VTX_LOC */
2842 radeon_ring_write(ring, 0xc0026f00);
2843 radeon_ring_write(ring, 0x00000000);
2844 radeon_ring_write(ring, 0x00000000);
2845 radeon_ring_write(ring, 0x00000000);
2846
2847 /* Clear consts */
2848 radeon_ring_write(ring, 0xc0036f00);
2849 radeon_ring_write(ring, 0x00000bc4);
2850 radeon_ring_write(ring, 0xffffffff);
2851 radeon_ring_write(ring, 0xffffffff);
2852 radeon_ring_write(ring, 0xffffffff);
2853
2854 radeon_ring_write(ring, 0xc0026900);
2855 radeon_ring_write(ring, 0x00000316);
2856 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2857 radeon_ring_write(ring, 0x00000010); /* */
2858
2859 radeon_ring_unlock_commit(rdev, ring);
2860
2861 return 0;
2862}
2863
2864static int evergreen_cp_resume(struct radeon_device *rdev)
2865{
2866 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2867 u32 tmp;
2868 u32 rb_bufsz;
2869 int r;
2870
2871 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2872 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2873 SOFT_RESET_PA |
2874 SOFT_RESET_SH |
2875 SOFT_RESET_VGT |
2876 SOFT_RESET_SPI |
2877 SOFT_RESET_SX));
2878 RREG32(GRBM_SOFT_RESET);
2879 DRM_MDELAY(15);
2880 WREG32(GRBM_SOFT_RESET, 0);
2881 RREG32(GRBM_SOFT_RESET);
2882
2883 /* Set ring buffer size */
2884 rb_bufsz = drm_order(ring->ring_size / 8);
2885 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2886#ifdef __BIG_ENDIAN
2887 tmp |= BUF_SWAP_32BIT;
2888#endif
2889 WREG32(CP_RB_CNTL, tmp);
2890 WREG32(CP_SEM_WAIT_TIMER, 0x0);
2891 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
2892
2893 /* Set the write pointer delay */
2894 WREG32(CP_RB_WPTR_DELAY, 0);
2895
2896 /* Initialize the ring buffer's read and write pointers */
2897 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2898 WREG32(CP_RB_RPTR_WR, 0);
2899 ring->wptr = 0;
2900 WREG32(CP_RB_WPTR, ring->wptr);
2901
2902 /* set the wb address whether it's enabled or not */
2903 WREG32(CP_RB_RPTR_ADDR,
2904 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
2905 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2906 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2907
2908 if (rdev->wb.enabled)
2909 WREG32(SCRATCH_UMSK, 0xff);
2910 else {
2911 tmp |= RB_NO_UPDATE;
2912 WREG32(SCRATCH_UMSK, 0);
2913 }
2914
2915 DRM_MDELAY(1);
2916 WREG32(CP_RB_CNTL, tmp);
2917
2918 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
2919 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2920
2921 ring->rptr = RREG32(CP_RB_RPTR);
2922
2923 evergreen_cp_start(rdev);
2924 ring->ready = true;
2925 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
2926 if (r) {
2927 ring->ready = false;
2928 return r;
2929 }
2930 return 0;
2931}
2932
2933/*
2934 * Core functions
2935 */
2936static void evergreen_gpu_init(struct radeon_device *rdev)
2937{
2938 u32 gb_addr_config;
2939 u32 mc_shared_chmap, mc_arb_ramcfg;
2940 u32 sx_debug_1;
2941 u32 smx_dc_ctl0;
2942 u32 sq_config;
2943 u32 sq_lds_resource_mgmt;
2944 u32 sq_gpr_resource_mgmt_1;
2945 u32 sq_gpr_resource_mgmt_2;
2946 u32 sq_gpr_resource_mgmt_3;
2947 u32 sq_thread_resource_mgmt;
2948 u32 sq_thread_resource_mgmt_2;
2949 u32 sq_stack_resource_mgmt_1;
2950 u32 sq_stack_resource_mgmt_2;
2951 u32 sq_stack_resource_mgmt_3;
2952 u32 vgt_cache_invalidation;
2953 u32 hdp_host_path_cntl, tmp;
2954 u32 disabled_rb_mask;
2955 int i, j, num_shader_engines, ps_thread_count;
2956
2957 switch (rdev->family) {
2958 case CHIP_CYPRESS:
2959 case CHIP_HEMLOCK:
2960 rdev->config.evergreen.num_ses = 2;
2961 rdev->config.evergreen.max_pipes = 4;
2962 rdev->config.evergreen.max_tile_pipes = 8;
2963 rdev->config.evergreen.max_simds = 10;
2964 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
2965 rdev->config.evergreen.max_gprs = 256;
2966 rdev->config.evergreen.max_threads = 248;
2967 rdev->config.evergreen.max_gs_threads = 32;
2968 rdev->config.evergreen.max_stack_entries = 512;
2969 rdev->config.evergreen.sx_num_of_sets = 4;
2970 rdev->config.evergreen.sx_max_export_size = 256;
2971 rdev->config.evergreen.sx_max_export_pos_size = 64;
2972 rdev->config.evergreen.sx_max_export_smx_size = 192;
2973 rdev->config.evergreen.max_hw_contexts = 8;
2974 rdev->config.evergreen.sq_num_cf_insts = 2;
2975
2976 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2977 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2978 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2979 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
2980 break;
2981 case CHIP_JUNIPER:
2982 rdev->config.evergreen.num_ses = 1;
2983 rdev->config.evergreen.max_pipes = 4;
2984 rdev->config.evergreen.max_tile_pipes = 4;
2985 rdev->config.evergreen.max_simds = 10;
2986 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
2987 rdev->config.evergreen.max_gprs = 256;
2988 rdev->config.evergreen.max_threads = 248;
2989 rdev->config.evergreen.max_gs_threads = 32;
2990 rdev->config.evergreen.max_stack_entries = 512;
2991 rdev->config.evergreen.sx_num_of_sets = 4;
2992 rdev->config.evergreen.sx_max_export_size = 256;
2993 rdev->config.evergreen.sx_max_export_pos_size = 64;
2994 rdev->config.evergreen.sx_max_export_smx_size = 192;
2995 rdev->config.evergreen.max_hw_contexts = 8;
2996 rdev->config.evergreen.sq_num_cf_insts = 2;
2997
2998 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2999 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3000 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3001 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3002 break;
3003 case CHIP_REDWOOD:
3004 rdev->config.evergreen.num_ses = 1;
3005 rdev->config.evergreen.max_pipes = 4;
3006 rdev->config.evergreen.max_tile_pipes = 4;
3007 rdev->config.evergreen.max_simds = 5;
3008 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3009 rdev->config.evergreen.max_gprs = 256;
3010 rdev->config.evergreen.max_threads = 248;
3011 rdev->config.evergreen.max_gs_threads = 32;
3012 rdev->config.evergreen.max_stack_entries = 256;
3013 rdev->config.evergreen.sx_num_of_sets = 4;
3014 rdev->config.evergreen.sx_max_export_size = 256;
3015 rdev->config.evergreen.sx_max_export_pos_size = 64;
3016 rdev->config.evergreen.sx_max_export_smx_size = 192;
3017 rdev->config.evergreen.max_hw_contexts = 8;
3018 rdev->config.evergreen.sq_num_cf_insts = 2;
3019
3020 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3021 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3022 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3023 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3024 break;
3025 case CHIP_CEDAR:
3026 default:
3027 rdev->config.evergreen.num_ses = 1;
3028 rdev->config.evergreen.max_pipes = 2;
3029 rdev->config.evergreen.max_tile_pipes = 2;
3030 rdev->config.evergreen.max_simds = 2;
3031 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3032 rdev->config.evergreen.max_gprs = 256;
3033 rdev->config.evergreen.max_threads = 192;
3034 rdev->config.evergreen.max_gs_threads = 16;
3035 rdev->config.evergreen.max_stack_entries = 256;
3036 rdev->config.evergreen.sx_num_of_sets = 4;
3037 rdev->config.evergreen.sx_max_export_size = 128;
3038 rdev->config.evergreen.sx_max_export_pos_size = 32;
3039 rdev->config.evergreen.sx_max_export_smx_size = 96;
3040 rdev->config.evergreen.max_hw_contexts = 4;
3041 rdev->config.evergreen.sq_num_cf_insts = 1;
3042
3043 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3044 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3045 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3046 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3047 break;
3048 case CHIP_PALM:
3049 rdev->config.evergreen.num_ses = 1;
3050 rdev->config.evergreen.max_pipes = 2;
3051 rdev->config.evergreen.max_tile_pipes = 2;
3052 rdev->config.evergreen.max_simds = 2;
3053 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3054 rdev->config.evergreen.max_gprs = 256;
3055 rdev->config.evergreen.max_threads = 192;
3056 rdev->config.evergreen.max_gs_threads = 16;
3057 rdev->config.evergreen.max_stack_entries = 256;
3058 rdev->config.evergreen.sx_num_of_sets = 4;
3059 rdev->config.evergreen.sx_max_export_size = 128;
3060 rdev->config.evergreen.sx_max_export_pos_size = 32;
3061 rdev->config.evergreen.sx_max_export_smx_size = 96;
3062 rdev->config.evergreen.max_hw_contexts = 4;
3063 rdev->config.evergreen.sq_num_cf_insts = 1;
3064
3065 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3066 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3067 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3068 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3069 break;
3070 case CHIP_SUMO:
3071 rdev->config.evergreen.num_ses = 1;
3072 rdev->config.evergreen.max_pipes = 4;
3073 rdev->config.evergreen.max_tile_pipes = 4;
3074 if (rdev->ddev->pci_device == 0x9648)
3075 rdev->config.evergreen.max_simds = 3;
3076 else if ((rdev->ddev->pci_device == 0x9647) ||
3077 (rdev->ddev->pci_device == 0x964a))
3078 rdev->config.evergreen.max_simds = 4;
3079 else
3080 rdev->config.evergreen.max_simds = 5;
3081 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3082 rdev->config.evergreen.max_gprs = 256;
3083 rdev->config.evergreen.max_threads = 248;
3084 rdev->config.evergreen.max_gs_threads = 32;
3085 rdev->config.evergreen.max_stack_entries = 256;
3086 rdev->config.evergreen.sx_num_of_sets = 4;
3087 rdev->config.evergreen.sx_max_export_size = 256;
3088 rdev->config.evergreen.sx_max_export_pos_size = 64;
3089 rdev->config.evergreen.sx_max_export_smx_size = 192;
3090 rdev->config.evergreen.max_hw_contexts = 8;
3091 rdev->config.evergreen.sq_num_cf_insts = 2;
3092
3093 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3094 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3095 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3096 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3097 break;
3098 case CHIP_SUMO2:
3099 rdev->config.evergreen.num_ses = 1;
3100 rdev->config.evergreen.max_pipes = 4;
3101 rdev->config.evergreen.max_tile_pipes = 4;
3102 rdev->config.evergreen.max_simds = 2;
3103 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3104 rdev->config.evergreen.max_gprs = 256;
3105 rdev->config.evergreen.max_threads = 248;
3106 rdev->config.evergreen.max_gs_threads = 32;
3107 rdev->config.evergreen.max_stack_entries = 512;
3108 rdev->config.evergreen.sx_num_of_sets = 4;
3109 rdev->config.evergreen.sx_max_export_size = 256;
3110 rdev->config.evergreen.sx_max_export_pos_size = 64;
3111 rdev->config.evergreen.sx_max_export_smx_size = 192;
3112 rdev->config.evergreen.max_hw_contexts = 8;
3113 rdev->config.evergreen.sq_num_cf_insts = 2;
3114
3115 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3116 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3117 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3118 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3119 break;
3120 case CHIP_BARTS:
3121 rdev->config.evergreen.num_ses = 2;
3122 rdev->config.evergreen.max_pipes = 4;
3123 rdev->config.evergreen.max_tile_pipes = 8;
3124 rdev->config.evergreen.max_simds = 7;
3125 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3126 rdev->config.evergreen.max_gprs = 256;
3127 rdev->config.evergreen.max_threads = 248;
3128 rdev->config.evergreen.max_gs_threads = 32;
3129 rdev->config.evergreen.max_stack_entries = 512;
3130 rdev->config.evergreen.sx_num_of_sets = 4;
3131 rdev->config.evergreen.sx_max_export_size = 256;
3132 rdev->config.evergreen.sx_max_export_pos_size = 64;
3133 rdev->config.evergreen.sx_max_export_smx_size = 192;
3134 rdev->config.evergreen.max_hw_contexts = 8;
3135 rdev->config.evergreen.sq_num_cf_insts = 2;
3136
3137 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3138 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3139 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3140 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3141 break;
3142 case CHIP_TURKS:
3143 rdev->config.evergreen.num_ses = 1;
3144 rdev->config.evergreen.max_pipes = 4;
3145 rdev->config.evergreen.max_tile_pipes = 4;
3146 rdev->config.evergreen.max_simds = 6;
3147 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3148 rdev->config.evergreen.max_gprs = 256;
3149 rdev->config.evergreen.max_threads = 248;
3150 rdev->config.evergreen.max_gs_threads = 32;
3151 rdev->config.evergreen.max_stack_entries = 256;
3152 rdev->config.evergreen.sx_num_of_sets = 4;
3153 rdev->config.evergreen.sx_max_export_size = 256;
3154 rdev->config.evergreen.sx_max_export_pos_size = 64;
3155 rdev->config.evergreen.sx_max_export_smx_size = 192;
3156 rdev->config.evergreen.max_hw_contexts = 8;
3157 rdev->config.evergreen.sq_num_cf_insts = 2;
3158
3159 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3160 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3161 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3162 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3163 break;
3164 case CHIP_CAICOS:
3165 rdev->config.evergreen.num_ses = 1;
3166 rdev->config.evergreen.max_pipes = 2;
3167 rdev->config.evergreen.max_tile_pipes = 2;
3168 rdev->config.evergreen.max_simds = 2;
3169 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3170 rdev->config.evergreen.max_gprs = 256;
3171 rdev->config.evergreen.max_threads = 192;
3172 rdev->config.evergreen.max_gs_threads = 16;
3173 rdev->config.evergreen.max_stack_entries = 256;
3174 rdev->config.evergreen.sx_num_of_sets = 4;
3175 rdev->config.evergreen.sx_max_export_size = 128;
3176 rdev->config.evergreen.sx_max_export_pos_size = 32;
3177 rdev->config.evergreen.sx_max_export_smx_size = 96;
3178 rdev->config.evergreen.max_hw_contexts = 4;
3179 rdev->config.evergreen.sq_num_cf_insts = 1;
3180
3181 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3182 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3183 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3184 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3185 break;
3186 }
3187
3188 /* Initialize HDP */
3189 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3190 WREG32((0x2c14 + j), 0x00000000);
3191 WREG32((0x2c18 + j), 0x00000000);
3192 WREG32((0x2c1c + j), 0x00000000);
3193 WREG32((0x2c20 + j), 0x00000000);
3194 WREG32((0x2c24 + j), 0x00000000);
3195 }
3196
3197 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3198
3199 evergreen_fix_pci_max_read_req_size(rdev);
3200
3201 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3202 if ((rdev->family == CHIP_PALM) ||
3203 (rdev->family == CHIP_SUMO) ||
3204 (rdev->family == CHIP_SUMO2))
3205 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3206 else
3207 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3208
3209 /* setup tiling info dword. gb_addr_config is not adequate since it does
3210 * not have bank info, so create a custom tiling dword.
3211 * bits 3:0 num_pipes
3212 * bits 7:4 num_banks
3213 * bits 11:8 group_size
3214 * bits 15:12 row_size
3215 */
3216 rdev->config.evergreen.tile_config = 0;
3217 switch (rdev->config.evergreen.max_tile_pipes) {
3218 case 1:
3219 default:
3220 rdev->config.evergreen.tile_config |= (0 << 0);
3221 break;
3222 case 2:
3223 rdev->config.evergreen.tile_config |= (1 << 0);
3224 break;
3225 case 4:
3226 rdev->config.evergreen.tile_config |= (2 << 0);
3227 break;
3228 case 8:
3229 rdev->config.evergreen.tile_config |= (3 << 0);
3230 break;
3231 }
3232 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3233 if (rdev->flags & RADEON_IS_IGP)
3234 rdev->config.evergreen.tile_config |= 1 << 4;
3235 else {
3236 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3237 case 0: /* four banks */
3238 rdev->config.evergreen.tile_config |= 0 << 4;
3239 break;
3240 case 1: /* eight banks */
3241 rdev->config.evergreen.tile_config |= 1 << 4;
3242 break;
3243 case 2: /* sixteen banks */
3244 default:
3245 rdev->config.evergreen.tile_config |= 2 << 4;
3246 break;
3247 }
3248 }
3249 rdev->config.evergreen.tile_config |= 0 << 8;
3250 rdev->config.evergreen.tile_config |=
3251 ((gb_addr_config & 0x30000000) >> 28) << 12;
3252
3253 num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1;
3254
3255 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3256 u32 efuse_straps_4;
3257 u32 efuse_straps_3;
3258
57e252bf
MN
3259 efuse_straps_4 = RREG32_RCU(0x204);
3260 efuse_straps_3 = RREG32_RCU(0x203);
926deccb
FT
3261 tmp = (((efuse_straps_4 & 0xf) << 4) |
3262 ((efuse_straps_3 & 0xf0000000) >> 28));
3263 } else {
3264 tmp = 0;
3265 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3266 u32 rb_disable_bitmap;
3267
3268 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3269 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3270 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3271 tmp <<= 4;
3272 tmp |= rb_disable_bitmap;
3273 }
3274 }
3275 /* enabled rb are just the one not disabled :) */
3276 disabled_rb_mask = tmp;
f43cf1b1
MN
3277 tmp = 0;
3278 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3279 tmp |= (1 << i);
3280 /* if all the backends are disabled, fix it up here */
3281 if ((disabled_rb_mask & tmp) == tmp) {
3282 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3283 disabled_rb_mask &= ~(1 << i);
3284 }
926deccb
FT
3285
3286 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3287 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3288
3289 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3290 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3291 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3292 WREG32(DMA_TILING_CONFIG, gb_addr_config);
f43cf1b1
MN
3293 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3294 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3295 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
926deccb
FT
3296
3297 if ((rdev->config.evergreen.max_backends == 1) &&
3298 (rdev->flags & RADEON_IS_IGP)) {
3299 if ((disabled_rb_mask & 3) == 1) {
3300 /* RB0 disabled, RB1 enabled */
3301 tmp = 0x11111111;
3302 } else {
3303 /* RB1 disabled, RB0 enabled */
3304 tmp = 0x00000000;
3305 }
3306 } else {
3307 tmp = gb_addr_config & NUM_PIPES_MASK;
3308 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3309 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3310 }
3311 WREG32(GB_BACKEND_MAP, tmp);
3312
3313 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3314 WREG32(CGTS_TCC_DISABLE, 0);
3315 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3316 WREG32(CGTS_USER_TCC_DISABLE, 0);
3317
3318 /* set HW defaults for 3D engine */
3319 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3320 ROQ_IB2_START(0x2b)));
3321
3322 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3323
3324 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3325 SYNC_GRADIENT |
3326 SYNC_WALKER |
3327 SYNC_ALIGNER));
3328
3329 sx_debug_1 = RREG32(SX_DEBUG_1);
3330 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3331 WREG32(SX_DEBUG_1, sx_debug_1);
3332
3333
3334 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3335 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3336 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3337 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3338
3339 if (rdev->family <= CHIP_SUMO2)
3340 WREG32(SMX_SAR_CTL0, 0x00010000);
3341
3342 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3343 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3344 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3345
3346 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3347 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3348 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3349
3350 WREG32(VGT_NUM_INSTANCES, 1);
3351 WREG32(SPI_CONFIG_CNTL, 0);
3352 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3353 WREG32(CP_PERFMON_CNTL, 0);
3354
3355 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3356 FETCH_FIFO_HIWATER(0x4) |
3357 DONE_FIFO_HIWATER(0xe0) |
3358 ALU_UPDATE_FIFO_HIWATER(0x8)));
3359
3360 sq_config = RREG32(SQ_CONFIG);
3361 sq_config &= ~(PS_PRIO(3) |
3362 VS_PRIO(3) |
3363 GS_PRIO(3) |
3364 ES_PRIO(3));
3365 sq_config |= (VC_ENABLE |
3366 EXPORT_SRC_C |
3367 PS_PRIO(0) |
3368 VS_PRIO(1) |
3369 GS_PRIO(2) |
3370 ES_PRIO(3));
3371
3372 switch (rdev->family) {
3373 case CHIP_CEDAR:
3374 case CHIP_PALM:
3375 case CHIP_SUMO:
3376 case CHIP_SUMO2:
3377 case CHIP_CAICOS:
3378 /* no vertex cache */
3379 sq_config &= ~VC_ENABLE;
3380 break;
3381 default:
3382 break;
3383 }
3384
3385 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3386
3387 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3388 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3389 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3390 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3391 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3392 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3393 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3394
3395 switch (rdev->family) {
3396 case CHIP_CEDAR:
3397 case CHIP_PALM:
3398 case CHIP_SUMO:
3399 case CHIP_SUMO2:
3400 ps_thread_count = 96;
3401 break;
3402 default:
3403 ps_thread_count = 128;
3404 break;
3405 }
3406
3407 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3408 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3409 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3410 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3411 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3412 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3413
3414 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3415 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3416 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3417 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3418 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3419 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3420
3421 WREG32(SQ_CONFIG, sq_config);
3422 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3423 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3424 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3425 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3426 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3427 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3428 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3429 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3430 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3431 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3432
3433 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3434 FORCE_EOV_MAX_REZ_CNT(255)));
3435
3436 switch (rdev->family) {
3437 case CHIP_CEDAR:
3438 case CHIP_PALM:
3439 case CHIP_SUMO:
3440 case CHIP_SUMO2:
3441 case CHIP_CAICOS:
3442 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3443 break;
3444 default:
3445 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3446 break;
3447 }
3448 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3449 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3450
3451 WREG32(VGT_GS_VERTEX_REUSE, 16);
3452 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3453 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3454
3455 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3456 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3457
3458 WREG32(CB_PERF_CTR0_SEL_0, 0);
3459 WREG32(CB_PERF_CTR0_SEL_1, 0);
3460 WREG32(CB_PERF_CTR1_SEL_0, 0);
3461 WREG32(CB_PERF_CTR1_SEL_1, 0);
3462 WREG32(CB_PERF_CTR2_SEL_0, 0);
3463 WREG32(CB_PERF_CTR2_SEL_1, 0);
3464 WREG32(CB_PERF_CTR3_SEL_0, 0);
3465 WREG32(CB_PERF_CTR3_SEL_1, 0);
3466
3467 /* clear render buffer base addresses */
3468 WREG32(CB_COLOR0_BASE, 0);
3469 WREG32(CB_COLOR1_BASE, 0);
3470 WREG32(CB_COLOR2_BASE, 0);
3471 WREG32(CB_COLOR3_BASE, 0);
3472 WREG32(CB_COLOR4_BASE, 0);
3473 WREG32(CB_COLOR5_BASE, 0);
3474 WREG32(CB_COLOR6_BASE, 0);
3475 WREG32(CB_COLOR7_BASE, 0);
3476 WREG32(CB_COLOR8_BASE, 0);
3477 WREG32(CB_COLOR9_BASE, 0);
3478 WREG32(CB_COLOR10_BASE, 0);
3479 WREG32(CB_COLOR11_BASE, 0);
3480
3481 /* set the shader const cache sizes to 0 */
3482 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3483 WREG32(i, 0);
3484 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3485 WREG32(i, 0);
3486
3487 tmp = RREG32(HDP_MISC_CNTL);
3488 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3489 WREG32(HDP_MISC_CNTL, tmp);
3490
3491 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3492 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3493
3494 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3495
3496 DRM_UDELAY(50);
3497
3498}
3499
3500int evergreen_mc_init(struct radeon_device *rdev)
3501{
3502 u32 tmp;
3503 int chansize, numchan;
3504
3505 /* Get VRAM informations */
3506 rdev->mc.vram_is_ddr = true;
3507 if ((rdev->family == CHIP_PALM) ||
3508 (rdev->family == CHIP_SUMO) ||
3509 (rdev->family == CHIP_SUMO2))
3510 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3511 else
3512 tmp = RREG32(MC_ARB_RAMCFG);
3513 if (tmp & CHANSIZE_OVERRIDE) {
3514 chansize = 16;
3515 } else if (tmp & CHANSIZE_MASK) {
3516 chansize = 64;
3517 } else {
3518 chansize = 32;
3519 }
3520 tmp = RREG32(MC_SHARED_CHMAP);
3521 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3522 case 0:
3523 default:
3524 numchan = 1;
3525 break;
3526 case 1:
3527 numchan = 2;
3528 break;
3529 case 2:
3530 numchan = 4;
3531 break;
3532 case 3:
3533 numchan = 8;
3534 break;
3535 }
3536 rdev->mc.vram_width = numchan * chansize;
3537 /* Could aper size report 0 ? */
3538 rdev->mc.aper_base = drm_get_resource_start(rdev->ddev, 0);
3539 rdev->mc.aper_size = drm_get_resource_len(rdev->ddev, 0);
3540 /* Setup GPU memory space */
3541 if ((rdev->family == CHIP_PALM) ||
3542 (rdev->family == CHIP_SUMO) ||
3543 (rdev->family == CHIP_SUMO2)) {
3544 /* size in bytes on fusion */
3545 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3546 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3547 } else {
3548 /* size in MB on evergreen/cayman/tn */
f43cf1b1
MN
3549 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3550 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
926deccb
FT
3551 }
3552 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3553 r700_vram_gtt_location(rdev, &rdev->mc);
3554 radeon_update_bandwidth_info(rdev);
3555
3556 return 0;
3557}
3558
b403bed8 3559void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
926deccb 3560{
926deccb
FT
3561 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
3562 RREG32(GRBM_STATUS));
3563 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
3564 RREG32(GRBM_STATUS_SE0));
3565 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
3566 RREG32(GRBM_STATUS_SE1));
3567 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
3568 RREG32(SRBM_STATUS));
b403bed8
MN
3569 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3570 RREG32(SRBM_STATUS2));
926deccb
FT
3571 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3572 RREG32(CP_STALLED_STAT1));
3573 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3574 RREG32(CP_STALLED_STAT2));
3575 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3576 RREG32(CP_BUSY_STAT));
3577 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3578 RREG32(CP_STAT));
b403bed8
MN
3579 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3580 RREG32(DMA_STATUS_REG));
3581 if (rdev->family >= CHIP_CAYMAN) {
3582 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3583 RREG32(DMA_STATUS_REG + 0x800));
3584 }
3585}
926deccb 3586
b403bed8
MN
3587bool evergreen_is_display_hung(struct radeon_device *rdev)
3588{
3589 u32 crtc_hung = 0;
3590 u32 crtc_status[6];
3591 u32 i, j, tmp;
926deccb 3592
b403bed8
MN
3593 for (i = 0; i < rdev->num_crtc; i++) {
3594 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3595 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3596 crtc_hung |= (1 << i);
3597 }
3598 }
926deccb 3599
b403bed8
MN
3600 for (j = 0; j < 10; j++) {
3601 for (i = 0; i < rdev->num_crtc; i++) {
3602 if (crtc_hung & (1 << i)) {
3603 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3604 if (tmp != crtc_status[i])
3605 crtc_hung &= ~(1 << i);
3606 }
3607 }
3608 if (crtc_hung == 0)
3609 return false;
3610 DRM_UDELAY(100);
3611 }
3612
3613 return true;
926deccb
FT
3614}
3615
b403bed8 3616static u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
926deccb 3617{
b403bed8 3618 u32 reset_mask = 0;
926deccb
FT
3619 u32 tmp;
3620
b403bed8
MN
3621 /* GRBM_STATUS */
3622 tmp = RREG32(GRBM_STATUS);
3623 if (tmp & (PA_BUSY | SC_BUSY |
3624 SH_BUSY | SX_BUSY |
3625 TA_BUSY | VGT_BUSY |
3626 DB_BUSY | CB_BUSY |
3627 SPI_BUSY | VGT_BUSY_NO_DMA))
3628 reset_mask |= RADEON_RESET_GFX;
926deccb 3629
b403bed8
MN
3630 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3631 CP_BUSY | CP_COHERENCY_BUSY))
3632 reset_mask |= RADEON_RESET_CP;
926deccb 3633
b403bed8
MN
3634 if (tmp & GRBM_EE_BUSY)
3635 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
926deccb 3636
b403bed8
MN
3637 /* DMA_STATUS_REG */
3638 tmp = RREG32(DMA_STATUS_REG);
3639 if (!(tmp & DMA_IDLE))
3640 reset_mask |= RADEON_RESET_DMA;
926deccb 3641
b403bed8
MN
3642 /* SRBM_STATUS2 */
3643 tmp = RREG32(SRBM_STATUS2);
3644 if (tmp & DMA_BUSY)
3645 reset_mask |= RADEON_RESET_DMA;
3646
3647 /* SRBM_STATUS */
3648 tmp = RREG32(SRBM_STATUS);
3649 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3650 reset_mask |= RADEON_RESET_RLC;
3651
3652 if (tmp & IH_BUSY)
3653 reset_mask |= RADEON_RESET_IH;
3654
3655 if (tmp & SEM_BUSY)
3656 reset_mask |= RADEON_RESET_SEM;
3657
3658 if (tmp & GRBM_RQ_PENDING)
3659 reset_mask |= RADEON_RESET_GRBM;
3660
3661 if (tmp & VMC_BUSY)
3662 reset_mask |= RADEON_RESET_VMC;
3663
3664 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3665 MCC_BUSY | MCD_BUSY))
3666 reset_mask |= RADEON_RESET_MC;
3667
3668 if (evergreen_is_display_hung(rdev))
3669 reset_mask |= RADEON_RESET_DISPLAY;
3670
3671 /* VM_L2_STATUS */
3672 tmp = RREG32(VM_L2_STATUS);
3673 if (tmp & L2_BUSY)
3674 reset_mask |= RADEON_RESET_VMC;
3675
3676 /* Skip MC reset as it's mostly likely not hung, just busy */
3677 if (reset_mask & RADEON_RESET_MC) {
3678 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3679 reset_mask &= ~RADEON_RESET_MC;
3680 }
3681
3682 return reset_mask;
926deccb
FT
3683}
3684
b403bed8 3685static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
926deccb
FT
3686{
3687 struct evergreen_mc_save save;
b403bed8
MN
3688 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3689 u32 tmp;
926deccb
FT
3690
3691 if (reset_mask == 0)
b403bed8 3692 return;
926deccb
FT
3693
3694 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3695
b403bed8
MN
3696 evergreen_print_gpu_status_regs(rdev);
3697
3698 /* Disable CP parsing/prefetching */
3699 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3700
3701 if (reset_mask & RADEON_RESET_DMA) {
3702 /* Disable DMA */
3703 tmp = RREG32(DMA_RB_CNTL);
3704 tmp &= ~DMA_RB_ENABLE;
3705 WREG32(DMA_RB_CNTL, tmp);
3706 }
3707
3708 DRM_UDELAY(50);
3709
926deccb
FT
3710 evergreen_mc_stop(rdev, &save);
3711 if (evergreen_mc_wait_for_idle(rdev)) {
3712 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3713 }
3714
b403bed8
MN
3715 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3716 grbm_soft_reset |= SOFT_RESET_DB |
3717 SOFT_RESET_CB |
3718 SOFT_RESET_PA |
3719 SOFT_RESET_SC |
3720 SOFT_RESET_SPI |
3721 SOFT_RESET_SX |
3722 SOFT_RESET_SH |
3723 SOFT_RESET_TC |
3724 SOFT_RESET_TA |
3725 SOFT_RESET_VC |
3726 SOFT_RESET_VGT;
3727 }
3728
3729 if (reset_mask & RADEON_RESET_CP) {
3730 grbm_soft_reset |= SOFT_RESET_CP |
3731 SOFT_RESET_VGT;
3732
3733 srbm_soft_reset |= SOFT_RESET_GRBM;
3734 }
926deccb
FT
3735
3736 if (reset_mask & RADEON_RESET_DMA)
b403bed8
MN
3737 srbm_soft_reset |= SOFT_RESET_DMA;
3738
3739 if (reset_mask & RADEON_RESET_DISPLAY)
3740 srbm_soft_reset |= SOFT_RESET_DC;
3741
3742 if (reset_mask & RADEON_RESET_RLC)
3743 srbm_soft_reset |= SOFT_RESET_RLC;
3744
3745 if (reset_mask & RADEON_RESET_SEM)
3746 srbm_soft_reset |= SOFT_RESET_SEM;
3747
3748 if (reset_mask & RADEON_RESET_IH)
3749 srbm_soft_reset |= SOFT_RESET_IH;
3750
3751 if (reset_mask & RADEON_RESET_GRBM)
3752 srbm_soft_reset |= SOFT_RESET_GRBM;
3753
3754 if (reset_mask & RADEON_RESET_VMC)
3755 srbm_soft_reset |= SOFT_RESET_VMC;
3756
3757 if (!(rdev->flags & RADEON_IS_IGP)) {
3758 if (reset_mask & RADEON_RESET_MC)
3759 srbm_soft_reset |= SOFT_RESET_MC;
3760 }
3761
3762 if (grbm_soft_reset) {
3763 tmp = RREG32(GRBM_SOFT_RESET);
3764 tmp |= grbm_soft_reset;
3765 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3766 WREG32(GRBM_SOFT_RESET, tmp);
3767 tmp = RREG32(GRBM_SOFT_RESET);
3768
3769 DRM_UDELAY(50);
3770
3771 tmp &= ~grbm_soft_reset;
3772 WREG32(GRBM_SOFT_RESET, tmp);
3773 tmp = RREG32(GRBM_SOFT_RESET);
3774 }
3775
3776 if (srbm_soft_reset) {
3777 tmp = RREG32(SRBM_SOFT_RESET);
3778 tmp |= srbm_soft_reset;
3779 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3780 WREG32(SRBM_SOFT_RESET, tmp);
3781 tmp = RREG32(SRBM_SOFT_RESET);
3782
3783 DRM_UDELAY(50);
3784
3785 tmp &= ~srbm_soft_reset;
3786 WREG32(SRBM_SOFT_RESET, tmp);
3787 tmp = RREG32(SRBM_SOFT_RESET);
3788 }
926deccb
FT
3789
3790 /* Wait a little for things to settle down */
3791 DRM_UDELAY(50);
3792
3793 evergreen_mc_resume(rdev, &save);
b403bed8
MN
3794 DRM_UDELAY(50);
3795
3796 evergreen_print_gpu_status_regs(rdev);
926deccb
FT
3797}
3798
3799int evergreen_asic_reset(struct radeon_device *rdev)
3800{
b403bed8
MN
3801 u32 reset_mask;
3802
3803 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3804
3805 if (reset_mask)
3806 r600_set_bios_scratch_engine_hung(rdev, true);
3807
3808 evergreen_gpu_soft_reset(rdev, reset_mask);
3809
3810 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3811
3812 if (!reset_mask)
3813 r600_set_bios_scratch_engine_hung(rdev, false);
3814
3815 return 0;
3816}
3817
3818/**
3819 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
3820 *
3821 * @rdev: radeon_device pointer
3822 * @ring: radeon_ring structure holding ring information
3823 *
3824 * Check if the GFX engine is locked up.
3825 * Returns true if the engine appears to be locked up, false if not.
3826 */
3827bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3828{
3829 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3830
3831 if (!(reset_mask & (RADEON_RESET_GFX |
3832 RADEON_RESET_COMPUTE |
3833 RADEON_RESET_CP))) {
3834 radeon_ring_lockup_update(ring);
3835 return false;
3836 }
3837 /* force CP activities */
3838 radeon_ring_force_activity(rdev, ring);
3839 return radeon_ring_test_lockup(rdev, ring);
3840}
3841
3842/**
3843 * evergreen_dma_is_lockup - Check if the DMA engine is locked up
3844 *
3845 * @rdev: radeon_device pointer
3846 * @ring: radeon_ring structure holding ring information
3847 *
3848 * Check if the async DMA engine is locked up.
3849 * Returns true if the engine appears to be locked up, false if not.
3850 */
3851bool evergreen_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3852{
3853 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3854
3855 if (!(reset_mask & RADEON_RESET_DMA)) {
3856 radeon_ring_lockup_update(ring);
3857 return false;
3858 }
3859 /* force ring activities */
3860 radeon_ring_force_activity(rdev, ring);
3861 return radeon_ring_test_lockup(rdev, ring);
926deccb
FT
3862}
3863
57e252bf
MN
3864/*
3865 * RLC
3866 */
3867#define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
3868#define RLC_CLEAR_STATE_END_MARKER 0x00000001
3869
3870void sumo_rlc_fini(struct radeon_device *rdev)
3871{
3872 int r;
3873
3874 /* save restore block */
3875 if (rdev->rlc.save_restore_obj) {
3876 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3877 if (unlikely(r != 0))
3878 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
3879 radeon_bo_unpin(rdev->rlc.save_restore_obj);
3880 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3881
3882 radeon_bo_unref(&rdev->rlc.save_restore_obj);
3883 rdev->rlc.save_restore_obj = NULL;
3884 }
3885
3886 /* clear state block */
3887 if (rdev->rlc.clear_state_obj) {
3888 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3889 if (unlikely(r != 0))
3890 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
3891 radeon_bo_unpin(rdev->rlc.clear_state_obj);
3892 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3893
3894 radeon_bo_unref(&rdev->rlc.clear_state_obj);
3895 rdev->rlc.clear_state_obj = NULL;
3896 }
3897}
3898
3899int sumo_rlc_init(struct radeon_device *rdev)
3900{
3901 u32 *src_ptr;
3902 volatile u32 *dst_ptr;
3903 u32 dws, data, i, j, k, reg_num;
3904 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index;
3905 u64 reg_list_mc_addr;
3906 struct cs_section_def *cs_data;
3907 int r;
3908 void *vptr;
3909
3910 vptr = NULL;
3911 src_ptr = rdev->rlc.reg_list;
3912 dws = rdev->rlc.reg_list_size;
3913 cs_data = rdev->rlc.cs_data;
3914
3915 /* save restore block */
3916 if (rdev->rlc.save_restore_obj == NULL) {
3917 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
3918 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.save_restore_obj);
3919 if (r) {
3920 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
3921 return r;
3922 }
3923 }
3924
3925 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3926 if (unlikely(r != 0)) {
3927 sumo_rlc_fini(rdev);
3928 return r;
3929 }
3930 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
3931 &rdev->rlc.save_restore_gpu_addr);
3932 if (r) {
3933 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3934 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
3935 sumo_rlc_fini(rdev);
3936 return r;
3937 }
3938 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void**)&vptr);
3939 if (r) {
3940 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
3941 sumo_rlc_fini(rdev);
3942 return r;
3943 }
3944 rdev->rlc.sr_ptr = vptr;
3945 /* write the sr buffer */
3946 dst_ptr = rdev->rlc.sr_ptr;
3947 /* format:
3948 * dw0: (reg2 << 16) | reg1
3949 * dw1: reg1 save space
3950 * dw2: reg2 save space
3951 */
3952 for (i = 0; i < dws; i++) {
3953 data = src_ptr[i] >> 2;
3954 i++;
3955 if (i < dws)
3956 data |= (src_ptr[i] >> 2) << 16;
3957 j = (((i - 1) * 3) / 2);
3958 dst_ptr[j] = data;
3959 }
3960 j = ((i * 3) / 2);