Pileus Git - ~andy/linux/blob - drivers/gpu/drm/radeon/ni.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include <drm/drmP.h>
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include <drm/radeon_drm.h>
  31 #include "nid.h"
  32 #include "atom.h"
  33 #include "ni_reg.h"
  34 #include "cayman_blit_shaders.h"
  35 #include "radeon_ucode.h"
  36 #include "clearstate_cayman.h"
  37
  38 static const u32 tn_rlc_save_restore_register_list[] =
  39 {
  40         0x98fc,
  41         0x98f0,
  42         0x9834,
  43         0x9838,
  44         0x9870,
  45         0x9874,
  46         0x8a14,
  47         0x8b24,
  48         0x8bcc,
  49         0x8b10,
  50         0x8c30,
  51         0x8d00,
  52         0x8d04,
  53         0x8c00,
  54         0x8c04,
  55         0x8c10,
  56         0x8c14,
  57         0x8d8c,
  58         0x8cf0,
  59         0x8e38,
  60         0x9508,
  61         0x9688,
  62         0x9608,
  63         0x960c,
  64         0x9610,
  65         0x9614,
  66         0x88c4,
  67         0x8978,
  68         0x88d4,
  69         0x900c,
  70         0x9100,
  71         0x913c,
  72         0x90e8,
  73         0x9354,
  74         0xa008,
  75         0x98f8,
  76         0x9148,
  77         0x914c,
  78         0x3f94,
  79         0x98f4,
  80         0x9b7c,
  81         0x3f8c,
  82         0x8950,
  83         0x8954,
  84         0x8a18,
  85         0x8b28,
  86         0x9144,
  87         0x3f90,
  88         0x915c,
  89         0x9160,
  90         0x9178,
  91         0x917c,
  92         0x9180,
  93         0x918c,
  94         0x9190,
  95         0x9194,
  96         0x9198,
  97         0x919c,
  98         0x91a8,
  99         0x91ac,
 100         0x91b0,
 101         0x91b4,
 102         0x91b8,
 103         0x91c4,
 104         0x91c8,
 105         0x91cc,
 106         0x91d0,
 107         0x91d4,
 108         0x91e0,
 109         0x91e4,
 110         0x91ec,
 111         0x91f0,
 112         0x91f4,
 113         0x9200,
 114         0x9204,
 115         0x929c,
 116         0x8030,
 117         0x9150,
 118         0x9a60,
 119         0x920c,
 120         0x9210,
 121         0x9228,
 122         0x922c,
 123         0x9244,
 124         0x9248,
 125         0x91e8,
 126         0x9294,
 127         0x9208,
 128         0x9224,
 129         0x9240,
 130         0x9220,
 131         0x923c,
 132         0x9258,
 133         0x9744,
 134         0xa200,
 135         0xa204,
 136         0xa208,
 137         0xa20c,
 138         0x8d58,
 139         0x9030,
 140         0x9034,
 141         0x9038,
 142         0x903c,
 143         0x9040,
 144         0x9654,
 145         0x897c,
 146         0xa210,
 147         0xa214,
 148         0x9868,
 149         0xa02c,
 150         0x9664,
 151         0x9698,
 152         0x949c,
 153         0x8e10,
 154         0x8e18,
 155         0x8c50,
 156         0x8c58,
 157         0x8c60,
 158         0x8c68,
 159         0x89b4,
 160         0x9830,
 161         0x802c,
 162 };
 163
 164 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
 165 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
 166 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
 167 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
 168 extern int evergreen_mc_wait_for_idle(struct radeon_device *rdev);
 169 extern void evergreen_mc_program(struct radeon_device *rdev);
 170 extern void evergreen_irq_suspend(struct radeon_device *rdev);
 171 extern int evergreen_mc_init(struct radeon_device *rdev);
 172 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
 173 extern void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 174 extern void evergreen_program_aspm(struct radeon_device *rdev);
 175 extern void sumo_rlc_fini(struct radeon_device *rdev);
 176 extern int sumo_rlc_init(struct radeon_device *rdev);
 177
 178 /* Firmware Names */
 179 MODULE_FIRMWARE("radeon/BARTS_pfp.bin");
 180 MODULE_FIRMWARE("radeon/BARTS_me.bin");
 181 MODULE_FIRMWARE("radeon/BARTS_mc.bin");
 182 MODULE_FIRMWARE("radeon/BARTS_smc.bin");
 183 MODULE_FIRMWARE("radeon/BTC_rlc.bin");
 184 MODULE_FIRMWARE("radeon/TURKS_pfp.bin");
 185 MODULE_FIRMWARE("radeon/TURKS_me.bin");
 186 MODULE_FIRMWARE("radeon/TURKS_mc.bin");
 187 MODULE_FIRMWARE("radeon/TURKS_smc.bin");
 188 MODULE_FIRMWARE("radeon/CAICOS_pfp.bin");
 189 MODULE_FIRMWARE("radeon/CAICOS_me.bin");
 190 MODULE_FIRMWARE("radeon/CAICOS_mc.bin");
 191 MODULE_FIRMWARE("radeon/CAICOS_smc.bin");
 192 MODULE_FIRMWARE("radeon/CAYMAN_pfp.bin");
 193 MODULE_FIRMWARE("radeon/CAYMAN_me.bin");
 194 MODULE_FIRMWARE("radeon/CAYMAN_mc.bin");
 195 MODULE_FIRMWARE("radeon/CAYMAN_rlc.bin");
 196 MODULE_FIRMWARE("radeon/CAYMAN_smc.bin");
 197 MODULE_FIRMWARE("radeon/ARUBA_pfp.bin");
 198 MODULE_FIRMWARE("radeon/ARUBA_me.bin");
 199 MODULE_FIRMWARE("radeon/ARUBA_rlc.bin");
 200
 201
 202 static const u32 cayman_golden_registers2[] =
 203 {
 204         0x3e5c, 0xffffffff, 0x00000000,
 205         0x3e48, 0xffffffff, 0x00000000,
 206         0x3e4c, 0xffffffff, 0x00000000,
 207         0x3e64, 0xffffffff, 0x00000000,
 208         0x3e50, 0xffffffff, 0x00000000,
 209         0x3e60, 0xffffffff, 0x00000000
 210 };
 211
 212 static const u32 cayman_golden_registers[] =
 213 {
 214         0x5eb4, 0xffffffff, 0x00000002,
 215         0x5e78, 0x8f311ff1, 0x001000f0,
 216         0x3f90, 0xffff0000, 0xff000000,
 217         0x9148, 0xffff0000, 0xff000000,
 218         0x3f94, 0xffff0000, 0xff000000,
 219         0x914c, 0xffff0000, 0xff000000,
 220         0xc78, 0x00000080, 0x00000080,
 221         0xbd4, 0x70073777, 0x00011003,
 222         0xd02c, 0xbfffff1f, 0x08421000,
 223         0xd0b8, 0x73773777, 0x02011003,
 224         0x5bc0, 0x00200000, 0x50100000,
 225         0x98f8, 0x33773777, 0x02011003,
 226         0x98fc, 0xffffffff, 0x76541032,
 227         0x7030, 0x31000311, 0x00000011,
 228         0x2f48, 0x33773777, 0x42010001,
 229         0x6b28, 0x00000010, 0x00000012,
 230         0x7728, 0x00000010, 0x00000012,
 231         0x10328, 0x00000010, 0x00000012,
 232         0x10f28, 0x00000010, 0x00000012,
 233         0x11b28, 0x00000010, 0x00000012,
 234         0x12728, 0x00000010, 0x00000012,
 235         0x240c, 0x000007ff, 0x00000000,
 236         0x8a14, 0xf000001f, 0x00000007,
 237         0x8b24, 0x3fff3fff, 0x00ff0fff,
 238         0x8b10, 0x0000ff0f, 0x00000000,
 239         0x28a4c, 0x07ffffff, 0x06000000,
 240         0x10c, 0x00000001, 0x00010003,
 241         0xa02c, 0xffffffff, 0x0000009b,
 242         0x913c, 0x0000010f, 0x01000100,
 243         0x8c04, 0xf8ff00ff, 0x40600060,
 244         0x28350, 0x00000f01, 0x00000000,
 245         0x9508, 0x3700001f, 0x00000002,
 246         0x960c, 0xffffffff, 0x54763210,
 247         0x88c4, 0x001f3ae3, 0x00000082,
 248         0x88d0, 0xffffffff, 0x0f40df40,
 249         0x88d4, 0x0000001f, 0x00000010,
 250         0x8974, 0xffffffff, 0x00000000
 251 };
 252
 253 static const u32 dvst_golden_registers2[] =
 254 {
 255         0x8f8, 0xffffffff, 0,
 256         0x8fc, 0x00380000, 0,
 257         0x8f8, 0xffffffff, 1,
 258         0x8fc, 0x0e000000, 0
 259 };
 260
 261 static const u32 dvst_golden_registers[] =
 262 {
 263         0x690, 0x3fff3fff, 0x20c00033,
 264         0x918c, 0x0fff0fff, 0x00010006,
 265         0x91a8, 0x0fff0fff, 0x00010006,
 266         0x9150, 0xffffdfff, 0x6e944040,
 267         0x917c, 0x0fff0fff, 0x00030002,
 268         0x9198, 0x0fff0fff, 0x00030002,
 269         0x915c, 0x0fff0fff, 0x00010000,
 270         0x3f90, 0xffff0001, 0xff000000,
 271         0x9178, 0x0fff0fff, 0x00070000,
 272         0x9194, 0x0fff0fff, 0x00070000,
 273         0x9148, 0xffff0001, 0xff000000,
 274         0x9190, 0x0fff0fff, 0x00090008,
 275         0x91ac, 0x0fff0fff, 0x00090008,
 276         0x3f94, 0xffff0000, 0xff000000,
 277         0x914c, 0xffff0000, 0xff000000,
 278         0x929c, 0x00000fff, 0x00000001,
 279         0x55e4, 0xff607fff, 0xfc000100,
 280         0x8a18, 0xff000fff, 0x00000100,
 281         0x8b28, 0xff000fff, 0x00000100,
 282         0x9144, 0xfffc0fff, 0x00000100,
 283         0x6ed8, 0x00010101, 0x00010000,
 284         0x9830, 0xffffffff, 0x00000000,
 285         0x9834, 0xf00fffff, 0x00000400,
 286         0x9838, 0xfffffffe, 0x00000000,
 287         0xd0c0, 0xff000fff, 0x00000100,
 288         0xd02c, 0xbfffff1f, 0x08421000,
 289         0xd0b8, 0x73773777, 0x12010001,
 290         0x5bb0, 0x000000f0, 0x00000070,
 291         0x98f8, 0x73773777, 0x12010001,
 292         0x98fc, 0xffffffff, 0x00000010,
 293         0x9b7c, 0x00ff0000, 0x00fc0000,
 294         0x8030, 0x00001f0f, 0x0000100a,
 295         0x2f48, 0x73773777, 0x12010001,
 296         0x2408, 0x00030000, 0x000c007f,
 297         0x8a14, 0xf000003f, 0x00000007,
 298         0x8b24, 0x3fff3fff, 0x00ff0fff,
 299         0x8b10, 0x0000ff0f, 0x00000000,
 300         0x28a4c, 0x07ffffff, 0x06000000,
 301         0x4d8, 0x00000fff, 0x00000100,
 302         0xa008, 0xffffffff, 0x00010000,
 303         0x913c, 0xffff03ff, 0x01000100,
 304         0x8c00, 0x000000ff, 0x00000003,
 305         0x8c04, 0xf8ff00ff, 0x40600060,
 306         0x8cf0, 0x1fff1fff, 0x08e00410,
 307         0x28350, 0x00000f01, 0x00000000,
 308         0x9508, 0xf700071f, 0x00000002,
 309         0x960c, 0xffffffff, 0x54763210,
 310         0x20ef8, 0x01ff01ff, 0x00000002,
 311         0x20e98, 0xfffffbff, 0x00200000,
 312         0x2015c, 0xffffffff, 0x00000f40,
 313         0x88c4, 0x001f3ae3, 0x00000082,
 314         0x8978, 0x3fffffff, 0x04050140,
 315         0x88d4, 0x0000001f, 0x00000010,
 316         0x8974, 0xffffffff, 0x00000000
 317 };
 318
 319 static const u32 scrapper_golden_registers[] =
 320 {
 321         0x690, 0x3fff3fff, 0x20c00033,
 322         0x918c, 0x0fff0fff, 0x00010006,
 323         0x918c, 0x0fff0fff, 0x00010006,
 324         0x91a8, 0x0fff0fff, 0x00010006,
 325         0x91a8, 0x0fff0fff, 0x00010006,
 326         0x9150, 0xffffdfff, 0x6e944040,
 327         0x9150, 0xffffdfff, 0x6e944040,
 328         0x917c, 0x0fff0fff, 0x00030002,
 329         0x917c, 0x0fff0fff, 0x00030002,
 330         0x9198, 0x0fff0fff, 0x00030002,
 331         0x9198, 0x0fff0fff, 0x00030002,
 332         0x915c, 0x0fff0fff, 0x00010000,
 333         0x915c, 0x0fff0fff, 0x00010000,
 334         0x3f90, 0xffff0001, 0xff000000,
 335         0x3f90, 0xffff0001, 0xff000000,
 336         0x9178, 0x0fff0fff, 0x00070000,
 337         0x9178, 0x0fff0fff, 0x00070000,
 338         0x9194, 0x0fff0fff, 0x00070000,
 339         0x9194, 0x0fff0fff, 0x00070000,
 340         0x9148, 0xffff0001, 0xff000000,
 341         0x9148, 0xffff0001, 0xff000000,
 342         0x9190, 0x0fff0fff, 0x00090008,
 343         0x9190, 0x0fff0fff, 0x00090008,
 344         0x91ac, 0x0fff0fff, 0x00090008,
 345         0x91ac, 0x0fff0fff, 0x00090008,
 346         0x3f94, 0xffff0000, 0xff000000,
 347         0x3f94, 0xffff0000, 0xff000000,
 348         0x914c, 0xffff0000, 0xff000000,
 349         0x914c, 0xffff0000, 0xff000000,
 350         0x929c, 0x00000fff, 0x00000001,
 351         0x929c, 0x00000fff, 0x00000001,
 352         0x55e4, 0xff607fff, 0xfc000100,
 353         0x8a18, 0xff000fff, 0x00000100,
 354         0x8a18, 0xff000fff, 0x00000100,
 355         0x8b28, 0xff000fff, 0x00000100,
 356         0x8b28, 0xff000fff, 0x00000100,
 357         0x9144, 0xfffc0fff, 0x00000100,
 358         0x9144, 0xfffc0fff, 0x00000100,
 359         0x6ed8, 0x00010101, 0x00010000,
 360         0x9830, 0xffffffff, 0x00000000,
 361         0x9830, 0xffffffff, 0x00000000,
 362         0x9834, 0xf00fffff, 0x00000400,
 363         0x9834, 0xf00fffff, 0x00000400,
 364         0x9838, 0xfffffffe, 0x00000000,
 365         0x9838, 0xfffffffe, 0x00000000,
 366         0xd0c0, 0xff000fff, 0x00000100,
 367         0xd02c, 0xbfffff1f, 0x08421000,
 368         0xd02c, 0xbfffff1f, 0x08421000,
 369         0xd0b8, 0x73773777, 0x12010001,
 370         0xd0b8, 0x73773777, 0x12010001,
 371         0x5bb0, 0x000000f0, 0x00000070,
 372         0x98f8, 0x73773777, 0x12010001,
 373         0x98f8, 0x73773777, 0x12010001,
 374         0x98fc, 0xffffffff, 0x00000010,
 375         0x98fc, 0xffffffff, 0x00000010,
 376         0x9b7c, 0x00ff0000, 0x00fc0000,
 377         0x9b7c, 0x00ff0000, 0x00fc0000,
 378         0x8030, 0x00001f0f, 0x0000100a,
 379         0x8030, 0x00001f0f, 0x0000100a,
 380         0x2f48, 0x73773777, 0x12010001,
 381         0x2f48, 0x73773777, 0x12010001,
 382         0x2408, 0x00030000, 0x000c007f,
 383         0x8a14, 0xf000003f, 0x00000007,
 384         0x8a14, 0xf000003f, 0x00000007,
 385         0x8b24, 0x3fff3fff, 0x00ff0fff,
 386         0x8b24, 0x3fff3fff, 0x00ff0fff,
 387         0x8b10, 0x0000ff0f, 0x00000000,
 388         0x8b10, 0x0000ff0f, 0x00000000,
 389         0x28a4c, 0x07ffffff, 0x06000000,
 390         0x28a4c, 0x07ffffff, 0x06000000,
 391         0x4d8, 0x00000fff, 0x00000100,
 392         0x4d8, 0x00000fff, 0x00000100,
 393         0xa008, 0xffffffff, 0x00010000,
 394         0xa008, 0xffffffff, 0x00010000,
 395         0x913c, 0xffff03ff, 0x01000100,
 396         0x913c, 0xffff03ff, 0x01000100,
 397         0x90e8, 0x001fffff, 0x010400c0,
 398         0x8c00, 0x000000ff, 0x00000003,
 399         0x8c00, 0x000000ff, 0x00000003,
 400         0x8c04, 0xf8ff00ff, 0x40600060,
 401         0x8c04, 0xf8ff00ff, 0x40600060,
 402         0x8c30, 0x0000000f, 0x00040005,
 403         0x8cf0, 0x1fff1fff, 0x08e00410,
 404         0x8cf0, 0x1fff1fff, 0x08e00410,
 405         0x900c, 0x00ffffff, 0x0017071f,
 406         0x28350, 0x00000f01, 0x00000000,
 407         0x28350, 0x00000f01, 0x00000000,
 408         0x9508, 0xf700071f, 0x00000002,
 409         0x9508, 0xf700071f, 0x00000002,
 410         0x9688, 0x00300000, 0x0017000f,
 411         0x960c, 0xffffffff, 0x54763210,
 412         0x960c, 0xffffffff, 0x54763210,
 413         0x20ef8, 0x01ff01ff, 0x00000002,
 414         0x20e98, 0xfffffbff, 0x00200000,
 415         0x2015c, 0xffffffff, 0x00000f40,
 416         0x88c4, 0x001f3ae3, 0x00000082,
 417         0x88c4, 0x001f3ae3, 0x00000082,
 418         0x8978, 0x3fffffff, 0x04050140,
 419         0x8978, 0x3fffffff, 0x04050140,
 420         0x88d4, 0x0000001f, 0x00000010,
 421         0x88d4, 0x0000001f, 0x00000010,
 422         0x8974, 0xffffffff, 0x00000000,
 423         0x8974, 0xffffffff, 0x00000000
 424 };
 425
 426 static void ni_init_golden_registers(struct radeon_device *rdev)
 427 {
 428         switch (rdev->family) {
 429         case CHIP_CAYMAN:
 430                 radeon_program_register_sequence(rdev,
 431                                                  cayman_golden_registers,
 432                                                  (const u32)ARRAY_SIZE(cayman_golden_registers));
 433                 radeon_program_register_sequence(rdev,
 434                                                  cayman_golden_registers2,
 435                                                  (const u32)ARRAY_SIZE(cayman_golden_registers2));
 436                 break;
 437         case CHIP_ARUBA:
 438                 if ((rdev->pdev->device == 0x9900) ||
 439                     (rdev->pdev->device == 0x9901) ||
 440                     (rdev->pdev->device == 0x9903) ||
 441                     (rdev->pdev->device == 0x9904) ||
 442                     (rdev->pdev->device == 0x9905) ||
 443                     (rdev->pdev->device == 0x9906) ||
 444                     (rdev->pdev->device == 0x9907) ||
 445                     (rdev->pdev->device == 0x9908) ||
 446                     (rdev->pdev->device == 0x9909) ||
 447                     (rdev->pdev->device == 0x990A) ||
 448                     (rdev->pdev->device == 0x990B) ||
 449                     (rdev->pdev->device == 0x990C) ||
 450                     (rdev->pdev->device == 0x990D) ||
 451                     (rdev->pdev->device == 0x990E) ||
 452                     (rdev->pdev->device == 0x990F) ||
 453                     (rdev->pdev->device == 0x9910) ||
 454                     (rdev->pdev->device == 0x9913) ||
 455                     (rdev->pdev->device == 0x9917) ||
 456                     (rdev->pdev->device == 0x9918)) {
 457                         radeon_program_register_sequence(rdev,
 458                                                          dvst_golden_registers,
 459                                                          (const u32)ARRAY_SIZE(dvst_golden_registers));
 460                         radeon_program_register_sequence(rdev,
 461                                                          dvst_golden_registers2,
 462                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 463                 } else {
 464                         radeon_program_register_sequence(rdev,
 465                                                          scrapper_golden_registers,
 466                                                          (const u32)ARRAY_SIZE(scrapper_golden_registers));
 467                         radeon_program_register_sequence(rdev,
 468                                                          dvst_golden_registers2,
 469                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 470                 }
 471                 break;
 472         default:
 473                 break;
 474         }
 475 }
 476
 477 #define BTC_IO_MC_REGS_SIZE 29
 478
 479 static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 480         {0x00000077, 0xff010100},
 481         {0x00000078, 0x00000000},
 482         {0x00000079, 0x00001434},
 483         {0x0000007a, 0xcc08ec08},
 484         {0x0000007b, 0x00040000},
 485         {0x0000007c, 0x000080c0},
 486         {0x0000007d, 0x09000000},
 487         {0x0000007e, 0x00210404},
 488         {0x00000081, 0x08a8e800},
 489         {0x00000082, 0x00030444},
 490         {0x00000083, 0x00000000},
 491         {0x00000085, 0x00000001},
 492         {0x00000086, 0x00000002},
 493         {0x00000087, 0x48490000},
 494         {0x00000088, 0x20244647},
 495         {0x00000089, 0x00000005},
 496         {0x0000008b, 0x66030000},
 497         {0x0000008c, 0x00006603},
 498         {0x0000008d, 0x00000100},
 499         {0x0000008f, 0x00001c0a},
 500         {0x00000090, 0xff000001},
 501         {0x00000094, 0x00101101},
 502         {0x00000095, 0x00000fff},
 503         {0x00000096, 0x00116fff},
 504         {0x00000097, 0x60010000},
 505         {0x00000098, 0x10010000},
 506         {0x00000099, 0x00006000},
 507         {0x0000009a, 0x00001000},
 508         {0x0000009f, 0x00946a00}
 509 };
 510
 511 static const u32 turks_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 512         {0x00000077, 0xff010100},
 513         {0x00000078, 0x00000000},
 514         {0x00000079, 0x00001434},
 515         {0x0000007a, 0xcc08ec08},
 516         {0x0000007b, 0x00040000},
 517         {0x0000007c, 0x000080c0},
 518         {0x0000007d, 0x09000000},
 519         {0x0000007e, 0x00210404},
 520         {0x00000081, 0x08a8e800},
 521         {0x00000082, 0x00030444},
 522         {0x00000083, 0x00000000},
 523         {0x00000085, 0x00000001},
 524         {0x00000086, 0x00000002},
 525         {0x00000087, 0x48490000},
 526         {0x00000088, 0x20244647},
 527         {0x00000089, 0x00000005},
 528         {0x0000008b, 0x66030000},
 529         {0x0000008c, 0x00006603},
 530         {0x0000008d, 0x00000100},
 531         {0x0000008f, 0x00001c0a},
 532         {0x00000090, 0xff000001},
 533         {0x00000094, 0x00101101},
 534         {0x00000095, 0x00000fff},
 535         {0x00000096, 0x00116fff},
 536         {0x00000097, 0x60010000},
 537         {0x00000098, 0x10010000},
 538         {0x00000099, 0x00006000},
 539         {0x0000009a, 0x00001000},
 540         {0x0000009f, 0x00936a00}
 541 };
 542
 543 static const u32 caicos_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 544         {0x00000077, 0xff010100},
 545         {0x00000078, 0x00000000},
 546         {0x00000079, 0x00001434},
 547         {0x0000007a, 0xcc08ec08},
 548         {0x0000007b, 0x00040000},
 549         {0x0000007c, 0x000080c0},
 550         {0x0000007d, 0x09000000},
 551         {0x0000007e, 0x00210404},
 552         {0x00000081, 0x08a8e800},
 553         {0x00000082, 0x00030444},
 554         {0x00000083, 0x00000000},
 555         {0x00000085, 0x00000001},
 556         {0x00000086, 0x00000002},
 557         {0x00000087, 0x48490000},
 558         {0x00000088, 0x20244647},
 559         {0x00000089, 0x00000005},
 560         {0x0000008b, 0x66030000},
 561         {0x0000008c, 0x00006603},
 562         {0x0000008d, 0x00000100},
 563         {0x0000008f, 0x00001c0a},
 564         {0x00000090, 0xff000001},
 565         {0x00000094, 0x00101101},
 566         {0x00000095, 0x00000fff},
 567         {0x00000096, 0x00116fff},
 568         {0x00000097, 0x60010000},
 569         {0x00000098, 0x10010000},
 570         {0x00000099, 0x00006000},
 571         {0x0000009a, 0x00001000},
 572         {0x0000009f, 0x00916a00}
 573 };
 574
 575 static const u32 cayman_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 576         {0x00000077, 0xff010100},
 577         {0x00000078, 0x00000000},
 578         {0x00000079, 0x00001434},
 579         {0x0000007a, 0xcc08ec08},
 580         {0x0000007b, 0x00040000},
 581         {0x0000007c, 0x000080c0},
 582         {0x0000007d, 0x09000000},
 583         {0x0000007e, 0x00210404},
 584         {0x00000081, 0x08a8e800},
 585         {0x00000082, 0x00030444},
 586         {0x00000083, 0x00000000},
 587         {0x00000085, 0x00000001},
 588         {0x00000086, 0x00000002},
 589         {0x00000087, 0x48490000},
 590         {0x00000088, 0x20244647},
 591         {0x00000089, 0x00000005},
 592         {0x0000008b, 0x66030000},
 593         {0x0000008c, 0x00006603},
 594         {0x0000008d, 0x00000100},
 595         {0x0000008f, 0x00001c0a},
 596         {0x00000090, 0xff000001},
 597         {0x00000094, 0x00101101},
 598         {0x00000095, 0x00000fff},
 599         {0x00000096, 0x00116fff},
 600         {0x00000097, 0x60010000},
 601         {0x00000098, 0x10010000},
 602         {0x00000099, 0x00006000},
 603         {0x0000009a, 0x00001000},
 604         {0x0000009f, 0x00976b00}
 605 };
 606
 607 int ni_mc_load_microcode(struct radeon_device *rdev)
 608 {
 609         const __be32 *fw_data;
 610         u32 mem_type, running, blackout = 0;
 611         u32 *io_mc_regs;
 612         int i, ucode_size, regs_size;
 613
 614         if (!rdev->mc_fw)
 615                 return -EINVAL;
 616
 617         switch (rdev->family) {
 618         case CHIP_BARTS:
 619                 io_mc_regs = (u32 *)&barts_io_mc_regs;
 620                 ucode_size = BTC_MC_UCODE_SIZE;
 621                 regs_size = BTC_IO_MC_REGS_SIZE;
 622                 break;
 623         case CHIP_TURKS:
 624                 io_mc_regs = (u32 *)&turks_io_mc_regs;
 625                 ucode_size = BTC_MC_UCODE_SIZE;
 626                 regs_size = BTC_IO_MC_REGS_SIZE;
 627                 break;
 628         case CHIP_CAICOS:
 629         default:
 630                 io_mc_regs = (u32 *)&caicos_io_mc_regs;
 631                 ucode_size = BTC_MC_UCODE_SIZE;
 632                 regs_size = BTC_IO_MC_REGS_SIZE;
 633                 break;
 634         case CHIP_CAYMAN:
 635                 io_mc_regs = (u32 *)&cayman_io_mc_regs;
 636                 ucode_size = CAYMAN_MC_UCODE_SIZE;
 637                 regs_size = BTC_IO_MC_REGS_SIZE;
 638                 break;
 639         }
 640
 641         mem_type = (RREG32(MC_SEQ_MISC0) & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT;
 642         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
 643
 644         if ((mem_type == MC_SEQ_MISC0_GDDR5_VALUE) && (running == 0)) {
 645                 if (running) {
 646                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
 647                         WREG32(MC_SHARED_BLACKOUT_CNTL, 1);
 648                 }
 649
 650                 /* reset the engine and set to writable */
 651                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 652                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
 653
 654                 /* load mc io regs */
 655                 for (i = 0; i < regs_size; i++) {
 656                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
 657                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
 658                 }
 659                 /* load the MC ucode */
 660                 fw_data = (const __be32 *)rdev->mc_fw->data;
 661                 for (i = 0; i < ucode_size; i++)
 662                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
 663
 664                 /* put the engine back into the active state */
 665                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 666                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
 667                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
 668
 669                 /* wait for training to complete */
 670                 for (i = 0; i < rdev->usec_timeout; i++) {
 671                         if (RREG32(MC_IO_PAD_CNTL_D0) & MEM_FALL_OUT_CMD)
 672                                 break;
 673                         udelay(1);
 674                 }
 675
 676                 if (running)
 677                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
 678         }
 679
 680         return 0;
 681 }
 682
 683 int ni_init_microcode(struct radeon_device *rdev)
 684 {
 685         const char *chip_name;
 686         const char *rlc_chip_name;
 687         size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size;
 688         size_t smc_req_size = 0;
 689         char fw_name[30];
 690         int err;
 691
 692         DRM_DEBUG("\n");
 693
 694         switch (rdev->family) {
 695         case CHIP_BARTS:
 696                 chip_name = "BARTS";
 697                 rlc_chip_name = "BTC";
 698                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 699                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 700                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 701                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 702                 smc_req_size = ALIGN(BARTS_SMC_UCODE_SIZE, 4);
 703                 break;
 704         case CHIP_TURKS:
 705                 chip_name = "TURKS";
 706                 rlc_chip_name = "BTC";
 707                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 708                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 709                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 710                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 711                 smc_req_size = ALIGN(TURKS_SMC_UCODE_SIZE, 4);
 712                 break;
 713         case CHIP_CAICOS:
 714                 chip_name = "CAICOS";
 715                 rlc_chip_name = "BTC";
 716                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 717                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 718                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 719                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 720                 smc_req_size = ALIGN(CAICOS_SMC_UCODE_SIZE, 4);
 721                 break;
 722         case CHIP_CAYMAN:
 723                 chip_name = "CAYMAN";
 724                 rlc_chip_name = "CAYMAN";
 725                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 726                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 727                 rlc_req_size = CAYMAN_RLC_UCODE_SIZE * 4;
 728                 mc_req_size = CAYMAN_MC_UCODE_SIZE * 4;
 729                 smc_req_size = ALIGN(CAYMAN_SMC_UCODE_SIZE, 4);
 730                 break;
 731         case CHIP_ARUBA:
 732                 chip_name = "ARUBA";
 733                 rlc_chip_name = "ARUBA";
 734                 /* pfp/me same size as CAYMAN */
 735                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 736                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 737                 rlc_req_size = ARUBA_RLC_UCODE_SIZE * 4;
 738                 mc_req_size = 0;
 739                 break;
 740         default: BUG();
 741         }
 742
 743         DRM_INFO("Loading %s Microcode\n", chip_name);
 744
 745         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
 746         err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
 747         if (err)
 748                 goto out;
 749         if (rdev->pfp_fw->size != pfp_req_size) {
 750                 printk(KERN_ERR
 751                        "ni_cp: Bogus length %zu in firmware \"%s\"\n",
 752                        rdev->pfp_fw->size, fw_name);
 753                 err = -EINVAL;
 754                 goto out;
 755         }
 756
 757         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
 758         err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
 759         if (err)
 760                 goto out;
 761         if (rdev->me_fw->size != me_req_size) {
 762                 printk(KERN_ERR
 763                        "ni_cp: Bogus length %zu in firmware \"%s\"\n",
 764                        rdev->me_fw->size, fw_name);
 765                 err = -EINVAL;
 766         }
 767
 768         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
 769         err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
 770         if (err)
 771                 goto out;
 772         if (rdev->rlc_fw->size != rlc_req_size) {
 773                 printk(KERN_ERR
 774                        "ni_rlc: Bogus length %zu in firmware \"%s\"\n",
 775                        rdev->rlc_fw->size, fw_name);
 776                 err = -EINVAL;
 777         }
 778
 779         /* no MC ucode on TN */
 780         if (!(rdev->flags & RADEON_IS_IGP)) {
 781                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
 782                 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
 783                 if (err)
 784                         goto out;
 785                 if (rdev->mc_fw->size != mc_req_size) {
 786                         printk(KERN_ERR
 787                                "ni_mc: Bogus length %zu in firmware \"%s\"\n",
 788                                rdev->mc_fw->size, fw_name);
 789                         err = -EINVAL;
 790                 }
 791         }
 792
 793         if ((rdev->family >= CHIP_BARTS) && (rdev->family <= CHIP_CAYMAN)) {
 794                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
 795                 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
 796                 if (err) {
 797                         printk(KERN_ERR
 798                                "smc: error loading firmware \"%s\"\n",
 799                                fw_name);
 800                         release_firmware(rdev->smc_fw);
 801                         rdev->smc_fw = NULL;
 802                         err = 0;
 803                 } else if (rdev->smc_fw->size != smc_req_size) {
 804                         printk(KERN_ERR
 805                                "ni_mc: Bogus length %zu in firmware \"%s\"\n",
 806                                rdev->mc_fw->size, fw_name);
 807                         err = -EINVAL;
 808                 }
 809         }
 810
 811 out:
 812         if (err) {
 813                 if (err != -EINVAL)
 814                         printk(KERN_ERR
 815                                "ni_cp: Failed to load firmware \"%s\"\n",
 816                                fw_name);
 817                 release_firmware(rdev->pfp_fw);
 818                 rdev->pfp_fw = NULL;
 819                 release_firmware(rdev->me_fw);
 820                 rdev->me_fw = NULL;
 821                 release_firmware(rdev->rlc_fw);
 822                 rdev->rlc_fw = NULL;
 823                 release_firmware(rdev->mc_fw);
 824                 rdev->mc_fw = NULL;
 825         }
 826         return err;
 827 }
 828
 829 int tn_get_temp(struct radeon_device *rdev)
 830 {
 831         u32 temp = RREG32_SMC(TN_CURRENT_GNB_TEMP) & 0x7ff;
 832         int actual_temp = (temp / 8) - 49;
 833
 834         return actual_temp * 1000;
 835 }
 836
 837 /*
 838  * Core functions
 839  */
 840 static void cayman_gpu_init(struct radeon_device *rdev)
 841 {
 842         u32 gb_addr_config = 0;
 843         u32 mc_shared_chmap, mc_arb_ramcfg;
 844         u32 cgts_tcc_disable;
 845         u32 sx_debug_1;
 846         u32 smx_dc_ctl0;
 847         u32 cgts_sm_ctrl_reg;
 848         u32 hdp_host_path_cntl;
 849         u32 tmp;
 850         u32 disabled_rb_mask;
 851         int i, j;
 852
 853         switch (rdev->family) {
 854         case CHIP_CAYMAN:
 855                 rdev->config.cayman.max_shader_engines = 2;
 856                 rdev->config.cayman.max_pipes_per_simd = 4;
 857                 rdev->config.cayman.max_tile_pipes = 8;
 858                 rdev->config.cayman.max_simds_per_se = 12;
 859                 rdev->config.cayman.max_backends_per_se = 4;
 860                 rdev->config.cayman.max_texture_channel_caches = 8;
 861                 rdev->config.cayman.max_gprs = 256;
 862                 rdev->config.cayman.max_threads = 256;
 863                 rdev->config.cayman.max_gs_threads = 32;
 864                 rdev->config.cayman.max_stack_entries = 512;
 865                 rdev->config.cayman.sx_num_of_sets = 8;
 866                 rdev->config.cayman.sx_max_export_size = 256;
 867                 rdev->config.cayman.sx_max_export_pos_size = 64;
 868                 rdev->config.cayman.sx_max_export_smx_size = 192;
 869                 rdev->config.cayman.max_hw_contexts = 8;
 870                 rdev->config.cayman.sq_num_cf_insts = 2;
 871
 872                 rdev->config.cayman.sc_prim_fifo_size = 0x100;
 873                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 874                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 875                 gb_addr_config = CAYMAN_GB_ADDR_CONFIG_GOLDEN;
 876                 break;
 877         case CHIP_ARUBA:
 878         default:
 879                 rdev->config.cayman.max_shader_engines = 1;
 880                 rdev->config.cayman.max_pipes_per_simd = 4;
 881                 rdev->config.cayman.max_tile_pipes = 2;
 882                 if ((rdev->pdev->device == 0x9900) ||
 883                     (rdev->pdev->device == 0x9901) ||
 884                     (rdev->pdev->device == 0x9905) ||
 885                     (rdev->pdev->device == 0x9906) ||
 886                     (rdev->pdev->device == 0x9907) ||
 887                     (rdev->pdev->device == 0x9908) ||
 888                     (rdev->pdev->device == 0x9909) ||
 889                     (rdev->pdev->device == 0x990B) ||
 890                     (rdev->pdev->device == 0x990C) ||
 891                     (rdev->pdev->device == 0x990F) ||
 892                     (rdev->pdev->device == 0x9910) ||
 893                     (rdev->pdev->device == 0x9917) ||
 894                     (rdev->pdev->device == 0x9999) ||
 895                     (rdev->pdev->device == 0x999C)) {
 896                         rdev->config.cayman.max_simds_per_se = 6;
 897                         rdev->config.cayman.max_backends_per_se = 2;
 898                 } else if ((rdev->pdev->device == 0x9903) ||
 899                            (rdev->pdev->device == 0x9904) ||
 900                            (rdev->pdev->device == 0x990A) ||
 901                            (rdev->pdev->device == 0x990D) ||
 902                            (rdev->pdev->device == 0x990E) ||
 903                            (rdev->pdev->device == 0x9913) ||
 904                            (rdev->pdev->device == 0x9918) ||
 905                            (rdev->pdev->device == 0x999D)) {
 906                         rdev->config.cayman.max_simds_per_se = 4;
 907                         rdev->config.cayman.max_backends_per_se = 2;
 908                 } else if ((rdev->pdev->device == 0x9919) ||
 909                            (rdev->pdev->device == 0x9990) ||
 910                            (rdev->pdev->device == 0x9991) ||
 911                            (rdev->pdev->device == 0x9994) ||
 912                            (rdev->pdev->device == 0x9995) ||
 913                            (rdev->pdev->device == 0x9996) ||
 914                            (rdev->pdev->device == 0x999A) ||
 915                            (rdev->pdev->device == 0x99A0)) {
 916                         rdev->config.cayman.max_simds_per_se = 3;
 917                         rdev->config.cayman.max_backends_per_se = 1;
 918                 } else {
 919                         rdev->config.cayman.max_simds_per_se = 2;
 920                         rdev->config.cayman.max_backends_per_se = 1;
 921                 }
 922                 rdev->config.cayman.max_texture_channel_caches = 2;
 923                 rdev->config.cayman.max_gprs = 256;
 924                 rdev->config.cayman.max_threads = 256;
 925                 rdev->config.cayman.max_gs_threads = 32;
 926                 rdev->config.cayman.max_stack_entries = 512;
 927                 rdev->config.cayman.sx_num_of_sets = 8;
 928                 rdev->config.cayman.sx_max_export_size = 256;
 929                 rdev->config.cayman.sx_max_export_pos_size = 64;
 930                 rdev->config.cayman.sx_max_export_smx_size = 192;
 931                 rdev->config.cayman.max_hw_contexts = 8;
 932                 rdev->config.cayman.sq_num_cf_insts = 2;
 933
 934                 rdev->config.cayman.sc_prim_fifo_size = 0x40;
 935                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 936                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 937                 gb_addr_config = ARUBA_GB_ADDR_CONFIG_GOLDEN;
 938                 break;
 939         }
 940
 941         /* Initialize HDP */
 942         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
 943                 WREG32((0x2c14 + j), 0x00000000);
 944                 WREG32((0x2c18 + j), 0x00000000);
 945                 WREG32((0x2c1c + j), 0x00000000);
 946                 WREG32((0x2c20 + j), 0x00000000);
 947                 WREG32((0x2c24 + j), 0x00000000);
 948         }
 949
 950         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
 951
 952         evergreen_fix_pci_max_read_req_size(rdev);
 953
 954         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
 955         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
 956
 957         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
 958         rdev->config.cayman.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
 959         if (rdev->config.cayman.mem_row_size_in_kb > 4)
 960                 rdev->config.cayman.mem_row_size_in_kb = 4;
 961         /* XXX use MC settings? */
 962         rdev->config.cayman.shader_engine_tile_size = 32;
 963         rdev->config.cayman.num_gpus = 1;
 964         rdev->config.cayman.multi_gpu_tile_size = 64;
 965
 966         tmp = (gb_addr_config & NUM_PIPES_MASK) >> NUM_PIPES_SHIFT;
 967         rdev->config.cayman.num_tile_pipes = (1 << tmp);
 968         tmp = (gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT;
 969         rdev->config.cayman.mem_max_burst_length_bytes = (tmp + 1) * 256;
 970         tmp = (gb_addr_config & NUM_SHADER_ENGINES_MASK) >> NUM_SHADER_ENGINES_SHIFT;
 971         rdev->config.cayman.num_shader_engines = tmp + 1;
 972         tmp = (gb_addr_config & NUM_GPUS_MASK) >> NUM_GPUS_SHIFT;
 973         rdev->config.cayman.num_gpus = tmp + 1;
 974         tmp = (gb_addr_config & MULTI_GPU_TILE_SIZE_MASK) >> MULTI_GPU_TILE_SIZE_SHIFT;
 975         rdev->config.cayman.multi_gpu_tile_size = 1 << tmp;
 976         tmp = (gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT;
 977         rdev->config.cayman.mem_row_size_in_kb = 1 << tmp;
 978
 979
 980         /* setup tiling info dword.  gb_addr_config is not adequate since it does
 981          * not have bank info, so create a custom tiling dword.
 982          * bits 3:0   num_pipes
 983          * bits 7:4   num_banks
 984          * bits 11:8  group_size
 985          * bits 15:12 row_size
 986          */
 987         rdev->config.cayman.tile_config = 0;
 988         switch (rdev->config.cayman.num_tile_pipes) {
 989         case 1:
 990         default:
 991                 rdev->config.cayman.tile_config |= (0 << 0);
 992                 break;
 993         case 2:
 994                 rdev->config.cayman.tile_config |= (1 << 0);
 995                 break;
 996         case 4:
 997                 rdev->config.cayman.tile_config |= (2 << 0);
 998                 break;
 999         case 8:
1000                 rdev->config.cayman.tile_config |= (3 << 0);
1001                 break;
1002         }
1003
1004         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1005         if (rdev->flags & RADEON_IS_IGP)
1006                 rdev->config.cayman.tile_config |= 1 << 4;
1007         else {
1008                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
1009                 case 0: /* four banks */
1010                         rdev->config.cayman.tile_config |= 0 << 4;
1011                         break;
1012                 case 1: /* eight banks */
1013                         rdev->config.cayman.tile_config |= 1 << 4;
1014                         break;
1015                 case 2: /* sixteen banks */
1016                 default:
1017                         rdev->config.cayman.tile_config |= 2 << 4;
1018                         break;
1019                 }
1020         }
1021         rdev->config.cayman.tile_config |=
1022                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
1023         rdev->config.cayman.tile_config |=
1024                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
1025
1026         tmp = 0;
1027         for (i = (rdev->config.cayman.max_shader_engines - 1); i >= 0; i--) {
1028                 u32 rb_disable_bitmap;
1029
1030                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1031                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1032                 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
1033                 tmp <<= 4;
1034                 tmp |= rb_disable_bitmap;
1035         }
1036         /* enabled rb are just the one not disabled :) */
1037         disabled_rb_mask = tmp;
1038         tmp = 0;
1039         for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1040                 tmp |= (1 << i);
1041         /* if all the backends are disabled, fix it up here */
1042         if ((disabled_rb_mask & tmp) == tmp) {
1043                 for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1044                         disabled_rb_mask &= ~(1 << i);
1045         }
1046
1047         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1048         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1049
1050         WREG32(GB_ADDR_CONFIG, gb_addr_config);
1051         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
1052         if (ASIC_IS_DCE6(rdev))
1053                 WREG32(DMIF_ADDR_CALC, gb_addr_config);
1054         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
1055         WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
1056         WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1057         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
1058         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
1059         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
1060
1061         if ((rdev->config.cayman.max_backends_per_se == 1) &&
1062             (rdev->flags & RADEON_IS_IGP)) {
1063                 if ((disabled_rb_mask & 3) == 1) {
1064                         /* RB0 disabled, RB1 enabled */
1065                         tmp = 0x11111111;
1066                 } else {
1067                         /* RB1 disabled, RB0 enabled */
1068                         tmp = 0x00000000;
1069                 }
1070         } else {
1071                 tmp = gb_addr_config & NUM_PIPES_MASK;
1072                 tmp = r6xx_remap_render_backend(rdev, tmp,
1073                                                 rdev->config.cayman.max_backends_per_se *
1074                                                 rdev->config.cayman.max_shader_engines,
1075                                                 CAYMAN_MAX_BACKENDS, disabled_rb_mask);
1076         }
1077         WREG32(GB_BACKEND_MAP, tmp);
1078
1079         cgts_tcc_disable = 0xffff0000;
1080         for (i = 0; i < rdev->config.cayman.max_texture_channel_caches; i++)
1081                 cgts_tcc_disable &= ~(1 << (16 + i));
1082         WREG32(CGTS_TCC_DISABLE, cgts_tcc_disable);
1083         WREG32(CGTS_SYS_TCC_DISABLE, cgts_tcc_disable);
1084         WREG32(CGTS_USER_SYS_TCC_DISABLE, cgts_tcc_disable);
1085         WREG32(CGTS_USER_TCC_DISABLE, cgts_tcc_disable);
1086
1087         /* reprogram the shader complex */
1088         cgts_sm_ctrl_reg = RREG32(CGTS_SM_CTRL_REG);
1089         for (i = 0; i < 16; i++)
1090                 WREG32(CGTS_SM_CTRL_REG, OVERRIDE);
1091         WREG32(CGTS_SM_CTRL_REG, cgts_sm_ctrl_reg);
1092
1093         /* set HW defaults for 3D engine */
1094         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
1095
1096         sx_debug_1 = RREG32(SX_DEBUG_1);
1097         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1098         WREG32(SX_DEBUG_1, sx_debug_1);
1099
1100         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1101         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
1102         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.cayman.sx_num_of_sets);
1103         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1104
1105         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4) | CRC_SIMD_ID_WADDR_DISABLE);
1106
1107         /* need to be explicitly zero-ed */
1108         WREG32(VGT_OFFCHIP_LDS_BASE, 0);
1109         WREG32(SQ_LSTMP_RING_BASE, 0);
1110         WREG32(SQ_HSTMP_RING_BASE, 0);
1111         WREG32(SQ_ESTMP_RING_BASE, 0);
1112         WREG32(SQ_GSTMP_RING_BASE, 0);
1113         WREG32(SQ_VSTMP_RING_BASE, 0);
1114         WREG32(SQ_PSTMP_RING_BASE, 0);
1115
1116         WREG32(TA_CNTL_AUX, DISABLE_CUBE_ANISO);
1117
1118         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.cayman.sx_max_export_size / 4) - 1) |
1119                                         POSITION_BUFFER_SIZE((rdev->config.cayman.sx_max_export_pos_size / 4) - 1) |
1120                                         SMX_BUFFER_SIZE((rdev->config.cayman.sx_max_export_smx_size / 4) - 1)));
1121
1122         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.cayman.sc_prim_fifo_size) |
1123                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_hiz_tile_fifo_size) |
1124                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_earlyz_tile_fifo_size)));
1125
1126
1127         WREG32(VGT_NUM_INSTANCES, 1);
1128
1129         WREG32(CP_PERFMON_CNTL, 0);
1130
1131         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.cayman.sq_num_cf_insts) |
1132                                   FETCH_FIFO_HIWATER(0x4) |
1133                                   DONE_FIFO_HIWATER(0xe0) |
1134                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
1135
1136         WREG32(SQ_GPR_RESOURCE_MGMT_1, NUM_CLAUSE_TEMP_GPRS(4));
1137         WREG32(SQ_CONFIG, (VC_ENABLE |
1138                            EXPORT_SRC_C |
1139                            GFX_PRIO(0) |
1140                            CS1_PRIO(0) |
1141                            CS2_PRIO(1)));
1142         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, DYN_GPR_ENABLE);
1143
1144         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1145                                           FORCE_EOV_MAX_REZ_CNT(255)));
1146
1147         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
1148                AUTO_INVLD_EN(ES_AND_GS_AUTO));
1149
1150         WREG32(VGT_GS_VERTEX_REUSE, 16);
1151         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1152
1153         WREG32(CB_PERF_CTR0_SEL_0, 0);
1154         WREG32(CB_PERF_CTR0_SEL_1, 0);
1155         WREG32(CB_PERF_CTR1_SEL_0, 0);
1156         WREG32(CB_PERF_CTR1_SEL_1, 0);
1157         WREG32(CB_PERF_CTR2_SEL_0, 0);
1158         WREG32(CB_PERF_CTR2_SEL_1, 0);
1159         WREG32(CB_PERF_CTR3_SEL_0, 0);
1160         WREG32(CB_PERF_CTR3_SEL_1, 0);
1161
1162         tmp = RREG32(HDP_MISC_CNTL);
1163         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
1164         WREG32(HDP_MISC_CNTL, tmp);
1165
1166         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1167         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1168
1169         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
1170
1171         udelay(50);
1172
1173         /* set clockgating golden values on TN */
1174         if (rdev->family == CHIP_ARUBA) {
1175                 tmp = RREG32_CG(CG_CGTT_LOCAL_0);
1176                 tmp &= ~0x00380000;
1177                 WREG32_CG(CG_CGTT_LOCAL_0, tmp);
1178                 tmp = RREG32_CG(CG_CGTT_LOCAL_1);
1179                 tmp &= ~0x0e000000;
1180                 WREG32_CG(CG_CGTT_LOCAL_1, tmp);
1181         }
1182 }
1183
1184 /*
1185  * GART
1186  */
1187 void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev)
1188 {
1189         /* flush hdp cache */
1190         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
1191
1192         /* bits 0-7 are the VM contexts0-7 */
1193         WREG32(VM_INVALIDATE_REQUEST, 1);
1194 }
1195
1196 static int cayman_pcie_gart_enable(struct radeon_device *rdev)
1197 {
1198         int i, r;
1199
1200         if (rdev->gart.robj == NULL) {
1201                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
1202                 return -EINVAL;
1203         }
1204         r = radeon_gart_table_vram_pin(rdev);
1205         if (r)
1206                 return r;
1207         radeon_gart_restore(rdev);
1208         /* Setup TLB control */
1209         WREG32(MC_VM_MX_L1_TLB_CNTL,
1210                (0xA << 7) |
1211                ENABLE_L1_TLB |
1212                ENABLE_L1_FRAGMENT_PROCESSING |
1213                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1214                ENABLE_ADVANCED_DRIVER_MODEL |
1215                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1216         /* Setup L2 cache */
1217         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
1218                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1219                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1220                EFFECTIVE_L2_QUEUE_SIZE(7) |
1221                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1222         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
1223         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1224                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1225         /* setup context0 */
1226         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
1227         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
1228         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
1229         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
1230                         (u32)(rdev->dummy_page.addr >> 12));
1231         WREG32(VM_CONTEXT0_CNTL2, 0);
1232         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
1233                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
1234
1235         WREG32(0x15D4, 0);
1236         WREG32(0x15D8, 0);
1237         WREG32(0x15DC, 0);
1238
1239         /* empty context1-7 */
1240         /* Assign the pt base to something valid for now; the pts used for
1241          * the VMs are determined by the application and setup and assigned
1242          * on the fly in the vm part of radeon_gart.c
1243          */
1244         for (i = 1; i < 8; i++) {
1245                 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (i << 2), 0);
1246                 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (i << 2), rdev->vm_manager.max_pfn);
1247                 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
1248                         rdev->gart.table_addr >> 12);
1249         }
1250
1251         /* enable context1-7 */
1252         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
1253                (u32)(rdev->dummy_page.addr >> 12));
1254         WREG32(VM_CONTEXT1_CNTL2, 4);
1255         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
1256                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1257                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1258                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1259                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1260                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
1261                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
1262                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
1263                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
1264                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
1265                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
1266                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1267                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1268
1269         cayman_pcie_gart_tlb_flush(rdev);
1270         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1271                  (unsigned)(rdev->mc.gtt_size >> 20),
1272                  (unsigned long long)rdev->gart.table_addr);
1273         rdev->gart.ready = true;
1274         return 0;
1275 }
1276
1277 static void cayman_pcie_gart_disable(struct radeon_device *rdev)
1278 {
1279         /* Disable all tables */
1280         WREG32(VM_CONTEXT0_CNTL, 0);
1281         WREG32(VM_CONTEXT1_CNTL, 0);
1282         /* Setup TLB control */
1283         WREG32(MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING |
1284                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1285                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1286         /* Setup L2 cache */
1287         WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1288                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1289                EFFECTIVE_L2_QUEUE_SIZE(7) |
1290                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1291         WREG32(VM_L2_CNTL2, 0);
1292         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1293                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1294         radeon_gart_table_vram_unpin(rdev);
1295 }
1296
1297 static void cayman_pcie_gart_fini(struct radeon_device *rdev)
1298 {
1299         cayman_pcie_gart_disable(rdev);
1300         radeon_gart_table_vram_free(rdev);
1301         radeon_gart_fini(rdev);
1302 }
1303
1304 void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
1305                               int ring, u32 cp_int_cntl)
1306 {
1307         u32 srbm_gfx_cntl = RREG32(SRBM_GFX_CNTL) & ~3;
1308
1309         WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl | (ring & 3));
1310         WREG32(CP_INT_CNTL, cp_int_cntl);
1311 }
1312
1313 /*
1314  * CP.
1315  */
1316 void cayman_fence_ring_emit(struct radeon_device *rdev,
1317                             struct radeon_fence *fence)
1318 {
1319         struct radeon_ring *ring = &rdev->ring[fence->ring];
1320         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
1321
1322         /* flush read cache over gart for this vmid */
1323         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1324         radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
1325         radeon_ring_write(ring, 0);
1326         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1327         radeon_ring_write(ring, PACKET3_TC_ACTION_ENA | PACKET3_SH_ACTION_ENA);
1328         radeon_ring_write(ring, 0xFFFFFFFF);
1329         radeon_ring_write(ring, 0);
1330         radeon_ring_write(ring, 10); /* poll interval */
1331         /* EVENT_WRITE_EOP - flush caches, send int */
1332         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
1333         radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
1334         radeon_ring_write(ring, addr & 0xffffffff);
1335         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
1336         radeon_ring_write(ring, fence->seq);
1337         radeon_ring_write(ring, 0);
1338 }
1339
1340 void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1341 {
1342         struct radeon_ring *ring = &rdev->ring[ib->ring];
1343
1344         /* set to DX10/11 mode */
1345         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
1346         radeon_ring_write(ring, 1);
1347
1348         if (ring->rptr_save_reg) {
1349                 uint32_t next_rptr = ring->wptr + 3 + 4 + 8;
1350                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1351                 radeon_ring_write(ring, ((ring->rptr_save_reg -
1352                                           PACKET3_SET_CONFIG_REG_START) >> 2));
1353                 radeon_ring_write(ring, next_rptr);
1354         }
1355
1356         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1357         radeon_ring_write(ring,
1358 #ifdef __BIG_ENDIAN
1359                           (2 << 0) |
1360 #endif
1361                           (ib->gpu_addr & 0xFFFFFFFC));
1362         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
1363         radeon_ring_write(ring, ib->length_dw |
1364                           (ib->vm ? (ib->vm->id << 24) : 0));
1365
1366         /* flush read cache over gart for this vmid */
1367         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1368         radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
1369         radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
1370         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1371         radeon_ring_write(ring, PACKET3_TC_ACTION_ENA | PACKET3_SH_ACTION_ENA);
1372         radeon_ring_write(ring, 0xFFFFFFFF);
1373         radeon_ring_write(ring, 0);
1374         radeon_ring_write(ring, 10); /* poll interval */
1375 }
1376
1377 static void cayman_cp_enable(struct radeon_device *rdev, bool enable)
1378 {
1379         if (enable)
1380                 WREG32(CP_ME_CNTL, 0);
1381         else {
1382                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1383                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1384                 WREG32(SCRATCH_UMSK, 0);
1385                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1386         }
1387 }
1388
1389 static int cayman_cp_load_microcode(struct radeon_device *rdev)
1390 {
1391         const __be32 *fw_data;
1392         int i;
1393
1394         if (!rdev->me_fw || !rdev->pfp_fw)
1395                 return -EINVAL;
1396
1397         cayman_cp_enable(rdev, false);
1398
1399         fw_data = (const __be32 *)rdev->pfp_fw->data;
1400         WREG32(CP_PFP_UCODE_ADDR, 0);
1401         for (i = 0; i < CAYMAN_PFP_UCODE_SIZE; i++)
1402                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1403         WREG32(CP_PFP_UCODE_ADDR, 0);
1404
1405         fw_data = (const __be32 *)rdev->me_fw->data;
1406         WREG32(CP_ME_RAM_WADDR, 0);
1407         for (i = 0; i < CAYMAN_PM4_UCODE_SIZE; i++)
1408                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1409
1410         WREG32(CP_PFP_UCODE_ADDR, 0);
1411         WREG32(CP_ME_RAM_WADDR, 0);
1412         WREG32(CP_ME_RAM_RADDR, 0);
1413         return 0;
1414 }
1415
1416 static int cayman_cp_start(struct radeon_device *rdev)
1417 {
1418         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1419         int r, i;
1420
1421         r = radeon_ring_lock(rdev, ring, 7);
1422         if (r) {
1423                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1424                 return r;
1425         }
1426         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
1427         radeon_ring_write(ring, 0x1);
1428         radeon_ring_write(ring, 0x0);
1429         radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1);
1430         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1431         radeon_ring_write(ring, 0);
1432         radeon_ring_write(ring, 0);
1433         radeon_ring_unlock_commit(rdev, ring);
1434
1435         cayman_cp_enable(rdev, true);
1436
1437         r = radeon_ring_lock(rdev, ring, cayman_default_size + 19);
1438         if (r) {
1439                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1440                 return r;
1441         }
1442
1443         /* setup clear context state */
1444         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1445         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1446
1447         for (i = 0; i < cayman_default_size; i++)
1448                 radeon_ring_write(ring, cayman_default_state[i]);
1449
1450         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1451         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1452
1453         /* set clear context state */
1454         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
1455         radeon_ring_write(ring, 0);
1456
1457         /* SQ_VTX_BASE_VTX_LOC */
1458         radeon_ring_write(ring, 0xc0026f00);
1459         radeon_ring_write(ring, 0x00000000);
1460         radeon_ring_write(ring, 0x00000000);
1461         radeon_ring_write(ring, 0x00000000);
1462
1463         /* Clear consts */
1464         radeon_ring_write(ring, 0xc0036f00);
1465         radeon_ring_write(ring, 0x00000bc4);
1466         radeon_ring_write(ring, 0xffffffff);
1467         radeon_ring_write(ring, 0xffffffff);
1468         radeon_ring_write(ring, 0xffffffff);
1469
1470         radeon_ring_write(ring, 0xc0026900);
1471         radeon_ring_write(ring, 0x00000316);
1472         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1473         radeon_ring_write(ring, 0x00000010); /*  */
1474
1475         radeon_ring_unlock_commit(rdev, ring);
1476
1477         /* XXX init other rings */
1478
1479         return 0;
1480 }
1481
1482 static void cayman_cp_fini(struct radeon_device *rdev)
1483 {
1484         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1485         cayman_cp_enable(rdev, false);
1486         radeon_ring_fini(rdev, ring);
1487         radeon_scratch_free(rdev, ring->rptr_save_reg);
1488 }
1489
1490 static int cayman_cp_resume(struct radeon_device *rdev)
1491 {
1492         static const int ridx[] = {
1493                 RADEON_RING_TYPE_GFX_INDEX,
1494                 CAYMAN_RING_TYPE_CP1_INDEX,
1495                 CAYMAN_RING_TYPE_CP2_INDEX
1496         };
1497         static const unsigned cp_rb_cntl[] = {
1498                 CP_RB0_CNTL,
1499                 CP_RB1_CNTL,
1500                 CP_RB2_CNTL,
1501         };
1502         static const unsigned cp_rb_rptr_addr[] = {
1503                 CP_RB0_RPTR_ADDR,
1504                 CP_RB1_RPTR_ADDR,
1505                 CP_RB2_RPTR_ADDR
1506         };
1507         static const unsigned cp_rb_rptr_addr_hi[] = {
1508                 CP_RB0_RPTR_ADDR_HI,
1509                 CP_RB1_RPTR_ADDR_HI,
1510                 CP_RB2_RPTR_ADDR_HI
1511         };
1512         static const unsigned cp_rb_base[] = {
1513                 CP_RB0_BASE,
1514                 CP_RB1_BASE,
1515                 CP_RB2_BASE
1516         };
1517         struct radeon_ring *ring;
1518         int i, r;
1519
1520         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1521         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1522                                  SOFT_RESET_PA |
1523                                  SOFT_RESET_SH |
1524                                  SOFT_RESET_VGT |
1525                                  SOFT_RESET_SPI |
1526                                  SOFT_RESET_SX));
1527         RREG32(GRBM_SOFT_RESET);
1528         mdelay(15);
1529         WREG32(GRBM_SOFT_RESET, 0);
1530         RREG32(GRBM_SOFT_RESET);
1531
1532         WREG32(CP_SEM_WAIT_TIMER, 0x0);
1533         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
1534
1535         /* Set the write pointer delay */
1536         WREG32(CP_RB_WPTR_DELAY, 0);
1537
1538         WREG32(CP_DEBUG, (1 << 27));
1539
1540         /* set the wb address whether it's enabled or not */
1541         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
1542         WREG32(SCRATCH_UMSK, 0xff);
1543
1544         for (i = 0; i < 3; ++i) {
1545                 uint32_t rb_cntl;
1546                 uint64_t addr;
1547
1548                 /* Set ring buffer size */
1549                 ring = &rdev->ring[ridx[i]];
1550                 rb_cntl = order_base_2(ring->ring_size / 8);
1551                 rb_cntl |= order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8;
1552 #ifdef __BIG_ENDIAN
1553                 rb_cntl |= BUF_SWAP_32BIT;
1554 #endif
1555                 WREG32(cp_rb_cntl[i], rb_cntl);
1556
1557                 /* set the wb address whether it's enabled or not */
1558                 addr = rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET;
1559                 WREG32(cp_rb_rptr_addr[i], addr & 0xFFFFFFFC);
1560                 WREG32(cp_rb_rptr_addr_hi[i], upper_32_bits(addr) & 0xFF);
1561         }
1562
1563         /* set the rb base addr, this causes an internal reset of ALL rings */
1564         for (i = 0; i < 3; ++i) {
1565                 ring = &rdev->ring[ridx[i]];
1566                 WREG32(cp_rb_base[i], ring->gpu_addr >> 8);
1567         }
1568
1569         for (i = 0; i < 3; ++i) {
1570                 /* Initialize the ring buffer's read and write pointers */
1571                 ring = &rdev->ring[ridx[i]];
1572                 WREG32_P(cp_rb_cntl[i], RB_RPTR_WR_ENA, ~RB_RPTR_WR_ENA);
1573
1574                 ring->rptr = ring->wptr = 0;
1575                 WREG32(ring->rptr_reg, ring->rptr);
1576                 WREG32(ring->wptr_reg, ring->wptr);
1577
1578                 mdelay(1);
1579                 WREG32_P(cp_rb_cntl[i], 0, ~RB_RPTR_WR_ENA);
1580         }
1581
1582         /* start the rings */
1583         cayman_cp_start(rdev);
1584         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
1585         rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1586         rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1587         /* this only test cp0 */
1588         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
1589         if (r) {
1590                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1591                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1592                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1593                 return r;
1594         }
1595
1596         return 0;
1597 }
1598
1599 u32 cayman_gpu_check_soft_reset(struct radeon_device *rdev)
1600 {
1601         u32 reset_mask = 0;
1602         u32 tmp;
1603
1604         /* GRBM_STATUS */
1605         tmp = RREG32(GRBM_STATUS);
1606         if (tmp & (PA_BUSY | SC_BUSY |
1607                    SH_BUSY | SX_BUSY |
1608                    TA_BUSY | VGT_BUSY |
1609                    DB_BUSY | CB_BUSY |
1610                    GDS_BUSY | SPI_BUSY |
1611                    IA_BUSY | IA_BUSY_NO_DMA))
1612                 reset_mask |= RADEON_RESET_GFX;
1613
1614         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
1615                    CP_BUSY | CP_COHERENCY_BUSY))
1616                 reset_mask |= RADEON_RESET_CP;
1617
1618         if (tmp & GRBM_EE_BUSY)
1619                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
1620
1621         /* DMA_STATUS_REG 0 */
1622         tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
1623         if (!(tmp & DMA_IDLE))
1624                 reset_mask |= RADEON_RESET_DMA;
1625
1626         /* DMA_STATUS_REG 1 */
1627         tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
1628         if (!(tmp & DMA_IDLE))
1629                 reset_mask |= RADEON_RESET_DMA1;
1630
1631         /* SRBM_STATUS2 */
1632         tmp = RREG32(SRBM_STATUS2);
1633         if (tmp & DMA_BUSY)
1634                 reset_mask |= RADEON_RESET_DMA;
1635
1636         if (tmp & DMA1_BUSY)
1637                 reset_mask |= RADEON_RESET_DMA1;
1638
1639         /* SRBM_STATUS */
1640         tmp = RREG32(SRBM_STATUS);
1641         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
1642                 reset_mask |= RADEON_RESET_RLC;
1643
1644         if (tmp & IH_BUSY)
1645                 reset_mask |= RADEON_RESET_IH;
1646
1647         if (tmp & SEM_BUSY)
1648                 reset_mask |= RADEON_RESET_SEM;
1649
1650         if (tmp & GRBM_RQ_PENDING)
1651                 reset_mask |= RADEON_RESET_GRBM;
1652
1653         if (tmp & VMC_BUSY)
1654                 reset_mask |= RADEON_RESET_VMC;
1655
1656         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
1657                    MCC_BUSY | MCD_BUSY))
1658                 reset_mask |= RADEON_RESET_MC;
1659
1660         if (evergreen_is_display_hung(rdev))
1661                 reset_mask |= RADEON_RESET_DISPLAY;
1662
1663         /* VM_L2_STATUS */
1664         tmp = RREG32(VM_L2_STATUS);
1665         if (tmp & L2_BUSY)
1666                 reset_mask |= RADEON_RESET_VMC;
1667
1668         /* Skip MC reset as it's mostly likely not hung, just busy */
1669         if (reset_mask & RADEON_RESET_MC) {
1670                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1671                 reset_mask &= ~RADEON_RESET_MC;
1672         }
1673
1674         return reset_mask;
1675 }
1676
1677 static void cayman_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1678 {
1679         struct evergreen_mc_save save;
1680         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1681         u32 tmp;
1682
1683         if (reset_mask == 0)
1684                 return;
1685
1686         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1687
1688         evergreen_print_gpu_status_regs(rdev);
1689         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_ADDR   0x%08X\n",
1690                  RREG32(0x14F8));
1691         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n",
1692                  RREG32(0x14D8));
1693         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
1694                  RREG32(0x14FC));
1695         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1696                  RREG32(0x14DC));
1697
1698         /* Disable CP parsing/prefetching */
1699         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
1700
1701         if (reset_mask & RADEON_RESET_DMA) {
1702                 /* dma0 */
1703                 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
1704                 tmp &= ~DMA_RB_ENABLE;
1705                 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
1706         }
1707
1708         if (reset_mask & RADEON_RESET_DMA1) {
1709                 /* dma1 */
1710                 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
1711                 tmp &= ~DMA_RB_ENABLE;
1712                 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
1713         }
1714
1715         udelay(50);
1716
1717         evergreen_mc_stop(rdev, &save);
1718         if (evergreen_mc_wait_for_idle(rdev)) {
1719                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1720         }
1721
1722         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
1723                 grbm_soft_reset = SOFT_RESET_CB |
1724                         SOFT_RESET_DB |
1725                         SOFT_RESET_GDS |
1726                         SOFT_RESET_PA |
1727                         SOFT_RESET_SC |
1728                         SOFT_RESET_SPI |
1729                         SOFT_RESET_SH |
1730                         SOFT_RESET_SX |
1731                         SOFT_RESET_TC |
1732                         SOFT_RESET_TA |
1733                         SOFT_RESET_VGT |
1734                         SOFT_RESET_IA;
1735         }
1736
1737         if (reset_mask & RADEON_RESET_CP) {
1738                 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
1739
1740                 srbm_soft_reset |= SOFT_RESET_GRBM;
1741         }
1742
1743         if (reset_mask & RADEON_RESET_DMA)
1744                 srbm_soft_reset |= SOFT_RESET_DMA;
1745
1746         if (reset_mask & RADEON_RESET_DMA1)
1747                 srbm_soft_reset |= SOFT_RESET_DMA1;
1748
1749         if (reset_mask & RADEON_RESET_DISPLAY)
1750                 srbm_soft_reset |= SOFT_RESET_DC;
1751
1752         if (reset_mask & RADEON_RESET_RLC)
1753                 srbm_soft_reset |= SOFT_RESET_RLC;
1754
1755         if (reset_mask & RADEON_RESET_SEM)
1756                 srbm_soft_reset |= SOFT_RESET_SEM;
1757
1758         if (reset_mask & RADEON_RESET_IH)
1759                 srbm_soft_reset |= SOFT_RESET_IH;
1760
1761         if (reset_mask & RADEON_RESET_GRBM)
1762                 srbm_soft_reset |= SOFT_RESET_GRBM;
1763
1764         if (reset_mask & RADEON_RESET_VMC)
1765                 srbm_soft_reset |= SOFT_RESET_VMC;
1766
1767         if (!(rdev->flags & RADEON_IS_IGP)) {
1768                 if (reset_mask & RADEON_RESET_MC)
1769                         srbm_soft_reset |= SOFT_RESET_MC;
1770         }
1771
1772         if (grbm_soft_reset) {
1773                 tmp = RREG32(GRBM_SOFT_RESET);
1774                 tmp |= grbm_soft_reset;
1775                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
1776                 WREG32(GRBM_SOFT_RESET, tmp);
1777                 tmp = RREG32(GRBM_SOFT_RESET);
1778
1779                 udelay(50);
1780
1781                 tmp &= ~grbm_soft_reset;
1782                 WREG32(GRBM_SOFT_RESET, tmp);
1783                 tmp = RREG32(GRBM_SOFT_RESET);
1784         }
1785
1786         if (srbm_soft_reset) {
1787                 tmp = RREG32(SRBM_SOFT_RESET);
1788                 tmp |= srbm_soft_reset;
1789                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1790                 WREG32(SRBM_SOFT_RESET, tmp);
1791                 tmp = RREG32(SRBM_SOFT_RESET);
1792
1793                 udelay(50);
1794
1795                 tmp &= ~srbm_soft_reset;
1796                 WREG32(SRBM_SOFT_RESET, tmp);
1797                 tmp = RREG32(SRBM_SOFT_RESET);
1798         }
1799
1800         /* Wait a little for things to settle down */
1801         udelay(50);
1802
1803         evergreen_mc_resume(rdev, &save);
1804         udelay(50);
1805
1806         evergreen_print_gpu_status_regs(rdev);
1807 }
1808
1809 int cayman_asic_reset(struct radeon_device *rdev)
1810 {
1811         u32 reset_mask;
1812
1813         reset_mask = cayman_gpu_check_soft_reset(rdev);
1814
1815         if (reset_mask)
1816                 r600_set_bios_scratch_engine_hung(rdev, true);
1817
1818         cayman_gpu_soft_reset(rdev, reset_mask);
1819
1820         reset_mask = cayman_gpu_check_soft_reset(rdev);
1821
1822         if (!reset_mask)
1823                 r600_set_bios_scratch_engine_hung(rdev, false);
1824
1825         return 0;
1826 }
1827
1828 /**
1829  * cayman_gfx_is_lockup - Check if the GFX engine is locked up
1830  *
1831  * @rdev: radeon_device pointer
1832  * @ring: radeon_ring structure holding ring information
1833  *
1834  * Check if the GFX engine is locked up.
1835  * Returns true if the engine appears to be locked up, false if not.
1836  */
1837 bool cayman_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1838 {
1839         u32 reset_mask = cayman_gpu_check_soft_reset(rdev);
1840
1841         if (!(reset_mask & (RADEON_RESET_GFX |
1842                             RADEON_RESET_COMPUTE |
1843                             RADEON_RESET_CP))) {
1844                 radeon_ring_lockup_update(ring);
1845                 return false;
1846         }
1847         /* force CP activities */
1848         radeon_ring_force_activity(rdev, ring);
1849         return radeon_ring_test_lockup(rdev, ring);
1850 }
1851
1852 static int cayman_startup(struct radeon_device *rdev)
1853 {
1854         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1855         int r;
1856
1857         /* enable pcie gen2 link */
1858         evergreen_pcie_gen2_enable(rdev);
1859         /* enable aspm */
1860         evergreen_program_aspm(rdev);
1861
1862         /* scratch needs to be initialized before MC */
1863         r = r600_vram_scratch_init(rdev);
1864         if (r)
1865                 return r;
1866
1867         evergreen_mc_program(rdev);
1868
1869         if (rdev->flags & RADEON_IS_IGP) {
1870                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
1871                         r = ni_init_microcode(rdev);
1872                         if (r) {
1873                                 DRM_ERROR("Failed to load firmware!\n");
1874                                 return r;
1875                         }
1876                 }
1877         } else {
1878                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
1879                         r = ni_init_microcode(rdev);
1880                         if (r) {
1881                                 DRM_ERROR("Failed to load firmware!\n");
1882                                 return r;
1883                         }
1884                 }
1885
1886                 r = ni_mc_load_microcode(rdev);
1887                 if (r) {
1888                         DRM_ERROR("Failed to load MC firmware!\n");
1889                         return r;
1890                 }
1891         }
1892
1893         r = cayman_pcie_gart_enable(rdev);
1894         if (r)
1895                 return r;
1896         cayman_gpu_init(rdev);
1897
1898         /* allocate rlc buffers */
1899         if (rdev->flags & RADEON_IS_IGP) {
1900                 rdev->rlc.reg_list = tn_rlc_save_restore_register_list;
1901                 rdev->rlc.reg_list_size =
1902                         (u32)ARRAY_SIZE(tn_rlc_save_restore_register_list);
1903                 rdev->rlc.cs_data = cayman_cs_data;
1904                 r = sumo_rlc_init(rdev);
1905                 if (r) {
1906                         DRM_ERROR("Failed to init rlc BOs!\n");
1907                         return r;
1908                 }
1909         }
1910
1911         /* allocate wb buffer */
1912         r = radeon_wb_init(rdev);
1913         if (r)
1914                 return r;
1915
1916         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
1917         if (r) {
1918                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
1919                 return r;
1920         }
1921
1922         r = uvd_v2_2_resume(rdev);
1923         if (!r) {
1924                 r = radeon_fence_driver_start_ring(rdev,
1925                                                    R600_RING_TYPE_UVD_INDEX);
1926                 if (r)
1927                         dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
1928         }
1929         if (r)
1930                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
1931
1932         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
1933         if (r) {
1934                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
1935                 return r;
1936         }
1937
1938         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
1939         if (r) {
1940                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
1941                 return r;
1942         }
1943
1944         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
1945         if (r) {
1946                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
1947                 return r;
1948         }
1949
1950         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
1951         if (r) {
1952                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
1953                 return r;
1954         }
1955
1956         /* Enable IRQ */
1957         if (!rdev->irq.installed) {
1958                 r = radeon_irq_kms_init(rdev);
1959                 if (r)
1960                         return r;
1961         }
1962
1963         r = r600_irq_init(rdev);
1964         if (r) {
1965                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
1966                 radeon_irq_kms_fini(rdev);
1967                 return r;
1968         }
1969         evergreen_irq_set(rdev);
1970
1971         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
1972                              CP_RB0_RPTR, CP_RB0_WPTR,
1973                              RADEON_CP_PACKET2);
1974         if (r)
1975                 return r;
1976
1977         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
1978         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
1979                              DMA_RB_RPTR + DMA0_REGISTER_OFFSET,
1980                              DMA_RB_WPTR + DMA0_REGISTER_OFFSET,
1981                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
1982         if (r)
1983                 return r;
1984
1985         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
1986         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
1987                              DMA_RB_RPTR + DMA1_REGISTER_OFFSET,
1988                              DMA_RB_WPTR + DMA1_REGISTER_OFFSET,
1989                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
1990         if (r)
1991                 return r;
1992
1993         r = cayman_cp_load_microcode(rdev);
1994         if (r)
1995                 return r;
1996         r = cayman_cp_resume(rdev);
1997         if (r)
1998                 return r;
1999
2000         r = cayman_dma_resume(rdev);
2001         if (r)
2002                 return r;
2003
2004         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2005         if (ring->ring_size) {
2006                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
2007                                      UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
2008                                      RADEON_CP_PACKET2);
2009                 if (!r)
2010                         r = uvd_v1_0_init(rdev);
2011                 if (r)
2012                         DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
2013         }
2014
2015         r = radeon_ib_pool_init(rdev);
2016         if (r) {
2017                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2018                 return r;
2019         }
2020
2021         r = radeon_vm_manager_init(rdev);
2022         if (r) {
2023                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
2024                 return r;
2025         }
2026
2027         if (ASIC_IS_DCE6(rdev)) {
2028                 r = dce6_audio_init(rdev);
2029                 if (r)
2030                         return r;
2031         } else {
2032                 r = r600_audio_init(rdev);
2033                 if (r)
2034                         return r;
2035         }
2036
2037         return 0;
2038 }
2039
2040 int cayman_resume(struct radeon_device *rdev)
2041 {
2042         int r;
2043
2044         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
2045          * posting will perform necessary task to bring back GPU into good
2046          * shape.
2047          */
2048         /* post card */
2049         atom_asic_init(rdev->mode_info.atom_context);
2050
2051         /* init golden registers */
2052         ni_init_golden_registers(rdev);
2053
2054         rdev->accel_working = true;
2055         r = cayman_startup(rdev);
2056         if (r) {
2057                 DRM_ERROR("cayman startup failed on resume\n");
2058                 rdev->accel_working = false;
2059                 return r;
2060         }
2061         return r;
2062 }
2063
2064 int cayman_suspend(struct radeon_device *rdev)
2065 {
2066         if (ASIC_IS_DCE6(rdev))
2067                 dce6_audio_fini(rdev);
2068         else
2069                 r600_audio_fini(rdev);
2070         radeon_vm_manager_fini(rdev);
2071         cayman_cp_enable(rdev, false);
2072         cayman_dma_stop(rdev);
2073         uvd_v1_0_fini(rdev);
2074         radeon_uvd_suspend(rdev);
2075         evergreen_irq_suspend(rdev);
2076         radeon_wb_disable(rdev);
2077         cayman_pcie_gart_disable(rdev);
2078         return 0;
2079 }
2080
2081 /* Plan is to move initialization in that function and use
2082  * helper function so that radeon_device_init pretty much
2083  * do nothing more than calling asic specific function. This
2084  * should also allow to remove a bunch of callback function
2085  * like vram_info.
2086  */
2087 int cayman_init(struct radeon_device *rdev)
2088 {
2089         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2090         int r;
2091
2092         /* Read BIOS */
2093         if (!radeon_get_bios(rdev)) {
2094                 if (ASIC_IS_AVIVO(rdev))
2095                         return -EINVAL;
2096         }
2097         /* Must be an ATOMBIOS */
2098         if (!rdev->is_atom_bios) {
2099                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
2100                 return -EINVAL;
2101         }
2102         r = radeon_atombios_init(rdev);
2103         if (r)
2104                 return r;
2105
2106         /* Post card if necessary */
2107         if (!radeon_card_posted(rdev)) {
2108                 if (!rdev->bios) {
2109                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
2110                         return -EINVAL;
2111                 }
2112                 DRM_INFO("GPU not posted. posting now...\n");
2113                 atom_asic_init(rdev->mode_info.atom_context);
2114         }
2115         /* init golden registers */
2116         ni_init_golden_registers(rdev);
2117         /* Initialize scratch registers */
2118         r600_scratch_init(rdev);
2119         /* Initialize surface registers */
2120         radeon_surface_init(rdev);
2121         /* Initialize clocks */
2122         radeon_get_clock_info(rdev->ddev);
2123         /* Fence driver */
2124         r = radeon_fence_driver_init(rdev);
2125         if (r)
2126                 return r;
2127         /* initialize memory controller */
2128         r = evergreen_mc_init(rdev);
2129         if (r)
2130                 return r;
2131         /* Memory manager */
2132         r = radeon_bo_init(rdev);
2133         if (r)
2134                 return r;
2135
2136         ring->ring_obj = NULL;
2137         r600_ring_init(rdev, ring, 1024 * 1024);
2138
2139         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2140         ring->ring_obj = NULL;
2141         r600_ring_init(rdev, ring, 64 * 1024);
2142
2143         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2144         ring->ring_obj = NULL;
2145         r600_ring_init(rdev, ring, 64 * 1024);
2146
2147         r = radeon_uvd_init(rdev);
2148         if (!r) {
2149                 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2150                 ring->ring_obj = NULL;
2151                 r600_ring_init(rdev, ring, 4096);
2152         }
2153
2154         rdev->ih.ring_obj = NULL;
2155         r600_ih_ring_init(rdev, 64 * 1024);
2156
2157         r = r600_pcie_gart_init(rdev);
2158         if (r)
2159                 return r;
2160
2161         rdev->accel_working = true;
2162         r = cayman_startup(rdev);
2163         if (r) {
2164                 dev_err(rdev->dev, "disabling GPU acceleration\n");
2165                 cayman_cp_fini(rdev);
2166                 cayman_dma_fini(rdev);
2167                 r600_irq_fini(rdev);
2168                 if (rdev->flags & RADEON_IS_IGP)
2169                         sumo_rlc_fini(rdev);
2170                 radeon_wb_fini(rdev);
2171                 radeon_ib_pool_fini(rdev);
2172                 radeon_vm_manager_fini(rdev);
2173                 radeon_irq_kms_fini(rdev);
2174                 cayman_pcie_gart_fini(rdev);
2175                 rdev->accel_working = false;
2176         }
2177
2178         /* Don't start up if the MC ucode is missing.
2179          * The default clocks and voltages before the MC ucode
2180          * is loaded are not suffient for advanced operations.
2181          *
2182          * We can skip this check for TN, because there is no MC
2183          * ucode.
2184          */
2185         if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
2186                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
2187                 return -EINVAL;
2188         }
2189
2190         return 0;
2191 }
2192
2193 void cayman_fini(struct radeon_device *rdev)
2194 {
2195         cayman_cp_fini(rdev);
2196         cayman_dma_fini(rdev);
2197         r600_irq_fini(rdev);
2198         if (rdev->flags & RADEON_IS_IGP)
2199                 sumo_rlc_fini(rdev);
2200         radeon_wb_fini(rdev);
2201         radeon_vm_manager_fini(rdev);
2202         radeon_ib_pool_fini(rdev);
2203         radeon_irq_kms_fini(rdev);
2204         uvd_v1_0_fini(rdev);
2205         radeon_uvd_fini(rdev);
2206         cayman_pcie_gart_fini(rdev);
2207         r600_vram_scratch_fini(rdev);
2208         radeon_gem_fini(rdev);
2209         radeon_fence_driver_fini(rdev);
2210         radeon_bo_fini(rdev);
2211         radeon_atombios_fini(rdev);
2212         kfree(rdev->bios);
2213         rdev->bios = NULL;
2214 }
2215
2216 /*
2217  * vm
2218  */
2219 int cayman_vm_init(struct radeon_device *rdev)
2220 {
2221         /* number of VMs */
2222         rdev->vm_manager.nvm = 8;
2223         /* base offset of vram pages */
2224         if (rdev->flags & RADEON_IS_IGP) {
2225                 u64 tmp = RREG32(FUS_MC_VM_FB_OFFSET);
2226                 tmp <<= 22;
2227                 rdev->vm_manager.vram_base_offset = tmp;
2228         } else
2229                 rdev->vm_manager.vram_base_offset = 0;
2230         return 0;
2231 }
2232
2233 void cayman_vm_fini(struct radeon_device *rdev)
2234 {
2235 }
2236
2237 /**
2238  * cayman_vm_decode_fault - print human readable fault info
2239  *
2240  * @rdev: radeon_device pointer
2241  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
2242  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
2243  *
2244  * Print human readable fault information (cayman/TN).
2245  */
2246 void cayman_vm_decode_fault(struct radeon_device *rdev,
2247                             u32 status, u32 addr)
2248 {
2249         u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
2250         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
2251         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
2252         char *block;
2253
2254         switch (mc_id) {
2255         case 32:
2256         case 16:
2257         case 96:
2258         case 80:
2259         case 160:
2260         case 144:
2261         case 224:
2262         case 208:
2263                 block = "CB";
2264                 break;
2265         case 33:
2266         case 17:
2267         case 97:
2268         case 81:
2269         case 161:
2270         case 145:
2271         case 225:
2272         case 209:
2273                 block = "CB_FMASK";
2274                 break;
2275         case 34:
2276         case 18:
2277         case 98:
2278         case 82:
2279         case 162:
2280         case 146:
2281         case 226:
2282         case 210:
2283                 block = "CB_CMASK";
2284                 break;
2285         case 35:
2286         case 19:
2287         case 99:
2288         case 83:
2289         case 163:
2290         case 147:
2291         case 227:
2292         case 211:
2293                 block = "CB_IMMED";
2294                 break;
2295         case 36:
2296         case 20:
2297         case 100:
2298         case 84:
2299         case 164:
2300         case 148:
2301         case 228:
2302         case 212:
2303                 block = "DB";
2304                 break;
2305         case 37:
2306         case 21:
2307         case 101:
2308         case 85:
2309         case 165:
2310         case 149:
2311         case 229:
2312         case 213:
2313                 block = "DB_HTILE";
2314                 break;
2315         case 38:
2316         case 22:
2317         case 102:
2318         case 86:
2319         case 166:
2320         case 150:
2321         case 230:
2322         case 214:
2323                 block = "SX";
2324                 break;
2325         case 39:
2326         case 23:
2327         case 103:
2328         case 87:
2329         case 167:
2330         case 151:
2331         case 231:
2332         case 215:
2333                 block = "DB_STEN";
2334                 break;
2335         case 40:
2336         case 24:
2337         case 104:
2338         case 88:
2339         case 232:
2340         case 216:
2341         case 168:
2342         case 152:
2343                 block = "TC_TFETCH";
2344                 break;
2345         case 41:
2346         case 25:
2347         case 105:
2348         case 89:
2349         case 233:
2350         case 217:
2351         case 169:
2352         case 153:
2353                 block = "TC_VFETCH";
2354                 break;
2355         case 42:
2356         case 26:
2357         case 106:
2358         case 90:
2359         case 234:
2360         case 218:
2361         case 170:
2362         case 154:
2363                 block = "VC";
2364                 break;
2365         case 112:
2366                 block = "CP";
2367                 break;
2368         case 113:
2369         case 114:
2370                 block = "SH";
2371                 break;
2372         case 115:
2373                 block = "VGT";
2374                 break;
2375         case 178:
2376                 block = "IH";
2377                 break;
2378         case 51:
2379                 block = "RLC";
2380                 break;
2381         case 55:
2382                 block = "DMA";
2383                 break;
2384         case 56:
2385                 block = "HDP";
2386                 break;
2387         default:
2388                 block = "unknown";
2389                 break;
2390         }
2391
2392         printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
2393                protections, vmid, addr,
2394                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
2395                block, mc_id);
2396 }
2397
2398 /**
2399  * cayman_vm_flush - vm flush using the CP
2400  *
2401  * @rdev: radeon_device pointer
2402  *
2403  * Update the page table base and flush the VM TLB
2404  * using the CP (cayman-si).
2405  */
2406 void cayman_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
2407 {
2408         struct radeon_ring *ring = &rdev->ring[ridx];
2409
2410         if (vm == NULL)
2411                 return;
2412
2413         radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2), 0));
2414         radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
2415
2416         /* flush hdp cache */
2417         radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
2418         radeon_ring_write(ring, 0x1);
2419
2420         /* bits 0-7 are the VM contexts0-7 */
2421         radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0));
2422         radeon_ring_write(ring, 1 << vm->id);
2423
2424         /* sync PFP to ME, otherwise we might get invalid PFP reads */
2425         radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
2426         radeon_ring_write(ring, 0x0);
2427 }