Pileus Git - ~andy/linux/blob - drivers/gpu/drm/radeon/ni_dpm.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include "drmP.h"
  25 #include "radeon.h"
  26 #include "nid.h"
  27 #include "r600_dpm.h"
  28 #include "ni_dpm.h"
  29 #include "atom.h"
  30 #include <linux/math64.h>
  31 #include <linux/seq_file.h>
  32
  33 #define MC_CG_ARB_FREQ_F0           0x0a
  34 #define MC_CG_ARB_FREQ_F1           0x0b
  35 #define MC_CG_ARB_FREQ_F2           0x0c
  36 #define MC_CG_ARB_FREQ_F3           0x0d
  37
  38 #define SMC_RAM_END 0xC000
  39
  40 static const struct ni_cac_weights cac_weights_cayman_xt =
  41 {
  42         0x15,
  43         0x2,
  44         0x19,
  45         0x2,
  46         0x8,
  47         0x14,
  48         0x2,
  49         0x16,
  50         0xE,
  51         0x17,
  52         0x13,
  53         0x2B,
  54         0x10,
  55         0x7,
  56         0x5,
  57         0x5,
  58         0x5,
  59         0x2,
  60         0x3,
  61         0x9,
  62         0x10,
  63         0x10,
  64         0x2B,
  65         0xA,
  66         0x9,
  67         0x4,
  68         0xD,
  69         0xD,
  70         0x3E,
  71         0x18,
  72         0x14,
  73         0,
  74         0x3,
  75         0x3,
  76         0x5,
  77         0,
  78         0x2,
  79         0,
  80         0,
  81         0,
  82         0,
  83         0,
  84         0,
  85         0,
  86         0,
  87         0,
  88         0x1CC,
  89         0,
  90         0x164,
  91         1,
  92         1,
  93         1,
  94         1,
  95         12,
  96         12,
  97         12,
  98         0x12,
  99         0x1F,
 100         132,
 101         5,
 102         7,
 103         0,
 104         { 0, 0, 0, 0, 0, 0, 0, 0 },
 105         { 0, 0, 0, 0 },
 106         true
 107 };
 108
 109 static const struct ni_cac_weights cac_weights_cayman_pro =
 110 {
 111         0x16,
 112         0x4,
 113         0x10,
 114         0x2,
 115         0xA,
 116         0x16,
 117         0x2,
 118         0x18,
 119         0x10,
 120         0x1A,
 121         0x16,
 122         0x2D,
 123         0x12,
 124         0xA,
 125         0x6,
 126         0x6,
 127         0x6,
 128         0x2,
 129         0x4,
 130         0xB,
 131         0x11,
 132         0x11,
 133         0x2D,
 134         0xC,
 135         0xC,
 136         0x7,
 137         0x10,
 138         0x10,
 139         0x3F,
 140         0x1A,
 141         0x16,
 142         0,
 143         0x7,
 144         0x4,
 145         0x6,
 146         1,
 147         0x2,
 148         0x1,
 149         0,
 150         0,
 151         0,
 152         0,
 153         0,
 154         0,
 155         0x30,
 156         0,
 157         0x1CF,
 158         0,
 159         0x166,
 160         1,
 161         1,
 162         1,
 163         1,
 164         12,
 165         12,
 166         12,
 167         0x15,
 168         0x1F,
 169         132,
 170         6,
 171         6,
 172         0,
 173         { 0, 0, 0, 0, 0, 0, 0, 0 },
 174         { 0, 0, 0, 0 },
 175         true
 176 };
 177
 178 static const struct ni_cac_weights cac_weights_cayman_le =
 179 {
 180         0x7,
 181         0xE,
 182         0x1,
 183         0xA,
 184         0x1,
 185         0x3F,
 186         0x2,
 187         0x18,
 188         0x10,
 189         0x1A,
 190         0x1,
 191         0x3F,
 192         0x1,
 193         0xE,
 194         0x6,
 195         0x6,
 196         0x6,
 197         0x2,
 198         0x4,
 199         0x9,
 200         0x1A,
 201         0x1A,
 202         0x2C,
 203         0xA,
 204         0x11,
 205         0x8,
 206         0x19,
 207         0x19,
 208         0x1,
 209         0x1,
 210         0x1A,
 211         0,
 212         0x8,
 213         0x5,
 214         0x8,
 215         0x1,
 216         0x3,
 217         0x1,
 218         0,
 219         0,
 220         0,
 221         0,
 222         0,
 223         0,
 224         0x38,
 225         0x38,
 226         0x239,
 227         0x3,
 228         0x18A,
 229         1,
 230         1,
 231         1,
 232         1,
 233         12,
 234         12,
 235         12,
 236         0x15,
 237         0x22,
 238         132,
 239         6,
 240         6,
 241         0,
 242         { 0, 0, 0, 0, 0, 0, 0, 0 },
 243         { 0, 0, 0, 0 },
 244         true
 245 };
 246
 247 #define NISLANDS_MGCG_SEQUENCE  300
 248
 249 static const u32 cayman_cgcg_cgls_default[] =
 250 {
 251         0x000008f8, 0x00000010, 0xffffffff,
 252         0x000008fc, 0x00000000, 0xffffffff,
 253         0x000008f8, 0x00000011, 0xffffffff,
 254         0x000008fc, 0x00000000, 0xffffffff,
 255         0x000008f8, 0x00000012, 0xffffffff,
 256         0x000008fc, 0x00000000, 0xffffffff,
 257         0x000008f8, 0x00000013, 0xffffffff,
 258         0x000008fc, 0x00000000, 0xffffffff,
 259         0x000008f8, 0x00000014, 0xffffffff,
 260         0x000008fc, 0x00000000, 0xffffffff,
 261         0x000008f8, 0x00000015, 0xffffffff,
 262         0x000008fc, 0x00000000, 0xffffffff,
 263         0x000008f8, 0x00000016, 0xffffffff,
 264         0x000008fc, 0x00000000, 0xffffffff,
 265         0x000008f8, 0x00000017, 0xffffffff,
 266         0x000008fc, 0x00000000, 0xffffffff,
 267         0x000008f8, 0x00000018, 0xffffffff,
 268         0x000008fc, 0x00000000, 0xffffffff,
 269         0x000008f8, 0x00000019, 0xffffffff,
 270         0x000008fc, 0x00000000, 0xffffffff,
 271         0x000008f8, 0x0000001a, 0xffffffff,
 272         0x000008fc, 0x00000000, 0xffffffff,
 273         0x000008f8, 0x0000001b, 0xffffffff,
 274         0x000008fc, 0x00000000, 0xffffffff,
 275         0x000008f8, 0x00000020, 0xffffffff,
 276         0x000008fc, 0x00000000, 0xffffffff,
 277         0x000008f8, 0x00000021, 0xffffffff,
 278         0x000008fc, 0x00000000, 0xffffffff,
 279         0x000008f8, 0x00000022, 0xffffffff,
 280         0x000008fc, 0x00000000, 0xffffffff,
 281         0x000008f8, 0x00000023, 0xffffffff,
 282         0x000008fc, 0x00000000, 0xffffffff,
 283         0x000008f8, 0x00000024, 0xffffffff,
 284         0x000008fc, 0x00000000, 0xffffffff,
 285         0x000008f8, 0x00000025, 0xffffffff,
 286         0x000008fc, 0x00000000, 0xffffffff,
 287         0x000008f8, 0x00000026, 0xffffffff,
 288         0x000008fc, 0x00000000, 0xffffffff,
 289         0x000008f8, 0x00000027, 0xffffffff,
 290         0x000008fc, 0x00000000, 0xffffffff,
 291         0x000008f8, 0x00000028, 0xffffffff,
 292         0x000008fc, 0x00000000, 0xffffffff,
 293         0x000008f8, 0x00000029, 0xffffffff,
 294         0x000008fc, 0x00000000, 0xffffffff,
 295         0x000008f8, 0x0000002a, 0xffffffff,
 296         0x000008fc, 0x00000000, 0xffffffff,
 297         0x000008f8, 0x0000002b, 0xffffffff,
 298         0x000008fc, 0x00000000, 0xffffffff
 299 };
 300 #define CAYMAN_CGCG_CGLS_DEFAULT_LENGTH sizeof(cayman_cgcg_cgls_default) / (3 * sizeof(u32))
 301
 302 static const u32 cayman_cgcg_cgls_disable[] =
 303 {
 304         0x000008f8, 0x00000010, 0xffffffff,
 305         0x000008fc, 0xffffffff, 0xffffffff,
 306         0x000008f8, 0x00000011, 0xffffffff,
 307         0x000008fc, 0xffffffff, 0xffffffff,
 308         0x000008f8, 0x00000012, 0xffffffff,
 309         0x000008fc, 0xffffffff, 0xffffffff,
 310         0x000008f8, 0x00000013, 0xffffffff,
 311         0x000008fc, 0xffffffff, 0xffffffff,
 312         0x000008f8, 0x00000014, 0xffffffff,
 313         0x000008fc, 0xffffffff, 0xffffffff,
 314         0x000008f8, 0x00000015, 0xffffffff,
 315         0x000008fc, 0xffffffff, 0xffffffff,
 316         0x000008f8, 0x00000016, 0xffffffff,
 317         0x000008fc, 0xffffffff, 0xffffffff,
 318         0x000008f8, 0x00000017, 0xffffffff,
 319         0x000008fc, 0xffffffff, 0xffffffff,
 320         0x000008f8, 0x00000018, 0xffffffff,
 321         0x000008fc, 0xffffffff, 0xffffffff,
 322         0x000008f8, 0x00000019, 0xffffffff,
 323         0x000008fc, 0xffffffff, 0xffffffff,
 324         0x000008f8, 0x0000001a, 0xffffffff,
 325         0x000008fc, 0xffffffff, 0xffffffff,
 326         0x000008f8, 0x0000001b, 0xffffffff,
 327         0x000008fc, 0xffffffff, 0xffffffff,
 328         0x000008f8, 0x00000020, 0xffffffff,
 329         0x000008fc, 0x00000000, 0xffffffff,
 330         0x000008f8, 0x00000021, 0xffffffff,
 331         0x000008fc, 0x00000000, 0xffffffff,
 332         0x000008f8, 0x00000022, 0xffffffff,
 333         0x000008fc, 0x00000000, 0xffffffff,
 334         0x000008f8, 0x00000023, 0xffffffff,
 335         0x000008fc, 0x00000000, 0xffffffff,
 336         0x000008f8, 0x00000024, 0xffffffff,
 337         0x000008fc, 0x00000000, 0xffffffff,
 338         0x000008f8, 0x00000025, 0xffffffff,
 339         0x000008fc, 0x00000000, 0xffffffff,
 340         0x000008f8, 0x00000026, 0xffffffff,
 341         0x000008fc, 0x00000000, 0xffffffff,
 342         0x000008f8, 0x00000027, 0xffffffff,
 343         0x000008fc, 0x00000000, 0xffffffff,
 344         0x000008f8, 0x00000028, 0xffffffff,
 345         0x000008fc, 0x00000000, 0xffffffff,
 346         0x000008f8, 0x00000029, 0xffffffff,
 347         0x000008fc, 0x00000000, 0xffffffff,
 348         0x000008f8, 0x0000002a, 0xffffffff,
 349         0x000008fc, 0x00000000, 0xffffffff,
 350         0x000008f8, 0x0000002b, 0xffffffff,
 351         0x000008fc, 0x00000000, 0xffffffff,
 352         0x00000644, 0x000f7902, 0x001f4180,
 353         0x00000644, 0x000f3802, 0x001f4180
 354 };
 355 #define CAYMAN_CGCG_CGLS_DISABLE_LENGTH sizeof(cayman_cgcg_cgls_disable) / (3 * sizeof(u32))
 356
 357 static const u32 cayman_cgcg_cgls_enable[] =
 358 {
 359         0x00000644, 0x000f7882, 0x001f4080,
 360         0x000008f8, 0x00000010, 0xffffffff,
 361         0x000008fc, 0x00000000, 0xffffffff,
 362         0x000008f8, 0x00000011, 0xffffffff,
 363         0x000008fc, 0x00000000, 0xffffffff,
 364         0x000008f8, 0x00000012, 0xffffffff,
 365         0x000008fc, 0x00000000, 0xffffffff,
 366         0x000008f8, 0x00000013, 0xffffffff,
 367         0x000008fc, 0x00000000, 0xffffffff,
 368         0x000008f8, 0x00000014, 0xffffffff,
 369         0x000008fc, 0x00000000, 0xffffffff,
 370         0x000008f8, 0x00000015, 0xffffffff,
 371         0x000008fc, 0x00000000, 0xffffffff,
 372         0x000008f8, 0x00000016, 0xffffffff,
 373         0x000008fc, 0x00000000, 0xffffffff,
 374         0x000008f8, 0x00000017, 0xffffffff,
 375         0x000008fc, 0x00000000, 0xffffffff,
 376         0x000008f8, 0x00000018, 0xffffffff,
 377         0x000008fc, 0x00000000, 0xffffffff,
 378         0x000008f8, 0x00000019, 0xffffffff,
 379         0x000008fc, 0x00000000, 0xffffffff,
 380         0x000008f8, 0x0000001a, 0xffffffff,
 381         0x000008fc, 0x00000000, 0xffffffff,
 382         0x000008f8, 0x0000001b, 0xffffffff,
 383         0x000008fc, 0x00000000, 0xffffffff,
 384         0x000008f8, 0x00000020, 0xffffffff,
 385         0x000008fc, 0xffffffff, 0xffffffff,
 386         0x000008f8, 0x00000021, 0xffffffff,
 387         0x000008fc, 0xffffffff, 0xffffffff,
 388         0x000008f8, 0x00000022, 0xffffffff,
 389         0x000008fc, 0xffffffff, 0xffffffff,
 390         0x000008f8, 0x00000023, 0xffffffff,
 391         0x000008fc, 0xffffffff, 0xffffffff,
 392         0x000008f8, 0x00000024, 0xffffffff,
 393         0x000008fc, 0xffffffff, 0xffffffff,
 394         0x000008f8, 0x00000025, 0xffffffff,
 395         0x000008fc, 0xffffffff, 0xffffffff,
 396         0x000008f8, 0x00000026, 0xffffffff,
 397         0x000008fc, 0xffffffff, 0xffffffff,
 398         0x000008f8, 0x00000027, 0xffffffff,
 399         0x000008fc, 0xffffffff, 0xffffffff,
 400         0x000008f8, 0x00000028, 0xffffffff,
 401         0x000008fc, 0xffffffff, 0xffffffff,
 402         0x000008f8, 0x00000029, 0xffffffff,
 403         0x000008fc, 0xffffffff, 0xffffffff,
 404         0x000008f8, 0x0000002a, 0xffffffff,
 405         0x000008fc, 0xffffffff, 0xffffffff,
 406         0x000008f8, 0x0000002b, 0xffffffff,
 407         0x000008fc, 0xffffffff, 0xffffffff
 408 };
 409 #define CAYMAN_CGCG_CGLS_ENABLE_LENGTH  sizeof(cayman_cgcg_cgls_enable) / (3 * sizeof(u32))
 410
 411 static const u32 cayman_mgcg_default[] =
 412 {
 413         0x0000802c, 0xc0000000, 0xffffffff,
 414         0x00003fc4, 0xc0000000, 0xffffffff,
 415         0x00005448, 0x00000100, 0xffffffff,
 416         0x000055e4, 0x00000100, 0xffffffff,
 417         0x0000160c, 0x00000100, 0xffffffff,
 418         0x00008984, 0x06000100, 0xffffffff,
 419         0x0000c164, 0x00000100, 0xffffffff,
 420         0x00008a18, 0x00000100, 0xffffffff,
 421         0x0000897c, 0x06000100, 0xffffffff,
 422         0x00008b28, 0x00000100, 0xffffffff,
 423         0x00009144, 0x00800200, 0xffffffff,
 424         0x00009a60, 0x00000100, 0xffffffff,
 425         0x00009868, 0x00000100, 0xffffffff,
 426         0x00008d58, 0x00000100, 0xffffffff,
 427         0x00009510, 0x00000100, 0xffffffff,
 428         0x0000949c, 0x00000100, 0xffffffff,
 429         0x00009654, 0x00000100, 0xffffffff,
 430         0x00009030, 0x00000100, 0xffffffff,
 431         0x00009034, 0x00000100, 0xffffffff,
 432         0x00009038, 0x00000100, 0xffffffff,
 433         0x0000903c, 0x00000100, 0xffffffff,
 434         0x00009040, 0x00000100, 0xffffffff,
 435         0x0000a200, 0x00000100, 0xffffffff,
 436         0x0000a204, 0x00000100, 0xffffffff,
 437         0x0000a208, 0x00000100, 0xffffffff,
 438         0x0000a20c, 0x00000100, 0xffffffff,
 439         0x00009744, 0x00000100, 0xffffffff,
 440         0x00003f80, 0x00000100, 0xffffffff,
 441         0x0000a210, 0x00000100, 0xffffffff,
 442         0x0000a214, 0x00000100, 0xffffffff,
 443         0x000004d8, 0x00000100, 0xffffffff,
 444         0x00009664, 0x00000100, 0xffffffff,
 445         0x00009698, 0x00000100, 0xffffffff,
 446         0x000004d4, 0x00000200, 0xffffffff,
 447         0x000004d0, 0x00000000, 0xffffffff,
 448         0x000030cc, 0x00000104, 0xffffffff,
 449         0x0000d0c0, 0x00000100, 0xffffffff,
 450         0x0000d8c0, 0x00000100, 0xffffffff,
 451         0x0000802c, 0x40000000, 0xffffffff,
 452         0x00003fc4, 0x40000000, 0xffffffff,
 453         0x0000915c, 0x00010000, 0xffffffff,
 454         0x00009160, 0x00030002, 0xffffffff,
 455         0x00009164, 0x00050004, 0xffffffff,
 456         0x00009168, 0x00070006, 0xffffffff,
 457         0x00009178, 0x00070000, 0xffffffff,
 458         0x0000917c, 0x00030002, 0xffffffff,
 459         0x00009180, 0x00050004, 0xffffffff,
 460         0x0000918c, 0x00010006, 0xffffffff,
 461         0x00009190, 0x00090008, 0xffffffff,
 462         0x00009194, 0x00070000, 0xffffffff,
 463         0x00009198, 0x00030002, 0xffffffff,
 464         0x0000919c, 0x00050004, 0xffffffff,
 465         0x000091a8, 0x00010006, 0xffffffff,
 466         0x000091ac, 0x00090008, 0xffffffff,
 467         0x000091b0, 0x00070000, 0xffffffff,
 468         0x000091b4, 0x00030002, 0xffffffff,
 469         0x000091b8, 0x00050004, 0xffffffff,
 470         0x000091c4, 0x00010006, 0xffffffff,
 471         0x000091c8, 0x00090008, 0xffffffff,
 472         0x000091cc, 0x00070000, 0xffffffff,
 473         0x000091d0, 0x00030002, 0xffffffff,
 474         0x000091d4, 0x00050004, 0xffffffff,
 475         0x000091e0, 0x00010006, 0xffffffff,
 476         0x000091e4, 0x00090008, 0xffffffff,
 477         0x000091e8, 0x00000000, 0xffffffff,
 478         0x000091ec, 0x00070000, 0xffffffff,
 479         0x000091f0, 0x00030002, 0xffffffff,
 480         0x000091f4, 0x00050004, 0xffffffff,
 481         0x00009200, 0x00010006, 0xffffffff,
 482         0x00009204, 0x00090008, 0xffffffff,
 483         0x00009208, 0x00070000, 0xffffffff,
 484         0x0000920c, 0x00030002, 0xffffffff,
 485         0x00009210, 0x00050004, 0xffffffff,
 486         0x0000921c, 0x00010006, 0xffffffff,
 487         0x00009220, 0x00090008, 0xffffffff,
 488         0x00009224, 0x00070000, 0xffffffff,
 489         0x00009228, 0x00030002, 0xffffffff,
 490         0x0000922c, 0x00050004, 0xffffffff,
 491         0x00009238, 0x00010006, 0xffffffff,
 492         0x0000923c, 0x00090008, 0xffffffff,
 493         0x00009240, 0x00070000, 0xffffffff,
 494         0x00009244, 0x00030002, 0xffffffff,
 495         0x00009248, 0x00050004, 0xffffffff,
 496         0x00009254, 0x00010006, 0xffffffff,
 497         0x00009258, 0x00090008, 0xffffffff,
 498         0x0000925c, 0x00070000, 0xffffffff,
 499         0x00009260, 0x00030002, 0xffffffff,
 500         0x00009264, 0x00050004, 0xffffffff,
 501         0x00009270, 0x00010006, 0xffffffff,
 502         0x00009274, 0x00090008, 0xffffffff,
 503         0x00009278, 0x00070000, 0xffffffff,
 504         0x0000927c, 0x00030002, 0xffffffff,
 505         0x00009280, 0x00050004, 0xffffffff,
 506         0x0000928c, 0x00010006, 0xffffffff,
 507         0x00009290, 0x00090008, 0xffffffff,
 508         0x000092a8, 0x00070000, 0xffffffff,
 509         0x000092ac, 0x00030002, 0xffffffff,
 510         0x000092b0, 0x00050004, 0xffffffff,
 511         0x000092bc, 0x00010006, 0xffffffff,
 512         0x000092c0, 0x00090008, 0xffffffff,
 513         0x000092c4, 0x00070000, 0xffffffff,
 514         0x000092c8, 0x00030002, 0xffffffff,
 515         0x000092cc, 0x00050004, 0xffffffff,
 516         0x000092d8, 0x00010006, 0xffffffff,
 517         0x000092dc, 0x00090008, 0xffffffff,
 518         0x00009294, 0x00000000, 0xffffffff,
 519         0x0000802c, 0x40010000, 0xffffffff,
 520         0x00003fc4, 0x40010000, 0xffffffff,
 521         0x0000915c, 0x00010000, 0xffffffff,
 522         0x00009160, 0x00030002, 0xffffffff,
 523         0x00009164, 0x00050004, 0xffffffff,
 524         0x00009168, 0x00070006, 0xffffffff,
 525         0x00009178, 0x00070000, 0xffffffff,
 526         0x0000917c, 0x00030002, 0xffffffff,
 527         0x00009180, 0x00050004, 0xffffffff,
 528         0x0000918c, 0x00010006, 0xffffffff,
 529         0x00009190, 0x00090008, 0xffffffff,
 530         0x00009194, 0x00070000, 0xffffffff,
 531         0x00009198, 0x00030002, 0xffffffff,
 532         0x0000919c, 0x00050004, 0xffffffff,
 533         0x000091a8, 0x00010006, 0xffffffff,
 534         0x000091ac, 0x00090008, 0xffffffff,
 535         0x000091b0, 0x00070000, 0xffffffff,
 536         0x000091b4, 0x00030002, 0xffffffff,
 537         0x000091b8, 0x00050004, 0xffffffff,
 538         0x000091c4, 0x00010006, 0xffffffff,
 539         0x000091c8, 0x00090008, 0xffffffff,
 540         0x000091cc, 0x00070000, 0xffffffff,
 541         0x000091d0, 0x00030002, 0xffffffff,
 542         0x000091d4, 0x00050004, 0xffffffff,
 543         0x000091e0, 0x00010006, 0xffffffff,
 544         0x000091e4, 0x00090008, 0xffffffff,
 545         0x000091e8, 0x00000000, 0xffffffff,
 546         0x000091ec, 0x00070000, 0xffffffff,
 547         0x000091f0, 0x00030002, 0xffffffff,
 548         0x000091f4, 0x00050004, 0xffffffff,
 549         0x00009200, 0x00010006, 0xffffffff,
 550         0x00009204, 0x00090008, 0xffffffff,
 551         0x00009208, 0x00070000, 0xffffffff,
 552         0x0000920c, 0x00030002, 0xffffffff,
 553         0x00009210, 0x00050004, 0xffffffff,
 554         0x0000921c, 0x00010006, 0xffffffff,
 555         0x00009220, 0x00090008, 0xffffffff,
 556         0x00009224, 0x00070000, 0xffffffff,
 557         0x00009228, 0x00030002, 0xffffffff,
 558         0x0000922c, 0x00050004, 0xffffffff,
 559         0x00009238, 0x00010006, 0xffffffff,
 560         0x0000923c, 0x00090008, 0xffffffff,
 561         0x00009240, 0x00070000, 0xffffffff,
 562         0x00009244, 0x00030002, 0xffffffff,
 563         0x00009248, 0x00050004, 0xffffffff,
 564         0x00009254, 0x00010006, 0xffffffff,
 565         0x00009258, 0x00090008, 0xffffffff,
 566         0x0000925c, 0x00070000, 0xffffffff,
 567         0x00009260, 0x00030002, 0xffffffff,
 568         0x00009264, 0x00050004, 0xffffffff,
 569         0x00009270, 0x00010006, 0xffffffff,
 570         0x00009274, 0x00090008, 0xffffffff,
 571         0x00009278, 0x00070000, 0xffffffff,
 572         0x0000927c, 0x00030002, 0xffffffff,
 573         0x00009280, 0x00050004, 0xffffffff,
 574         0x0000928c, 0x00010006, 0xffffffff,
 575         0x00009290, 0x00090008, 0xffffffff,
 576         0x000092a8, 0x00070000, 0xffffffff,
 577         0x000092ac, 0x00030002, 0xffffffff,
 578         0x000092b0, 0x00050004, 0xffffffff,
 579         0x000092bc, 0x00010006, 0xffffffff,
 580         0x000092c0, 0x00090008, 0xffffffff,
 581         0x000092c4, 0x00070000, 0xffffffff,
 582         0x000092c8, 0x00030002, 0xffffffff,
 583         0x000092cc, 0x00050004, 0xffffffff,
 584         0x000092d8, 0x00010006, 0xffffffff,
 585         0x000092dc, 0x00090008, 0xffffffff,
 586         0x00009294, 0x00000000, 0xffffffff,
 587         0x0000802c, 0xc0000000, 0xffffffff,
 588         0x00003fc4, 0xc0000000, 0xffffffff,
 589         0x000008f8, 0x00000010, 0xffffffff,
 590         0x000008fc, 0x00000000, 0xffffffff,
 591         0x000008f8, 0x00000011, 0xffffffff,
 592         0x000008fc, 0x00000000, 0xffffffff,
 593         0x000008f8, 0x00000012, 0xffffffff,
 594         0x000008fc, 0x00000000, 0xffffffff,
 595         0x000008f8, 0x00000013, 0xffffffff,
 596         0x000008fc, 0x00000000, 0xffffffff,
 597         0x000008f8, 0x00000014, 0xffffffff,
 598         0x000008fc, 0x00000000, 0xffffffff,
 599         0x000008f8, 0x00000015, 0xffffffff,
 600         0x000008fc, 0x00000000, 0xffffffff,
 601         0x000008f8, 0x00000016, 0xffffffff,
 602         0x000008fc, 0x00000000, 0xffffffff,
 603         0x000008f8, 0x00000017, 0xffffffff,
 604         0x000008fc, 0x00000000, 0xffffffff,
 605         0x000008f8, 0x00000018, 0xffffffff,
 606         0x000008fc, 0x00000000, 0xffffffff,
 607         0x000008f8, 0x00000019, 0xffffffff,
 608         0x000008fc, 0x00000000, 0xffffffff,
 609         0x000008f8, 0x0000001a, 0xffffffff,
 610         0x000008fc, 0x00000000, 0xffffffff,
 611         0x000008f8, 0x0000001b, 0xffffffff,
 612         0x000008fc, 0x00000000, 0xffffffff
 613 };
 614 #define CAYMAN_MGCG_DEFAULT_LENGTH sizeof(cayman_mgcg_default) / (3 * sizeof(u32))
 615
 616 static const u32 cayman_mgcg_disable[] =
 617 {
 618         0x0000802c, 0xc0000000, 0xffffffff,
 619         0x000008f8, 0x00000000, 0xffffffff,
 620         0x000008fc, 0xffffffff, 0xffffffff,
 621         0x000008f8, 0x00000001, 0xffffffff,
 622         0x000008fc, 0xffffffff, 0xffffffff,
 623         0x000008f8, 0x00000002, 0xffffffff,
 624         0x000008fc, 0xffffffff, 0xffffffff,
 625         0x000008f8, 0x00000003, 0xffffffff,
 626         0x000008fc, 0xffffffff, 0xffffffff,
 627         0x00009150, 0x00600000, 0xffffffff
 628 };
 629 #define CAYMAN_MGCG_DISABLE_LENGTH   sizeof(cayman_mgcg_disable) / (3 * sizeof(u32))
 630
 631 static const u32 cayman_mgcg_enable[] =
 632 {
 633         0x0000802c, 0xc0000000, 0xffffffff,
 634         0x000008f8, 0x00000000, 0xffffffff,
 635         0x000008fc, 0x00000000, 0xffffffff,
 636         0x000008f8, 0x00000001, 0xffffffff,
 637         0x000008fc, 0x00000000, 0xffffffff,
 638         0x000008f8, 0x00000002, 0xffffffff,
 639         0x000008fc, 0x00600000, 0xffffffff,
 640         0x000008f8, 0x00000003, 0xffffffff,
 641         0x000008fc, 0x00000000, 0xffffffff,
 642         0x00009150, 0x96944200, 0xffffffff
 643 };
 644
 645 #define CAYMAN_MGCG_ENABLE_LENGTH   sizeof(cayman_mgcg_enable) / (3 * sizeof(u32))
 646
 647 #define NISLANDS_SYSLS_SEQUENCE  100
 648
 649 static const u32 cayman_sysls_default[] =
 650 {
 651         /* Register,   Value,     Mask bits */
 652         0x000055e8, 0x00000000, 0xffffffff,
 653         0x0000d0bc, 0x00000000, 0xffffffff,
 654         0x0000d8bc, 0x00000000, 0xffffffff,
 655         0x000015c0, 0x000c1401, 0xffffffff,
 656         0x0000264c, 0x000c0400, 0xffffffff,
 657         0x00002648, 0x000c0400, 0xffffffff,
 658         0x00002650, 0x000c0400, 0xffffffff,
 659         0x000020b8, 0x000c0400, 0xffffffff,
 660         0x000020bc, 0x000c0400, 0xffffffff,
 661         0x000020c0, 0x000c0c80, 0xffffffff,
 662         0x0000f4a0, 0x000000c0, 0xffffffff,
 663         0x0000f4a4, 0x00680fff, 0xffffffff,
 664         0x00002f50, 0x00000404, 0xffffffff,
 665         0x000004c8, 0x00000001, 0xffffffff,
 666         0x000064ec, 0x00000000, 0xffffffff,
 667         0x00000c7c, 0x00000000, 0xffffffff,
 668         0x00008dfc, 0x00000000, 0xffffffff
 669 };
 670 #define CAYMAN_SYSLS_DEFAULT_LENGTH sizeof(cayman_sysls_default) / (3 * sizeof(u32))
 671
 672 static const u32 cayman_sysls_disable[] =
 673 {
 674         /* Register,   Value,     Mask bits */
 675         0x0000d0c0, 0x00000000, 0xffffffff,
 676         0x0000d8c0, 0x00000000, 0xffffffff,
 677         0x000055e8, 0x00000000, 0xffffffff,
 678         0x0000d0bc, 0x00000000, 0xffffffff,
 679         0x0000d8bc, 0x00000000, 0xffffffff,
 680         0x000015c0, 0x00041401, 0xffffffff,
 681         0x0000264c, 0x00040400, 0xffffffff,
 682         0x00002648, 0x00040400, 0xffffffff,
 683         0x00002650, 0x00040400, 0xffffffff,
 684         0x000020b8, 0x00040400, 0xffffffff,
 685         0x000020bc, 0x00040400, 0xffffffff,
 686         0x000020c0, 0x00040c80, 0xffffffff,
 687         0x0000f4a0, 0x000000c0, 0xffffffff,
 688         0x0000f4a4, 0x00680000, 0xffffffff,
 689         0x00002f50, 0x00000404, 0xffffffff,
 690         0x000004c8, 0x00000001, 0xffffffff,
 691         0x000064ec, 0x00007ffd, 0xffffffff,
 692         0x00000c7c, 0x0000ff00, 0xffffffff,
 693         0x00008dfc, 0x0000007f, 0xffffffff
 694 };
 695 #define CAYMAN_SYSLS_DISABLE_LENGTH sizeof(cayman_sysls_disable) / (3 * sizeof(u32))
 696
 697 static const u32 cayman_sysls_enable[] =
 698 {
 699         /* Register,   Value,     Mask bits */
 700         0x000055e8, 0x00000001, 0xffffffff,
 701         0x0000d0bc, 0x00000100, 0xffffffff,
 702         0x0000d8bc, 0x00000100, 0xffffffff,
 703         0x000015c0, 0x000c1401, 0xffffffff,
 704         0x0000264c, 0x000c0400, 0xffffffff,
 705         0x00002648, 0x000c0400, 0xffffffff,
 706         0x00002650, 0x000c0400, 0xffffffff,
 707         0x000020b8, 0x000c0400, 0xffffffff,
 708         0x000020bc, 0x000c0400, 0xffffffff,
 709         0x000020c0, 0x000c0c80, 0xffffffff,
 710         0x0000f4a0, 0x000000c0, 0xffffffff,
 711         0x0000f4a4, 0x00680fff, 0xffffffff,
 712         0x00002f50, 0x00000903, 0xffffffff,
 713         0x000004c8, 0x00000000, 0xffffffff,
 714         0x000064ec, 0x00000000, 0xffffffff,
 715         0x00000c7c, 0x00000000, 0xffffffff,
 716         0x00008dfc, 0x00000000, 0xffffffff
 717 };
 718 #define CAYMAN_SYSLS_ENABLE_LENGTH sizeof(cayman_sysls_enable) / (3 * sizeof(u32))
 719
 720 struct rv7xx_power_info *rv770_get_pi(struct radeon_device *rdev);
 721 struct evergreen_power_info *evergreen_get_pi(struct radeon_device *rdev);
 722
 723 extern int ni_mc_load_microcode(struct radeon_device *rdev);
 724
 725 struct ni_power_info *ni_get_pi(struct radeon_device *rdev)
 726 {
 727         struct ni_power_info *pi = rdev->pm.dpm.priv;
 728
 729         return pi;
 730 }
 731
 732 struct ni_ps *ni_get_ps(struct radeon_ps *rps)
 733 {
 734         struct ni_ps *ps = rps->ps_priv;
 735
 736         return ps;
 737 }
 738
 739 static void ni_calculate_leakage_for_v_and_t_formula(const struct ni_leakage_coeffients *coeff,
 740                                                      u16 v, s32 t,
 741                                                      u32 ileakage,
 742                                                      u32 *leakage)
 743 {
 744         s64 kt, kv, leakage_w, i_leakage, vddc, temperature;
 745
 746         i_leakage = div64_s64(drm_int2fixp(ileakage), 1000);
 747         vddc = div64_s64(drm_int2fixp(v), 1000);
 748         temperature = div64_s64(drm_int2fixp(t), 1000);
 749
 750         kt = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->at), 1000),
 751                           drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bt), 1000), temperature)));
 752         kv = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->av), 1000),
 753                           drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bv), 1000), vddc)));
 754
 755         leakage_w = drm_fixp_mul(drm_fixp_mul(drm_fixp_mul(i_leakage, kt), kv), vddc);
 756
 757         *leakage = drm_fixp2int(leakage_w * 1000);
 758 }
 759
 760 static void ni_calculate_leakage_for_v_and_t(struct radeon_device *rdev,
 761                                              const struct ni_leakage_coeffients *coeff,
 762                                              u16 v,
 763                                              s32 t,
 764                                              u32 i_leakage,
 765                                              u32 *leakage)
 766 {
 767         ni_calculate_leakage_for_v_and_t_formula(coeff, v, t, i_leakage, leakage);
 768 }
 769
 770 bool ni_dpm_vblank_too_short(struct radeon_device *rdev)
 771 {
 772         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 773         u32 vblank_time = r600_dpm_get_vblank_time(rdev);
 774         /* we never hit the non-gddr5 limit so disable it */
 775         u32 switch_limit = pi->mem_gddr5 ? 450 : 0;
 776
 777         if (vblank_time < switch_limit)
 778                 return true;
 779         else
 780                 return false;
 781
 782 }
 783
 784 static void ni_apply_state_adjust_rules(struct radeon_device *rdev,
 785                                         struct radeon_ps *rps)
 786 {
 787         struct ni_ps *ps = ni_get_ps(rps);
 788         struct radeon_clock_and_voltage_limits *max_limits;
 789         bool disable_mclk_switching;
 790         u32 mclk;
 791         u16 vddci;
 792         u32 max_sclk_vddc, max_mclk_vddci, max_mclk_vddc;
 793         int i;
 794
 795         if ((rdev->pm.dpm.new_active_crtc_count > 1) ||
 796             ni_dpm_vblank_too_short(rdev))
 797                 disable_mclk_switching = true;
 798         else
 799                 disable_mclk_switching = false;
 800
 801         if (rdev->pm.dpm.ac_power)
 802                 max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
 803         else
 804                 max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc;
 805
 806         if (rdev->pm.dpm.ac_power == false) {
 807                 for (i = 0; i < ps->performance_level_count; i++) {
 808                         if (ps->performance_levels[i].mclk > max_limits->mclk)
 809                                 ps->performance_levels[i].mclk = max_limits->mclk;
 810                         if (ps->performance_levels[i].sclk > max_limits->sclk)
 811                                 ps->performance_levels[i].sclk = max_limits->sclk;
 812                         if (ps->performance_levels[i].vddc > max_limits->vddc)
 813                                 ps->performance_levels[i].vddc = max_limits->vddc;
 814                         if (ps->performance_levels[i].vddci > max_limits->vddci)
 815                                 ps->performance_levels[i].vddci = max_limits->vddci;
 816                 }
 817         }
 818
 819         /* limit clocks to max supported clocks based on voltage dependency tables */
 820         btc_get_max_clock_from_voltage_dependency_table(&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk,
 821                                                         &max_sclk_vddc);
 822         btc_get_max_clock_from_voltage_dependency_table(&rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk,
 823                                                         &max_mclk_vddci);
 824         btc_get_max_clock_from_voltage_dependency_table(&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk,
 825                                                         &max_mclk_vddc);
 826
 827         for (i = 0; i < ps->performance_level_count; i++) {
 828                 if (max_sclk_vddc) {
 829                         if (ps->performance_levels[i].sclk > max_sclk_vddc)
 830                                 ps->performance_levels[i].sclk = max_sclk_vddc;
 831                 }
 832                 if (max_mclk_vddci) {
 833                         if (ps->performance_levels[i].mclk > max_mclk_vddci)
 834                                 ps->performance_levels[i].mclk = max_mclk_vddci;
 835                 }
 836                 if (max_mclk_vddc) {
 837                         if (ps->performance_levels[i].mclk > max_mclk_vddc)
 838                                 ps->performance_levels[i].mclk = max_mclk_vddc;
 839                 }
 840         }
 841
 842         /* XXX validate the min clocks required for display */
 843
 844         /* adjust low state */
 845         if (disable_mclk_switching) {
 846                 ps->performance_levels[0].mclk =
 847                         ps->performance_levels[ps->performance_level_count - 1].mclk;
 848                 ps->performance_levels[0].vddci =
 849                         ps->performance_levels[ps->performance_level_count - 1].vddci;
 850         }
 851
 852         btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk,
 853                                   &ps->performance_levels[0].sclk,
 854                                   &ps->performance_levels[0].mclk);
 855
 856         for (i = 1; i < ps->performance_level_count; i++) {
 857                 if (ps->performance_levels[i].sclk < ps->performance_levels[i - 1].sclk)
 858                         ps->performance_levels[i].sclk = ps->performance_levels[i - 1].sclk;
 859                 if (ps->performance_levels[i].vddc < ps->performance_levels[i - 1].vddc)
 860                         ps->performance_levels[i].vddc = ps->performance_levels[i - 1].vddc;
 861         }
 862
 863         /* adjust remaining states */
 864         if (disable_mclk_switching) {
 865                 mclk = ps->performance_levels[0].mclk;
 866                 vddci = ps->performance_levels[0].vddci;
 867                 for (i = 1; i < ps->performance_level_count; i++) {
 868                         if (mclk < ps->performance_levels[i].mclk)
 869                                 mclk = ps->performance_levels[i].mclk;
 870                         if (vddci < ps->performance_levels[i].vddci)
 871                                 vddci = ps->performance_levels[i].vddci;
 872                 }
 873                 for (i = 0; i < ps->performance_level_count; i++) {
 874                         ps->performance_levels[i].mclk = mclk;
 875                         ps->performance_levels[i].vddci = vddci;
 876                 }
 877         } else {
 878                 for (i = 1; i < ps->performance_level_count; i++) {
 879                         if (ps->performance_levels[i].mclk < ps->performance_levels[i - 1].mclk)
 880                                 ps->performance_levels[i].mclk = ps->performance_levels[i - 1].mclk;
 881                         if (ps->performance_levels[i].vddci < ps->performance_levels[i - 1].vddci)
 882                                 ps->performance_levels[i].vddci = ps->performance_levels[i - 1].vddci;
 883                 }
 884         }
 885
 886         for (i = 1; i < ps->performance_level_count; i++)
 887                 btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk,
 888                                           &ps->performance_levels[i].sclk,
 889                                           &ps->performance_levels[i].mclk);
 890
 891         for (i = 0; i < ps->performance_level_count; i++)
 892                 btc_adjust_clock_combinations(rdev, max_limits,
 893                                               &ps->performance_levels[i]);
 894
 895         for (i = 0; i < ps->performance_level_count; i++) {
 896                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk,
 897                                                    ps->performance_levels[i].sclk,
 898                                                    max_limits->vddc,  &ps->performance_levels[i].vddc);
 899                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk,
 900                                                    ps->performance_levels[i].mclk,
 901                                                    max_limits->vddci, &ps->performance_levels[i].vddci);
 902                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk,
 903                                                    ps->performance_levels[i].mclk,
 904                                                    max_limits->vddc,  &ps->performance_levels[i].vddc);
 905                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk,
 906                                                    rdev->clock.current_dispclk,
 907                                                    max_limits->vddc,  &ps->performance_levels[i].vddc);
 908         }
 909
 910         for (i = 0; i < ps->performance_level_count; i++) {
 911                 btc_apply_voltage_delta_rules(rdev,
 912                                               max_limits->vddc, max_limits->vddci,
 913                                               &ps->performance_levels[i].vddc,
 914                                               &ps->performance_levels[i].vddci);
 915         }
 916
 917         ps->dc_compatible = true;
 918         for (i = 0; i < ps->performance_level_count; i++) {
 919                 if (ps->performance_levels[i].vddc > rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.vddc)
 920                         ps->dc_compatible = false;
 921
 922                 if (ps->performance_levels[i].vddc < rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2)
 923                         ps->performance_levels[i].flags &= ~ATOM_PPLIB_R600_FLAGS_PCIEGEN2;
 924         }
 925 }
 926
 927 static void ni_cg_clockgating_default(struct radeon_device *rdev)
 928 {
 929         u32 count;
 930         const u32 *ps = NULL;
 931
 932         ps = (const u32 *)&cayman_cgcg_cgls_default;
 933         count = CAYMAN_CGCG_CGLS_DEFAULT_LENGTH;
 934
 935         btc_program_mgcg_hw_sequence(rdev, ps, count);
 936 }
 937
 938 static void ni_gfx_clockgating_enable(struct radeon_device *rdev,
 939                                       bool enable)
 940 {
 941         u32 count;
 942         const u32 *ps = NULL;
 943
 944         if (enable) {
 945                 ps = (const u32 *)&cayman_cgcg_cgls_enable;
 946                 count = CAYMAN_CGCG_CGLS_ENABLE_LENGTH;
 947         } else {
 948                 ps = (const u32 *)&cayman_cgcg_cgls_disable;
 949                 count = CAYMAN_CGCG_CGLS_DISABLE_LENGTH;
 950         }
 951
 952         btc_program_mgcg_hw_sequence(rdev, ps, count);
 953 }
 954
 955 static void ni_mg_clockgating_default(struct radeon_device *rdev)
 956 {
 957         u32 count;
 958         const u32 *ps = NULL;
 959
 960         ps = (const u32 *)&cayman_mgcg_default;
 961         count = CAYMAN_MGCG_DEFAULT_LENGTH;
 962
 963         btc_program_mgcg_hw_sequence(rdev, ps, count);
 964 }
 965
 966 static void ni_mg_clockgating_enable(struct radeon_device *rdev,
 967                                      bool enable)
 968 {
 969         u32 count;
 970         const u32 *ps = NULL;
 971
 972         if (enable) {
 973                 ps = (const u32 *)&cayman_mgcg_enable;
 974                 count = CAYMAN_MGCG_ENABLE_LENGTH;
 975         } else {
 976                 ps = (const u32 *)&cayman_mgcg_disable;
 977                 count = CAYMAN_MGCG_DISABLE_LENGTH;
 978         }
 979
 980         btc_program_mgcg_hw_sequence(rdev, ps, count);
 981 }
 982
 983 static void ni_ls_clockgating_default(struct radeon_device *rdev)
 984 {
 985         u32 count;
 986         const u32 *ps = NULL;
 987
 988         ps = (const u32 *)&cayman_sysls_default;
 989         count = CAYMAN_SYSLS_DEFAULT_LENGTH;
 990
 991         btc_program_mgcg_hw_sequence(rdev, ps, count);
 992 }
 993
 994 static void ni_ls_clockgating_enable(struct radeon_device *rdev,
 995                                      bool enable)
 996 {
 997         u32 count;
 998         const u32 *ps = NULL;
 999
1000         if (enable) {
1001                 ps = (const u32 *)&cayman_sysls_enable;
1002                 count = CAYMAN_SYSLS_ENABLE_LENGTH;
1003         } else {
1004                 ps = (const u32 *)&cayman_sysls_disable;
1005                 count = CAYMAN_SYSLS_DISABLE_LENGTH;
1006         }
1007
1008         btc_program_mgcg_hw_sequence(rdev, ps, count);
1009
1010 }
1011
1012 static int ni_patch_single_dependency_table_based_on_leakage(struct radeon_device *rdev,
1013                                                              struct radeon_clock_voltage_dependency_table *table)
1014 {
1015         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1016         u32 i;
1017
1018         if (table) {
1019                 for (i = 0; i < table->count; i++) {
1020                         if (0xff01 == table->entries[i].v) {
1021                                 if (pi->max_vddc == 0)
1022                                         return -EINVAL;
1023                                 table->entries[i].v = pi->max_vddc;
1024                         }
1025                 }
1026         }
1027         return 0;
1028 }
1029
1030 static int ni_patch_dependency_tables_based_on_leakage(struct radeon_device *rdev)
1031 {
1032         int ret = 0;
1033
1034         ret = ni_patch_single_dependency_table_based_on_leakage(rdev,
1035                                                                 &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk);
1036
1037         ret = ni_patch_single_dependency_table_based_on_leakage(rdev,
1038                                                                 &rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk);
1039         return ret;
1040 }
1041
1042 static void ni_stop_dpm(struct radeon_device *rdev)
1043 {
1044         WREG32_P(GENERAL_PWRMGT, 0, ~GLOBAL_PWRMGT_EN);
1045 }
1046
1047 #if 0
1048 static int ni_notify_hw_of_power_source(struct radeon_device *rdev,
1049                                         bool ac_power)
1050 {
1051         if (ac_power)
1052                 return (rv770_send_msg_to_smc(rdev, PPSMC_MSG_RunningOnAC) == PPSMC_Result_OK) ?
1053                         0 : -EINVAL;
1054
1055         return 0;
1056 }
1057 #endif
1058
1059 static PPSMC_Result ni_send_msg_to_smc_with_parameter(struct radeon_device *rdev,
1060                                                       PPSMC_Msg msg, u32 parameter)
1061 {
1062         WREG32(SMC_SCRATCH0, parameter);
1063         return rv770_send_msg_to_smc(rdev, msg);
1064 }
1065
1066 static int ni_restrict_performance_levels_before_switch(struct radeon_device *rdev)
1067 {
1068         if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_NoForcedLevel) != PPSMC_Result_OK)
1069                 return -EINVAL;
1070
1071         return (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 1) == PPSMC_Result_OK) ?
1072                 0 : -EINVAL;
1073 }
1074
1075 int ni_dpm_force_performance_level(struct radeon_device *rdev,
1076                                    enum radeon_dpm_forced_level level)
1077 {
1078         if (level == RADEON_DPM_FORCED_LEVEL_HIGH) {
1079                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 0) != PPSMC_Result_OK)
1080                         return -EINVAL;
1081
1082                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 1) != PPSMC_Result_OK)
1083                         return -EINVAL;
1084         } else if (level == RADEON_DPM_FORCED_LEVEL_LOW) {
1085                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK)
1086                         return -EINVAL;
1087
1088                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 1) != PPSMC_Result_OK)
1089                         return -EINVAL;
1090         } else if (level == RADEON_DPM_FORCED_LEVEL_AUTO) {
1091                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK)
1092                         return -EINVAL;
1093
1094                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 0) != PPSMC_Result_OK)
1095                         return -EINVAL;
1096         }
1097
1098         rdev->pm.dpm.forced_level = level;
1099
1100         return 0;
1101 }
1102
1103 static void ni_stop_smc(struct radeon_device *rdev)
1104 {
1105         u32 tmp;
1106         int i;
1107
1108         for (i = 0; i < rdev->usec_timeout; i++) {
1109                 tmp = RREG32(LB_SYNC_RESET_SEL) & LB_SYNC_RESET_SEL_MASK;
1110                 if (tmp != 1)
1111                         break;
1112                 udelay(1);
1113         }
1114
1115         udelay(100);
1116
1117         r7xx_stop_smc(rdev);
1118 }
1119
1120 static int ni_process_firmware_header(struct radeon_device *rdev)
1121 {
1122         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1123         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1124         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1125         u32 tmp;
1126         int ret;
1127
1128         ret = rv770_read_smc_sram_dword(rdev,
1129                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1130                                         NISLANDS_SMC_FIRMWARE_HEADER_stateTable,
1131                                         &tmp, pi->sram_end);
1132
1133         if (ret)
1134                 return ret;
1135
1136         pi->state_table_start = (u16)tmp;
1137
1138         ret = rv770_read_smc_sram_dword(rdev,
1139                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1140                                         NISLANDS_SMC_FIRMWARE_HEADER_softRegisters,
1141                                         &tmp, pi->sram_end);
1142
1143         if (ret)
1144                 return ret;
1145
1146         pi->soft_regs_start = (u16)tmp;
1147
1148         ret = rv770_read_smc_sram_dword(rdev,
1149                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1150                                         NISLANDS_SMC_FIRMWARE_HEADER_mcRegisterTable,
1151                                         &tmp, pi->sram_end);
1152
1153         if (ret)
1154                 return ret;
1155
1156         eg_pi->mc_reg_table_start = (u16)tmp;
1157
1158         ret = rv770_read_smc_sram_dword(rdev,
1159                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1160                                         NISLANDS_SMC_FIRMWARE_HEADER_fanTable,
1161                                         &tmp, pi->sram_end);
1162
1163         if (ret)
1164                 return ret;
1165
1166         ni_pi->fan_table_start = (u16)tmp;
1167
1168         ret = rv770_read_smc_sram_dword(rdev,
1169                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1170                                         NISLANDS_SMC_FIRMWARE_HEADER_mcArbDramAutoRefreshTable,
1171                                         &tmp, pi->sram_end);
1172
1173         if (ret)
1174                 return ret;
1175
1176         ni_pi->arb_table_start = (u16)tmp;
1177
1178         ret = rv770_read_smc_sram_dword(rdev,
1179                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1180                                         NISLANDS_SMC_FIRMWARE_HEADER_cacTable,
1181                                         &tmp, pi->sram_end);
1182
1183         if (ret)
1184                 return ret;
1185
1186         ni_pi->cac_table_start = (u16)tmp;
1187
1188         ret = rv770_read_smc_sram_dword(rdev,
1189                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1190                                         NISLANDS_SMC_FIRMWARE_HEADER_spllTable,
1191                                         &tmp, pi->sram_end);
1192
1193         if (ret)
1194                 return ret;
1195
1196         ni_pi->spll_table_start = (u16)tmp;
1197
1198
1199         return ret;
1200 }
1201
1202 static void ni_read_clock_registers(struct radeon_device *rdev)
1203 {
1204         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1205
1206         ni_pi->clock_registers.cg_spll_func_cntl = RREG32(CG_SPLL_FUNC_CNTL);
1207         ni_pi->clock_registers.cg_spll_func_cntl_2 = RREG32(CG_SPLL_FUNC_CNTL_2);
1208         ni_pi->clock_registers.cg_spll_func_cntl_3 = RREG32(CG_SPLL_FUNC_CNTL_3);
1209         ni_pi->clock_registers.cg_spll_func_cntl_4 = RREG32(CG_SPLL_FUNC_CNTL_4);
1210         ni_pi->clock_registers.cg_spll_spread_spectrum = RREG32(CG_SPLL_SPREAD_SPECTRUM);
1211         ni_pi->clock_registers.cg_spll_spread_spectrum_2 = RREG32(CG_SPLL_SPREAD_SPECTRUM_2);
1212         ni_pi->clock_registers.mpll_ad_func_cntl = RREG32(MPLL_AD_FUNC_CNTL);
1213         ni_pi->clock_registers.mpll_ad_func_cntl_2 = RREG32(MPLL_AD_FUNC_CNTL_2);
1214         ni_pi->clock_registers.mpll_dq_func_cntl = RREG32(MPLL_DQ_FUNC_CNTL);
1215         ni_pi->clock_registers.mpll_dq_func_cntl_2 = RREG32(MPLL_DQ_FUNC_CNTL_2);
1216         ni_pi->clock_registers.mclk_pwrmgt_cntl = RREG32(MCLK_PWRMGT_CNTL);
1217         ni_pi->clock_registers.dll_cntl = RREG32(DLL_CNTL);
1218         ni_pi->clock_registers.mpll_ss1 = RREG32(MPLL_SS1);
1219         ni_pi->clock_registers.mpll_ss2 = RREG32(MPLL_SS2);
1220 }
1221
1222 #if 0
1223 static int ni_enter_ulp_state(struct radeon_device *rdev)
1224 {
1225         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1226
1227         if (pi->gfx_clock_gating) {
1228                 WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
1229                 WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON);
1230                 WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON);
1231                 RREG32(GB_ADDR_CONFIG);
1232         }
1233
1234         WREG32_P(SMC_MSG, HOST_SMC_MSG(PPSMC_MSG_SwitchToMinimumPower),
1235                  ~HOST_SMC_MSG_MASK);
1236
1237         udelay(25000);
1238
1239         return 0;
1240 }
1241 #endif
1242
1243 static void ni_program_response_times(struct radeon_device *rdev)
1244 {
1245         u32 voltage_response_time, backbias_response_time, acpi_delay_time, vbi_time_out;
1246         u32 vddc_dly, bb_dly, acpi_dly, vbi_dly, mclk_switch_limit;
1247         u32 reference_clock;
1248
1249         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mvdd_chg_time, 1);
1250
1251         voltage_response_time = (u32)rdev->pm.dpm.voltage_response_time;
1252         backbias_response_time = (u32)rdev->pm.dpm.backbias_response_time;
1253
1254         if (voltage_response_time == 0)
1255                 voltage_response_time = 1000;
1256
1257         if (backbias_response_time == 0)
1258                 backbias_response_time = 1000;
1259
1260         acpi_delay_time = 15000;
1261         vbi_time_out = 100000;
1262
1263         reference_clock = radeon_get_xclk(rdev);
1264
1265         vddc_dly = (voltage_response_time  * reference_clock) / 1600;
1266         bb_dly   = (backbias_response_time * reference_clock) / 1600;
1267         acpi_dly = (acpi_delay_time * reference_clock) / 1600;
1268         vbi_dly  = (vbi_time_out * reference_clock) / 1600;
1269
1270         mclk_switch_limit = (460 * reference_clock) / 100;
1271
1272         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_vreg,  vddc_dly);
1273         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_bbias, bb_dly);
1274         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_acpi,  acpi_dly);
1275         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mclk_chg_timeout, vbi_dly);
1276         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mc_block_delay, 0xAA);
1277         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mclk_switch_lim, mclk_switch_limit);
1278 }
1279
1280 static void ni_populate_smc_voltage_table(struct radeon_device *rdev,
1281                                           struct atom_voltage_table *voltage_table,
1282                                           NISLANDS_SMC_STATETABLE *table)
1283 {
1284         unsigned int i;
1285
1286         for (i = 0; i < voltage_table->count; i++) {
1287                 table->highSMIO[i] = 0;
1288                 table->lowSMIO[i] |= cpu_to_be32(voltage_table->entries[i].smio_low);
1289         }
1290 }
1291
1292 static void ni_populate_smc_voltage_tables(struct radeon_device *rdev,
1293                                            NISLANDS_SMC_STATETABLE *table)
1294 {
1295         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1296         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1297         unsigned char i;
1298
1299         if (eg_pi->vddc_voltage_table.count) {
1300                 ni_populate_smc_voltage_table(rdev, &eg_pi->vddc_voltage_table, table);
1301                 table->voltageMaskTable.highMask[NISLANDS_SMC_VOLTAGEMASK_VDDC] = 0;
1302                 table->voltageMaskTable.lowMask[NISLANDS_SMC_VOLTAGEMASK_VDDC] =
1303                         cpu_to_be32(eg_pi->vddc_voltage_table.mask_low);
1304
1305                 for (i = 0; i < eg_pi->vddc_voltage_table.count; i++) {
1306                         if (pi->max_vddc_in_table <= eg_pi->vddc_voltage_table.entries[i].value) {
1307                                 table->maxVDDCIndexInPPTable = i;
1308                                 break;
1309                         }
1310                 }
1311         }
1312
1313         if (eg_pi->vddci_voltage_table.count) {
1314                 ni_populate_smc_voltage_table(rdev, &eg_pi->vddci_voltage_table, table);
1315
1316                 table->voltageMaskTable.highMask[NISLANDS_SMC_VOLTAGEMASK_VDDCI] = 0;
1317                 table->voltageMaskTable.lowMask[NISLANDS_SMC_VOLTAGEMASK_VDDCI] =
1318                         cpu_to_be32(eg_pi->vddc_voltage_table.mask_low);
1319         }
1320 }
1321
1322 static int ni_populate_voltage_value(struct radeon_device *rdev,
1323                                      struct atom_voltage_table *table,
1324                                      u16 value,
1325                                      NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1326 {
1327         unsigned int i;
1328
1329         for (i = 0; i < table->count; i++) {
1330                 if (value <= table->entries[i].value) {
1331                         voltage->index = (u8)i;
1332                         voltage->value = cpu_to_be16(table->entries[i].value);
1333                         break;
1334                 }
1335         }
1336
1337         if (i >= table->count)
1338                 return -EINVAL;
1339
1340         return 0;
1341 }
1342
1343 static void ni_populate_mvdd_value(struct radeon_device *rdev,
1344                                    u32 mclk,
1345                                    NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1346 {
1347         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1348         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1349
1350         if (!pi->mvdd_control) {
1351                 voltage->index = eg_pi->mvdd_high_index;
1352                 voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1353                 return;
1354         }
1355
1356         if (mclk <= pi->mvdd_split_frequency) {
1357                 voltage->index = eg_pi->mvdd_low_index;
1358                 voltage->value = cpu_to_be16(MVDD_LOW_VALUE);
1359         } else {
1360                 voltage->index = eg_pi->mvdd_high_index;
1361                 voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1362         }
1363 }
1364
1365 static int ni_get_std_voltage_value(struct radeon_device *rdev,
1366                                     NISLANDS_SMC_VOLTAGE_VALUE *voltage,
1367                                     u16 *std_voltage)
1368 {
1369         if (rdev->pm.dpm.dyn_state.cac_leakage_table.entries &&
1370             ((u32)voltage->index < rdev->pm.dpm.dyn_state.cac_leakage_table.count))
1371                 *std_voltage = rdev->pm.dpm.dyn_state.cac_leakage_table.entries[voltage->index].vddc;
1372         else
1373                 *std_voltage = be16_to_cpu(voltage->value);
1374
1375         return 0;
1376 }
1377
1378 static void ni_populate_std_voltage_value(struct radeon_device *rdev,
1379                                           u16 value, u8 index,
1380                                           NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1381 {
1382         voltage->index = index;
1383         voltage->value = cpu_to_be16(value);
1384 }
1385
1386 static u32 ni_get_smc_power_scaling_factor(struct radeon_device *rdev)
1387 {
1388         u32 xclk_period;
1389         u32 xclk = radeon_get_xclk(rdev);
1390         u32 tmp = RREG32(CG_CAC_CTRL) & TID_CNT_MASK;
1391
1392         xclk_period = (1000000000UL / xclk);
1393         xclk_period /= 10000UL;
1394
1395         return tmp * xclk_period;
1396 }
1397
1398 static u32 ni_scale_power_for_smc(u32 power_in_watts, u32 scaling_factor)
1399 {
1400         return (power_in_watts * scaling_factor) << 2;
1401 }
1402
1403 static u32 ni_calculate_power_boost_limit(struct radeon_device *rdev,
1404                                           struct radeon_ps *radeon_state,
1405                                           u32 near_tdp_limit)
1406 {
1407         struct ni_ps *state = ni_get_ps(radeon_state);
1408         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1409         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1410         u32 power_boost_limit = 0;
1411         int ret;
1412
1413         if (ni_pi->enable_power_containment &&
1414             ni_pi->use_power_boost_limit) {
1415                 NISLANDS_SMC_VOLTAGE_VALUE vddc;
1416                 u16 std_vddc_med;
1417                 u16 std_vddc_high;
1418                 u64 tmp, n, d;
1419
1420                 if (state->performance_level_count < 3)
1421                         return 0;
1422
1423                 ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1424                                                 state->performance_levels[state->performance_level_count - 2].vddc,
1425                                                 &vddc);
1426                 if (ret)
1427                         return 0;
1428
1429                 ret = ni_get_std_voltage_value(rdev, &vddc, &std_vddc_med);
1430                 if (ret)
1431                         return 0;
1432
1433                 ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1434                                                 state->performance_levels[state->performance_level_count - 1].vddc,
1435                                                 &vddc);
1436                 if (ret)
1437                         return 0;
1438
1439                 ret = ni_get_std_voltage_value(rdev, &vddc, &std_vddc_high);
1440                 if (ret)
1441                         return 0;
1442
1443                 n = ((u64)near_tdp_limit * ((u64)std_vddc_med * (u64)std_vddc_med) * 90);
1444                 d = ((u64)std_vddc_high * (u64)std_vddc_high * 100);
1445                 tmp = div64_u64(n, d);
1446
1447                 if (tmp >> 32)
1448                         return 0;
1449                 power_boost_limit = (u32)tmp;
1450         }
1451
1452         return power_boost_limit;
1453 }
1454
1455 static int ni_calculate_adjusted_tdp_limits(struct radeon_device *rdev,
1456                                             bool adjust_polarity,
1457                                             u32 tdp_adjustment,
1458                                             u32 *tdp_limit,
1459                                             u32 *near_tdp_limit)
1460 {
1461         if (tdp_adjustment > (u32)rdev->pm.dpm.tdp_od_limit)
1462                 return -EINVAL;
1463
1464         if (adjust_polarity) {
1465                 *tdp_limit = ((100 + tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100;
1466                 *near_tdp_limit = rdev->pm.dpm.near_tdp_limit + (*tdp_limit - rdev->pm.dpm.tdp_limit);
1467         } else {
1468                 *tdp_limit = ((100 - tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100;
1469                 *near_tdp_limit = rdev->pm.dpm.near_tdp_limit - (rdev->pm.dpm.tdp_limit - *tdp_limit);
1470         }
1471
1472         return 0;
1473 }
1474
1475 static int ni_populate_smc_tdp_limits(struct radeon_device *rdev,
1476                                       struct radeon_ps *radeon_state)
1477 {
1478         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1479         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1480
1481         if (ni_pi->enable_power_containment) {
1482                 NISLANDS_SMC_STATETABLE *smc_table = &ni_pi->smc_statetable;
1483                 u32 scaling_factor = ni_get_smc_power_scaling_factor(rdev);
1484                 u32 tdp_limit;
1485                 u32 near_tdp_limit;
1486                 u32 power_boost_limit;
1487                 int ret;
1488
1489                 if (scaling_factor == 0)
1490                         return -EINVAL;
1491
1492                 memset(smc_table, 0, sizeof(NISLANDS_SMC_STATETABLE));
1493
1494                 ret = ni_calculate_adjusted_tdp_limits(rdev,
1495                                                        false, /* ??? */
1496                                                        rdev->pm.dpm.tdp_adjustment,
1497                                                        &tdp_limit,
1498                                                        &near_tdp_limit);
1499                 if (ret)
1500                         return ret;
1501
1502                 power_boost_limit = ni_calculate_power_boost_limit(rdev, radeon_state,
1503                                                                    near_tdp_limit);
1504
1505                 smc_table->dpm2Params.TDPLimit =
1506                         cpu_to_be32(ni_scale_power_for_smc(tdp_limit, scaling_factor));
1507                 smc_table->dpm2Params.NearTDPLimit =
1508                         cpu_to_be32(ni_scale_power_for_smc(near_tdp_limit, scaling_factor));
1509                 smc_table->dpm2Params.SafePowerLimit =
1510                         cpu_to_be32(ni_scale_power_for_smc((near_tdp_limit * NISLANDS_DPM2_TDP_SAFE_LIMIT_PERCENT) / 100,
1511                                                            scaling_factor));
1512                 smc_table->dpm2Params.PowerBoostLimit =
1513                         cpu_to_be32(ni_scale_power_for_smc(power_boost_limit, scaling_factor));
1514
1515                 ret = rv770_copy_bytes_to_smc(rdev,
1516                                               (u16)(pi->state_table_start + offsetof(NISLANDS_SMC_STATETABLE, dpm2Params) +
1517                                                     offsetof(PP_NIslands_DPM2Parameters, TDPLimit)),
1518                                               (u8 *)(&smc_table->dpm2Params.TDPLimit),
1519                                               sizeof(u32) * 4, pi->sram_end);
1520                 if (ret)
1521                         return ret;
1522         }
1523
1524         return 0;
1525 }
1526
1527 int ni_copy_and_switch_arb_sets(struct radeon_device *rdev,
1528                                 u32 arb_freq_src, u32 arb_freq_dest)
1529 {
1530         u32 mc_arb_dram_timing;
1531         u32 mc_arb_dram_timing2;
1532         u32 burst_time;
1533         u32 mc_cg_config;
1534
1535         switch (arb_freq_src) {
1536         case MC_CG_ARB_FREQ_F0:
1537                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING);
1538                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
1539                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE0_MASK) >> STATE0_SHIFT;
1540                 break;
1541         case MC_CG_ARB_FREQ_F1:
1542                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_1);
1543                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_1);
1544                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE1_MASK) >> STATE1_SHIFT;
1545                 break;
1546         case MC_CG_ARB_FREQ_F2:
1547                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_2);
1548                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_2);
1549                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE2_MASK) >> STATE2_SHIFT;
1550                 break;
1551         case MC_CG_ARB_FREQ_F3:
1552                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_3);
1553                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_3);
1554                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE3_MASK) >> STATE3_SHIFT;
1555                 break;
1556         default:
1557                 return -EINVAL;
1558         }
1559
1560         switch (arb_freq_dest) {
1561         case MC_CG_ARB_FREQ_F0:
1562                 WREG32(MC_ARB_DRAM_TIMING, mc_arb_dram_timing);
1563                 WREG32(MC_ARB_DRAM_TIMING2, mc_arb_dram_timing2);
1564                 WREG32_P(MC_ARB_BURST_TIME, STATE0(burst_time), ~STATE0_MASK);
1565                 break;
1566         case MC_CG_ARB_FREQ_F1:
1567                 WREG32(MC_ARB_DRAM_TIMING_1, mc_arb_dram_timing);
1568                 WREG32(MC_ARB_DRAM_TIMING2_1, mc_arb_dram_timing2);
1569                 WREG32_P(MC_ARB_BURST_TIME, STATE1(burst_time), ~STATE1_MASK);
1570                 break;
1571         case MC_CG_ARB_FREQ_F2:
1572                 WREG32(MC_ARB_DRAM_TIMING_2, mc_arb_dram_timing);
1573                 WREG32(MC_ARB_DRAM_TIMING2_2, mc_arb_dram_timing2);
1574                 WREG32_P(MC_ARB_BURST_TIME, STATE2(burst_time), ~STATE2_MASK);
1575                 break;
1576         case MC_CG_ARB_FREQ_F3:
1577                 WREG32(MC_ARB_DRAM_TIMING_3, mc_arb_dram_timing);
1578                 WREG32(MC_ARB_DRAM_TIMING2_3, mc_arb_dram_timing2);
1579                 WREG32_P(MC_ARB_BURST_TIME, STATE3(burst_time), ~STATE3_MASK);
1580                 break;
1581         default:
1582                 return -EINVAL;
1583         }
1584
1585         mc_cg_config = RREG32(MC_CG_CONFIG) | 0x0000000F;
1586         WREG32(MC_CG_CONFIG, mc_cg_config);
1587         WREG32_P(MC_ARB_CG, CG_ARB_REQ(arb_freq_dest), ~CG_ARB_REQ_MASK);
1588
1589         return 0;
1590 }
1591
1592 static int ni_init_arb_table_index(struct radeon_device *rdev)
1593 {
1594         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1595         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1596         u32 tmp;
1597         int ret;
1598
1599         ret = rv770_read_smc_sram_dword(rdev, ni_pi->arb_table_start,
1600                                         &tmp, pi->sram_end);
1601         if (ret)
1602                 return ret;
1603
1604         tmp &= 0x00FFFFFF;
1605         tmp |= ((u32)MC_CG_ARB_FREQ_F1) << 24;
1606
1607         return rv770_write_smc_sram_dword(rdev, ni_pi->arb_table_start,
1608                                           tmp, pi->sram_end);
1609 }
1610
1611 static int ni_initial_switch_from_arb_f0_to_f1(struct radeon_device *rdev)
1612 {
1613         return ni_copy_and_switch_arb_sets(rdev, MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1);
1614 }
1615
1616 static int ni_force_switch_to_arb_f0(struct radeon_device *rdev)
1617 {
1618         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1619         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1620         u32 tmp;
1621         int ret;
1622
1623         ret = rv770_read_smc_sram_dword(rdev, ni_pi->arb_table_start,
1624                                         &tmp, pi->sram_end);
1625         if (ret)
1626                 return ret;
1627
1628         tmp = (tmp >> 24) & 0xff;
1629
1630         if (tmp == MC_CG_ARB_FREQ_F0)
1631                 return 0;
1632
1633         return ni_copy_and_switch_arb_sets(rdev, tmp, MC_CG_ARB_FREQ_F0);
1634 }
1635
1636 static int ni_populate_memory_timing_parameters(struct radeon_device *rdev,
1637                                                 struct rv7xx_pl *pl,
1638                                                 SMC_NIslands_MCArbDramTimingRegisterSet *arb_regs)
1639 {
1640         u32 dram_timing;
1641         u32 dram_timing2;
1642
1643         arb_regs->mc_arb_rfsh_rate =
1644                 (u8)rv770_calculate_memory_refresh_rate(rdev, pl->sclk);
1645
1646
1647         radeon_atom_set_engine_dram_timings(rdev,
1648                                             pl->sclk,
1649                                             pl->mclk);
1650
1651         dram_timing = RREG32(MC_ARB_DRAM_TIMING);
1652         dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
1653
1654         arb_regs->mc_arb_dram_timing  = cpu_to_be32(dram_timing);
1655         arb_regs->mc_arb_dram_timing2 = cpu_to_be32(dram_timing2);
1656
1657         return 0;
1658 }
1659
1660 static int ni_do_program_memory_timing_parameters(struct radeon_device *rdev,
1661                                                   struct radeon_ps *radeon_state,
1662                                                   unsigned int first_arb_set)
1663 {
1664         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1665         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1666         struct ni_ps *state = ni_get_ps(radeon_state);
1667         SMC_NIslands_MCArbDramTimingRegisterSet arb_regs = { 0 };
1668         int i, ret = 0;
1669
1670         for (i = 0; i < state->performance_level_count; i++) {
1671                 ret = ni_populate_memory_timing_parameters(rdev, &state->performance_levels[i], &arb_regs);
1672                 if (ret)
1673                         break;
1674
1675                 ret = rv770_copy_bytes_to_smc(rdev,
1676                                               (u16)(ni_pi->arb_table_start +
1677                                                     offsetof(SMC_NIslands_MCArbDramTimingRegisters, data) +
1678                                                     sizeof(SMC_NIslands_MCArbDramTimingRegisterSet) * (first_arb_set + i)),
1679                                               (u8 *)&arb_regs,
1680                                               (u16)sizeof(SMC_NIslands_MCArbDramTimingRegisterSet),
1681                                               pi->sram_end);
1682                 if (ret)
1683                         break;
1684         }
1685         return ret;
1686 }
1687
1688 static int ni_program_memory_timing_parameters(struct radeon_device *rdev,
1689                                                struct radeon_ps *radeon_new_state)
1690 {
1691         return ni_do_program_memory_timing_parameters(rdev, radeon_new_state,
1692                                                       NISLANDS_DRIVER_STATE_ARB_INDEX);
1693 }
1694
1695 static void ni_populate_initial_mvdd_value(struct radeon_device *rdev,
1696                                            struct NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1697 {
1698         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1699
1700         voltage->index = eg_pi->mvdd_high_index;
1701         voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1702 }
1703
1704 static int ni_populate_smc_initial_state(struct radeon_device *rdev,
1705                                          struct radeon_ps *radeon_initial_state,
1706                                          NISLANDS_SMC_STATETABLE *table)
1707 {
1708         struct ni_ps *initial_state = ni_get_ps(radeon_initial_state);
1709         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1710         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1711         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1712         u32 reg;
1713         int ret;
1714
1715         table->initialState.levels[0].mclk.vMPLL_AD_FUNC_CNTL =
1716                 cpu_to_be32(ni_pi->clock_registers.mpll_ad_func_cntl);
1717         table->initialState.levels[0].mclk.vMPLL_AD_FUNC_CNTL_2 =
1718                 cpu_to_be32(ni_pi->clock_registers.mpll_ad_func_cntl_2);
1719         table->initialState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL =
1720                 cpu_to_be32(ni_pi->clock_registers.mpll_dq_func_cntl);
1721         table->initialState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL_2 =
1722                 cpu_to_be32(ni_pi->clock_registers.mpll_dq_func_cntl_2);
1723         table->initialState.levels[0].mclk.vMCLK_PWRMGT_CNTL =
1724                 cpu_to_be32(ni_pi->clock_registers.mclk_pwrmgt_cntl);
1725         table->initialState.levels[0].mclk.vDLL_CNTL =
1726                 cpu_to_be32(ni_pi->clock_registers.dll_cntl);
1727         table->initialState.levels[0].mclk.vMPLL_SS =
1728                 cpu_to_be32(ni_pi->clock_registers.mpll_ss1);
1729         table->initialState.levels[0].mclk.vMPLL_SS2 =
1730                 cpu_to_be32(ni_pi->clock_registers.mpll_ss2);
1731         table->initialState.levels[0].mclk.mclk_value =
1732                 cpu_to_be32(initial_state->performance_levels[0].mclk);
1733
1734         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL =
1735                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl);
1736         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 =
1737                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_2);
1738         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 =
1739                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_3);
1740         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_4 =
1741                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_4);
1742         table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM =
1743                 cpu_to_be32(ni_pi->clock_registers.cg_spll_spread_spectrum);
1744         table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM_2 =
1745                 cpu_to_be32(ni_pi->clock_registers.cg_spll_spread_spectrum_2);
1746         table->initialState.levels[0].sclk.sclk_value =
1747                 cpu_to_be32(initial_state->performance_levels[0].sclk);
1748         table->initialState.levels[0].arbRefreshState =
1749                 NISLANDS_INITIAL_STATE_ARB_INDEX;
1750
1751         table->initialState.levels[0].ACIndex = 0;
1752
1753         ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1754                                         initial_state->performance_levels[0].vddc,
1755                                         &table->initialState.levels[0].vddc);
1756         if (!ret) {
1757                 u16 std_vddc;
1758
1759                 ret = ni_get_std_voltage_value(rdev,
1760                                                &table->initialState.levels[0].vddc,
1761                                                &std_vddc);
1762                 if (!ret)
1763                         ni_populate_std_voltage_value(rdev, std_vddc,
1764                                                       table->initialState.levels[0].vddc.index,
1765                                                       &table->initialState.levels[0].std_vddc);
1766         }
1767
1768         if (eg_pi->vddci_control)
1769                 ni_populate_voltage_value(rdev,
1770                                           &eg_pi->vddci_voltage_table,
1771                                           initial_state->performance_levels[0].vddci,
1772                                           &table->initialState.levels[0].vddci);
1773
1774         ni_populate_initial_mvdd_value(rdev, &table->initialState.levels[0].mvdd);
1775
1776         reg = CG_R(0xffff) | CG_L(0);
1777         table->initialState.levels[0].aT = cpu_to_be32(reg);
1778
1779         table->initialState.levels[0].bSP = cpu_to_be32(pi->dsp);
1780
1781         if (pi->boot_in_gen2)
1782                 table->initialState.levels[0].gen2PCIE = 1;
1783         else
1784                 table->initialState.levels[0].gen2PCIE = 0;
1785
1786         if (pi->mem_gddr5) {
1787                 table->initialState.levels[0].strobeMode =
1788                         cypress_get_strobe_mode_settings(rdev,
1789                                                          initial_state->performance_levels[0].mclk);
1790
1791                 if (initial_state->performance_levels[0].mclk > pi->mclk_edc_enable_threshold)
1792                         table->initialState.levels[0].mcFlags = NISLANDS_SMC_MC_EDC_RD_FLAG | NISLANDS_SMC_MC_EDC_WR_FLAG;
1793                 else
1794                         table->initialState.levels[0].mcFlags =  0;
1795         }
1796
1797         table->initialState.levelCount = 1;
1798
1799         table->initialState.flags |= PPSMC_SWSTATE_FLAG_DC;
1800
1801         table->initialState.levels[0].dpm2.MaxPS = 0;
1802         table->initialState.levels[0].dpm2.NearTDPDec = 0;
1803         table->initialState.levels[0].dpm2.AboveSafeInc = 0;
1804         table->initialState.levels[0].dpm2.BelowSafeInc = 0;
1805
1806         reg = MIN_POWER_MASK | MAX_POWER_MASK;
1807         table->initialState.levels[0].SQPowerThrottle = cpu_to_be32(reg);
1808
1809         reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
1810         table->initialState.levels[0].SQPowerThrottle_2 = cpu_to_be32(reg);
1811
1812         return 0;
1813 }
1814
1815 static int ni_populate_smc_acpi_state(struct radeon_device *rdev,
1816                                       NISLANDS_SMC_STATETABLE *table)
1817 {
1818         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1819         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1820         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1821         u32 mpll_ad_func_cntl   = ni_pi->clock_registers.mpll_ad_func_cntl;
1822         u32 mpll_ad_func_cntl_2 = ni_pi->clock_registers.mpll_ad_func_cntl_2;
1823         u32 mpll_dq_func_cntl   = ni_pi->clock_registers.mpll_dq_func_cntl;
1824         u32 mpll_dq_func_cntl_2 = ni_pi->clock_registers.mpll_dq_func_cntl_2;
1825         u32 spll_func_cntl      = ni_pi->clock_registers.cg_spll_func_cntl;
1826         u32 spll_func_cntl_2    = ni_pi->clock_registers.cg_spll_func_cntl_2;
1827         u32 spll_func_cntl_3    = ni_pi->clock_registers.cg_spll_func_cntl_3;
1828         u32 spll_func_cntl_4    = ni_pi->clock_registers.cg_spll_func_cntl_4;
1829         u32 mclk_pwrmgt_cntl    = ni_pi->clock_registers.mclk_pwrmgt_cntl;
1830         u32 dll_cntl            = ni_pi->clock_registers.dll_cntl;
1831         u32 reg;
1832         int ret;
1833
1834         table->ACPIState = table->initialState;
1835
1836         table->ACPIState.flags &= ~PPSMC_SWSTATE_FLAG_DC;
1837
1838         if (pi->acpi_vddc) {
1839                 ret = ni_populate_voltage_value(rdev,
1840                                                 &eg_pi->vddc_voltage_table,
1841                                                 pi->acpi_vddc, &table->ACPIState.levels[0].vddc);
1842                 if (!ret) {
1843                         u16 std_vddc;
1844
1845                         ret = ni_get_std_voltage_value(rdev,
1846                                                        &table->ACPIState.levels[0].vddc, &std_vddc);
1847                         if (!ret)
1848                                 ni_populate_std_voltage_value(rdev, std_vddc,
1849                                                               table->ACPIState.levels[0].vddc.index,
1850                                                               &table->ACPIState.levels[0].std_vddc);
1851                 }
1852
1853                 if (pi->pcie_gen2) {
1854                         if (pi->acpi_pcie_gen2)
1855                                 table->ACPIState.levels[0].gen2PCIE = 1;
1856                         else
1857                                 table->ACPIState.levels[0].gen2PCIE = 0;
1858                 } else {
1859                         table->ACPIState.levels[0].gen2PCIE = 0;
1860                 }
1861         } else {
1862                 ret = ni_populate_voltage_value(rdev,
1863                                                 &eg_pi->vddc_voltage_table,
1864                                                 pi->min_vddc_in_table,
1865                                                 &table->ACPIState.levels[0].vddc);
1866                 if (!ret) {
1867                         u16 std_vddc;
1868
1869                         ret = ni_get_std_voltage_value(rdev,
1870                                                        &table->ACPIState.levels[0].vddc,
1871                                                        &std_vddc);
1872                         if (!ret)
1873                                 ni_populate_std_voltage_value(rdev, std_vddc,
1874                                                               table->ACPIState.levels[0].vddc.index,
1875                                                               &table->ACPIState.levels[0].std_vddc);
1876                 }
1877                 table->ACPIState.levels[0].gen2PCIE = 0;
1878         }
1879
1880         if (eg_pi->acpi_vddci) {
1881                 if (eg_pi->vddci_control)
1882                         ni_populate_voltage_value(rdev,
1883                                                   &eg_pi->vddci_voltage_table,
1884                                                   eg_pi->acpi_vddci,
1885                                                   &table->ACPIState.levels[0].vddci);
1886         }
1887
1888
1889         mpll_ad_func_cntl &= ~PDNB;
1890
1891         mpll_ad_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN;
1892
1893         if (pi->mem_gddr5)
1894                 mpll_dq_func_cntl &= ~PDNB;
1895         mpll_dq_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN | BYPASS;
1896
1897
1898         mclk_pwrmgt_cntl |= (MRDCKA0_RESET |
1899                              MRDCKA1_RESET |
1900                              MRDCKB0_RESET |
1901                              MRDCKB1_RESET |
1902                              MRDCKC0_RESET |
1903                              MRDCKC1_RESET |
1904                              MRDCKD0_RESET |
1905                              MRDCKD1_RESET);
1906
1907         mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB |
1908                               MRDCKA1_PDNB |
1909                               MRDCKB0_PDNB |
1910                               MRDCKB1_PDNB |
1911                               MRDCKC0_PDNB |
1912                               MRDCKC1_PDNB |
1913                               MRDCKD0_PDNB |
1914                               MRDCKD1_PDNB);
1915
1916         dll_cntl |= (MRDCKA0_BYPASS |
1917                      MRDCKA1_BYPASS |
1918                      MRDCKB0_BYPASS |
1919                      MRDCKB1_BYPASS |
1920                      MRDCKC0_BYPASS |
1921                      MRDCKC1_BYPASS |
1922                      MRDCKD0_BYPASS |
1923                      MRDCKD1_BYPASS);
1924
1925         spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
1926         spll_func_cntl_2 |= SCLK_MUX_SEL(4);
1927
1928         table->ACPIState.levels[0].mclk.vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
1929         table->ACPIState.levels[0].mclk.vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
1930         table->ACPIState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
1931         table->ACPIState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
1932         table->ACPIState.levels[0].mclk.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
1933         table->ACPIState.levels[0].mclk.vDLL_CNTL = cpu_to_be32(dll_cntl);
1934
1935         table->ACPIState.levels[0].mclk.mclk_value = 0;
1936
1937         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl);
1938         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2);
1939         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3);
1940         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(spll_func_cntl_4);
1941
1942         table->ACPIState.levels[0].sclk.sclk_value = 0;
1943
1944         ni_populate_mvdd_value(rdev, 0, &table->ACPIState.levels[0].mvdd);
1945
1946         if (eg_pi->dynamic_ac_timing)
1947                 table->ACPIState.levels[0].ACIndex = 1;
1948
1949         table->ACPIState.levels[0].dpm2.MaxPS = 0;
1950         table->ACPIState.levels[0].dpm2.NearTDPDec = 0;
1951         table->ACPIState.levels[0].dpm2.AboveSafeInc = 0;
1952         table->ACPIState.levels[0].dpm2.BelowSafeInc = 0;
1953
1954         reg = MIN_POWER_MASK | MAX_POWER_MASK;
1955         table->ACPIState.levels[0].SQPowerThrottle = cpu_to_be32(reg);
1956
1957         reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
1958         table->ACPIState.levels[0].SQPowerThrottle_2 = cpu_to_be32(reg);
1959
1960         return 0;
1961 }
1962
1963 static int ni_init_smc_table(struct radeon_device *rdev)
1964 {
1965         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1966         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1967         int ret;
1968         struct radeon_ps *radeon_boot_state = rdev->pm.dpm.boot_ps;
1969         NISLANDS_SMC_STATETABLE *table = &ni_pi->smc_statetable;
1970
1971         memset(table, 0, sizeof(NISLANDS_SMC_STATETABLE));
1972
1973         ni_populate_smc_voltage_tables(rdev, table);
1974
1975         switch (rdev->pm.int_thermal_type) {
1976         case THERMAL_TYPE_NI:
1977         case THERMAL_TYPE_EMC2103_WITH_INTERNAL:
1978                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_INTERNAL;
1979                 break;
1980         case THERMAL_TYPE_NONE:
1981                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_NONE;
1982                 break;
1983         default:
1984                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL;
1985                 break;
1986         }
1987
1988         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC)
1989                 table->systemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
1990
1991         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_REGULATOR_HOT)
1992                 table->systemFlags |= PPSMC_SYSTEMFLAG_REGULATOR_HOT;
1993
1994         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC)
1995                 table->systemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
1996
1997         if (pi->mem_gddr5)
1998                 table->systemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
1999
2000         ret = ni_populate_smc_initial_state(rdev, radeon_boot_state, table);
2001         if (ret)
2002                 return ret;
2003
2004         ret = ni_populate_smc_acpi_state(rdev, table);
2005         if (ret)
2006                 return ret;
2007
2008         table->driverState = table->initialState;
2009
2010         table->ULVState = table->initialState;
2011
2012         ret = ni_do_program_memory_timing_parameters(rdev, radeon_boot_state,
2013                                                      NISLANDS_INITIAL_STATE_ARB_INDEX);
2014         if (ret)
2015                 return ret;
2016
2017         return rv770_copy_bytes_to_smc(rdev, pi->state_table_start, (u8 *)table,
2018                                        sizeof(NISLANDS_SMC_STATETABLE), pi->sram_end);
2019 }
2020
2021 static int ni_calculate_sclk_params(struct radeon_device *rdev,
2022                                     u32 engine_clock,
2023                                     NISLANDS_SMC_SCLK_VALUE *sclk)
2024 {
2025         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2026         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2027         struct atom_clock_dividers dividers;
2028         u32 spll_func_cntl = ni_pi->clock_registers.cg_spll_func_cntl;
2029         u32 spll_func_cntl_2 = ni_pi->clock_registers.cg_spll_func_cntl_2;
2030         u32 spll_func_cntl_3 = ni_pi->clock_registers.cg_spll_func_cntl_3;
2031         u32 spll_func_cntl_4 = ni_pi->clock_registers.cg_spll_func_cntl_4;
2032         u32 cg_spll_spread_spectrum = ni_pi->clock_registers.cg_spll_spread_spectrum;
2033         u32 cg_spll_spread_spectrum_2 = ni_pi->clock_registers.cg_spll_spread_spectrum_2;
2034         u64 tmp;
2035         u32 reference_clock = rdev->clock.spll.reference_freq;
2036         u32 reference_divider;
2037         u32 fbdiv;
2038         int ret;
2039
2040         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
2041                                              engine_clock, false, &dividers);
2042         if (ret)
2043                 return ret;
2044
2045         reference_divider = 1 + dividers.ref_div;
2046
2047
2048         tmp = (u64) engine_clock * reference_divider * dividers.post_div * 16834;
2049         do_div(tmp, reference_clock);
2050         fbdiv = (u32) tmp;
2051
2052         spll_func_cntl &= ~(SPLL_PDIV_A_MASK | SPLL_REF_DIV_MASK);
2053         spll_func_cntl |= SPLL_REF_DIV(dividers.ref_div);
2054         spll_func_cntl |= SPLL_PDIV_A(dividers.post_div);
2055
2056         spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
2057         spll_func_cntl_2 |= SCLK_MUX_SEL(2);
2058
2059         spll_func_cntl_3 &= ~SPLL_FB_DIV_MASK;
2060         spll_func_cntl_3 |= SPLL_FB_DIV(fbdiv);
2061         spll_func_cntl_3 |= SPLL_DITHEN;
2062
2063         if (pi->sclk_ss) {
2064                 struct radeon_atom_ss ss;
2065                 u32 vco_freq = engine_clock * dividers.post_div;
2066
2067                 if (radeon_atombios_get_asic_ss_info(rdev, &ss,
2068                                                      ASIC_INTERNAL_ENGINE_SS, vco_freq)) {
2069                         u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate);
2070                         u32 clk_v = 4 * ss.percentage * fbdiv / (clk_s * 10000);
2071
2072                         cg_spll_spread_spectrum &= ~CLK_S_MASK;
2073                         cg_spll_spread_spectrum |= CLK_S(clk_s);
2074                         cg_spll_spread_spectrum |= SSEN;
2075
2076                         cg_spll_spread_spectrum_2 &= ~CLK_V_MASK;
2077                         cg_spll_spread_spectrum_2 |= CLK_V(clk_v);
2078                 }
2079         }
2080
2081         sclk->sclk_value = engine_clock;
2082         sclk->vCG_SPLL_FUNC_CNTL = spll_func_cntl;
2083         sclk->vCG_SPLL_FUNC_CNTL_2 = spll_func_cntl_2;
2084         sclk->vCG_SPLL_FUNC_CNTL_3 = spll_func_cntl_3;
2085         sclk->vCG_SPLL_FUNC_CNTL_4 = spll_func_cntl_4;
2086         sclk->vCG_SPLL_SPREAD_SPECTRUM = cg_spll_spread_spectrum;
2087         sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cg_spll_spread_spectrum_2;
2088
2089         return 0;
2090 }
2091
2092 static int ni_populate_sclk_value(struct radeon_device *rdev,
2093                                   u32 engine_clock,
2094                                   NISLANDS_SMC_SCLK_VALUE *sclk)
2095 {
2096         NISLANDS_SMC_SCLK_VALUE sclk_tmp;
2097         int ret;
2098
2099         ret = ni_calculate_sclk_params(rdev, engine_clock, &sclk_tmp);
2100         if (!ret) {
2101                 sclk->sclk_value = cpu_to_be32(sclk_tmp.sclk_value);
2102                 sclk->vCG_SPLL_FUNC_CNTL = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL);
2103                 sclk->vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_2);
2104                 sclk->vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_3);
2105                 sclk->vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_4);
2106                 sclk->vCG_SPLL_SPREAD_SPECTRUM = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM);
2107                 sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM_2);
2108         }
2109
2110         return ret;
2111 }
2112
2113 static int ni_init_smc_spll_table(struct radeon_device *rdev)
2114 {
2115         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2116         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2117         SMC_NISLANDS_SPLL_DIV_TABLE *spll_table;
2118         NISLANDS_SMC_SCLK_VALUE sclk_params;
2119         u32 fb_div;
2120         u32 p_div;
2121         u32 clk_s;
2122         u32 clk_v;
2123         u32 sclk = 0;
2124         int i, ret;
2125         u32 tmp;
2126
2127         if (ni_pi->spll_table_start == 0)
2128                 return -EINVAL;
2129
2130         spll_table = kzalloc(sizeof(SMC_NISLANDS_SPLL_DIV_TABLE), GFP_KERNEL);
2131         if (spll_table == NULL)
2132                 return -ENOMEM;
2133
2134         for (i = 0; i < 256; i++) {
2135                 ret = ni_calculate_sclk_params(rdev, sclk, &sclk_params);
2136                 if (ret)
2137                         break;
2138
2139                 p_div = (sclk_params.vCG_SPLL_FUNC_CNTL & SPLL_PDIV_A_MASK) >> SPLL_PDIV_A_SHIFT;
2140                 fb_div = (sclk_params.vCG_SPLL_FUNC_CNTL_3 & SPLL_FB_DIV_MASK) >> SPLL_FB_DIV_SHIFT;
2141                 clk_s = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM & CLK_S_MASK) >> CLK_S_SHIFT;
2142                 clk_v = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM_2 & CLK_V_MASK) >> CLK_V_SHIFT;
2143
2144                 fb_div &= ~0x00001FFF;
2145                 fb_div >>= 1;
2146                 clk_v >>= 6;
2147
2148                 if (p_div & ~(SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT))
2149                         ret = -EINVAL;
2150
2151                 if (clk_s & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT))
2152                         ret = -EINVAL;
2153
2154                 if (clk_s & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT))
2155                         ret = -EINVAL;
2156
2157                 if (clk_v & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT))
2158                         ret = -EINVAL;
2159
2160                 if (ret)
2161                         break;
2162
2163                 tmp = ((fb_div << SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_MASK) |
2164                         ((p_div << SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_MASK);
2165                 spll_table->freq[i] = cpu_to_be32(tmp);
2166
2167                 tmp = ((clk_v << SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_MASK) |
2168                         ((clk_s << SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK);
2169                 spll_table->ss[i] = cpu_to_be32(tmp);
2170
2171                 sclk += 512;
2172         }
2173
2174         if (!ret)
2175                 ret = rv770_copy_bytes_to_smc(rdev, ni_pi->spll_table_start, (u8 *)spll_table,
2176                                               sizeof(SMC_NISLANDS_SPLL_DIV_TABLE), pi->sram_end);
2177
2178         kfree(spll_table);
2179
2180         return ret;
2181 }
2182
2183 static int ni_populate_mclk_value(struct radeon_device *rdev,
2184                                   u32 engine_clock,
2185                                   u32 memory_clock,
2186                                   NISLANDS_SMC_MCLK_VALUE *mclk,
2187                                   bool strobe_mode,
2188                                   bool dll_state_on)
2189 {
2190         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2191         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2192         u32 mpll_ad_func_cntl = ni_pi->clock_registers.mpll_ad_func_cntl;
2193         u32 mpll_ad_func_cntl_2 = ni_pi->clock_registers.mpll_ad_func_cntl_2;
2194         u32 mpll_dq_func_cntl = ni_pi->clock_registers.mpll_dq_func_cntl;
2195         u32 mpll_dq_func_cntl_2 = ni_pi->clock_registers.mpll_dq_func_cntl_2;
2196         u32 mclk_pwrmgt_cntl = ni_pi->clock_registers.mclk_pwrmgt_cntl;
2197         u32 dll_cntl = ni_pi->clock_registers.dll_cntl;
2198         u32 mpll_ss1 = ni_pi->clock_registers.mpll_ss1;
2199         u32 mpll_ss2 = ni_pi->clock_registers.mpll_ss2;
2200         struct atom_clock_dividers dividers;
2201         u32 ibias;
2202         u32 dll_speed;
2203         int ret;
2204         u32 mc_seq_misc7;
2205
2206         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM,
2207                                              memory_clock, strobe_mode, &dividers);
2208         if (ret)
2209                 return ret;
2210
2211         if (!strobe_mode) {
2212                 mc_seq_misc7 = RREG32(MC_SEQ_MISC7);
2213
2214                 if (mc_seq_misc7 & 0x8000000)
2215                         dividers.post_div = 1;
2216         }
2217
2218         ibias = cypress_map_clkf_to_ibias(rdev, dividers.whole_fb_div);
2219
2220         mpll_ad_func_cntl &= ~(CLKR_MASK |
2221                                YCLK_POST_DIV_MASK |
2222                                CLKF_MASK |
2223                                CLKFRAC_MASK |
2224                                IBIAS_MASK);
2225         mpll_ad_func_cntl |= CLKR(dividers.ref_div);
2226         mpll_ad_func_cntl |= YCLK_POST_DIV(dividers.post_div);
2227         mpll_ad_func_cntl |= CLKF(dividers.whole_fb_div);
2228         mpll_ad_func_cntl |= CLKFRAC(dividers.frac_fb_div);
2229         mpll_ad_func_cntl |= IBIAS(ibias);
2230
2231         if (dividers.vco_mode)
2232                 mpll_ad_func_cntl_2 |= VCO_MODE;
2233         else
2234                 mpll_ad_func_cntl_2 &= ~VCO_MODE;
2235
2236         if (pi->mem_gddr5) {
2237                 mpll_dq_func_cntl &= ~(CLKR_MASK |
2238                                        YCLK_POST_DIV_MASK |
2239                                        CLKF_MASK |
2240                                        CLKFRAC_MASK |
2241                                        IBIAS_MASK);
2242                 mpll_dq_func_cntl |= CLKR(dividers.ref_div);
2243                 mpll_dq_func_cntl |= YCLK_POST_DIV(dividers.post_div);
2244                 mpll_dq_func_cntl |= CLKF(dividers.whole_fb_div);
2245                 mpll_dq_func_cntl |= CLKFRAC(dividers.frac_fb_div);
2246                 mpll_dq_func_cntl |= IBIAS(ibias);
2247
2248                 if (strobe_mode)
2249                         mpll_dq_func_cntl &= ~PDNB;
2250                 else
2251                         mpll_dq_func_cntl |= PDNB;
2252
2253                 if (dividers.vco_mode)
2254                         mpll_dq_func_cntl_2 |= VCO_MODE;
2255                 else
2256                         mpll_dq_func_cntl_2 &= ~VCO_MODE;
2257         }
2258
2259         if (pi->mclk_ss) {
2260                 struct radeon_atom_ss ss;
2261                 u32 vco_freq = memory_clock * dividers.post_div;
2262
2263                 if (radeon_atombios_get_asic_ss_info(rdev, &ss,
2264                                                      ASIC_INTERNAL_MEMORY_SS, vco_freq)) {
2265                         u32 reference_clock = rdev->clock.mpll.reference_freq;
2266                         u32 decoded_ref = rv740_get_decoded_reference_divider(dividers.ref_div);
2267                         u32 clk_s = reference_clock * 5 / (decoded_ref * ss.rate);
2268                         u32 clk_v = ss.percentage *
2269                                 (0x4000 * dividers.whole_fb_div + 0x800 * dividers.frac_fb_div) / (clk_s * 625);
2270
2271                         mpll_ss1 &= ~CLKV_MASK;
2272                         mpll_ss1 |= CLKV(clk_v);
2273
2274                         mpll_ss2 &= ~CLKS_MASK;
2275                         mpll_ss2 |= CLKS(clk_s);
2276                 }
2277         }
2278
2279         dll_speed = rv740_get_dll_speed(pi->mem_gddr5,
2280                                         memory_clock);
2281
2282         mclk_pwrmgt_cntl &= ~DLL_SPEED_MASK;
2283         mclk_pwrmgt_cntl |= DLL_SPEED(dll_speed);
2284         if (dll_state_on)
2285                 mclk_pwrmgt_cntl |= (MRDCKA0_PDNB |
2286                                      MRDCKA1_PDNB |
2287                                      MRDCKB0_PDNB |
2288                                      MRDCKB1_PDNB |
2289                                      MRDCKC0_PDNB |
2290                                      MRDCKC1_PDNB |
2291                                      MRDCKD0_PDNB |
2292                                      MRDCKD1_PDNB);
2293         else
2294                 mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB |
2295                                       MRDCKA1_PDNB |
2296                                       MRDCKB0_PDNB |
2297                                       MRDCKB1_PDNB |
2298                                       MRDCKC0_PDNB |
2299                                       MRDCKC1_PDNB |
2300                                       MRDCKD0_PDNB |
2301                                       MRDCKD1_PDNB);
2302
2303
2304         mclk->mclk_value = cpu_to_be32(memory_clock);
2305         mclk->vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
2306         mclk->vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
2307         mclk->vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
2308         mclk->vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
2309         mclk->vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
2310         mclk->vDLL_CNTL = cpu_to_be32(dll_cntl);
2311         mclk->vMPLL_SS = cpu_to_be32(mpll_ss1);
2312         mclk->vMPLL_SS2 = cpu_to_be32(mpll_ss2);
2313
2314         return 0;
2315 }
2316
2317 static void ni_populate_smc_sp(struct radeon_device *rdev,
2318                                struct radeon_ps *radeon_state,
2319                                NISLANDS_SMC_SWSTATE *smc_state)
2320 {
2321         struct ni_ps *ps = ni_get_ps(radeon_state);
2322         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2323         int i;
2324
2325         for (i = 0; i < ps->performance_level_count - 1; i++)
2326                 smc_state->levels[i].bSP = cpu_to_be32(pi->dsp);
2327
2328         smc_state->levels[ps->performance_level_count - 1].bSP =
2329                 cpu_to_be32(pi->psp);
2330 }
2331
2332 static int ni_convert_power_level_to_smc(struct radeon_device *rdev,
2333                                          struct rv7xx_pl *pl,
2334                                          NISLANDS_SMC_HW_PERFORMANCE_LEVEL *level)
2335 {
2336         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2337         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2338         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2339         int ret;
2340         bool dll_state_on;
2341         u16 std_vddc;
2342         u32 tmp = RREG32(DC_STUTTER_CNTL);
2343
2344         level->gen2PCIE = pi->pcie_gen2 ?
2345                 ((pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? 1 : 0) : 0;
2346
2347         ret = ni_populate_sclk_value(rdev, pl->sclk, &level->sclk);
2348         if (ret)
2349                 return ret;
2350
2351         level->mcFlags =  0;
2352         if (pi->mclk_stutter_mode_threshold &&
2353             (pl->mclk <= pi->mclk_stutter_mode_threshold) &&
2354             !eg_pi->uvd_enabled &&
2355             (tmp & DC_STUTTER_ENABLE_A) &&
2356             (tmp & DC_STUTTER_ENABLE_B))
2357                 level->mcFlags |= NISLANDS_SMC_MC_STUTTER_EN;
2358
2359         if (pi->mem_gddr5) {
2360                 if (pl->mclk > pi->mclk_edc_enable_threshold)
2361                         level->mcFlags |= NISLANDS_SMC_MC_EDC_RD_FLAG;
2362                 if (pl->mclk > eg_pi->mclk_edc_wr_enable_threshold)
2363                         level->mcFlags |= NISLANDS_SMC_MC_EDC_WR_FLAG;
2364
2365                 level->strobeMode = cypress_get_strobe_mode_settings(rdev, pl->mclk);
2366
2367                 if (level->strobeMode & NISLANDS_SMC_STROBE_ENABLE) {
2368                         if (cypress_get_mclk_frequency_ratio(rdev, pl->mclk, true) >=
2369                             ((RREG32(MC_SEQ_MISC7) >> 16) & 0xf))
2370                                 dll_state_on = ((RREG32(MC_SEQ_MISC5) >> 1) & 0x1) ? true : false;
2371                         else
2372                                 dll_state_on = ((RREG32(MC_SEQ_MISC6) >> 1) & 0x1) ? true : false;
2373                 } else {
2374                         dll_state_on = false;
2375                         if (pl->mclk > ni_pi->mclk_rtt_mode_threshold)
2376                                 level->mcFlags |= NISLANDS_SMC_MC_RTT_ENABLE;
2377                 }
2378
2379                 ret = ni_populate_mclk_value(rdev, pl->sclk, pl->mclk,
2380                                              &level->mclk,
2381                                              (level->strobeMode & NISLANDS_SMC_STROBE_ENABLE) != 0,
2382                                              dll_state_on);
2383         } else
2384                 ret = ni_populate_mclk_value(rdev, pl->sclk, pl->mclk, &level->mclk, 1, 1);
2385
2386         if (ret)
2387                 return ret;
2388
2389         ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
2390                                         pl->vddc, &level->vddc);
2391         if (ret)
2392                 return ret;
2393
2394         ret = ni_get_std_voltage_value(rdev, &level->vddc, &std_vddc);
2395         if (ret)
2396                 return ret;
2397
2398         ni_populate_std_voltage_value(rdev, std_vddc,
2399                                       level->vddc.index, &level->std_vddc);
2400
2401         if (eg_pi->vddci_control) {
2402                 ret = ni_populate_voltage_value(rdev, &eg_pi->vddci_voltage_table,
2403                                                 pl->vddci, &level->vddci);
2404                 if (ret)
2405                         return ret;
2406         }
2407
2408         ni_populate_mvdd_value(rdev, pl->mclk, &level->mvdd);
2409
2410         return ret;
2411 }
2412
2413 static int ni_populate_smc_t(struct radeon_device *rdev,
2414                              struct radeon_ps *radeon_state,
2415                              NISLANDS_SMC_SWSTATE *smc_state)
2416 {
2417         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2418         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2419         struct ni_ps *state = ni_get_ps(radeon_state);
2420         u32 a_t;
2421         u32 t_l, t_h;
2422         u32 high_bsp;
2423         int i, ret;
2424
2425         if (state->performance_level_count >= 9)
2426                 return -EINVAL;
2427
2428         if (state->performance_level_count < 2) {
2429                 a_t = CG_R(0xffff) | CG_L(0);
2430                 smc_state->levels[0].aT = cpu_to_be32(a_t);
2431                 return 0;
2432         }
2433
2434         smc_state->levels[0].aT = cpu_to_be32(0);
2435
2436         for (i = 0; i <= state->performance_level_count - 2; i++) {
2437                 if (eg_pi->uvd_enabled)
2438                         ret = r600_calculate_at(
2439                                 1000 * (i * (eg_pi->smu_uvd_hs ? 2 : 8) + 2),
2440                                 100 * R600_AH_DFLT,
2441                                 state->performance_levels[i + 1].sclk,
2442                                 state->performance_levels[i].sclk,
2443                                 &t_l,
2444                                 &t_h);
2445                 else
2446                         ret = r600_calculate_at(
2447                                 1000 * (i + 1),
2448                                 100 * R600_AH_DFLT,
2449                                 state->performance_levels[i + 1].sclk,
2450                                 state->performance_levels[i].sclk,
2451                                 &t_l,
2452                                 &t_h);
2453
2454                 if (ret) {
2455                         t_h = (i + 1) * 1000 - 50 * R600_AH_DFLT;
2456                         t_l = (i + 1) * 1000 + 50 * R600_AH_DFLT;
2457                 }
2458
2459                 a_t = be32_to_cpu(smc_state->levels[i].aT) & ~CG_R_MASK;
2460                 a_t |= CG_R(t_l * pi->bsp / 20000);
2461                 smc_state->levels[i].aT = cpu_to_be32(a_t);
2462
2463                 high_bsp = (i == state->performance_level_count - 2) ?
2464                         pi->pbsp : pi->bsp;
2465
2466                 a_t = CG_R(0xffff) | CG_L(t_h * high_bsp / 20000);
2467                 smc_state->levels[i + 1].aT = cpu_to_be32(a_t);
2468         }
2469
2470         return 0;
2471 }
2472
2473 static int ni_populate_power_containment_values(struct radeon_device *rdev,
2474                                                 struct radeon_ps *radeon_state,
2475                                                 NISLANDS_SMC_SWSTATE *smc_state)
2476 {
2477         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2478         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2479         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2480         struct ni_ps *state = ni_get_ps(radeon_state);
2481         u32 prev_sclk;
2482         u32 max_sclk;
2483         u32 min_sclk;
2484         int i, ret;
2485         u32 tdp_limit;
2486         u32 near_tdp_limit;
2487         u32 power_boost_limit;
2488         u8 max_ps_percent;
2489
2490         if (ni_pi->enable_power_containment == false)
2491                 return 0;
2492
2493         if (state->performance_level_count == 0)
2494                 return -EINVAL;
2495
2496         if (smc_state->levelCount != state->performance_level_count)
2497                 return -EINVAL;
2498
2499         ret = ni_calculate_adjusted_tdp_limits(rdev,
2500                                                false, /* ??? */
2501                                                rdev->pm.dpm.tdp_adjustment,
2502                                                &tdp_limit,
2503                                                &near_tdp_limit);
2504         if (ret)
2505                 return ret;
2506
2507         power_boost_limit = ni_calculate_power_boost_limit(rdev, radeon_state, near_tdp_limit);
2508
2509         ret = rv770_write_smc_sram_dword(rdev,
2510                                          pi->state_table_start +
2511                                          offsetof(NISLANDS_SMC_STATETABLE, dpm2Params) +
2512                                          offsetof(PP_NIslands_DPM2Parameters, PowerBoostLimit),
2513                                          ni_scale_power_for_smc(power_boost_limit, ni_get_smc_power_scaling_factor(rdev)),
2514                                          pi->sram_end);
2515         if (ret)
2516                 power_boost_limit = 0;
2517
2518         smc_state->levels[0].dpm2.MaxPS = 0;
2519         smc_state->levels[0].dpm2.NearTDPDec = 0;
2520         smc_state->levels[0].dpm2.AboveSafeInc = 0;
2521         smc_state->levels[0].dpm2.BelowSafeInc = 0;
2522         smc_state->levels[0].stateFlags |= power_boost_limit ? PPSMC_STATEFLAG_POWERBOOST : 0;
2523
2524         for (i = 1; i < state->performance_level_count; i++) {
2525                 prev_sclk = state->performance_levels[i-1].sclk;
2526                 max_sclk  = state->performance_levels[i].sclk;
2527                 max_ps_percent = (i != (state->performance_level_count - 1)) ?
2528                         NISLANDS_DPM2_MAXPS_PERCENT_M : NISLANDS_DPM2_MAXPS_PERCENT_H;
2529
2530                 if (max_sclk < prev_sclk)
2531                         return -EINVAL;
2532
2533                 if ((max_ps_percent == 0) || (prev_sclk == max_sclk) || eg_pi->uvd_enabled)
2534                         min_sclk = max_sclk;
2535                 else if (1 == i)
2536                         min_sclk = prev_sclk;
2537                 else
2538                         min_sclk = (prev_sclk * (u32)max_ps_percent) / 100;
2539
2540                 if (min_sclk < state->performance_levels[0].sclk)
2541                         min_sclk = state->performance_levels[0].sclk;
2542
2543                 if (min_sclk == 0)
2544                         return -EINVAL;
2545
2546                 smc_state->levels[i].dpm2.MaxPS =
2547                         (u8)((NISLANDS_DPM2_MAX_PULSE_SKIP * (max_sclk - min_sclk)) / max_sclk);
2548                 smc_state->levels[i].dpm2.NearTDPDec = NISLANDS_DPM2_NEAR_TDP_DEC;
2549                 smc_state->levels[i].dpm2.AboveSafeInc = NISLANDS_DPM2_ABOVE_SAFE_INC;
2550                 smc_state->levels[i].dpm2.BelowSafeInc = NISLANDS_DPM2_BELOW_SAFE_INC;
2551                 smc_state->levels[i].stateFlags |=
2552                         ((i != (state->performance_level_count - 1)) && power_boost_limit) ?
2553                         PPSMC_STATEFLAG_POWERBOOST : 0;
2554         }
2555
2556         return 0;
2557 }
2558
2559 static int ni_populate_sq_ramping_values(struct radeon_device *rdev,
2560                                          struct radeon_ps *radeon_state,
2561                                          NISLANDS_SMC_SWSTATE *smc_state)
2562 {
2563         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2564         struct ni_ps *state = ni_get_ps(radeon_state);
2565         u32 sq_power_throttle;
2566         u32 sq_power_throttle2;
2567         bool enable_sq_ramping = ni_pi->enable_sq_ramping;
2568         int i;
2569
2570         if (state->performance_level_count == 0)
2571                 return -EINVAL;
2572
2573         if (smc_state->levelCount != state->performance_level_count)
2574                 return -EINVAL;
2575
2576         if (rdev->pm.dpm.sq_ramping_threshold == 0)
2577                 return -EINVAL;
2578
2579         if (NISLANDS_DPM2_SQ_RAMP_MAX_POWER > (MAX_POWER_MASK >> MAX_POWER_SHIFT))
2580                 enable_sq_ramping = false;
2581
2582         if (NISLANDS_DPM2_SQ_RAMP_MIN_POWER > (MIN_POWER_MASK >> MIN_POWER_SHIFT))
2583                 enable_sq_ramping = false;
2584
2585         if (NISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA > (MAX_POWER_DELTA_MASK >> MAX_POWER_DELTA_SHIFT))
2586                 enable_sq_ramping = false;
2587
2588         if (NISLANDS_DPM2_SQ_RAMP_STI_SIZE > (STI_SIZE_MASK >> STI_SIZE_SHIFT))
2589                 enable_sq_ramping = false;
2590
2591         if (NISLANDS_DPM2_SQ_RAMP_LTI_RATIO > (LTI_RATIO_MASK >> LTI_RATIO_SHIFT))
2592                 enable_sq_ramping = false;
2593
2594         for (i = 0; i < state->performance_level_count; i++) {
2595                 sq_power_throttle  = 0;
2596                 sq_power_throttle2 = 0;
2597
2598                 if ((state->performance_levels[i].sclk >= rdev->pm.dpm.sq_ramping_threshold) &&
2599                     enable_sq_ramping) {
2600                         sq_power_throttle |= MAX_POWER(NISLANDS_DPM2_SQ_RAMP_MAX_POWER);
2601                         sq_power_throttle |= MIN_POWER(NISLANDS_DPM2_SQ_RAMP_MIN_POWER);
2602                         sq_power_throttle2 |= MAX_POWER_DELTA(NISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA);
2603                         sq_power_throttle2 |= STI_SIZE(NISLANDS_DPM2_SQ_RAMP_STI_SIZE);
2604                         sq_power_throttle2 |= LTI_RATIO(NISLANDS_DPM2_SQ_RAMP_LTI_RATIO);
2605                 } else {
2606                         sq_power_throttle |= MAX_POWER_MASK | MIN_POWER_MASK;
2607                         sq_power_throttle2 |= MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
2608                 }
2609
2610                 smc_state->levels[i].SQPowerThrottle   = cpu_to_be32(sq_power_throttle);
2611                 smc_state->levels[i].SQPowerThrottle_2 = cpu_to_be32(sq_power_throttle2);
2612         }
2613
2614         return 0;
2615 }
2616
2617 static int ni_enable_power_containment(struct radeon_device *rdev,
2618                                        struct radeon_ps *radeon_new_state,
2619                                        bool enable)
2620 {
2621         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2622         PPSMC_Result smc_result;
2623         int ret = 0;
2624
2625         if (ni_pi->enable_power_containment) {
2626                 if (enable) {
2627                         if (!r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) {
2628                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_TDPClampingActive);
2629                                 if (smc_result != PPSMC_Result_OK) {
2630                                         ret = -EINVAL;
2631                                         ni_pi->pc_enabled = false;
2632                                 } else {
2633                                         ni_pi->pc_enabled = true;
2634                                 }
2635                         }
2636                 } else {
2637                         smc_result = rv770_send_msg_to_smc(rdev, PPSMC_TDPClampingInactive);
2638                         if (smc_result != PPSMC_Result_OK)
2639                                 ret = -EINVAL;
2640                         ni_pi->pc_enabled = false;
2641                 }
2642         }
2643
2644         return ret;
2645 }
2646
2647 static int ni_convert_power_state_to_smc(struct radeon_device *rdev,
2648                                          struct radeon_ps *radeon_state,
2649                                          NISLANDS_SMC_SWSTATE *smc_state)
2650 {
2651         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2652         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2653         struct ni_ps *state = ni_get_ps(radeon_state);
2654         int i, ret;
2655         u32 threshold = state->performance_levels[state->performance_level_count - 1].sclk * 100 / 100;
2656
2657         if (!(radeon_state->caps & ATOM_PPLIB_DISALLOW_ON_DC))
2658                 smc_state->flags |= PPSMC_SWSTATE_FLAG_DC;
2659
2660         smc_state->levelCount = 0;
2661
2662         if (state->performance_level_count > NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE)
2663                 return -EINVAL;
2664
2665         for (i = 0; i < state->performance_level_count; i++) {
2666                 ret = ni_convert_power_level_to_smc(rdev, &state->performance_levels[i],
2667                                                     &smc_state->levels[i]);
2668                 smc_state->levels[i].arbRefreshState =
2669                         (u8)(NISLANDS_DRIVER_STATE_ARB_INDEX + i);
2670
2671                 if (ret)
2672                         return ret;
2673
2674                 if (ni_pi->enable_power_containment)
2675                         smc_state->levels[i].displayWatermark =
2676                                 (state->performance_levels[i].sclk < threshold) ?
2677                                 PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH;
2678                 else
2679                         smc_state->levels[i].displayWatermark = (i < 2) ?
2680                                 PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH;
2681
2682                 if (eg_pi->dynamic_ac_timing)
2683                         smc_state->levels[i].ACIndex = NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i;
2684                 else
2685                         smc_state->levels[i].ACIndex = 0;
2686
2687                 smc_state->levelCount++;
2688         }
2689
2690         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_watermark_threshold,
2691                                       cpu_to_be32(threshold / 512));
2692
2693         ni_populate_smc_sp(rdev, radeon_state, smc_state);
2694
2695         ret = ni_populate_power_containment_values(rdev, radeon_state, smc_state);
2696         if (ret)
2697                 ni_pi->enable_power_containment = false;
2698
2699         ret = ni_populate_sq_ramping_values(rdev, radeon_state, smc_state);
2700         if (ret)
2701                 ni_pi->enable_sq_ramping = false;
2702
2703         return ni_populate_smc_t(rdev, radeon_state, smc_state);
2704 }
2705
2706 static int ni_upload_sw_state(struct radeon_device *rdev,
2707                               struct radeon_ps *radeon_new_state)
2708 {
2709         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2710         u16 address = pi->state_table_start +
2711                 offsetof(NISLANDS_SMC_STATETABLE, driverState);
2712         u16 state_size = sizeof(NISLANDS_SMC_SWSTATE) +
2713                 ((NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1) * sizeof(NISLANDS_SMC_HW_PERFORMANCE_LEVEL));
2714         int ret;
2715         NISLANDS_SMC_SWSTATE *smc_state = kzalloc(state_size, GFP_KERNEL);
2716
2717         if (smc_state == NULL)
2718                 return -ENOMEM;
2719
2720         ret = ni_convert_power_state_to_smc(rdev, radeon_new_state, smc_state);
2721         if (ret)
2722                 goto done;
2723
2724         ret = rv770_copy_bytes_to_smc(rdev, address, (u8 *)smc_state, state_size, pi->sram_end);
2725
2726 done:
2727         kfree(smc_state);
2728
2729         return ret;
2730 }
2731
2732 static int ni_set_mc_special_registers(struct radeon_device *rdev,
2733                                        struct ni_mc_reg_table *table)
2734 {
2735         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2736         u8 i, j, k;
2737         u32 temp_reg;
2738
2739         for (i = 0, j = table->last; i < table->last; i++) {
2740                 switch (table->mc_reg_address[i].s1) {
2741                 case MC_SEQ_MISC1 >> 2:
2742                         if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2743                                 return -EINVAL;
2744                         temp_reg = RREG32(MC_PMG_CMD_EMRS);
2745                         table->mc_reg_address[j].s1 = MC_PMG_CMD_EMRS >> 2;
2746                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
2747                         for (k = 0; k < table->num_entries; k++)
2748                                 table->mc_reg_table_entry[k].mc_data[j] =
2749                                         ((temp_reg & 0xffff0000)) |
2750                                         ((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16);
2751                         j++;
2752                         if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2753                                 return -EINVAL;
2754
2755                         temp_reg = RREG32(MC_PMG_CMD_MRS);
2756                         table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS >> 2;
2757                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS_LP >> 2;
2758                         for(k = 0; k < table->num_entries; k++) {
2759                                 table->mc_reg_table_entry[k].mc_data[j] =
2760                                         (temp_reg & 0xffff0000) |
2761                                         (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
2762                                 if (!pi->mem_gddr5)
2763                                         table->mc_reg_table_entry[k].mc_data[j] |= 0x100;
2764                         }
2765                         j++;
2766                         if (j > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2767                                 return -EINVAL;
2768                         break;
2769                 case MC_SEQ_RESERVE_M >> 2:
2770                         temp_reg = RREG32(MC_PMG_CMD_MRS1);
2771                         table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS1 >> 2;
2772                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
2773                         for (k = 0; k < table->num_entries; k++)
2774                                 table->mc_reg_table_entry[k].mc_data[j] =
2775                                         (temp_reg & 0xffff0000) |
2776                                         (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
2777                         j++;
2778                         if (j > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2779                                 return -EINVAL;
2780                         break;
2781                 default:
2782                         break;
2783                 }
2784         }
2785
2786         table->last = j;
2787
2788         return 0;
2789 }
2790
2791 static bool ni_check_s0_mc_reg_index(u16 in_reg, u16 *out_reg)
2792 {
2793         bool result = true;
2794
2795         switch (in_reg) {
2796         case  MC_SEQ_RAS_TIMING >> 2:
2797                 *out_reg = MC_SEQ_RAS_TIMING_LP >> 2;
2798                 break;
2799         case MC_SEQ_CAS_TIMING >> 2:
2800                 *out_reg = MC_SEQ_CAS_TIMING_LP >> 2;
2801                 break;
2802         case MC_SEQ_MISC_TIMING >> 2:
2803                 *out_reg = MC_SEQ_MISC_TIMING_LP >> 2;
2804                 break;
2805         case MC_SEQ_MISC_TIMING2 >> 2:
2806                 *out_reg = MC_SEQ_MISC_TIMING2_LP >> 2;
2807                 break;
2808         case MC_SEQ_RD_CTL_D0 >> 2:
2809                 *out_reg = MC_SEQ_RD_CTL_D0_LP >> 2;
2810                 break;
2811         case MC_SEQ_RD_CTL_D1 >> 2:
2812                 *out_reg = MC_SEQ_RD_CTL_D1_LP >> 2;
2813                 break;
2814         case MC_SEQ_WR_CTL_D0 >> 2:
2815                 *out_reg = MC_SEQ_WR_CTL_D0_LP >> 2;
2816                 break;
2817         case MC_SEQ_WR_CTL_D1 >> 2:
2818                 *out_reg = MC_SEQ_WR_CTL_D1_LP >> 2;
2819                 break;
2820         case MC_PMG_CMD_EMRS >> 2:
2821                 *out_reg = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
2822                 break;
2823         case MC_PMG_CMD_MRS >> 2:
2824                 *out_reg = MC_SEQ_PMG_CMD_MRS_LP >> 2;
2825                 break;
2826         case MC_PMG_CMD_MRS1 >> 2:
2827                 *out_reg = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
2828                 break;
2829         case MC_SEQ_PMG_TIMING >> 2:
2830                 *out_reg = MC_SEQ_PMG_TIMING_LP >> 2;
2831                 break;
2832         case MC_PMG_CMD_MRS2 >> 2:
2833                 *out_reg = MC_SEQ_PMG_CMD_MRS2_LP >> 2;
2834                 break;
2835         default:
2836                 result = false;
2837                 break;
2838         }
2839
2840         return result;
2841 }
2842
2843 static void ni_set_valid_flag(struct ni_mc_reg_table *table)
2844 {
2845         u8 i, j;
2846
2847         for (i = 0; i < table->last; i++) {
2848                 for (j = 1; j < table->num_entries; j++) {
2849                         if (table->mc_reg_table_entry[j-1].mc_data[i] != table->mc_reg_table_entry[j].mc_data[i]) {
2850                                 table->valid_flag |= 1 << i;
2851                                 break;
2852                         }
2853                 }
2854         }
2855 }
2856
2857 static void ni_set_s0_mc_reg_index(struct ni_mc_reg_table *table)
2858 {
2859         u32 i;
2860         u16 address;
2861
2862         for (i = 0; i < table->last; i++)
2863                 table->mc_reg_address[i].s0 =
2864                         ni_check_s0_mc_reg_index(table->mc_reg_address[i].s1, &address) ?
2865                         address : table->mc_reg_address[i].s1;
2866 }
2867
2868 static int ni_copy_vbios_mc_reg_table(struct atom_mc_reg_table *table,
2869                                       struct ni_mc_reg_table *ni_table)
2870 {
2871         u8 i, j;
2872
2873         if (table->last > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2874                 return -EINVAL;
2875         if (table->num_entries > MAX_AC_TIMING_ENTRIES)
2876                 return -EINVAL;
2877
2878         for (i = 0; i < table->last; i++)
2879                 ni_table->mc_reg_address[i].s1 = table->mc_reg_address[i].s1;
2880         ni_table->last = table->last;
2881
2882         for (i = 0; i < table->num_entries; i++) {
2883                 ni_table->mc_reg_table_entry[i].mclk_max =
2884                         table->mc_reg_table_entry[i].mclk_max;
2885                 for (j = 0; j < table->last; j++)
2886                         ni_table->mc_reg_table_entry[i].mc_data[j] =
2887                                 table->mc_reg_table_entry[i].mc_data[j];
2888         }
2889         ni_table->num_entries = table->num_entries;
2890
2891         return 0;
2892 }
2893
2894 static int ni_initialize_mc_reg_table(struct radeon_device *rdev)
2895 {
2896         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2897         int ret;
2898         struct atom_mc_reg_table *table;
2899         struct ni_mc_reg_table *ni_table = &ni_pi->mc_reg_table;
2900         u8 module_index = rv770_get_memory_module_index(rdev);
2901
2902         table = kzalloc(sizeof(struct atom_mc_reg_table), GFP_KERNEL);
2903         if (!table)
2904                 return -ENOMEM;
2905
2906         WREG32(MC_SEQ_RAS_TIMING_LP, RREG32(MC_SEQ_RAS_TIMING));
2907         WREG32(MC_SEQ_CAS_TIMING_LP, RREG32(MC_SEQ_CAS_TIMING));
2908         WREG32(MC_SEQ_MISC_TIMING_LP, RREG32(MC_SEQ_MISC_TIMING));
2909         WREG32(MC_SEQ_MISC_TIMING2_LP, RREG32(MC_SEQ_MISC_TIMING2));
2910         WREG32(MC_SEQ_PMG_CMD_EMRS_LP, RREG32(MC_PMG_CMD_EMRS));
2911         WREG32(MC_SEQ_PMG_CMD_MRS_LP, RREG32(MC_PMG_CMD_MRS));
2912         WREG32(MC_SEQ_PMG_CMD_MRS1_LP, RREG32(MC_PMG_CMD_MRS1));
2913         WREG32(MC_SEQ_WR_CTL_D0_LP, RREG32(MC_SEQ_WR_CTL_D0));
2914         WREG32(MC_SEQ_WR_CTL_D1_LP, RREG32(MC_SEQ_WR_CTL_D1));
2915         WREG32(MC_SEQ_RD_CTL_D0_LP, RREG32(MC_SEQ_RD_CTL_D0));
2916         WREG32(MC_SEQ_RD_CTL_D1_LP, RREG32(MC_SEQ_RD_CTL_D1));
2917         WREG32(MC_SEQ_PMG_TIMING_LP, RREG32(MC_SEQ_PMG_TIMING));
2918         WREG32(MC_SEQ_PMG_CMD_MRS2_LP, RREG32(MC_PMG_CMD_MRS2));
2919
2920         ret = radeon_atom_init_mc_reg_table(rdev, module_index, table);
2921
2922         if (ret)
2923                 goto init_mc_done;
2924
2925         ret = ni_copy_vbios_mc_reg_table(table, ni_table);
2926
2927         if (ret)
2928                 goto init_mc_done;
2929
2930         ni_set_s0_mc_reg_index(ni_table);
2931
2932         ret = ni_set_mc_special_registers(rdev, ni_table);
2933
2934         if (ret)
2935                 goto init_mc_done;
2936
2937         ni_set_valid_flag(ni_table);
2938
2939 init_mc_done:
2940         kfree(table);
2941
2942         return ret;
2943 }
2944
2945 static void ni_populate_mc_reg_addresses(struct radeon_device *rdev,
2946                                          SMC_NIslands_MCRegisters *mc_reg_table)
2947 {
2948         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2949         u32 i, j;
2950
2951         for (i = 0, j = 0; j < ni_pi->mc_reg_table.last; j++) {
2952                 if (ni_pi->mc_reg_table.valid_flag & (1 << j)) {
2953                         if (i >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2954                                 break;
2955                         mc_reg_table->address[i].s0 =
2956                                 cpu_to_be16(ni_pi->mc_reg_table.mc_reg_address[j].s0);
2957                         mc_reg_table->address[i].s1 =
2958                                 cpu_to_be16(ni_pi->mc_reg_table.mc_reg_address[j].s1);
2959                         i++;
2960                 }
2961         }
2962         mc_reg_table->last = (u8)i;
2963 }
2964
2965
2966 static void ni_convert_mc_registers(struct ni_mc_reg_entry *entry,
2967                                     SMC_NIslands_MCRegisterSet *data,
2968                                     u32 num_entries, u32 valid_flag)
2969 {
2970         u32 i, j;
2971
2972         for (i = 0, j = 0; j < num_entries; j++) {
2973                 if (valid_flag & (1 << j)) {
2974                         data->value[i] = cpu_to_be32(entry->mc_data[j]);
2975                         i++;
2976                 }
2977         }
2978 }
2979
2980 static void ni_convert_mc_reg_table_entry_to_smc(struct radeon_device *rdev,
2981                                                  struct rv7xx_pl *pl,
2982                                                  SMC_NIslands_MCRegisterSet *mc_reg_table_data)
2983 {
2984         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2985         u32 i = 0;
2986
2987         for (i = 0; i < ni_pi->mc_reg_table.num_entries; i++) {
2988                 if (pl->mclk <= ni_pi->mc_reg_table.mc_reg_table_entry[i].mclk_max)
2989                         break;
2990         }
2991
2992         if ((i == ni_pi->mc_reg_table.num_entries) && (i > 0))
2993                 --i;
2994
2995         ni_convert_mc_registers(&ni_pi->mc_reg_table.mc_reg_table_entry[i],
2996                                 mc_reg_table_data,
2997                                 ni_pi->mc_reg_table.last,
2998                                 ni_pi->mc_reg_table.valid_flag);
2999 }
3000
3001 static void ni_convert_mc_reg_table_to_smc(struct radeon_device *rdev,
3002                                            struct radeon_ps *radeon_state,
3003                                            SMC_NIslands_MCRegisters *mc_reg_table)
3004 {
3005         struct ni_ps *state = ni_get_ps(radeon_state);
3006         int i;
3007
3008         for (i = 0; i < state->performance_level_count; i++) {
3009                 ni_convert_mc_reg_table_entry_to_smc(rdev,
3010                                                      &state->performance_levels[i],
3011                                                      &mc_reg_table->data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i]);
3012         }
3013 }
3014
3015 static int ni_populate_mc_reg_table(struct radeon_device *rdev,
3016                                     struct radeon_ps *radeon_boot_state)
3017 {
3018         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3019         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3020         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3021         struct ni_ps *boot_state = ni_get_ps(radeon_boot_state);
3022         SMC_NIslands_MCRegisters *mc_reg_table = &ni_pi->smc_mc_reg_table;
3023
3024         memset(mc_reg_table, 0, sizeof(SMC_NIslands_MCRegisters));
3025
3026         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_seq_index, 1);
3027
3028         ni_populate_mc_reg_addresses(rdev, mc_reg_table);
3029
3030         ni_convert_mc_reg_table_entry_to_smc(rdev, &boot_state->performance_levels[0],
3031                                              &mc_reg_table->data[0]);
3032
3033         ni_convert_mc_registers(&ni_pi->mc_reg_table.mc_reg_table_entry[0],
3034                                 &mc_reg_table->data[1],
3035                                 ni_pi->mc_reg_table.last,
3036                                 ni_pi->mc_reg_table.valid_flag);
3037
3038         ni_convert_mc_reg_table_to_smc(rdev, radeon_boot_state, mc_reg_table);
3039
3040         return rv770_copy_bytes_to_smc(rdev, eg_pi->mc_reg_table_start,
3041                                        (u8 *)mc_reg_table,
3042                                        sizeof(SMC_NIslands_MCRegisters),
3043                                        pi->sram_end);
3044 }
3045
3046 static int ni_upload_mc_reg_table(struct radeon_device *rdev,
3047                                   struct radeon_ps *radeon_new_state)
3048 {
3049         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3050         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3051         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3052         struct ni_ps *ni_new_state = ni_get_ps(radeon_new_state);
3053         SMC_NIslands_MCRegisters *mc_reg_table = &ni_pi->smc_mc_reg_table;
3054         u16 address;
3055
3056         memset(mc_reg_table, 0, sizeof(SMC_NIslands_MCRegisters));
3057
3058         ni_convert_mc_reg_table_to_smc(rdev, radeon_new_state, mc_reg_table);
3059
3060         address = eg_pi->mc_reg_table_start +
3061                 (u16)offsetof(SMC_NIslands_MCRegisters, data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT]);
3062
3063         return rv770_copy_bytes_to_smc(rdev, address,
3064                                        (u8 *)&mc_reg_table->data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT],
3065                                        sizeof(SMC_NIslands_MCRegisterSet) * ni_new_state->performance_level_count,
3066                                        pi->sram_end);
3067 }
3068
3069 static int ni_init_driver_calculated_leakage_table(struct radeon_device *rdev,
3070                                                    PP_NIslands_CACTABLES *cac_tables)
3071 {
3072         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3073         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3074         u32 leakage = 0;
3075         unsigned int i, j, table_size;
3076         s32 t;
3077         u32 smc_leakage, max_leakage = 0;
3078         u32 scaling_factor;
3079
3080         table_size = eg_pi->vddc_voltage_table.count;
3081
3082         if (SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES < table_size)
3083                 table_size = SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES;
3084
3085         scaling_factor = ni_get_smc_power_scaling_factor(rdev);
3086
3087         for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++) {
3088                 for (j = 0; j < table_size; j++) {
3089                         t = (1000 * ((i + 1) * 8));
3090
3091                         if (t < ni_pi->cac_data.leakage_minimum_temperature)
3092                                 t = ni_pi->cac_data.leakage_minimum_temperature;
3093
3094                         ni_calculate_leakage_for_v_and_t(rdev,
3095                                                          &ni_pi->cac_data.leakage_coefficients,
3096                                                          eg_pi->vddc_voltage_table.entries[j].value,
3097                                                          t,
3098                                                          ni_pi->cac_data.i_leakage,
3099                                                          &leakage);
3100
3101                         smc_leakage = ni_scale_power_for_smc(leakage, scaling_factor) / 1000;
3102                         if (smc_leakage > max_leakage)
3103                                 max_leakage = smc_leakage;
3104
3105                         cac_tables->cac_lkge_lut[i][j] = cpu_to_be32(smc_leakage);
3106                 }
3107         }
3108
3109         for (j = table_size; j < SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) {
3110                 for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3111                         cac_tables->cac_lkge_lut[i][j] = cpu_to_be32(max_leakage);
3112         }
3113         return 0;
3114 }
3115
3116 static int ni_init_simplified_leakage_table(struct radeon_device *rdev,
3117                                             PP_NIslands_CACTABLES *cac_tables)
3118 {
3119         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3120         struct radeon_cac_leakage_table *leakage_table =
3121                 &rdev->pm.dpm.dyn_state.cac_leakage_table;
3122         u32 i, j, table_size;
3123         u32 smc_leakage, max_leakage = 0;
3124         u32 scaling_factor;
3125
3126         if (!leakage_table)
3127                 return -EINVAL;
3128
3129         table_size = leakage_table->count;
3130
3131         if (eg_pi->vddc_voltage_table.count != table_size)
3132                 table_size = (eg_pi->vddc_voltage_table.count < leakage_table->count) ?
3133                         eg_pi->vddc_voltage_table.count : leakage_table->count;
3134
3135         if (SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES < table_size)
3136                 table_size = SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES;
3137
3138         if (table_size == 0)
3139                 return -EINVAL;
3140
3141         scaling_factor = ni_get_smc_power_scaling_factor(rdev);
3142
3143         for (j = 0; j < table_size; j++) {
3144                 smc_leakage = leakage_table->entries[j].leakage;
3145
3146                 if (smc_leakage > max_leakage)
3147                         max_leakage = smc_leakage;
3148
3149                 for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3150                         cac_tables->cac_lkge_lut[i][j] =
3151                                 cpu_to_be32(ni_scale_power_for_smc(smc_leakage, scaling_factor));
3152         }
3153
3154         for (j = table_size; j < SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) {
3155                 for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3156                         cac_tables->cac_lkge_lut[i][j] =
3157                                 cpu_to_be32(ni_scale_power_for_smc(max_leakage, scaling_factor));
3158         }
3159         return 0;
3160 }
3161
3162 static int ni_initialize_smc_cac_tables(struct radeon_device *rdev)
3163 {
3164         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3165         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3166         PP_NIslands_CACTABLES *cac_tables = NULL;
3167         int i, ret;
3168         u32 reg;
3169
3170         if (ni_pi->enable_cac == false)
3171                 return 0;
3172
3173         cac_tables = kzalloc(sizeof(PP_NIslands_CACTABLES), GFP_KERNEL);
3174         if (!cac_tables)
3175                 return -ENOMEM;
3176
3177         reg = RREG32(CG_CAC_CTRL) & ~(TID_CNT_MASK | TID_UNIT_MASK);
3178         reg |= (TID_CNT(ni_pi->cac_weights->tid_cnt) |
3179                 TID_UNIT(ni_pi->cac_weights->tid_unit));
3180         WREG32(CG_CAC_CTRL, reg);
3181
3182         for (i = 0; i < NISLANDS_DCCAC_MAX_LEVELS; i++)
3183                 ni_pi->dc_cac_table[i] = ni_pi->cac_weights->dc_cac[i];
3184
3185         for (i = 0; i < SMC_NISLANDS_BIF_LUT_NUM_OF_ENTRIES; i++)
3186                 cac_tables->cac_bif_lut[i] = ni_pi->cac_weights->pcie_cac[i];
3187
3188         ni_pi->cac_data.i_leakage = rdev->pm.dpm.cac_leakage;
3189         ni_pi->cac_data.pwr_const = 0;
3190         ni_pi->cac_data.dc_cac_value = ni_pi->dc_cac_table[NISLANDS_DCCAC_LEVEL_0];
3191         ni_pi->cac_data.bif_cac_value = 0;
3192         ni_pi->cac_data.mc_wr_weight = ni_pi->cac_weights->mc_write_weight;
3193         ni_pi->cac_data.mc_rd_weight = ni_pi->cac_weights->mc_read_weight;
3194         ni_pi->cac_data.allow_ovrflw = 0;
3195         ni_pi->cac_data.l2num_win_tdp = ni_pi->lta_window_size;
3196         ni_pi->cac_data.num_win_tdp = 0;
3197         ni_pi->cac_data.lts_truncate_n = ni_pi->lts_truncate;
3198
3199         if (ni_pi->driver_calculate_cac_leakage)
3200                 ret = ni_init_driver_calculated_leakage_table(rdev, cac_tables);
3201         else
3202                 ret = ni_init_simplified_leakage_table(rdev, cac_tables);
3203
3204         if (ret)
3205                 goto done_free;
3206
3207         cac_tables->pwr_const      = cpu_to_be32(ni_pi->cac_data.pwr_const);
3208         cac_tables->dc_cacValue    = cpu_to_be32(ni_pi->cac_data.dc_cac_value);
3209         cac_tables->bif_cacValue   = cpu_to_be32(ni_pi->cac_data.bif_cac_value);
3210         cac_tables->AllowOvrflw    = ni_pi->cac_data.allow_ovrflw;
3211         cac_tables->MCWrWeight     = ni_pi->cac_data.mc_wr_weight;
3212         cac_tables->MCRdWeight     = ni_pi->cac_data.mc_rd_weight;
3213         cac_tables->numWin_TDP     = ni_pi->cac_data.num_win_tdp;
3214         cac_tables->l2numWin_TDP   = ni_pi->cac_data.l2num_win_tdp;
3215         cac_tables->lts_truncate_n = ni_pi->cac_data.lts_truncate_n;
3216
3217         ret = rv770_copy_bytes_to_smc(rdev, ni_pi->cac_table_start, (u8 *)cac_tables,
3218                                       sizeof(PP_NIslands_CACTABLES), pi->sram_end);
3219
3220 done_free:
3221         if (ret) {
3222                 ni_pi->enable_cac = false;
3223                 ni_pi->enable_power_containment = false;
3224         }
3225
3226         kfree(cac_tables);
3227
3228         return 0;
3229 }
3230
3231 static int ni_initialize_hardware_cac_manager(struct radeon_device *rdev)
3232 {
3233         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3234         u32 reg;
3235
3236         if (!ni_pi->enable_cac ||
3237             !ni_pi->cac_configuration_required)
3238                 return 0;
3239
3240         if (ni_pi->cac_weights == NULL)
3241                 return -EINVAL;
3242
3243         reg = RREG32_CG(CG_CAC_REGION_1_WEIGHT_0) & ~(WEIGHT_TCP_SIG0_MASK |
3244                                                       WEIGHT_TCP_SIG1_MASK |
3245                                                       WEIGHT_TA_SIG_MASK);
3246         reg |= (WEIGHT_TCP_SIG0(ni_pi->cac_weights->weight_tcp_sig0) |
3247                 WEIGHT_TCP_SIG1(ni_pi->cac_weights->weight_tcp_sig1) |
3248                 WEIGHT_TA_SIG(ni_pi->cac_weights->weight_ta_sig));
3249         WREG32_CG(CG_CAC_REGION_1_WEIGHT_0, reg);
3250
3251         reg = RREG32_CG(CG_CAC_REGION_1_WEIGHT_1) & ~(WEIGHT_TCC_EN0_MASK |
3252                                                       WEIGHT_TCC_EN1_MASK |
3253                                                       WEIGHT_TCC_EN2_MASK);
3254         reg |= (WEIGHT_TCC_EN0(ni_pi->cac_weights->weight_tcc_en0) |
3255                 WEIGHT_TCC_EN1(ni_pi->cac_weights->weight_tcc_en1) |
3256                 WEIGHT_TCC_EN2(ni_pi->cac_weights->weight_tcc_en2));
3257         WREG32_CG(CG_CAC_REGION_1_WEIGHT_1, reg);
3258
3259         reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_0) & ~(WEIGHT_CB_EN0_MASK |
3260                                                       WEIGHT_CB_EN1_MASK |
3261                                                       WEIGHT_CB_EN2_MASK |
3262                                                       WEIGHT_CB_EN3_MASK);
3263         reg |= (WEIGHT_CB_EN0(ni_pi->cac_weights->weight_cb_en0) |
3264                 WEIGHT_CB_EN1(ni_pi->cac_weights->weight_cb_en1) |
3265                 WEIGHT_CB_EN2(ni_pi->cac_weights->weight_cb_en2) |
3266                 WEIGHT_CB_EN3(ni_pi->cac_weights->weight_cb_en3));
3267         WREG32_CG(CG_CAC_REGION_2_WEIGHT_0, reg);
3268
3269         reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_1) & ~(WEIGHT_DB_SIG0_MASK |
3270                                                       WEIGHT_DB_SIG1_MASK |
3271                                                       WEIGHT_DB_SIG2_MASK |
3272                                                       WEIGHT_DB_SIG3_MASK);
3273         reg |= (WEIGHT_DB_SIG0(ni_pi->cac_weights->weight_db_sig0) |
3274                 WEIGHT_DB_SIG1(ni_pi->cac_weights->weight_db_sig1) |
3275                 WEIGHT_DB_SIG2(ni_pi->cac_weights->weight_db_sig2) |
3276                 WEIGHT_DB_SIG3(ni_pi->cac_weights->weight_db_sig3));
3277         WREG32_CG(CG_CAC_REGION_2_WEIGHT_1, reg);
3278
3279         reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_2) & ~(WEIGHT_SXM_SIG0_MASK |
3280                                                       WEIGHT_SXM_SIG1_MASK |
3281                                                       WEIGHT_SXM_SIG2_MASK |
3282                                                       WEIGHT_SXS_SIG0_MASK |
3283                                                       WEIGHT_SXS_SIG1_MASK);
3284         reg |= (WEIGHT_SXM_SIG0(ni_pi->cac_weights->weight_sxm_sig0) |
3285                 WEIGHT_SXM_SIG1(ni_pi->cac_weights->weight_sxm_sig1) |
3286                 WEIGHT_SXM_SIG2(ni_pi->cac_weights->weight_sxm_sig2) |
3287                 WEIGHT_SXS_SIG0(ni_pi->cac_weights->weight_sxs_sig0) |
3288                 WEIGHT_SXS_SIG1(ni_pi->cac_weights->weight_sxs_sig1));
3289         WREG32_CG(CG_CAC_REGION_2_WEIGHT_2, reg);
3290
3291         reg = RREG32_CG(CG_CAC_REGION_3_WEIGHT_0) & ~(WEIGHT_XBR_0_MASK |
3292                                                       WEIGHT_XBR_1_MASK |
3293                                                       WEIGHT_XBR_2_MASK |
3294                                                       WEIGHT_SPI_SIG0_MASK);
3295         reg |= (WEIGHT_XBR_0(ni_pi->cac_weights->weight_xbr_0) |
3296                 WEIGHT_XBR_1(ni_pi->cac_weights->weight_xbr_1) |
3297                 WEIGHT_XBR_2(ni_pi->cac_weights->weight_xbr_2) |
3298                 WEIGHT_SPI_SIG0(ni_pi->cac_weights->weight_spi_sig0));
3299         WREG32_CG(CG_CAC_REGION_3_WEIGHT_0, reg);
3300
3301         reg = RREG32_CG(CG_CAC_REGION_3_WEIGHT_1) & ~(WEIGHT_SPI_SIG1_MASK |
3302                                                       WEIGHT_SPI_SIG2_MASK |
3303                                                       WEIGHT_SPI_SIG3_MASK |
3304                                                       WEIGHT_SPI_SIG4_MASK |
3305                                                       WEIGHT_SPI_SIG5_MASK);
3306         reg |= (WEIGHT_SPI_SIG1(ni_pi->cac_weights->weight_spi_sig1) |
3307                 WEIGHT_SPI_SIG2(ni_pi->cac_weights->weight_spi_sig2) |
3308                 WEIGHT_SPI_SIG3(ni_pi->cac_weights->weight_spi_sig3) |
3309                 WEIGHT_SPI_SIG4(ni_pi->cac_weights->weight_spi_sig4) |
3310                 WEIGHT_SPI_SIG5(ni_pi->cac_weights->weight_spi_sig5));
3311         WREG32_CG(CG_CAC_REGION_3_WEIGHT_1, reg);
3312
3313         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_0) & ~(WEIGHT_LDS_SIG0_MASK |
3314                                                       WEIGHT_LDS_SIG1_MASK |
3315                                                       WEIGHT_SC_MASK);
3316         reg |= (WEIGHT_LDS_SIG0(ni_pi->cac_weights->weight_lds_sig0) |
3317                 WEIGHT_LDS_SIG1(ni_pi->cac_weights->weight_lds_sig1) |
3318                 WEIGHT_SC(ni_pi->cac_weights->weight_sc));
3319         WREG32_CG(CG_CAC_REGION_4_WEIGHT_0, reg);
3320
3321         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_1) & ~(WEIGHT_BIF_MASK |
3322                                                       WEIGHT_CP_MASK |
3323                                                       WEIGHT_PA_SIG0_MASK |
3324                                                       WEIGHT_PA_SIG1_MASK |
3325                                                       WEIGHT_VGT_SIG0_MASK);
3326         reg |= (WEIGHT_BIF(ni_pi->cac_weights->weight_bif) |
3327                 WEIGHT_CP(ni_pi->cac_weights->weight_cp) |
3328                 WEIGHT_PA_SIG0(ni_pi->cac_weights->weight_pa_sig0) |
3329                 WEIGHT_PA_SIG1(ni_pi->cac_weights->weight_pa_sig1) |
3330                 WEIGHT_VGT_SIG0(ni_pi->cac_weights->weight_vgt_sig0));
3331         WREG32_CG(CG_CAC_REGION_4_WEIGHT_1, reg);
3332
3333         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_2) & ~(WEIGHT_VGT_SIG1_MASK |
3334                                                       WEIGHT_VGT_SIG2_MASK |
3335                                                       WEIGHT_DC_SIG0_MASK |
3336                                                       WEIGHT_DC_SIG1_MASK |
3337                                                       WEIGHT_DC_SIG2_MASK);
3338         reg |= (WEIGHT_VGT_SIG1(ni_pi->cac_weights->weight_vgt_sig1) |
3339                 WEIGHT_VGT_SIG2(ni_pi->cac_weights->weight_vgt_sig2) |
3340                 WEIGHT_DC_SIG0(ni_pi->cac_weights->weight_dc_sig0) |
3341                 WEIGHT_DC_SIG1(ni_pi->cac_weights->weight_dc_sig1) |
3342                 WEIGHT_DC_SIG2(ni_pi->cac_weights->weight_dc_sig2));
3343         WREG32_CG(CG_CAC_REGION_4_WEIGHT_2, reg);
3344
3345         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_3) & ~(WEIGHT_DC_SIG3_MASK |
3346                                                       WEIGHT_UVD_SIG0_MASK |
3347                                                       WEIGHT_UVD_SIG1_MASK |
3348                                                       WEIGHT_SPARE0_MASK |
3349                                                       WEIGHT_SPARE1_MASK);
3350         reg |= (WEIGHT_DC_SIG3(ni_pi->cac_weights->weight_dc_sig3) |
3351                 WEIGHT_UVD_SIG0(ni_pi->cac_weights->weight_uvd_sig0) |
3352                 WEIGHT_UVD_SIG1(ni_pi->cac_weights->weight_uvd_sig1) |
3353                 WEIGHT_SPARE0(ni_pi->cac_weights->weight_spare0) |
3354                 WEIGHT_SPARE1(ni_pi->cac_weights->weight_spare1));
3355         WREG32_CG(CG_CAC_REGION_4_WEIGHT_3, reg);
3356
3357         reg = RREG32_CG(CG_CAC_REGION_5_WEIGHT_0) & ~(WEIGHT_SQ_VSP_MASK |
3358                                                       WEIGHT_SQ_VSP0_MASK);
3359         reg |= (WEIGHT_SQ_VSP(ni_pi->cac_weights->weight_sq_vsp) |
3360                 WEIGHT_SQ_VSP0(ni_pi->cac_weights->weight_sq_vsp0));
3361         WREG32_CG(CG_CAC_REGION_5_WEIGHT_0, reg);
3362
3363         reg = RREG32_CG(CG_CAC_REGION_5_WEIGHT_1) & ~(WEIGHT_SQ_GPR_MASK);
3364         reg |= WEIGHT_SQ_GPR(ni_pi->cac_weights->weight_sq_gpr);
3365         WREG32_CG(CG_CAC_REGION_5_WEIGHT_1, reg);
3366
3367         reg = RREG32_CG(CG_CAC_REGION_4_OVERRIDE_4) & ~(OVR_MODE_SPARE_0_MASK |
3368                                                         OVR_VAL_SPARE_0_MASK |
3369                                                         OVR_MODE_SPARE_1_MASK |
3370                                                         OVR_VAL_SPARE_1_MASK);
3371         reg |= (OVR_MODE_SPARE_0(ni_pi->cac_weights->ovr_mode_spare_0) |
3372                 OVR_VAL_SPARE_0(ni_pi->cac_weights->ovr_val_spare_0) |
3373                 OVR_MODE_SPARE_1(ni_pi->cac_weights->ovr_mode_spare_1) |
3374                 OVR_VAL_SPARE_1(ni_pi->cac_weights->ovr_val_spare_1));
3375         WREG32_CG(CG_CAC_REGION_4_OVERRIDE_4, reg);
3376
3377         reg = RREG32(SQ_CAC_THRESHOLD) & ~(VSP_MASK |
3378                                            VSP0_MASK |
3379                                            GPR_MASK);
3380         reg |= (VSP(ni_pi->cac_weights->vsp) |
3381                 VSP0(ni_pi->cac_weights->vsp0) |
3382                 GPR(ni_pi->cac_weights->gpr));
3383         WREG32(SQ_CAC_THRESHOLD, reg);
3384
3385         reg = (MCDW_WR_ENABLE |
3386                MCDX_WR_ENABLE |
3387                MCDY_WR_ENABLE |
3388                MCDZ_WR_ENABLE |
3389                INDEX(0x09D4));
3390         WREG32(MC_CG_CONFIG, reg);
3391
3392         reg = (READ_WEIGHT(ni_pi->cac_weights->mc_read_weight) |
3393                WRITE_WEIGHT(ni_pi->cac_weights->mc_write_weight) |
3394                ALLOW_OVERFLOW);
3395         WREG32(MC_CG_DATAPORT, reg);
3396
3397         return 0;
3398 }
3399
3400 static int ni_enable_smc_cac(struct radeon_device *rdev,
3401                              struct radeon_ps *radeon_new_state,
3402                              bool enable)
3403 {
3404         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3405         int ret = 0;
3406         PPSMC_Result smc_result;
3407
3408         if (ni_pi->enable_cac) {
3409                 if (enable) {
3410                         if (!r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) {
3411                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_CollectCAC_PowerCorreln);
3412
3413                                 if (ni_pi->support_cac_long_term_average) {
3414                                         smc_result = rv770_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgEnable);
3415                                         if (PPSMC_Result_OK != smc_result)
3416                                                 ni_pi->support_cac_long_term_average = false;
3417                                 }
3418
3419                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_EnableCac);
3420                                 if (PPSMC_Result_OK != smc_result)
3421                                         ret = -EINVAL;
3422
3423                                 ni_pi->cac_enabled = (PPSMC_Result_OK == smc_result) ? true : false;
3424                         }
3425                 } else if (ni_pi->cac_enabled) {
3426                         smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_DisableCac);
3427
3428                         ni_pi->cac_enabled = false;
3429
3430                         if (ni_pi->support_cac_long_term_average) {
3431                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgDisable);
3432                                 if (PPSMC_Result_OK != smc_result)
3433                                         ni_pi->support_cac_long_term_average = false;
3434                         }
3435                 }
3436         }
3437
3438         return ret;
3439 }
3440
3441 static int ni_pcie_performance_request(struct radeon_device *rdev,
3442                                        u8 perf_req, bool advertise)
3443 {
3444 #if defined(CONFIG_ACPI)
3445         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3446
3447         if ((perf_req == PCIE_PERF_REQ_PECI_GEN1) ||
3448             (perf_req == PCIE_PERF_REQ_PECI_GEN2)) {
3449                 if (eg_pi->pcie_performance_request_registered == false)
3450                         radeon_acpi_pcie_notify_device_ready(rdev);
3451                 eg_pi->pcie_performance_request_registered = true;
3452                 return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise);
3453         } else if ((perf_req == PCIE_PERF_REQ_REMOVE_REGISTRY) &&
3454                    eg_pi->pcie_performance_request_registered) {
3455                 eg_pi->pcie_performance_request_registered = false;
3456                 return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise);
3457         }
3458 #endif
3459         return 0;
3460 }
3461
3462 static int ni_advertise_gen2_capability(struct radeon_device *rdev)
3463 {
3464         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3465         u32 tmp;
3466
3467         tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3468
3469         if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
3470             (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2))
3471                 pi->pcie_gen2 = true;
3472         else
3473                 pi->pcie_gen2 = false;
3474
3475         if (!pi->pcie_gen2)
3476                 ni_pcie_performance_request(rdev, PCIE_PERF_REQ_PECI_GEN2, true);
3477
3478         return 0;
3479 }
3480
3481 static void ni_enable_bif_dynamic_pcie_gen2(struct radeon_device *rdev,
3482                                             bool enable)
3483 {
3484         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3485         u32 tmp, bif;
3486
3487         tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3488
3489         if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
3490             (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
3491                 if (enable) {
3492                         if (!pi->boot_in_gen2) {
3493                                 bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK;
3494                                 bif |= CG_CLIENT_REQ(0xd);
3495                                 WREG32(CG_BIF_REQ_AND_RSP, bif);
3496                         }
3497                         tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
3498                         tmp |= LC_HW_VOLTAGE_IF_CONTROL(1);
3499                         tmp |= LC_GEN2_EN_STRAP;
3500
3501                         tmp |= LC_CLR_FAILED_SPD_CHANGE_CNT;
3502                         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3503                         udelay(10);
3504                         tmp &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
3505                         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3506                 } else {
3507                         if (!pi->boot_in_gen2) {
3508                                 bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK;
3509                                 bif |= CG_CLIENT_REQ(0xd);
3510                                 WREG32(CG_BIF_REQ_AND_RSP, bif);
3511
3512                                 tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
3513                                 tmp &= ~LC_GEN2_EN_STRAP;
3514                         }
3515                         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3516                 }
3517         }
3518 }
3519
3520 static void ni_enable_dynamic_pcie_gen2(struct radeon_device *rdev,
3521                                         bool enable)
3522 {
3523         ni_enable_bif_dynamic_pcie_gen2(rdev, enable);
3524
3525         if (enable)
3526                 WREG32_P(GENERAL_PWRMGT, ENABLE_GEN2PCIE, ~ENABLE_GEN2PCIE);
3527         else
3528                 WREG32_P(GENERAL_PWRMGT, 0, ~ENABLE_GEN2PCIE);
3529 }
3530
3531 void ni_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev,
3532                                            struct radeon_ps *new_ps,
3533                                            struct radeon_ps *old_ps)
3534 {
3535         struct ni_ps *new_state = ni_get_ps(new_ps);
3536         struct ni_ps *current_state = ni_get_ps(old_ps);
3537
3538         if ((new_ps->vclk == old_ps->vclk) &&
3539             (new_ps->dclk == old_ps->dclk))
3540                 return;
3541
3542         if (new_state->performance_levels[new_state->performance_level_count - 1].sclk >=
3543             current_state->performance_levels[current_state->performance_level_count - 1].sclk)
3544                 return;
3545
3546         radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
3547 }
3548
3549 void ni_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev,
3550                                           struct radeon_ps *new_ps,
3551                                           struct radeon_ps *old_ps)
3552 {
3553         struct ni_ps *new_state = ni_get_ps(new_ps);
3554         struct ni_ps *current_state = ni_get_ps(old_ps);
3555
3556         if ((new_ps->vclk == old_ps->vclk) &&
3557             (new_ps->dclk == old_ps->dclk))
3558                 return;
3559
3560         if (new_state->performance_levels[new_state->performance_level_count - 1].sclk <
3561             current_state->performance_levels[current_state->performance_level_count - 1].sclk)
3562                 return;
3563
3564         radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
3565 }
3566
3567 void ni_dpm_setup_asic(struct radeon_device *rdev)
3568 {
3569         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3570         int r;
3571
3572         r = ni_mc_load_microcode(rdev);
3573         if (r)
3574                 DRM_ERROR("Failed to load MC firmware!\n");
3575         ni_read_clock_registers(rdev);
3576         btc_read_arb_registers(rdev);
3577         rv770_get_memory_type(rdev);
3578         if (eg_pi->pcie_performance_request)
3579                 ni_advertise_gen2_capability(rdev);
3580         rv770_get_pcie_gen2_status(rdev);
3581         rv770_enable_acpi_pm(rdev);
3582 }
3583
3584 void ni_update_current_ps(struct radeon_device *rdev,
3585                           struct radeon_ps *rps)
3586 {
3587         struct ni_ps *new_ps = ni_get_ps(rps);
3588         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3589         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3590
3591         eg_pi->current_rps = *rps;
3592         ni_pi->current_ps = *new_ps;
3593         eg_pi->current_rps.ps_priv = &ni_pi->current_ps;
3594 }
3595
3596 void ni_update_requested_ps(struct radeon_device *rdev,
3597                             struct radeon_ps *rps)
3598 {
3599         struct ni_ps *new_ps = ni_get_ps(rps);
3600         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3601         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3602
3603         eg_pi->requested_rps = *rps;
3604         ni_pi->requested_ps = *new_ps;
3605         eg_pi->requested_rps.ps_priv = &ni_pi->requested_ps;
3606 }
3607
3608 int ni_dpm_enable(struct radeon_device *rdev)
3609 {
3610         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3611         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3612         struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
3613         int ret;
3614
3615         if (pi->gfx_clock_gating)
3616                 ni_cg_clockgating_default(rdev);
3617         if (btc_dpm_enabled(rdev))
3618                 return -EINVAL;
3619         if (pi->mg_clock_gating)
3620                 ni_mg_clockgating_default(rdev);
3621         if (eg_pi->ls_clock_gating)
3622                 ni_ls_clockgating_default(rdev);
3623         if (pi->voltage_control) {
3624                 rv770_enable_voltage_control(rdev, true);
3625                 ret = cypress_construct_voltage_tables(rdev);
3626                 if (ret) {
3627                         DRM_ERROR("cypress_construct_voltage_tables failed\n");
3628                         return ret;
3629                 }
3630         }
3631         if (eg_pi->dynamic_ac_timing) {
3632                 ret = ni_initialize_mc_reg_table(rdev);
3633                 if (ret)
3634                         eg_pi->dynamic_ac_timing = false;
3635         }
3636         if (pi->dynamic_ss)
3637                 cypress_enable_spread_spectrum(rdev, true);
3638         if (pi->thermal_protection)
3639                 rv770_enable_thermal_protection(rdev, true);
3640         rv770_setup_bsp(rdev);
3641         rv770_program_git(rdev);
3642         rv770_program_tp(rdev);
3643         rv770_program_tpp(rdev);
3644         rv770_program_sstp(rdev);
3645         cypress_enable_display_gap(rdev);
3646         rv770_program_vc(rdev);
3647         if (pi->dynamic_pcie_gen2)
3648                 ni_enable_dynamic_pcie_gen2(rdev, true);
3649         ret = rv770_upload_firmware(rdev);
3650         if (ret) {
3651                 DRM_ERROR("rv770_upload_firmware failed\n");
3652                 return ret;
3653         }
3654         ret = ni_process_firmware_header(rdev);
3655         if (ret) {
3656                 DRM_ERROR("ni_process_firmware_header failed\n");
3657                 return ret;
3658         }
3659         ret = ni_initial_switch_from_arb_f0_to_f1(rdev);
3660         if (ret) {
3661                 DRM_ERROR("ni_initial_switch_from_arb_f0_to_f1 failed\n");
3662                 return ret;
3663         }
3664         ret = ni_init_smc_table(rdev);
3665         if (ret) {
3666                 DRM_ERROR("ni_init_smc_table failed\n");
3667                 return ret;
3668         }
3669         ret = ni_init_smc_spll_table(rdev);
3670         if (ret) {
3671                 DRM_ERROR("ni_init_smc_spll_table failed\n");
3672                 return ret;
3673         }
3674         ret = ni_init_arb_table_index(rdev);
3675         if (ret) {
3676                 DRM_ERROR("ni_init_arb_table_index failed\n");
3677                 return ret;
3678         }
3679         if (eg_pi->dynamic_ac_timing) {
3680                 ret = ni_populate_mc_reg_table(rdev, boot_ps);
3681                 if (ret) {
3682                         DRM_ERROR("ni_populate_mc_reg_table failed\n");
3683                         return ret;
3684                 }
3685         }
3686         ret = ni_initialize_smc_cac_tables(rdev);
3687         if (ret) {
3688                 DRM_ERROR("ni_initialize_smc_cac_tables failed\n");
3689                 return ret;
3690         }
3691         ret = ni_initialize_hardware_cac_manager(rdev);
3692         if (ret) {
3693                 DRM_ERROR("ni_initialize_hardware_cac_manager failed\n");
3694                 return ret;
3695         }
3696         ret = ni_populate_smc_tdp_limits(rdev, boot_ps);
3697         if (ret) {
3698                 DRM_ERROR("ni_populate_smc_tdp_limits failed\n");
3699                 return ret;
3700         }
3701         ni_program_response_times(rdev);
3702         r7xx_start_smc(rdev);
3703         ret = cypress_notify_smc_display_change(rdev, false);
3704         if (ret) {
3705                 DRM_ERROR("cypress_notify_smc_display_change failed\n");
3706                 return ret;
3707         }
3708         cypress_enable_sclk_control(rdev, true);
3709         if (eg_pi->memory_transition)
3710                 cypress_enable_mclk_control(rdev, true);
3711         cypress_start_dpm(rdev);
3712         if (pi->gfx_clock_gating)
3713                 ni_gfx_clockgating_enable(rdev, true);
3714         if (pi->mg_clock_gating)
3715                 ni_mg_clockgating_enable(rdev, true);
3716         if (eg_pi->ls_clock_gating)
3717                 ni_ls_clockgating_enable(rdev, true);
3718
3719         rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true);
3720
3721         ni_update_current_ps(rdev, boot_ps);
3722
3723         return 0;
3724 }
3725
3726 void ni_dpm_disable(struct radeon_device *rdev)
3727 {
3728         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3729         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3730         struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
3731
3732         if (!btc_dpm_enabled(rdev))
3733                 return;
3734         rv770_clear_vc(rdev);
3735         if (pi->thermal_protection)
3736                 rv770_enable_thermal_protection(rdev, false);
3737         ni_enable_power_containment(rdev, boot_ps, false);
3738         ni_enable_smc_cac(rdev, boot_ps, false);
3739         cypress_enable_spread_spectrum(rdev, false);
3740         rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, false);
3741         if (pi->dynamic_pcie_gen2)
3742                 ni_enable_dynamic_pcie_gen2(rdev, false);
3743
3744         if (rdev->irq.installed &&
3745             r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
3746                 rdev->irq.dpm_thermal = false;
3747                 radeon_irq_set(rdev);
3748         }
3749
3750         if (pi->gfx_clock_gating)
3751                 ni_gfx_clockgating_enable(rdev, false);
3752         if (pi->mg_clock_gating)
3753                 ni_mg_clockgating_enable(rdev, false);
3754         if (eg_pi->ls_clock_gating)
3755                 ni_ls_clockgating_enable(rdev, false);
3756         ni_stop_dpm(rdev);
3757         btc_reset_to_default(rdev);
3758         ni_stop_smc(rdev);
3759         ni_force_switch_to_arb_f0(rdev);
3760
3761         ni_update_current_ps(rdev, boot_ps);
3762 }
3763
3764 static int ni_power_control_set_level(struct radeon_device *rdev)
3765 {
3766         struct radeon_ps *new_ps = rdev->pm.dpm.requested_ps;
3767         int ret;
3768
3769         ret = ni_restrict_performance_levels_before_switch(rdev);
3770         if (ret)
3771                 return ret;
3772         ret = rv770_halt_smc(rdev);
3773         if (ret)
3774                 return ret;
3775         ret = ni_populate_smc_tdp_limits(rdev, new_ps);
3776         if (ret)
3777                 return ret;
3778         ret = rv770_resume_smc(rdev);
3779         if (ret)
3780                 return ret;
3781         ret = rv770_set_sw_state(rdev);
3782         if (ret)
3783                 return ret;
3784
3785         return 0;
3786 }
3787
3788 int ni_dpm_pre_set_power_state(struct radeon_device *rdev)
3789 {
3790         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3791         struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps;
3792         struct radeon_ps *new_ps = &requested_ps;
3793
3794         ni_update_requested_ps(rdev, new_ps);
3795
3796         ni_apply_state_adjust_rules(rdev, &eg_pi->requested_rps);
3797
3798         return 0;
3799 }
3800
3801 int ni_dpm_set_power_state(struct radeon_device *rdev)
3802 {
3803         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3804         struct radeon_ps *new_ps = &eg_pi->requested_rps;
3805         struct radeon_ps *old_ps = &eg_pi->current_rps;
3806         int ret;
3807
3808         ret = ni_restrict_performance_levels_before_switch(rdev);
3809         if (ret) {
3810                 DRM_ERROR("ni_restrict_performance_levels_before_switch failed\n");
3811                 return ret;
3812         }
3813         ni_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
3814         ret = ni_enable_power_containment(rdev, new_ps, false);
3815         if (ret) {
3816                 DRM_ERROR("ni_enable_power_containment failed\n");
3817                 return ret;
3818         }
3819         ret = ni_enable_smc_cac(rdev, new_ps, false);
3820         if (ret) {
3821                 DRM_ERROR("ni_enable_smc_cac failed\n");
3822                 return ret;
3823         }
3824         ret = rv770_halt_smc(rdev);
3825         if (ret) {
3826                 DRM_ERROR("rv770_halt_smc failed\n");
3827                 return ret;
3828         }
3829         if (eg_pi->smu_uvd_hs)
3830                 btc_notify_uvd_to_smc(rdev, new_ps);
3831         ret = ni_upload_sw_state(rdev, new_ps);
3832         if (ret) {
3833                 DRM_ERROR("ni_upload_sw_state failed\n");
3834                 return ret;
3835         }
3836         if (eg_pi->dynamic_ac_timing) {
3837                 ret = ni_upload_mc_reg_table(rdev, new_ps);
3838                 if (ret) {
3839                         DRM_ERROR("ni_upload_mc_reg_table failed\n");
3840                         return ret;
3841                 }
3842         }
3843         ret = ni_program_memory_timing_parameters(rdev, new_ps);
3844         if (ret) {
3845                 DRM_ERROR("ni_program_memory_timing_parameters failed\n");
3846                 return ret;
3847         }
3848         ret = rv770_resume_smc(rdev);
3849         if (ret) {
3850                 DRM_ERROR("rv770_resume_smc failed\n");
3851                 return ret;
3852         }
3853         ret = rv770_set_sw_state(rdev);
3854         if (ret) {
3855                 DRM_ERROR("rv770_set_sw_state failed\n");
3856                 return ret;
3857         }
3858         ni_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
3859         ret = ni_enable_smc_cac(rdev, new_ps, true);
3860         if (ret) {
3861                 DRM_ERROR("ni_enable_smc_cac failed\n");
3862                 return ret;
3863         }
3864         ret = ni_enable_power_containment(rdev, new_ps, true);
3865         if (ret) {
3866                 DRM_ERROR("ni_enable_power_containment failed\n");
3867                 return ret;
3868         }
3869
3870         /* update tdp */
3871         ret = ni_power_control_set_level(rdev);
3872         if (ret) {
3873                 DRM_ERROR("ni_power_control_set_level failed\n");
3874                 return ret;
3875         }
3876
3877         return 0;
3878 }
3879
3880 void ni_dpm_post_set_power_state(struct radeon_device *rdev)
3881 {
3882         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3883         struct radeon_ps *new_ps = &eg_pi->requested_rps;
3884
3885         ni_update_current_ps(rdev, new_ps);
3886 }
3887
3888 void ni_dpm_reset_asic(struct radeon_device *rdev)
3889 {
3890         ni_restrict_performance_levels_before_switch(rdev);
3891         rv770_set_boot_state(rdev);
3892 }
3893
3894 union power_info {
3895         struct _ATOM_POWERPLAY_INFO info;
3896         struct _ATOM_POWERPLAY_INFO_V2 info_2;
3897         struct _ATOM_POWERPLAY_INFO_V3 info_3;
3898         struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
3899         struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
3900         struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
3901 };
3902
3903 union pplib_clock_info {
3904         struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
3905         struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
3906         struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
3907         struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
3908 };
3909
3910 union pplib_power_state {
3911         struct _ATOM_PPLIB_STATE v1;
3912         struct _ATOM_PPLIB_STATE_V2 v2;
3913 };
3914
3915 static void ni_parse_pplib_non_clock_info(struct radeon_device *rdev,
3916                                           struct radeon_ps *rps,
3917                                           struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
3918                                           u8 table_rev)
3919 {
3920         rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
3921         rps->class = le16_to_cpu(non_clock_info->usClassification);
3922         rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
3923
3924         if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
3925                 rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
3926                 rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
3927         } else if (r600_is_uvd_state(rps->class, rps->class2)) {
3928                 rps->vclk = RV770_DEFAULT_VCLK_FREQ;
3929                 rps->dclk = RV770_DEFAULT_DCLK_FREQ;
3930         } else {
3931                 rps->vclk = 0;
3932                 rps->dclk = 0;
3933         }
3934
3935         if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT)
3936                 rdev->pm.dpm.boot_ps = rps;
3937         if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
3938                 rdev->pm.dpm.uvd_ps = rps;
3939 }
3940
3941 static void ni_parse_pplib_clock_info(struct radeon_device *rdev,
3942                                       struct radeon_ps *rps, int index,
3943                                       union pplib_clock_info *clock_info)
3944 {
3945         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3946         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3947         struct ni_ps *ps = ni_get_ps(rps);
3948         struct rv7xx_pl *pl = &ps->performance_levels[index];
3949
3950         ps->performance_level_count = index + 1;
3951
3952         pl->sclk = le16_to_cpu(clock_info->evergreen.usEngineClockLow);
3953         pl->sclk |= clock_info->evergreen.ucEngineClockHigh << 16;
3954         pl->mclk = le16_to_cpu(clock_info->evergreen.usMemoryClockLow);
3955         pl->mclk |= clock_info->evergreen.ucMemoryClockHigh << 16;
3956
3957         pl->vddc = le16_to_cpu(clock_info->evergreen.usVDDC);
3958         pl->vddci = le16_to_cpu(clock_info->evergreen.usVDDCI);
3959         pl->flags = le32_to_cpu(clock_info->evergreen.ulFlags);
3960
3961         /* patch up vddc if necessary */
3962         if (pl->vddc == 0xff01) {
3963                 if (pi->max_vddc)
3964                         pl->vddc = pi->max_vddc;
3965         }
3966
3967         if (rps->class & ATOM_PPLIB_CLASSIFICATION_ACPI) {
3968                 pi->acpi_vddc = pl->vddc;
3969                 eg_pi->acpi_vddci = pl->vddci;
3970                 if (ps->performance_levels[0].flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2)
3971                         pi->acpi_pcie_gen2 = true;
3972                 else
3973                         pi->acpi_pcie_gen2 = false;
3974         }
3975
3976         if (rps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) {
3977                 eg_pi->ulv.supported = true;
3978                 eg_pi->ulv.pl = pl;
3979         }
3980
3981         if (pi->min_vddc_in_table > pl->vddc)
3982                 pi->min_vddc_in_table = pl->vddc;
3983
3984         if (pi->max_vddc_in_table < pl->vddc)
3985                 pi->max_vddc_in_table = pl->vddc;
3986
3987         /* patch up boot state */
3988         if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
3989                 u16 vddc, vddci, mvdd;
3990                 radeon_atombios_get_default_voltages(rdev, &vddc, &vddci, &mvdd);
3991                 pl->mclk = rdev->clock.default_mclk;
3992                 pl->sclk = rdev->clock.default_sclk;
3993                 pl->vddc = vddc;
3994                 pl->vddci = vddci;
3995         }
3996
3997         if ((rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) ==
3998             ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE) {
3999                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk = pl->sclk;
4000                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.mclk = pl->mclk;
4001                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddc = pl->vddc;
4002                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddci = pl->vddci;
4003         }
4004 }
4005
4006 static int ni_parse_power_table(struct radeon_device *rdev)
4007 {
4008         struct radeon_mode_info *mode_info = &rdev->mode_info;
4009         struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
4010         union pplib_power_state *power_state;
4011         int i, j;
4012         union pplib_clock_info *clock_info;
4013         union power_info *power_info;
4014         int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
4015         u16 data_offset;
4016         u8 frev, crev;
4017         struct ni_ps *ps;
4018
4019         if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
4020                                    &frev, &crev, &data_offset))
4021                 return -EINVAL;
4022         power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
4023
4024         rdev->pm.dpm.ps = kzalloc(sizeof(struct radeon_ps) *
4025                                   power_info->pplib.ucNumStates, GFP_KERNEL);
4026         if (!rdev->pm.dpm.ps)
4027                 return -ENOMEM;
4028         rdev->pm.dpm.platform_caps = le32_to_cpu(power_info->pplib.ulPlatformCaps);
4029         rdev->pm.dpm.backbias_response_time = le16_to_cpu(power_info->pplib.usBackbiasTime);
4030         rdev->pm.dpm.voltage_response_time = le16_to_cpu(power_info->pplib.usVoltageTime);
4031
4032         for (i = 0; i < power_info->pplib.ucNumStates; i++) {
4033                 power_state = (union pplib_power_state *)
4034                         (mode_info->atom_context->bios + data_offset +
4035                          le16_to_cpu(power_info->pplib.usStateArrayOffset) +
4036                          i * power_info->pplib.ucStateEntrySize);
4037                 non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
4038                         (mode_info->atom_context->bios + data_offset +
4039                          le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset) +
4040                          (power_state->v1.ucNonClockStateIndex *
4041                           power_info->pplib.ucNonClockSize));
4042                 if (power_info->pplib.ucStateEntrySize - 1) {
4043                         u8 *idx;
4044                         ps = kzalloc(sizeof(struct ni_ps), GFP_KERNEL);
4045                         if (ps == NULL) {
4046                                 kfree(rdev->pm.dpm.ps);
4047                                 return -ENOMEM;
4048                         }
4049                         rdev->pm.dpm.ps[i].ps_priv = ps;
4050                         ni_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
4051                                                          non_clock_info,
4052                                                          power_info->pplib.ucNonClockSize);
4053                         idx = (u8 *)&power_state->v1.ucClockStateIndices[0];
4054                         for (j = 0; j < (power_info->pplib.ucStateEntrySize - 1); j++) {
4055                                 clock_info = (union pplib_clock_info *)
4056                                         (mode_info->atom_context->bios + data_offset +
4057                                          le16_to_cpu(power_info->pplib.usClockInfoArrayOffset) +
4058                                          (idx[j] * power_info->pplib.ucClockInfoSize));
4059                                 ni_parse_pplib_clock_info(rdev,
4060                                                           &rdev->pm.dpm.ps[i], j,
4061                                                           clock_info);
4062                         }
4063                 }
4064         }
4065         rdev->pm.dpm.num_ps = power_info->pplib.ucNumStates;
4066         return 0;
4067 }
4068
4069 int ni_dpm_init(struct radeon_device *rdev)
4070 {
4071         struct rv7xx_power_info *pi;
4072         struct evergreen_power_info *eg_pi;
4073         struct ni_power_info *ni_pi;
4074         struct atom_clock_dividers dividers;
4075         int ret;
4076
4077         ni_pi = kzalloc(sizeof(struct ni_power_info), GFP_KERNEL);
4078         if (ni_pi == NULL)
4079                 return -ENOMEM;
4080         rdev->pm.dpm.priv = ni_pi;
4081         eg_pi = &ni_pi->eg;
4082         pi = &eg_pi->rv7xx;
4083
4084         rv770_get_max_vddc(rdev);
4085
4086         eg_pi->ulv.supported = false;
4087         pi->acpi_vddc = 0;
4088         eg_pi->acpi_vddci = 0;
4089         pi->min_vddc_in_table = 0;
4090         pi->max_vddc_in_table = 0;
4091
4092         ret = ni_parse_power_table(rdev);
4093         if (ret)
4094                 return ret;
4095         ret = r600_parse_extended_power_table(rdev);
4096         if (ret)
4097                 return ret;
4098
4099         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries =
4100                 kzalloc(4 * sizeof(struct radeon_clock_voltage_dependency_entry), GFP_KERNEL);
4101         if (!rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries) {
4102                 r600_free_extended_power_table(rdev);
4103                 return -ENOMEM;
4104         }
4105         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.count = 4;
4106         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].clk = 0;
4107         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].v = 0;
4108         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].clk = 36000;
4109         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].v = 720;
4110         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].clk = 54000;
4111         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].v = 810;
4112         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].clk = 72000;
4113         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].v = 900;
4114
4115         ni_patch_dependency_tables_based_on_leakage(rdev);
4116
4117         if (rdev->pm.dpm.voltage_response_time == 0)
4118                 rdev->pm.dpm.voltage_response_time = R600_VOLTAGERESPONSETIME_DFLT;
4119         if (rdev->pm.dpm.backbias_response_time == 0)
4120                 rdev->pm.dpm.backbias_response_time = R600_BACKBIASRESPONSETIME_DFLT;
4121
4122         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
4123                                              0, false, &dividers);
4124         if (ret)
4125                 pi->ref_div = dividers.ref_div + 1;
4126         else
4127                 pi->ref_div = R600_REFERENCEDIVIDER_DFLT;
4128
4129         pi->rlp = RV770_RLP_DFLT;
4130         pi->rmp = RV770_RMP_DFLT;
4131         pi->lhp = RV770_LHP_DFLT;
4132         pi->lmp = RV770_LMP_DFLT;
4133
4134         eg_pi->ats[0].rlp = RV770_RLP_DFLT;
4135         eg_pi->ats[0].rmp = RV770_RMP_DFLT;
4136         eg_pi->ats[0].lhp = RV770_LHP_DFLT;
4137         eg_pi->ats[0].lmp = RV770_LMP_DFLT;
4138
4139         eg_pi->ats[1].rlp = BTC_RLP_UVD_DFLT;
4140         eg_pi->ats[1].rmp = BTC_RMP_UVD_DFLT;
4141         eg_pi->ats[1].lhp = BTC_LHP_UVD_DFLT;
4142         eg_pi->ats[1].lmp = BTC_LMP_UVD_DFLT;
4143
4144         eg_pi->smu_uvd_hs = true;
4145
4146         if (rdev->pdev->device == 0x6707) {
4147                 pi->mclk_strobe_mode_threshold = 55000;
4148                 pi->mclk_edc_enable_threshold = 55000;
4149                 eg_pi->mclk_edc_wr_enable_threshold = 55000;
4150         } else {
4151                 pi->mclk_strobe_mode_threshold = 40000;
4152                 pi->mclk_edc_enable_threshold = 40000;
4153                 eg_pi->mclk_edc_wr_enable_threshold = 40000;
4154         }
4155         ni_pi->mclk_rtt_mode_threshold = eg_pi->mclk_edc_wr_enable_threshold;
4156
4157         pi->voltage_control =
4158                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, 0);
4159
4160         pi->mvdd_control =
4161                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_MVDDC, 0);
4162
4163         eg_pi->vddci_control =
4164                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDCI, 0);
4165
4166         rv770_get_engine_memory_ss(rdev);
4167
4168         pi->asi = RV770_ASI_DFLT;
4169         pi->pasi = CYPRESS_HASI_DFLT;
4170         pi->vrc = CYPRESS_VRC_DFLT;
4171
4172         pi->power_gating = false;
4173
4174         pi->gfx_clock_gating = true;
4175
4176         pi->mg_clock_gating = true;
4177         pi->mgcgtssm = true;
4178         eg_pi->ls_clock_gating = false;
4179         eg_pi->sclk_deep_sleep = false;
4180
4181         pi->dynamic_pcie_gen2 = true;
4182
4183         if (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE)
4184                 pi->thermal_protection = true;
4185         else
4186                 pi->thermal_protection = false;
4187
4188         pi->display_gap = true;
4189
4190         pi->dcodt = true;
4191
4192         pi->ulps = true;
4193
4194         eg_pi->dynamic_ac_timing = true;
4195         eg_pi->abm = true;
4196         eg_pi->mcls = true;
4197         eg_pi->light_sleep = true;
4198         eg_pi->memory_transition = true;
4199 #if defined(CONFIG_ACPI)
4200         eg_pi->pcie_performance_request =
4201                 radeon_acpi_is_pcie_performance_request_supported(rdev);
4202 #else
4203         eg_pi->pcie_performance_request = false;
4204 #endif
4205
4206         eg_pi->dll_default_on = false;
4207
4208         eg_pi->sclk_deep_sleep = false;
4209
4210         pi->mclk_stutter_mode_threshold = 0;
4211
4212         pi->sram_end = SMC_RAM_END;
4213
4214         rdev->pm.dpm.dyn_state.mclk_sclk_ratio = 3;
4215         rdev->pm.dpm.dyn_state.vddc_vddci_delta = 200;
4216         rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2 = 900;
4217         rdev->pm.dpm.dyn_state.valid_sclk_values.count = ARRAY_SIZE(btc_valid_sclk);
4218         rdev->pm.dpm.dyn_state.valid_sclk_values.values = btc_valid_sclk;
4219         rdev->pm.dpm.dyn_state.valid_mclk_values.count = 0;
4220         rdev->pm.dpm.dyn_state.valid_mclk_values.values = NULL;
4221         rdev->pm.dpm.dyn_state.sclk_mclk_delta = 12500;
4222
4223         ni_pi->cac_data.leakage_coefficients.at = 516;
4224         ni_pi->cac_data.leakage_coefficients.bt = 18;
4225         ni_pi->cac_data.leakage_coefficients.av = 51;
4226         ni_pi->cac_data.leakage_coefficients.bv = 2957;
4227
4228         switch (rdev->pdev->device) {
4229         case 0x6700:
4230         case 0x6701:
4231         case 0x6702:
4232         case 0x6703:
4233         case 0x6718:
4234                 ni_pi->cac_weights = &cac_weights_cayman_xt;
4235                 break;
4236         case 0x6705:
4237         case 0x6719:
4238         case 0x671D:
4239         case 0x671C:
4240         default:
4241                 ni_pi->cac_weights = &cac_weights_cayman_pro;
4242                 break;
4243         case 0x6704:
4244         case 0x6706:
4245         case 0x6707:
4246         case 0x6708:
4247         case 0x6709:
4248                 ni_pi->cac_weights = &cac_weights_cayman_le;
4249                 break;
4250         }
4251
4252         if (ni_pi->cac_weights->enable_power_containment_by_default) {
4253                 ni_pi->enable_power_containment = true;
4254                 ni_pi->enable_cac = true;
4255                 ni_pi->enable_sq_ramping = true;
4256         } else {
4257                 ni_pi->enable_power_containment = false;
4258                 ni_pi->enable_cac = false;
4259                 ni_pi->enable_sq_ramping = false;
4260         }
4261
4262         ni_pi->driver_calculate_cac_leakage = false;
4263         ni_pi->cac_configuration_required = true;
4264
4265         if (ni_pi->cac_configuration_required) {
4266                 ni_pi->support_cac_long_term_average = true;
4267                 ni_pi->lta_window_size = ni_pi->cac_weights->l2_lta_window_size;
4268                 ni_pi->lts_truncate = ni_pi->cac_weights->lts_truncate;
4269         } else {
4270                 ni_pi->support_cac_long_term_average = false;
4271                 ni_pi->lta_window_size = 0;
4272                 ni_pi->lts_truncate = 0;
4273         }
4274
4275         ni_pi->use_power_boost_limit = true;
4276
4277         /* make sure dc limits are valid */
4278         if ((rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.sclk == 0) ||
4279             (rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.mclk == 0))
4280                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc =
4281                         rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
4282
4283         return 0;
4284 }
4285
4286 void ni_dpm_fini(struct radeon_device *rdev)
4287 {
4288         int i;
4289
4290         for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
4291                 kfree(rdev->pm.dpm.ps[i].ps_priv);
4292         }
4293         kfree(rdev->pm.dpm.ps);
4294         kfree(rdev->pm.dpm.priv);
4295         kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries);
4296         r600_free_extended_power_table(rdev);
4297 }
4298
4299 void ni_dpm_print_power_state(struct radeon_device *rdev,
4300                               struct radeon_ps *rps)
4301 {
4302         struct ni_ps *ps = ni_get_ps(rps);
4303         struct rv7xx_pl *pl;
4304         int i;
4305
4306         r600_dpm_print_class_info(rps->class, rps->class2);
4307         r600_dpm_print_cap_info(rps->caps);
4308         printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
4309         for (i = 0; i < ps->performance_level_count; i++) {
4310                 pl = &ps->performance_levels[i];
4311                 if (rdev->family >= CHIP_TAHITI)
4312                         printk("\t\tpower level %d    sclk: %u mclk: %u vddc: %u vddci: %u pcie gen: %u\n",
4313                                i, pl->sclk, pl->mclk, pl->vddc, pl->vddci, pl->pcie_gen + 1);
4314                 else
4315                         printk("\t\tpower level %d    sclk: %u mclk: %u vddc: %u vddci: %u\n",
4316                                i, pl->sclk, pl->mclk, pl->vddc, pl->vddci);
4317         }
4318         r600_dpm_print_ps_status(rdev, rps);
4319 }
4320
4321 void ni_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
4322                                                     struct seq_file *m)
4323 {
4324         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4325         struct radeon_ps *rps = &eg_pi->current_rps;
4326         struct ni_ps *ps = ni_get_ps(rps);
4327         struct rv7xx_pl *pl;
4328         u32 current_index =
4329                 (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >>
4330                 CURRENT_STATE_INDEX_SHIFT;
4331
4332         if (current_index >= ps->performance_level_count) {
4333                 seq_printf(m, "invalid dpm profile %d\n", current_index);
4334         } else {
4335                 pl = &ps->performance_levels[current_index];
4336                 seq_printf(m, "uvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
4337                 seq_printf(m, "power level %d    sclk: %u mclk: %u vddc: %u vddci: %u\n",
4338                            current_index, pl->sclk, pl->mclk, pl->vddc, pl->vddci);
4339         }
4340 }
4341
4342 u32 ni_dpm_get_sclk(struct radeon_device *rdev, bool low)
4343 {
4344         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4345         struct ni_ps *requested_state = ni_get_ps(&eg_pi->requested_rps);
4346
4347         if (low)
4348                 return requested_state->performance_levels[0].sclk;
4349         else
4350                 return requested_state->performance_levels[requested_state->performance_level_count - 1].sclk;
4351 }
4352
4353 u32 ni_dpm_get_mclk(struct radeon_device *rdev, bool low)
4354 {
4355         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4356         struct ni_ps *requested_state = ni_get_ps(&eg_pi->requested_rps);
4357
4358         if (low)
4359                 return requested_state->performance_levels[0].mclk;
4360         else
4361                 return requested_state->performance_levels[requested_state->performance_level_count - 1].mclk;
4362 }
4363