Pileus Git - ~andy/linux/blob - arch/arm/kernel/perf_event_v7.c

   1 /*
   2  * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code.
   3  *
   4  * ARMv7 support: Jean Pihet <jpihet@mvista.com>
   5  * 2010 (c) MontaVista Software, LLC.
   6  *
   7  * Copied from ARMv6 code, with the low level code inspired
   8  *  by the ARMv7 Oprofile code.
   9  *
  10  * Cortex-A8 has up to 4 configurable performance counters and
  11  *  a single cycle counter.
  12  * Cortex-A9 has up to 31 configurable performance counters and
  13  *  a single cycle counter.
  14  *
  15  * All counters can be enabled/disabled and IRQ masked separately. The cycle
  16  *  counter and all 4 performance counters together can be reset separately.
  17  */
  18
  19 #ifdef CONFIG_CPU_V7
  20
  21 static struct arm_pmu armv7pmu;
  22
  23 /*
  24  * Common ARMv7 event types
  25  *
  26  * Note: An implementation may not be able to count all of these events
  27  * but the encodings are considered to be `reserved' in the case that
  28  * they are not available.
  29  */
  30 enum armv7_perf_types {
  31         ARMV7_PERFCTR_PMNC_SW_INCR                      = 0x00,
  32         ARMV7_PERFCTR_L1_ICACHE_REFILL                  = 0x01,
  33         ARMV7_PERFCTR_ITLB_REFILL                       = 0x02,
  34         ARMV7_PERFCTR_L1_DCACHE_REFILL                  = 0x03,
  35         ARMV7_PERFCTR_L1_DCACHE_ACCESS                  = 0x04,
  36         ARMV7_PERFCTR_DTLB_REFILL                       = 0x05,
  37         ARMV7_PERFCTR_MEM_READ                          = 0x06,
  38         ARMV7_PERFCTR_MEM_WRITE                         = 0x07,
  39         ARMV7_PERFCTR_INSTR_EXECUTED                    = 0x08,
  40         ARMV7_PERFCTR_EXC_TAKEN                         = 0x09,
  41         ARMV7_PERFCTR_EXC_EXECUTED                      = 0x0A,
  42         ARMV7_PERFCTR_CID_WRITE                         = 0x0B,
  43
  44         /*
  45          * ARMV7_PERFCTR_PC_WRITE is equivalent to HW_BRANCH_INSTRUCTIONS.
  46          * It counts:
  47          *  - all (taken) branch instructions,
  48          *  - instructions that explicitly write the PC,
  49          *  - exception generating instructions.
  50          */
  51         ARMV7_PERFCTR_PC_WRITE                          = 0x0C,
  52         ARMV7_PERFCTR_PC_IMM_BRANCH                     = 0x0D,
  53         ARMV7_PERFCTR_PC_PROC_RETURN                    = 0x0E,
  54         ARMV7_PERFCTR_MEM_UNALIGNED_ACCESS              = 0x0F,
  55         ARMV7_PERFCTR_PC_BRANCH_MIS_PRED                = 0x10,
  56         ARMV7_PERFCTR_CLOCK_CYCLES                      = 0x11,
  57         ARMV7_PERFCTR_PC_BRANCH_PRED                    = 0x12,
  58
  59         /* These events are defined by the PMUv2 supplement (ARM DDI 0457A). */
  60         ARMV7_PERFCTR_MEM_ACCESS                        = 0x13,
  61         ARMV7_PERFCTR_L1_ICACHE_ACCESS                  = 0x14,
  62         ARMV7_PERFCTR_L1_DCACHE_WB                      = 0x15,
  63         ARMV7_PERFCTR_L2_CACHE_ACCESS                   = 0x16,
  64         ARMV7_PERFCTR_L2_CACHE_REFILL                   = 0x17,
  65         ARMV7_PERFCTR_L2_CACHE_WB                       = 0x18,
  66         ARMV7_PERFCTR_BUS_ACCESS                        = 0x19,
  67         ARMV7_PERFCTR_MEM_ERROR                         = 0x1A,
  68         ARMV7_PERFCTR_INSTR_SPEC                        = 0x1B,
  69         ARMV7_PERFCTR_TTBR_WRITE                        = 0x1C,
  70         ARMV7_PERFCTR_BUS_CYCLES                        = 0x1D,
  71
  72         ARMV7_PERFCTR_CPU_CYCLES                        = 0xFF
  73 };
  74
  75 /* ARMv7 Cortex-A8 specific event types */
  76 enum armv7_a8_perf_types {
  77         ARMV7_A8_PERFCTR_L2_CACHE_ACCESS                = 0x43,
  78         ARMV7_A8_PERFCTR_L2_CACHE_REFILL                = 0x44,
  79         ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS               = 0x50,
  80         ARMV7_A8_PERFCTR_STALL_ISIDE                    = 0x56,
  81 };
  82
  83 /* ARMv7 Cortex-A9 specific event types */
  84 enum armv7_a9_perf_types {
  85         ARMV7_A9_PERFCTR_INSTR_CORE_RENAME              = 0x68,
  86         ARMV7_A9_PERFCTR_STALL_ICACHE                   = 0x60,
  87         ARMV7_A9_PERFCTR_STALL_DISPATCH                 = 0x66,
  88 };
  89
  90 /* ARMv7 Cortex-A5 specific event types */
  91 enum armv7_a5_perf_types {
  92         ARMV7_A5_PERFCTR_PREFETCH_LINEFILL              = 0xc2,
  93         ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP         = 0xc3,
  94 };
  95
  96 /* ARMv7 Cortex-A15 specific event types */
  97 enum armv7_a15_perf_types {
  98         ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ         = 0x40,
  99         ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE        = 0x41,
 100         ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ         = 0x42,
 101         ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE        = 0x43,
 102
 103         ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ           = 0x4C,
 104         ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE          = 0x4D,
 105
 106         ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ          = 0x50,
 107         ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE         = 0x51,
 108         ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ          = 0x52,
 109         ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE         = 0x53,
 110
 111         ARMV7_A15_PERFCTR_PC_WRITE_SPEC                 = 0x76,
 112 };
 113
 114 /*
 115  * Cortex-A8 HW events mapping
 116  *
 117  * The hardware events that we support. We do support cache operations but
 118  * we have harvard caches and no way to combine instruction and data
 119  * accesses/misses in hardware.
 120  */
 121 static const unsigned armv7_a8_perf_map[PERF_COUNT_HW_MAX] = {
 122         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 123         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
 124         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 125         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 126         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_PERFCTR_PC_WRITE,
 127         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 128         [PERF_COUNT_HW_BUS_CYCLES]              = HW_OP_UNSUPPORTED,
 129         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A8_PERFCTR_STALL_ISIDE,
 130         [PERF_COUNT_HW_STALLED_CYCLES_BACKEND]  = HW_OP_UNSUPPORTED,
 131 };
 132
 133 static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 134                                           [PERF_COUNT_HW_CACHE_OP_MAX]
 135                                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 136         [C(L1D)] = {
 137                 /*
 138                  * The performance counters don't differentiate between read
 139                  * and write accesses/misses so this isn't strictly correct,
 140                  * but it's the best we can do. Writes and reads get
 141                  * combined.
 142                  */
 143                 [C(OP_READ)] = {
 144                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 145                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 146                 },
 147                 [C(OP_WRITE)] = {
 148                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 149                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 150                 },
 151                 [C(OP_PREFETCH)] = {
 152                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 153                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 154                 },
 155         },
 156         [C(L1I)] = {
 157                 [C(OP_READ)] = {
 158                         [C(RESULT_ACCESS)]      = ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS,
 159                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 160                 },
 161                 [C(OP_WRITE)] = {
 162                         [C(RESULT_ACCESS)]      = ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS,
 163                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 164                 },
 165                 [C(OP_PREFETCH)] = {
 166                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 167                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 168                 },
 169         },
 170         [C(LL)] = {
 171                 [C(OP_READ)] = {
 172                         [C(RESULT_ACCESS)]      = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,
 173                         [C(RESULT_MISS)]        = ARMV7_A8_PERFCTR_L2_CACHE_REFILL,
 174                 },
 175                 [C(OP_WRITE)] = {
 176                         [C(RESULT_ACCESS)]      = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,
 177                         [C(RESULT_MISS)]        = ARMV7_A8_PERFCTR_L2_CACHE_REFILL,
 178                 },
 179                 [C(OP_PREFETCH)] = {
 180                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 181                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 182                 },
 183         },
 184         [C(DTLB)] = {
 185                 [C(OP_READ)] = {
 186                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 187                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
 188                 },
 189                 [C(OP_WRITE)] = {
 190                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 191                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
 192                 },
 193                 [C(OP_PREFETCH)] = {
 194                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 195                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 196                 },
 197         },
 198         [C(ITLB)] = {
 199                 [C(OP_READ)] = {
 200                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 201                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_REFILL,
 202                 },
 203                 [C(OP_WRITE)] = {
 204                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 205                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_REFILL,
 206                 },
 207                 [C(OP_PREFETCH)] = {
 208                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 209                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 210                 },
 211         },
 212         [C(BPU)] = {
 213                 [C(OP_READ)] = {
 214                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_BRANCH_PRED,
 215                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 216                 },
 217                 [C(OP_WRITE)] = {
 218                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_BRANCH_PRED,
 219                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 220                 },
 221                 [C(OP_PREFETCH)] = {
 222                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 223                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 224                 },
 225         },
 226         [C(NODE)] = {
 227                 [C(OP_READ)] = {
 228                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 229                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 230                 },
 231                 [C(OP_WRITE)] = {
 232                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 233                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 234                 },
 235                 [C(OP_PREFETCH)] = {
 236                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 237                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 238                 },
 239         },
 240 };
 241
 242 /*
 243  * Cortex-A9 HW events mapping
 244  */
 245 static const unsigned armv7_a9_perf_map[PERF_COUNT_HW_MAX] = {
 246         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 247         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_A9_PERFCTR_INSTR_CORE_RENAME,
 248         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 249         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 250         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_PERFCTR_PC_WRITE,
 251         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 252         [PERF_COUNT_HW_BUS_CYCLES]              = HW_OP_UNSUPPORTED,
 253         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A9_PERFCTR_STALL_ICACHE,
 254         [PERF_COUNT_HW_STALLED_CYCLES_BACKEND]  = ARMV7_A9_PERFCTR_STALL_DISPATCH,
 255 };
 256
 257 static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 258                                           [PERF_COUNT_HW_CACHE_OP_MAX]
 259                                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 260         [C(L1D)] = {
 261                 /*
 262                  * The performance counters don't differentiate between read
 263                  * and write accesses/misses so this isn't strictly correct,
 264                  * but it's the best we can do. Writes and reads get
 265                  * combined.
 266                  */
 267                 [C(OP_READ)] = {
 268                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 269                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 270                 },
 271                 [C(OP_WRITE)] = {
 272                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 273                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 274                 },
 275                 [C(OP_PREFETCH)] = {
 276                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 277                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 278                 },
 279         },
 280         [C(L1I)] = {
 281                 [C(OP_READ)] = {
 282                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 283                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 284                 },
 285                 [C(OP_WRITE)] = {
 286                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 287                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 288                 },
 289                 [C(OP_PREFETCH)] = {
 290                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 291                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 292                 },
 293         },
 294         [C(LL)] = {
 295                 [C(OP_READ)] = {
 296                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 297                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 298                 },
 299                 [C(OP_WRITE)] = {
 300                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 301                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 302                 },
 303                 [C(OP_PREFETCH)] = {
 304                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 305                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 306                 },
 307         },
 308         [C(DTLB)] = {
 309                 [C(OP_READ)] = {
 310                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 311                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
 312                 },
 313                 [C(OP_WRITE)] = {
 314                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 315                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
 316                 },
 317                 [C(OP_PREFETCH)] = {
 318                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 319                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 320                 },
 321         },
 322         [C(ITLB)] = {
 323                 [C(OP_READ)] = {
 324                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 325                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_REFILL,
 326                 },
 327                 [C(OP_WRITE)] = {
 328                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 329                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_REFILL,
 330                 },
 331                 [C(OP_PREFETCH)] = {
 332                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 333                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 334                 },
 335         },
 336         [C(BPU)] = {
 337                 [C(OP_READ)] = {
 338                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_BRANCH_PRED,
 339                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 340                 },
 341                 [C(OP_WRITE)] = {
 342                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_BRANCH_PRED,
 343                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 344                 },
 345                 [C(OP_PREFETCH)] = {
 346                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 347                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 348                 },
 349         },
 350         [C(NODE)] = {
 351                 [C(OP_READ)] = {
 352                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 353                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 354                 },
 355                 [C(OP_WRITE)] = {
 356                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 357                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 358                 },
 359                 [C(OP_PREFETCH)] = {
 360                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 361                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 362                 },
 363         },
 364 };
 365
 366 /*
 367  * Cortex-A5 HW events mapping
 368  */
 369 static const unsigned armv7_a5_perf_map[PERF_COUNT_HW_MAX] = {
 370         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 371         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
 372         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 373         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 374         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_PERFCTR_PC_WRITE,
 375         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 376         [PERF_COUNT_HW_BUS_CYCLES]              = HW_OP_UNSUPPORTED,
 377         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = HW_OP_UNSUPPORTED,
 378         [PERF_COUNT_HW_STALLED_CYCLES_BACKEND]  = HW_OP_UNSUPPORTED,
 379 };
 380
 381 static const unsigned armv7_a5_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 382                                         [PERF_COUNT_HW_CACHE_OP_MAX]
 383                                         [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 384         [C(L1D)] = {
 385                 [C(OP_READ)] = {
 386                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 387                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 388                 },
 389                 [C(OP_WRITE)] = {
 390                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 391                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 392                 },
 393                 [C(OP_PREFETCH)] = {
 394                         [C(RESULT_ACCESS)]      = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,
 395                         [C(RESULT_MISS)]        = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,
 396                 },
 397         },
 398         [C(L1I)] = {
 399                 [C(OP_READ)] = {
 400                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
 401                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 402                 },
 403                 [C(OP_WRITE)] = {
 404                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
 405                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 406                 },
 407                 /*
 408                  * The prefetch counters don't differentiate between the I
 409                  * side and the D side.
 410                  */
 411                 [C(OP_PREFETCH)] = {
 412                         [C(RESULT_ACCESS)]      = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,
 413                         [C(RESULT_MISS)]        = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,
 414                 },
 415         },
 416         [C(LL)] = {
 417                 [C(OP_READ)] = {
 418                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 419                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 420                 },
 421                 [C(OP_WRITE)] = {
 422                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 423                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 424                 },
 425                 [C(OP_PREFETCH)] = {
 426                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 427                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 428                 },
 429         },
 430         [C(DTLB)] = {
 431                 [C(OP_READ)] = {
 432                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 433                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
 434                 },
 435                 [C(OP_WRITE)] = {
 436                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 437                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
 438                 },
 439                 [C(OP_PREFETCH)] = {
 440                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 441                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 442                 },
 443         },
 444         [C(ITLB)] = {
 445                 [C(OP_READ)] = {
 446                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 447                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_REFILL,
 448                 },
 449                 [C(OP_WRITE)] = {
 450                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 451                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_REFILL,
 452                 },
 453                 [C(OP_PREFETCH)] = {
 454                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 455                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 456                 },
 457         },
 458         [C(BPU)] = {
 459                 [C(OP_READ)] = {
 460                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_BRANCH_PRED,
 461                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 462                 },
 463                 [C(OP_WRITE)] = {
 464                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_BRANCH_PRED,
 465                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 466                 },
 467                 [C(OP_PREFETCH)] = {
 468                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 469                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 470                 },
 471         },
 472         [C(NODE)] = {
 473                 [C(OP_READ)] = {
 474                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 475                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 476                 },
 477                 [C(OP_WRITE)] = {
 478                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 479                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 480                 },
 481                 [C(OP_PREFETCH)] = {
 482                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 483                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 484                 },
 485         },
 486 };
 487
 488 /*
 489  * Cortex-A15 HW events mapping
 490  */
 491 static const unsigned armv7_a15_perf_map[PERF_COUNT_HW_MAX] = {
 492         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 493         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
 494         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 495         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 496         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_A15_PERFCTR_PC_WRITE_SPEC,
 497         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 498         [PERF_COUNT_HW_BUS_CYCLES]              = ARMV7_PERFCTR_BUS_CYCLES,
 499         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = HW_OP_UNSUPPORTED,
 500         [PERF_COUNT_HW_STALLED_CYCLES_BACKEND]  = HW_OP_UNSUPPORTED,
 501 };
 502
 503 static const unsigned armv7_a15_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 504                                         [PERF_COUNT_HW_CACHE_OP_MAX]
 505                                         [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 506         [C(L1D)] = {
 507                 [C(OP_READ)] = {
 508                         [C(RESULT_ACCESS)]      = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ,
 509                         [C(RESULT_MISS)]        = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ,
 510                 },
 511                 [C(OP_WRITE)] = {
 512                         [C(RESULT_ACCESS)]      = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE,
 513                         [C(RESULT_MISS)]        = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE,
 514                 },
 515                 [C(OP_PREFETCH)] = {
 516                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 517                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 518                 },
 519         },
 520         [C(L1I)] = {
 521                 /*
 522                  * Not all performance counters differentiate between read
 523                  * and write accesses/misses so we're not always strictly
 524                  * correct, but it's the best we can do. Writes and reads get
 525                  * combined in these cases.
 526                  */
 527                 [C(OP_READ)] = {
 528                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
 529                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 530                 },
 531                 [C(OP_WRITE)] = {
 532                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
 533                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 534                 },
 535                 [C(OP_PREFETCH)] = {
 536                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 537                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 538                 },
 539         },
 540         [C(LL)] = {
 541                 [C(OP_READ)] = {
 542                         [C(RESULT_ACCESS)]      = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ,
 543                         [C(RESULT_MISS)]        = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ,
 544                 },
 545                 [C(OP_WRITE)] = {
 546                         [C(RESULT_ACCESS)]      = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE,
 547                         [C(RESULT_MISS)]        = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE,
 548                 },
 549                 [C(OP_PREFETCH)] = {
 550                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 551                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 552                 },
 553         },
 554         [C(DTLB)] = {
 555                 [C(OP_READ)] = {
 556                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 557                         [C(RESULT_MISS)]        = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ,
 558                 },
 559                 [C(OP_WRITE)] = {
 560                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 561                         [C(RESULT_MISS)]        = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE,
 562                 },
 563                 [C(OP_PREFETCH)] = {
 564                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 565                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 566                 },
 567         },
 568         [C(ITLB)] = {
 569                 [C(OP_READ)] = {
 570                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 571                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_REFILL,
 572                 },
 573                 [C(OP_WRITE)] = {
 574                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 575                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_REFILL,
 576                 },
 577                 [C(OP_PREFETCH)] = {
 578                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 579                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 580                 },
 581         },
 582         [C(BPU)] = {
 583                 [C(OP_READ)] = {
 584                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_BRANCH_PRED,
 585                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 586                 },
 587                 [C(OP_WRITE)] = {
 588                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_BRANCH_PRED,
 589                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 590                 },
 591                 [C(OP_PREFETCH)] = {
 592                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 593                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 594                 },
 595         },
 596         [C(NODE)] = {
 597                 [C(OP_READ)] = {
 598                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 599                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 600                 },
 601                 [C(OP_WRITE)] = {
 602                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 603                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 604                 },
 605                 [C(OP_PREFETCH)] = {
 606                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 607                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 608                 },
 609         },
 610 };
 611
 612 /*
 613  * Cortex-A7 HW events mapping
 614  */
 615 static const unsigned armv7_a7_perf_map[PERF_COUNT_HW_MAX] = {
 616         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 617         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
 618         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 619         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 620         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_PERFCTR_PC_WRITE,
 621         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 622         [PERF_COUNT_HW_BUS_CYCLES]              = ARMV7_PERFCTR_BUS_CYCLES,
 623         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = HW_OP_UNSUPPORTED,
 624         [PERF_COUNT_HW_STALLED_CYCLES_BACKEND]  = HW_OP_UNSUPPORTED,
 625 };
 626
 627 static const unsigned armv7_a7_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 628                                         [PERF_COUNT_HW_CACHE_OP_MAX]
 629                                         [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 630         [C(L1D)] = {
 631                 /*
 632                  * The performance counters don't differentiate between read
 633                  * and write accesses/misses so this isn't strictly correct,
 634                  * but it's the best we can do. Writes and reads get
 635                  * combined.
 636                  */
 637                 [C(OP_READ)] = {
 638                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 639                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 640                 },
 641                 [C(OP_WRITE)] = {
 642                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 643                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 644                 },
 645                 [C(OP_PREFETCH)] = {
 646                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 647                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 648                 },
 649         },
 650         [C(L1I)] = {
 651                 [C(OP_READ)] = {
 652                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
 653                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 654                 },
 655                 [C(OP_WRITE)] = {
 656                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
 657                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 658                 },
 659                 [C(OP_PREFETCH)] = {
 660                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 661                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 662                 },
 663         },
 664         [C(LL)] = {
 665                 [C(OP_READ)] = {
 666                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L2_CACHE_ACCESS,
 667                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L2_CACHE_REFILL,
 668                 },
 669                 [C(OP_WRITE)] = {
 670                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L2_CACHE_ACCESS,
 671                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L2_CACHE_REFILL,
 672                 },
 673                 [C(OP_PREFETCH)] = {
 674                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 675                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 676                 },
 677         },
 678         [C(DTLB)] = {
 679                 [C(OP_READ)] = {
 680                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 681                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
 682                 },
 683                 [C(OP_WRITE)] = {
 684                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 685                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
 686                 },
 687                 [C(OP_PREFETCH)] = {
 688                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 689                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 690                 },
 691         },
 692         [C(ITLB)] = {
 693                 [C(OP_READ)] = {
 694                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 695                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_REFILL,
 696                 },
 697                 [C(OP_WRITE)] = {
 698                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 699                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_REFILL,
 700                 },
 701                 [C(OP_PREFETCH)] = {
 702                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 703                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 704                 },
 705         },
 706         [C(BPU)] = {
 707                 [C(OP_READ)] = {
 708                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_BRANCH_PRED,
 709                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 710                 },
 711                 [C(OP_WRITE)] = {
 712                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_BRANCH_PRED,
 713                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 714                 },
 715                 [C(OP_PREFETCH)] = {
 716                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 717                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 718                 },
 719         },
 720         [C(NODE)] = {
 721                 [C(OP_READ)] = {
 722                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 723                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 724                 },
 725                 [C(OP_WRITE)] = {
 726                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 727                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 728                 },
 729                 [C(OP_PREFETCH)] = {
 730                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 731                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 732                 },
 733         },
 734 };
 735
 736 /*
 737  * Perf Events' indices
 738  */
 739 #define ARMV7_IDX_CYCLE_COUNTER 0
 740 #define ARMV7_IDX_COUNTER0      1
 741 #define ARMV7_IDX_COUNTER_LAST  (ARMV7_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1)
 742
 743 #define ARMV7_MAX_COUNTERS      32
 744 #define ARMV7_COUNTER_MASK      (ARMV7_MAX_COUNTERS - 1)
 745
 746 /*
 747  * ARMv7 low level PMNC access
 748  */
 749
 750 /*
 751  * Perf Event to low level counters mapping
 752  */
 753 #define ARMV7_IDX_TO_COUNTER(x) \
 754         (((x) - ARMV7_IDX_COUNTER0) & ARMV7_COUNTER_MASK)
 755
 756 /*
 757  * Per-CPU PMNC: config reg
 758  */
 759 #define ARMV7_PMNC_E            (1 << 0) /* Enable all counters */
 760 #define ARMV7_PMNC_P            (1 << 1) /* Reset all counters */
 761 #define ARMV7_PMNC_C            (1 << 2) /* Cycle counter reset */
 762 #define ARMV7_PMNC_D            (1 << 3) /* CCNT counts every 64th cpu cycle */
 763 #define ARMV7_PMNC_X            (1 << 4) /* Export to ETM */
 764 #define ARMV7_PMNC_DP           (1 << 5) /* Disable CCNT if non-invasive debug*/
 765 #define ARMV7_PMNC_N_SHIFT      11       /* Number of counters supported */
 766 #define ARMV7_PMNC_N_MASK       0x1f
 767 #define ARMV7_PMNC_MASK         0x3f     /* Mask for writable bits */
 768
 769 /*
 770  * FLAG: counters overflow flag status reg
 771  */
 772 #define ARMV7_FLAG_MASK         0xffffffff      /* Mask for writable bits */
 773 #define ARMV7_OVERFLOWED_MASK   ARMV7_FLAG_MASK
 774
 775 /*
 776  * PMXEVTYPER: Event selection reg
 777  */
 778 #define ARMV7_EVTYPE_MASK       0xc00000ff      /* Mask for writable bits */
 779 #define ARMV7_EVTYPE_EVENT      0xff            /* Mask for EVENT bits */
 780
 781 /*
 782  * Event filters for PMUv2
 783  */
 784 #define ARMV7_EXCLUDE_PL1       (1 << 31)
 785 #define ARMV7_EXCLUDE_USER      (1 << 30)
 786 #define ARMV7_INCLUDE_HYP       (1 << 27)
 787
 788 static inline u32 armv7_pmnc_read(void)
 789 {
 790         u32 val;
 791         asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(val));
 792         return val;
 793 }
 794
 795 static inline void armv7_pmnc_write(u32 val)
 796 {
 797         val &= ARMV7_PMNC_MASK;
 798         isb();
 799         asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(val));
 800 }
 801
 802 static inline int armv7_pmnc_has_overflowed(u32 pmnc)
 803 {
 804         return pmnc & ARMV7_OVERFLOWED_MASK;
 805 }
 806
 807 static inline int armv7_pmnc_counter_valid(int idx)
 808 {
 809         return idx >= ARMV7_IDX_CYCLE_COUNTER && idx <= ARMV7_IDX_COUNTER_LAST;
 810 }
 811
 812 static inline int armv7_pmnc_counter_has_overflowed(u32 pmnc, int idx)
 813 {
 814         int ret = 0;
 815         u32 counter;
 816
 817         if (!armv7_pmnc_counter_valid(idx)) {
 818                 pr_err("CPU%u checking wrong counter %d overflow status\n",
 819                         smp_processor_id(), idx);
 820         } else {
 821                 counter = ARMV7_IDX_TO_COUNTER(idx);
 822                 ret = pmnc & BIT(counter);
 823         }
 824
 825         return ret;
 826 }
 827
 828 static inline int armv7_pmnc_select_counter(int idx)
 829 {
 830         u32 counter;
 831
 832         if (!armv7_pmnc_counter_valid(idx)) {
 833                 pr_err("CPU%u selecting wrong PMNC counter %d\n",
 834                         smp_processor_id(), idx);
 835                 return -EINVAL;
 836         }
 837
 838         counter = ARMV7_IDX_TO_COUNTER(idx);
 839         asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (counter));
 840         isb();
 841
 842         return idx;
 843 }
 844
 845 static inline u32 armv7pmu_read_counter(int idx)
 846 {
 847         u32 value = 0;
 848
 849         if (!armv7_pmnc_counter_valid(idx))
 850                 pr_err("CPU%u reading wrong counter %d\n",
 851                         smp_processor_id(), idx);
 852         else if (idx == ARMV7_IDX_CYCLE_COUNTER)
 853                 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (value));
 854         else if (armv7_pmnc_select_counter(idx) == idx)
 855                 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (value));
 856
 857         return value;
 858 }
 859
 860 static inline void armv7pmu_write_counter(int idx, u32 value)
 861 {
 862         if (!armv7_pmnc_counter_valid(idx))
 863                 pr_err("CPU%u writing wrong counter %d\n",
 864                         smp_processor_id(), idx);
 865         else if (idx == ARMV7_IDX_CYCLE_COUNTER)
 866                 asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (value));
 867         else if (armv7_pmnc_select_counter(idx) == idx)
 868                 asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (value));
 869 }
 870
 871 static inline void armv7_pmnc_write_evtsel(int idx, u32 val)
 872 {
 873         if (armv7_pmnc_select_counter(idx) == idx) {
 874                 val &= ARMV7_EVTYPE_MASK;
 875                 asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val));
 876         }
 877 }
 878
 879 static inline int armv7_pmnc_enable_counter(int idx)
 880 {
 881         u32 counter;
 882
 883         if (!armv7_pmnc_counter_valid(idx)) {
 884                 pr_err("CPU%u enabling wrong PMNC counter %d\n",
 885                         smp_processor_id(), idx);
 886                 return -EINVAL;
 887         }
 888
 889         counter = ARMV7_IDX_TO_COUNTER(idx);
 890         asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (BIT(counter)));
 891         return idx;
 892 }
 893
 894 static inline int armv7_pmnc_disable_counter(int idx)
 895 {
 896         u32 counter;
 897
 898         if (!armv7_pmnc_counter_valid(idx)) {
 899                 pr_err("CPU%u disabling wrong PMNC counter %d\n",
 900                         smp_processor_id(), idx);
 901                 return -EINVAL;
 902         }
 903
 904         counter = ARMV7_IDX_TO_COUNTER(idx);
 905         asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (BIT(counter)));
 906         return idx;
 907 }
 908
 909 static inline int armv7_pmnc_enable_intens(int idx)
 910 {
 911         u32 counter;
 912
 913         if (!armv7_pmnc_counter_valid(idx)) {
 914                 pr_err("CPU%u enabling wrong PMNC counter IRQ enable %d\n",
 915                         smp_processor_id(), idx);
 916                 return -EINVAL;
 917         }
 918
 919         counter = ARMV7_IDX_TO_COUNTER(idx);
 920         asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (BIT(counter)));
 921         return idx;
 922 }
 923
 924 static inline int armv7_pmnc_disable_intens(int idx)
 925 {
 926         u32 counter;
 927
 928         if (!armv7_pmnc_counter_valid(idx)) {
 929                 pr_err("CPU%u disabling wrong PMNC counter IRQ enable %d\n",
 930                         smp_processor_id(), idx);
 931                 return -EINVAL;
 932         }
 933
 934         counter = ARMV7_IDX_TO_COUNTER(idx);
 935         asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (BIT(counter)));
 936         isb();
 937         /* Clear the overflow flag in case an interrupt is pending. */
 938         asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (BIT(counter)));
 939         isb();
 940
 941         return idx;
 942 }
 943
 944 static inline u32 armv7_pmnc_getreset_flags(void)
 945 {
 946         u32 val;
 947
 948         /* Read */
 949         asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
 950
 951         /* Write to clear flags */
 952         val &= ARMV7_FLAG_MASK;
 953         asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (val));
 954
 955         return val;
 956 }
 957
 958 #ifdef DEBUG
 959 static void armv7_pmnc_dump_regs(void)
 960 {
 961         u32 val;
 962         unsigned int cnt;
 963
 964         printk(KERN_INFO "PMNC registers dump:\n");
 965
 966         asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val));
 967         printk(KERN_INFO "PMNC  =0x%08x\n", val);
 968
 969         asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r" (val));
 970         printk(KERN_INFO "CNTENS=0x%08x\n", val);
 971
 972         asm volatile("mrc p15, 0, %0, c9, c14, 1" : "=r" (val));
 973         printk(KERN_INFO "INTENS=0x%08x\n", val);
 974
 975         asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
 976         printk(KERN_INFO "FLAGS =0x%08x\n", val);
 977
 978         asm volatile("mrc p15, 0, %0, c9, c12, 5" : "=r" (val));
 979         printk(KERN_INFO "SELECT=0x%08x\n", val);
 980
 981         asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val));
 982         printk(KERN_INFO "CCNT  =0x%08x\n", val);
 983
 984         for (cnt = ARMV7_IDX_COUNTER0; cnt <= ARMV7_IDX_COUNTER_LAST; cnt++) {
 985                 armv7_pmnc_select_counter(cnt);
 986                 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val));
 987                 printk(KERN_INFO "CNT[%d] count =0x%08x\n",
 988                         ARMV7_IDX_TO_COUNTER(cnt), val);
 989                 asm volatile("mrc p15, 0, %0, c9, c13, 1" : "=r" (val));
 990                 printk(KERN_INFO "CNT[%d] evtsel=0x%08x\n",
 991                         ARMV7_IDX_TO_COUNTER(cnt), val);
 992         }
 993 }
 994 #endif
 995
 996 static void armv7pmu_enable_event(struct hw_perf_event *hwc, int idx)
 997 {
 998         unsigned long flags;
 999         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
1000
1001         /*
1002          * Enable counter and interrupt, and set the counter to count
1003          * the event that we're interested in.
1004          */
1005         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1006
1007         /*
1008          * Disable counter
1009          */
1010         armv7_pmnc_disable_counter(idx);
1011
1012         /*
1013          * Set event (if destined for PMNx counters)
1014          * We only need to set the event for the cycle counter if we
1015          * have the ability to perform event filtering.
1016          */
1017         if (armv7pmu.set_event_filter || idx != ARMV7_IDX_CYCLE_COUNTER)
1018                 armv7_pmnc_write_evtsel(idx, hwc->config_base);
1019
1020         /*
1021          * Enable interrupt for this counter
1022          */
1023         armv7_pmnc_enable_intens(idx);
1024
1025         /*
1026          * Enable counter
1027          */
1028         armv7_pmnc_enable_counter(idx);
1029
1030         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1031 }
1032
1033 static void armv7pmu_disable_event(struct hw_perf_event *hwc, int idx)
1034 {
1035         unsigned long flags;
1036         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
1037
1038         /*
1039          * Disable counter and interrupt
1040          */
1041         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1042
1043         /*
1044          * Disable counter
1045          */
1046         armv7_pmnc_disable_counter(idx);
1047
1048         /*
1049          * Disable interrupt for this counter
1050          */
1051         armv7_pmnc_disable_intens(idx);
1052
1053         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1054 }
1055
1056 static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev)
1057 {
1058         u32 pmnc;
1059         struct perf_sample_data data;
1060         struct pmu_hw_events *cpuc;
1061         struct pt_regs *regs;
1062         int idx;
1063
1064         /*
1065          * Get and reset the IRQ flags
1066          */
1067         pmnc = armv7_pmnc_getreset_flags();
1068
1069         /*
1070          * Did an overflow occur?
1071          */
1072         if (!armv7_pmnc_has_overflowed(pmnc))
1073                 return IRQ_NONE;
1074
1075         /*
1076          * Handle the counter(s) overflow(s)
1077          */
1078         regs = get_irq_regs();
1079
1080         cpuc = &__get_cpu_var(cpu_hw_events);
1081         for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
1082                 struct perf_event *event = cpuc->events[idx];
1083                 struct hw_perf_event *hwc;
1084
1085                 /* Ignore if we don't have an event. */
1086                 if (!event)
1087                         continue;
1088
1089                 /*
1090                  * We have a single interrupt for all counters. Check that
1091                  * each counter has overflowed before we process it.
1092                  */
1093                 if (!armv7_pmnc_counter_has_overflowed(pmnc, idx))
1094                         continue;
1095
1096                 hwc = &event->hw;
1097                 armpmu_event_update(event, hwc, idx);
1098                 perf_sample_data_init(&data, 0, hwc->last_period);
1099                 if (!armpmu_event_set_period(event, hwc, idx))
1100                         continue;
1101
1102                 if (perf_event_overflow(event, &data, regs))
1103                         cpu_pmu->disable(hwc, idx);
1104         }
1105
1106         /*
1107          * Handle the pending perf events.
1108          *
1109          * Note: this call *must* be run with interrupts disabled. For
1110          * platforms that can have the PMU interrupts raised as an NMI, this
1111          * will not work.
1112          */
1113         irq_work_run();
1114
1115         return IRQ_HANDLED;
1116 }
1117
1118 static void armv7pmu_start(void)
1119 {
1120         unsigned long flags;
1121         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
1122
1123         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1124         /* Enable all counters */
1125         armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E);
1126         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1127 }
1128
1129 static void armv7pmu_stop(void)
1130 {
1131         unsigned long flags;
1132         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
1133
1134         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1135         /* Disable all counters */
1136         armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E);
1137         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1138 }
1139
1140 static int armv7pmu_get_event_idx(struct pmu_hw_events *cpuc,
1141                                   struct hw_perf_event *event)
1142 {
1143         int idx;
1144         unsigned long evtype = event->config_base & ARMV7_EVTYPE_EVENT;
1145
1146         /* Always place a cycle counter into the cycle counter. */
1147         if (evtype == ARMV7_PERFCTR_CPU_CYCLES) {
1148                 if (test_and_set_bit(ARMV7_IDX_CYCLE_COUNTER, cpuc->used_mask))
1149                         return -EAGAIN;
1150
1151                 return ARMV7_IDX_CYCLE_COUNTER;
1152         }
1153
1154         /*
1155          * For anything other than a cycle counter, try and use
1156          * the events counters
1157          */
1158         for (idx = ARMV7_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) {
1159                 if (!test_and_set_bit(idx, cpuc->used_mask))
1160                         return idx;
1161         }
1162
1163         /* The counters are all in use. */
1164         return -EAGAIN;
1165 }
1166
1167 /*
1168  * Add an event filter to a given event. This will only work for PMUv2 PMUs.
1169  */
1170 static int armv7pmu_set_event_filter(struct hw_perf_event *event,
1171                                      struct perf_event_attr *attr)
1172 {
1173         unsigned long config_base = 0;
1174
1175         if (attr->exclude_idle)
1176                 return -EPERM;
1177         if (attr->exclude_user)
1178                 config_base |= ARMV7_EXCLUDE_USER;
1179         if (attr->exclude_kernel)
1180                 config_base |= ARMV7_EXCLUDE_PL1;
1181         if (!attr->exclude_hv)
1182                 config_base |= ARMV7_INCLUDE_HYP;
1183
1184         /*
1185          * Install the filter into config_base as this is used to
1186          * construct the event type.
1187          */
1188         event->config_base = config_base;
1189
1190         return 0;
1191 }
1192
1193 static void armv7pmu_reset(void *info)
1194 {
1195         u32 idx, nb_cnt = cpu_pmu->num_events;
1196
1197         /* The counter and interrupt enable registers are unknown at reset. */
1198         for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx)
1199                 armv7pmu_disable_event(NULL, idx);
1200
1201         /* Initialize & Reset PMNC: C and P bits */
1202         armv7_pmnc_write(ARMV7_PMNC_P | ARMV7_PMNC_C);
1203 }
1204
1205 static int armv7_a8_map_event(struct perf_event *event)
1206 {
1207         return map_cpu_event(event, &armv7_a8_perf_map,
1208                                 &armv7_a8_perf_cache_map, 0xFF);
1209 }
1210
1211 static int armv7_a9_map_event(struct perf_event *event)
1212 {
1213         return map_cpu_event(event, &armv7_a9_perf_map,
1214                                 &armv7_a9_perf_cache_map, 0xFF);
1215 }
1216
1217 static int armv7_a5_map_event(struct perf_event *event)
1218 {
1219         return map_cpu_event(event, &armv7_a5_perf_map,
1220                                 &armv7_a5_perf_cache_map, 0xFF);
1221 }
1222
1223 static int armv7_a15_map_event(struct perf_event *event)
1224 {
1225         return map_cpu_event(event, &armv7_a15_perf_map,
1226                                 &armv7_a15_perf_cache_map, 0xFF);
1227 }
1228
1229 static int armv7_a7_map_event(struct perf_event *event)
1230 {
1231         return map_cpu_event(event, &armv7_a7_perf_map,
1232                                 &armv7_a7_perf_cache_map, 0xFF);
1233 }
1234
1235 static struct arm_pmu armv7pmu = {
1236         .handle_irq             = armv7pmu_handle_irq,
1237         .enable                 = armv7pmu_enable_event,
1238         .disable                = armv7pmu_disable_event,
1239         .read_counter           = armv7pmu_read_counter,
1240         .write_counter          = armv7pmu_write_counter,
1241         .get_event_idx          = armv7pmu_get_event_idx,
1242         .start                  = armv7pmu_start,
1243         .stop                   = armv7pmu_stop,
1244         .reset                  = armv7pmu_reset,
1245         .max_period             = (1LLU << 32) - 1,
1246 };
1247
1248 static u32 __init armv7_read_num_pmnc_events(void)
1249 {
1250         u32 nb_cnt;
1251
1252         /* Read the nb of CNTx counters supported from PMNC */
1253         nb_cnt = (armv7_pmnc_read() >> ARMV7_PMNC_N_SHIFT) & ARMV7_PMNC_N_MASK;
1254
1255         /* Add the CPU cycles counter and return */
1256         return nb_cnt + 1;
1257 }
1258
1259 static struct arm_pmu *__init armv7_a8_pmu_init(void)
1260 {
1261         armv7pmu.id             = ARM_PERF_PMU_ID_CA8;
1262         armv7pmu.name           = "ARMv7 Cortex-A8";
1263         armv7pmu.map_event      = armv7_a8_map_event;
1264         armv7pmu.num_events     = armv7_read_num_pmnc_events();
1265         return &armv7pmu;
1266 }
1267
1268 static struct arm_pmu *__init armv7_a9_pmu_init(void)
1269 {
1270         armv7pmu.id             = ARM_PERF_PMU_ID_CA9;
1271         armv7pmu.name           = "ARMv7 Cortex-A9";
1272         armv7pmu.map_event      = armv7_a9_map_event;
1273         armv7pmu.num_events     = armv7_read_num_pmnc_events();
1274         return &armv7pmu;
1275 }
1276
1277 static struct arm_pmu *__init armv7_a5_pmu_init(void)
1278 {
1279         armv7pmu.id             = ARM_PERF_PMU_ID_CA5;
1280         armv7pmu.name           = "ARMv7 Cortex-A5";
1281         armv7pmu.map_event      = armv7_a5_map_event;
1282         armv7pmu.num_events     = armv7_read_num_pmnc_events();
1283         return &armv7pmu;
1284 }
1285
1286 static struct arm_pmu *__init armv7_a15_pmu_init(void)
1287 {
1288         armv7pmu.id             = ARM_PERF_PMU_ID_CA15;
1289         armv7pmu.name           = "ARMv7 Cortex-A15";
1290         armv7pmu.map_event      = armv7_a15_map_event;
1291         armv7pmu.num_events     = armv7_read_num_pmnc_events();
1292         armv7pmu.set_event_filter = armv7pmu_set_event_filter;
1293         return &armv7pmu;
1294 }
1295
1296 static struct arm_pmu *__init armv7_a7_pmu_init(void)
1297 {
1298         armv7pmu.id             = ARM_PERF_PMU_ID_CA7;
1299         armv7pmu.name           = "ARMv7 Cortex-A7";
1300         armv7pmu.map_event      = armv7_a7_map_event;
1301         armv7pmu.num_events     = armv7_read_num_pmnc_events();
1302         armv7pmu.set_event_filter = armv7pmu_set_event_filter;
1303         return &armv7pmu;
1304 }
1305 #else
1306 static struct arm_pmu *__init armv7_a8_pmu_init(void)
1307 {
1308         return NULL;
1309 }
1310
1311 static struct arm_pmu *__init armv7_a9_pmu_init(void)
1312 {
1313         return NULL;
1314 }
1315
1316 static struct arm_pmu *__init armv7_a5_pmu_init(void)
1317 {
1318         return NULL;
1319 }
1320
1321 static struct arm_pmu *__init armv7_a15_pmu_init(void)
1322 {
1323         return NULL;
1324 }
1325
1326 static struct arm_pmu *__init armv7_a7_pmu_init(void)
1327 {
1328         return NULL;
1329 }
1330 #endif  /* CONFIG_CPU_V7 */