Pileus Git - ~andy/linux/blob - arch/arm/kernel/perf_event_v7.c

   1 /*
   2  * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code.
   3  *
   4  * ARMv7 support: Jean Pihet <jpihet@mvista.com>
   5  * 2010 (c) MontaVista Software, LLC.
   6  *
   7  * Copied from ARMv6 code, with the low level code inspired
   8  *  by the ARMv7 Oprofile code.
   9  *
  10  * Cortex-A8 has up to 4 configurable performance counters and
  11  *  a single cycle counter.
  12  * Cortex-A9 has up to 31 configurable performance counters and
  13  *  a single cycle counter.
  14  *
  15  * All counters can be enabled/disabled and IRQ masked separately. The cycle
  16  *  counter and all 4 performance counters together can be reset separately.
  17  */
  18
  19 #ifdef CONFIG_CPU_V7
  20 /* Common ARMv7 event types */
  21 enum armv7_perf_types {
  22         ARMV7_PERFCTR_PMNC_SW_INCR              = 0x00,
  23         ARMV7_PERFCTR_IFETCH_MISS               = 0x01,
  24         ARMV7_PERFCTR_ITLB_MISS                 = 0x02,
  25         ARMV7_PERFCTR_DCACHE_REFILL             = 0x03,
  26         ARMV7_PERFCTR_DCACHE_ACCESS             = 0x04,
  27         ARMV7_PERFCTR_DTLB_REFILL               = 0x05,
  28         ARMV7_PERFCTR_DREAD                     = 0x06,
  29         ARMV7_PERFCTR_DWRITE                    = 0x07,
  30
  31         ARMV7_PERFCTR_EXC_TAKEN                 = 0x09,
  32         ARMV7_PERFCTR_EXC_EXECUTED              = 0x0A,
  33         ARMV7_PERFCTR_CID_WRITE                 = 0x0B,
  34         /* ARMV7_PERFCTR_PC_WRITE is equivalent to HW_BRANCH_INSTRUCTIONS.
  35          * It counts:
  36          *  - all branch instructions,
  37          *  - instructions that explicitly write the PC,
  38          *  - exception generating instructions.
  39          */
  40         ARMV7_PERFCTR_PC_WRITE                  = 0x0C,
  41         ARMV7_PERFCTR_PC_IMM_BRANCH             = 0x0D,
  42         ARMV7_PERFCTR_UNALIGNED_ACCESS          = 0x0F,
  43         ARMV7_PERFCTR_PC_BRANCH_MIS_PRED        = 0x10,
  44         ARMV7_PERFCTR_CLOCK_CYCLES              = 0x11,
  45
  46         ARMV7_PERFCTR_PC_BRANCH_MIS_USED        = 0x12,
  47
  48         ARMV7_PERFCTR_CPU_CYCLES                = 0xFF
  49 };
  50
  51 /* ARMv7 Cortex-A8 specific event types */
  52 enum armv7_a8_perf_types {
  53         ARMV7_PERFCTR_INSTR_EXECUTED            = 0x08,
  54
  55         ARMV7_PERFCTR_PC_PROC_RETURN            = 0x0E,
  56
  57         ARMV7_PERFCTR_WRITE_BUFFER_FULL         = 0x40,
  58         ARMV7_PERFCTR_L2_STORE_MERGED           = 0x41,
  59         ARMV7_PERFCTR_L2_STORE_BUFF             = 0x42,
  60         ARMV7_PERFCTR_L2_ACCESS                 = 0x43,
  61         ARMV7_PERFCTR_L2_CACH_MISS              = 0x44,
  62         ARMV7_PERFCTR_AXI_READ_CYCLES           = 0x45,
  63         ARMV7_PERFCTR_AXI_WRITE_CYCLES          = 0x46,
  64         ARMV7_PERFCTR_MEMORY_REPLAY             = 0x47,
  65         ARMV7_PERFCTR_UNALIGNED_ACCESS_REPLAY   = 0x48,
  66         ARMV7_PERFCTR_L1_DATA_MISS              = 0x49,
  67         ARMV7_PERFCTR_L1_INST_MISS              = 0x4A,
  68         ARMV7_PERFCTR_L1_DATA_COLORING          = 0x4B,
  69         ARMV7_PERFCTR_L1_NEON_DATA              = 0x4C,
  70         ARMV7_PERFCTR_L1_NEON_CACH_DATA         = 0x4D,
  71         ARMV7_PERFCTR_L2_NEON                   = 0x4E,
  72         ARMV7_PERFCTR_L2_NEON_HIT               = 0x4F,
  73         ARMV7_PERFCTR_L1_INST                   = 0x50,
  74         ARMV7_PERFCTR_PC_RETURN_MIS_PRED        = 0x51,
  75         ARMV7_PERFCTR_PC_BRANCH_FAILED          = 0x52,
  76         ARMV7_PERFCTR_PC_BRANCH_TAKEN           = 0x53,
  77         ARMV7_PERFCTR_PC_BRANCH_EXECUTED        = 0x54,
  78         ARMV7_PERFCTR_OP_EXECUTED               = 0x55,
  79         ARMV7_PERFCTR_CYCLES_INST_STALL         = 0x56,
  80         ARMV7_PERFCTR_CYCLES_INST               = 0x57,
  81         ARMV7_PERFCTR_CYCLES_NEON_DATA_STALL    = 0x58,
  82         ARMV7_PERFCTR_CYCLES_NEON_INST_STALL    = 0x59,
  83         ARMV7_PERFCTR_NEON_CYCLES               = 0x5A,
  84
  85         ARMV7_PERFCTR_PMU0_EVENTS               = 0x70,
  86         ARMV7_PERFCTR_PMU1_EVENTS               = 0x71,
  87         ARMV7_PERFCTR_PMU_EVENTS                = 0x72,
  88 };
  89
  90 /* ARMv7 Cortex-A9 specific event types */
  91 enum armv7_a9_perf_types {
  92         ARMV7_PERFCTR_JAVA_HW_BYTECODE_EXEC     = 0x40,
  93         ARMV7_PERFCTR_JAVA_SW_BYTECODE_EXEC     = 0x41,
  94         ARMV7_PERFCTR_JAZELLE_BRANCH_EXEC       = 0x42,
  95
  96         ARMV7_PERFCTR_COHERENT_LINE_MISS        = 0x50,
  97         ARMV7_PERFCTR_COHERENT_LINE_HIT         = 0x51,
  98
  99         ARMV7_PERFCTR_ICACHE_DEP_STALL_CYCLES   = 0x60,
 100         ARMV7_PERFCTR_DCACHE_DEP_STALL_CYCLES   = 0x61,
 101         ARMV7_PERFCTR_TLB_MISS_DEP_STALL_CYCLES = 0x62,
 102         ARMV7_PERFCTR_STREX_EXECUTED_PASSED     = 0x63,
 103         ARMV7_PERFCTR_STREX_EXECUTED_FAILED     = 0x64,
 104         ARMV7_PERFCTR_DATA_EVICTION             = 0x65,
 105         ARMV7_PERFCTR_ISSUE_STAGE_NO_INST       = 0x66,
 106         ARMV7_PERFCTR_ISSUE_STAGE_EMPTY         = 0x67,
 107         ARMV7_PERFCTR_INST_OUT_OF_RENAME_STAGE  = 0x68,
 108
 109         ARMV7_PERFCTR_PREDICTABLE_FUNCT_RETURNS = 0x6E,
 110
 111         ARMV7_PERFCTR_MAIN_UNIT_EXECUTED_INST   = 0x70,
 112         ARMV7_PERFCTR_SECOND_UNIT_EXECUTED_INST = 0x71,
 113         ARMV7_PERFCTR_LD_ST_UNIT_EXECUTED_INST  = 0x72,
 114         ARMV7_PERFCTR_FP_EXECUTED_INST          = 0x73,
 115         ARMV7_PERFCTR_NEON_EXECUTED_INST        = 0x74,
 116
 117         ARMV7_PERFCTR_PLD_FULL_DEP_STALL_CYCLES = 0x80,
 118         ARMV7_PERFCTR_DATA_WR_DEP_STALL_CYCLES  = 0x81,
 119         ARMV7_PERFCTR_ITLB_MISS_DEP_STALL_CYCLES        = 0x82,
 120         ARMV7_PERFCTR_DTLB_MISS_DEP_STALL_CYCLES        = 0x83,
 121         ARMV7_PERFCTR_MICRO_ITLB_MISS_DEP_STALL_CYCLES  = 0x84,
 122         ARMV7_PERFCTR_MICRO_DTLB_MISS_DEP_STALL_CYCLES  = 0x85,
 123         ARMV7_PERFCTR_DMB_DEP_STALL_CYCLES      = 0x86,
 124
 125         ARMV7_PERFCTR_INTGR_CLK_ENABLED_CYCLES  = 0x8A,
 126         ARMV7_PERFCTR_DATA_ENGINE_CLK_EN_CYCLES = 0x8B,
 127
 128         ARMV7_PERFCTR_ISB_INST                  = 0x90,
 129         ARMV7_PERFCTR_DSB_INST                  = 0x91,
 130         ARMV7_PERFCTR_DMB_INST                  = 0x92,
 131         ARMV7_PERFCTR_EXT_INTERRUPTS            = 0x93,
 132
 133         ARMV7_PERFCTR_PLE_CACHE_LINE_RQST_COMPLETED     = 0xA0,
 134         ARMV7_PERFCTR_PLE_CACHE_LINE_RQST_SKIPPED       = 0xA1,
 135         ARMV7_PERFCTR_PLE_FIFO_FLUSH            = 0xA2,
 136         ARMV7_PERFCTR_PLE_RQST_COMPLETED        = 0xA3,
 137         ARMV7_PERFCTR_PLE_FIFO_OVERFLOW         = 0xA4,
 138         ARMV7_PERFCTR_PLE_RQST_PROG             = 0xA5
 139 };
 140
 141 /*
 142  * Cortex-A8 HW events mapping
 143  *
 144  * The hardware events that we support. We do support cache operations but
 145  * we have harvard caches and no way to combine instruction and data
 146  * accesses/misses in hardware.
 147  */
 148 static const unsigned armv7_a8_perf_map[PERF_COUNT_HW_MAX] = {
 149         [PERF_COUNT_HW_CPU_CYCLES]          = ARMV7_PERFCTR_CPU_CYCLES,
 150         [PERF_COUNT_HW_INSTRUCTIONS]        = ARMV7_PERFCTR_INSTR_EXECUTED,
 151         [PERF_COUNT_HW_CACHE_REFERENCES]    = HW_OP_UNSUPPORTED,
 152         [PERF_COUNT_HW_CACHE_MISSES]        = HW_OP_UNSUPPORTED,
 153         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
 154         [PERF_COUNT_HW_BRANCH_MISSES]       = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 155         [PERF_COUNT_HW_BUS_CYCLES]          = ARMV7_PERFCTR_CLOCK_CYCLES,
 156 };
 157
 158 static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 159                                           [PERF_COUNT_HW_CACHE_OP_MAX]
 160                                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 161         [C(L1D)] = {
 162                 /*
 163                  * The performance counters don't differentiate between read
 164                  * and write accesses/misses so this isn't strictly correct,
 165                  * but it's the best we can do. Writes and reads get
 166                  * combined.
 167                  */
 168                 [C(OP_READ)] = {
 169                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_DCACHE_ACCESS,
 170                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DCACHE_REFILL,
 171                 },
 172                 [C(OP_WRITE)] = {
 173                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_DCACHE_ACCESS,
 174                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DCACHE_REFILL,
 175                 },
 176                 [C(OP_PREFETCH)] = {
 177                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 178                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 179                 },
 180         },
 181         [C(L1I)] = {
 182                 [C(OP_READ)] = {
 183                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_INST,
 184                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_INST_MISS,
 185                 },
 186                 [C(OP_WRITE)] = {
 187                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_INST,
 188                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_INST_MISS,
 189                 },
 190                 [C(OP_PREFETCH)] = {
 191                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 192                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 193                 },
 194         },
 195         [C(LL)] = {
 196                 [C(OP_READ)] = {
 197                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L2_ACCESS,
 198                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L2_CACH_MISS,
 199                 },
 200                 [C(OP_WRITE)] = {
 201                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L2_ACCESS,
 202                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L2_CACH_MISS,
 203                 },
 204                 [C(OP_PREFETCH)] = {
 205                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 206                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 207                 },
 208         },
 209         [C(DTLB)] = {
 210                 [C(OP_READ)] = {
 211                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 212                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
 213                 },
 214                 [C(OP_WRITE)] = {
 215                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 216                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
 217                 },
 218                 [C(OP_PREFETCH)] = {
 219                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 220                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 221                 },
 222         },
 223         [C(ITLB)] = {
 224                 [C(OP_READ)] = {
 225                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 226                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_MISS,
 227                 },
 228                 [C(OP_WRITE)] = {
 229                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 230                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_MISS,
 231                 },
 232                 [C(OP_PREFETCH)] = {
 233                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 234                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 235                 },
 236         },
 237         [C(BPU)] = {
 238                 [C(OP_READ)] = {
 239                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_WRITE,
 240                         [C(RESULT_MISS)]
 241                                         = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 242                 },
 243                 [C(OP_WRITE)] = {
 244                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_WRITE,
 245                         [C(RESULT_MISS)]
 246                                         = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 247                 },
 248                 [C(OP_PREFETCH)] = {
 249                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 250                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 251                 },
 252         },
 253 };
 254
 255 /*
 256  * Cortex-A9 HW events mapping
 257  */
 258 static const unsigned armv7_a9_perf_map[PERF_COUNT_HW_MAX] = {
 259         [PERF_COUNT_HW_CPU_CYCLES]          = ARMV7_PERFCTR_CPU_CYCLES,
 260         [PERF_COUNT_HW_INSTRUCTIONS]        =
 261                                         ARMV7_PERFCTR_INST_OUT_OF_RENAME_STAGE,
 262         [PERF_COUNT_HW_CACHE_REFERENCES]    = ARMV7_PERFCTR_COHERENT_LINE_HIT,
 263         [PERF_COUNT_HW_CACHE_MISSES]        = ARMV7_PERFCTR_COHERENT_LINE_MISS,
 264         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
 265         [PERF_COUNT_HW_BRANCH_MISSES]       = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 266         [PERF_COUNT_HW_BUS_CYCLES]          = ARMV7_PERFCTR_CLOCK_CYCLES,
 267 };
 268
 269 static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 270                                           [PERF_COUNT_HW_CACHE_OP_MAX]
 271                                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 272         [C(L1D)] = {
 273                 /*
 274                  * The performance counters don't differentiate between read
 275                  * and write accesses/misses so this isn't strictly correct,
 276                  * but it's the best we can do. Writes and reads get
 277                  * combined.
 278                  */
 279                 [C(OP_READ)] = {
 280                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_DCACHE_ACCESS,
 281                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DCACHE_REFILL,
 282                 },
 283                 [C(OP_WRITE)] = {
 284                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_DCACHE_ACCESS,
 285                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DCACHE_REFILL,
 286                 },
 287                 [C(OP_PREFETCH)] = {
 288                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 289                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 290                 },
 291         },
 292         [C(L1I)] = {
 293                 [C(OP_READ)] = {
 294                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 295                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_IFETCH_MISS,
 296                 },
 297                 [C(OP_WRITE)] = {
 298                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 299                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_IFETCH_MISS,
 300                 },
 301                 [C(OP_PREFETCH)] = {
 302                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 303                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 304                 },
 305         },
 306         [C(LL)] = {
 307                 [C(OP_READ)] = {
 308                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 309                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 310                 },
 311                 [C(OP_WRITE)] = {
 312                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 313                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 314                 },
 315                 [C(OP_PREFETCH)] = {
 316                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 317                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 318                 },
 319         },
 320         [C(DTLB)] = {
 321                 [C(OP_READ)] = {
 322                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 323                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
 324                 },
 325                 [C(OP_WRITE)] = {
 326                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 327                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
 328                 },
 329                 [C(OP_PREFETCH)] = {
 330                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 331                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 332                 },
 333         },
 334         [C(ITLB)] = {
 335                 [C(OP_READ)] = {
 336                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 337                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_MISS,
 338                 },
 339                 [C(OP_WRITE)] = {
 340                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 341                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_MISS,
 342                 },
 343                 [C(OP_PREFETCH)] = {
 344                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 345                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 346                 },
 347         },
 348         [C(BPU)] = {
 349                 [C(OP_READ)] = {
 350                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_WRITE,
 351                         [C(RESULT_MISS)]
 352                                         = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 353                 },
 354                 [C(OP_WRITE)] = {
 355                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_WRITE,
 356                         [C(RESULT_MISS)]
 357                                         = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 358                 },
 359                 [C(OP_PREFETCH)] = {
 360                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 361                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 362                 },
 363         },
 364 };
 365
 366 /*
 367  * Perf Events counters
 368  */
 369 enum armv7_counters {
 370         ARMV7_CYCLE_COUNTER             = 1,    /* Cycle counter */
 371         ARMV7_COUNTER0                  = 2,    /* First event counter */
 372 };
 373
 374 /*
 375  * The cycle counter is ARMV7_CYCLE_COUNTER.
 376  * The first event counter is ARMV7_COUNTER0.
 377  * The last event counter is (ARMV7_COUNTER0 + armpmu->num_events - 1).
 378  */
 379 #define ARMV7_COUNTER_LAST      (ARMV7_COUNTER0 + armpmu->num_events - 1)
 380
 381 /*
 382  * ARMv7 low level PMNC access
 383  */
 384
 385 /*
 386  * Per-CPU PMNC: config reg
 387  */
 388 #define ARMV7_PMNC_E            (1 << 0) /* Enable all counters */
 389 #define ARMV7_PMNC_P            (1 << 1) /* Reset all counters */
 390 #define ARMV7_PMNC_C            (1 << 2) /* Cycle counter reset */
 391 #define ARMV7_PMNC_D            (1 << 3) /* CCNT counts every 64th cpu cycle */
 392 #define ARMV7_PMNC_X            (1 << 4) /* Export to ETM */
 393 #define ARMV7_PMNC_DP           (1 << 5) /* Disable CCNT if non-invasive debug*/
 394 #define ARMV7_PMNC_N_SHIFT      11       /* Number of counters supported */
 395 #define ARMV7_PMNC_N_MASK       0x1f
 396 #define ARMV7_PMNC_MASK         0x3f     /* Mask for writable bits */
 397
 398 /*
 399  * Available counters
 400  */
 401 #define ARMV7_CNT0              0       /* First event counter */
 402 #define ARMV7_CCNT              31      /* Cycle counter */
 403
 404 /* Perf Event to low level counters mapping */
 405 #define ARMV7_EVENT_CNT_TO_CNTx (ARMV7_COUNTER0 - ARMV7_CNT0)
 406
 407 /*
 408  * CNTENS: counters enable reg
 409  */
 410 #define ARMV7_CNTENS_P(idx)     (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx))
 411 #define ARMV7_CNTENS_C          (1 << ARMV7_CCNT)
 412
 413 /*
 414  * CNTENC: counters disable reg
 415  */
 416 #define ARMV7_CNTENC_P(idx)     (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx))
 417 #define ARMV7_CNTENC_C          (1 << ARMV7_CCNT)
 418
 419 /*
 420  * INTENS: counters overflow interrupt enable reg
 421  */
 422 #define ARMV7_INTENS_P(idx)     (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx))
 423 #define ARMV7_INTENS_C          (1 << ARMV7_CCNT)
 424
 425 /*
 426  * INTENC: counters overflow interrupt disable reg
 427  */
 428 #define ARMV7_INTENC_P(idx)     (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx))
 429 #define ARMV7_INTENC_C          (1 << ARMV7_CCNT)
 430
 431 /*
 432  * EVTSEL: Event selection reg
 433  */
 434 #define ARMV7_EVTSEL_MASK       0xff            /* Mask for writable bits */
 435
 436 /*
 437  * SELECT: Counter selection reg
 438  */
 439 #define ARMV7_SELECT_MASK       0x1f            /* Mask for writable bits */
 440
 441 /*
 442  * FLAG: counters overflow flag status reg
 443  */
 444 #define ARMV7_FLAG_P(idx)       (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx))
 445 #define ARMV7_FLAG_C            (1 << ARMV7_CCNT)
 446 #define ARMV7_FLAG_MASK         0xffffffff      /* Mask for writable bits */
 447 #define ARMV7_OVERFLOWED_MASK   ARMV7_FLAG_MASK
 448
 449 static inline unsigned long armv7_pmnc_read(void)
 450 {
 451         u32 val;
 452         asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(val));
 453         return val;
 454 }
 455
 456 static inline void armv7_pmnc_write(unsigned long val)
 457 {
 458         val &= ARMV7_PMNC_MASK;
 459         isb();
 460         asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(val));
 461 }
 462
 463 static inline int armv7_pmnc_has_overflowed(unsigned long pmnc)
 464 {
 465         return pmnc & ARMV7_OVERFLOWED_MASK;
 466 }
 467
 468 static inline int armv7_pmnc_counter_has_overflowed(unsigned long pmnc,
 469                                         enum armv7_counters counter)
 470 {
 471         int ret = 0;
 472
 473         if (counter == ARMV7_CYCLE_COUNTER)
 474                 ret = pmnc & ARMV7_FLAG_C;
 475         else if ((counter >= ARMV7_COUNTER0) && (counter <= ARMV7_COUNTER_LAST))
 476                 ret = pmnc & ARMV7_FLAG_P(counter);
 477         else
 478                 pr_err("CPU%u checking wrong counter %d overflow status\n",
 479                         smp_processor_id(), counter);
 480
 481         return ret;
 482 }
 483
 484 static inline int armv7_pmnc_select_counter(unsigned int idx)
 485 {
 486         u32 val;
 487
 488         if ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST)) {
 489                 pr_err("CPU%u selecting wrong PMNC counter"
 490                         " %d\n", smp_processor_id(), idx);
 491                 return -1;
 492         }
 493
 494         val = (idx - ARMV7_EVENT_CNT_TO_CNTx) & ARMV7_SELECT_MASK;
 495         asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (val));
 496         isb();
 497
 498         return idx;
 499 }
 500
 501 static inline u32 armv7pmu_read_counter(int idx)
 502 {
 503         unsigned long value = 0;
 504
 505         if (idx == ARMV7_CYCLE_COUNTER)
 506                 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (value));
 507         else if ((idx >= ARMV7_COUNTER0) && (idx <= ARMV7_COUNTER_LAST)) {
 508                 if (armv7_pmnc_select_counter(idx) == idx)
 509                         asm volatile("mrc p15, 0, %0, c9, c13, 2"
 510                                      : "=r" (value));
 511         } else
 512                 pr_err("CPU%u reading wrong counter %d\n",
 513                         smp_processor_id(), idx);
 514
 515         return value;
 516 }
 517
 518 static inline void armv7pmu_write_counter(int idx, u32 value)
 519 {
 520         if (idx == ARMV7_CYCLE_COUNTER)
 521                 asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (value));
 522         else if ((idx >= ARMV7_COUNTER0) && (idx <= ARMV7_COUNTER_LAST)) {
 523                 if (armv7_pmnc_select_counter(idx) == idx)
 524                         asm volatile("mcr p15, 0, %0, c9, c13, 2"
 525                                      : : "r" (value));
 526         } else
 527                 pr_err("CPU%u writing wrong counter %d\n",
 528                         smp_processor_id(), idx);
 529 }
 530
 531 static inline void armv7_pmnc_write_evtsel(unsigned int idx, u32 val)
 532 {
 533         if (armv7_pmnc_select_counter(idx) == idx) {
 534                 val &= ARMV7_EVTSEL_MASK;
 535                 asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val));
 536         }
 537 }
 538
 539 static inline u32 armv7_pmnc_enable_counter(unsigned int idx)
 540 {
 541         u32 val;
 542
 543         if ((idx != ARMV7_CYCLE_COUNTER) &&
 544             ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) {
 545                 pr_err("CPU%u enabling wrong PMNC counter"
 546                         " %d\n", smp_processor_id(), idx);
 547                 return -1;
 548         }
 549
 550         if (idx == ARMV7_CYCLE_COUNTER)
 551                 val = ARMV7_CNTENS_C;
 552         else
 553                 val = ARMV7_CNTENS_P(idx);
 554
 555         asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (val));
 556
 557         return idx;
 558 }
 559
 560 static inline u32 armv7_pmnc_disable_counter(unsigned int idx)
 561 {
 562         u32 val;
 563
 564
 565         if ((idx != ARMV7_CYCLE_COUNTER) &&
 566             ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) {
 567                 pr_err("CPU%u disabling wrong PMNC counter"
 568                         " %d\n", smp_processor_id(), idx);
 569                 return -1;
 570         }
 571
 572         if (idx == ARMV7_CYCLE_COUNTER)
 573                 val = ARMV7_CNTENC_C;
 574         else
 575                 val = ARMV7_CNTENC_P(idx);
 576
 577         asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (val));
 578
 579         return idx;
 580 }
 581
 582 static inline u32 armv7_pmnc_enable_intens(unsigned int idx)
 583 {
 584         u32 val;
 585
 586         if ((idx != ARMV7_CYCLE_COUNTER) &&
 587             ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) {
 588                 pr_err("CPU%u enabling wrong PMNC counter"
 589                         " interrupt enable %d\n", smp_processor_id(), idx);
 590                 return -1;
 591         }
 592
 593         if (idx == ARMV7_CYCLE_COUNTER)
 594                 val = ARMV7_INTENS_C;
 595         else
 596                 val = ARMV7_INTENS_P(idx);
 597
 598         asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (val));
 599
 600         return idx;
 601 }
 602
 603 static inline u32 armv7_pmnc_disable_intens(unsigned int idx)
 604 {
 605         u32 val;
 606
 607         if ((idx != ARMV7_CYCLE_COUNTER) &&
 608             ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) {
 609                 pr_err("CPU%u disabling wrong PMNC counter"
 610                         " interrupt enable %d\n", smp_processor_id(), idx);
 611                 return -1;
 612         }
 613
 614         if (idx == ARMV7_CYCLE_COUNTER)
 615                 val = ARMV7_INTENC_C;
 616         else
 617                 val = ARMV7_INTENC_P(idx);
 618
 619         asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (val));
 620
 621         return idx;
 622 }
 623
 624 static inline u32 armv7_pmnc_getreset_flags(void)
 625 {
 626         u32 val;
 627
 628         /* Read */
 629         asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
 630
 631         /* Write to clear flags */
 632         val &= ARMV7_FLAG_MASK;
 633         asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (val));
 634
 635         return val;
 636 }
 637
 638 #ifdef DEBUG
 639 static void armv7_pmnc_dump_regs(void)
 640 {
 641         u32 val;
 642         unsigned int cnt;
 643
 644         printk(KERN_INFO "PMNC registers dump:\n");
 645
 646         asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val));
 647         printk(KERN_INFO "PMNC  =0x%08x\n", val);
 648
 649         asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r" (val));
 650         printk(KERN_INFO "CNTENS=0x%08x\n", val);
 651
 652         asm volatile("mrc p15, 0, %0, c9, c14, 1" : "=r" (val));
 653         printk(KERN_INFO "INTENS=0x%08x\n", val);
 654
 655         asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
 656         printk(KERN_INFO "FLAGS =0x%08x\n", val);
 657
 658         asm volatile("mrc p15, 0, %0, c9, c12, 5" : "=r" (val));
 659         printk(KERN_INFO "SELECT=0x%08x\n", val);
 660
 661         asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val));
 662         printk(KERN_INFO "CCNT  =0x%08x\n", val);
 663
 664         for (cnt = ARMV7_COUNTER0; cnt < ARMV7_COUNTER_LAST; cnt++) {
 665                 armv7_pmnc_select_counter(cnt);
 666                 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val));
 667                 printk(KERN_INFO "CNT[%d] count =0x%08x\n",
 668                         cnt-ARMV7_EVENT_CNT_TO_CNTx, val);
 669                 asm volatile("mrc p15, 0, %0, c9, c13, 1" : "=r" (val));
 670                 printk(KERN_INFO "CNT[%d] evtsel=0x%08x\n",
 671                         cnt-ARMV7_EVENT_CNT_TO_CNTx, val);
 672         }
 673 }
 674 #endif
 675
 676 static void armv7pmu_enable_event(struct hw_perf_event *hwc, int idx)
 677 {
 678         unsigned long flags;
 679
 680         /*
 681          * Enable counter and interrupt, and set the counter to count
 682          * the event that we're interested in.
 683          */
 684         raw_spin_lock_irqsave(&pmu_lock, flags);
 685
 686         /*
 687          * Disable counter
 688          */
 689         armv7_pmnc_disable_counter(idx);
 690
 691         /*
 692          * Set event (if destined for PMNx counters)
 693          * We don't need to set the event if it's a cycle count
 694          */
 695         if (idx != ARMV7_CYCLE_COUNTER)
 696                 armv7_pmnc_write_evtsel(idx, hwc->config_base);
 697
 698         /*
 699          * Enable interrupt for this counter
 700          */
 701         armv7_pmnc_enable_intens(idx);
 702
 703         /*
 704          * Enable counter
 705          */
 706         armv7_pmnc_enable_counter(idx);
 707
 708         raw_spin_unlock_irqrestore(&pmu_lock, flags);
 709 }
 710
 711 static void armv7pmu_disable_event(struct hw_perf_event *hwc, int idx)
 712 {
 713         unsigned long flags;
 714
 715         /*
 716          * Disable counter and interrupt
 717          */
 718         raw_spin_lock_irqsave(&pmu_lock, flags);
 719
 720         /*
 721          * Disable counter
 722          */
 723         armv7_pmnc_disable_counter(idx);
 724
 725         /*
 726          * Disable interrupt for this counter
 727          */
 728         armv7_pmnc_disable_intens(idx);
 729
 730         raw_spin_unlock_irqrestore(&pmu_lock, flags);
 731 }
 732
 733 static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev)
 734 {
 735         unsigned long pmnc;
 736         struct perf_sample_data data;
 737         struct cpu_hw_events *cpuc;
 738         struct pt_regs *regs;
 739         int idx;
 740
 741         /*
 742          * Get and reset the IRQ flags
 743          */
 744         pmnc = armv7_pmnc_getreset_flags();
 745
 746         /*
 747          * Did an overflow occur?
 748          */
 749         if (!armv7_pmnc_has_overflowed(pmnc))
 750                 return IRQ_NONE;
 751
 752         /*
 753          * Handle the counter(s) overflow(s)
 754          */
 755         regs = get_irq_regs();
 756
 757         perf_sample_data_init(&data, 0);
 758
 759         cpuc = &__get_cpu_var(cpu_hw_events);
 760         for (idx = 0; idx <= armpmu->num_events; ++idx) {
 761                 struct perf_event *event = cpuc->events[idx];
 762                 struct hw_perf_event *hwc;
 763
 764                 if (!test_bit(idx, cpuc->active_mask))
 765                         continue;
 766
 767                 /*
 768                  * We have a single interrupt for all counters. Check that
 769                  * each counter has overflowed before we process it.
 770                  */
 771                 if (!armv7_pmnc_counter_has_overflowed(pmnc, idx))
 772                         continue;
 773
 774                 hwc = &event->hw;
 775                 armpmu_event_update(event, hwc, idx, 1);
 776                 data.period = event->hw.last_period;
 777                 if (!armpmu_event_set_period(event, hwc, idx))
 778                         continue;
 779
 780                 if (perf_event_overflow(event, 0, &data, regs))
 781                         armpmu->disable(hwc, idx);
 782         }
 783
 784         /*
 785          * Handle the pending perf events.
 786          *
 787          * Note: this call *must* be run with interrupts disabled. For
 788          * platforms that can have the PMU interrupts raised as an NMI, this
 789          * will not work.
 790          */
 791         irq_work_run();
 792
 793         return IRQ_HANDLED;
 794 }
 795
 796 static void armv7pmu_start(void)
 797 {
 798         unsigned long flags;
 799
 800         raw_spin_lock_irqsave(&pmu_lock, flags);
 801         /* Enable all counters */
 802         armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E);
 803         raw_spin_unlock_irqrestore(&pmu_lock, flags);
 804 }
 805
 806 static void armv7pmu_stop(void)
 807 {
 808         unsigned long flags;
 809
 810         raw_spin_lock_irqsave(&pmu_lock, flags);
 811         /* Disable all counters */
 812         armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E);
 813         raw_spin_unlock_irqrestore(&pmu_lock, flags);
 814 }
 815
 816 static int armv7pmu_get_event_idx(struct cpu_hw_events *cpuc,
 817                                   struct hw_perf_event *event)
 818 {
 819         int idx;
 820
 821         /* Always place a cycle counter into the cycle counter. */
 822         if (event->config_base == ARMV7_PERFCTR_CPU_CYCLES) {
 823                 if (test_and_set_bit(ARMV7_CYCLE_COUNTER, cpuc->used_mask))
 824                         return -EAGAIN;
 825
 826                 return ARMV7_CYCLE_COUNTER;
 827         } else {
 828                 /*
 829                  * For anything other than a cycle counter, try and use
 830                  * the events counters
 831                  */
 832                 for (idx = ARMV7_COUNTER0; idx <= armpmu->num_events; ++idx) {
 833                         if (!test_and_set_bit(idx, cpuc->used_mask))
 834                                 return idx;
 835                 }
 836
 837                 /* The counters are all in use. */
 838                 return -EAGAIN;
 839         }
 840 }
 841
 842 static void armv7pmu_reset(void *info)
 843 {
 844         u32 idx, nb_cnt = armpmu->num_events;
 845
 846         /* The counter and interrupt enable registers are unknown at reset. */
 847         for (idx = 1; idx < nb_cnt; ++idx)
 848                 armv7pmu_disable_event(NULL, idx);
 849
 850         /* Initialize & Reset PMNC: C and P bits */
 851         armv7_pmnc_write(ARMV7_PMNC_P | ARMV7_PMNC_C);
 852 }
 853
 854 static struct arm_pmu armv7pmu = {
 855         .handle_irq             = armv7pmu_handle_irq,
 856         .enable                 = armv7pmu_enable_event,
 857         .disable                = armv7pmu_disable_event,
 858         .read_counter           = armv7pmu_read_counter,
 859         .write_counter          = armv7pmu_write_counter,
 860         .get_event_idx          = armv7pmu_get_event_idx,
 861         .start                  = armv7pmu_start,
 862         .stop                   = armv7pmu_stop,
 863         .reset                  = armv7pmu_reset,
 864         .raw_event_mask         = 0xFF,
 865         .max_period             = (1LLU << 32) - 1,
 866 };
 867
 868 static u32 __init armv7_read_num_pmnc_events(void)
 869 {
 870         u32 nb_cnt;
 871
 872         /* Read the nb of CNTx counters supported from PMNC */
 873         nb_cnt = (armv7_pmnc_read() >> ARMV7_PMNC_N_SHIFT) & ARMV7_PMNC_N_MASK;
 874
 875         /* Add the CPU cycles counter and return */
 876         return nb_cnt + 1;
 877 }
 878
 879 static const struct arm_pmu *__init armv7_a8_pmu_init(void)
 880 {
 881         armv7pmu.id             = ARM_PERF_PMU_ID_CA8;
 882         armv7pmu.name           = "ARMv7 Cortex-A8";
 883         armv7pmu.cache_map      = &armv7_a8_perf_cache_map;
 884         armv7pmu.event_map      = &armv7_a8_perf_map;
 885         armv7pmu.num_events     = armv7_read_num_pmnc_events();
 886         return &armv7pmu;
 887 }
 888
 889 static const struct arm_pmu *__init armv7_a9_pmu_init(void)
 890 {
 891         armv7pmu.id             = ARM_PERF_PMU_ID_CA9;
 892         armv7pmu.name           = "ARMv7 Cortex-A9";
 893         armv7pmu.cache_map      = &armv7_a9_perf_cache_map;
 894         armv7pmu.event_map      = &armv7_a9_perf_map;
 895         armv7pmu.num_events     = armv7_read_num_pmnc_events();
 896         return &armv7pmu;
 897 }
 898 #else
 899 static const struct arm_pmu *__init armv7_a8_pmu_init(void)
 900 {
 901         return NULL;
 902 }
 903
 904 static const struct arm_pmu *__init armv7_a9_pmu_init(void)
 905 {
 906         return NULL;
 907 }
 908 #endif  /* CONFIG_CPU_V7 */