Pileus Git - ~andy/linux/blob - tools/perf/builtin-annotate.c

   1 /*
   2  * builtin-annotate.c
   3  *
   4  * Builtin annotate command: Analyze the perf.data input file,
   5  * look up and read DSOs and symbol information and display
   6  * a histogram of results, along various sorting keys.
   7  */
   8 #include "builtin.h"
   9
  10 #include "util/util.h"
  11
  12 #include "util/color.h"
  13 #include <linux/list.h>
  14 #include "util/cache.h"
  15 #include <linux/rbtree.h>
  16 #include "util/symbol.h"
  17 #include "util/string.h"
  18
  19 #include "perf.h"
  20 #include "util/debug.h"
  21
  22 #include "util/parse-options.h"
  23 #include "util/parse-events.h"
  24 #include "util/thread.h"
  25 #include "util/sort.h"
  26 #include "util/hist.h"
  27
  28 static char             const *input_name = "perf.data";
  29
  30 static int              force;
  31 static int              input;
  32
  33 static int              full_paths;
  34
  35 static int              print_line;
  36
  37 static unsigned long    page_size;
  38 static unsigned long    mmap_window = 32;
  39
  40 struct sym_hist {
  41         u64             sum;
  42         u64             ip[0];
  43 };
  44
  45 struct sym_ext {
  46         struct rb_node  node;
  47         double          percent;
  48         char            *path;
  49 };
  50
  51 struct sym_priv {
  52         struct sym_hist *hist;
  53         struct sym_ext  *ext;
  54 };
  55
  56 static const char *sym_hist_filter;
  57
  58 static int symbol_filter(struct map *map, struct symbol *sym)
  59 {
  60         if (strcmp(sym->name, sym_hist_filter) == 0) {
  61                 struct sym_priv *priv = dso__sym_priv(map->dso, sym);
  62                 const int size = (sizeof(*priv->hist) +
  63                                   (sym->end - sym->start) * sizeof(u64));
  64
  65                 priv->hist = malloc(size);
  66                 if (priv->hist)
  67                         memset(priv->hist, 0, size);
  68                 return 0;
  69         }
  70         /*
  71          * FIXME: We should really filter it out, as we don't want to go thru symbols
  72          * we're not interested, and if a DSO ends up with no symbols, delete it too,
  73          * but right now the kernel loading routines in symbol.c bail out if no symbols
  74          * are found, fix it later.
  75          */
  76         return 0;
  77 }
  78
  79 /*
  80  * collect histogram counts
  81  */
  82 static void hist_hit(struct hist_entry *he, u64 ip)
  83 {
  84         unsigned int sym_size, offset;
  85         struct symbol *sym = he->sym;
  86         struct sym_priv *priv;
  87         struct sym_hist *h;
  88
  89         he->count++;
  90
  91         if (!sym || !he->map)
  92                 return;
  93
  94         priv = dso__sym_priv(he->map->dso, sym);
  95         if (!priv->hist)
  96                 return;
  97
  98         sym_size = sym->end - sym->start;
  99         offset = ip - sym->start;
 100
 101         if (verbose)
 102                 fprintf(stderr, "%s: ip=%Lx\n", __func__,
 103                         he->map->unmap_ip(he->map, ip));
 104
 105         if (offset >= sym_size)
 106                 return;
 107
 108         h = priv->hist;
 109         h->sum++;
 110         h->ip[offset]++;
 111
 112         if (verbose >= 3)
 113                 printf("%p %s: count++ [ip: %p, %08Lx] => %Ld\n",
 114                         (void *)(unsigned long)he->sym->start,
 115                         he->sym->name,
 116                         (void *)(unsigned long)ip, ip - he->sym->start,
 117                         h->ip[offset]);
 118 }
 119
 120 static int hist_entry__add(struct thread *thread, struct map *map,
 121                            struct symbol *sym, u64 ip, u64 count, char level)
 122 {
 123         bool hit;
 124         struct hist_entry *he = __hist_entry__add(thread, map, sym, NULL, ip,
 125                                                   count, level, &hit);
 126         if (he == NULL)
 127                 return -ENOMEM;
 128         hist_hit(he, ip);
 129         return 0;
 130 }
 131
 132 static int
 133 process_sample_event(event_t *event, unsigned long offset, unsigned long head)
 134 {
 135         char level;
 136         u64 ip = event->ip.ip;
 137         struct map *map = NULL;
 138         struct symbol *sym = NULL;
 139         struct thread *thread = threads__findnew(event->ip.pid);
 140
 141         dump_printf("%p [%p]: PERF_EVENT (IP, %d): %d: %p\n",
 142                 (void *)(offset + head),
 143                 (void *)(long)(event->header.size),
 144                 event->header.misc,
 145                 event->ip.pid,
 146                 (void *)(long)ip);
 147
 148         if (thread == NULL) {
 149                 fprintf(stderr, "problem processing %d event, skipping it.\n",
 150                         event->header.type);
 151                 return -1;
 152         }
 153
 154         dump_printf(" ... thread: %s:%d\n", thread->comm, thread->pid);
 155
 156         if (event->header.misc & PERF_RECORD_MISC_KERNEL) {
 157                 level = 'k';
 158                 sym = kernel_maps__find_symbol(ip, &map);
 159                 dump_printf(" ...... dso: %s\n",
 160                             map ? map->dso->long_name : "<not found>");
 161         } else if (event->header.misc & PERF_RECORD_MISC_USER) {
 162                 level = '.';
 163                 map = thread__find_map(thread, ip);
 164                 if (map != NULL) {
 165 got_map:
 166                         ip = map->map_ip(map, ip);
 167                         sym = map->dso->find_symbol(map->dso, ip);
 168                 } else {
 169                         /*
 170                          * If this is outside of all known maps,
 171                          * and is a negative address, try to look it
 172                          * up in the kernel dso, as it might be a
 173                          * vsyscall or vdso (which executes in user-mode).
 174                          *
 175                          * XXX This is nasty, we should have a symbol list in
 176                          * the "[vdso]" dso, but for now lets use the old
 177                          * trick of looking in the whole kernel symbol list.
 178                          */
 179                         if ((long long)ip < 0) {
 180                                 map = kernel_map;
 181                                 goto got_map;
 182                         }
 183                 }
 184                 dump_printf(" ...... dso: %s\n",
 185                             map ? map->dso->long_name : "<not found>");
 186         } else {
 187                 level = 'H';
 188                 dump_printf(" ...... dso: [hypervisor]\n");
 189         }
 190
 191         if (hist_entry__add(thread, map, sym, ip, 1, level)) {
 192                 fprintf(stderr, "problem incrementing symbol count, "
 193                                 "skipping event\n");
 194                 return -1;
 195         }
 196         total++;
 197
 198         return 0;
 199 }
 200
 201 static int
 202 process_mmap_event(event_t *event, unsigned long offset, unsigned long head)
 203 {
 204         struct map *map = map__new(&event->mmap, NULL, 0,
 205                                    sizeof(struct sym_priv), symbol_filter,
 206                                    verbose);
 207         struct thread *thread = threads__findnew(event->mmap.pid);
 208
 209         dump_printf("%p [%p]: PERF_RECORD_MMAP %d: [%p(%p) @ %p]: %s\n",
 210                 (void *)(offset + head),
 211                 (void *)(long)(event->header.size),
 212                 event->mmap.pid,
 213                 (void *)(long)event->mmap.start,
 214                 (void *)(long)event->mmap.len,
 215                 (void *)(long)event->mmap.pgoff,
 216                 event->mmap.filename);
 217
 218         if (thread == NULL || map == NULL) {
 219                 dump_printf("problem processing PERF_RECORD_MMAP, skipping event.\n");
 220                 return 0;
 221         }
 222
 223         thread__insert_map(thread, map);
 224         total_mmap++;
 225
 226         return 0;
 227 }
 228
 229 static int
 230 process_comm_event(event_t *event, unsigned long offset, unsigned long head)
 231 {
 232         struct thread *thread = threads__findnew(event->comm.pid);
 233
 234         dump_printf("%p [%p]: PERF_RECORD_COMM: %s:%d\n",
 235                 (void *)(offset + head),
 236                 (void *)(long)(event->header.size),
 237                 event->comm.comm, event->comm.pid);
 238
 239         if (thread == NULL ||
 240             thread__set_comm(thread, event->comm.comm)) {
 241                 dump_printf("problem processing PERF_RECORD_COMM, skipping event.\n");
 242                 return -1;
 243         }
 244         total_comm++;
 245
 246         return 0;
 247 }
 248
 249 static int
 250 process_fork_event(event_t *event, unsigned long offset, unsigned long head)
 251 {
 252         struct thread *thread = threads__findnew(event->fork.pid);
 253         struct thread *parent = threads__findnew(event->fork.ppid);
 254
 255         dump_printf("%p [%p]: PERF_RECORD_FORK: %d:%d\n",
 256                 (void *)(offset + head),
 257                 (void *)(long)(event->header.size),
 258                 event->fork.pid, event->fork.ppid);
 259
 260         /*
 261          * A thread clone will have the same PID for both
 262          * parent and child.
 263          */
 264         if (thread == parent)
 265                 return 0;
 266
 267         if (!thread || !parent || thread__fork(thread, parent)) {
 268                 dump_printf("problem processing PERF_RECORD_FORK, skipping event.\n");
 269                 return -1;
 270         }
 271         total_fork++;
 272
 273         return 0;
 274 }
 275
 276 static int
 277 process_event(event_t *event, unsigned long offset, unsigned long head)
 278 {
 279         switch (event->header.type) {
 280         case PERF_RECORD_SAMPLE:
 281                 return process_sample_event(event, offset, head);
 282
 283         case PERF_RECORD_MMAP:
 284                 return process_mmap_event(event, offset, head);
 285
 286         case PERF_RECORD_COMM:
 287                 return process_comm_event(event, offset, head);
 288
 289         case PERF_RECORD_FORK:
 290                 return process_fork_event(event, offset, head);
 291         /*
 292          * We dont process them right now but they are fine:
 293          */
 294
 295         case PERF_RECORD_THROTTLE:
 296         case PERF_RECORD_UNTHROTTLE:
 297                 return 0;
 298
 299         default:
 300                 return -1;
 301         }
 302
 303         return 0;
 304 }
 305
 306 static int parse_line(FILE *file, struct hist_entry *he, u64 len)
 307 {
 308         struct symbol *sym = he->sym;
 309         char *line = NULL, *tmp, *tmp2;
 310         static const char *prev_line;
 311         static const char *prev_color;
 312         unsigned int offset;
 313         size_t line_len;
 314         u64 start;
 315         s64 line_ip;
 316         int ret;
 317         char *c;
 318
 319         if (getline(&line, &line_len, file) < 0)
 320                 return -1;
 321         if (!line)
 322                 return -1;
 323
 324         c = strchr(line, '\n');
 325         if (c)
 326                 *c = 0;
 327
 328         line_ip = -1;
 329         offset = 0;
 330         ret = -2;
 331
 332         /*
 333          * Strip leading spaces:
 334          */
 335         tmp = line;
 336         while (*tmp) {
 337                 if (*tmp != ' ')
 338                         break;
 339                 tmp++;
 340         }
 341
 342         if (*tmp) {
 343                 /*
 344                  * Parse hexa addresses followed by ':'
 345                  */
 346                 line_ip = strtoull(tmp, &tmp2, 16);
 347                 if (*tmp2 != ':')
 348                         line_ip = -1;
 349         }
 350
 351         start = he->map->unmap_ip(he->map, sym->start);
 352
 353         if (line_ip != -1) {
 354                 const char *path = NULL;
 355                 unsigned int hits = 0;
 356                 double percent = 0.0;
 357                 const char *color;
 358                 struct sym_priv *priv = dso__sym_priv(he->map->dso, sym);
 359                 struct sym_ext *sym_ext = priv->ext;
 360                 struct sym_hist *h = priv->hist;
 361
 362                 offset = line_ip - start;
 363                 if (offset < len)
 364                         hits = h->ip[offset];
 365
 366                 if (offset < len && sym_ext) {
 367                         path = sym_ext[offset].path;
 368                         percent = sym_ext[offset].percent;
 369                 } else if (h->sum)
 370                         percent = 100.0 * hits / h->sum;
 371
 372                 color = get_percent_color(percent);
 373
 374                 /*
 375                  * Also color the filename and line if needed, with
 376                  * the same color than the percentage. Don't print it
 377                  * twice for close colored ip with the same filename:line
 378                  */
 379                 if (path) {
 380                         if (!prev_line || strcmp(prev_line, path)
 381                                        || color != prev_color) {
 382                                 color_fprintf(stdout, color, " %s", path);
 383                                 prev_line = path;
 384                                 prev_color = color;
 385                         }
 386                 }
 387
 388                 color_fprintf(stdout, color, " %7.2f", percent);
 389                 printf(" :      ");
 390                 color_fprintf(stdout, PERF_COLOR_BLUE, "%s\n", line);
 391         } else {
 392                 if (!*line)
 393                         printf("         :\n");
 394                 else
 395                         printf("         :      %s\n", line);
 396         }
 397
 398         return 0;
 399 }
 400
 401 static struct rb_root root_sym_ext;
 402
 403 static void insert_source_line(struct sym_ext *sym_ext)
 404 {
 405         struct sym_ext *iter;
 406         struct rb_node **p = &root_sym_ext.rb_node;
 407         struct rb_node *parent = NULL;
 408
 409         while (*p != NULL) {
 410                 parent = *p;
 411                 iter = rb_entry(parent, struct sym_ext, node);
 412
 413                 if (sym_ext->percent > iter->percent)
 414                         p = &(*p)->rb_left;
 415                 else
 416                         p = &(*p)->rb_right;
 417         }
 418
 419         rb_link_node(&sym_ext->node, parent, p);
 420         rb_insert_color(&sym_ext->node, &root_sym_ext);
 421 }
 422
 423 static void free_source_line(struct hist_entry *he, int len)
 424 {
 425         struct sym_priv *priv = dso__sym_priv(he->map->dso, he->sym);
 426         struct sym_ext *sym_ext = priv->ext;
 427         int i;
 428
 429         if (!sym_ext)
 430                 return;
 431
 432         for (i = 0; i < len; i++)
 433                 free(sym_ext[i].path);
 434         free(sym_ext);
 435
 436         priv->ext = NULL;
 437         root_sym_ext = RB_ROOT;
 438 }
 439
 440 /* Get the filename:line for the colored entries */
 441 static void
 442 get_source_line(struct hist_entry *he, int len, const char *filename)
 443 {
 444         struct symbol *sym = he->sym;
 445         u64 start;
 446         int i;
 447         char cmd[PATH_MAX * 2];
 448         struct sym_ext *sym_ext;
 449         struct sym_priv *priv = dso__sym_priv(he->map->dso, sym);
 450         struct sym_hist *h = priv->hist;
 451
 452         if (!h->sum)
 453                 return;
 454
 455         sym_ext = priv->ext = calloc(len, sizeof(struct sym_ext));
 456         if (!priv->ext)
 457                 return;
 458
 459         start = he->map->unmap_ip(he->map, sym->start);
 460
 461         for (i = 0; i < len; i++) {
 462                 char *path = NULL;
 463                 size_t line_len;
 464                 u64 offset;
 465                 FILE *fp;
 466
 467                 sym_ext[i].percent = 100.0 * h->ip[i] / h->sum;
 468                 if (sym_ext[i].percent <= 0.5)
 469                         continue;
 470
 471                 offset = start + i;
 472                 sprintf(cmd, "addr2line -e %s %016llx", filename, offset);
 473                 fp = popen(cmd, "r");
 474                 if (!fp)
 475                         continue;
 476
 477                 if (getline(&path, &line_len, fp) < 0 || !line_len)
 478                         goto next;
 479
 480                 sym_ext[i].path = malloc(sizeof(char) * line_len + 1);
 481                 if (!sym_ext[i].path)
 482                         goto next;
 483
 484                 strcpy(sym_ext[i].path, path);
 485                 insert_source_line(&sym_ext[i]);
 486
 487         next:
 488                 pclose(fp);
 489         }
 490 }
 491
 492 static void print_summary(const char *filename)
 493 {
 494         struct sym_ext *sym_ext;
 495         struct rb_node *node;
 496
 497         printf("\nSorted summary for file %s\n", filename);
 498         printf("----------------------------------------------\n\n");
 499
 500         if (RB_EMPTY_ROOT(&root_sym_ext)) {
 501                 printf(" Nothing higher than %1.1f%%\n", MIN_GREEN);
 502                 return;
 503         }
 504
 505         node = rb_first(&root_sym_ext);
 506         while (node) {
 507                 double percent;
 508                 const char *color;
 509                 char *path;
 510
 511                 sym_ext = rb_entry(node, struct sym_ext, node);
 512                 percent = sym_ext->percent;
 513                 color = get_percent_color(percent);
 514                 path = sym_ext->path;
 515
 516                 color_fprintf(stdout, color, " %7.2f %s", percent, path);
 517                 node = rb_next(node);
 518         }
 519 }
 520
 521 static void annotate_sym(struct hist_entry *he)
 522 {
 523         struct map *map = he->map;
 524         struct dso *dso = map->dso;
 525         struct symbol *sym = he->sym;
 526         const char *filename = dso->long_name, *d_filename;
 527         u64 len;
 528         char command[PATH_MAX*2];
 529         FILE *file;
 530
 531         if (!filename)
 532                 return;
 533
 534         if (verbose)
 535                 fprintf(stderr, "%s: filename=%s, sym=%s, start=%Lx, end=%Lx\n",
 536                         __func__, filename, sym->name,
 537                         map->unmap_ip(map, sym->start),
 538                         map->unmap_ip(map, sym->end));
 539
 540         if (full_paths)
 541                 d_filename = filename;
 542         else
 543                 d_filename = basename(filename);
 544
 545         len = sym->end - sym->start;
 546
 547         if (print_line) {
 548                 get_source_line(he, len, filename);
 549                 print_summary(filename);
 550         }
 551
 552         printf("\n\n------------------------------------------------\n");
 553         printf(" Percent |      Source code & Disassembly of %s\n", d_filename);
 554         printf("------------------------------------------------\n");
 555
 556         if (verbose >= 2)
 557                 printf("annotating [%p] %30s : [%p] %30s\n",
 558                        dso, dso->long_name, sym, sym->name);
 559
 560         sprintf(command, "objdump --start-address=0x%016Lx --stop-address=0x%016Lx -dS %s|grep -v %s",
 561                 map->unmap_ip(map, sym->start), map->unmap_ip(map, sym->end),
 562                 filename, filename);
 563
 564         if (verbose >= 3)
 565                 printf("doing: %s\n", command);
 566
 567         file = popen(command, "r");
 568         if (!file)
 569                 return;
 570
 571         while (!feof(file)) {
 572                 if (parse_line(file, he, len) < 0)
 573                         break;
 574         }
 575
 576         pclose(file);
 577         if (print_line)
 578                 free_source_line(he, len);
 579 }
 580
 581 static void find_annotations(void)
 582 {
 583         struct rb_node *nd;
 584         int count = 0;
 585
 586         for (nd = rb_first(&output_hists); nd; nd = rb_next(nd)) {
 587                 struct hist_entry *he = rb_entry(nd, struct hist_entry, rb_node);
 588                 struct sym_priv *priv;
 589
 590                 if (he->sym == NULL)
 591                         continue;
 592
 593                 priv = dso__sym_priv(he->map->dso, he->sym);
 594                 if (priv->hist == NULL)
 595                         continue;
 596
 597                 annotate_sym(he);
 598                 count++;
 599                 /*
 600                  * Since we have a hist_entry per IP for the same symbol, free
 601                  * he->sym->hist to signal we already processed this symbol.
 602                  */
 603                 free(priv->hist);
 604                 priv->hist = NULL;
 605         }
 606
 607         if (!count)
 608                 printf(" Error: symbol '%s' not present amongst the samples.\n", sym_hist_filter);
 609 }
 610
 611 static int __cmd_annotate(void)
 612 {
 613         int ret, rc = EXIT_FAILURE;
 614         unsigned long offset = 0;
 615         unsigned long head = 0;
 616         struct stat input_stat;
 617         event_t *event;
 618         uint32_t size;
 619         char *buf;
 620
 621         register_idle_thread();
 622
 623         input = open(input_name, O_RDONLY);
 624         if (input < 0) {
 625                 perror("failed to open file");
 626                 exit(-1);
 627         }
 628
 629         ret = fstat(input, &input_stat);
 630         if (ret < 0) {
 631                 perror("failed to stat file");
 632                 exit(-1);
 633         }
 634
 635         if (!force && input_stat.st_uid && (input_stat.st_uid != geteuid())) {
 636                 fprintf(stderr, "file: %s not owned by current user or root\n", input_name);
 637                 exit(-1);
 638         }
 639
 640         if (!input_stat.st_size) {
 641                 fprintf(stderr, "zero-sized file, nothing to do!\n");
 642                 exit(0);
 643         }
 644
 645         if (load_kernel(sizeof(struct sym_priv), symbol_filter) < 0) {
 646                 perror("failed to load kernel symbols");
 647                 return EXIT_FAILURE;
 648         }
 649
 650 remap:
 651         buf = (char *)mmap(NULL, page_size * mmap_window, PROT_READ,
 652                            MAP_SHARED, input, offset);
 653         if (buf == MAP_FAILED) {
 654                 perror("failed to mmap file");
 655                 exit(-1);
 656         }
 657
 658 more:
 659         event = (event_t *)(buf + head);
 660
 661         size = event->header.size;
 662         if (!size)
 663                 size = 8;
 664
 665         if (head + event->header.size >= page_size * mmap_window) {
 666                 unsigned long shift = page_size * (head / page_size);
 667                 int munmap_ret;
 668
 669                 munmap_ret = munmap(buf, page_size * mmap_window);
 670                 assert(munmap_ret == 0);
 671
 672                 offset += shift;
 673                 head -= shift;
 674                 goto remap;
 675         }
 676
 677         size = event->header.size;
 678
 679         dump_printf("%p [%p]: event: %d\n",
 680                         (void *)(offset + head),
 681                         (void *)(long)event->header.size,
 682                         event->header.type);
 683
 684         if (!size || process_event(event, offset, head) < 0) {
 685
 686                 dump_printf("%p [%p]: skipping unknown header type: %d\n",
 687                         (void *)(offset + head),
 688                         (void *)(long)(event->header.size),
 689                         event->header.type);
 690
 691                 total_unknown++;
 692
 693                 /*
 694                  * assume we lost track of the stream, check alignment, and
 695                  * increment a single u64 in the hope to catch on again 'soon'.
 696                  */
 697
 698                 if (unlikely(head & 7))
 699                         head &= ~7ULL;
 700
 701                 size = 8;
 702         }
 703
 704         head += size;
 705
 706         if (offset + head < (unsigned long)input_stat.st_size)
 707                 goto more;
 708
 709         rc = EXIT_SUCCESS;
 710         close(input);
 711
 712         dump_printf("      IP events: %10ld\n", total);
 713         dump_printf("    mmap events: %10ld\n", total_mmap);
 714         dump_printf("    comm events: %10ld\n", total_comm);
 715         dump_printf("    fork events: %10ld\n", total_fork);
 716         dump_printf(" unknown events: %10ld\n", total_unknown);
 717
 718         if (dump_trace)
 719                 return 0;
 720
 721         if (verbose > 3)
 722                 threads__fprintf(stdout);
 723
 724         if (verbose > 2)
 725                 dsos__fprintf(stdout);
 726
 727         collapse__resort();
 728         output__resort(total);
 729
 730         find_annotations();
 731
 732         return rc;
 733 }
 734
 735 static const char * const annotate_usage[] = {
 736         "perf annotate [<options>] <command>",
 737         NULL
 738 };
 739
 740 static const struct option options[] = {
 741         OPT_STRING('i', "input", &input_name, "file",
 742                     "input file name"),
 743         OPT_STRING('s', "symbol", &sym_hist_filter, "symbol",
 744                     "symbol to annotate"),
 745         OPT_BOOLEAN('f', "force", &force, "don't complain, do it"),
 746         OPT_BOOLEAN('v', "verbose", &verbose,
 747                     "be more verbose (show symbol address, etc)"),
 748         OPT_BOOLEAN('D', "dump-raw-trace", &dump_trace,
 749                     "dump raw trace in ASCII"),
 750         OPT_STRING('k', "vmlinux", &vmlinux_name, "file", "vmlinux pathname"),
 751         OPT_BOOLEAN('m', "modules", &modules,
 752                     "load module symbols - WARNING: use only with -k and LIVE kernel"),
 753         OPT_BOOLEAN('l', "print-line", &print_line,
 754                     "print matching source lines (may be slow)"),
 755         OPT_BOOLEAN('P', "full-paths", &full_paths,
 756                     "Don't shorten the displayed pathnames"),
 757         OPT_END()
 758 };
 759
 760 static void setup_sorting(void)
 761 {
 762         char *tmp, *tok, *str = strdup(sort_order);
 763
 764         for (tok = strtok_r(str, ", ", &tmp);
 765                         tok; tok = strtok_r(NULL, ", ", &tmp)) {
 766                 if (sort_dimension__add(tok) < 0) {
 767                         error("Unknown --sort key: `%s'", tok);
 768                         usage_with_options(annotate_usage, options);
 769                 }
 770         }
 771
 772         free(str);
 773 }
 774
 775 int cmd_annotate(int argc, const char **argv, const char *prefix __used)
 776 {
 777         symbol__init();
 778
 779         page_size = getpagesize();
 780
 781         argc = parse_options(argc, argv, options, annotate_usage, 0);
 782
 783         setup_sorting();
 784
 785         if (argc) {
 786                 /*
 787                  * Special case: if there's an argument left then assume tha
 788                  * it's a symbol filter:
 789                  */
 790                 if (argc > 1)
 791                         usage_with_options(annotate_usage, options);
 792
 793                 sym_hist_filter = argv[0];
 794         }
 795
 796         if (!sym_hist_filter)
 797                 usage_with_options(annotate_usage, options);
 798
 799         setup_pager();
 800
 801         if (field_sep && *field_sep == '.') {
 802                 fputs("'.' is the only non valid --field-separator argument\n",
 803                                 stderr);
 804                 exit(129);
 805         }
 806
 807         return __cmd_annotate();
 808 }