Pileus Git - ~andy/linux/blob - kernel/posix-cpu-timers.c

   1 /*
   2  * Implement CPU time clocks for the POSIX clock interface.
   3  */
   4
   5 #include <linux/sched.h>
   6 #include <linux/posix-timers.h>
   7 #include <linux/errno.h>
   8 #include <linux/math64.h>
   9 #include <asm/uaccess.h>
  10 #include <linux/kernel_stat.h>
  11 #include <trace/events/timer.h>
  12 #include <linux/random.h>
  13 #include <linux/tick.h>
  14 #include <linux/workqueue.h>
  15
  16 /*
  17  * Called after updating RLIMIT_CPU to run cpu timer and update
  18  * tsk->signal->cputime_expires expiration cache if necessary. Needs
  19  * siglock protection since other code may update expiration cache as
  20  * well.
  21  */
  22 void update_rlimit_cpu(struct task_struct *task, unsigned long rlim_new)
  23 {
  24         cputime_t cputime = secs_to_cputime(rlim_new);
  25
  26         spin_lock_irq(&task->sighand->siglock);
  27         set_process_cpu_timer(task, CPUCLOCK_PROF, &cputime, NULL);
  28         spin_unlock_irq(&task->sighand->siglock);
  29 }
  30
  31 static int check_clock(const clockid_t which_clock)
  32 {
  33         int error = 0;
  34         struct task_struct *p;
  35         const pid_t pid = CPUCLOCK_PID(which_clock);
  36
  37         if (CPUCLOCK_WHICH(which_clock) >= CPUCLOCK_MAX)
  38                 return -EINVAL;
  39
  40         if (pid == 0)
  41                 return 0;
  42
  43         rcu_read_lock();
  44         p = find_task_by_vpid(pid);
  45         if (!p || !(CPUCLOCK_PERTHREAD(which_clock) ?
  46                    same_thread_group(p, current) : has_group_leader_pid(p))) {
  47                 error = -EINVAL;
  48         }
  49         rcu_read_unlock();
  50
  51         return error;
  52 }
  53
  54 static inline union cpu_time_count
  55 timespec_to_sample(const clockid_t which_clock, const struct timespec *tp)
  56 {
  57         union cpu_time_count ret;
  58         ret.sched = 0;          /* high half always zero when .cpu used */
  59         if (CPUCLOCK_WHICH(which_clock) == CPUCLOCK_SCHED) {
  60                 ret.sched = (unsigned long long)tp->tv_sec * NSEC_PER_SEC + tp->tv_nsec;
  61         } else {
  62                 ret.cpu = timespec_to_cputime(tp);
  63         }
  64         return ret;
  65 }
  66
  67 static void sample_to_timespec(const clockid_t which_clock,
  68                                union cpu_time_count cpu,
  69                                struct timespec *tp)
  70 {
  71         if (CPUCLOCK_WHICH(which_clock) == CPUCLOCK_SCHED)
  72                 *tp = ns_to_timespec(cpu.sched);
  73         else
  74                 cputime_to_timespec(cpu.cpu, tp);
  75 }
  76
  77 static inline int cpu_time_before(const clockid_t which_clock,
  78                                   union cpu_time_count now,
  79                                   union cpu_time_count then)
  80 {
  81         if (CPUCLOCK_WHICH(which_clock) == CPUCLOCK_SCHED) {
  82                 return now.sched < then.sched;
  83         }  else {
  84                 return now.cpu < then.cpu;
  85         }
  86 }
  87 static inline void cpu_time_add(const clockid_t which_clock,
  88                                 union cpu_time_count *acc,
  89                                 union cpu_time_count val)
  90 {
  91         if (CPUCLOCK_WHICH(which_clock) == CPUCLOCK_SCHED) {
  92                 acc->sched += val.sched;
  93         }  else {
  94                 acc->cpu += val.cpu;
  95         }
  96 }
  97 static inline union cpu_time_count cpu_time_sub(const clockid_t which_clock,
  98                                                 union cpu_time_count a,
  99                                                 union cpu_time_count b)
 100 {
 101         if (CPUCLOCK_WHICH(which_clock) == CPUCLOCK_SCHED) {
 102                 a.sched -= b.sched;
 103         }  else {
 104                 a.cpu -= b.cpu;
 105         }
 106         return a;
 107 }
 108
 109 /*
 110  * Update expiry time from increment, and increase overrun count,
 111  * given the current clock sample.
 112  */
 113 static void bump_cpu_timer(struct k_itimer *timer,
 114                                   union cpu_time_count now)
 115 {
 116         int i;
 117
 118         if (timer->it.cpu.incr.sched == 0)
 119                 return;
 120
 121         if (CPUCLOCK_WHICH(timer->it_clock) == CPUCLOCK_SCHED) {
 122                 unsigned long long delta, incr;
 123
 124                 if (now.sched < timer->it.cpu.expires.sched)
 125                         return;
 126                 incr = timer->it.cpu.incr.sched;
 127                 delta = now.sched + incr - timer->it.cpu.expires.sched;
 128                 /* Don't use (incr*2 < delta), incr*2 might overflow. */
 129                 for (i = 0; incr < delta - incr; i++)
 130                         incr = incr << 1;
 131                 for (; i >= 0; incr >>= 1, i--) {
 132                         if (delta < incr)
 133                                 continue;
 134                         timer->it.cpu.expires.sched += incr;
 135                         timer->it_overrun += 1 << i;
 136                         delta -= incr;
 137                 }
 138         } else {
 139                 cputime_t delta, incr;
 140
 141                 if (now.cpu < timer->it.cpu.expires.cpu)
 142                         return;
 143                 incr = timer->it.cpu.incr.cpu;
 144                 delta = now.cpu + incr - timer->it.cpu.expires.cpu;
 145                 /* Don't use (incr*2 < delta), incr*2 might overflow. */
 146                 for (i = 0; incr < delta - incr; i++)
 147                              incr += incr;
 148                 for (; i >= 0; incr = incr >> 1, i--) {
 149                         if (delta < incr)
 150                                 continue;
 151                         timer->it.cpu.expires.cpu += incr;
 152                         timer->it_overrun += 1 << i;
 153                         delta -= incr;
 154                 }
 155         }
 156 }
 157
 158 static inline cputime_t prof_ticks(struct task_struct *p)
 159 {
 160         cputime_t utime, stime;
 161
 162         task_cputime(p, &utime, &stime);
 163
 164         return utime + stime;
 165 }
 166 static inline cputime_t virt_ticks(struct task_struct *p)
 167 {
 168         cputime_t utime;
 169
 170         task_cputime(p, &utime, NULL);
 171
 172         return utime;
 173 }
 174
 175 static int
 176 posix_cpu_clock_getres(const clockid_t which_clock, struct timespec *tp)
 177 {
 178         int error = check_clock(which_clock);
 179         if (!error) {
 180                 tp->tv_sec = 0;
 181                 tp->tv_nsec = ((NSEC_PER_SEC + HZ - 1) / HZ);
 182                 if (CPUCLOCK_WHICH(which_clock) == CPUCLOCK_SCHED) {
 183                         /*
 184                          * If sched_clock is using a cycle counter, we
 185                          * don't have any idea of its true resolution
 186                          * exported, but it is much more than 1s/HZ.
 187                          */
 188                         tp->tv_nsec = 1;
 189                 }
 190         }
 191         return error;
 192 }
 193
 194 static int
 195 posix_cpu_clock_set(const clockid_t which_clock, const struct timespec *tp)
 196 {
 197         /*
 198          * You can never reset a CPU clock, but we check for other errors
 199          * in the call before failing with EPERM.
 200          */
 201         int error = check_clock(which_clock);
 202         if (error == 0) {
 203                 error = -EPERM;
 204         }
 205         return error;
 206 }
 207
 208
 209 /*
 210  * Sample a per-thread clock for the given task.
 211  */
 212 static int cpu_clock_sample(const clockid_t which_clock, struct task_struct *p,
 213                             union cpu_time_count *cpu)
 214 {
 215         switch (CPUCLOCK_WHICH(which_clock)) {
 216         default:
 217                 return -EINVAL;
 218         case CPUCLOCK_PROF:
 219                 cpu->cpu = prof_ticks(p);
 220                 break;
 221         case CPUCLOCK_VIRT:
 222                 cpu->cpu = virt_ticks(p);
 223                 break;
 224         case CPUCLOCK_SCHED:
 225                 cpu->sched = task_sched_runtime(p);
 226                 break;
 227         }
 228         return 0;
 229 }
 230
 231 static void update_gt_cputime(struct task_cputime *a, struct task_cputime *b)
 232 {
 233         if (b->utime > a->utime)
 234                 a->utime = b->utime;
 235
 236         if (b->stime > a->stime)
 237                 a->stime = b->stime;
 238
 239         if (b->sum_exec_runtime > a->sum_exec_runtime)
 240                 a->sum_exec_runtime = b->sum_exec_runtime;
 241 }
 242
 243 void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times)
 244 {
 245         struct thread_group_cputimer *cputimer = &tsk->signal->cputimer;
 246         struct task_cputime sum;
 247         unsigned long flags;
 248
 249         if (!cputimer->running) {
 250                 /*
 251                  * The POSIX timer interface allows for absolute time expiry
 252                  * values through the TIMER_ABSTIME flag, therefore we have
 253                  * to synchronize the timer to the clock every time we start
 254                  * it.
 255                  */
 256                 thread_group_cputime(tsk, &sum);
 257                 raw_spin_lock_irqsave(&cputimer->lock, flags);
 258                 cputimer->running = 1;
 259                 update_gt_cputime(&cputimer->cputime, &sum);
 260         } else
 261                 raw_spin_lock_irqsave(&cputimer->lock, flags);
 262         *times = cputimer->cputime;
 263         raw_spin_unlock_irqrestore(&cputimer->lock, flags);
 264 }
 265
 266 /*
 267  * Sample a process (thread group) clock for the given group_leader task.
 268  * Must be called with tasklist_lock held for reading.
 269  */
 270 static int cpu_clock_sample_group(const clockid_t which_clock,
 271                                   struct task_struct *p,
 272                                   union cpu_time_count *cpu)
 273 {
 274         struct task_cputime cputime;
 275
 276         switch (CPUCLOCK_WHICH(which_clock)) {
 277         default:
 278                 return -EINVAL;
 279         case CPUCLOCK_PROF:
 280                 thread_group_cputime(p, &cputime);
 281                 cpu->cpu = cputime.utime + cputime.stime;
 282                 break;
 283         case CPUCLOCK_VIRT:
 284                 thread_group_cputime(p, &cputime);
 285                 cpu->cpu = cputime.utime;
 286                 break;
 287         case CPUCLOCK_SCHED:
 288                 thread_group_cputime(p, &cputime);
 289                 cpu->sched = cputime.sum_exec_runtime;
 290                 break;
 291         }
 292         return 0;
 293 }
 294
 295
 296 static int posix_cpu_clock_get(const clockid_t which_clock, struct timespec *tp)
 297 {
 298         const pid_t pid = CPUCLOCK_PID(which_clock);
 299         int error = -EINVAL;
 300         union cpu_time_count rtn;
 301
 302         if (pid == 0) {
 303                 /*
 304                  * Special case constant value for our own clocks.
 305                  * We don't have to do any lookup to find ourselves.
 306                  */
 307                 if (CPUCLOCK_PERTHREAD(which_clock)) {
 308                         /*
 309                          * Sampling just ourselves we can do with no locking.
 310                          */
 311                         error = cpu_clock_sample(which_clock,
 312                                                  current, &rtn);
 313                 } else {
 314                         read_lock(&tasklist_lock);
 315                         error = cpu_clock_sample_group(which_clock,
 316                                                        current, &rtn);
 317                         read_unlock(&tasklist_lock);
 318                 }
 319         } else {
 320                 /*
 321                  * Find the given PID, and validate that the caller
 322                  * should be able to see it.
 323                  */
 324                 struct task_struct *p;
 325                 rcu_read_lock();
 326                 p = find_task_by_vpid(pid);
 327                 if (p) {
 328                         if (CPUCLOCK_PERTHREAD(which_clock)) {
 329                                 if (same_thread_group(p, current)) {
 330                                         error = cpu_clock_sample(which_clock,
 331                                                                  p, &rtn);
 332                                 }
 333                         } else {
 334                                 read_lock(&tasklist_lock);
 335                                 if (thread_group_leader(p) && p->sighand) {
 336                                         error =
 337                                             cpu_clock_sample_group(which_clock,
 338                                                                    p, &rtn);
 339                                 }
 340                                 read_unlock(&tasklist_lock);
 341                         }
 342                 }
 343                 rcu_read_unlock();
 344         }
 345
 346         if (error)
 347                 return error;
 348         sample_to_timespec(which_clock, rtn, tp);
 349         return 0;
 350 }
 351
 352
 353 /*
 354  * Validate the clockid_t for a new CPU-clock timer, and initialize the timer.
 355  * This is called from sys_timer_create() and do_cpu_nanosleep() with the
 356  * new timer already all-zeros initialized.
 357  */
 358 static int posix_cpu_timer_create(struct k_itimer *new_timer)
 359 {
 360         int ret = 0;
 361         const pid_t pid = CPUCLOCK_PID(new_timer->it_clock);
 362         struct task_struct *p;
 363
 364         if (CPUCLOCK_WHICH(new_timer->it_clock) >= CPUCLOCK_MAX)
 365                 return -EINVAL;
 366
 367         INIT_LIST_HEAD(&new_timer->it.cpu.entry);
 368
 369         rcu_read_lock();
 370         if (CPUCLOCK_PERTHREAD(new_timer->it_clock)) {
 371                 if (pid == 0) {
 372                         p = current;
 373                 } else {
 374                         p = find_task_by_vpid(pid);
 375                         if (p && !same_thread_group(p, current))
 376                                 p = NULL;
 377                 }
 378         } else {
 379                 if (pid == 0) {
 380                         p = current->group_leader;
 381                 } else {
 382                         p = find_task_by_vpid(pid);
 383                         if (p && !has_group_leader_pid(p))
 384                                 p = NULL;
 385                 }
 386         }
 387         new_timer->it.cpu.task = p;
 388         if (p) {
 389                 get_task_struct(p);
 390         } else {
 391                 ret = -EINVAL;
 392         }
 393         rcu_read_unlock();
 394
 395         return ret;
 396 }
 397
 398 /*
 399  * Clean up a CPU-clock timer that is about to be destroyed.
 400  * This is called from timer deletion with the timer already locked.
 401  * If we return TIMER_RETRY, it's necessary to release the timer's lock
 402  * and try again.  (This happens when the timer is in the middle of firing.)
 403  */
 404 static int posix_cpu_timer_del(struct k_itimer *timer)
 405 {
 406         struct task_struct *p = timer->it.cpu.task;
 407         int ret = 0;
 408
 409         if (likely(p != NULL)) {
 410                 read_lock(&tasklist_lock);
 411                 if (unlikely(p->sighand == NULL)) {
 412                         /*
 413                          * We raced with the reaping of the task.
 414                          * The deletion should have cleared us off the list.
 415                          */
 416                         BUG_ON(!list_empty(&timer->it.cpu.entry));
 417                 } else {
 418                         spin_lock(&p->sighand->siglock);
 419                         if (timer->it.cpu.firing)
 420                                 ret = TIMER_RETRY;
 421                         else
 422                                 list_del(&timer->it.cpu.entry);
 423                         spin_unlock(&p->sighand->siglock);
 424                 }
 425                 read_unlock(&tasklist_lock);
 426
 427                 if (!ret)
 428                         put_task_struct(p);
 429         }
 430
 431         return ret;
 432 }
 433
 434 /*
 435  * Clean out CPU timers still ticking when a thread exited.  The task
 436  * pointer is cleared, and the expiry time is replaced with the residual
 437  * time for later timer_gettime calls to return.
 438  * This must be called with the siglock held.
 439  */
 440 static void cleanup_timers(struct list_head *head,
 441                            cputime_t utime, cputime_t stime,
 442                            unsigned long long sum_exec_runtime)
 443 {
 444         struct cpu_timer_list *timer, *next;
 445         cputime_t ptime = utime + stime;
 446
 447         list_for_each_entry_safe(timer, next, head, entry) {
 448                 list_del_init(&timer->entry);
 449                 if (timer->expires.cpu < ptime) {
 450                         timer->expires.cpu = 0;
 451                 } else {
 452                         timer->expires.cpu -= ptime;
 453                 }
 454         }
 455
 456         ++head;
 457         list_for_each_entry_safe(timer, next, head, entry) {
 458                 list_del_init(&timer->entry);
 459                 if (timer->expires.cpu < utime) {
 460                         timer->expires.cpu = 0;
 461                 } else {
 462                         timer->expires.cpu -= utime;
 463                 }
 464         }
 465
 466         ++head;
 467         list_for_each_entry_safe(timer, next, head, entry) {
 468                 list_del_init(&timer->entry);
 469                 if (timer->expires.sched < sum_exec_runtime) {
 470                         timer->expires.sched = 0;
 471                 } else {
 472                         timer->expires.sched -= sum_exec_runtime;
 473                 }
 474         }
 475 }
 476
 477 /*
 478  * These are both called with the siglock held, when the current thread
 479  * is being reaped.  When the final (leader) thread in the group is reaped,
 480  * posix_cpu_timers_exit_group will be called after posix_cpu_timers_exit.
 481  */
 482 void posix_cpu_timers_exit(struct task_struct *tsk)
 483 {
 484         cputime_t utime, stime;
 485
 486         add_device_randomness((const void*) &tsk->se.sum_exec_runtime,
 487                                                 sizeof(unsigned long long));
 488         task_cputime(tsk, &utime, &stime);
 489         cleanup_timers(tsk->cpu_timers,
 490                        utime, stime, tsk->se.sum_exec_runtime);
 491
 492 }
 493 void posix_cpu_timers_exit_group(struct task_struct *tsk)
 494 {
 495         struct signal_struct *const sig = tsk->signal;
 496         cputime_t utime, stime;
 497
 498         task_cputime(tsk, &utime, &stime);
 499         cleanup_timers(tsk->signal->cpu_timers,
 500                        utime + sig->utime, stime + sig->stime,
 501                        tsk->se.sum_exec_runtime + sig->sum_sched_runtime);
 502 }
 503
 504 static void clear_dead_task(struct k_itimer *timer, union cpu_time_count now)
 505 {
 506         /*
 507          * That's all for this thread or process.
 508          * We leave our residual in expires to be reported.
 509          */
 510         put_task_struct(timer->it.cpu.task);
 511         timer->it.cpu.task = NULL;
 512         timer->it.cpu.expires = cpu_time_sub(timer->it_clock,
 513                                              timer->it.cpu.expires,
 514                                              now);
 515 }
 516
 517 static inline int expires_gt(cputime_t expires, cputime_t new_exp)
 518 {
 519         return expires == 0 || expires > new_exp;
 520 }
 521
 522 /*
 523  * Insert the timer on the appropriate list before any timers that
 524  * expire later.  This must be called with the tasklist_lock held
 525  * for reading, interrupts disabled and p->sighand->siglock taken.
 526  */
 527 static void arm_timer(struct k_itimer *timer)
 528 {
 529         struct task_struct *p = timer->it.cpu.task;
 530         struct list_head *head, *listpos;
 531         struct task_cputime *cputime_expires;
 532         struct cpu_timer_list *const nt = &timer->it.cpu;
 533         struct cpu_timer_list *next;
 534
 535         if (CPUCLOCK_PERTHREAD(timer->it_clock)) {
 536                 head = p->cpu_timers;
 537                 cputime_expires = &p->cputime_expires;
 538         } else {
 539                 head = p->signal->cpu_timers;
 540                 cputime_expires = &p->signal->cputime_expires;
 541         }
 542         head += CPUCLOCK_WHICH(timer->it_clock);
 543
 544         listpos = head;
 545         list_for_each_entry(next, head, entry) {
 546                 if (cpu_time_before(timer->it_clock, nt->expires, next->expires))
 547                         break;
 548                 listpos = &next->entry;
 549         }
 550         list_add(&nt->entry, listpos);
 551
 552         if (listpos == head) {
 553                 union cpu_time_count *exp = &nt->expires;
 554
 555                 /*
 556                  * We are the new earliest-expiring POSIX 1.b timer, hence
 557                  * need to update expiration cache. Take into account that
 558                  * for process timers we share expiration cache with itimers
 559                  * and RLIMIT_CPU and for thread timers with RLIMIT_RTTIME.
 560                  */
 561
 562                 switch (CPUCLOCK_WHICH(timer->it_clock)) {
 563                 case CPUCLOCK_PROF:
 564                         if (expires_gt(cputime_expires->prof_exp, exp->cpu))
 565                                 cputime_expires->prof_exp = exp->cpu;
 566                         break;
 567                 case CPUCLOCK_VIRT:
 568                         if (expires_gt(cputime_expires->virt_exp, exp->cpu))
 569                                 cputime_expires->virt_exp = exp->cpu;
 570                         break;
 571                 case CPUCLOCK_SCHED:
 572                         if (cputime_expires->sched_exp == 0 ||
 573                             cputime_expires->sched_exp > exp->sched)
 574                                 cputime_expires->sched_exp = exp->sched;
 575                         break;
 576                 }
 577         }
 578 }
 579
 580 /*
 581  * The timer is locked, fire it and arrange for its reload.
 582  */
 583 static void cpu_timer_fire(struct k_itimer *timer)
 584 {
 585         if ((timer->it_sigev_notify & ~SIGEV_THREAD_ID) == SIGEV_NONE) {
 586                 /*
 587                  * User don't want any signal.
 588                  */
 589                 timer->it.cpu.expires.sched = 0;
 590         } else if (unlikely(timer->sigq == NULL)) {
 591                 /*
 592                  * This a special case for clock_nanosleep,
 593                  * not a normal timer from sys_timer_create.
 594                  */
 595                 wake_up_process(timer->it_process);
 596                 timer->it.cpu.expires.sched = 0;
 597         } else if (timer->it.cpu.incr.sched == 0) {
 598                 /*
 599                  * One-shot timer.  Clear it as soon as it's fired.
 600                  */
 601                 posix_timer_event(timer, 0);
 602                 timer->it.cpu.expires.sched = 0;
 603         } else if (posix_timer_event(timer, ++timer->it_requeue_pending)) {
 604                 /*
 605                  * The signal did not get queued because the signal
 606                  * was ignored, so we won't get any callback to
 607                  * reload the timer.  But we need to keep it
 608                  * ticking in case the signal is deliverable next time.
 609                  */
 610                 posix_cpu_timer_schedule(timer);
 611         }
 612 }
 613
 614 /*
 615  * Sample a process (thread group) timer for the given group_leader task.
 616  * Must be called with tasklist_lock held for reading.
 617  */
 618 static int cpu_timer_sample_group(const clockid_t which_clock,
 619                                   struct task_struct *p,
 620                                   union cpu_time_count *cpu)
 621 {
 622         struct task_cputime cputime;
 623
 624         thread_group_cputimer(p, &cputime);
 625         switch (CPUCLOCK_WHICH(which_clock)) {
 626         default:
 627                 return -EINVAL;
 628         case CPUCLOCK_PROF:
 629                 cpu->cpu = cputime.utime + cputime.stime;
 630                 break;
 631         case CPUCLOCK_VIRT:
 632                 cpu->cpu = cputime.utime;
 633                 break;
 634         case CPUCLOCK_SCHED:
 635                 cpu->sched = cputime.sum_exec_runtime + task_delta_exec(p);
 636                 break;
 637         }
 638         return 0;
 639 }
 640
 641 #ifdef CONFIG_NO_HZ_FULL
 642 static void nohz_kick_work_fn(struct work_struct *work)
 643 {
 644         tick_nohz_full_kick_all();
 645 }
 646
 647 static DECLARE_WORK(nohz_kick_work, nohz_kick_work_fn);
 648
 649 /*
 650  * We need the IPIs to be sent from sane process context.
 651  * The posix cpu timers are always set with irqs disabled.
 652  */
 653 static void posix_cpu_timer_kick_nohz(void)
 654 {
 655         schedule_work(&nohz_kick_work);
 656 }
 657 #else
 658 static inline void posix_cpu_timer_kick_nohz(void) { }
 659 #endif
 660
 661 /*
 662  * Guts of sys_timer_settime for CPU timers.
 663  * This is called with the timer locked and interrupts disabled.
 664  * If we return TIMER_RETRY, it's necessary to release the timer's lock
 665  * and try again.  (This happens when the timer is in the middle of firing.)
 666  */
 667 static int posix_cpu_timer_set(struct k_itimer *timer, int flags,
 668                                struct itimerspec *new, struct itimerspec *old)
 669 {
 670         struct task_struct *p = timer->it.cpu.task;
 671         union cpu_time_count old_expires, new_expires, old_incr, val;
 672         int ret;
 673
 674         if (unlikely(p == NULL)) {
 675                 /*
 676                  * Timer refers to a dead task's clock.
 677                  */
 678                 return -ESRCH;
 679         }
 680
 681         new_expires = timespec_to_sample(timer->it_clock, &new->it_value);
 682
 683         read_lock(&tasklist_lock);
 684         /*
 685          * We need the tasklist_lock to protect against reaping that
 686          * clears p->sighand.  If p has just been reaped, we can no
 687          * longer get any information about it at all.
 688          */
 689         if (unlikely(p->sighand == NULL)) {
 690                 read_unlock(&tasklist_lock);
 691                 put_task_struct(p);
 692                 timer->it.cpu.task = NULL;
 693                 return -ESRCH;
 694         }
 695
 696         /*
 697          * Disarm any old timer after extracting its expiry time.
 698          */
 699         BUG_ON(!irqs_disabled());
 700
 701         ret = 0;
 702         old_incr = timer->it.cpu.incr;
 703         spin_lock(&p->sighand->siglock);
 704         old_expires = timer->it.cpu.expires;
 705         if (unlikely(timer->it.cpu.firing)) {
 706                 timer->it.cpu.firing = -1;
 707                 ret = TIMER_RETRY;
 708         } else
 709                 list_del_init(&timer->it.cpu.entry);
 710
 711         /*
 712          * We need to sample the current value to convert the new
 713          * value from to relative and absolute, and to convert the
 714          * old value from absolute to relative.  To set a process
 715          * timer, we need a sample to balance the thread expiry
 716          * times (in arm_timer).  With an absolute time, we must
 717          * check if it's already passed.  In short, we need a sample.
 718          */
 719         if (CPUCLOCK_PERTHREAD(timer->it_clock)) {
 720                 cpu_clock_sample(timer->it_clock, p, &val);
 721         } else {
 722                 cpu_timer_sample_group(timer->it_clock, p, &val);
 723         }
 724
 725         if (old) {
 726                 if (old_expires.sched == 0) {
 727                         old->it_value.tv_sec = 0;
 728                         old->it_value.tv_nsec = 0;
 729                 } else {
 730                         /*
 731                          * Update the timer in case it has
 732                          * overrun already.  If it has,
 733                          * we'll report it as having overrun
 734                          * and with the next reloaded timer
 735                          * already ticking, though we are
 736                          * swallowing that pending
 737                          * notification here to install the
 738                          * new setting.
 739                          */
 740                         bump_cpu_timer(timer, val);
 741                         if (cpu_time_before(timer->it_clock, val,
 742                                             timer->it.cpu.expires)) {
 743                                 old_expires = cpu_time_sub(
 744                                         timer->it_clock,
 745                                         timer->it.cpu.expires, val);
 746                                 sample_to_timespec(timer->it_clock,
 747                                                    old_expires,
 748                                                    &old->it_value);
 749                         } else {
 750                                 old->it_value.tv_nsec = 1;
 751                                 old->it_value.tv_sec = 0;
 752                         }
 753                 }
 754         }
 755
 756         if (unlikely(ret)) {
 757                 /*
 758                  * We are colliding with the timer actually firing.
 759                  * Punt after filling in the timer's old value, and
 760                  * disable this firing since we are already reporting
 761                  * it as an overrun (thanks to bump_cpu_timer above).
 762                  */
 763                 spin_unlock(&p->sighand->siglock);
 764                 read_unlock(&tasklist_lock);
 765                 goto out;
 766         }
 767
 768         if (new_expires.sched != 0 && !(flags & TIMER_ABSTIME)) {
 769                 cpu_time_add(timer->it_clock, &new_expires, val);
 770         }
 771
 772         /*
 773          * Install the new expiry time (or zero).
 774          * For a timer with no notification action, we don't actually
 775          * arm the timer (we'll just fake it for timer_gettime).
 776          */
 777         timer->it.cpu.expires = new_expires;
 778         if (new_expires.sched != 0 &&
 779             cpu_time_before(timer->it_clock, val, new_expires)) {
 780                 arm_timer(timer);
 781         }
 782
 783         spin_unlock(&p->sighand->siglock);
 784         read_unlock(&tasklist_lock);
 785
 786         /*
 787          * Install the new reload setting, and
 788          * set up the signal and overrun bookkeeping.
 789          */
 790         timer->it.cpu.incr = timespec_to_sample(timer->it_clock,
 791                                                 &new->it_interval);
 792
 793         /*
 794          * This acts as a modification timestamp for the timer,
 795          * so any automatic reload attempt will punt on seeing
 796          * that we have reset the timer manually.
 797          */
 798         timer->it_requeue_pending = (timer->it_requeue_pending + 2) &
 799                 ~REQUEUE_PENDING;
 800         timer->it_overrun_last = 0;
 801         timer->it_overrun = -1;
 802
 803         if (new_expires.sched != 0 &&
 804             !cpu_time_before(timer->it_clock, val, new_expires)) {
 805                 /*
 806                  * The designated time already passed, so we notify
 807                  * immediately, even if the thread never runs to
 808                  * accumulate more time on this clock.
 809                  */
 810                 cpu_timer_fire(timer);
 811         }
 812
 813         ret = 0;
 814  out:
 815         if (old) {
 816                 sample_to_timespec(timer->it_clock,
 817                                    old_incr, &old->it_interval);
 818         }
 819         if (!ret)
 820                 posix_cpu_timer_kick_nohz();
 821         return ret;
 822 }
 823
 824 static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec *itp)
 825 {
 826         union cpu_time_count now;
 827         struct task_struct *p = timer->it.cpu.task;
 828         int clear_dead;
 829
 830         /*
 831          * Easy part: convert the reload time.
 832          */
 833         sample_to_timespec(timer->it_clock,
 834                            timer->it.cpu.incr, &itp->it_interval);
 835
 836         if (timer->it.cpu.expires.sched == 0) { /* Timer not armed at all.  */
 837                 itp->it_value.tv_sec = itp->it_value.tv_nsec = 0;
 838                 return;
 839         }
 840
 841         if (unlikely(p == NULL)) {
 842                 /*
 843                  * This task already died and the timer will never fire.
 844                  * In this case, expires is actually the dead value.
 845                  */
 846         dead:
 847                 sample_to_timespec(timer->it_clock, timer->it.cpu.expires,
 848                                    &itp->it_value);
 849                 return;
 850         }
 851
 852         /*
 853          * Sample the clock to take the difference with the expiry time.
 854          */
 855         if (CPUCLOCK_PERTHREAD(timer->it_clock)) {
 856                 cpu_clock_sample(timer->it_clock, p, &now);
 857                 clear_dead = p->exit_state;
 858         } else {
 859                 read_lock(&tasklist_lock);
 860                 if (unlikely(p->sighand == NULL)) {
 861                         /*
 862                          * The process has been reaped.
 863                          * We can't even collect a sample any more.
 864                          * Call the timer disarmed, nothing else to do.
 865                          */
 866                         put_task_struct(p);
 867                         timer->it.cpu.task = NULL;
 868                         timer->it.cpu.expires.sched = 0;
 869                         read_unlock(&tasklist_lock);
 870                         goto dead;
 871                 } else {
 872                         cpu_timer_sample_group(timer->it_clock, p, &now);
 873                         clear_dead = (unlikely(p->exit_state) &&
 874                                       thread_group_empty(p));
 875                 }
 876                 read_unlock(&tasklist_lock);
 877         }
 878
 879         if (unlikely(clear_dead)) {
 880                 /*
 881                  * We've noticed that the thread is dead, but
 882                  * not yet reaped.  Take this opportunity to
 883                  * drop our task ref.
 884                  */
 885                 clear_dead_task(timer, now);
 886                 goto dead;
 887         }
 888
 889         if (cpu_time_before(timer->it_clock, now, timer->it.cpu.expires)) {
 890                 sample_to_timespec(timer->it_clock,
 891                                    cpu_time_sub(timer->it_clock,
 892                                                 timer->it.cpu.expires, now),
 893                                    &itp->it_value);
 894         } else {
 895                 /*
 896                  * The timer should have expired already, but the firing
 897                  * hasn't taken place yet.  Say it's just about to expire.
 898                  */
 899                 itp->it_value.tv_nsec = 1;
 900                 itp->it_value.tv_sec = 0;
 901         }
 902 }
 903
 904 /*
 905  * Check for any per-thread CPU timers that have fired and move them off
 906  * the tsk->cpu_timers[N] list onto the firing list.  Here we update the
 907  * tsk->it_*_expires values to reflect the remaining thread CPU timers.
 908  */
 909 static void check_thread_timers(struct task_struct *tsk,
 910                                 struct list_head *firing)
 911 {
 912         int maxfire;
 913         struct list_head *timers = tsk->cpu_timers;
 914         struct signal_struct *const sig = tsk->signal;
 915         unsigned long soft;
 916
 917         maxfire = 20;
 918         tsk->cputime_expires.prof_exp = 0;
 919         while (!list_empty(timers)) {
 920                 struct cpu_timer_list *t = list_first_entry(timers,
 921                                                       struct cpu_timer_list,
 922                                                       entry);
 923                 if (!--maxfire || prof_ticks(tsk) < t->expires.cpu) {
 924                         tsk->cputime_expires.prof_exp = t->expires.cpu;
 925                         break;
 926                 }
 927                 t->firing = 1;
 928                 list_move_tail(&t->entry, firing);
 929         }
 930
 931         ++timers;
 932         maxfire = 20;
 933         tsk->cputime_expires.virt_exp = 0;
 934         while (!list_empty(timers)) {
 935                 struct cpu_timer_list *t = list_first_entry(timers,
 936                                                       struct cpu_timer_list,
 937                                                       entry);
 938                 if (!--maxfire || virt_ticks(tsk) < t->expires.cpu) {
 939                         tsk->cputime_expires.virt_exp = t->expires.cpu;
 940                         break;
 941                 }
 942                 t->firing = 1;
 943                 list_move_tail(&t->entry, firing);
 944         }
 945
 946         ++timers;
 947         maxfire = 20;
 948         tsk->cputime_expires.sched_exp = 0;
 949         while (!list_empty(timers)) {
 950                 struct cpu_timer_list *t = list_first_entry(timers,
 951                                                       struct cpu_timer_list,
 952                                                       entry);
 953                 if (!--maxfire || tsk->se.sum_exec_runtime < t->expires.sched) {
 954                         tsk->cputime_expires.sched_exp = t->expires.sched;
 955                         break;
 956                 }
 957                 t->firing = 1;
 958                 list_move_tail(&t->entry, firing);
 959         }
 960
 961         /*
 962          * Check for the special case thread timers.
 963          */
 964         soft = ACCESS_ONCE(sig->rlim[RLIMIT_RTTIME].rlim_cur);
 965         if (soft != RLIM_INFINITY) {
 966                 unsigned long hard =
 967                         ACCESS_ONCE(sig->rlim[RLIMIT_RTTIME].rlim_max);
 968
 969                 if (hard != RLIM_INFINITY &&
 970                     tsk->rt.timeout > DIV_ROUND_UP(hard, USEC_PER_SEC/HZ)) {
 971                         /*
 972                          * At the hard limit, we just die.
 973                          * No need to calculate anything else now.
 974                          */
 975                         __group_send_sig_info(SIGKILL, SEND_SIG_PRIV, tsk);
 976                         return;
 977                 }
 978                 if (tsk->rt.timeout > DIV_ROUND_UP(soft, USEC_PER_SEC/HZ)) {
 979                         /*
 980                          * At the soft limit, send a SIGXCPU every second.
 981                          */
 982                         if (soft < hard) {
 983                                 soft += USEC_PER_SEC;
 984                                 sig->rlim[RLIMIT_RTTIME].rlim_cur = soft;
 985                         }
 986                         printk(KERN_INFO
 987                                 "RT Watchdog Timeout: %s[%d]\n",
 988                                 tsk->comm, task_pid_nr(tsk));
 989                         __group_send_sig_info(SIGXCPU, SEND_SIG_PRIV, tsk);
 990                 }
 991         }
 992 }
 993
 994 static void stop_process_timers(struct signal_struct *sig)
 995 {
 996         struct thread_group_cputimer *cputimer = &sig->cputimer;
 997         unsigned long flags;
 998
 999         raw_spin_lock_irqsave(&cputimer->lock, flags);
1000         cputimer->running = 0;
1001         raw_spin_unlock_irqrestore(&cputimer->lock, flags);
1002 }
1003
1004 static u32 onecputick;
1005
1006 static void check_cpu_itimer(struct task_struct *tsk, struct cpu_itimer *it,
1007                              cputime_t *expires, cputime_t cur_time, int signo)
1008 {
1009         if (!it->expires)
1010                 return;
1011
1012         if (cur_time >= it->expires) {
1013                 if (it->incr) {
1014                         it->expires += it->incr;
1015                         it->error += it->incr_error;
1016                         if (it->error >= onecputick) {
1017                                 it->expires -= cputime_one_jiffy;
1018                                 it->error -= onecputick;
1019                         }
1020                 } else {
1021                         it->expires = 0;
1022                 }
1023
1024                 trace_itimer_expire(signo == SIGPROF ?
1025                                     ITIMER_PROF : ITIMER_VIRTUAL,
1026                                     tsk->signal->leader_pid, cur_time);
1027                 __group_send_sig_info(signo, SEND_SIG_PRIV, tsk);
1028         }
1029
1030         if (it->expires && (!*expires || it->expires < *expires)) {
1031                 *expires = it->expires;
1032         }
1033 }
1034
1035 /**
1036  * task_cputime_zero - Check a task_cputime struct for all zero fields.
1037  *
1038  * @cputime:    The struct to compare.
1039  *
1040  * Checks @cputime to see if all fields are zero.  Returns true if all fields
1041  * are zero, false if any field is nonzero.
1042  */
1043 static inline int task_cputime_zero(const struct task_cputime *cputime)
1044 {
1045         if (!cputime->utime && !cputime->stime && !cputime->sum_exec_runtime)
1046                 return 1;
1047         return 0;
1048 }
1049
1050 /*
1051  * Check for any per-thread CPU timers that have fired and move them
1052  * off the tsk->*_timers list onto the firing list.  Per-thread timers
1053  * have already been taken off.
1054  */
1055 static void check_process_timers(struct task_struct *tsk,
1056                                  struct list_head *firing)
1057 {
1058         int maxfire;
1059         struct signal_struct *const sig = tsk->signal;
1060         cputime_t utime, ptime, virt_expires, prof_expires;
1061         unsigned long long sum_sched_runtime, sched_expires;
1062         struct list_head *timers = sig->cpu_timers;
1063         struct task_cputime cputime;
1064         unsigned long soft;
1065
1066         /*
1067          * Collect the current process totals.
1068          */
1069         thread_group_cputimer(tsk, &cputime);
1070         utime = cputime.utime;
1071         ptime = utime + cputime.stime;
1072         sum_sched_runtime = cputime.sum_exec_runtime;
1073         maxfire = 20;
1074         prof_expires = 0;
1075         while (!list_empty(timers)) {
1076                 struct cpu_timer_list *tl = list_first_entry(timers,
1077                                                       struct cpu_timer_list,
1078                                                       entry);
1079                 if (!--maxfire || ptime < tl->expires.cpu) {
1080                         prof_expires = tl->expires.cpu;
1081                         break;
1082                 }
1083                 tl->firing = 1;
1084                 list_move_tail(&tl->entry, firing);
1085         }
1086
1087         ++timers;
1088         maxfire = 20;
1089         virt_expires = 0;
1090         while (!list_empty(timers)) {
1091                 struct cpu_timer_list *tl = list_first_entry(timers,
1092                                                       struct cpu_timer_list,
1093                                                       entry);
1094                 if (!--maxfire || utime < tl->expires.cpu) {
1095                         virt_expires = tl->expires.cpu;
1096                         break;
1097                 }
1098                 tl->firing = 1;
1099                 list_move_tail(&tl->entry, firing);
1100         }
1101
1102         ++timers;
1103         maxfire = 20;
1104         sched_expires = 0;
1105         while (!list_empty(timers)) {
1106                 struct cpu_timer_list *tl = list_first_entry(timers,
1107                                                       struct cpu_timer_list,
1108                                                       entry);
1109                 if (!--maxfire || sum_sched_runtime < tl->expires.sched) {
1110                         sched_expires = tl->expires.sched;
1111                         break;
1112                 }
1113                 tl->firing = 1;
1114                 list_move_tail(&tl->entry, firing);
1115         }
1116
1117         /*
1118          * Check for the special case process timers.
1119          */
1120         check_cpu_itimer(tsk, &sig->it[CPUCLOCK_PROF], &prof_expires, ptime,
1121                          SIGPROF);
1122         check_cpu_itimer(tsk, &sig->it[CPUCLOCK_VIRT], &virt_expires, utime,
1123                          SIGVTALRM);
1124         soft = ACCESS_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur);
1125         if (soft != RLIM_INFINITY) {
1126                 unsigned long psecs = cputime_to_secs(ptime);
1127                 unsigned long hard =
1128                         ACCESS_ONCE(sig->rlim[RLIMIT_CPU].rlim_max);
1129                 cputime_t x;
1130                 if (psecs >= hard) {
1131                         /*
1132                          * At the hard limit, we just die.
1133                          * No need to calculate anything else now.
1134                          */
1135                         __group_send_sig_info(SIGKILL, SEND_SIG_PRIV, tsk);
1136                         return;
1137                 }
1138                 if (psecs >= soft) {
1139                         /*
1140                          * At the soft limit, send a SIGXCPU every second.
1141                          */
1142                         __group_send_sig_info(SIGXCPU, SEND_SIG_PRIV, tsk);
1143                         if (soft < hard) {
1144                                 soft++;
1145                                 sig->rlim[RLIMIT_CPU].rlim_cur = soft;
1146                         }
1147                 }
1148                 x = secs_to_cputime(soft);
1149                 if (!prof_expires || x < prof_expires) {
1150                         prof_expires = x;
1151                 }
1152         }
1153
1154         sig->cputime_expires.prof_exp = prof_expires;
1155         sig->cputime_expires.virt_exp = virt_expires;
1156         sig->cputime_expires.sched_exp = sched_expires;
1157         if (task_cputime_zero(&sig->cputime_expires))
1158                 stop_process_timers(sig);
1159 }
1160
1161 /*
1162  * This is called from the signal code (via do_schedule_next_timer)
1163  * when the last timer signal was delivered and we have to reload the timer.
1164  */
1165 void posix_cpu_timer_schedule(struct k_itimer *timer)
1166 {
1167         struct task_struct *p = timer->it.cpu.task;
1168         union cpu_time_count now;
1169
1170         if (unlikely(p == NULL))
1171                 /*
1172                  * The task was cleaned up already, no future firings.
1173                  */
1174                 goto out;
1175
1176         /*
1177          * Fetch the current sample and update the timer's expiry time.
1178          */
1179         if (CPUCLOCK_PERTHREAD(timer->it_clock)) {
1180                 cpu_clock_sample(timer->it_clock, p, &now);
1181                 bump_cpu_timer(timer, now);
1182                 if (unlikely(p->exit_state)) {
1183                         clear_dead_task(timer, now);
1184                         goto out;
1185                 }
1186                 read_lock(&tasklist_lock); /* arm_timer needs it.  */
1187                 spin_lock(&p->sighand->siglock);
1188         } else {
1189                 read_lock(&tasklist_lock);
1190                 if (unlikely(p->sighand == NULL)) {
1191                         /*
1192                          * The process has been reaped.
1193                          * We can't even collect a sample any more.
1194                          */
1195                         put_task_struct(p);
1196                         timer->it.cpu.task = p = NULL;
1197                         timer->it.cpu.expires.sched = 0;
1198                         goto out_unlock;
1199                 } else if (unlikely(p->exit_state) && thread_group_empty(p)) {
1200                         /*
1201                          * We've noticed that the thread is dead, but
1202                          * not yet reaped.  Take this opportunity to
1203                          * drop our task ref.
1204                          */
1205                         clear_dead_task(timer, now);
1206                         goto out_unlock;
1207                 }
1208                 spin_lock(&p->sighand->siglock);
1209                 cpu_timer_sample_group(timer->it_clock, p, &now);
1210                 bump_cpu_timer(timer, now);
1211                 /* Leave the tasklist_lock locked for the call below.  */
1212         }
1213
1214         /*
1215          * Now re-arm for the new expiry time.
1216          */
1217         BUG_ON(!irqs_disabled());
1218         arm_timer(timer);
1219         spin_unlock(&p->sighand->siglock);
1220
1221 out_unlock:
1222         read_unlock(&tasklist_lock);
1223
1224 out:
1225         timer->it_overrun_last = timer->it_overrun;
1226         timer->it_overrun = -1;
1227         ++timer->it_requeue_pending;
1228 }
1229
1230 /**
1231  * task_cputime_expired - Compare two task_cputime entities.
1232  *
1233  * @sample:     The task_cputime structure to be checked for expiration.
1234  * @expires:    Expiration times, against which @sample will be checked.
1235  *
1236  * Checks @sample against @expires to see if any field of @sample has expired.
1237  * Returns true if any field of the former is greater than the corresponding
1238  * field of the latter if the latter field is set.  Otherwise returns false.
1239  */
1240 static inline int task_cputime_expired(const struct task_cputime *sample,
1241                                         const struct task_cputime *expires)
1242 {
1243         if (expires->utime && sample->utime >= expires->utime)
1244                 return 1;
1245         if (expires->stime && sample->utime + sample->stime >= expires->stime)
1246                 return 1;
1247         if (expires->sum_exec_runtime != 0 &&
1248             sample->sum_exec_runtime >= expires->sum_exec_runtime)
1249                 return 1;
1250         return 0;
1251 }
1252
1253 /**
1254  * fastpath_timer_check - POSIX CPU timers fast path.
1255  *
1256  * @tsk:        The task (thread) being checked.
1257  *
1258  * Check the task and thread group timers.  If both are zero (there are no
1259  * timers set) return false.  Otherwise snapshot the task and thread group
1260  * timers and compare them with the corresponding expiration times.  Return
1261  * true if a timer has expired, else return false.
1262  */
1263 static inline int fastpath_timer_check(struct task_struct *tsk)
1264 {
1265         struct signal_struct *sig;
1266         cputime_t utime, stime;
1267
1268         task_cputime(tsk, &utime, &stime);
1269
1270         if (!task_cputime_zero(&tsk->cputime_expires)) {
1271                 struct task_cputime task_sample = {
1272                         .utime = utime,
1273                         .stime = stime,
1274                         .sum_exec_runtime = tsk->se.sum_exec_runtime
1275                 };
1276
1277                 if (task_cputime_expired(&task_sample, &tsk->cputime_expires))
1278                         return 1;
1279         }
1280
1281         sig = tsk->signal;
1282         if (sig->cputimer.running) {
1283                 struct task_cputime group_sample;
1284
1285                 raw_spin_lock(&sig->cputimer.lock);
1286                 group_sample = sig->cputimer.cputime;
1287                 raw_spin_unlock(&sig->cputimer.lock);
1288
1289                 if (task_cputime_expired(&group_sample, &sig->cputime_expires))
1290                         return 1;
1291         }
1292
1293         return 0;
1294 }
1295
1296 /*
1297  * This is called from the timer interrupt handler.  The irq handler has
1298  * already updated our counts.  We need to check if any timers fire now.
1299  * Interrupts are disabled.
1300  */
1301 void run_posix_cpu_timers(struct task_struct *tsk)
1302 {
1303         LIST_HEAD(firing);
1304         struct k_itimer *timer, *next;
1305         unsigned long flags;
1306
1307         BUG_ON(!irqs_disabled());
1308
1309         /*
1310          * The fast path checks that there are no expired thread or thread
1311          * group timers.  If that's so, just return.
1312          */
1313         if (!fastpath_timer_check(tsk))
1314                 return;
1315
1316         if (!lock_task_sighand(tsk, &flags))
1317                 return;
1318         /*
1319          * Here we take off tsk->signal->cpu_timers[N] and
1320          * tsk->cpu_timers[N] all the timers that are firing, and
1321          * put them on the firing list.
1322          */
1323         check_thread_timers(tsk, &firing);
1324         /*
1325          * If there are any active process wide timers (POSIX 1.b, itimers,
1326          * RLIMIT_CPU) cputimer must be running.
1327          */
1328         if (tsk->signal->cputimer.running)
1329                 check_process_timers(tsk, &firing);
1330
1331         /*
1332          * We must release these locks before taking any timer's lock.
1333          * There is a potential race with timer deletion here, as the
1334          * siglock now protects our private firing list.  We have set
1335          * the firing flag in each timer, so that a deletion attempt
1336          * that gets the timer lock before we do will give it up and
1337          * spin until we've taken care of that timer below.
1338          */
1339         unlock_task_sighand(tsk, &flags);
1340
1341         /*
1342          * Now that all the timers on our list have the firing flag,
1343          * no one will touch their list entries but us.  We'll take
1344          * each timer's lock before clearing its firing flag, so no
1345          * timer call will interfere.
1346          */
1347         list_for_each_entry_safe(timer, next, &firing, it.cpu.entry) {
1348                 int cpu_firing;
1349
1350                 spin_lock(&timer->it_lock);
1351                 list_del_init(&timer->it.cpu.entry);
1352                 cpu_firing = timer->it.cpu.firing;
1353                 timer->it.cpu.firing = 0;
1354                 /*
1355                  * The firing flag is -1 if we collided with a reset
1356                  * of the timer, which already reported this
1357                  * almost-firing as an overrun.  So don't generate an event.
1358                  */
1359                 if (likely(cpu_firing >= 0))
1360                         cpu_timer_fire(timer);
1361                 spin_unlock(&timer->it_lock);
1362         }
1363
1364         /*
1365          * In case some timers were rescheduled after the queue got emptied,
1366          * wake up full dynticks CPUs.
1367          */
1368         if (tsk->signal->cputimer.running)
1369                 posix_cpu_timer_kick_nohz();
1370 }
1371
1372 /*
1373  * Set one of the process-wide special case CPU timers or RLIMIT_CPU.
1374  * The tsk->sighand->siglock must be held by the caller.
1375  */
1376 void set_process_cpu_timer(struct task_struct *tsk, unsigned int clock_idx,
1377                            cputime_t *newval, cputime_t *oldval)
1378 {
1379         union cpu_time_count now;
1380
1381         BUG_ON(clock_idx == CPUCLOCK_SCHED);
1382         cpu_timer_sample_group(clock_idx, tsk, &now);
1383
1384         if (oldval) {
1385                 /*
1386                  * We are setting itimer. The *oldval is absolute and we update
1387                  * it to be relative, *newval argument is relative and we update
1388                  * it to be absolute.
1389                  */
1390                 if (*oldval) {
1391                         if (*oldval <= now.cpu) {
1392                                 /* Just about to fire. */
1393                                 *oldval = cputime_one_jiffy;
1394                         } else {
1395                                 *oldval -= now.cpu;
1396                         }
1397                 }
1398
1399                 if (!*newval)
1400                         goto out;
1401                 *newval += now.cpu;
1402         }
1403
1404         /*
1405          * Update expiration cache if we are the earliest timer, or eventually
1406          * RLIMIT_CPU limit is earlier than prof_exp cpu timer expire.
1407          */
1408         switch (clock_idx) {
1409         case CPUCLOCK_PROF:
1410                 if (expires_gt(tsk->signal->cputime_expires.prof_exp, *newval))
1411                         tsk->signal->cputime_expires.prof_exp = *newval;
1412                 break;
1413         case CPUCLOCK_VIRT:
1414                 if (expires_gt(tsk->signal->cputime_expires.virt_exp, *newval))
1415                         tsk->signal->cputime_expires.virt_exp = *newval;
1416                 break;
1417         }
1418 out:
1419         posix_cpu_timer_kick_nohz();
1420 }
1421
1422 static int do_cpu_nanosleep(const clockid_t which_clock, int flags,
1423                             struct timespec *rqtp, struct itimerspec *it)
1424 {
1425         struct k_itimer timer;
1426         int error;
1427
1428         /*
1429          * Set up a temporary timer and then wait for it to go off.
1430          */
1431         memset(&timer, 0, sizeof timer);
1432         spin_lock_init(&timer.it_lock);
1433         timer.it_clock = which_clock;
1434         timer.it_overrun = -1;
1435         error = posix_cpu_timer_create(&timer);
1436         timer.it_process = current;
1437         if (!error) {
1438                 static struct itimerspec zero_it;
1439
1440                 memset(it, 0, sizeof *it);
1441                 it->it_value = *rqtp;
1442
1443                 spin_lock_irq(&timer.it_lock);
1444                 error = posix_cpu_timer_set(&timer, flags, it, NULL);
1445                 if (error) {
1446                         spin_unlock_irq(&timer.it_lock);
1447                         return error;
1448                 }
1449
1450                 while (!signal_pending(current)) {
1451                         if (timer.it.cpu.expires.sched == 0) {
1452                                 /*
1453                                  * Our timer fired and was reset, below
1454                                  * deletion can not fail.
1455                                  */
1456                                 posix_cpu_timer_del(&timer);
1457                                 spin_unlock_irq(&timer.it_lock);
1458                                 return 0;
1459                         }
1460
1461                         /*
1462                          * Block until cpu_timer_fire (or a signal) wakes us.
1463                          */
1464                         __set_current_state(TASK_INTERRUPTIBLE);
1465                         spin_unlock_irq(&timer.it_lock);
1466                         schedule();
1467                         spin_lock_irq(&timer.it_lock);
1468                 }
1469
1470                 /*
1471                  * We were interrupted by a signal.
1472                  */
1473                 sample_to_timespec(which_clock, timer.it.cpu.expires, rqtp);
1474                 error = posix_cpu_timer_set(&timer, 0, &zero_it, it);
1475                 if (!error) {
1476                         /*
1477                          * Timer is now unarmed, deletion can not fail.
1478                          */
1479                         posix_cpu_timer_del(&timer);
1480                 }
1481                 spin_unlock_irq(&timer.it_lock);
1482
1483                 while (error == TIMER_RETRY) {
1484                         /*
1485                          * We need to handle case when timer was or is in the
1486                          * middle of firing. In other cases we already freed
1487                          * resources.
1488                          */
1489                         spin_lock_irq(&timer.it_lock);
1490                         error = posix_cpu_timer_del(&timer);
1491                         spin_unlock_irq(&timer.it_lock);
1492                 }
1493
1494                 if ((it->it_value.tv_sec | it->it_value.tv_nsec) == 0) {
1495                         /*
1496                          * It actually did fire already.
1497                          */
1498                         return 0;
1499                 }
1500
1501                 error = -ERESTART_RESTARTBLOCK;
1502         }
1503
1504         return error;
1505 }
1506
1507 static long posix_cpu_nsleep_restart(struct restart_block *restart_block);
1508
1509 static int posix_cpu_nsleep(const clockid_t which_clock, int flags,
1510                             struct timespec *rqtp, struct timespec __user *rmtp)
1511 {
1512         struct restart_block *restart_block =
1513                 &current_thread_info()->restart_block;
1514         struct itimerspec it;
1515         int error;
1516
1517         /*
1518          * Diagnose required errors first.
1519          */
1520         if (CPUCLOCK_PERTHREAD(which_clock) &&
1521             (CPUCLOCK_PID(which_clock) == 0 ||
1522              CPUCLOCK_PID(which_clock) == current->pid))
1523                 return -EINVAL;
1524
1525         error = do_cpu_nanosleep(which_clock, flags, rqtp, &it);
1526
1527         if (error == -ERESTART_RESTARTBLOCK) {
1528
1529                 if (flags & TIMER_ABSTIME)
1530                         return -ERESTARTNOHAND;
1531                 /*
1532                  * Report back to the user the time still remaining.
1533                  */
1534                 if (rmtp && copy_to_user(rmtp, &it.it_value, sizeof *rmtp))
1535                         return -EFAULT;
1536
1537                 restart_block->fn = posix_cpu_nsleep_restart;
1538                 restart_block->nanosleep.clockid = which_clock;
1539                 restart_block->nanosleep.rmtp = rmtp;
1540                 restart_block->nanosleep.expires = timespec_to_ns(rqtp);
1541         }
1542         return error;
1543 }
1544
1545 static long posix_cpu_nsleep_restart(struct restart_block *restart_block)
1546 {
1547         clockid_t which_clock = restart_block->nanosleep.clockid;
1548         struct timespec t;
1549         struct itimerspec it;
1550         int error;
1551
1552         t = ns_to_timespec(restart_block->nanosleep.expires);
1553
1554         error = do_cpu_nanosleep(which_clock, TIMER_ABSTIME, &t, &it);
1555
1556         if (error == -ERESTART_RESTARTBLOCK) {
1557                 struct timespec __user *rmtp = restart_block->nanosleep.rmtp;
1558                 /*
1559                  * Report back to the user the time still remaining.
1560                  */
1561                 if (rmtp && copy_to_user(rmtp, &it.it_value, sizeof *rmtp))
1562                         return -EFAULT;
1563
1564                 restart_block->nanosleep.expires = timespec_to_ns(&t);
1565         }
1566         return error;
1567
1568 }
1569
1570 #define PROCESS_CLOCK   MAKE_PROCESS_CPUCLOCK(0, CPUCLOCK_SCHED)
1571 #define THREAD_CLOCK    MAKE_THREAD_CPUCLOCK(0, CPUCLOCK_SCHED)
1572
1573 static int process_cpu_clock_getres(const clockid_t which_clock,
1574                                     struct timespec *tp)
1575 {
1576         return posix_cpu_clock_getres(PROCESS_CLOCK, tp);
1577 }
1578 static int process_cpu_clock_get(const clockid_t which_clock,
1579                                  struct timespec *tp)
1580 {
1581         return posix_cpu_clock_get(PROCESS_CLOCK, tp);
1582 }
1583 static int process_cpu_timer_create(struct k_itimer *timer)
1584 {
1585         timer->it_clock = PROCESS_CLOCK;
1586         return posix_cpu_timer_create(timer);
1587 }
1588 static int process_cpu_nsleep(const clockid_t which_clock, int flags,
1589                               struct timespec *rqtp,
1590                               struct timespec __user *rmtp)
1591 {
1592         return posix_cpu_nsleep(PROCESS_CLOCK, flags, rqtp, rmtp);
1593 }
1594 static long process_cpu_nsleep_restart(struct restart_block *restart_block)
1595 {
1596         return -EINVAL;
1597 }
1598 static int thread_cpu_clock_getres(const clockid_t which_clock,
1599                                    struct timespec *tp)
1600 {
1601         return posix_cpu_clock_getres(THREAD_CLOCK, tp);
1602 }
1603 static int thread_cpu_clock_get(const clockid_t which_clock,
1604                                 struct timespec *tp)
1605 {
1606         return posix_cpu_clock_get(THREAD_CLOCK, tp);
1607 }
1608 static int thread_cpu_timer_create(struct k_itimer *timer)
1609 {
1610         timer->it_clock = THREAD_CLOCK;
1611         return posix_cpu_timer_create(timer);
1612 }
1613
1614 struct k_clock clock_posix_cpu = {
1615         .clock_getres   = posix_cpu_clock_getres,
1616         .clock_set      = posix_cpu_clock_set,
1617         .clock_get      = posix_cpu_clock_get,
1618         .timer_create   = posix_cpu_timer_create,
1619         .nsleep         = posix_cpu_nsleep,
1620         .nsleep_restart = posix_cpu_nsleep_restart,
1621         .timer_set      = posix_cpu_timer_set,
1622         .timer_del      = posix_cpu_timer_del,
1623         .timer_get      = posix_cpu_timer_get,
1624 };
1625
1626 static __init int init_posix_cpu_timers(void)
1627 {
1628         struct k_clock process = {
1629                 .clock_getres   = process_cpu_clock_getres,
1630                 .clock_get      = process_cpu_clock_get,
1631                 .timer_create   = process_cpu_timer_create,
1632                 .nsleep         = process_cpu_nsleep,
1633                 .nsleep_restart = process_cpu_nsleep_restart,
1634         };
1635         struct k_clock thread = {
1636                 .clock_getres   = thread_cpu_clock_getres,
1637                 .clock_get      = thread_cpu_clock_get,
1638                 .timer_create   = thread_cpu_timer_create,
1639         };
1640         struct timespec ts;
1641
1642         posix_timers_register_clock(CLOCK_PROCESS_CPUTIME_ID, &process);
1643         posix_timers_register_clock(CLOCK_THREAD_CPUTIME_ID, &thread);
1644
1645         cputime_to_timespec(cputime_one_jiffy, &ts);
1646         onecputick = ts.tv_nsec;
1647         WARN_ON(ts.tv_sec != 0);
1648
1649         return 0;
1650 }
1651 __initcall(init_posix_cpu_timers);