Pileus Git - ~andy/linux/blob - kernel/softirq.c

   1 /*
   2  *      linux/kernel/softirq.c
   3  *
   4  *      Copyright (C) 1992 Linus Torvalds
   5  *
   6  *      Distribute under GPLv2.
   7  *
   8  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   9  *
  10  *      Remote softirq infrastructure is by Jens Axboe.
  11  */
  12
  13 #include <linux/export.h>
  14 #include <linux/kernel_stat.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/init.h>
  17 #include <linux/mm.h>
  18 #include <linux/notifier.h>
  19 #include <linux/percpu.h>
  20 #include <linux/cpu.h>
  21 #include <linux/freezer.h>
  22 #include <linux/kthread.h>
  23 #include <linux/rcupdate.h>
  24 #include <linux/ftrace.h>
  25 #include <linux/smp.h>
  26 #include <linux/smpboot.h>
  27 #include <linux/tick.h>
  28
  29 #define CREATE_TRACE_POINTS
  30 #include <trace/events/irq.h>
  31
  32 #include <asm/irq.h>
  33 /*
  34    - No shared variables, all the data are CPU local.
  35    - If a softirq needs serialization, let it serialize itself
  36      by its own spinlocks.
  37    - Even if softirq is serialized, only local cpu is marked for
  38      execution. Hence, we get something sort of weak cpu binding.
  39      Though it is still not clear, will it result in better locality
  40      or will not.
  41
  42    Examples:
  43    - NET RX softirq. It is multithreaded and does not require
  44      any global serialization.
  45    - NET TX softirq. It kicks software netdevice queues, hence
  46      it is logically serialized per device, but this serialization
  47      is invisible to common code.
  48    - Tasklets: serialized wrt itself.
  49  */
  50
  51 #ifndef __ARCH_IRQ_STAT
  52 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  53 EXPORT_SYMBOL(irq_stat);
  54 #endif
  55
  56 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
  57
  58 DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  59
  60 char *softirq_to_name[NR_SOFTIRQS] = {
  61         "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
  62         "TASKLET", "SCHED", "HRTIMER", "RCU"
  63 };
  64
  65 /*
  66  * we cannot loop indefinitely here to avoid userspace starvation,
  67  * but we also don't want to introduce a worst case 1/HZ latency
  68  * to the pending events, so lets the scheduler to balance
  69  * the softirq load for us.
  70  */
  71 static void wakeup_softirqd(void)
  72 {
  73         /* Interrupts are disabled: no need to stop preemption */
  74         struct task_struct *tsk = __this_cpu_read(ksoftirqd);
  75
  76         if (tsk && tsk->state != TASK_RUNNING)
  77                 wake_up_process(tsk);
  78 }
  79
  80 /*
  81  * preempt_count and SOFTIRQ_OFFSET usage:
  82  * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
  83  *   softirq processing.
  84  * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
  85  *   on local_bh_disable or local_bh_enable.
  86  * This lets us distinguish between whether we are currently processing
  87  * softirq and whether we just have bh disabled.
  88  */
  89
  90 /*
  91  * This one is for softirq.c-internal use,
  92  * where hardirqs are disabled legitimately:
  93  */
  94 #ifdef CONFIG_TRACE_IRQFLAGS
  95 static void __local_bh_disable(unsigned long ip, unsigned int cnt)
  96 {
  97         unsigned long flags;
  98
  99         WARN_ON_ONCE(in_irq());
 100
 101         raw_local_irq_save(flags);
 102         /*
 103          * The preempt tracer hooks into add_preempt_count and will break
 104          * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
 105          * is set and before current->softirq_enabled is cleared.
 106          * We must manually increment preempt_count here and manually
 107          * call the trace_preempt_off later.
 108          */
 109         preempt_count() += cnt;
 110         /*
 111          * Were softirqs turned off above:
 112          */
 113         if (softirq_count() == cnt)
 114                 trace_softirqs_off(ip);
 115         raw_local_irq_restore(flags);
 116
 117         if (preempt_count() == cnt)
 118                 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
 119 }
 120 #else /* !CONFIG_TRACE_IRQFLAGS */
 121 static inline void __local_bh_disable(unsigned long ip, unsigned int cnt)
 122 {
 123         add_preempt_count(cnt);
 124         barrier();
 125 }
 126 #endif /* CONFIG_TRACE_IRQFLAGS */
 127
 128 void local_bh_disable(void)
 129 {
 130         __local_bh_disable(_RET_IP_, SOFTIRQ_DISABLE_OFFSET);
 131 }
 132
 133 EXPORT_SYMBOL(local_bh_disable);
 134
 135 static void __local_bh_enable(unsigned int cnt)
 136 {
 137         WARN_ON_ONCE(in_irq());
 138         WARN_ON_ONCE(!irqs_disabled());
 139
 140         if (softirq_count() == cnt)
 141                 trace_softirqs_on(_RET_IP_);
 142         sub_preempt_count(cnt);
 143 }
 144
 145 /*
 146  * Special-case - softirqs can safely be enabled in
 147  * cond_resched_softirq(), or by __do_softirq(),
 148  * without processing still-pending softirqs:
 149  */
 150 void _local_bh_enable(void)
 151 {
 152         __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
 153 }
 154
 155 EXPORT_SYMBOL(_local_bh_enable);
 156
 157 static inline void _local_bh_enable_ip(unsigned long ip)
 158 {
 159         WARN_ON_ONCE(in_irq() || irqs_disabled());
 160 #ifdef CONFIG_TRACE_IRQFLAGS
 161         local_irq_disable();
 162 #endif
 163         /*
 164          * Are softirqs going to be turned on now:
 165          */
 166         if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
 167                 trace_softirqs_on(ip);
 168         /*
 169          * Keep preemption disabled until we are done with
 170          * softirq processing:
 171          */
 172         sub_preempt_count(SOFTIRQ_DISABLE_OFFSET - 1);
 173
 174         if (unlikely(!in_interrupt() && local_softirq_pending()))
 175                 do_softirq();
 176
 177         dec_preempt_count();
 178 #ifdef CONFIG_TRACE_IRQFLAGS
 179         local_irq_enable();
 180 #endif
 181         preempt_check_resched();
 182 }
 183
 184 void local_bh_enable(void)
 185 {
 186         _local_bh_enable_ip(_RET_IP_);
 187 }
 188 EXPORT_SYMBOL(local_bh_enable);
 189
 190 void local_bh_enable_ip(unsigned long ip)
 191 {
 192         _local_bh_enable_ip(ip);
 193 }
 194 EXPORT_SYMBOL(local_bh_enable_ip);
 195
 196 /*
 197  * We restart softirq processing for at most 2 ms,
 198  * and if need_resched() is not set.
 199  *
 200  * These limits have been established via experimentation.
 201  * The two things to balance is latency against fairness -
 202  * we want to handle softirqs as soon as possible, but they
 203  * should not be able to lock up the box.
 204  */
 205 #define MAX_SOFTIRQ_TIME  msecs_to_jiffies(2)
 206
 207 asmlinkage void __do_softirq(void)
 208 {
 209         struct softirq_action *h;
 210         __u32 pending;
 211         unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
 212         int cpu;
 213         unsigned long old_flags = current->flags;
 214
 215         /*
 216          * Mask out PF_MEMALLOC s current task context is borrowed for the
 217          * softirq. A softirq handled such as network RX might set PF_MEMALLOC
 218          * again if the socket is related to swap
 219          */
 220         current->flags &= ~PF_MEMALLOC;
 221
 222         pending = local_softirq_pending();
 223         account_irq_enter_time(current);
 224
 225         __local_bh_disable(_RET_IP_, SOFTIRQ_OFFSET);
 226         lockdep_softirq_enter();
 227
 228         cpu = smp_processor_id();
 229 restart:
 230         /* Reset the pending bitmask before enabling irqs */
 231         set_softirq_pending(0);
 232
 233         local_irq_enable();
 234
 235         h = softirq_vec;
 236
 237         do {
 238                 if (pending & 1) {
 239                         unsigned int vec_nr = h - softirq_vec;
 240                         int prev_count = preempt_count();
 241
 242                         kstat_incr_softirqs_this_cpu(vec_nr);
 243
 244                         trace_softirq_entry(vec_nr);
 245                         h->action(h);
 246                         trace_softirq_exit(vec_nr);
 247                         if (unlikely(prev_count != preempt_count())) {
 248                                 printk(KERN_ERR "huh, entered softirq %u %s %p"
 249                                        "with preempt_count %08x,"
 250                                        " exited with %08x?\n", vec_nr,
 251                                        softirq_to_name[vec_nr], h->action,
 252                                        prev_count, preempt_count());
 253                                 preempt_count() = prev_count;
 254                         }
 255
 256                         rcu_bh_qs(cpu);
 257                 }
 258                 h++;
 259                 pending >>= 1;
 260         } while (pending);
 261
 262         local_irq_disable();
 263
 264         pending = local_softirq_pending();
 265         if (pending) {
 266                 if (time_before(jiffies, end) && !need_resched())
 267                         goto restart;
 268
 269                 wakeup_softirqd();
 270         }
 271
 272         lockdep_softirq_exit();
 273
 274         account_irq_exit_time(current);
 275         __local_bh_enable(SOFTIRQ_OFFSET);
 276         tsk_restore_flags(current, old_flags, PF_MEMALLOC);
 277 }
 278
 279 #ifndef __ARCH_HAS_DO_SOFTIRQ
 280
 281 asmlinkage void do_softirq(void)
 282 {
 283         __u32 pending;
 284         unsigned long flags;
 285
 286         if (in_interrupt())
 287                 return;
 288
 289         local_irq_save(flags);
 290
 291         pending = local_softirq_pending();
 292
 293         if (pending)
 294                 __do_softirq();
 295
 296         local_irq_restore(flags);
 297 }
 298
 299 #endif
 300
 301 /*
 302  * Enter an interrupt context.
 303  */
 304 void irq_enter(void)
 305 {
 306         int cpu = smp_processor_id();
 307
 308         rcu_irq_enter();
 309         if (is_idle_task(current) && !in_interrupt()) {
 310                 /*
 311                  * Prevent raise_softirq from needlessly waking up ksoftirqd
 312                  * here, as softirq will be serviced on return from interrupt.
 313                  */
 314                 local_bh_disable();
 315                 tick_check_idle(cpu);
 316                 _local_bh_enable();
 317         }
 318
 319         __irq_enter();
 320 }
 321
 322 static inline void invoke_softirq(void)
 323 {
 324         if (!force_irqthreads)
 325                 __do_softirq();
 326         else
 327                 wakeup_softirqd();
 328 }
 329
 330 static inline void tick_irq_exit(void)
 331 {
 332 #ifdef CONFIG_NO_HZ_COMMON
 333         int cpu = smp_processor_id();
 334
 335         /* Make sure that timer wheel updates are propagated */
 336         if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
 337                 if (!in_interrupt())
 338                         tick_nohz_irq_exit();
 339         }
 340 #endif
 341 }
 342
 343 /*
 344  * Exit an interrupt context. Process softirqs if needed and possible:
 345  */
 346 void irq_exit(void)
 347 {
 348 #ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
 349         local_irq_disable();
 350 #else
 351         WARN_ON_ONCE(!irqs_disabled());
 352 #endif
 353
 354         account_irq_exit_time(current);
 355         trace_hardirq_exit();
 356         sub_preempt_count(HARDIRQ_OFFSET);
 357         if (!in_interrupt() && local_softirq_pending())
 358                 invoke_softirq();
 359
 360         tick_irq_exit();
 361         rcu_irq_exit();
 362 }
 363
 364 /*
 365  * This function must run with irqs disabled!
 366  */
 367 inline void raise_softirq_irqoff(unsigned int nr)
 368 {
 369         __raise_softirq_irqoff(nr);
 370
 371         /*
 372          * If we're in an interrupt or softirq, we're done
 373          * (this also catches softirq-disabled code). We will
 374          * actually run the softirq once we return from
 375          * the irq or softirq.
 376          *
 377          * Otherwise we wake up ksoftirqd to make sure we
 378          * schedule the softirq soon.
 379          */
 380         if (!in_interrupt())
 381                 wakeup_softirqd();
 382 }
 383
 384 void raise_softirq(unsigned int nr)
 385 {
 386         unsigned long flags;
 387
 388         local_irq_save(flags);
 389         raise_softirq_irqoff(nr);
 390         local_irq_restore(flags);
 391 }
 392
 393 void __raise_softirq_irqoff(unsigned int nr)
 394 {
 395         trace_softirq_raise(nr);
 396         or_softirq_pending(1UL << nr);
 397 }
 398
 399 void open_softirq(int nr, void (*action)(struct softirq_action *))
 400 {
 401         softirq_vec[nr].action = action;
 402 }
 403
 404 /*
 405  * Tasklets
 406  */
 407 struct tasklet_head
 408 {
 409         struct tasklet_struct *head;
 410         struct tasklet_struct **tail;
 411 };
 412
 413 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
 414 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 415
 416 void __tasklet_schedule(struct tasklet_struct *t)
 417 {
 418         unsigned long flags;
 419
 420         local_irq_save(flags);
 421         t->next = NULL;
 422         *__this_cpu_read(tasklet_vec.tail) = t;
 423         __this_cpu_write(tasklet_vec.tail, &(t->next));
 424         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 425         local_irq_restore(flags);
 426 }
 427
 428 EXPORT_SYMBOL(__tasklet_schedule);
 429
 430 void __tasklet_hi_schedule(struct tasklet_struct *t)
 431 {
 432         unsigned long flags;
 433
 434         local_irq_save(flags);
 435         t->next = NULL;
 436         *__this_cpu_read(tasklet_hi_vec.tail) = t;
 437         __this_cpu_write(tasklet_hi_vec.tail,  &(t->next));
 438         raise_softirq_irqoff(HI_SOFTIRQ);
 439         local_irq_restore(flags);
 440 }
 441
 442 EXPORT_SYMBOL(__tasklet_hi_schedule);
 443
 444 void __tasklet_hi_schedule_first(struct tasklet_struct *t)
 445 {
 446         BUG_ON(!irqs_disabled());
 447
 448         t->next = __this_cpu_read(tasklet_hi_vec.head);
 449         __this_cpu_write(tasklet_hi_vec.head, t);
 450         __raise_softirq_irqoff(HI_SOFTIRQ);
 451 }
 452
 453 EXPORT_SYMBOL(__tasklet_hi_schedule_first);
 454
 455 static void tasklet_action(struct softirq_action *a)
 456 {
 457         struct tasklet_struct *list;
 458
 459         local_irq_disable();
 460         list = __this_cpu_read(tasklet_vec.head);
 461         __this_cpu_write(tasklet_vec.head, NULL);
 462         __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
 463         local_irq_enable();
 464
 465         while (list) {
 466                 struct tasklet_struct *t = list;
 467
 468                 list = list->next;
 469
 470                 if (tasklet_trylock(t)) {
 471                         if (!atomic_read(&t->count)) {
 472                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 473                                         BUG();
 474                                 t->func(t->data);
 475                                 tasklet_unlock(t);
 476                                 continue;
 477                         }
 478                         tasklet_unlock(t);
 479                 }
 480
 481                 local_irq_disable();
 482                 t->next = NULL;
 483                 *__this_cpu_read(tasklet_vec.tail) = t;
 484                 __this_cpu_write(tasklet_vec.tail, &(t->next));
 485                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 486                 local_irq_enable();
 487         }
 488 }
 489
 490 static void tasklet_hi_action(struct softirq_action *a)
 491 {
 492         struct tasklet_struct *list;
 493
 494         local_irq_disable();
 495         list = __this_cpu_read(tasklet_hi_vec.head);
 496         __this_cpu_write(tasklet_hi_vec.head, NULL);
 497         __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
 498         local_irq_enable();
 499
 500         while (list) {
 501                 struct tasklet_struct *t = list;
 502
 503                 list = list->next;
 504
 505                 if (tasklet_trylock(t)) {
 506                         if (!atomic_read(&t->count)) {
 507                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 508                                         BUG();
 509                                 t->func(t->data);
 510                                 tasklet_unlock(t);
 511                                 continue;
 512                         }
 513                         tasklet_unlock(t);
 514                 }
 515
 516                 local_irq_disable();
 517                 t->next = NULL;
 518                 *__this_cpu_read(tasklet_hi_vec.tail) = t;
 519                 __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
 520                 __raise_softirq_irqoff(HI_SOFTIRQ);
 521                 local_irq_enable();
 522         }
 523 }
 524
 525
 526 void tasklet_init(struct tasklet_struct *t,
 527                   void (*func)(unsigned long), unsigned long data)
 528 {
 529         t->next = NULL;
 530         t->state = 0;
 531         atomic_set(&t->count, 0);
 532         t->func = func;
 533         t->data = data;
 534 }
 535
 536 EXPORT_SYMBOL(tasklet_init);
 537
 538 void tasklet_kill(struct tasklet_struct *t)
 539 {
 540         if (in_interrupt())
 541                 printk("Attempt to kill tasklet from interrupt\n");
 542
 543         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 544                 do {
 545                         yield();
 546                 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
 547         }
 548         tasklet_unlock_wait(t);
 549         clear_bit(TASKLET_STATE_SCHED, &t->state);
 550 }
 551
 552 EXPORT_SYMBOL(tasklet_kill);
 553
 554 /*
 555  * tasklet_hrtimer
 556  */
 557
 558 /*
 559  * The trampoline is called when the hrtimer expires. It schedules a tasklet
 560  * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
 561  * hrtimer callback, but from softirq context.
 562  */
 563 static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
 564 {
 565         struct tasklet_hrtimer *ttimer =
 566                 container_of(timer, struct tasklet_hrtimer, timer);
 567
 568         tasklet_hi_schedule(&ttimer->tasklet);
 569         return HRTIMER_NORESTART;
 570 }
 571
 572 /*
 573  * Helper function which calls the hrtimer callback from
 574  * tasklet/softirq context
 575  */
 576 static void __tasklet_hrtimer_trampoline(unsigned long data)
 577 {
 578         struct tasklet_hrtimer *ttimer = (void *)data;
 579         enum hrtimer_restart restart;
 580
 581         restart = ttimer->function(&ttimer->timer);
 582         if (restart != HRTIMER_NORESTART)
 583                 hrtimer_restart(&ttimer->timer);
 584 }
 585
 586 /**
 587  * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
 588  * @ttimer:      tasklet_hrtimer which is initialized
 589  * @function:    hrtimer callback function which gets called from softirq context
 590  * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
 591  * @mode:        hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
 592  */
 593 void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
 594                           enum hrtimer_restart (*function)(struct hrtimer *),
 595                           clockid_t which_clock, enum hrtimer_mode mode)
 596 {
 597         hrtimer_init(&ttimer->timer, which_clock, mode);
 598         ttimer->timer.function = __hrtimer_tasklet_trampoline;
 599         tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
 600                      (unsigned long)ttimer);
 601         ttimer->function = function;
 602 }
 603 EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
 604
 605 /*
 606  * Remote softirq bits
 607  */
 608
 609 DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
 610 EXPORT_PER_CPU_SYMBOL(softirq_work_list);
 611
 612 static void __local_trigger(struct call_single_data *cp, int softirq)
 613 {
 614         struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
 615
 616         list_add_tail(&cp->list, head);
 617
 618         /* Trigger the softirq only if the list was previously empty.  */
 619         if (head->next == &cp->list)
 620                 raise_softirq_irqoff(softirq);
 621 }
 622
 623 #ifdef CONFIG_USE_GENERIC_SMP_HELPERS
 624 static void remote_softirq_receive(void *data)
 625 {
 626         struct call_single_data *cp = data;
 627         unsigned long flags;
 628         int softirq;
 629
 630         softirq = *(int *)cp->info;
 631         local_irq_save(flags);
 632         __local_trigger(cp, softirq);
 633         local_irq_restore(flags);
 634 }
 635
 636 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 637 {
 638         if (cpu_online(cpu)) {
 639                 cp->func = remote_softirq_receive;
 640                 cp->info = &softirq;
 641                 cp->flags = 0;
 642
 643                 __smp_call_function_single(cpu, cp, 0);
 644                 return 0;
 645         }
 646         return 1;
 647 }
 648 #else /* CONFIG_USE_GENERIC_SMP_HELPERS */
 649 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 650 {
 651         return 1;
 652 }
 653 #endif
 654
 655 /**
 656  * __send_remote_softirq - try to schedule softirq work on a remote cpu
 657  * @cp: private SMP call function data area
 658  * @cpu: the remote cpu
 659  * @this_cpu: the currently executing cpu
 660  * @softirq: the softirq for the work
 661  *
 662  * Attempt to schedule softirq work on a remote cpu.  If this cannot be
 663  * done, the work is instead queued up on the local cpu.
 664  *
 665  * Interrupts must be disabled.
 666  */
 667 void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
 668 {
 669         if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
 670                 __local_trigger(cp, softirq);
 671 }
 672 EXPORT_SYMBOL(__send_remote_softirq);
 673
 674 /**
 675  * send_remote_softirq - try to schedule softirq work on a remote cpu
 676  * @cp: private SMP call function data area
 677  * @cpu: the remote cpu
 678  * @softirq: the softirq for the work
 679  *
 680  * Like __send_remote_softirq except that disabling interrupts and
 681  * computing the current cpu is done for the caller.
 682  */
 683 void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 684 {
 685         unsigned long flags;
 686         int this_cpu;
 687
 688         local_irq_save(flags);
 689         this_cpu = smp_processor_id();
 690         __send_remote_softirq(cp, cpu, this_cpu, softirq);
 691         local_irq_restore(flags);
 692 }
 693 EXPORT_SYMBOL(send_remote_softirq);
 694
 695 static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
 696                                                unsigned long action, void *hcpu)
 697 {
 698         /*
 699          * If a CPU goes away, splice its entries to the current CPU
 700          * and trigger a run of the softirq
 701          */
 702         if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
 703                 int cpu = (unsigned long) hcpu;
 704                 int i;
 705
 706                 local_irq_disable();
 707                 for (i = 0; i < NR_SOFTIRQS; i++) {
 708                         struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
 709                         struct list_head *local_head;
 710
 711                         if (list_empty(head))
 712                                 continue;
 713
 714                         local_head = &__get_cpu_var(softirq_work_list[i]);
 715                         list_splice_init(head, local_head);
 716                         raise_softirq_irqoff(i);
 717                 }
 718                 local_irq_enable();
 719         }
 720
 721         return NOTIFY_OK;
 722 }
 723
 724 static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
 725         .notifier_call  = remote_softirq_cpu_notify,
 726 };
 727
 728 void __init softirq_init(void)
 729 {
 730         int cpu;
 731
 732         for_each_possible_cpu(cpu) {
 733                 int i;
 734
 735                 per_cpu(tasklet_vec, cpu).tail =
 736                         &per_cpu(tasklet_vec, cpu).head;
 737                 per_cpu(tasklet_hi_vec, cpu).tail =
 738                         &per_cpu(tasklet_hi_vec, cpu).head;
 739                 for (i = 0; i < NR_SOFTIRQS; i++)
 740                         INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
 741         }
 742
 743         register_hotcpu_notifier(&remote_softirq_cpu_notifier);
 744
 745         open_softirq(TASKLET_SOFTIRQ, tasklet_action);
 746         open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 747 }
 748
 749 static int ksoftirqd_should_run(unsigned int cpu)
 750 {
 751         return local_softirq_pending();
 752 }
 753
 754 static void run_ksoftirqd(unsigned int cpu)
 755 {
 756         local_irq_disable();
 757         if (local_softirq_pending()) {
 758                 __do_softirq();
 759                 rcu_note_context_switch(cpu);
 760                 local_irq_enable();
 761                 cond_resched();
 762                 return;
 763         }
 764         local_irq_enable();
 765 }
 766
 767 #ifdef CONFIG_HOTPLUG_CPU
 768 /*
 769  * tasklet_kill_immediate is called to remove a tasklet which can already be
 770  * scheduled for execution on @cpu.
 771  *
 772  * Unlike tasklet_kill, this function removes the tasklet
 773  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 774  *
 775  * When this function is called, @cpu must be in the CPU_DEAD state.
 776  */
 777 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 778 {
 779         struct tasklet_struct **i;
 780
 781         BUG_ON(cpu_online(cpu));
 782         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 783
 784         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 785                 return;
 786
 787         /* CPU is dead, so no lock needed. */
 788         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
 789                 if (*i == t) {
 790                         *i = t->next;
 791                         /* If this was the tail element, move the tail ptr */
 792                         if (*i == NULL)
 793                                 per_cpu(tasklet_vec, cpu).tail = i;
 794                         return;
 795                 }
 796         }
 797         BUG();
 798 }
 799
 800 static void takeover_tasklets(unsigned int cpu)
 801 {
 802         /* CPU is dead, so no lock needed. */
 803         local_irq_disable();
 804
 805         /* Find end, append list for that CPU. */
 806         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
 807                 *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
 808                 this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
 809                 per_cpu(tasklet_vec, cpu).head = NULL;
 810                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
 811         }
 812         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 813
 814         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
 815                 *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
 816                 __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
 817                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
 818                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
 819         }
 820         raise_softirq_irqoff(HI_SOFTIRQ);
 821
 822         local_irq_enable();
 823 }
 824 #endif /* CONFIG_HOTPLUG_CPU */
 825
 826 static int __cpuinit cpu_callback(struct notifier_block *nfb,
 827                                   unsigned long action,
 828                                   void *hcpu)
 829 {
 830         switch (action) {
 831 #ifdef CONFIG_HOTPLUG_CPU
 832         case CPU_DEAD:
 833         case CPU_DEAD_FROZEN:
 834                 takeover_tasklets((unsigned long)hcpu);
 835                 break;
 836 #endif /* CONFIG_HOTPLUG_CPU */
 837         }
 838         return NOTIFY_OK;
 839 }
 840
 841 static struct notifier_block __cpuinitdata cpu_nfb = {
 842         .notifier_call = cpu_callback
 843 };
 844
 845 static struct smp_hotplug_thread softirq_threads = {
 846         .store                  = &ksoftirqd,
 847         .thread_should_run      = ksoftirqd_should_run,
 848         .thread_fn              = run_ksoftirqd,
 849         .thread_comm            = "ksoftirqd/%u",
 850 };
 851
 852 static __init int spawn_ksoftirqd(void)
 853 {
 854         register_cpu_notifier(&cpu_nfb);
 855
 856         BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
 857
 858         return 0;
 859 }
 860 early_initcall(spawn_ksoftirqd);
 861
 862 /*
 863  * [ These __weak aliases are kept in a separate compilation unit, so that
 864  *   GCC does not inline them incorrectly. ]
 865  */
 866
 867 int __init __weak early_irq_init(void)
 868 {
 869         return 0;
 870 }
 871
 872 #ifdef CONFIG_GENERIC_HARDIRQS
 873 int __init __weak arch_probe_nr_irqs(void)
 874 {
 875         return NR_IRQS_LEGACY;
 876 }
 877
 878 int __init __weak arch_early_irq_init(void)
 879 {
 880         return 0;
 881 }
 882 #endif