Pileus Git - ~andy/linux/blob - kernel/softirq.c

   1 /*
   2  *      linux/kernel/softirq.c
   3  *
   4  *      Copyright (C) 1992 Linus Torvalds
   5  *
   6  *      Distribute under GPLv2.
   7  *
   8  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   9  *
  10  *      Remote softirq infrastructure is by Jens Axboe.
  11  */
  12
  13 #include <linux/module.h>
  14 #include <linux/kernel_stat.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/init.h>
  17 #include <linux/mm.h>
  18 #include <linux/notifier.h>
  19 #include <linux/percpu.h>
  20 #include <linux/cpu.h>
  21 #include <linux/freezer.h>
  22 #include <linux/kthread.h>
  23 #include <linux/rcupdate.h>
  24 #include <linux/ftrace.h>
  25 #include <linux/smp.h>
  26 #include <linux/tick.h>
  27
  28 #define CREATE_TRACE_POINTS
  29 #include <trace/events/irq.h>
  30
  31 #include <asm/irq.h>
  32 /*
  33    - No shared variables, all the data are CPU local.
  34    - If a softirq needs serialization, let it serialize itself
  35      by its own spinlocks.
  36    - Even if softirq is serialized, only local cpu is marked for
  37      execution. Hence, we get something sort of weak cpu binding.
  38      Though it is still not clear, will it result in better locality
  39      or will not.
  40
  41    Examples:
  42    - NET RX softirq. It is multithreaded and does not require
  43      any global serialization.
  44    - NET TX softirq. It kicks software netdevice queues, hence
  45      it is logically serialized per device, but this serialization
  46      is invisible to common code.
  47    - Tasklets: serialized wrt itself.
  48  */
  49
  50 #ifndef __ARCH_IRQ_STAT
  51 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  52 EXPORT_SYMBOL(irq_stat);
  53 #endif
  54
  55 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
  56
  57 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  58
  59 char *softirq_to_name[NR_SOFTIRQS] = {
  60         "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
  61         "TASKLET", "SCHED", "HRTIMER",  "RCU"
  62 };
  63
  64 /*
  65  * we cannot loop indefinitely here to avoid userspace starvation,
  66  * but we also don't want to introduce a worst case 1/HZ latency
  67  * to the pending events, so lets the scheduler to balance
  68  * the softirq load for us.
  69  */
  70 void wakeup_softirqd(void)
  71 {
  72         /* Interrupts are disabled: no need to stop preemption */
  73         struct task_struct *tsk = __get_cpu_var(ksoftirqd);
  74
  75         if (tsk && tsk->state != TASK_RUNNING)
  76                 wake_up_process(tsk);
  77 }
  78
  79 /*
  80  * This one is for softirq.c-internal use,
  81  * where hardirqs are disabled legitimately:
  82  */
  83 #ifdef CONFIG_TRACE_IRQFLAGS
  84 static void __local_bh_disable(unsigned long ip)
  85 {
  86         unsigned long flags;
  87
  88         WARN_ON_ONCE(in_irq());
  89
  90         raw_local_irq_save(flags);
  91         /*
  92          * The preempt tracer hooks into add_preempt_count and will break
  93          * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
  94          * is set and before current->softirq_enabled is cleared.
  95          * We must manually increment preempt_count here and manually
  96          * call the trace_preempt_off later.
  97          */
  98         preempt_count() += SOFTIRQ_OFFSET;
  99         /*
 100          * Were softirqs turned off above:
 101          */
 102         if (softirq_count() == SOFTIRQ_OFFSET)
 103                 trace_softirqs_off(ip);
 104         raw_local_irq_restore(flags);
 105
 106         if (preempt_count() == SOFTIRQ_OFFSET)
 107                 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
 108 }
 109 #else /* !CONFIG_TRACE_IRQFLAGS */
 110 static inline void __local_bh_disable(unsigned long ip)
 111 {
 112         add_preempt_count(SOFTIRQ_OFFSET);
 113         barrier();
 114 }
 115 #endif /* CONFIG_TRACE_IRQFLAGS */
 116
 117 void local_bh_disable(void)
 118 {
 119         __local_bh_disable((unsigned long)__builtin_return_address(0));
 120 }
 121
 122 EXPORT_SYMBOL(local_bh_disable);
 123
 124 /*
 125  * Special-case - softirqs can safely be enabled in
 126  * cond_resched_softirq(), or by __do_softirq(),
 127  * without processing still-pending softirqs:
 128  */
 129 void _local_bh_enable(void)
 130 {
 131         WARN_ON_ONCE(in_irq());
 132         WARN_ON_ONCE(!irqs_disabled());
 133
 134         if (softirq_count() == SOFTIRQ_OFFSET)
 135                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
 136         sub_preempt_count(SOFTIRQ_OFFSET);
 137 }
 138
 139 EXPORT_SYMBOL(_local_bh_enable);
 140
 141 static inline void _local_bh_enable_ip(unsigned long ip)
 142 {
 143         WARN_ON_ONCE(in_irq() || irqs_disabled());
 144 #ifdef CONFIG_TRACE_IRQFLAGS
 145         local_irq_disable();
 146 #endif
 147         /*
 148          * Are softirqs going to be turned on now:
 149          */
 150         if (softirq_count() == SOFTIRQ_OFFSET)
 151                 trace_softirqs_on(ip);
 152         /*
 153          * Keep preemption disabled until we are done with
 154          * softirq processing:
 155          */
 156         sub_preempt_count(SOFTIRQ_OFFSET - 1);
 157
 158         if (unlikely(!in_interrupt() && local_softirq_pending()))
 159                 do_softirq();
 160
 161         dec_preempt_count();
 162 #ifdef CONFIG_TRACE_IRQFLAGS
 163         local_irq_enable();
 164 #endif
 165         preempt_check_resched();
 166 }
 167
 168 void local_bh_enable(void)
 169 {
 170         _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 171 }
 172 EXPORT_SYMBOL(local_bh_enable);
 173
 174 void local_bh_enable_ip(unsigned long ip)
 175 {
 176         _local_bh_enable_ip(ip);
 177 }
 178 EXPORT_SYMBOL(local_bh_enable_ip);
 179
 180 /*
 181  * We restart softirq processing MAX_SOFTIRQ_RESTART times,
 182  * and we fall back to softirqd after that.
 183  *
 184  * This number has been established via experimentation.
 185  * The two things to balance is latency against fairness -
 186  * we want to handle softirqs as soon as possible, but they
 187  * should not be able to lock up the box.
 188  */
 189 #define MAX_SOFTIRQ_RESTART 10
 190
 191 asmlinkage void __do_softirq(void)
 192 {
 193         struct softirq_action *h;
 194         __u32 pending;
 195         int max_restart = MAX_SOFTIRQ_RESTART;
 196         int cpu;
 197
 198         pending = local_softirq_pending();
 199         account_system_vtime(current);
 200
 201         __local_bh_disable((unsigned long)__builtin_return_address(0));
 202         lockdep_softirq_enter();
 203
 204         cpu = smp_processor_id();
 205 restart:
 206         /* Reset the pending bitmask before enabling irqs */
 207         set_softirq_pending(0);
 208
 209         local_irq_enable();
 210
 211         h = softirq_vec;
 212
 213         do {
 214                 if (pending & 1) {
 215                         unsigned int vec_nr = h - softirq_vec;
 216                         int prev_count = preempt_count();
 217
 218                         kstat_incr_softirqs_this_cpu(vec_nr);
 219
 220                         trace_softirq_entry(vec_nr);
 221                         h->action(h);
 222                         trace_softirq_exit(vec_nr);
 223                         if (unlikely(prev_count != preempt_count())) {
 224                                 printk(KERN_ERR "huh, entered softirq %u %s %p"
 225                                        "with preempt_count %08x,"
 226                                        " exited with %08x?\n", vec_nr,
 227                                        softirq_to_name[vec_nr], h->action,
 228                                        prev_count, preempt_count());
 229                                 preempt_count() = prev_count;
 230                         }
 231
 232                         rcu_bh_qs(cpu);
 233                 }
 234                 h++;
 235                 pending >>= 1;
 236         } while (pending);
 237
 238         local_irq_disable();
 239
 240         pending = local_softirq_pending();
 241         if (pending && --max_restart)
 242                 goto restart;
 243
 244         if (pending)
 245                 wakeup_softirqd();
 246
 247         lockdep_softirq_exit();
 248
 249         account_system_vtime(current);
 250         _local_bh_enable();
 251 }
 252
 253 #ifndef __ARCH_HAS_DO_SOFTIRQ
 254
 255 asmlinkage void do_softirq(void)
 256 {
 257         __u32 pending;
 258         unsigned long flags;
 259
 260         if (in_interrupt())
 261                 return;
 262
 263         local_irq_save(flags);
 264
 265         pending = local_softirq_pending();
 266
 267         if (pending)
 268                 __do_softirq();
 269
 270         local_irq_restore(flags);
 271 }
 272
 273 #endif
 274
 275 /*
 276  * Enter an interrupt context.
 277  */
 278 void irq_enter(void)
 279 {
 280         int cpu = smp_processor_id();
 281
 282         rcu_irq_enter();
 283         if (idle_cpu(cpu) && !in_interrupt()) {
 284                 __irq_enter();
 285                 tick_check_idle(cpu);
 286         } else
 287                 __irq_enter();
 288 }
 289
 290 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
 291 # define invoke_softirq()       __do_softirq()
 292 #else
 293 # define invoke_softirq()       do_softirq()
 294 #endif
 295
 296 /*
 297  * Exit an interrupt context. Process softirqs if needed and possible:
 298  */
 299 void irq_exit(void)
 300 {
 301         account_system_vtime(current);
 302         trace_hardirq_exit();
 303         sub_preempt_count(IRQ_EXIT_OFFSET);
 304         if (!in_interrupt() && local_softirq_pending())
 305                 invoke_softirq();
 306
 307         rcu_irq_exit();
 308 #ifdef CONFIG_NO_HZ
 309         /* Make sure that timer wheel updates are propagated */
 310         if (idle_cpu(smp_processor_id()) && !in_interrupt() && !need_resched())
 311                 tick_nohz_stop_sched_tick(0);
 312 #endif
 313         preempt_enable_no_resched();
 314 }
 315
 316 /*
 317  * This function must run with irqs disabled!
 318  */
 319 inline void raise_softirq_irqoff(unsigned int nr)
 320 {
 321         __raise_softirq_irqoff(nr);
 322
 323         /*
 324          * If we're in an interrupt or softirq, we're done
 325          * (this also catches softirq-disabled code). We will
 326          * actually run the softirq once we return from
 327          * the irq or softirq.
 328          *
 329          * Otherwise we wake up ksoftirqd to make sure we
 330          * schedule the softirq soon.
 331          */
 332         if (!in_interrupt())
 333                 wakeup_softirqd();
 334 }
 335
 336 void raise_softirq(unsigned int nr)
 337 {
 338         unsigned long flags;
 339
 340         local_irq_save(flags);
 341         raise_softirq_irqoff(nr);
 342         local_irq_restore(flags);
 343 }
 344
 345 void open_softirq(int nr, void (*action)(struct softirq_action *))
 346 {
 347         softirq_vec[nr].action = action;
 348 }
 349
 350 /*
 351  * Tasklets
 352  */
 353 struct tasklet_head
 354 {
 355         struct tasklet_struct *head;
 356         struct tasklet_struct **tail;
 357 };
 358
 359 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
 360 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 361
 362 void __tasklet_schedule(struct tasklet_struct *t)
 363 {
 364         unsigned long flags;
 365
 366         local_irq_save(flags);
 367         t->next = NULL;
 368         *__get_cpu_var(tasklet_vec).tail = t;
 369         __get_cpu_var(tasklet_vec).tail = &(t->next);
 370         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 371         local_irq_restore(flags);
 372 }
 373
 374 EXPORT_SYMBOL(__tasklet_schedule);
 375
 376 void __tasklet_hi_schedule(struct tasklet_struct *t)
 377 {
 378         unsigned long flags;
 379
 380         local_irq_save(flags);
 381         t->next = NULL;
 382         *__get_cpu_var(tasklet_hi_vec).tail = t;
 383         __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
 384         raise_softirq_irqoff(HI_SOFTIRQ);
 385         local_irq_restore(flags);
 386 }
 387
 388 EXPORT_SYMBOL(__tasklet_hi_schedule);
 389
 390 void __tasklet_hi_schedule_first(struct tasklet_struct *t)
 391 {
 392         BUG_ON(!irqs_disabled());
 393
 394         t->next = __get_cpu_var(tasklet_hi_vec).head;
 395         __get_cpu_var(tasklet_hi_vec).head = t;
 396         __raise_softirq_irqoff(HI_SOFTIRQ);
 397 }
 398
 399 EXPORT_SYMBOL(__tasklet_hi_schedule_first);
 400
 401 static void tasklet_action(struct softirq_action *a)
 402 {
 403         struct tasklet_struct *list;
 404
 405         local_irq_disable();
 406         list = __get_cpu_var(tasklet_vec).head;
 407         __get_cpu_var(tasklet_vec).head = NULL;
 408         __get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
 409         local_irq_enable();
 410
 411         while (list) {
 412                 struct tasklet_struct *t = list;
 413
 414                 list = list->next;
 415
 416                 if (tasklet_trylock(t)) {
 417                         if (!atomic_read(&t->count)) {
 418                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 419                                         BUG();
 420                                 t->func(t->data);
 421                                 tasklet_unlock(t);
 422                                 continue;
 423                         }
 424                         tasklet_unlock(t);
 425                 }
 426
 427                 local_irq_disable();
 428                 t->next = NULL;
 429                 *__get_cpu_var(tasklet_vec).tail = t;
 430                 __get_cpu_var(tasklet_vec).tail = &(t->next);
 431                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 432                 local_irq_enable();
 433         }
 434 }
 435
 436 static void tasklet_hi_action(struct softirq_action *a)
 437 {
 438         struct tasklet_struct *list;
 439
 440         local_irq_disable();
 441         list = __get_cpu_var(tasklet_hi_vec).head;
 442         __get_cpu_var(tasklet_hi_vec).head = NULL;
 443         __get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
 444         local_irq_enable();
 445
 446         while (list) {
 447                 struct tasklet_struct *t = list;
 448
 449                 list = list->next;
 450
 451                 if (tasklet_trylock(t)) {
 452                         if (!atomic_read(&t->count)) {
 453                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 454                                         BUG();
 455                                 t->func(t->data);
 456                                 tasklet_unlock(t);
 457                                 continue;
 458                         }
 459                         tasklet_unlock(t);
 460                 }
 461
 462                 local_irq_disable();
 463                 t->next = NULL;
 464                 *__get_cpu_var(tasklet_hi_vec).tail = t;
 465                 __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
 466                 __raise_softirq_irqoff(HI_SOFTIRQ);
 467                 local_irq_enable();
 468         }
 469 }
 470
 471
 472 void tasklet_init(struct tasklet_struct *t,
 473                   void (*func)(unsigned long), unsigned long data)
 474 {
 475         t->next = NULL;
 476         t->state = 0;
 477         atomic_set(&t->count, 0);
 478         t->func = func;
 479         t->data = data;
 480 }
 481
 482 EXPORT_SYMBOL(tasklet_init);
 483
 484 void tasklet_kill(struct tasklet_struct *t)
 485 {
 486         if (in_interrupt())
 487                 printk("Attempt to kill tasklet from interrupt\n");
 488
 489         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 490                 do {
 491                         yield();
 492                 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
 493         }
 494         tasklet_unlock_wait(t);
 495         clear_bit(TASKLET_STATE_SCHED, &t->state);
 496 }
 497
 498 EXPORT_SYMBOL(tasklet_kill);
 499
 500 /*
 501  * tasklet_hrtimer
 502  */
 503
 504 /*
 505  * The trampoline is called when the hrtimer expires. It schedules a tasklet
 506  * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
 507  * hrtimer callback, but from softirq context.
 508  */
 509 static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
 510 {
 511         struct tasklet_hrtimer *ttimer =
 512                 container_of(timer, struct tasklet_hrtimer, timer);
 513
 514         tasklet_hi_schedule(&ttimer->tasklet);
 515         return HRTIMER_NORESTART;
 516 }
 517
 518 /*
 519  * Helper function which calls the hrtimer callback from
 520  * tasklet/softirq context
 521  */
 522 static void __tasklet_hrtimer_trampoline(unsigned long data)
 523 {
 524         struct tasklet_hrtimer *ttimer = (void *)data;
 525         enum hrtimer_restart restart;
 526
 527         restart = ttimer->function(&ttimer->timer);
 528         if (restart != HRTIMER_NORESTART)
 529                 hrtimer_restart(&ttimer->timer);
 530 }
 531
 532 /**
 533  * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
 534  * @ttimer:      tasklet_hrtimer which is initialized
 535  * @function:    hrtimer callback funtion which gets called from softirq context
 536  * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
 537  * @mode:        hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
 538  */
 539 void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
 540                           enum hrtimer_restart (*function)(struct hrtimer *),
 541                           clockid_t which_clock, enum hrtimer_mode mode)
 542 {
 543         hrtimer_init(&ttimer->timer, which_clock, mode);
 544         ttimer->timer.function = __hrtimer_tasklet_trampoline;
 545         tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
 546                      (unsigned long)ttimer);
 547         ttimer->function = function;
 548 }
 549 EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
 550
 551 /*
 552  * Remote softirq bits
 553  */
 554
 555 DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
 556 EXPORT_PER_CPU_SYMBOL(softirq_work_list);
 557
 558 static void __local_trigger(struct call_single_data *cp, int softirq)
 559 {
 560         struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
 561
 562         list_add_tail(&cp->list, head);
 563
 564         /* Trigger the softirq only if the list was previously empty.  */
 565         if (head->next == &cp->list)
 566                 raise_softirq_irqoff(softirq);
 567 }
 568
 569 #ifdef CONFIG_USE_GENERIC_SMP_HELPERS
 570 static void remote_softirq_receive(void *data)
 571 {
 572         struct call_single_data *cp = data;
 573         unsigned long flags;
 574         int softirq;
 575
 576         softirq = cp->priv;
 577
 578         local_irq_save(flags);
 579         __local_trigger(cp, softirq);
 580         local_irq_restore(flags);
 581 }
 582
 583 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 584 {
 585         if (cpu_online(cpu)) {
 586                 cp->func = remote_softirq_receive;
 587                 cp->info = cp;
 588                 cp->flags = 0;
 589                 cp->priv = softirq;
 590
 591                 __smp_call_function_single(cpu, cp, 0);
 592                 return 0;
 593         }
 594         return 1;
 595 }
 596 #else /* CONFIG_USE_GENERIC_SMP_HELPERS */
 597 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 598 {
 599         return 1;
 600 }
 601 #endif
 602
 603 /**
 604  * __send_remote_softirq - try to schedule softirq work on a remote cpu
 605  * @cp: private SMP call function data area
 606  * @cpu: the remote cpu
 607  * @this_cpu: the currently executing cpu
 608  * @softirq: the softirq for the work
 609  *
 610  * Attempt to schedule softirq work on a remote cpu.  If this cannot be
 611  * done, the work is instead queued up on the local cpu.
 612  *
 613  * Interrupts must be disabled.
 614  */
 615 void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
 616 {
 617         if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
 618                 __local_trigger(cp, softirq);
 619 }
 620 EXPORT_SYMBOL(__send_remote_softirq);
 621
 622 /**
 623  * send_remote_softirq - try to schedule softirq work on a remote cpu
 624  * @cp: private SMP call function data area
 625  * @cpu: the remote cpu
 626  * @softirq: the softirq for the work
 627  *
 628  * Like __send_remote_softirq except that disabling interrupts and
 629  * computing the current cpu is done for the caller.
 630  */
 631 void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 632 {
 633         unsigned long flags;
 634         int this_cpu;
 635
 636         local_irq_save(flags);
 637         this_cpu = smp_processor_id();
 638         __send_remote_softirq(cp, cpu, this_cpu, softirq);
 639         local_irq_restore(flags);
 640 }
 641 EXPORT_SYMBOL(send_remote_softirq);
 642
 643 static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
 644                                                unsigned long action, void *hcpu)
 645 {
 646         /*
 647          * If a CPU goes away, splice its entries to the current CPU
 648          * and trigger a run of the softirq
 649          */
 650         if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
 651                 int cpu = (unsigned long) hcpu;
 652                 int i;
 653
 654                 local_irq_disable();
 655                 for (i = 0; i < NR_SOFTIRQS; i++) {
 656                         struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
 657                         struct list_head *local_head;
 658
 659                         if (list_empty(head))
 660                                 continue;
 661
 662                         local_head = &__get_cpu_var(softirq_work_list[i]);
 663                         list_splice_init(head, local_head);
 664                         raise_softirq_irqoff(i);
 665                 }
 666                 local_irq_enable();
 667         }
 668
 669         return NOTIFY_OK;
 670 }
 671
 672 static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
 673         .notifier_call  = remote_softirq_cpu_notify,
 674 };
 675
 676 void __init softirq_init(void)
 677 {
 678         int cpu;
 679
 680         for_each_possible_cpu(cpu) {
 681                 int i;
 682
 683                 per_cpu(tasklet_vec, cpu).tail =
 684                         &per_cpu(tasklet_vec, cpu).head;
 685                 per_cpu(tasklet_hi_vec, cpu).tail =
 686                         &per_cpu(tasklet_hi_vec, cpu).head;
 687                 for (i = 0; i < NR_SOFTIRQS; i++)
 688                         INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
 689         }
 690
 691         register_hotcpu_notifier(&remote_softirq_cpu_notifier);
 692
 693         open_softirq(TASKLET_SOFTIRQ, tasklet_action);
 694         open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 695 }
 696
 697 static int run_ksoftirqd(void * __bind_cpu)
 698 {
 699         set_current_state(TASK_INTERRUPTIBLE);
 700
 701         while (!kthread_should_stop()) {
 702                 preempt_disable();
 703                 if (!local_softirq_pending()) {
 704                         preempt_enable_no_resched();
 705                         schedule();
 706                         preempt_disable();
 707                 }
 708
 709                 __set_current_state(TASK_RUNNING);
 710
 711                 while (local_softirq_pending()) {
 712                         /* Preempt disable stops cpu going offline.
 713                            If already offline, we'll be on wrong CPU:
 714                            don't process */
 715                         if (cpu_is_offline((long)__bind_cpu))
 716                                 goto wait_to_die;
 717                         do_softirq();
 718                         preempt_enable_no_resched();
 719                         cond_resched();
 720                         preempt_disable();
 721                         rcu_note_context_switch((long)__bind_cpu);
 722                 }
 723                 preempt_enable();
 724                 set_current_state(TASK_INTERRUPTIBLE);
 725         }
 726         __set_current_state(TASK_RUNNING);
 727         return 0;
 728
 729 wait_to_die:
 730         preempt_enable();
 731         /* Wait for kthread_stop */
 732         set_current_state(TASK_INTERRUPTIBLE);
 733         while (!kthread_should_stop()) {
 734                 schedule();
 735                 set_current_state(TASK_INTERRUPTIBLE);
 736         }
 737         __set_current_state(TASK_RUNNING);
 738         return 0;
 739 }
 740
 741 #ifdef CONFIG_HOTPLUG_CPU
 742 /*
 743  * tasklet_kill_immediate is called to remove a tasklet which can already be
 744  * scheduled for execution on @cpu.
 745  *
 746  * Unlike tasklet_kill, this function removes the tasklet
 747  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 748  *
 749  * When this function is called, @cpu must be in the CPU_DEAD state.
 750  */
 751 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 752 {
 753         struct tasklet_struct **i;
 754
 755         BUG_ON(cpu_online(cpu));
 756         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 757
 758         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 759                 return;
 760
 761         /* CPU is dead, so no lock needed. */
 762         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
 763                 if (*i == t) {
 764                         *i = t->next;
 765                         /* If this was the tail element, move the tail ptr */
 766                         if (*i == NULL)
 767                                 per_cpu(tasklet_vec, cpu).tail = i;
 768                         return;
 769                 }
 770         }
 771         BUG();
 772 }
 773
 774 static void takeover_tasklets(unsigned int cpu)
 775 {
 776         /* CPU is dead, so no lock needed. */
 777         local_irq_disable();
 778
 779         /* Find end, append list for that CPU. */
 780         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
 781                 *(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
 782                 __get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
 783                 per_cpu(tasklet_vec, cpu).head = NULL;
 784                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
 785         }
 786         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 787
 788         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
 789                 *__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
 790                 __get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
 791                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
 792                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
 793         }
 794         raise_softirq_irqoff(HI_SOFTIRQ);
 795
 796         local_irq_enable();
 797 }
 798 #endif /* CONFIG_HOTPLUG_CPU */
 799
 800 static int __cpuinit cpu_callback(struct notifier_block *nfb,
 801                                   unsigned long action,
 802                                   void *hcpu)
 803 {
 804         int hotcpu = (unsigned long)hcpu;
 805         struct task_struct *p;
 806
 807         switch (action) {
 808         case CPU_UP_PREPARE:
 809         case CPU_UP_PREPARE_FROZEN:
 810                 p = kthread_create(run_ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
 811                 if (IS_ERR(p)) {
 812                         printk("ksoftirqd for %i failed\n", hotcpu);
 813                         return notifier_from_errno(PTR_ERR(p));
 814                 }
 815                 kthread_bind(p, hotcpu);
 816                 per_cpu(ksoftirqd, hotcpu) = p;
 817                 break;
 818         case CPU_ONLINE:
 819         case CPU_ONLINE_FROZEN:
 820                 wake_up_process(per_cpu(ksoftirqd, hotcpu));
 821                 break;
 822 #ifdef CONFIG_HOTPLUG_CPU
 823         case CPU_UP_CANCELED:
 824         case CPU_UP_CANCELED_FROZEN:
 825                 if (!per_cpu(ksoftirqd, hotcpu))
 826                         break;
 827                 /* Unbind so it can run.  Fall thru. */
 828                 kthread_bind(per_cpu(ksoftirqd, hotcpu),
 829                              cpumask_any(cpu_online_mask));
 830         case CPU_DEAD:
 831         case CPU_DEAD_FROZEN: {
 832                 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
 833
 834                 p = per_cpu(ksoftirqd, hotcpu);
 835                 per_cpu(ksoftirqd, hotcpu) = NULL;
 836                 sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
 837                 kthread_stop(p);
 838                 takeover_tasklets(hotcpu);
 839                 break;
 840         }
 841 #endif /* CONFIG_HOTPLUG_CPU */
 842         }
 843         return NOTIFY_OK;
 844 }
 845
 846 static struct notifier_block __cpuinitdata cpu_nfb = {
 847         .notifier_call = cpu_callback
 848 };
 849
 850 static __init int spawn_ksoftirqd(void)
 851 {
 852         void *cpu = (void *)(long)smp_processor_id();
 853         int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
 854
 855         BUG_ON(err != NOTIFY_OK);
 856         cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
 857         register_cpu_notifier(&cpu_nfb);
 858         return 0;
 859 }
 860 early_initcall(spawn_ksoftirqd);
 861
 862 #ifdef CONFIG_SMP
 863 /*
 864  * Call a function on all processors
 865  */
 866 int on_each_cpu(void (*func) (void *info), void *info, int wait)
 867 {
 868         int ret = 0;
 869
 870         preempt_disable();
 871         ret = smp_call_function(func, info, wait);
 872         local_irq_disable();
 873         func(info);
 874         local_irq_enable();
 875         preempt_enable();
 876         return ret;
 877 }
 878 EXPORT_SYMBOL(on_each_cpu);
 879 #endif
 880
 881 /*
 882  * [ These __weak aliases are kept in a separate compilation unit, so that
 883  *   GCC does not inline them incorrectly. ]
 884  */
 885
 886 int __init __weak early_irq_init(void)
 887 {
 888         return 0;
 889 }
 890
 891 int __init __weak arch_probe_nr_irqs(void)
 892 {
 893         return 0;
 894 }
 895
 896 int __init __weak arch_early_irq_init(void)
 897 {
 898         return 0;
 899 }
 900
 901 int __weak arch_init_chip_data(struct irq_desc *desc, int node)
 902 {
 903         return 0;
 904 }