Impact: fix rare lost events problem
There are CPUs whose performance counters misbehave on CSTATE transitions,
so provide a way to just disable/enable them around deep idle methods.
(hw_perf_enable_all() is cheap on x86.)
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
#include <linux/notifier.h>
#include <linux/hardirq.h>
#include <linux/kprobes.h>
#include <linux/notifier.h>
#include <linux/hardirq.h>
#include <linux/kprobes.h>
+#include <linux/module.h>
#include <linux/kdebug.h>
#include <linux/sched.h>
#include <linux/kdebug.h>
#include <linux/sched.h>
wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, perf_counter_mask, 0);
}
wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, perf_counter_mask, 0);
}
-void hw_perf_disable_all(void)
+void hw_perf_restore_ctrl(u64 ctrl)
+ wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, ctrl, 0);
+}
+EXPORT_SYMBOL_GPL(hw_perf_restore_ctrl);
+
+u64 hw_perf_disable_all(void)
+{
+ u64 ctrl;
+
+ rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0, 0);
wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0, 0);
+EXPORT_SYMBOL_GPL(hw_perf_disable_all);
static inline void
__hw_perf_counter_disable(struct hw_perf_counter *hwc, unsigned int idx)
static inline void
__hw_perf_counter_disable(struct hw_perf_counter *hwc, unsigned int idx)
/* Common C-state entry for C2, C3, .. */
static void acpi_cstate_enter(struct acpi_processor_cx *cstate)
{
/* Common C-state entry for C2, C3, .. */
static void acpi_cstate_enter(struct acpi_processor_cx *cstate)
{
/* Don't trace irqs off for idle */
stop_critical_timings();
/* Don't trace irqs off for idle */
stop_critical_timings();
+ pctrl = hw_perf_disable_all();
if (cstate->entry_method == ACPI_CSTATE_FFH) {
/* Call into architectural FFH based C-state */
acpi_processor_ffh_cstate_enter(cstate);
if (cstate->entry_method == ACPI_CSTATE_FFH) {
/* Call into architectural FFH based C-state */
acpi_processor_ffh_cstate_enter(cstate);
gets asserted in time to freeze execution properly. */
unused = inl(acpi_gbl_FADT.xpm_timer_block.address);
}
gets asserted in time to freeze execution properly. */
unused = inl(acpi_gbl_FADT.xpm_timer_block.address);
}
+ hw_perf_restore_ctrl(pctrl);
start_critical_timings();
}
#endif /* !CONFIG_CPU_IDLE */
start_critical_timings();
}
#endif /* !CONFIG_CPU_IDLE */
*/
static inline void acpi_idle_do_entry(struct acpi_processor_cx *cx)
{
*/
static inline void acpi_idle_do_entry(struct acpi_processor_cx *cx)
{
/* Don't trace irqs off for idle */
stop_critical_timings();
/* Don't trace irqs off for idle */
stop_critical_timings();
+ pctrl = hw_perf_disable_all();
if (cx->entry_method == ACPI_CSTATE_FFH) {
/* Call into architectural FFH based C-state */
acpi_processor_ffh_cstate_enter(cx);
if (cx->entry_method == ACPI_CSTATE_FFH) {
/* Call into architectural FFH based C-state */
acpi_processor_ffh_cstate_enter(cx);
gets asserted in time to freeze execution properly. */
unused = inl(acpi_gbl_FADT.xpm_timer_block.address);
}
gets asserted in time to freeze execution properly. */
unused = inl(acpi_gbl_FADT.xpm_timer_block.address);
}
+ hw_perf_restore_ctrl(pctrl);
start_critical_timings();
}
start_critical_timings();
}
extern void perf_counter_init_task(struct task_struct *task);
extern void perf_counter_notify(struct pt_regs *regs);
extern void perf_counter_print_debug(void);
extern void perf_counter_init_task(struct task_struct *task);
extern void perf_counter_notify(struct pt_regs *regs);
extern void perf_counter_print_debug(void);
+extern void hw_perf_restore_ctrl(u64 ctrl);
+extern u64 hw_perf_disable_all(void);
#else
static inline void
perf_counter_task_sched_in(struct task_struct *task, int cpu) { }
#else
static inline void
perf_counter_task_sched_in(struct task_struct *task, int cpu) { }
static inline void perf_counter_init_task(struct task_struct *task) { }
static inline void perf_counter_notify(struct pt_regs *regs) { }
static inline void perf_counter_print_debug(void) { }
static inline void perf_counter_init_task(struct task_struct *task) { }
static inline void perf_counter_notify(struct pt_regs *regs) { }
static inline void perf_counter_print_debug(void) { }
+static inline void hw_perf_restore_ctrl(u64 ctrl) { }
+static inline u64 hw_perf_disable_all(void) { return 0; }
#endif
#endif /* _LINUX_PERF_COUNTER_H */
#endif
#endif /* _LINUX_PERF_COUNTER_H */