So here's a boot tested patch on top of Jason's series that does
all the cleanups I talked about and turns jump labels into a
more intuitive to use facility. It should also address the
various misconceptions and confusions that surround jump labels.
Typical usage scenarios:
        #include <linux/static_key.h>
        struct static_key key = STATIC_KEY_INIT_TRUE;
        if (static_key_false(&key))
                do unlikely code
        else
                do likely code
Or:
        if (static_key_true(&key))
                do likely code
        else
                do unlikely code
The static key is modified via:
        static_key_slow_inc(&key);
        ...
        static_key_slow_dec(&key);
The 'slow' prefix makes it abundantly clear that this is an
expensive operation.
I've updated all in-kernel code to use this everywhere. Note
that I (intentionally) have not pushed through the rename
blindly through to the lowest levels: the actual jump-label
patching arch facility should be named like that, so we want to
decouple jump labels from the static-key facility a bit.
On non-jump-label enabled architectures static keys default to
likely()/unlikely() branches.
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Acked-by: Jason Baron <jbaron@redhat.com>
Acked-by: Steven Rostedt <rostedt@goodmis.org>
Cc: a.p.zijlstra@chello.nl
Cc: mathieu.desnoyers@efficios.com
Cc: davem@davemloft.net
Cc: ddaney.cavm@gmail.com
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: http://lkml.kernel.org/r/20120222085809.GA26397@elte.hu
Signed-off-by: Ingo Molnar <mingo@elte.hu>
		
	
			
		
			
				
	
	
		
			322 lines
		
	
	
	
		
			7.6 KiB
			
		
	
	
	
		
			C
		
	
	
	
	
	
			
		
		
	
	
			322 lines
		
	
	
	
		
			7.6 KiB
			
		
	
	
	
		
			C
		
	
	
	
	
	
/******************************************************************************
 | 
						|
 * Copyright (c) 2008 Isaku Yamahata <yamahata at valinux co jp>
 | 
						|
 *                    VA Linux Systems Japan K.K.
 | 
						|
 *
 | 
						|
 * This program is free software; you can redistribute it and/or modify
 | 
						|
 * it under the terms of the GNU General Public License as published by
 | 
						|
 * the Free Software Foundation; either version 2 of the License, or
 | 
						|
 * (at your option) any later version.
 | 
						|
 *
 | 
						|
 * This program is distributed in the hope that it will be useful,
 | 
						|
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 | 
						|
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 | 
						|
 * GNU General Public License for more details.
 | 
						|
 *
 | 
						|
 * You should have received a copy of the GNU General Public License
 | 
						|
 * along with this program; if not, write to the Free Software
 | 
						|
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 | 
						|
 *
 | 
						|
 */
 | 
						|
 | 
						|
 | 
						|
#ifndef __ASM_PARAVIRT_H
 | 
						|
#define __ASM_PARAVIRT_H
 | 
						|
 | 
						|
#ifndef __ASSEMBLY__
 | 
						|
/******************************************************************************
 | 
						|
 * fsys related addresses
 | 
						|
 */
 | 
						|
struct pv_fsys_data {
 | 
						|
	unsigned long *fsyscall_table;
 | 
						|
	void *fsys_bubble_down;
 | 
						|
};
 | 
						|
 | 
						|
extern struct pv_fsys_data pv_fsys_data;
 | 
						|
 | 
						|
unsigned long *paravirt_get_fsyscall_table(void);
 | 
						|
char *paravirt_get_fsys_bubble_down(void);
 | 
						|
 | 
						|
/******************************************************************************
 | 
						|
 * patchlist addresses for gate page
 | 
						|
 */
 | 
						|
enum pv_gate_patchlist {
 | 
						|
	PV_GATE_START_FSYSCALL,
 | 
						|
	PV_GATE_END_FSYSCALL,
 | 
						|
 | 
						|
	PV_GATE_START_BRL_FSYS_BUBBLE_DOWN,
 | 
						|
	PV_GATE_END_BRL_FSYS_BUBBLE_DOWN,
 | 
						|
 | 
						|
	PV_GATE_START_VTOP,
 | 
						|
	PV_GATE_END_VTOP,
 | 
						|
 | 
						|
	PV_GATE_START_MCKINLEY_E9,
 | 
						|
	PV_GATE_END_MCKINLEY_E9,
 | 
						|
};
 | 
						|
 | 
						|
struct pv_patchdata {
 | 
						|
	unsigned long start_fsyscall_patchlist;
 | 
						|
	unsigned long end_fsyscall_patchlist;
 | 
						|
	unsigned long start_brl_fsys_bubble_down_patchlist;
 | 
						|
	unsigned long end_brl_fsys_bubble_down_patchlist;
 | 
						|
	unsigned long start_vtop_patchlist;
 | 
						|
	unsigned long end_vtop_patchlist;
 | 
						|
	unsigned long start_mckinley_e9_patchlist;
 | 
						|
	unsigned long end_mckinley_e9_patchlist;
 | 
						|
 | 
						|
	void *gate_section;
 | 
						|
};
 | 
						|
 | 
						|
extern struct pv_patchdata pv_patchdata;
 | 
						|
 | 
						|
unsigned long paravirt_get_gate_patchlist(enum pv_gate_patchlist type);
 | 
						|
void *paravirt_get_gate_section(void);
 | 
						|
#endif
 | 
						|
 | 
						|
#ifdef CONFIG_PARAVIRT_GUEST
 | 
						|
 | 
						|
#define PARAVIRT_HYPERVISOR_TYPE_DEFAULT	0
 | 
						|
#define PARAVIRT_HYPERVISOR_TYPE_XEN		1
 | 
						|
 | 
						|
#ifndef __ASSEMBLY__
 | 
						|
 | 
						|
#include <asm/hw_irq.h>
 | 
						|
#include <asm/meminit.h>
 | 
						|
 | 
						|
/******************************************************************************
 | 
						|
 * general info
 | 
						|
 */
 | 
						|
struct pv_info {
 | 
						|
	unsigned int kernel_rpl;
 | 
						|
	int paravirt_enabled;
 | 
						|
	const char *name;
 | 
						|
};
 | 
						|
 | 
						|
extern struct pv_info pv_info;
 | 
						|
 | 
						|
static inline int paravirt_enabled(void)
 | 
						|
{
 | 
						|
	return pv_info.paravirt_enabled;
 | 
						|
}
 | 
						|
 | 
						|
static inline unsigned int get_kernel_rpl(void)
 | 
						|
{
 | 
						|
	return pv_info.kernel_rpl;
 | 
						|
}
 | 
						|
 | 
						|
/******************************************************************************
 | 
						|
 * initialization hooks.
 | 
						|
 */
 | 
						|
struct rsvd_region;
 | 
						|
 | 
						|
struct pv_init_ops {
 | 
						|
	void (*banner)(void);
 | 
						|
 | 
						|
	int (*reserve_memory)(struct rsvd_region *region);
 | 
						|
 | 
						|
	void (*arch_setup_early)(void);
 | 
						|
	void (*arch_setup_console)(char **cmdline_p);
 | 
						|
	int (*arch_setup_nomca)(void);
 | 
						|
 | 
						|
	void (*post_smp_prepare_boot_cpu)(void);
 | 
						|
 | 
						|
#ifdef ASM_SUPPORTED
 | 
						|
	unsigned long (*patch_bundle)(void *sbundle, void *ebundle,
 | 
						|
				      unsigned long type);
 | 
						|
	unsigned long (*patch_inst)(unsigned long stag, unsigned long etag,
 | 
						|
				    unsigned long type);
 | 
						|
#endif
 | 
						|
	void (*patch_branch)(unsigned long tag, unsigned long type);
 | 
						|
};
 | 
						|
 | 
						|
extern struct pv_init_ops pv_init_ops;
 | 
						|
 | 
						|
static inline void paravirt_banner(void)
 | 
						|
{
 | 
						|
	if (pv_init_ops.banner)
 | 
						|
		pv_init_ops.banner();
 | 
						|
}
 | 
						|
 | 
						|
static inline int paravirt_reserve_memory(struct rsvd_region *region)
 | 
						|
{
 | 
						|
	if (pv_init_ops.reserve_memory)
 | 
						|
		return pv_init_ops.reserve_memory(region);
 | 
						|
	return 0;
 | 
						|
}
 | 
						|
 | 
						|
static inline void paravirt_arch_setup_early(void)
 | 
						|
{
 | 
						|
	if (pv_init_ops.arch_setup_early)
 | 
						|
		pv_init_ops.arch_setup_early();
 | 
						|
}
 | 
						|
 | 
						|
static inline void paravirt_arch_setup_console(char **cmdline_p)
 | 
						|
{
 | 
						|
	if (pv_init_ops.arch_setup_console)
 | 
						|
		pv_init_ops.arch_setup_console(cmdline_p);
 | 
						|
}
 | 
						|
 | 
						|
static inline int paravirt_arch_setup_nomca(void)
 | 
						|
{
 | 
						|
	if (pv_init_ops.arch_setup_nomca)
 | 
						|
		return pv_init_ops.arch_setup_nomca();
 | 
						|
	return 0;
 | 
						|
}
 | 
						|
 | 
						|
static inline void paravirt_post_smp_prepare_boot_cpu(void)
 | 
						|
{
 | 
						|
	if (pv_init_ops.post_smp_prepare_boot_cpu)
 | 
						|
		pv_init_ops.post_smp_prepare_boot_cpu();
 | 
						|
}
 | 
						|
 | 
						|
/******************************************************************************
 | 
						|
 * replacement of iosapic operations.
 | 
						|
 */
 | 
						|
 | 
						|
struct pv_iosapic_ops {
 | 
						|
	void (*pcat_compat_init)(void);
 | 
						|
 | 
						|
	struct irq_chip *(*__get_irq_chip)(unsigned long trigger);
 | 
						|
 | 
						|
	unsigned int (*__read)(char __iomem *iosapic, unsigned int reg);
 | 
						|
	void (*__write)(char __iomem *iosapic, unsigned int reg, u32 val);
 | 
						|
};
 | 
						|
 | 
						|
extern struct pv_iosapic_ops pv_iosapic_ops;
 | 
						|
 | 
						|
static inline void
 | 
						|
iosapic_pcat_compat_init(void)
 | 
						|
{
 | 
						|
	if (pv_iosapic_ops.pcat_compat_init)
 | 
						|
		pv_iosapic_ops.pcat_compat_init();
 | 
						|
}
 | 
						|
 | 
						|
static inline struct irq_chip*
 | 
						|
iosapic_get_irq_chip(unsigned long trigger)
 | 
						|
{
 | 
						|
	return pv_iosapic_ops.__get_irq_chip(trigger);
 | 
						|
}
 | 
						|
 | 
						|
static inline unsigned int
 | 
						|
__iosapic_read(char __iomem *iosapic, unsigned int reg)
 | 
						|
{
 | 
						|
	return pv_iosapic_ops.__read(iosapic, reg);
 | 
						|
}
 | 
						|
 | 
						|
static inline void
 | 
						|
__iosapic_write(char __iomem *iosapic, unsigned int reg, u32 val)
 | 
						|
{
 | 
						|
	return pv_iosapic_ops.__write(iosapic, reg, val);
 | 
						|
}
 | 
						|
 | 
						|
/******************************************************************************
 | 
						|
 * replacement of irq operations.
 | 
						|
 */
 | 
						|
 | 
						|
struct pv_irq_ops {
 | 
						|
	void (*register_ipi)(void);
 | 
						|
 | 
						|
	int (*assign_irq_vector)(int irq);
 | 
						|
	void (*free_irq_vector)(int vector);
 | 
						|
 | 
						|
	void (*register_percpu_irq)(ia64_vector vec,
 | 
						|
				    struct irqaction *action);
 | 
						|
 | 
						|
	void (*resend_irq)(unsigned int vector);
 | 
						|
};
 | 
						|
 | 
						|
extern struct pv_irq_ops pv_irq_ops;
 | 
						|
 | 
						|
static inline void
 | 
						|
ia64_register_ipi(void)
 | 
						|
{
 | 
						|
	pv_irq_ops.register_ipi();
 | 
						|
}
 | 
						|
 | 
						|
static inline int
 | 
						|
assign_irq_vector(int irq)
 | 
						|
{
 | 
						|
	return pv_irq_ops.assign_irq_vector(irq);
 | 
						|
}
 | 
						|
 | 
						|
static inline void
 | 
						|
free_irq_vector(int vector)
 | 
						|
{
 | 
						|
	return pv_irq_ops.free_irq_vector(vector);
 | 
						|
}
 | 
						|
 | 
						|
static inline void
 | 
						|
register_percpu_irq(ia64_vector vec, struct irqaction *action)
 | 
						|
{
 | 
						|
	pv_irq_ops.register_percpu_irq(vec, action);
 | 
						|
}
 | 
						|
 | 
						|
static inline void
 | 
						|
ia64_resend_irq(unsigned int vector)
 | 
						|
{
 | 
						|
	pv_irq_ops.resend_irq(vector);
 | 
						|
}
 | 
						|
 | 
						|
/******************************************************************************
 | 
						|
 * replacement of time operations.
 | 
						|
 */
 | 
						|
 | 
						|
extern struct itc_jitter_data_t itc_jitter_data;
 | 
						|
extern volatile int time_keeper_id;
 | 
						|
 | 
						|
struct pv_time_ops {
 | 
						|
	void (*init_missing_ticks_accounting)(int cpu);
 | 
						|
	int (*do_steal_accounting)(unsigned long *new_itm);
 | 
						|
 | 
						|
	void (*clocksource_resume)(void);
 | 
						|
 | 
						|
	unsigned long long (*sched_clock)(void);
 | 
						|
};
 | 
						|
 | 
						|
extern struct pv_time_ops pv_time_ops;
 | 
						|
 | 
						|
static inline void
 | 
						|
paravirt_init_missing_ticks_accounting(int cpu)
 | 
						|
{
 | 
						|
	if (pv_time_ops.init_missing_ticks_accounting)
 | 
						|
		pv_time_ops.init_missing_ticks_accounting(cpu);
 | 
						|
}
 | 
						|
 | 
						|
struct static_key;
 | 
						|
extern struct static_key paravirt_steal_enabled;
 | 
						|
extern struct static_key paravirt_steal_rq_enabled;
 | 
						|
 | 
						|
static inline int
 | 
						|
paravirt_do_steal_accounting(unsigned long *new_itm)
 | 
						|
{
 | 
						|
	return pv_time_ops.do_steal_accounting(new_itm);
 | 
						|
}
 | 
						|
 | 
						|
static inline unsigned long long paravirt_sched_clock(void)
 | 
						|
{
 | 
						|
	return pv_time_ops.sched_clock();
 | 
						|
}
 | 
						|
 | 
						|
#endif /* !__ASSEMBLY__ */
 | 
						|
 | 
						|
#else
 | 
						|
/* fallback for native case */
 | 
						|
 | 
						|
#ifndef __ASSEMBLY__
 | 
						|
 | 
						|
#define paravirt_banner()				do { } while (0)
 | 
						|
#define paravirt_reserve_memory(region)			0
 | 
						|
 | 
						|
#define paravirt_arch_setup_early()			do { } while (0)
 | 
						|
#define paravirt_arch_setup_console(cmdline_p)		do { } while (0)
 | 
						|
#define paravirt_arch_setup_nomca()			0
 | 
						|
#define paravirt_post_smp_prepare_boot_cpu()		do { } while (0)
 | 
						|
 | 
						|
#define paravirt_init_missing_ticks_accounting(cpu)	do { } while (0)
 | 
						|
#define paravirt_do_steal_accounting(new_itm)		0
 | 
						|
 | 
						|
#endif /* __ASSEMBLY__ */
 | 
						|
 | 
						|
 | 
						|
#endif /* CONFIG_PARAVIRT_GUEST */
 | 
						|
 | 
						|
#endif /* __ASM_PARAVIRT_H */
 |