 c5905afb0e
			
		
	
	
	c5905afb0e
	
	
	
		
			
			So here's a boot tested patch on top of Jason's series that does
all the cleanups I talked about and turns jump labels into a
more intuitive to use facility. It should also address the
various misconceptions and confusions that surround jump labels.
Typical usage scenarios:
        #include <linux/static_key.h>
        struct static_key key = STATIC_KEY_INIT_TRUE;
        if (static_key_false(&key))
                do unlikely code
        else
                do likely code
Or:
        if (static_key_true(&key))
                do likely code
        else
                do unlikely code
The static key is modified via:
        static_key_slow_inc(&key);
        ...
        static_key_slow_dec(&key);
The 'slow' prefix makes it abundantly clear that this is an
expensive operation.
I've updated all in-kernel code to use this everywhere. Note
that I (intentionally) have not pushed through the rename
blindly through to the lowest levels: the actual jump-label
patching arch facility should be named like that, so we want to
decouple jump labels from the static-key facility a bit.
On non-jump-label enabled architectures static keys default to
likely()/unlikely() branches.
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Acked-by: Jason Baron <jbaron@redhat.com>
Acked-by: Steven Rostedt <rostedt@goodmis.org>
Cc: a.p.zijlstra@chello.nl
Cc: mathieu.desnoyers@efficios.com
Cc: davem@davemloft.net
Cc: ddaney.cavm@gmail.com
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: http://lkml.kernel.org/r/20120222085809.GA26397@elte.hu
Signed-off-by: Ingo Molnar <mingo@elte.hu>
		
	
			
		
			
				
	
	
		
			322 lines
		
	
	
	
		
			7.6 KiB
			
		
	
	
	
		
			C
		
	
	
	
	
	
			
		
		
	
	
			322 lines
		
	
	
	
		
			7.6 KiB
			
		
	
	
	
		
			C
		
	
	
	
	
	
| /******************************************************************************
 | |
|  * Copyright (c) 2008 Isaku Yamahata <yamahata at valinux co jp>
 | |
|  *                    VA Linux Systems Japan K.K.
 | |
|  *
 | |
|  * This program is free software; you can redistribute it and/or modify
 | |
|  * it under the terms of the GNU General Public License as published by
 | |
|  * the Free Software Foundation; either version 2 of the License, or
 | |
|  * (at your option) any later version.
 | |
|  *
 | |
|  * This program is distributed in the hope that it will be useful,
 | |
|  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 | |
|  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 | |
|  * GNU General Public License for more details.
 | |
|  *
 | |
|  * You should have received a copy of the GNU General Public License
 | |
|  * along with this program; if not, write to the Free Software
 | |
|  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 | |
|  *
 | |
|  */
 | |
| 
 | |
| 
 | |
| #ifndef __ASM_PARAVIRT_H
 | |
| #define __ASM_PARAVIRT_H
 | |
| 
 | |
| #ifndef __ASSEMBLY__
 | |
| /******************************************************************************
 | |
|  * fsys related addresses
 | |
|  */
 | |
| struct pv_fsys_data {
 | |
| 	unsigned long *fsyscall_table;
 | |
| 	void *fsys_bubble_down;
 | |
| };
 | |
| 
 | |
| extern struct pv_fsys_data pv_fsys_data;
 | |
| 
 | |
| unsigned long *paravirt_get_fsyscall_table(void);
 | |
| char *paravirt_get_fsys_bubble_down(void);
 | |
| 
 | |
| /******************************************************************************
 | |
|  * patchlist addresses for gate page
 | |
|  */
 | |
| enum pv_gate_patchlist {
 | |
| 	PV_GATE_START_FSYSCALL,
 | |
| 	PV_GATE_END_FSYSCALL,
 | |
| 
 | |
| 	PV_GATE_START_BRL_FSYS_BUBBLE_DOWN,
 | |
| 	PV_GATE_END_BRL_FSYS_BUBBLE_DOWN,
 | |
| 
 | |
| 	PV_GATE_START_VTOP,
 | |
| 	PV_GATE_END_VTOP,
 | |
| 
 | |
| 	PV_GATE_START_MCKINLEY_E9,
 | |
| 	PV_GATE_END_MCKINLEY_E9,
 | |
| };
 | |
| 
 | |
| struct pv_patchdata {
 | |
| 	unsigned long start_fsyscall_patchlist;
 | |
| 	unsigned long end_fsyscall_patchlist;
 | |
| 	unsigned long start_brl_fsys_bubble_down_patchlist;
 | |
| 	unsigned long end_brl_fsys_bubble_down_patchlist;
 | |
| 	unsigned long start_vtop_patchlist;
 | |
| 	unsigned long end_vtop_patchlist;
 | |
| 	unsigned long start_mckinley_e9_patchlist;
 | |
| 	unsigned long end_mckinley_e9_patchlist;
 | |
| 
 | |
| 	void *gate_section;
 | |
| };
 | |
| 
 | |
| extern struct pv_patchdata pv_patchdata;
 | |
| 
 | |
| unsigned long paravirt_get_gate_patchlist(enum pv_gate_patchlist type);
 | |
| void *paravirt_get_gate_section(void);
 | |
| #endif
 | |
| 
 | |
| #ifdef CONFIG_PARAVIRT_GUEST
 | |
| 
 | |
| #define PARAVIRT_HYPERVISOR_TYPE_DEFAULT	0
 | |
| #define PARAVIRT_HYPERVISOR_TYPE_XEN		1
 | |
| 
 | |
| #ifndef __ASSEMBLY__
 | |
| 
 | |
| #include <asm/hw_irq.h>
 | |
| #include <asm/meminit.h>
 | |
| 
 | |
| /******************************************************************************
 | |
|  * general info
 | |
|  */
 | |
| struct pv_info {
 | |
| 	unsigned int kernel_rpl;
 | |
| 	int paravirt_enabled;
 | |
| 	const char *name;
 | |
| };
 | |
| 
 | |
| extern struct pv_info pv_info;
 | |
| 
 | |
| static inline int paravirt_enabled(void)
 | |
| {
 | |
| 	return pv_info.paravirt_enabled;
 | |
| }
 | |
| 
 | |
| static inline unsigned int get_kernel_rpl(void)
 | |
| {
 | |
| 	return pv_info.kernel_rpl;
 | |
| }
 | |
| 
 | |
| /******************************************************************************
 | |
|  * initialization hooks.
 | |
|  */
 | |
| struct rsvd_region;
 | |
| 
 | |
| struct pv_init_ops {
 | |
| 	void (*banner)(void);
 | |
| 
 | |
| 	int (*reserve_memory)(struct rsvd_region *region);
 | |
| 
 | |
| 	void (*arch_setup_early)(void);
 | |
| 	void (*arch_setup_console)(char **cmdline_p);
 | |
| 	int (*arch_setup_nomca)(void);
 | |
| 
 | |
| 	void (*post_smp_prepare_boot_cpu)(void);
 | |
| 
 | |
| #ifdef ASM_SUPPORTED
 | |
| 	unsigned long (*patch_bundle)(void *sbundle, void *ebundle,
 | |
| 				      unsigned long type);
 | |
| 	unsigned long (*patch_inst)(unsigned long stag, unsigned long etag,
 | |
| 				    unsigned long type);
 | |
| #endif
 | |
| 	void (*patch_branch)(unsigned long tag, unsigned long type);
 | |
| };
 | |
| 
 | |
| extern struct pv_init_ops pv_init_ops;
 | |
| 
 | |
| static inline void paravirt_banner(void)
 | |
| {
 | |
| 	if (pv_init_ops.banner)
 | |
| 		pv_init_ops.banner();
 | |
| }
 | |
| 
 | |
| static inline int paravirt_reserve_memory(struct rsvd_region *region)
 | |
| {
 | |
| 	if (pv_init_ops.reserve_memory)
 | |
| 		return pv_init_ops.reserve_memory(region);
 | |
| 	return 0;
 | |
| }
 | |
| 
 | |
| static inline void paravirt_arch_setup_early(void)
 | |
| {
 | |
| 	if (pv_init_ops.arch_setup_early)
 | |
| 		pv_init_ops.arch_setup_early();
 | |
| }
 | |
| 
 | |
| static inline void paravirt_arch_setup_console(char **cmdline_p)
 | |
| {
 | |
| 	if (pv_init_ops.arch_setup_console)
 | |
| 		pv_init_ops.arch_setup_console(cmdline_p);
 | |
| }
 | |
| 
 | |
| static inline int paravirt_arch_setup_nomca(void)
 | |
| {
 | |
| 	if (pv_init_ops.arch_setup_nomca)
 | |
| 		return pv_init_ops.arch_setup_nomca();
 | |
| 	return 0;
 | |
| }
 | |
| 
 | |
| static inline void paravirt_post_smp_prepare_boot_cpu(void)
 | |
| {
 | |
| 	if (pv_init_ops.post_smp_prepare_boot_cpu)
 | |
| 		pv_init_ops.post_smp_prepare_boot_cpu();
 | |
| }
 | |
| 
 | |
| /******************************************************************************
 | |
|  * replacement of iosapic operations.
 | |
|  */
 | |
| 
 | |
| struct pv_iosapic_ops {
 | |
| 	void (*pcat_compat_init)(void);
 | |
| 
 | |
| 	struct irq_chip *(*__get_irq_chip)(unsigned long trigger);
 | |
| 
 | |
| 	unsigned int (*__read)(char __iomem *iosapic, unsigned int reg);
 | |
| 	void (*__write)(char __iomem *iosapic, unsigned int reg, u32 val);
 | |
| };
 | |
| 
 | |
| extern struct pv_iosapic_ops pv_iosapic_ops;
 | |
| 
 | |
| static inline void
 | |
| iosapic_pcat_compat_init(void)
 | |
| {
 | |
| 	if (pv_iosapic_ops.pcat_compat_init)
 | |
| 		pv_iosapic_ops.pcat_compat_init();
 | |
| }
 | |
| 
 | |
| static inline struct irq_chip*
 | |
| iosapic_get_irq_chip(unsigned long trigger)
 | |
| {
 | |
| 	return pv_iosapic_ops.__get_irq_chip(trigger);
 | |
| }
 | |
| 
 | |
| static inline unsigned int
 | |
| __iosapic_read(char __iomem *iosapic, unsigned int reg)
 | |
| {
 | |
| 	return pv_iosapic_ops.__read(iosapic, reg);
 | |
| }
 | |
| 
 | |
| static inline void
 | |
| __iosapic_write(char __iomem *iosapic, unsigned int reg, u32 val)
 | |
| {
 | |
| 	return pv_iosapic_ops.__write(iosapic, reg, val);
 | |
| }
 | |
| 
 | |
| /******************************************************************************
 | |
|  * replacement of irq operations.
 | |
|  */
 | |
| 
 | |
| struct pv_irq_ops {
 | |
| 	void (*register_ipi)(void);
 | |
| 
 | |
| 	int (*assign_irq_vector)(int irq);
 | |
| 	void (*free_irq_vector)(int vector);
 | |
| 
 | |
| 	void (*register_percpu_irq)(ia64_vector vec,
 | |
| 				    struct irqaction *action);
 | |
| 
 | |
| 	void (*resend_irq)(unsigned int vector);
 | |
| };
 | |
| 
 | |
| extern struct pv_irq_ops pv_irq_ops;
 | |
| 
 | |
| static inline void
 | |
| ia64_register_ipi(void)
 | |
| {
 | |
| 	pv_irq_ops.register_ipi();
 | |
| }
 | |
| 
 | |
| static inline int
 | |
| assign_irq_vector(int irq)
 | |
| {
 | |
| 	return pv_irq_ops.assign_irq_vector(irq);
 | |
| }
 | |
| 
 | |
| static inline void
 | |
| free_irq_vector(int vector)
 | |
| {
 | |
| 	return pv_irq_ops.free_irq_vector(vector);
 | |
| }
 | |
| 
 | |
| static inline void
 | |
| register_percpu_irq(ia64_vector vec, struct irqaction *action)
 | |
| {
 | |
| 	pv_irq_ops.register_percpu_irq(vec, action);
 | |
| }
 | |
| 
 | |
| static inline void
 | |
| ia64_resend_irq(unsigned int vector)
 | |
| {
 | |
| 	pv_irq_ops.resend_irq(vector);
 | |
| }
 | |
| 
 | |
| /******************************************************************************
 | |
|  * replacement of time operations.
 | |
|  */
 | |
| 
 | |
| extern struct itc_jitter_data_t itc_jitter_data;
 | |
| extern volatile int time_keeper_id;
 | |
| 
 | |
| struct pv_time_ops {
 | |
| 	void (*init_missing_ticks_accounting)(int cpu);
 | |
| 	int (*do_steal_accounting)(unsigned long *new_itm);
 | |
| 
 | |
| 	void (*clocksource_resume)(void);
 | |
| 
 | |
| 	unsigned long long (*sched_clock)(void);
 | |
| };
 | |
| 
 | |
| extern struct pv_time_ops pv_time_ops;
 | |
| 
 | |
| static inline void
 | |
| paravirt_init_missing_ticks_accounting(int cpu)
 | |
| {
 | |
| 	if (pv_time_ops.init_missing_ticks_accounting)
 | |
| 		pv_time_ops.init_missing_ticks_accounting(cpu);
 | |
| }
 | |
| 
 | |
| struct static_key;
 | |
| extern struct static_key paravirt_steal_enabled;
 | |
| extern struct static_key paravirt_steal_rq_enabled;
 | |
| 
 | |
| static inline int
 | |
| paravirt_do_steal_accounting(unsigned long *new_itm)
 | |
| {
 | |
| 	return pv_time_ops.do_steal_accounting(new_itm);
 | |
| }
 | |
| 
 | |
| static inline unsigned long long paravirt_sched_clock(void)
 | |
| {
 | |
| 	return pv_time_ops.sched_clock();
 | |
| }
 | |
| 
 | |
| #endif /* !__ASSEMBLY__ */
 | |
| 
 | |
| #else
 | |
| /* fallback for native case */
 | |
| 
 | |
| #ifndef __ASSEMBLY__
 | |
| 
 | |
| #define paravirt_banner()				do { } while (0)
 | |
| #define paravirt_reserve_memory(region)			0
 | |
| 
 | |
| #define paravirt_arch_setup_early()			do { } while (0)
 | |
| #define paravirt_arch_setup_console(cmdline_p)		do { } while (0)
 | |
| #define paravirt_arch_setup_nomca()			0
 | |
| #define paravirt_post_smp_prepare_boot_cpu()		do { } while (0)
 | |
| 
 | |
| #define paravirt_init_missing_ticks_accounting(cpu)	do { } while (0)
 | |
| #define paravirt_do_steal_accounting(new_itm)		0
 | |
| 
 | |
| #endif /* __ASSEMBLY__ */
 | |
| 
 | |
| 
 | |
| #endif /* CONFIG_PARAVIRT_GUEST */
 | |
| 
 | |
| #endif /* __ASM_PARAVIRT_H */
 |