79e5f05edc
ARC irqsave/restore macros were missing the compiler barrier, causing a
stale load in irq-enabled region be used in irq-safe region, despite
being changed, because the register holding the value was still live.
The problem manifested as random crashes in timer code when stress
testing ARCLinux (3.9-rc3) on a !SMP && !PREEMPT_COUNT
Here's the exact sequence which caused this:
(0). tv1[x] <----> t1 <---> t2
(1). mod_timer(t1) interrupted after it calls timer_pending()
(2). mod_timer(t2) completes
(3). mod_timer(t1) resumes but messes up the list
(4). __runt_timers( ) uses bogus timer_list entry / crashes in
timer->function
Essentially mod_timer() was racing against itself and while the spinlock
serialized the tv1[] timer link list, timer_pending() called outside the
spinlock, cached timer link list element in a register.
With low register pressure (and a deep register file), lack of barrier
in raw_local_irqsave() as well as preempt_disable (!PREEMPT_COUNT
version), there was nothing to force gcc to reload across the spinlock,
causing a stale value in reg be used for link list manipulation - ensuing
a corruption.
ARcompact disassembly which shows the culprit generated code:
mod_timer:
push_s blink
mov_s r13,r0 # timer, timer
..
###### timer_pending( )
ld_s r3,[r13] # <------ <variable>.entry.next LOADED
brne r3, 0, @.L163
.L163:
..
###### spin_lock_irq( )
lr r5, [status32] # flags
bic r4, r5, 6 # temp, flags,
and.f 0, r5, 6 # flags,
flag.nz r4
###### detach_if_pending( ) begins
tst_s r3,r3 <--------------
# timer_pending( ) checks timer->entry.next
# r3 is NOT reloaded by gcc, using stale value
beq.d @.L169
mov.eq r0,0
##### detach_timer( ): __list_del( )
ld r4,[r13,4] # <variable>.entry.prev, D.31439
st r4,[r3,4] # <variable>.prev, D.31439
st r3,[r4] # <variable>.next, D.30246
We initially tried to fix this by adding barrier() to preempt_* macros
for !PREEMPT_COUNT but Linus clarified that it was anything but wrong.
http://www.spinics.net/lists/kernel/msg1512709.html
[vgupta: updated commitlog]
Reported-by/Signed-off-by: Christian Ruppert <christian.ruppert@abilis.com>
Cc: Christian Ruppert <christian.ruppert@abilis.com>
Cc: Pierrick Hascoet <pierrick.hascoet@abilis.com>
Debugged-by/Signed-off-by: Vineet Gupta <vgupta@synopsys.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
157 lines
3 KiB
C
157 lines
3 KiB
C
/*
|
|
* Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
* published by the Free Software Foundation.
|
|
*/
|
|
|
|
#ifndef __ASM_ARC_IRQFLAGS_H
|
|
#define __ASM_ARC_IRQFLAGS_H
|
|
|
|
/* vineetg: March 2010 : local_irq_save( ) optimisation
|
|
* -Remove explicit mov of current status32 into reg, that is not needed
|
|
* -Use BIC insn instead of INVERTED + AND
|
|
* -Conditionally disable interrupts (if they are not enabled, don't disable)
|
|
*/
|
|
|
|
#ifdef __KERNEL__
|
|
|
|
#include <asm/arcregs.h>
|
|
|
|
#ifndef __ASSEMBLY__
|
|
|
|
/******************************************************************
|
|
* IRQ Control Macros
|
|
******************************************************************/
|
|
|
|
/*
|
|
* Save IRQ state and disable IRQs
|
|
*/
|
|
static inline long arch_local_irq_save(void)
|
|
{
|
|
unsigned long temp, flags;
|
|
|
|
__asm__ __volatile__(
|
|
" lr %1, [status32] \n"
|
|
" bic %0, %1, %2 \n"
|
|
" and.f 0, %1, %2 \n"
|
|
" flag.nz %0 \n"
|
|
: "=r"(temp), "=r"(flags)
|
|
: "n"((STATUS_E1_MASK | STATUS_E2_MASK))
|
|
: "memory", "cc");
|
|
|
|
return flags;
|
|
}
|
|
|
|
/*
|
|
* restore saved IRQ state
|
|
*/
|
|
static inline void arch_local_irq_restore(unsigned long flags)
|
|
{
|
|
|
|
__asm__ __volatile__(
|
|
" flag %0 \n"
|
|
:
|
|
: "r"(flags)
|
|
: "memory");
|
|
}
|
|
|
|
/*
|
|
* Unconditionally Enable IRQs
|
|
*/
|
|
extern void arch_local_irq_enable(void);
|
|
|
|
/*
|
|
* Unconditionally Disable IRQs
|
|
*/
|
|
static inline void arch_local_irq_disable(void)
|
|
{
|
|
unsigned long temp;
|
|
|
|
__asm__ __volatile__(
|
|
" lr %0, [status32] \n"
|
|
" and %0, %0, %1 \n"
|
|
" flag %0 \n"
|
|
: "=&r"(temp)
|
|
: "n"(~(STATUS_E1_MASK | STATUS_E2_MASK))
|
|
: "memory");
|
|
}
|
|
|
|
/*
|
|
* save IRQ state
|
|
*/
|
|
static inline long arch_local_save_flags(void)
|
|
{
|
|
unsigned long temp;
|
|
|
|
__asm__ __volatile__(
|
|
" lr %0, [status32] \n"
|
|
: "=&r"(temp)
|
|
:
|
|
: "memory");
|
|
|
|
return temp;
|
|
}
|
|
|
|
/*
|
|
* Query IRQ state
|
|
*/
|
|
static inline int arch_irqs_disabled_flags(unsigned long flags)
|
|
{
|
|
return !(flags & (STATUS_E1_MASK
|
|
#ifdef CONFIG_ARC_COMPACT_IRQ_LEVELS
|
|
| STATUS_E2_MASK
|
|
#endif
|
|
));
|
|
}
|
|
|
|
static inline int arch_irqs_disabled(void)
|
|
{
|
|
return arch_irqs_disabled_flags(arch_local_save_flags());
|
|
}
|
|
|
|
static inline void arch_mask_irq(unsigned int irq)
|
|
{
|
|
unsigned int ienb;
|
|
|
|
ienb = read_aux_reg(AUX_IENABLE);
|
|
ienb &= ~(1 << irq);
|
|
write_aux_reg(AUX_IENABLE, ienb);
|
|
}
|
|
|
|
static inline void arch_unmask_irq(unsigned int irq)
|
|
{
|
|
unsigned int ienb;
|
|
|
|
ienb = read_aux_reg(AUX_IENABLE);
|
|
ienb |= (1 << irq);
|
|
write_aux_reg(AUX_IENABLE, ienb);
|
|
}
|
|
|
|
#else
|
|
|
|
.macro IRQ_DISABLE scratch
|
|
lr \scratch, [status32]
|
|
bic \scratch, \scratch, (STATUS_E1_MASK | STATUS_E2_MASK)
|
|
flag \scratch
|
|
.endm
|
|
|
|
.macro IRQ_DISABLE_SAVE scratch, save
|
|
lr \scratch, [status32]
|
|
mov \save, \scratch /* Make a copy */
|
|
bic \scratch, \scratch, (STATUS_E1_MASK | STATUS_E2_MASK)
|
|
flag \scratch
|
|
.endm
|
|
|
|
.macro IRQ_ENABLE scratch
|
|
lr \scratch, [status32]
|
|
or \scratch, \scratch, (STATUS_E1_MASK | STATUS_E2_MASK)
|
|
flag \scratch
|
|
.endm
|
|
|
|
#endif /* __ASSEMBLY__ */
|
|
|
|
#endif /* KERNEL */
|
|
|
|
#endif
|