In order to optimize the spinning step, we need to set the lock owner as soon as the lock is acquired; after a successful counter cmpxchg operation, that is. This is particularly useful as rwsems need to set the owner to nil for readers, so there is a greater chance of falling out of the spinning. Currently we only set the owner much later in the game, in the more generic level -- latency can be specially bad when waiting for a node->next pointer when releasing the osq in up_write calls. As such, update the owner inside rwsem_try_write_lock (when the lock is obtained after blocking) and rwsem_try_write_lock_unqueued (when the lock is obtained while spinning). This requires creating a new internal rwsem.h header to share the owner related calls. Also cleanup some headers for mutex and rwsem. Suggested-by: Peter Zijlstra <peterz@infradead.org> Signed-off-by: Davidlohr Bueso <dbueso@suse.de> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Cc: Jason Low <jason.low2@hp.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Michel Lespinasse <walken@google.com> Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Cc: Tim Chen <tim.c.chen@linux.intel.com> Link: http://lkml.kernel.org/r/1422609267-15102-4-git-send-email-dave@stgolabs.net Signed-off-by: Ingo Molnar <mingo@kernel.org>
		
			
				
	
	
		
			166 lines
		
	
	
	
		
			3 KiB
			
		
	
	
	
		
			C
		
	
	
	
	
	
			
		
		
	
	
			166 lines
		
	
	
	
		
			3 KiB
			
		
	
	
	
		
			C
		
	
	
	
	
	
/* kernel/rwsem.c: R/W semaphores, public implementation
 | 
						|
 *
 | 
						|
 * Written by David Howells (dhowells@redhat.com).
 | 
						|
 * Derived from asm-i386/semaphore.h
 | 
						|
 */
 | 
						|
 | 
						|
#include <linux/types.h>
 | 
						|
#include <linux/kernel.h>
 | 
						|
#include <linux/sched.h>
 | 
						|
#include <linux/export.h>
 | 
						|
#include <linux/rwsem.h>
 | 
						|
#include <linux/atomic.h>
 | 
						|
 | 
						|
#include "rwsem.h"
 | 
						|
 | 
						|
/*
 | 
						|
 * lock for reading
 | 
						|
 */
 | 
						|
void __sched down_read(struct rw_semaphore *sem)
 | 
						|
{
 | 
						|
	might_sleep();
 | 
						|
	rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);
 | 
						|
 | 
						|
	LOCK_CONTENDED(sem, __down_read_trylock, __down_read);
 | 
						|
}
 | 
						|
 | 
						|
EXPORT_SYMBOL(down_read);
 | 
						|
 | 
						|
/*
 | 
						|
 * trylock for reading -- returns 1 if successful, 0 if contention
 | 
						|
 */
 | 
						|
int down_read_trylock(struct rw_semaphore *sem)
 | 
						|
{
 | 
						|
	int ret = __down_read_trylock(sem);
 | 
						|
 | 
						|
	if (ret == 1)
 | 
						|
		rwsem_acquire_read(&sem->dep_map, 0, 1, _RET_IP_);
 | 
						|
	return ret;
 | 
						|
}
 | 
						|
 | 
						|
EXPORT_SYMBOL(down_read_trylock);
 | 
						|
 | 
						|
/*
 | 
						|
 * lock for writing
 | 
						|
 */
 | 
						|
void __sched down_write(struct rw_semaphore *sem)
 | 
						|
{
 | 
						|
	might_sleep();
 | 
						|
	rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);
 | 
						|
 | 
						|
	LOCK_CONTENDED(sem, __down_write_trylock, __down_write);
 | 
						|
	rwsem_set_owner(sem);
 | 
						|
}
 | 
						|
 | 
						|
EXPORT_SYMBOL(down_write);
 | 
						|
 | 
						|
/*
 | 
						|
 * trylock for writing -- returns 1 if successful, 0 if contention
 | 
						|
 */
 | 
						|
int down_write_trylock(struct rw_semaphore *sem)
 | 
						|
{
 | 
						|
	int ret = __down_write_trylock(sem);
 | 
						|
 | 
						|
	if (ret == 1) {
 | 
						|
		rwsem_acquire(&sem->dep_map, 0, 1, _RET_IP_);
 | 
						|
		rwsem_set_owner(sem);
 | 
						|
	}
 | 
						|
 | 
						|
	return ret;
 | 
						|
}
 | 
						|
 | 
						|
EXPORT_SYMBOL(down_write_trylock);
 | 
						|
 | 
						|
/*
 | 
						|
 * release a read lock
 | 
						|
 */
 | 
						|
void up_read(struct rw_semaphore *sem)
 | 
						|
{
 | 
						|
	rwsem_release(&sem->dep_map, 1, _RET_IP_);
 | 
						|
 | 
						|
	__up_read(sem);
 | 
						|
}
 | 
						|
 | 
						|
EXPORT_SYMBOL(up_read);
 | 
						|
 | 
						|
/*
 | 
						|
 * release a write lock
 | 
						|
 */
 | 
						|
void up_write(struct rw_semaphore *sem)
 | 
						|
{
 | 
						|
	rwsem_release(&sem->dep_map, 1, _RET_IP_);
 | 
						|
 | 
						|
	rwsem_clear_owner(sem);
 | 
						|
	__up_write(sem);
 | 
						|
}
 | 
						|
 | 
						|
EXPORT_SYMBOL(up_write);
 | 
						|
 | 
						|
/*
 | 
						|
 * downgrade write lock to read lock
 | 
						|
 */
 | 
						|
void downgrade_write(struct rw_semaphore *sem)
 | 
						|
{
 | 
						|
	/*
 | 
						|
	 * lockdep: a downgraded write will live on as a write
 | 
						|
	 * dependency.
 | 
						|
	 */
 | 
						|
	rwsem_clear_owner(sem);
 | 
						|
	__downgrade_write(sem);
 | 
						|
}
 | 
						|
 | 
						|
EXPORT_SYMBOL(downgrade_write);
 | 
						|
 | 
						|
#ifdef CONFIG_DEBUG_LOCK_ALLOC
 | 
						|
 | 
						|
void down_read_nested(struct rw_semaphore *sem, int subclass)
 | 
						|
{
 | 
						|
	might_sleep();
 | 
						|
	rwsem_acquire_read(&sem->dep_map, subclass, 0, _RET_IP_);
 | 
						|
 | 
						|
	LOCK_CONTENDED(sem, __down_read_trylock, __down_read);
 | 
						|
}
 | 
						|
 | 
						|
EXPORT_SYMBOL(down_read_nested);
 | 
						|
 | 
						|
void _down_write_nest_lock(struct rw_semaphore *sem, struct lockdep_map *nest)
 | 
						|
{
 | 
						|
	might_sleep();
 | 
						|
	rwsem_acquire_nest(&sem->dep_map, 0, 0, nest, _RET_IP_);
 | 
						|
 | 
						|
	LOCK_CONTENDED(sem, __down_write_trylock, __down_write);
 | 
						|
	rwsem_set_owner(sem);
 | 
						|
}
 | 
						|
 | 
						|
EXPORT_SYMBOL(_down_write_nest_lock);
 | 
						|
 | 
						|
void down_read_non_owner(struct rw_semaphore *sem)
 | 
						|
{
 | 
						|
	might_sleep();
 | 
						|
 | 
						|
	__down_read(sem);
 | 
						|
}
 | 
						|
 | 
						|
EXPORT_SYMBOL(down_read_non_owner);
 | 
						|
 | 
						|
void down_write_nested(struct rw_semaphore *sem, int subclass)
 | 
						|
{
 | 
						|
	might_sleep();
 | 
						|
	rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_);
 | 
						|
 | 
						|
	LOCK_CONTENDED(sem, __down_write_trylock, __down_write);
 | 
						|
	rwsem_set_owner(sem);
 | 
						|
}
 | 
						|
 | 
						|
EXPORT_SYMBOL(down_write_nested);
 | 
						|
 | 
						|
void up_read_non_owner(struct rw_semaphore *sem)
 | 
						|
{
 | 
						|
	__up_read(sem);
 | 
						|
}
 | 
						|
 | 
						|
EXPORT_SYMBOL(up_read_non_owner);
 | 
						|
 | 
						|
#endif
 | 
						|
 | 
						|
 |