forge/linux-pinenote
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

powerpc: Add VDSO version of getcpu

Browse source 
We have a request for a fast method of getting CPU and NUMA node IDs
from userspace. This patch implements a getcpu VDSO function,
similar to x86.

Ben suggested we use SPRG3 which is userspace readable. SPRG3 can be
modified by a KVM guest, so we save the SPRG3 value in the paca and
restore it when transitioning from the guest to the host.

I have a glibc patch that implements sched_getcpu on top of this.
Testing on a POWER7:

baseline: 538 cycles
vdso:      30 cycles

Signed-off-by: Anton Blanchard <anton@samba.org>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
This commit is contained in:
Anton Blanchard 2012-07-04 20:37:11 +00:00 • committed by Benjamin Herrenschmidt
commit 18ad51dd34
13 changed files with 140 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

1
arch/powerpc/include/asm/kvm_book3s_asm.h
View file

@ -74,6 +74,7 @@ struct kvmppc_host_state {
ulong vmhandler;
ulong scratch0;
ulong scratch1;
ulong sprg3;
u8 in_guest;
u8 restore_hid5;
u8 napping;

5
arch/powerpc/include/asm/reg.h
View file

@ -491,6 +491,7 @@
#define SPRN_SPRG1 0x111 /* Special Purpose Register General 1 */
#define SPRN_SPRG2 0x112 /* Special Purpose Register General 2 */
#define SPRN_SPRG3 0x113 /* Special Purpose Register General 3 */
#define SPRN_USPRG3 0x103 /* SPRG3 userspace read */
#define SPRN_SPRG4 0x114 /* Special Purpose Register General 4 */
#define SPRN_SPRG5 0x115 /* Special Purpose Register General 5 */
#define SPRN_SPRG6 0x116 /* Special Purpose Register General 6 */
@ -753,14 +754,14 @@
* 64-bit server:
* - SPRG0 unused (reserved for HV on Power4)
* - SPRG2 scratch for exception vectors
* - SPRG3 unused (user visible)
* - SPRG3 CPU and NUMA node for VDSO getcpu (user visible)
* - HSPRG0 stores PACA in HV mode
* - HSPRG1 scratch for "HV" exceptions
*
* 64-bit embedded
* - SPRG0 generic exception scratch
* - SPRG2 TLB exception stack
* - SPRG3 unused (user visible)
* - SPRG3 CPU and NUMA node for VDSO getcpu (user visible)
* - SPRG4 unused (user visible)
* - SPRG6 TLB miss scratch (user visible, sorry !)
* - SPRG7 critical exception scratch

2
arch/powerpc/include/asm/vdso.h
View file

@ -22,6 +22,8 @@ extern unsigned long vdso64_rt_sigtramp;
extern unsigned long vdso32_sigtramp;
extern unsigned long vdso32_rt_sigtramp;
int __cpuinit vdso_getcpu_init(void);
#else /* __ASSEMBLY__ */
#ifdef __VDSO64__

1
arch/powerpc/kernel/asm-offsets.c
View file

@ -533,6 +533,7 @@ int main(void)
HSTATE_FIELD(HSTATE_VMHANDLER, vmhandler);
HSTATE_FIELD(HSTATE_SCRATCH0, scratch0);
HSTATE_FIELD(HSTATE_SCRATCH1, scratch1);
HSTATE_FIELD(HSTATE_SPRG3, sprg3);
HSTATE_FIELD(HSTATE_IN_GUEST, in_guest);
HSTATE_FIELD(HSTATE_RESTORE_HID5, restore_hid5);
HSTATE_FIELD(HSTATE_NAPPING, napping);

3
arch/powerpc/kernel/smp.c
View file

@ -48,6 +48,7 @@
#ifdef CONFIG_PPC64
#include <asm/paca.h>
#endif
#include <asm/vdso.h>
#include <asm/debug.h>
#ifdef DEBUG
@ -570,6 +571,8 @@ void __devinit start_secondary(void *unused)
#ifdef CONFIG_PPC64
if (system_state == SYSTEM_RUNNING)
vdso_data->processorCount++;
vdso_getcpu_init();
#endif
notify_cpu_starting(cpu);
set_cpu_online(cpu, true);

28
arch/powerpc/kernel/vdso.c
View file

@ -706,6 +706,34 @@ static void __init vdso_setup_syscall_map(void)
}
}
#ifdef CONFIG_PPC64
int __cpuinit vdso_getcpu_init(void)
{
unsigned long cpu, node, val;
/*
* SPRG3 contains the CPU in the bottom 16 bits and the NUMA node in
* the next 16 bits. The VDSO uses this to implement getcpu().
*/
cpu = get_cpu();
WARN_ON_ONCE(cpu > 0xffff);
node = cpu_to_node(cpu);
WARN_ON_ONCE(node > 0xffff);
val = (cpu & 0xfff) | ((node & 0xffff) << 16);
mtspr(SPRN_SPRG3, val);
#ifdef CONFIG_KVM_BOOK3S_HANDLER
get_paca()->kvm_hstate.sprg3 = val;
#endif
put_cpu();
return 0;
}
/* We need to call this before SMP init */
early_initcall(vdso_getcpu_init);
#endif
static int __init vdso_init(void)
{

4
arch/powerpc/kernel/vdso32/Makefile
View file

@ -1,7 +1,9 @@
# List of files in the vdso, has to be asm only for now
obj-vdso32 = sigtramp.o gettimeofday.o datapage.o cacheflush.o note.o
obj-vdso32-$(CONFIG_PPC64) = getcpu.o
obj-vdso32 = sigtramp.o gettimeofday.o datapage.o cacheflush.o note.o \
$(obj-vdso32-y)
# Build rules

45
arch/powerpc/kernel/vdso32/getcpu.S Normal file
View file

@ -0,0 +1,45 @@
/*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*
* Copyright (C) IBM Corporation, 2012
*
* Author: Anton Blanchard <anton@au.ibm.com>
*/
#include <asm/ppc_asm.h>
#include <asm/vdso.h>
.text
/*
* Exact prototype of getcpu
*
* int __kernel_getcpu(unsigned *cpu, unsigned *node);
*
*/
V_FUNCTION_BEGIN(__kernel_getcpu)
.cfi_startproc
mfspr r5,SPRN_USPRG3
cmpdi cr0,r3,0
cmpdi cr1,r4,0
clrlwi r6,r5,16
rlwinm r7,r5,16,31-15,31-0
beq cr0,1f
stw r6,0(r3)
1: beq cr1,2f
stw r7,0(r4)
2: crclr cr0*4+so
li r3,0 /* always success */
blr
.cfi_endproc
V_FUNCTION_END(__kernel_getcpu)

3
arch/powerpc/kernel/vdso32/vdso32.lds.S
View file

@ -147,6 +147,9 @@ VERSION
__kernel_sync_dicache_p5;
__kernel_sigtramp32;
__kernel_sigtramp_rt32;
#ifdef CONFIG_PPC64
__kernel_getcpu;
#endif
local: *;
};

2
arch/powerpc/kernel/vdso64/Makefile
View file

@ -1,6 +1,6 @@
# List of files in the vdso, has to be asm only for now
obj-vdso64 = sigtramp.o gettimeofday.o datapage.o cacheflush.o note.o
obj-vdso64 = sigtramp.o gettimeofday.o datapage.o cacheflush.o note.o getcpu.o
# Build rules

45
arch/powerpc/kernel/vdso64/getcpu.S Normal file
View file

@ -0,0 +1,45 @@
/*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*
* Copyright (C) IBM Corporation, 2012
*
* Author: Anton Blanchard <anton@au.ibm.com>
*/
#include <asm/ppc_asm.h>
#include <asm/vdso.h>
.text
/*
* Exact prototype of getcpu
*
* int __kernel_getcpu(unsigned *cpu, unsigned *node);
*
*/
V_FUNCTION_BEGIN(__kernel_getcpu)
.cfi_startproc
mfspr r5,SPRN_USPRG3
cmpdi cr0,r3,0
cmpdi cr1,r4,0
clrlwi r6,r5,16
rlwinm r7,r5,16,31-15,31-0
beq cr0,1f
stw r6,0(r3)
1: beq cr1,2f
stw r7,0(r4)
2: crclr cr0*4+so
li r3,0 /* always success */
blr
.cfi_endproc
V_FUNCTION_END(__kernel_getcpu)

1
arch/powerpc/kernel/vdso64/vdso64.lds.S
View file

@ -146,6 +146,7 @@ VERSION
__kernel_sync_dicache;
__kernel_sync_dicache_p5;
__kernel_sigtramp_rt64;
__kernel_getcpu;
local: *;
};

4
arch/powerpc/kvm/book3s_hv_rmhandlers.S
View file

@ -1064,6 +1064,10 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
mtspr SPRN_DABR,r5
mtspr SPRN_DABRX,r6
/* Restore SPRG3 */
ld r3,HSTATE_SPRG3(r13)
mtspr SPRN_SPRG3,r3
/*
* Reload DEC. HDEC interrupts were disabled when
* we reloaded the host's LPCR value.