From 8d496bf981b2cacb9c21ccdb66a9991978c986ea Mon Sep 17 00:00:00 2001 From: Matthew Dillon Date: Mon, 1 Mar 2010 16:09:41 -0800 Subject: [PATCH] kernel - Move grow_stack code in fault path to improve fault performance * grow_stack() calls were being performed unconditionally during a page fault. Shift the code over to vm_fault() and only run it if the vm_map_lookup() call fails. * Also take this opportunity to remove trapwrite() (added by Matt). Submitted-by: Venkatesh Srinivas --- sys/emulation/linux/i386/linux_sysvec.c | 4 +- sys/platform/pc32/i386/trap.c | 60 ------------------------ sys/platform/pc32/i386/vm_machdep.c | 12 ----- sys/platform/pc64/x86_64/trap.c | 15 ------ sys/platform/pc64/x86_64/vm_machdep.c | 12 ----- sys/platform/vkernel/i386/trap.c | 61 ------------------------- sys/platform/vkernel/i386/vm_machdep.c | 12 ----- sys/vm/vm_extern.h | 1 - sys/vm/vm_fault.c | 19 ++++++-- 9 files changed, 17 insertions(+), 179 deletions(-) diff --git a/sys/emulation/linux/i386/linux_sysvec.c b/sys/emulation/linux/i386/linux_sysvec.c index c136b4c181..41a5a1a73c 100644 --- a/sys/emulation/linux/i386/linux_sysvec.c +++ b/sys/emulation/linux/i386/linux_sysvec.c @@ -279,7 +279,7 @@ linux_rt_sendsig(sig_t catcher, int sig, sigset_t *mask, u_long code) * and the stack can not be grown. useracc will return FALSE * if access is denied. */ - if ((grow_stack (p, (int)fp) == FALSE) || + if ((vm_map_growstack(p, (vm_offset_t)fp) != KERN_SUCCESS) || !useracc((caddr_t)fp, sizeof (struct l_rt_sigframe), VM_PROT_WRITE)) { /* @@ -442,7 +442,7 @@ linux_sendsig(sig_t catcher, int sig, sigset_t *mask, u_long code) * and the stack can not be grown. useracc will return FALSE * if access is denied. */ - if ((grow_stack (p, (int)fp) == FALSE) || + if ((vm_map_growstack(p, (vm_offset_t)fp) != KERN_SUCCESS) || !useracc((caddr_t)fp, sizeof (struct l_sigframe), VM_PROT_WRITE)) { /* diff --git a/sys/platform/pc32/i386/trap.c b/sys/platform/pc32/i386/trap.c index 194c627a73..bd6c3f59f0 100644 --- a/sys/platform/pc32/i386/trap.c +++ b/sys/platform/pc32/i386/trap.c @@ -124,7 +124,6 @@ int (*pmath_emulate) (struct trapframe *); extern void trap (struct trapframe *frame); -extern int trapwrite (unsigned addr); extern void syscall2 (struct trapframe *frame); static int trap_pfault (struct trapframe *, int, vm_offset_t); @@ -988,21 +987,6 @@ trap_pfault(struct trapframe *frame, int usermode, vm_offset_t eva) */ PHOLD(lp->lwp_proc); - /* - * Grow the stack if necessary - */ - /* grow_stack returns false only if va falls into - * a growable stack region and the stack growth - * fails. It returns true if va was not within - * a growable stack region, or if the stack - * growth succeeded. - */ - if (!grow_stack(lp->lwp_proc, va)) { - rv = KERN_FAILURE; - PRELE(lp->lwp_proc); - goto nogo; - } - /* * Issue fault */ @@ -1166,50 +1150,6 @@ dblfault_handler(void) panic("double fault"); } -/* - * Compensate for 386 brain damage (missing URKR). - * This is a little simpler than the pagefault handler in trap() because - * it the page tables have already been faulted in and high addresses - * are thrown out early for other reasons. - */ -int -trapwrite(unsigned addr) -{ - struct lwp *lp; - vm_offset_t va; - struct vmspace *vm; - int rv; - - va = trunc_page((vm_offset_t)addr); - /* - * XXX - MAX is END. Changed > to >= for temp. fix. - */ - if (va >= VM_MAX_USER_ADDRESS) - return (1); - - lp = curthread->td_lwp; - vm = lp->lwp_vmspace; - - PHOLD(lp->lwp_proc); - - if (!grow_stack(lp->lwp_proc, va)) { - PRELE(lp->lwp_proc); - return (1); - } - - /* - * fault the data page - */ - rv = vm_fault(&vm->vm_map, va, VM_PROT_WRITE, VM_FAULT_DIRTY); - - PRELE(lp->lwp_proc); - - if (rv != KERN_SUCCESS) - return 1; - - return (0); -} - /* * syscall2 - MP aware system call request C handler * diff --git a/sys/platform/pc32/i386/vm_machdep.c b/sys/platform/pc32/i386/vm_machdep.c index a59a2a0ded..80d5d14cbd 100644 --- a/sys/platform/pc32/i386/vm_machdep.c +++ b/sys/platform/pc32/i386/vm_machdep.c @@ -472,18 +472,6 @@ cpu_reset_real(void) while(1); } -int -grow_stack(struct proc *p, vm_offset_t sp) -{ - int rv; - - rv = vm_map_growstack (p, sp); - if (rv != KERN_SUCCESS) - return (0); - - return (1); -} - SYSCTL_DECL(_vm_stats_misc); static int cnt_prezero; diff --git a/sys/platform/pc64/x86_64/trap.c b/sys/platform/pc64/x86_64/trap.c index 0e02a10023..08afdeec51 100644 --- a/sys/platform/pc64/x86_64/trap.c +++ b/sys/platform/pc64/x86_64/trap.c @@ -856,21 +856,6 @@ trap_pfault(struct trapframe *frame, int usermode) */ PHOLD(lp->lwp_proc); - /* - * Grow the stack if necessary - */ - /* grow_stack returns false only if va falls into - * a growable stack region and the stack growth - * fails. It returns true if va was not within - * a growable stack region, or if the stack - * growth succeeded. - */ - if (!grow_stack(lp->lwp_proc, va)) { - rv = KERN_FAILURE; - PRELE(lp->lwp_proc); - goto nogo; - } - /* * Issue fault */ diff --git a/sys/platform/pc64/x86_64/vm_machdep.c b/sys/platform/pc64/x86_64/vm_machdep.c index 278d06d45e..c02cb2d8c1 100644 --- a/sys/platform/pc64/x86_64/vm_machdep.c +++ b/sys/platform/pc64/x86_64/vm_machdep.c @@ -347,18 +347,6 @@ kvtop(void *addr) return (pa); } -int -grow_stack(struct proc *p, vm_offset_t sp) -{ - int rv; - - rv = vm_map_growstack (p, sp); - if (rv != KERN_SUCCESS) - return (0); - - return (1); -} - /* * Tell whether this address is in some physical memory region. * Currently used by the kernel coredump code in order to avoid diff --git a/sys/platform/vkernel/i386/trap.c b/sys/platform/vkernel/i386/trap.c index 377e5a84a8..49fbd831f2 100644 --- a/sys/platform/vkernel/i386/trap.c +++ b/sys/platform/vkernel/i386/trap.c @@ -113,8 +113,6 @@ int (*pmath_emulate) (struct trapframe *); -extern int trapwrite (unsigned addr); - static int trap_pfault (struct trapframe *, int, vm_offset_t); static void trap_fatal (struct trapframe *, int, vm_offset_t); void dblfault_handler (void); @@ -924,21 +922,6 @@ trap_pfault(struct trapframe *frame, int usermode, vm_offset_t eva) */ PHOLD(lp->lwp_proc); - /* - * Grow the stack if necessary - */ - /* grow_stack returns false only if va falls into - * a growable stack region and the stack growth - * fails. It returns true if va was not within - * a growable stack region, or if the stack - * growth succeeded. - */ - if (!grow_stack (lp->lwp_proc, va)) { - rv = KERN_FAILURE; - PRELE(lp->lwp_proc); - goto nogo; - } - /* * Issue fault */ @@ -1090,50 +1073,6 @@ dblfault_handler(void) panic("double fault"); } -/* - * Compensate for 386 brain damage (missing URKR). - * This is a little simpler than the pagefault handler in trap() because - * it the page tables have already been faulted in and high addresses - * are thrown out early for other reasons. - */ -int -trapwrite(unsigned addr) -{ - struct lwp *lp; - vm_offset_t va; - struct vmspace *vm; - int rv; - - va = trunc_page((vm_offset_t)addr); - /* - * XXX - MAX is END. Changed > to >= for temp. fix. - */ - if (va >= VM_MAX_USER_ADDRESS) - return (1); - - lp = curthread->td_lwp; - vm = lp->lwp_vmspace; - - PHOLD(lp->lwp_proc); - - if (!grow_stack (lp->lwp_proc, va)) { - PRELE(lp->lwp_proc); - return (1); - } - - /* - * fault the data page - */ - rv = vm_fault(&vm->vm_map, va, VM_PROT_WRITE, VM_FAULT_DIRTY); - - PRELE(lp->lwp_proc); - - if (rv != KERN_SUCCESS) - return 1; - - return (0); -} - /* * syscall2 - MP aware system call request C handler * diff --git a/sys/platform/vkernel/i386/vm_machdep.c b/sys/platform/vkernel/i386/vm_machdep.c index 266c765fbb..54df1b5548 100644 --- a/sys/platform/vkernel/i386/vm_machdep.c +++ b/sys/platform/vkernel/i386/vm_machdep.c @@ -367,18 +367,6 @@ kvtop(void *addr) return (pa); } -int -grow_stack(struct proc *p, vm_offset_t sp) -{ - int rv; - - rv = vm_map_growstack (p, sp); - if (rv != KERN_SUCCESS) - return (0); - - return (1); -} - SYSCTL_DECL(_vm_stats_misc); static int cnt_prezero; diff --git a/sys/vm/vm_extern.h b/sys/vm/vm_extern.h index 0c452e30c5..5bcb989afa 100644 --- a/sys/vm/vm_extern.h +++ b/sys/vm/vm_extern.h @@ -74,7 +74,6 @@ int swapon (struct proc *, void *, int *); #endif int grow (struct proc *, size_t); -int grow_stack (struct proc *, vm_offset_t); int kernacc(c_caddr_t, int, int); vm_offset_t kmem_alloc3 (vm_map_t, vm_size_t, int flags); vm_offset_t kmem_alloc_nofault (vm_map_t, vm_size_t); diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c index 02f3da04c6..b55b0da686 100644 --- a/sys/vm/vm_fault.c +++ b/sys/vm/vm_fault.c @@ -225,12 +225,14 @@ vm_fault(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type, int fault_flags) int result; vm_pindex_t first_pindex; struct faultstate fs; + int growstack; mycpu->gd_cnt.v_vm_faults++; fs.didlimit = 0; fs.hardfault = 0; fs.fault_flags = fault_flags; + growstack = 1; RetryFault: /* @@ -258,10 +260,19 @@ RetryFault: * to do a user wiring we have more work to do. */ if (result != KERN_SUCCESS) { - if (result != KERN_PROTECTION_FAILURE) - return result; - if ((fs.fault_flags & VM_FAULT_WIRE_MASK) != VM_FAULT_USER_WIRE) - return result; + if (result != KERN_PROTECTION_FAILURE || + (fs.fault_flags & VM_FAULT_WIRE_MASK) != VM_FAULT_USER_WIRE) + { + if (result == KERN_INVALID_ADDRESS && growstack && + map != &kernel_map && curproc != NULL) { + result = vm_map_growstack(curproc, vaddr); + if (result != KERN_SUCCESS) + return (KERN_FAILURE); + growstack = 0; + goto RetryFault; + } + return (result); + } /* * If we are user-wiring a r/w segment, and it is COW, then -- 2.41.0