From: Matthew Dillon Date: Thu, 20 Jan 2011 01:27:46 +0000 (-0800) Subject: kernel - Optimize the x86-64 lwbuf API X-Git-Tag: v2.10.0~291 X-Git-Url: http://gitweb.dragonflybsd.org/dragonfly.git/commitdiff_plain/7a683a249a628667d7a484f26af36786efbdd87f kernel - Optimize the x86-64 lwbuf API * Change lwbuf_alloc(m) to lwbuf_alloc(m, &lwb_cache), passing a pointer to a struct lwb which lwbuf_alloc() may used if it desires. * The x86-64 lwbuf_alloc() now just fills in the passed lwb and returns it. The i386 lwbuf_alloc() still uses the objcache w/ its kva mappings. This removes objcache calls from the critical path. * The x86-64 lwbuf_alloc()/lwbuf_free() functions are now inlines (ALL x86-64 lwbuf functions are now inlines). --- diff --git a/sys/cpu/i386/include/lwbuf.h b/sys/cpu/i386/include/lwbuf.h index e253ba6..63b75d7 100644 --- a/sys/cpu/i386/include/lwbuf.h +++ b/sys/cpu/i386/include/lwbuf.h @@ -75,7 +75,7 @@ lwbuf_page(struct lwbuf *lwb) #if defined(_KERNEL) -struct lwbuf *lwbuf_alloc(vm_page_t); +struct lwbuf *lwbuf_alloc(vm_page_t, struct lwbuf *); void lwbuf_free(struct lwbuf *); vm_offset_t lwbuf_kva(struct lwbuf *lwb); void lwbuf_set_global(struct lwbuf *); diff --git a/sys/cpu/i386/misc/lwbuf.c b/sys/cpu/i386/misc/lwbuf.c index 5025730..5decd98 100644 --- a/sys/cpu/i386/misc/lwbuf.c +++ b/sys/cpu/i386/misc/lwbuf.c @@ -120,7 +120,7 @@ lwbuf_init(void *arg) } struct lwbuf * -lwbuf_alloc(vm_page_t m) +lwbuf_alloc(vm_page_t m, struct lwbuf *lwb_dummy __unused) { struct mdglobaldata *gd = mdcpu; struct lwbuf *lwb; diff --git a/sys/cpu/x86_64/include/lwbuf.h b/sys/cpu/x86_64/include/lwbuf.h index 2554bbe..eb6203e 100644 --- a/sys/cpu/x86_64/include/lwbuf.h +++ b/sys/cpu/x86_64/include/lwbuf.h @@ -51,33 +51,52 @@ #ifndef _MACHINE_ATOMIC_H_ #include #endif +#ifndef _MACHINE_VMPARAM_H_ +#include +#endif #if !defined(_KERNEL) && !defined(_KERNEL_STRUCTURES) #error "This file should not be included by userland programs." #endif struct lwbuf { - vm_page_t m; /* currently mapped page */ - vm_offset_t kva; /* va of mapping */ + vm_page_t m; /* currently mapped page */ + vm_offset_t kva; /* va of mapping */ }; static __inline vm_page_t -lwbuf_page(struct lwbuf *lwb) { - return (lwb->m); +lwbuf_page(struct lwbuf *lwb) +{ + return (lwb->m); } static __inline vm_offset_t -lwbuf_kva(struct lwbuf *lwb) { - return (lwb->kva); +lwbuf_kva(struct lwbuf *lwb) +{ + return (lwb->kva); +} + +static __inline struct lwbuf * +lwbuf_alloc(vm_page_t m, struct lwbuf *lwb_cache) +{ + struct lwbuf *lwb = lwb_cache; + + lwb->m = m; + lwb->kva = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(lwb->m)); + + return (lwb); +} + +static __inline void +lwbuf_free(struct lwbuf *lwb) +{ + lwb->m = NULL; /* safety */ } #define lwbuf_set_global(lwb) #if defined(_KERNEL) -struct lwbuf *lwbuf_alloc(vm_page_t); -void lwbuf_free(struct lwbuf *); - #endif #endif /* !_CPU_LWBUF_H_ */ diff --git a/sys/cpu/x86_64/misc/lwbuf.c b/sys/cpu/x86_64/misc/lwbuf.c index e4326f4..bee143e 100644 --- a/sys/cpu/x86_64/misc/lwbuf.c +++ b/sys/cpu/x86_64/misc/lwbuf.c @@ -45,6 +45,11 @@ #include #include +#if 0 +/* + * NO LONGER USED - See inlines + */ + static void lwbuf_init(void *); SYSINIT(sock_lwb, SI_BOOT2_MACHDEP, SI_ORDER_ANY, lwbuf_init, NULL); @@ -74,13 +79,17 @@ lwbuf_init(void *arg) &lwbuf_malloc_args); } +#endif + +#if 0 +/* + * NO LONGER USED - See inlines + */ + struct lwbuf * -lwbuf_alloc(vm_page_t m) +lwbuf_alloc(vm_page_t m, struct lwbuf *lwb_cache) { - struct lwbuf *lwb; - - if ((lwb = objcache_get(lwbuf_cache, M_WAITOK)) == NULL) - return (NULL); + struct lwbuf *lwb = lwb_cache; lwb->m = m; lwb->kva = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(lwb->m)); @@ -91,7 +100,7 @@ lwbuf_alloc(vm_page_t m) void lwbuf_free(struct lwbuf *lwb) { - lwb->m = NULL; - - objcache_put(lwbuf_cache, lwb); + lwb->m = NULL; /* safety */ } + +#endif diff --git a/sys/kern/imgact_elf.c b/sys/kern/imgact_elf.c index 767c61a..72d3ef1 100644 --- a/sys/kern/imgact_elf.c +++ b/sys/kern/imgact_elf.c @@ -262,6 +262,7 @@ elf_check_abi_note(struct image_params *imgp, const Elf_Phdr *ph) Elf_Brandinfo *match = NULL; const Elf_Note *tmp_note; struct lwbuf *lwb; + struct lwbuf lwb_cache; const char *page; char *data = NULL; Elf_Off off; @@ -278,6 +279,7 @@ elf_check_abi_note(struct image_params *imgp, const Elf_Phdr *ph) if (len < sizeof(Elf_Note) || len > PAGE_SIZE) return NULL; /* ENOEXEC? */ + lwb = &lwb_cache; if (exec_map_page(imgp, off >> PAGE_SHIFT, &lwb, &page)) return NULL; @@ -290,6 +292,7 @@ elf_check_abi_note(struct image_params *imgp, const Elf_Phdr *ph) bcopy(page + firstoff, data, firstlen); exec_unmap_page(lwb); + lwb = &lwb_cache; if (exec_map_page(imgp, (off >> PAGE_SHIFT) + 1, &lwb, &page)) { kfree(data, M_TEMP); return NULL; @@ -442,11 +445,12 @@ elf_load_section(struct proc *p, struct vmspace *vmspace, struct vnode *vp, if (copy_len != 0) { vm_page_t m; struct lwbuf *lwb; + struct lwbuf lwb_cache; m = vm_fault_object_page(object, trunc_page(offset + filsz), VM_PROT_READ, 0, &error); if (m) { - lwb = lwbuf_alloc(m); + lwb = lwbuf_alloc(m, &lwb_cache); error = copyout((caddr_t)lwbuf_kva(lwb), (caddr_t)map_addr, copy_len); lwbuf_free(lwb); diff --git a/sys/kern/kern_exec.c b/sys/kern/kern_exec.c index 4bbb198..08e9a10 100644 --- a/sys/kern/kern_exec.c +++ b/sys/kern/kern_exec.c @@ -630,7 +630,7 @@ exec_map_page(struct image_params *imgp, vm_pindex_t pageno, vm_page_wakeup(m); /* unbusy the page */ lwkt_reltoken(&vm_token); - *plwb = lwbuf_alloc(m); + *plwb = lwbuf_alloc(m, *plwb); *pdata = (void *)lwbuf_kva(*plwb); return (0); @@ -644,6 +644,7 @@ exec_map_first_page(struct image_params *imgp) if (imgp->firstpage) exec_unmap_first_page(imgp); + imgp->firstpage = &imgp->firstpage_cache; err = exec_map_page(imgp, 0, &imgp->firstpage, &imgp->image_header); if (err) diff --git a/sys/kern/kern_sfbuf.c b/sys/kern/kern_sfbuf.c index 494e55b..45c91da 100644 --- a/sys/kern/kern_sfbuf.c +++ b/sys/kern/kern_sfbuf.c @@ -87,7 +87,7 @@ sf_buf_alloc(struct vm_page *m) if ((sf = objcache_get(sf_buf_cache, M_WAITOK)) == NULL) goto done; - if ((sf->lwbuf = lwbuf_alloc(m)) == NULL) { + if ((sf->lwbuf = lwbuf_alloc(m, &sf->lwbuf_cache)) == NULL) { objcache_put(sf_buf_cache, sf); sf = NULL; goto done; diff --git a/sys/kern/kern_subr.c b/sys/kern/kern_subr.c index 129d890..666e13f 100644 --- a/sys/kern/kern_subr.c +++ b/sys/kern/kern_subr.c @@ -448,6 +448,7 @@ iovec_copyin(struct iovec *uiov, struct iovec **kiov, struct iovec *siov, int uiomove_fromphys(vm_page_t *ma, vm_offset_t offset, size_t n, struct uio *uio) { + struct lwbuf lwb_cache; struct lwbuf *lwb; struct thread *td = curthread; struct iovec *iov; @@ -481,7 +482,7 @@ uiomove_fromphys(vm_page_t *ma, vm_offset_t offset, size_t n, struct uio *uio) page_offset = offset & PAGE_MASK; cnt = min(cnt, PAGE_SIZE - page_offset); m = ma[offset >> PAGE_SHIFT]; - lwb = lwbuf_alloc(m); + lwb = lwbuf_alloc(m, &lwb_cache); cp = (char *)lwbuf_kva(lwb) + page_offset; switch (uio->uio_segflg) { case UIO_USERSPACE: diff --git a/sys/kern/kern_umtx.c b/sys/kern/kern_umtx.c index b0cbc36..c3c3904 100644 --- a/sys/kern/kern_umtx.c +++ b/sys/kern/kern_umtx.c @@ -100,13 +100,14 @@ static void umtx_sleep_page_action_cow(vm_page_t m, vm_page_action_t action); int sys_umtx_sleep(struct umtx_sleep_args *uap) { - int error = EBUSY; + struct lwbuf lwb_cache; struct lwbuf *lwb; struct vm_page_action action; vm_page_t m; void *waddr; int offset; int timeout; + int error = EBUSY; if (uap->timeout < 0) return (EINVAL); @@ -124,7 +125,7 @@ sys_umtx_sleep(struct umtx_sleep_args *uap) error = EFAULT; goto done; } - lwb = lwbuf_alloc(m); + lwb = lwbuf_alloc(m, &lwb_cache); offset = (vm_offset_t)uap->ptr & PAGE_MASK; /* diff --git a/sys/kern/kern_xio.c b/sys/kern/kern_xio.c index f2bbe91..64416d5 100644 --- a/sys/kern/kern_xio.c +++ b/sys/kern/kern_xio.c @@ -309,6 +309,7 @@ xio_copy_xtou(xio_t xio, int uoffset, void *uptr, int bytes) int offset; vm_page_t m; struct lwbuf *lwb; + struct lwbuf lwb_cache; if (uoffset + bytes > xio->xio_bytes) return(EFAULT); @@ -323,7 +324,7 @@ xio_copy_xtou(xio_t xio, int uoffset, void *uptr, int bytes) ++i ) { m = xio->xio_pages[i]; - lwb = lwbuf_alloc(m); + lwb = lwbuf_alloc(m, &lwb_cache); error = copyout((char *)lwbuf_kva(lwb) + offset, uptr, n); lwbuf_free(lwb); if (error) @@ -357,6 +358,7 @@ xio_copy_xtok(xio_t xio, int uoffset, void *kptr, int bytes) int offset; vm_page_t m; struct lwbuf *lwb; + struct lwbuf lwb_cache; if (bytes + uoffset > xio->xio_bytes) return(EFAULT); @@ -371,7 +373,7 @@ xio_copy_xtok(xio_t xio, int uoffset, void *kptr, int bytes) ++i ) { m = xio->xio_pages[i]; - lwb = lwbuf_alloc(m); + lwb = lwbuf_alloc(m, &lwb_cache); bcopy((char *)lwbuf_kva(lwb) + offset, kptr, n); lwbuf_free(lwb); bytes -= n; @@ -403,6 +405,7 @@ xio_copy_utox(xio_t xio, int uoffset, const void *uptr, int bytes) int offset; vm_page_t m; struct lwbuf *lwb; + struct lwbuf lwb_cache; if (uoffset + bytes > xio->xio_bytes) return(EFAULT); @@ -417,7 +420,7 @@ xio_copy_utox(xio_t xio, int uoffset, const void *uptr, int bytes) ++i ) { m = xio->xio_pages[i]; - lwb = lwbuf_alloc(m); + lwb = lwbuf_alloc(m, &lwb_cache); error = copyin(uptr, (char *)lwbuf_kva(lwb) + offset, n); lwbuf_free(lwb); if (error) @@ -451,6 +454,7 @@ xio_copy_ktox(xio_t xio, int uoffset, const void *kptr, int bytes) int offset; vm_page_t m; struct lwbuf *lwb; + struct lwbuf lwb_cache; if (uoffset + bytes > xio->xio_bytes) return(EFAULT); @@ -465,7 +469,7 @@ xio_copy_ktox(xio_t xio, int uoffset, const void *kptr, int bytes) ++i ) { m = xio->xio_pages[i]; - lwb = lwbuf_alloc(m); + lwb = lwbuf_alloc(m, &lwb_cache); bcopy(kptr, (char *)lwbuf_kva(lwb) + offset, n); lwbuf_free(lwb); bytes -= n; diff --git a/sys/platform/vkernel/platform/copyio.c b/sys/platform/vkernel/platform/copyio.c index 9c8de1b..66b2be6 100644 --- a/sys/platform/vkernel/platform/copyio.c +++ b/sys/platform/vkernel/platform/copyio.c @@ -125,6 +125,7 @@ copyin(const void *udaddr, void *kaddr, size_t len) { struct vmspace *vm = curproc->p_vmspace; struct lwbuf *lwb; + struct lwbuf lwb_cache; vm_page_t m; int error; size_t n; @@ -139,7 +140,7 @@ copyin(const void *udaddr, void *kaddr, size_t len) n = PAGE_SIZE - ((vm_offset_t)udaddr & PAGE_MASK); if (n > len) n = len; - lwb = lwbuf_alloc(m); + lwb = lwbuf_alloc(m, &lwb_cache); bcopy((char *)lwbuf_kva(lwb)+((vm_offset_t)udaddr & PAGE_MASK), kaddr, n); len -= n; @@ -161,6 +162,7 @@ copyout(const void *kaddr, void *udaddr, size_t len) { struct vmspace *vm = curproc->p_vmspace; struct lwbuf *lwb; + struct lwbuf lwb_cache; vm_page_t m; int error; size_t n; @@ -175,7 +177,7 @@ copyout(const void *kaddr, void *udaddr, size_t len) n = PAGE_SIZE - ((vm_offset_t)udaddr & PAGE_MASK); if (n > len) n = len; - lwb = lwbuf_alloc(m); + lwb = lwbuf_alloc(m, &lwb_cache); bcopy(kaddr, (char *)lwbuf_kva(lwb) + ((vm_offset_t)udaddr & PAGE_MASK), n); len -= n; diff --git a/sys/platform/vkernel64/platform/copyio.c b/sys/platform/vkernel64/platform/copyio.c index b83e169..4cf62f2 100644 --- a/sys/platform/vkernel64/platform/copyio.c +++ b/sys/platform/vkernel64/platform/copyio.c @@ -125,6 +125,7 @@ copyin(const void *udaddr, void *kaddr, size_t len) { struct vmspace *vm = curproc->p_vmspace; struct lwbuf *lwb; + struct lwbuf_cache lwb_cache; vm_page_t m; int error; size_t n; @@ -139,7 +140,7 @@ copyin(const void *udaddr, void *kaddr, size_t len) n = PAGE_SIZE - ((vm_offset_t)udaddr & PAGE_MASK); if (n > len) n = len; - lwb = lwbuf_alloc(m); + lwb = lwbuf_alloc(m, &lwb_cache); bcopy((char *)lwbuf_kva(lwb)+((vm_offset_t)udaddr & PAGE_MASK), kaddr, n); len -= n; @@ -161,6 +162,7 @@ copyout(const void *kaddr, void *udaddr, size_t len) { struct vmspace *vm = curproc->p_vmspace; struct lwbuf *lwb; + struct lwbuf lwb_cache; vm_page_t m; int error; size_t n; @@ -175,7 +177,7 @@ copyout(const void *kaddr, void *udaddr, size_t len) n = PAGE_SIZE - ((vm_offset_t)udaddr & PAGE_MASK); if (n > len) n = len; - lwb = lwbuf_alloc(m); + lwb = lwbuf_alloc(m, &lwb_cache); bcopy(kaddr, (char *)lwbuf_kva(lwb) + ((vm_offset_t)udaddr & PAGE_MASK), n); len -= n; diff --git a/sys/sys/imgact.h b/sys/sys/imgact.h index e70c9a0..40ca7c3 100644 --- a/sys/sys/imgact.h +++ b/sys/sys/imgact.h @@ -38,6 +38,7 @@ #define _SYS_IMGACT_H_ #include +#include #define MAXSHELLCMDLEN 128 @@ -65,6 +66,7 @@ struct image_params { char interpreter_name[MAXSHELLCMDLEN]; /* name of the interpreter */ void *auxargs; /* ELF Auxinfo structure pointer */ struct lwbuf *firstpage; /* first page that we mapped */ + struct lwbuf firstpage_cache; unsigned long ps_strings; /* PS_STRINGS for BSD/OS binaries */ }; diff --git a/sys/sys/sfbuf.h b/sys/sys/sfbuf.h index 8141229..99a3877 100644 --- a/sys/sys/sfbuf.h +++ b/sys/sys/sfbuf.h @@ -47,8 +47,9 @@ #endif struct sf_buf { - struct lwbuf *lwbuf; /* lightweight buffer */ - struct kref ref; + struct lwbuf *lwbuf; /* lightweight buffer */ + struct kref ref; + struct lwbuf lwbuf_cache; }; #define sf_buf_kva(sf) (lwbuf_kva((sf)->lwbuf)) diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c index b62fcc3..5c621db 100644 --- a/sys/vm/vm_fault.c +++ b/sys/vm/vm_fault.c @@ -865,6 +865,7 @@ vm_fault_vpagetable(struct faultstate *fs, vm_pindex_t *pindex, vpte_t vpte, int fault_type) { struct lwbuf *lwb; + struct lwbuf lwb_cache; int vshift = VPTE_FRAME_END - PAGE_SHIFT; /* index bits remaining */ int result = KERN_SUCCESS; vpte_t *ptep; @@ -907,7 +908,7 @@ vm_fault_vpagetable(struct faultstate *fs, vm_pindex_t *pindex, * entry in the page table page. */ vshift -= VPTE_PAGE_BITS; - lwb = lwbuf_alloc(fs->m); + lwb = lwbuf_alloc(fs->m, &lwb_cache); ptep = ((vpte_t *)lwbuf_kva(lwb) + ((*pindex >> vshift) & VPTE_PAGE_MASK)); vpte = *ptep; diff --git a/sys/vm/vm_zeroidle.c b/sys/vm/vm_zeroidle.c index c99dee9..ece448b 100644 --- a/sys/vm/vm_zeroidle.c +++ b/sys/vm/vm_zeroidle.c @@ -149,7 +149,8 @@ static void vm_pagezero(void __unused *arg) { vm_page_t m = NULL; - struct lwbuf *buf = NULL; + struct lwbuf *lwb = NULL; + struct lwbuf lwb_cache; enum zeroidle_state state = STATE_IDLE; char *pg = NULL; int npages = 0; @@ -198,8 +199,8 @@ vm_pagezero(void __unused *arg) state = STATE_IDLE; } else { state = STATE_ZERO_PAGE; - buf = lwbuf_alloc(m); - pg = (char *)lwbuf_kva(buf); + lwb = lwbuf_alloc(m, &lwb_cache); + pg = (char *)lwbuf_kva(lwb); i = 0; } } @@ -219,7 +220,7 @@ vm_pagezero(void __unused *arg) state = STATE_RELEASE_PAGE; break; case STATE_RELEASE_PAGE: - lwbuf_free(buf); + lwbuf_free(lwb); vm_page_flag_set(m, PG_ZERO); vm_page_free_toq(m); state = STATE_GET_PAGE; diff --git a/sys/vm/vnode_pager.c b/sys/vm/vnode_pager.c index 292f8aa..6be7dd6 100644 --- a/sys/vm/vnode_pager.c +++ b/sys/vm/vnode_pager.c @@ -401,6 +401,7 @@ vnode_pager_setsize(struct vnode *vp, vm_ooffset_t nsize) int base = (int)nsize & PAGE_MASK; int size = PAGE_SIZE - base; struct lwbuf *lwb; + struct lwbuf lwb_cache; /* * Clear out partial-page garbage in case @@ -409,7 +410,7 @@ vnode_pager_setsize(struct vnode *vp, vm_ooffset_t nsize) * This is byte aligned. */ vm_page_busy(m); - lwb = lwbuf_alloc(m); + lwb = lwbuf_alloc(m, &lwb_cache); kva = lwbuf_kva(lwb); bzero((caddr_t)kva + base, size); lwbuf_free(lwb);