kernel - Optimize the x86-64 lwbuf API

* Change lwbuf_alloc(m) to lwbuf_alloc(m, &lwb_cache), passing a pointer to
  a struct lwb which lwbuf_alloc() may used if it desires.

* The x86-64 lwbuf_alloc() now just fills in the passed lwb and returns it.
  The i386 lwbuf_alloc() still uses the objcache w/ its kva mappings.  This
  removes objcache calls from the critical path.

* The x86-64 lwbuf_alloc()/lwbuf_free() functions are now inlines (ALL x86-64
  lwbuf functions are now inlines).

diff --git a/sys/cpu/i386/include/lwbuf.h b/sys/cpu/i386/include/lwbuf.h
index e253ba6..63b75d7 100644 (file)
--- a/sys/cpu/i386/include/lwbuf.h
+++ b/sys/cpu/i386/include/lwbuf.h
@@ -75,7 +75,7 @@ lwbuf_page(struct lwbuf *lwb)
 
 #if defined(_KERNEL)
 
-struct lwbuf   *lwbuf_alloc(vm_page_t);
+struct lwbuf   *lwbuf_alloc(vm_page_t, struct lwbuf *);
 void            lwbuf_free(struct lwbuf *);
 vm_offset_t     lwbuf_kva(struct lwbuf *lwb);
 void            lwbuf_set_global(struct lwbuf *);

diff --git a/sys/cpu/i386/misc/lwbuf.c b/sys/cpu/i386/misc/lwbuf.c
index 5025730..5decd98 100644 (file)
--- a/sys/cpu/i386/misc/lwbuf.c
+++ b/sys/cpu/i386/misc/lwbuf.c
@@ -120,7 +120,7 @@ lwbuf_init(void *arg)
 }
 
 struct lwbuf *
-lwbuf_alloc(vm_page_t m)
+lwbuf_alloc(vm_page_t m, struct lwbuf *lwb_dummy __unused)
 {
     struct mdglobaldata *gd = mdcpu;
     struct lwbuf *lwb;

diff --git a/sys/cpu/x86_64/include/lwbuf.h b/sys/cpu/x86_64/include/lwbuf.h
index 2554bbe..eb6203e 100644 (file)
--- a/sys/cpu/x86_64/include/lwbuf.h
+++ b/sys/cpu/x86_64/include/lwbuf.h
@@ -51,33 +51,52 @@
 #ifndef _MACHINE_ATOMIC_H_
 #include <machine/atomic.h>
 #endif
+#ifndef _MACHINE_VMPARAM_H_
+#include <machine/vmparam.h>
+#endif
 
 #if !defined(_KERNEL) && !defined(_KERNEL_STRUCTURES)
 #error "This file should not be included by userland programs."
 #endif
 
 struct lwbuf {
-    vm_page_t          m;              /* currently mapped page */
-    vm_offset_t                kva;            /* va of mapping */
+       vm_page_t       m;      /* currently mapped page */
+       vm_offset_t     kva;    /* va of mapping */
 };
 
 static __inline vm_page_t
-lwbuf_page(struct lwbuf *lwb) {
-    return (lwb->m);
+lwbuf_page(struct lwbuf *lwb)
+{
+       return (lwb->m);
 }
 
 static __inline vm_offset_t
-lwbuf_kva(struct lwbuf *lwb) {
-    return (lwb->kva);
+lwbuf_kva(struct lwbuf *lwb)
+{
+       return (lwb->kva);
+}
+
+static __inline struct lwbuf *
+lwbuf_alloc(vm_page_t m, struct lwbuf *lwb_cache)
+{
+       struct lwbuf *lwb = lwb_cache;
+
+       lwb->m = m;
+       lwb->kva = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(lwb->m));
+
+       return (lwb);
+}
+
+static __inline void
+lwbuf_free(struct lwbuf *lwb)
+{
+       lwb->m = NULL;  /* safety */
 }
 
 #define lwbuf_set_global(lwb)
 
 #if defined(_KERNEL)
 
-struct lwbuf   *lwbuf_alloc(vm_page_t);
-void            lwbuf_free(struct lwbuf *);
-
 #endif
 
 #endif /* !_CPU_LWBUF_H_ */

diff --git a/sys/cpu/x86_64/misc/lwbuf.c b/sys/cpu/x86_64/misc/lwbuf.c
index e4326f4..bee143e 100644 (file)
--- a/sys/cpu/x86_64/misc/lwbuf.c
+++ b/sys/cpu/x86_64/misc/lwbuf.c
@@ -45,6 +45,11 @@
 #include <cpu/lwbuf.h>
 #include <machine/param.h>
 
+#if 0
+/*
+ * NO LONGER USED - See inlines
+ */
+
 static void lwbuf_init(void *);
 SYSINIT(sock_lwb, SI_BOOT2_MACHDEP, SI_ORDER_ANY, lwbuf_init, NULL);
 
@@ -74,13 +79,17 @@ lwbuf_init(void *arg)
        &lwbuf_malloc_args);
 }
 
+#endif
+
+#if 0
+/*
+ * NO LONGER USED - See inlines
+ */
+
 struct lwbuf *
-lwbuf_alloc(vm_page_t m)
+lwbuf_alloc(vm_page_t m, struct lwbuf *lwb_cache)
 {
-    struct lwbuf *lwb;
-
-    if ((lwb = objcache_get(lwbuf_cache, M_WAITOK)) == NULL)
-        return (NULL);
+    struct lwbuf *lwb = lwb_cache;
 
     lwb->m = m;
     lwb->kva = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(lwb->m));
@@ -91,7 +100,7 @@ lwbuf_alloc(vm_page_t m)
 void
 lwbuf_free(struct lwbuf *lwb)
 {
-    lwb->m = NULL;
-
-    objcache_put(lwbuf_cache, lwb);
+    lwb->m = NULL;     /* safety */
 }
+
+#endif

diff --git a/sys/kern/imgact_elf.c b/sys/kern/imgact_elf.c
index 767c61a..72d3ef1 100644 (file)
--- a/sys/kern/imgact_elf.c
+++ b/sys/kern/imgact_elf.c
@@ -262,6 +262,7 @@ elf_check_abi_note(struct image_params *imgp, const Elf_Phdr *ph)
        Elf_Brandinfo *match = NULL;
        const Elf_Note *tmp_note;
        struct lwbuf *lwb;
+       struct lwbuf lwb_cache;
        const char *page;
        char *data = NULL;
        Elf_Off off;
@@ -278,6 +279,7 @@ elf_check_abi_note(struct image_params *imgp, const Elf_Phdr *ph)
        if (len < sizeof(Elf_Note) || len > PAGE_SIZE)
                return NULL; /* ENOEXEC? */
 
+       lwb = &lwb_cache;
        if (exec_map_page(imgp, off >> PAGE_SHIFT, &lwb, &page))
                return NULL;
 
@@ -290,6 +292,7 @@ elf_check_abi_note(struct image_params *imgp, const Elf_Phdr *ph)
                bcopy(page + firstoff, data, firstlen);
 
                exec_unmap_page(lwb);
+               lwb = &lwb_cache;
                if (exec_map_page(imgp, (off >> PAGE_SHIFT) + 1, &lwb, &page)) {
                        kfree(data, M_TEMP);
                        return NULL;
@@ -442,11 +445,12 @@ elf_load_section(struct proc *p, struct vmspace *vmspace, struct vnode *vp,
        if (copy_len != 0) {
                vm_page_t m;
                struct lwbuf *lwb;
+               struct lwbuf lwb_cache;
 
                m = vm_fault_object_page(object, trunc_page(offset + filsz),
                                         VM_PROT_READ, 0, &error);
                if (m) {
-                       lwb = lwbuf_alloc(m);
+                       lwb = lwbuf_alloc(m, &lwb_cache);
                        error = copyout((caddr_t)lwbuf_kva(lwb),
                                        (caddr_t)map_addr, copy_len);
                        lwbuf_free(lwb);

diff --git a/sys/kern/kern_exec.c b/sys/kern/kern_exec.c
index 4bbb198..08e9a10 100644 (file)
--- a/sys/kern/kern_exec.c
+++ b/sys/kern/kern_exec.c
@@ -630,7 +630,7 @@ exec_map_page(struct image_params *imgp, vm_pindex_t pageno,
        vm_page_wakeup(m);      /* unbusy the page */
        lwkt_reltoken(&vm_token);
 
-       *plwb = lwbuf_alloc(m);
+       *plwb = lwbuf_alloc(m, *plwb);
        *pdata = (void *)lwbuf_kva(*plwb);
 
        return (0);
@@ -644,6 +644,7 @@ exec_map_first_page(struct image_params *imgp)
        if (imgp->firstpage)
                exec_unmap_first_page(imgp);
 
+       imgp->firstpage = &imgp->firstpage_cache;
        err = exec_map_page(imgp, 0, &imgp->firstpage, &imgp->image_header);
 
        if (err)

diff --git a/sys/kern/kern_sfbuf.c b/sys/kern/kern_sfbuf.c
index 494e55b..45c91da 100644 (file)
--- a/sys/kern/kern_sfbuf.c
+++ b/sys/kern/kern_sfbuf.c
@@ -87,7 +87,7 @@ sf_buf_alloc(struct vm_page *m)
        if ((sf = objcache_get(sf_buf_cache, M_WAITOK)) == NULL)
                goto done;
 
-       if ((sf->lwbuf = lwbuf_alloc(m)) == NULL) {
+       if ((sf->lwbuf = lwbuf_alloc(m, &sf->lwbuf_cache)) == NULL) {
                objcache_put(sf_buf_cache, sf);
                sf = NULL;
                goto done;

diff --git a/sys/kern/kern_subr.c b/sys/kern/kern_subr.c
index 129d890..666e13f 100644 (file)
--- a/sys/kern/kern_subr.c
+++ b/sys/kern/kern_subr.c
@@ -448,6 +448,7 @@ iovec_copyin(struct iovec *uiov, struct iovec **kiov, struct iovec *siov,
 int
 uiomove_fromphys(vm_page_t *ma, vm_offset_t offset, size_t n, struct uio *uio)
 {
+       struct lwbuf lwb_cache;
        struct lwbuf *lwb;
        struct thread *td = curthread;
        struct iovec *iov;
@@ -481,7 +482,7 @@ uiomove_fromphys(vm_page_t *ma, vm_offset_t offset, size_t n, struct uio *uio)
                page_offset = offset & PAGE_MASK;
                cnt = min(cnt, PAGE_SIZE - page_offset);
                m = ma[offset >> PAGE_SHIFT];
-               lwb = lwbuf_alloc(m);
+               lwb = lwbuf_alloc(m, &lwb_cache);
                cp = (char *)lwbuf_kva(lwb) + page_offset;
                switch (uio->uio_segflg) {
                case UIO_USERSPACE:

diff --git a/sys/kern/kern_umtx.c b/sys/kern/kern_umtx.c
index b0cbc36..c3c3904 100644 (file)
--- a/sys/kern/kern_umtx.c
+++ b/sys/kern/kern_umtx.c
@@ -100,13 +100,14 @@ static void umtx_sleep_page_action_cow(vm_page_t m, vm_page_action_t action);
 int
 sys_umtx_sleep(struct umtx_sleep_args *uap)
 {
-    int error = EBUSY;
+    struct lwbuf lwb_cache;
     struct lwbuf *lwb;
     struct vm_page_action action;
     vm_page_t m;
     void *waddr;
     int offset;
     int timeout;
+    int error = EBUSY;
 
     if (uap->timeout < 0)
        return (EINVAL);
@@ -124,7 +125,7 @@ sys_umtx_sleep(struct umtx_sleep_args *uap)
        error = EFAULT;
        goto done;
     }
-    lwb = lwbuf_alloc(m);
+    lwb = lwbuf_alloc(m, &lwb_cache);
     offset = (vm_offset_t)uap->ptr & PAGE_MASK;
 
     /*

diff --git a/sys/kern/kern_xio.c b/sys/kern/kern_xio.c
index f2bbe91..64416d5 100644 (file)
--- a/sys/kern/kern_xio.c
+++ b/sys/kern/kern_xio.c
@@ -309,6 +309,7 @@ xio_copy_xtou(xio_t xio, int uoffset, void *uptr, int bytes)
     int offset;
     vm_page_t m;
     struct lwbuf *lwb;
+    struct lwbuf lwb_cache;
 
     if (uoffset + bytes > xio->xio_bytes)
        return(EFAULT);
@@ -323,7 +324,7 @@ xio_copy_xtou(xio_t xio, int uoffset, void *uptr, int bytes)
         ++i
     ) {
        m = xio->xio_pages[i];
-       lwb = lwbuf_alloc(m);
+       lwb = lwbuf_alloc(m, &lwb_cache);
        error = copyout((char *)lwbuf_kva(lwb) + offset, uptr, n);
        lwbuf_free(lwb);
        if (error)
@@ -357,6 +358,7 @@ xio_copy_xtok(xio_t xio, int uoffset, void *kptr, int bytes)
     int offset;
     vm_page_t m;
     struct lwbuf *lwb;
+    struct lwbuf lwb_cache;
 
     if (bytes + uoffset > xio->xio_bytes)
        return(EFAULT);
@@ -371,7 +373,7 @@ xio_copy_xtok(xio_t xio, int uoffset, void *kptr, int bytes)
         ++i
     ) {
        m = xio->xio_pages[i];
-       lwb = lwbuf_alloc(m);
+       lwb = lwbuf_alloc(m, &lwb_cache);
        bcopy((char *)lwbuf_kva(lwb) + offset, kptr, n);
        lwbuf_free(lwb);
        bytes -= n;
@@ -403,6 +405,7 @@ xio_copy_utox(xio_t xio, int uoffset, const void *uptr, int bytes)
     int offset;
     vm_page_t m;
     struct lwbuf *lwb;
+    struct lwbuf lwb_cache;
 
     if (uoffset + bytes > xio->xio_bytes)
        return(EFAULT);
@@ -417,7 +420,7 @@ xio_copy_utox(xio_t xio, int uoffset, const void *uptr, int bytes)
         ++i
     ) {
        m = xio->xio_pages[i];
-       lwb = lwbuf_alloc(m);
+       lwb = lwbuf_alloc(m, &lwb_cache);
        error = copyin(uptr, (char *)lwbuf_kva(lwb) + offset, n);
        lwbuf_free(lwb);
        if (error)
@@ -451,6 +454,7 @@ xio_copy_ktox(xio_t xio, int uoffset, const void *kptr, int bytes)
     int offset;
     vm_page_t m;
     struct lwbuf *lwb;
+    struct lwbuf lwb_cache;
 
     if (uoffset + bytes > xio->xio_bytes)
        return(EFAULT);
@@ -465,7 +469,7 @@ xio_copy_ktox(xio_t xio, int uoffset, const void *kptr, int bytes)
         ++i
     ) {
        m = xio->xio_pages[i];
-       lwb = lwbuf_alloc(m);
+       lwb = lwbuf_alloc(m, &lwb_cache);
        bcopy(kptr, (char *)lwbuf_kva(lwb) + offset, n);
        lwbuf_free(lwb);
        bytes -= n;

diff --git a/sys/platform/vkernel/platform/copyio.c b/sys/platform/vkernel/platform/copyio.c
index 9c8de1b..66b2be6 100644 (file)
--- a/sys/platform/vkernel/platform/copyio.c
+++ b/sys/platform/vkernel/platform/copyio.c
@@ -125,6 +125,7 @@ copyin(const void *udaddr, void *kaddr, size_t len)
 {
        struct vmspace *vm = curproc->p_vmspace;
        struct lwbuf *lwb;
+       struct lwbuf lwb_cache;
        vm_page_t m;
        int error;
        size_t n;
@@ -139,7 +140,7 @@ copyin(const void *udaddr, void *kaddr, size_t len)
                n = PAGE_SIZE - ((vm_offset_t)udaddr & PAGE_MASK);
                if (n > len)
                        n = len;
-               lwb = lwbuf_alloc(m);
+               lwb = lwbuf_alloc(m, &lwb_cache);
                bcopy((char *)lwbuf_kva(lwb)+((vm_offset_t)udaddr & PAGE_MASK),
                      kaddr, n);
                len -= n;
@@ -161,6 +162,7 @@ copyout(const void *kaddr, void *udaddr, size_t len)
 {
        struct vmspace *vm = curproc->p_vmspace;
        struct lwbuf *lwb;
+       struct lwbuf lwb_cache;
        vm_page_t m;
        int error;
        size_t n;
@@ -175,7 +177,7 @@ copyout(const void *kaddr, void *udaddr, size_t len)
                n = PAGE_SIZE - ((vm_offset_t)udaddr & PAGE_MASK);
                if (n > len)
                        n = len;
-               lwb = lwbuf_alloc(m);
+               lwb = lwbuf_alloc(m, &lwb_cache);
                bcopy(kaddr, (char *)lwbuf_kva(lwb) +
                             ((vm_offset_t)udaddr & PAGE_MASK), n);
                len -= n;

diff --git a/sys/platform/vkernel64/platform/copyio.c b/sys/platform/vkernel64/platform/copyio.c
index b83e169..4cf62f2 100644 (file)
--- a/sys/platform/vkernel64/platform/copyio.c
+++ b/sys/platform/vkernel64/platform/copyio.c
@@ -125,6 +125,7 @@ copyin(const void *udaddr, void *kaddr, size_t len)
 {
        struct vmspace *vm = curproc->p_vmspace;
        struct lwbuf *lwb;
+       struct lwbuf_cache lwb_cache;
        vm_page_t m;
        int error;
        size_t n;
@@ -139,7 +140,7 @@ copyin(const void *udaddr, void *kaddr, size_t len)
                n = PAGE_SIZE - ((vm_offset_t)udaddr & PAGE_MASK);
                if (n > len)
                        n = len;
-               lwb = lwbuf_alloc(m);
+               lwb = lwbuf_alloc(m, &lwb_cache);
                bcopy((char *)lwbuf_kva(lwb)+((vm_offset_t)udaddr & PAGE_MASK),
                      kaddr, n);
                len -= n;
@@ -161,6 +162,7 @@ copyout(const void *kaddr, void *udaddr, size_t len)
 {
        struct vmspace *vm = curproc->p_vmspace;
        struct lwbuf *lwb;
+       struct lwbuf lwb_cache;
        vm_page_t m;
        int error;
        size_t n;
@@ -175,7 +177,7 @@ copyout(const void *kaddr, void *udaddr, size_t len)
                n = PAGE_SIZE - ((vm_offset_t)udaddr & PAGE_MASK);
                if (n > len)
                        n = len;
-               lwb = lwbuf_alloc(m);
+               lwb = lwbuf_alloc(m, &lwb_cache);
                bcopy(kaddr, (char *)lwbuf_kva(lwb) +
                             ((vm_offset_t)udaddr & PAGE_MASK), n);
                len -= n;

diff --git a/sys/sys/imgact.h b/sys/sys/imgact.h
index e70c9a0..40ca7c3 100644 (file)
--- a/sys/sys/imgact.h
+++ b/sys/sys/imgact.h
@@ -38,6 +38,7 @@
 #define        _SYS_IMGACT_H_
 
 #include <sys/mount.h>
+#include <cpu/lwbuf.h>
 
 #define MAXSHELLCMDLEN 128
 
@@ -65,6 +66,7 @@ struct image_params {
        char interpreter_name[MAXSHELLCMDLEN]; /* name of the interpreter */
        void *auxargs;          /* ELF Auxinfo structure pointer */
        struct lwbuf *firstpage;        /* first page that we mapped */
+       struct lwbuf firstpage_cache;
        unsigned long ps_strings; /* PS_STRINGS for BSD/OS binaries */
 };
 

diff --git a/sys/sys/sfbuf.h b/sys/sys/sfbuf.h
index 8141229..99a3877 100644 (file)
--- a/sys/sys/sfbuf.h
+++ b/sys/sys/sfbuf.h
@@ -47,8 +47,9 @@
 #endif
 
 struct sf_buf {
-       struct  lwbuf *lwbuf;   /* lightweight buffer */
-       struct  kref ref;
+       struct lwbuf *lwbuf;    /* lightweight buffer */
+       struct kref ref;
+       struct lwbuf lwbuf_cache;
 };
 
 #define sf_buf_kva(sf) (lwbuf_kva((sf)->lwbuf))

diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c
index b62fcc3..5c621db 100644 (file)
--- a/sys/vm/vm_fault.c
+++ b/sys/vm/vm_fault.c
@@ -865,6 +865,7 @@ vm_fault_vpagetable(struct faultstate *fs, vm_pindex_t *pindex,
                    vpte_t vpte, int fault_type)
 {
        struct lwbuf *lwb;
+       struct lwbuf lwb_cache;
        int vshift = VPTE_FRAME_END - PAGE_SHIFT; /* index bits remaining */
        int result = KERN_SUCCESS;
        vpte_t *ptep;
@@ -907,7 +908,7 @@ vm_fault_vpagetable(struct faultstate *fs, vm_pindex_t *pindex,
                 * entry in the page table page.
                 */
                vshift -= VPTE_PAGE_BITS;
-               lwb = lwbuf_alloc(fs->m);
+               lwb = lwbuf_alloc(fs->m, &lwb_cache);
                ptep = ((vpte_t *)lwbuf_kva(lwb) +
                        ((*pindex >> vshift) & VPTE_PAGE_MASK));
                vpte = *ptep;

diff --git a/sys/vm/vm_zeroidle.c b/sys/vm/vm_zeroidle.c
index c99dee9..ece448b 100644 (file)
--- a/sys/vm/vm_zeroidle.c
+++ b/sys/vm/vm_zeroidle.c
@@ -149,7 +149,8 @@ static void
 vm_pagezero(void __unused *arg)
 {
        vm_page_t m = NULL;
-       struct lwbuf *buf = NULL;
+       struct lwbuf *lwb = NULL;
+       struct lwbuf lwb_cache;
        enum zeroidle_state state = STATE_IDLE;
        char *pg = NULL;
        int npages = 0;
@@ -198,8 +199,8 @@ vm_pagezero(void __unused *arg)
                                        state = STATE_IDLE;
                                } else {
                                        state = STATE_ZERO_PAGE;
-                                       buf = lwbuf_alloc(m);
-                                       pg = (char *)lwbuf_kva(buf);
+                                       lwb = lwbuf_alloc(m, &lwb_cache);
+                                       pg = (char *)lwbuf_kva(lwb);
                                        i = 0;
                                }
                        }
@@ -219,7 +220,7 @@ vm_pagezero(void __unused *arg)
                        state = STATE_RELEASE_PAGE;
                        break;
                case STATE_RELEASE_PAGE:
-                       lwbuf_free(buf);
+                       lwbuf_free(lwb);
                        vm_page_flag_set(m, PG_ZERO);
                        vm_page_free_toq(m);
                        state = STATE_GET_PAGE;

diff --git a/sys/vm/vnode_pager.c b/sys/vm/vnode_pager.c
index 292f8aa..6be7dd6 100644 (file)
--- a/sys/vm/vnode_pager.c
+++ b/sys/vm/vnode_pager.c
@@ -401,6 +401,7 @@ vnode_pager_setsize(struct vnode *vp, vm_ooffset_t nsize)
                                int base = (int)nsize & PAGE_MASK;
                                int size = PAGE_SIZE - base;
                                struct lwbuf *lwb;
+                               struct lwbuf lwb_cache;
 
                                /*
                                 * Clear out partial-page garbage in case
@@ -409,7 +410,7 @@ vnode_pager_setsize(struct vnode *vp, vm_ooffset_t nsize)
                                 * This is byte aligned.
                                 */
                                vm_page_busy(m);
-                               lwb = lwbuf_alloc(m);
+                               lwb = lwbuf_alloc(m, &lwb_cache);
                                kva = lwbuf_kva(lwb);
                                bzero((caddr_t)kva + base, size);
                                lwbuf_free(lwb);