ksignal.9 \
ktr.9 \
lock.9 \
+ lwbuf.9 \
make_autoclone_dev.9 \
make_dev.9 \
mbuf.9 \
--- /dev/null
+.\" Copyright (c) 2010 by The DragonFly Project and Samuel J. Greear.
+.\" All rights reserved.
+.\"
+.\" This code is derived from software contributed to The DragonFly Project
+.\" by Samuel J. Greear <sjg@thesjg.com>
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright
+.\" notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\" notice, this list of conditions and the following disclaimer in
+.\" the documentation and/or other materials provided with the
+.\" distribution.
+.\" 3. Neither the name of The DragonFly Project nor the names of its
+.\" contributors may be used to endorse or promote products derived
+.\" from this software without specific, prior written permission.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\" "
+.Dd March 17, 2010
+.Os
+.Dt lwbuf 9
+.Sh NAME
+.Nm lwbuf_alloc ,
+.Nm lwbuf_free ,
+.Nm lwbuf_page ,
+.Nm lwbuf_kva ,
+.Nm lwbuf_set_global
+.Nd "Lightweight buffers"
+.Sh SYNOPSIS
+.In cpu/lwbuf.h
+.Ft "struct lwbuf *"
+.Fn lwbuf_alloc "vm_page_t m"
+.Ft void
+.Fn lwbuf_free "struct lwbuf *lwb"
+.Ft vm_page_t
+.Fn lwbuf_page "struct lwbuf *lwb"
+.Ft vm_offset_t
+.Fn lwbuf_kva "struct lwbuf *lwb"
+.Ft void
+.Fn lwbuf_set_global "struct lwbuf *lwb"
+.Sh DESCRIPTION
+The
+.Fa lwbuf
+kernel functions are used for maintaining a lightweight reference to and
+accessing an arbitrary
+.Fa vm_page_t .
+.Pp
+.Fn lwbuf_alloc
+returns a pointer to a lightweight buffer representing
+.Fa m .
+.Pp
+.Fn lwbuf_free
+frees all resources associated with the lightweight buffer
+.Fa lwb .
+.Pp
+.Fn lwbuf_page
+and
+.Fn lwbuf_kva
+return the associated
+.Fa vm_page_t
+or
+.Fa vm_offset_t
+of the lightweight buffer
+.Fa lwb .
+.Pp
+.Fn lwbuf_set_global
+ensures that a vm_offset_t previously obtained through
+.Fa lwbuf_kva
+will be valid on all processors without subsequent calls to
+.Fa lwbuf_kva .
+It should not be used.
+.Sh IMPLEMENTATION NOTES
+The implementation of
+.Fa lwbuf
+is cpu-dependent. On i386, pages taken from
+per-processor pools of kernel virtual address space (KVA) are used to map
+arbitrary
+.Fa vm_page_t
+objects. On x86_64 such tricks are unnecessary, the
+kernel maintains a direct map of KVA covering all physical memory.
+.Pp
+Lightweight buffers are thread and cross-processor safe with a number of
+limitations. Allocated buffers are not internally cached or reference counted.
+Any consumer of lightweight buffers may elect to share allocated buffers
+or allow them to be used in other threads or on other processors, but care
+must be taken. Buffers must be externally refcounted or in some other manner
+freed only after last use.
+.Sh HISTORY
+A
+.Nm lwbuf
+implementation first appeared in
+.Dx 2.5 .
+.Sh Authors
+The
+.Nm lwbuf
+implementation and this manpage were written by
+.An Samuel J. Greear .
cannot be set below the default determined when the kernel
was compiled.
- set kern.ipc.nsfbufs=<value> NSFBUFS
-
- Set the number of sendfile buffers to be allocated. This
- overrides the value determined when the kernel was compiled.
-
set kern.vm.kmem.size=<value>
Sets the size of kernel memory (bytes). This overrides
determined when the kernel was compiled.
Modifies
.Va NMBCLUSTERS .
-.It Va kern.ipc.nsfbufs
-Set the number of
-.Xr sendfile 2
-buffers to be allocated.
-Overrides
-.Dv NSFBUFS .
.It Va kern.mmxopt
Toggles the mmx optimizations for the bcopy/copyin/copyout routines
.It Va kern.vm.kmem.size
#kern.ipc.maxsockets="" # Set the maximum number of sockets avaliable
#kern.ipc.nmbclusters="" # Set the number of mbuf clusters
#kern.ipc.nmbufs="" # Set the maximum number of mbufs
-#kern.ipc.nsfbufs="" # Set the number of sendfile(2) bufs
#kern.vm.kmem.size="" # Sets the size of kernel memory (bytes)
#net.inet.tcp.tcbhashsize="" # Set the value of TCBHASHSIZE
#vfs.root.mountfrom="" # Specify root partition in a way the
#kern.ipc.maxsockets="" # Set the maximum number of sockets avaliable
#kern.ipc.nmbclusters="" # Set the number of mbuf clusters
#kern.ipc.nmbufs="" # Set the maximum number of mbufs
-#kern.ipc.nsfbufs="" # Set the number of sendfile(2) bufs
#kern.vm.kmem.size="" # Sets the size of kernel memory (bytes)
#net.inet.tcp.tcbhashsize="" # Set the value of TCBHASHSIZE
#vfs.root.mountfrom="" # Specify root partition in a way the
--- /dev/null
+/*
+ * Copyright (c) 2010 by The DragonFly Project and Samuel J. Greear.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The DragonFly Project
+ * by Samuel J. Greear <sjg@thesjg.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * 3. Neither the name of The DragonFly Project nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific, prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _CPU_LWBUF_H_
+#define _CPU_LWBUF_H_
+
+#ifndef _SYS_TYPES_H_
+#include <sys/types.h>
+#endif
+#ifndef _SYS_QUEUE_H_
+#include <sys/queue.h>
+#endif
+#ifndef _SYS_GLOBALDATA_H_
+#include <sys/globaldata.h>
+#endif
+#ifndef _VM_PMAP_H_
+#include <vm/pmap.h>
+#endif
+#ifndef _MACHINE_ATOMIC_H_
+#include <machine/atomic.h>
+#endif
+
+#if !defined(_KERNEL) && !defined(_KERNEL_STRUCTURES)
+#error "This file should not be included by userland programs."
+#endif
+
+struct lwbuf {
+ vm_page_t m; /* currently mapped page */
+ vm_offset_t kva; /* va of mapping */
+ cpumask_t cpumask; /* cpu mapping synchronization */
+};
+
+struct lwbuf_free_kvp {
+ vm_offset_t kva;
+ SLIST_ENTRY(lwbuf_free_kvp) next;
+};
+SLIST_HEAD(lwbuf_free_kvp_list, lwbuf_free_kvp);
+
+static __inline vm_page_t
+lwbuf_page(struct lwbuf *lwb) {
+ return (lwb->m);
+}
+
+#if defined(_KERNEL)
+
+struct lwbuf *lwbuf_alloc(vm_page_t);
+void lwbuf_free(struct lwbuf *);
+vm_offset_t lwbuf_kva(struct lwbuf *lwb);
+void lwbuf_set_global(struct lwbuf *);
+
+#endif
+
+#endif /* !_CPU_LWBUF_H_ */
--- /dev/null
+/*
+ * Copyright (c) 2010 by The DragonFly Project and Samuel J. Greear.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The DragonFly Project
+ * by Samuel J. Greear <sjg@thesjg.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * 3. Neither the name of The DragonFly Project nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific, prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/types.h>
+#include <sys/kernel.h>
+#include <sys/objcache.h>
+#include <sys/sysctl.h>
+#include <sys/param.h>
+#include <sys/serialize.h>
+#include <sys/systm.h>
+#include <sys/proc.h>
+#include <sys/queue.h>
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_page.h>
+#include <cpu/lwbuf.h>
+#include <machine/globaldata.h>
+#include <machine/atomic.h>
+#include <machine/param.h>
+
+static void lwbuf_init(void *);
+SYSINIT(sock_lwb, SI_BOOT2_MACHDEP, SI_ORDER_ANY, lwbuf_init, NULL);
+
+/* Number of pages of KVA to allocate at boot per cpu (1MB) */
+#define LWBUF_BOOT_PAGES 256
+/* Number to allocate incrementally (128KB) */
+#define LWBUF_ALLOC_PAGES 32
+
+static struct objcache *lwbuf_cache;
+
+MALLOC_DEFINE(M_LWBUF, "lwbuf", "Lightweight buffers");
+struct objcache_malloc_args lwbuf_malloc_args = { sizeof(struct lwbuf), M_LWBUF };
+
+
+static boolean_t
+lwbuf_cache_ctor(void *obj, void *pdata, int ocflags)
+{
+ struct lwbuf *lwb = (struct lwbuf *)obj;
+
+ lwb->m = NULL;
+ lwb->kva = 0;
+ lwb->cpumask = 0;
+
+ return (TRUE);
+}
+
+static boolean_t
+lwbuf_initpages(struct lwbuf_free_kvp_list *fkvpl, int pages)
+{
+ struct lwbuf_free_kvp *free_kvp;
+ vm_offset_t k;
+ int i;
+
+ k = kmem_alloc_nofault(&kernel_map, PAGE_SIZE * pages, PAGE_SIZE);
+ if (k == 0)
+ return (FALSE);
+
+ for (i = 0; i < pages; ++i) {
+ free_kvp = (struct lwbuf_free_kvp *)
+ kmalloc(sizeof(*free_kvp), M_LWBUF, M_WAITOK | M_ZERO);
+
+ free_kvp->kva = k + (i * PAGE_SIZE);
+ SLIST_INSERT_HEAD(fkvpl, free_kvp, next);
+ }
+
+ return (TRUE);
+}
+
+static void
+lwbuf_init(void *arg)
+{
+ int i;
+
+ lwbuf_cache = objcache_create("lwbuf", 0, 0,
+ lwbuf_cache_ctor, NULL, NULL,
+ objcache_malloc_alloc, objcache_malloc_free,
+ &lwbuf_malloc_args);
+
+ /* Should probably be in cpu_gdinit */
+ for (i = 0; i < SMP_MAXCPU; ++i) {
+ SLIST_INIT(&mdcpu->gd_lwbuf_fpages);
+ lwbuf_initpages(&mdcpu->gd_lwbuf_fpages, LWBUF_BOOT_PAGES);
+ }
+}
+
+struct lwbuf *
+lwbuf_alloc(vm_page_t m)
+{
+ struct mdglobaldata *gd = mdcpu;
+ struct lwbuf_free_kvp *free_kvp;
+ struct lwbuf *lwb;
+
+ if ((lwb = objcache_get(lwbuf_cache, M_WAITOK)) == NULL)
+ return (NULL);
+
+ lwb->m = m;
+
+check_slist:
+ if (!SLIST_EMPTY(&gd->gd_lwbuf_fpages)) {
+ free_kvp = SLIST_FIRST(&gd->gd_lwbuf_fpages);
+ SLIST_REMOVE_HEAD(&gd->gd_lwbuf_fpages, next);
+
+ lwb->kva = free_kvp->kva;
+
+ kfree(free_kvp, M_LWBUF);
+ } else {
+ if (lwbuf_initpages(&gd->gd_lwbuf_fpages,
+ LWBUF_ALLOC_PAGES) == FALSE)
+ tsleep(&gd->gd_lwbuf_fpages, 0, "lwbuf", hz);
+
+ goto check_slist;
+ }
+
+ pmap_kenter_quick(lwb->kva, lwb->m->phys_addr);
+ lwb->cpumask |= gd->mi.gd_cpumask;
+
+ return (lwb);
+}
+
+void
+lwbuf_free(struct lwbuf *lwb)
+{
+ struct lwbuf_free_kvp *free_kvp;
+
+ free_kvp = (struct lwbuf_free_kvp *)
+ kmalloc(sizeof(*free_kvp), M_LWBUF, M_WAITOK);
+ free_kvp->kva = lwb->kva;
+ SLIST_INSERT_HEAD(&mdcpu->gd_lwbuf_fpages, free_kvp, next);
+ wakeup_one(&mdcpu->gd_lwbuf_fpages);
+
+ lwb->m = NULL;
+ lwb->kva = 0;
+ lwb->cpumask = 0;
+
+ objcache_put(lwbuf_cache, lwb);
+}
+
+void
+lwbuf_set_global(struct lwbuf *lwb)
+{
+ pmap_kenter_sync(lwb->kva);
+ lwb->cpumask = (cpumask_t)-1;
+}
+
+static vm_offset_t
+_lwbuf_kva(struct lwbuf *lwb, struct mdglobaldata *gd)
+{
+ cpumask_t old, new;
+
+ pmap_kenter_sync_quick(lwb->kva);
+
+ do {
+ old = lwb->cpumask;
+ new = old | gd->mi.gd_cpumask;
+ } while (atomic_cmpset_int(&lwb->cpumask, old, new) == 0);
+
+ return (lwb->kva);
+}
+
+__inline vm_offset_t
+lwbuf_kva(struct lwbuf *lwb)
+{
+ struct mdglobaldata *gd = mdcpu;
+
+ if (lwb->cpumask & gd->mi.gd_cpumask)
+ return (lwb->kva);
+
+ return (_lwbuf_kva(lwb, gd));
+}
--- /dev/null
+/*
+ * Copyright (c) 2010 by The DragonFly Project and Samuel J. Greear.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The DragonFly Project
+ * by Samuel J. Greear <sjg@thesjg.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * 3. Neither the name of The DragonFly Project nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific, prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _CPU_LWBUF_H_
+#define _CPU_LWBUF_H_
+
+#ifndef _SYS_TYPES_H_
+#include <sys/types.h>
+#endif
+#ifndef _SYS_GLOBALDATA_H_
+#include <sys/globaldata.h>
+#endif
+#ifndef _VM_PMAP_H_
+#include <vm/pmap.h>
+#endif
+#ifndef _MACHINE_ATOMIC_H_
+#include <machine/atomic.h>
+#endif
+
+#if !defined(_KERNEL) && !defined(_KERNEL_STRUCTURES)
+#error "This file should not be included by userland programs."
+#endif
+
+struct lwbuf {
+ vm_page_t m; /* currently mapped page */
+ vm_offset_t kva; /* va of mapping */
+};
+
+static __inline vm_page_t
+lwbuf_page(struct lwbuf *lwb) {
+ return (lwb->m);
+}
+
+static __inline vm_offset_t
+lwbuf_kva(struct lwbuf *lwb) {
+ return (lwb->kva);
+}
+
+#define lwbuf_set_global(lwb)
+
+#if defined(_KERNEL)
+
+struct lwbuf *lwbuf_alloc(vm_page_t);
+void lwbuf_free(struct lwbuf *);
+
+#endif
+
+#endif /* !_CPU_LWBUF_H_ */
--- /dev/null
+/*
+ * Copyright (c) 2010 by The DragonFly Project and Samuel J. Greear.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The DragonFly Project
+ * by Samuel J. Greear <sjg@thesjg.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * 3. Neither the name of The DragonFly Project nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific, prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/types.h>
+#include <sys/kernel.h>
+#include <sys/objcache.h>
+#include <sys/systm.h>
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_page.h>
+#include <cpu/lwbuf.h>
+#include <machine/param.h>
+
+static void lwbuf_init(void *);
+SYSINIT(sock_lwb, SI_BOOT2_MACHDEP, SI_ORDER_ANY, lwbuf_init, NULL);
+
+static struct objcache *lwbuf_cache;
+
+MALLOC_DEFINE(M_LWBUF, "lwbuf", "Lightweight buffers");
+struct objcache_malloc_args lwbuf_malloc_args = { sizeof(struct lwbuf), M_LWBUF };
+
+
+static boolean_t
+lwbuf_cache_ctor(void *obj, void *pdata, int ocflags)
+{
+ struct lwbuf *lwb = (struct lwbuf *)obj;
+
+ lwb->m = NULL;
+ lwb->kva = 0;
+
+ return (TRUE);
+}
+
+static void
+lwbuf_init(void *arg)
+{
+ lwbuf_cache = objcache_create("lwbuf", 0, 0,
+ lwbuf_cache_ctor, NULL, NULL,
+ objcache_malloc_alloc, objcache_malloc_free,
+ &lwbuf_malloc_args);
+}
+
+struct lwbuf *
+lwbuf_alloc(vm_page_t m)
+{
+ struct lwbuf *lwb;
+
+ if ((lwb = objcache_get(lwbuf_cache, M_WAITOK)) == NULL)
+ return (NULL);
+
+ lwb->m = m;
+ lwb->kva = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(lwb->m));
+
+ return (lwb);
+}
+
+void
+lwbuf_free(struct lwbuf *lwb)
+{
+ lwb->m = NULL;
+
+ objcache_put(lwbuf_cache, lwb);
+}
#include <sys/sysctl.h>
#include <sys/sysent.h>
#include <sys/vnode.h>
-#include <sys/sfbuf.h>
#include <sys/eventhandler.h>
+#include <cpu/lwbuf.h>
+
#include <vm/vm.h>
#include <vm/vm_kern.h>
#include <vm/vm_param.h>
{
Elf_Brandinfo *match = NULL;
const Elf_Note *tmp_note;
- struct sf_buf *sfb;
+ struct lwbuf *lwb;
const char *page;
char *data = NULL;
Elf_Off off;
if (len < sizeof(Elf_Note) || len > PAGE_SIZE)
return NULL; /* ENOEXEC? */
- if (exec_map_page(imgp, off >> PAGE_SHIFT, &sfb, &page))
+ if (exec_map_page(imgp, off >> PAGE_SHIFT, &lwb, &page))
return NULL;
/*
bcopy(page + firstoff, data, firstlen);
- exec_unmap_page(sfb);
- if (exec_map_page(imgp, (off >> PAGE_SHIFT) + 1, &sfb, &page)) {
+ exec_unmap_page(lwb);
+ if (exec_map_page(imgp, (off >> PAGE_SHIFT) + 1, &lwb, &page)) {
kfree(data, M_TEMP);
return NULL;
}
if (data != NULL)
kfree(data, M_TEMP);
- exec_unmap_page(sfb);
+ exec_unmap_page(lwb);
return (match);
}
if (copy_len != 0) {
vm_page_t m;
- struct sf_buf *sf;
+ struct lwbuf *lwb;
m = vm_fault_object_page(object, trunc_page(offset + filsz),
VM_PROT_READ, 0, &error);
if (m) {
- sf = sf_buf_alloc(m, SFB_CPUPRIVATE);
- error = copyout((caddr_t)sf_buf_kva(sf),
+ lwb = lwbuf_alloc(m);
+ error = copyout((caddr_t)lwbuf_kva(lwb),
(caddr_t)map_addr, copy_len);
- sf_buf_free(sf);
+ lwbuf_free(lwb);
vm_page_unhold(m);
}
if (error) {
#include <sys/signalvar.h>
#include <sys/pioctl.h>
#include <sys/nlookup.h>
-#include <sys/sfbuf.h>
#include <sys/sysent.h>
#include <sys/shm.h>
#include <sys/sysctl.h>
#include <sys/aio.h>
#include <sys/libkern.h>
+#include <cpu/lwbuf.h>
+
#include <vm/vm.h>
#include <vm/vm_param.h>
#include <sys/lock.h>
int
exec_map_page(struct image_params *imgp, vm_pindex_t pageno,
- struct sf_buf **psfb, const char **pdata)
+ struct lwbuf **plwb, const char **pdata)
{
int rv;
vm_page_t ma;
vm_page_wakeup(m); /* unbusy the page */
crit_exit();
- *psfb = sf_buf_alloc(m, SFB_CPUPRIVATE);
- *pdata = (void *)sf_buf_kva(*psfb);
+ *plwb = lwbuf_alloc(m);
+ *pdata = (void *)lwbuf_kva(*plwb);
return (0);
}
}
void
-exec_unmap_page(struct sf_buf *sfb)
+exec_unmap_page(struct lwbuf *lwb)
{
vm_page_t m;
crit_enter();
- if (sfb != NULL) {
- m = sf_buf_page(sfb);
- sf_buf_free(sfb);
+ if (lwb != NULL) {
+ m = lwbuf_page(lwb);
+ lwbuf_free(lwb);
vm_page_unhold(m);
}
crit_exit();
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
-#include <sys/queue.h>
#include <sys/sfbuf.h>
-#include <sys/globaldata.h>
-#include <sys/thread.h>
-#include <sys/sysctl.h>
+#include <sys/objcache.h>
+
+#include <cpu/lwbuf.h>
+
#include <vm/vm.h>
#include <vm/vm_extern.h>
#include <vm/vm_kern.h>
#include <vm/vm_page.h>
#include <vm/pmap.h>
-#include <sys/thread2.h>
static void sf_buf_init(void *arg);
SYSINIT(sock_sf, SI_BOOT2_MACHDEP, SI_ORDER_ANY, sf_buf_init, NULL)
LIST_HEAD(sf_buf_list, sf_buf);
-SYSCTL_INT(_kern_ipc, OID_AUTO, nsfbufs, CTLFLAG_RD, &nsfbufs, 0,
- "Maximum number of sf_bufs available to the system");
+static struct objcache *sf_buf_cache;
-/*
- * A hash table of active sendfile(2) buffers
- */
-static struct sf_buf_list *sf_buf_hashtable;
-static u_long sf_buf_hashmask;
+MALLOC_DEFINE(M_SFBUF, "sfbuf", "Sendfile buffer structures");
+struct objcache_malloc_args sf_buf_malloc_args = { sizeof(struct sf_buf), M_SFBUF };
-static TAILQ_HEAD(, sf_buf) sf_buf_freelist;
-static u_int sf_buf_alloc_want;
-static vm_offset_t sf_base;
-static struct sf_buf *sf_bufs;
-
-static int sfbuf_quick = 1;
-SYSCTL_INT(_debug, OID_AUTO, sfbuf_quick, CTLFLAG_RW, &sfbuf_quick, 0, "");
-static int nsffree;
-SYSCTL_INT(_kern_ipc, OID_AUTO, nsffree, CTLFLAG_RD, &nsffree, 0,
- "Number of free sf_bufs available to the system");
-
-static __inline
-int
-sf_buf_hash(vm_page_t m)
+static boolean_t
+sf_buf_cache_ctor(void *obj, void *pdata, int ocflags)
{
- int hv;
+ struct sf_buf *sf = (struct sf_buf *)obj;
- hv = ((int)(intptr_t)m / sizeof(vm_page_t)) + ((int)(intptr_t)m >> 12);
- return(hv & sf_buf_hashmask);
+ sf->lwbuf = NULL;
+ sf->refs = 0;
+
+ return (TRUE);
}
/*
- * Allocate a pool of sf_bufs (sendfile(2) or "super-fast" if you prefer. :-))
+ * Init objcache of sf_bufs (sendfile(2) or "super-fast" if you prefer. :-))
*/
static void
sf_buf_init(void *arg)
{
- int i;
-
- sf_buf_hashtable = hashinit(nsfbufs, M_TEMP, &sf_buf_hashmask);
- TAILQ_INIT(&sf_buf_freelist);
- sf_base = kmem_alloc_nofault(&kernel_map, nsfbufs * PAGE_SIZE,
- PAGE_SIZE);
- sf_bufs = kmalloc(nsfbufs * sizeof(struct sf_buf), M_TEMP,
- M_WAITOK | M_ZERO);
- for (i = 0; i < nsfbufs; i++) {
- sf_bufs[i].kva = sf_base + i * PAGE_SIZE;
- sf_bufs[i].flags |= SFBA_ONFREEQ;
- TAILQ_INSERT_TAIL(&sf_buf_freelist, &sf_bufs[i], free_entry);
- ++nsffree;
- }
+ sf_buf_cache = objcache_create("sf_buf", 0, 0,
+ sf_buf_cache_ctor, NULL, NULL,
+ objcache_malloc_alloc, objcache_malloc_free,
+ &sf_buf_malloc_args);
}
/*
- * Get an sf_buf from the freelist. Will block if none are available.
+ * Acquire an sf_buf reference for a vm_page.
*/
struct sf_buf *
-sf_buf_alloc(struct vm_page *m, int flags)
+sf_buf_alloc(struct vm_page *m)
{
- struct sf_buf_list *hash_chain;
struct sf_buf *sf;
- globaldata_t gd;
- int error;
- int pflags;
-
- gd = mycpu;
- crit_enter();
- hash_chain = &sf_buf_hashtable[sf_buf_hash(m)];
- LIST_FOREACH(sf, hash_chain, list_entry) {
- if (sf->m == m) {
- /*
- * cache hit
- *
- * We must invalidate the TLB entry based on whether
- * it need only be valid on the local cpu (SFB_CPUPRIVATE),
- * or on all cpus. This is conditionalized and in
- * most cases no system-wide invalidation should be
- * needed.
- *
- * Note: we do not remove the entry from the freelist
- * on the 0->1 transition.
- */
- ++sf->refcnt;
- if ((flags & SFB_CPUPRIVATE) && sfbuf_quick) {
- if ((sf->cpumask & gd->gd_cpumask) == 0) {
- pmap_kenter_sync_quick(sf->kva);
- sf->cpumask |= gd->gd_cpumask;
- }
- } else {
- if (sf->cpumask != (cpumask_t)-1) {
- pmap_kenter_sync(sf->kva);
- sf->cpumask = (cpumask_t)-1;
- }
- }
- goto done; /* found existing mapping */
- }
+
+ if ((sf = objcache_get(sf_buf_cache, M_WAITOK)) == NULL)
+ goto done;
+
+ if ((sf->lwbuf = lwbuf_alloc(m)) == NULL) {
+ objcache_put(sf_buf_cache, sf);
+ sf = NULL;
+ goto done;
}
/*
- * Didn't find old mapping. Get a buffer off the freelist. We
- * may have to remove and skip buffers with non-zero ref counts
- * that were lazily allocated.
+ * Force invalidation of the TLB entry on all CPU's
*/
- for (;;) {
- if ((sf = TAILQ_FIRST(&sf_buf_freelist)) == NULL) {
- pflags = (flags & SFB_CATCH) ? PCATCH : 0;
- ++sf_buf_alloc_want;
- error = tsleep(&sf_buf_freelist, pflags, "sfbufa", 0);
- --sf_buf_alloc_want;
- if (error)
- goto done;
- } else {
- /*
- * We may have to do delayed removals for referenced
- * sf_buf's here in addition to locating a sf_buf
- * to reuse. The sf_bufs must be removed.
- *
- * We are finished when we find an sf_buf with a
- * refcnt of 0. We theoretically do not have to
- * remove it from the freelist but it's a good idea
- * to do so to preserve LRU operation for the
- * (1) never before seen before case and (2)
- * accidently recycled due to prior cached uses not
- * removing the buffer case.
- */
- KKASSERT(sf->flags & SFBA_ONFREEQ);
- TAILQ_REMOVE(&sf_buf_freelist, sf, free_entry);
- --nsffree;
- sf->flags &= ~SFBA_ONFREEQ;
- if (sf->refcnt == 0)
- break;
- }
- }
- if (sf->m != NULL) /* remove previous mapping from hash table */
- LIST_REMOVE(sf, list_entry);
- LIST_INSERT_HEAD(hash_chain, sf, list_entry);
- sf->refcnt = 1;
- sf->m = m;
- if ((flags & SFB_CPUPRIVATE) && sfbuf_quick) {
- pmap_kenter_quick(sf->kva, sf->m->phys_addr);
- sf->cpumask = gd->gd_cpumask;
- } else {
- pmap_kenter(sf->kva, sf->m->phys_addr);
- sf->cpumask = (cpumask_t)-1;
- }
+ lwbuf_set_global(sf->lwbuf);
+
+ sf->refs = 1;
+
done:
- crit_exit();
return (sf);
}
-#if 0
-
-/*
- * Add a reference to a buffer (currently unused)
- */
void
-sf_buf_ref(struct sf_buf *sf)
+sf_buf_ref(void *arg)
{
- if (sf->refcnt == 0)
- panic("sf_buf_ref: referencing a free sf_buf");
- crit_enter();
- sf->refcnt++;
- crit_exit();
-}
+ struct sf_buf *sf = arg;
-#endif
+ atomic_add_int(&sf->refs, 1);
+}
/*
- * Lose a reference to an sf_buf. When none left, detach mapped page
- * and release resources back to the system. Note that the sfbuf's
- * removal from the freelist is delayed, so it may in fact already be
- * on the free list. This is the optimal (and most likely) scenario.
+ * Detach mapped page and release resources back to the system.
*
* Must be called at splimp.
*/
-void
-sf_buf_free(struct sf_buf *sf)
+int
+sf_buf_free(void *arg)
{
- if (sf->refcnt == 0)
- panic("sf_buf_free: freeing free sf_buf");
- crit_enter();
- sf->refcnt--;
- if (sf->refcnt == 0 && (sf->flags & SFBA_ONFREEQ) == 0) {
- TAILQ_INSERT_TAIL(&sf_buf_freelist, sf, free_entry);
- ++nsffree;
- sf->flags |= SFBA_ONFREEQ;
- if (sf_buf_alloc_want > 0)
- wakeup_one(&sf_buf_freelist);
+ struct sf_buf *sf = arg;
+
+ KKASSERT(sf->refs > 0);
+ if (atomic_fetchadd_int(&sf->refs, -1) == 1) {
+ lwbuf_free(sf->lwbuf);
+ objcache_put(sf_buf_cache, sf);
+ return (0);
}
- crit_exit();
-}
+ return (1);
+}
#include <sys/sysctl.h>
#include <sys/uio.h>
#include <sys/vnode.h>
-#include <sys/sfbuf.h>
#include <sys/thread2.h>
#include <machine/limits.h>
+#include <cpu/lwbuf.h>
+
#include <vm/vm.h>
#include <vm/vm_page.h>
#include <vm/vm_map.h>
*/
/*
- * Implement uiomove(9) from physical memory using sf_bufs to reduce
+ * Implement uiomove(9) from physical memory using lwbuf's to reduce
* the creation and destruction of ephemeral mappings.
*/
int
uiomove_fromphys(vm_page_t *ma, vm_offset_t offset, size_t n, struct uio *uio)
{
- struct sf_buf *sf;
+ struct lwbuf *lwb;
struct thread *td = curthread;
struct iovec *iov;
void *cp;
page_offset = offset & PAGE_MASK;
cnt = min(cnt, PAGE_SIZE - page_offset);
m = ma[offset >> PAGE_SHIFT];
- sf = sf_buf_alloc(m, SFB_CPUPRIVATE);
- cp = (char *)sf_buf_kva(sf) + page_offset;
+ lwb = lwbuf_alloc(m);
+ cp = (char *)lwbuf_kva(lwb) + page_offset;
switch (uio->uio_segflg) {
case UIO_USERSPACE:
/*
else
error = copyin(iov->iov_base, cp, cnt);
if (error) {
- sf_buf_free(sf);
+ lwbuf_free(lwb);
goto out;
}
break;
case UIO_NOCOPY:
break;
}
- sf_buf_free(sf);
+ lwbuf_free(lwb);
iov->iov_base = (char *)iov->iov_base + cnt;
iov->iov_len -= cnt;
uio->uio_resid -= cnt;
#include <sys/sysunion.h>
#include <sys/sysent.h>
#include <sys/syscall.h>
-#include <sys/sfbuf.h>
#include <sys/module.h>
+#include <cpu/lwbuf.h>
+
#include <vm/vm.h>
#include <vm/vm_param.h>
#include <sys/lock.h>
sys_umtx_sleep(struct umtx_sleep_args *uap)
{
int error = EBUSY;
- struct sf_buf *sf;
+ struct lwbuf *lwb;
struct vm_page_action action;
vm_page_t m;
void *waddr;
error = EFAULT;
goto done;
}
- sf = sf_buf_alloc(m, SFB_CPUPRIVATE);
+ lwb = lwbuf_alloc(m);
offset = (vm_offset_t)uap->ptr & PAGE_MASK;
/*
* The critical section is required to interlock the tsleep against
* a wakeup from another cpu. The lfence forces synchronization.
*/
- if (*(int *)(sf_buf_kva(sf) + offset) == uap->value) {
+ if (*(int *)(lwbuf_kva(lwb) + offset) == uap->value) {
if ((timeout = uap->timeout) != 0) {
timeout = (timeout / 1000000) * hz +
((timeout % 1000000) * hz + 999999) / 1000000;
waddr = (void *)((intptr_t)VM_PAGE_TO_PHYS(m) + offset);
crit_enter();
tsleep_interlock(waddr, PCATCH | PDOMAIN_UMTX);
- if (*(int *)(sf_buf_kva(sf) + offset) == uap->value) {
+ if (*(int *)(lwbuf_kva(lwb) + offset) == uap->value) {
vm_page_init_action(&action, umtx_sleep_page_action_cow, waddr);
vm_page_register_action(m, &action, VMEVENT_COW);
error = tsleep(waddr, PCATCH | PINTERLOCKED | PDOMAIN_UMTX,
error = EBUSY;
}
- sf_buf_free(sf);
+ lwbuf_free(lwb);
/*vm_page_dirty(m); we don't actually dirty the page */
vm_page_unhold(m);
done:
#include <sys/vmmeter.h>
#include <sys/vnode.h>
#include <sys/xio.h>
-#include <sys/sfbuf.h>
+
+#include <cpu/lwbuf.h>
#include <vm/vm.h>
#include <vm/vm_param.h>
int error;
int offset;
vm_page_t m;
- struct sf_buf *sf;
+ struct lwbuf *lwb;
if (uoffset + bytes > xio->xio_bytes)
return(EFAULT);
++i
) {
m = xio->xio_pages[i];
- sf = sf_buf_alloc(m, SFB_CPUPRIVATE);
- error = copyout((char *)sf_buf_kva(sf) + offset, uptr, n);
- sf_buf_free(sf);
+ lwb = lwbuf_alloc(m);
+ error = copyout((char *)lwbuf_kva(lwb) + offset, uptr, n);
+ lwbuf_free(lwb);
if (error)
break;
bytes -= n;
int error;
int offset;
vm_page_t m;
- struct sf_buf *sf;
+ struct lwbuf *lwb;
if (bytes + uoffset > xio->xio_bytes)
return(EFAULT);
++i
) {
m = xio->xio_pages[i];
- sf = sf_buf_alloc(m, SFB_CPUPRIVATE);
- bcopy((char *)sf_buf_kva(sf) + offset, kptr, n);
- sf_buf_free(sf);
+ lwb = lwbuf_alloc(m);
+ bcopy((char *)lwbuf_kva(lwb) + offset, kptr, n);
+ lwbuf_free(lwb);
bytes -= n;
kptr = (char *)kptr + n;
if (bytes == 0)
int error;
int offset;
vm_page_t m;
- struct sf_buf *sf;
+ struct lwbuf *lwb;
if (uoffset + bytes > xio->xio_bytes)
return(EFAULT);
++i
) {
m = xio->xio_pages[i];
- sf = sf_buf_alloc(m, SFB_CPUPRIVATE);
- error = copyin(uptr, (char *)sf_buf_kva(sf) + offset, n);
- sf_buf_free(sf);
+ lwb = lwbuf_alloc(m);
+ error = copyin(uptr, (char *)lwbuf_kva(lwb) + offset, n);
+ lwbuf_free(lwb);
if (error)
break;
bytes -= n;
int error;
int offset;
vm_page_t m;
- struct sf_buf *sf;
+ struct lwbuf *lwb;
if (uoffset + bytes > xio->xio_bytes)
return(EFAULT);
++i
) {
m = xio->xio_pages[i];
- sf = sf_buf_alloc(m, SFB_CPUPRIVATE);
- bcopy(kptr, (char *)sf_buf_kva(sf) + offset, n);
- sf_buf_free(sf);
+ lwb = lwbuf_alloc(m);
+ bcopy(kptr, (char *)lwbuf_kva(lwb) + offset, n);
+ lwbuf_free(lwb);
bytes -= n;
kptr = (const char *)kptr + n;
if (bytes == 0)
#ifndef MAXFILES
#define MAXFILES (maxproc * 16)
#endif
-#ifndef NSFBUFS
-#define NSFBUFS (512 + maxusers * 16)
-#endif
#ifndef MAXPOSIXLOCKSPERUID
#define MAXPOSIXLOCKSPERUID (maxusers * 64) /* Should be a safe value */
#endif
u_quad_t maxssiz; /* max stack size */
u_quad_t sgrowsiz; /* amount to grow stack */
-/* maximum # of sf_bufs (sendfile(2) zero-copy virtual buffers) */
-int nsfbufs;
-
/*
* These have to be allocated somewhere; allocating
* them here forces loader errors if this file is omitted
TUNABLE_INT_FETCH("kern.maxposixlocksperuid", &maxposixlocksperuid);
/*
- * Cannot be changed after boot. Unless overriden, NSFBUFS is based
- * on maxusers and NBUF is typically 0 (auto-sized later).
+ * Unless overriden, NBUF is typically 0 (auto-sized later).
*/
- nsfbufs = NSFBUFS;
- TUNABLE_INT_FETCH("kern.ipc.nsfbufs", &nsfbufs);
nbuf = NBUF;
TUNABLE_INT_FETCH("kern.nbuf", &nbuf);
#include <netinet/sctp_peeloff.h>
#endif /* SCTP */
-struct sfbuf_mref {
- struct sf_buf *sf;
- int mref_count;
-};
-
-static MALLOC_DEFINE(M_SENDFILE, "sendfile", "sendfile sfbuf ref structures");
-
/*
* System call interface to the socket abstraction.
*/
* XXX vm_page_*() routines are not MPSAFE yet, the MP lock is required.
*/
static void
-sf_buf_mref(void *arg)
-{
- struct sfbuf_mref *sfm = arg;
-
- /*
- * We must already hold a ref so there is no race to 0, just
- * atomically increment the count.
- */
- atomic_add_int(&sfm->mref_count, 1);
-}
-
-static void
sf_buf_mfree(void *arg)
{
- struct sfbuf_mref *sfm = arg;
+ struct sf_buf *sf = arg;
vm_page_t m;
- KKASSERT(sfm->mref_count > 0);
- if (atomic_fetchadd_int(&sfm->mref_count, -1) == 1) {
- /*
- * XXX vm_page_*() and SFBUF routines not MPSAFE yet.
- */
- get_mplock();
- crit_enter();
- m = sf_buf_page(sfm->sf);
- sf_buf_free(sfm->sf);
+ /*
+ * XXX vm_page_*() and SFBUF routines not MPSAFE yet.
+ */
+ get_mplock();
+ crit_enter();
+ m = sf_buf_page(sf);
+ if (sf_buf_free(sf) == 0) {
vm_page_unwire(m, 0);
if (m->wire_count == 0 && m->object == NULL)
vm_page_try_to_free(m);
- crit_exit();
- rel_mplock();
- kfree(sfm, M_SENDFILE);
}
+ crit_exit();
+ rel_mplock();
}
/*
struct file *fp;
struct mbuf *m;
struct sf_buf *sf;
- struct sfbuf_mref *sfm;
struct vm_page *pg;
off_t off, xfsize;
off_t hbytes = 0;
* Get a sendfile buf. We usually wait as long as necessary,
* but this wait can be interrupted.
*/
- if ((sf = sf_buf_alloc(pg, SFB_CATCH)) == NULL) {
+ if ((sf = sf_buf_alloc(pg)) == NULL) {
crit_enter();
vm_page_unwire(pg, 0);
vm_page_try_to_free(pg);
goto done;
}
- /*
- * sfm is a temporary hack, use a per-cpu cache for this.
- */
- sfm = kmalloc(sizeof(struct sfbuf_mref), M_SENDFILE, M_WAITOK);
- sfm->sf = sf;
- sfm->mref_count = 1;
-
m->m_ext.ext_free = sf_buf_mfree;
- m->m_ext.ext_ref = sf_buf_mref;
- m->m_ext.ext_arg = sfm;
- m->m_ext.ext_buf = (void *)sf->kva;
+ m->m_ext.ext_ref = sf_buf_ref;
+ m->m_ext.ext_arg = sf;
+ m->m_ext.ext_buf = (void *)sf_buf_kva(sf);
m->m_ext.ext_size = PAGE_SIZE;
- m->m_data = (char *) sf->kva + pgoff;
+ m->m_data = (char *)sf_buf_kva(sf) + pgoff;
m->m_flags |= M_EXT;
m->m_pkthdr.len = m->m_len = xfsize;
KKASSERT((m->m_flags & (M_EXT_CLUSTER)) == 0);
cpu/i386/misc/atomic.c standard \
compile-with "${CC} -c ${CFLAGS} ${WERROR} ${DEFINED_PROF:S/^$/-fomit-frame-pointer/} ${.IMPSRC}"
cpu/i386/misc/elf_machdep.c standard
+cpu/i386/misc/lwbuf.c standard
cpu/i386/misc/in_cksum2.s optional inet
cpu/i386/misc/ktr.c optional ktr
cpu/i386/misc/db_disasm.c optional ddb
ASSYM(GD_COMMON_TSS, offsetof(struct mdglobaldata, gd_common_tss));
ASSYM(GD_COMMON_TSSD, offsetof(struct mdglobaldata, gd_common_tssd));
ASSYM(GD_TSS_GDT, offsetof(struct mdglobaldata, gd_tss_gdt));
+ASSYM(GD_LWBUF_FPAGES, offsetof(struct mdglobaldata, gd_lwbuf_fpages));
ASSYM(GD_NPXTHREAD, offsetof(struct mdglobaldata, gd_npxthread));
ASSYM(GD_FPU_LOCK, offsetof(struct mdglobaldata, gd_fpu_lock));
ASSYM(GD_SAVEFPU, offsetof(struct mdglobaldata, gd_savefpu));
.set gd_reqflags,globaldata + GD_REQFLAGS
.set gd_common_tss,globaldata + GD_COMMON_TSS
+ .globl gd_lwbuf_fpages
+ .set gd_lwbuf_fpages,globaldata + GD_LWBUF_FPAGES
+
.globl gd_common_tssd, gd_tss_gdt
.set gd_common_tssd,globaldata + GD_COMMON_TSSD
.set gd_tss_gdt,globaldata + GD_TSS_GDT
#ifndef _MACHINE_NPX_H_
#include <machine/npx.h>
#endif
+#ifndef _CPU_LWBUF_H_
+#include <cpu/lwbuf.h>
+#endif
/*
* Note on interrupt control. Pending interrupts not yet dispatched are
struct segment_descriptor *gd_tss_gdt;
struct thread *gd_npxthread;
struct i386tss gd_common_tss;
+ struct lwbuf_free_kvp_list gd_lwbuf_fpages; /* lwbuf: free kva */
union savefpu gd_savefpu; /* fast bcopy/zero temp fpu save area */
int gd_fpu_lock; /* fast bcopy/zero cpu lock */
int gd_fpending; /* fast interrupt pending */
# DDB XXX
cpu/x86_64/misc/x86_64-gdbstub.c optional ddb
+cpu/x86_64/misc/lwbuf.c standard
platform/pc64/x86_64/elf_machdep.c standard
platform/pc64/x86_64/in_cksum2.s optional inet
platform/pc64/x86_64/ktr.c optional ktr
#
# DDB XXX
cpu/i386/misc/elf_machdep.c standard
+cpu/i386/misc/lwbuf.c standard
cpu/i386/misc/in_cksum2.s optional inet
cpu/i386/misc/ktr.c optional ktr
cpu/i386/misc/db_disasm.c optional ddb
ASSYM(GD_COMMON_TSS, offsetof(struct mdglobaldata, gd_common_tss));
ASSYM(GD_COMMON_TSSD, offsetof(struct mdglobaldata, gd_common_tssd));
ASSYM(GD_TSS_GDT, offsetof(struct mdglobaldata, gd_tss_gdt));
+ASSYM(GD_LWBUF_FPAGES, offsetof(struct mdglobaldata, gd_lwbuf_fpages));
ASSYM(GD_NPXTHREAD, offsetof(struct mdglobaldata, gd_npxthread));
ASSYM(GD_FPU_LOCK, offsetof(struct mdglobaldata, gd_fpu_lock));
ASSYM(GD_SAVEFPU, offsetof(struct mdglobaldata, gd_savefpu));
.set gd_reqflags,globaldata + GD_REQFLAGS
.set gd_common_tss,globaldata + GD_COMMON_TSS
+ .globl gd_lwbuf_fpages
+ .set gd_lwbuf_fpages,globaldata + GD_LWBUF_FPAGES
+
.globl gd_common_tssd, gd_tss_gdt
.set gd_common_tssd,globaldata + GD_COMMON_TSSD
.set gd_tss_gdt,globaldata + GD_TSS_GDT
#ifndef _MACHINE_NPX_H_
#include <machine/npx.h>
#endif
+#ifndef _CPU_LWBUF_H_
+#include <cpu/lwbuf.h>
+#endif
/*
* Note on interrupt control. Pending interrupts not yet dispatched are
struct segment_descriptor *gd_tss_gdt;
struct thread *gd_npxthread;
struct i386tss gd_common_tss;
+ struct lwbuf_free_kvp_list gd_lwbuf_fpages; /* lwbuf: free kva */
union savefpu gd_savefpu; /* fast bcopy/zero temp fpu save area */
int gd_fpu_lock; /* fast bcopy/zero cpu lock */
int gd_fpending; /* fast interrupt pending */
#include <sys/types.h>
#include <sys/systm.h>
-#include <sys/sfbuf.h>
+#include <cpu/lwbuf.h>
#include <vm/vm_page.h>
#include <vm/vm_extern.h>
#include <assert.h>
copyin(const void *udaddr, void *kaddr, size_t len)
{
struct vmspace *vm = curproc->p_vmspace;
- struct sf_buf *sf;
+ struct lwbuf *lwb;
vm_page_t m;
int error;
size_t n;
n = PAGE_SIZE - ((vm_offset_t)udaddr & PAGE_MASK);
if (n > len)
n = len;
- sf = sf_buf_alloc(m, SFB_CPUPRIVATE);
- bcopy((char *)sf_buf_kva(sf)+((vm_offset_t)udaddr & PAGE_MASK),
+ lwb = lwbuf_alloc(m);
+ bcopy((char *)lwbuf_kva(lwb)+((vm_offset_t)udaddr & PAGE_MASK),
kaddr, n);
len -= n;
udaddr = (const char *)udaddr + n;
kaddr = (char *)kaddr + n;
vm_page_unhold(m);
- sf_buf_free(sf);
+ lwbuf_free(lwb);
}
rel_mplock();
return (error);
copyout(const void *kaddr, void *udaddr, size_t len)
{
struct vmspace *vm = curproc->p_vmspace;
- struct sf_buf *sf;
+ struct lwbuf *lwb;
vm_page_t m;
int error;
size_t n;
n = PAGE_SIZE - ((vm_offset_t)udaddr & PAGE_MASK);
if (n > len)
n = len;
- sf = sf_buf_alloc(m, SFB_CPUPRIVATE);
- bcopy(kaddr, (char *)sf_buf_kva(sf) +
+ lwb = lwbuf_alloc(m);
+ bcopy(kaddr, (char *)lwbuf_kva(lwb) +
((vm_offset_t)udaddr & PAGE_MASK), n);
len -= n;
udaddr = (char *)udaddr + n;
kaddr = (const char *)kaddr + n;
vm_page_dirty(m);
vm_page_unhold(m);
- sf_buf_free(sf);
+ lwbuf_free(lwb);
}
rel_mplock();
return (error);
#ifdef _KERNEL
#include <sys/cdefs.h>
-struct sf_buf;
-int exec_map_page(struct image_params *, vm_pindex_t, struct sf_buf **,
+struct lwbuf;
+int exec_map_page(struct image_params *, vm_pindex_t, struct lwbuf **,
const char **);
-void exec_unmap_page(struct sf_buf *);
+void exec_unmap_page(struct lwbuf *);
int exec_map_first_page (struct image_params *);
void exec_unmap_first_page (struct image_params *);
char interpreted; /* flag - this executable is interpreted */
char interpreter_name[MAXSHELLCMDLEN]; /* name of the interpreter */
void *auxargs; /* ELF Auxinfo structure pointer */
- struct sf_buf *firstpage; /* first page that we mapped */
+ struct lwbuf *firstpage; /* first page that we mapped */
unsigned long ps_strings; /* PS_STRINGS for BSD/OS binaries */
};
#ifndef _SYS_QUEUE_H_
#include <sys/queue.h>
#endif
+#ifndef _CPU_LWBUF_H_
+#include <cpu/lwbuf.h>
+#endif
#if !defined(_KERNEL) && !defined(_KERNEL_STRUCTURES)
#endif
struct sf_buf {
- LIST_ENTRY(sf_buf) list_entry; /* hash chain of active buffers */
- TAILQ_ENTRY(sf_buf) free_entry; /* list of free buffers */
- struct vm_page *m; /* currently mapped page */
- vm_offset_t kva; /* va of mapping */
- int refcnt; /* usage of this mapping */
- int flags; /* global SFBA flags */
- cpumask_t cpumask; /* cpu mapping synchronization */
+ struct lwbuf *lwbuf; /* lightweight buffer */
+ int refs;
};
-/*
- * sf_buf_alloc() flags (not all are stored in sf->flags)
- */
-#define SFB_CPUPRIVATE 0x0001 /* sync mapping to current cpu only */
-#define SFBA_ONFREEQ 0x0002 /* on the free queue (lazy move) */
-#define SFB_CATCH 0x0004 /* allow interruption */
-
-static __inline vm_offset_t
-sf_buf_kva(struct sf_buf *sf)
-{
- return(sf->kva);
-}
-
-static __inline struct vm_page *
-sf_buf_page(struct sf_buf *sf)
-{
- return(sf->m);
-}
+#define sf_buf_kva(sf) (lwbuf_kva((sf)->lwbuf))
+#define sf_buf_page(sf) (lwbuf_page((sf)->lwbuf))
#if defined(_KERNEL)
-extern int nsfbufs;
-
-struct sf_buf *sf_buf_alloc(struct vm_page *, int flags);
-void sf_buf_free(struct sf_buf *);
-void sf_buf_ref(struct sf_buf *);
+struct sf_buf *sf_buf_alloc(struct vm_page *);
+void sf_buf_ref(void *);
+int sf_buf_free(void *);
#endif
#include <sys/proc.h>
#include <sys/resourcevar.h>
#include <sys/sched.h>
-#include <sys/sfbuf.h>
#include <sys/stat.h>
#include <sys/systm.h>
#include <sys/unistd.h>
#include <sys/resourcevar.h>
#include <sys/vmmeter.h>
#include <sys/vkernel.h>
-#include <sys/sfbuf.h>
#include <sys/lock.h>
#include <sys/sysctl.h>
+#include <cpu/lwbuf.h>
+
#include <vm/vm.h>
#include <vm/vm_param.h>
#include <vm/pmap.h>
vm_fault_vpagetable(struct faultstate *fs, vm_pindex_t *pindex,
vpte_t vpte, int fault_type)
{
- struct sf_buf *sf;
+ struct lwbuf *lwb;
int vshift = 32 - PAGE_SHIFT; /* page index bits remaining */
int result = KERN_SUCCESS;
vpte_t *ptep;
* entry in the page table page.
*/
vshift -= VPTE_PAGE_BITS;
- sf = sf_buf_alloc(fs->m, SFB_CPUPRIVATE);
- ptep = ((vpte_t *)sf_buf_kva(sf) +
+ lwb = lwbuf_alloc(fs->m);
+ ptep = ((vpte_t *)lwbuf_kva(lwb) +
((*pindex >> vshift) & VPTE_PAGE_MASK));
vpte = *ptep;
vm_page_dirty(fs->m);
}
}
- sf_buf_free(sf);
+ lwbuf_free(lwb);
vm_page_flag_set(fs->m, PG_REFERENCED);
vm_page_activate(fs->m);
vm_page_wakeup(fs->m);
#include <sys/buf.h>
#include <sys/vmmeter.h>
#include <sys/conf.h>
-#include <sys/sfbuf.h>
+
+#include <cpu/lwbuf.h>
#include <vm/vm.h>
#include <vm/vm_object.h>
if (m && m->valid) {
int base = (int)nsize & PAGE_MASK;
int size = PAGE_SIZE - base;
- struct sf_buf *sf;
+ struct lwbuf *lwb;
/*
* Clear out partial-page garbage in case
* This is byte aligned.
*/
vm_page_busy(m);
- sf = sf_buf_alloc(m, SFB_CPUPRIVATE);
- kva = sf_buf_kva(sf);
+ lwb = lwbuf_alloc(m);
+ kva = lwbuf_kva(lwb);
bzero((caddr_t)kva + base, size);
- sf_buf_free(sf);
+ lwbuf_free(lwb);
/*
* XXX work around SMP data integrity race
projects / dragonfly.git / commitdiff