From d6a46bb7c40983b9dc0aa256e0730c79872a41a2 Mon Sep 17 00:00:00 2001 From: Matthew Dillon Date: Sat, 5 Jun 2004 19:57:40 +0000 Subject: [PATCH] Add the MSFBUF API. MSFBUFs are like SFBUFs but they manage ephermal multi-page mappings instead of single-page mappings. MSFBUFs have the same caching and page invalidation optimizations that SFBUFs have and are considered to be SMP-friendly. Whereas XIO manages pure page lists, MSFBUFs manage KVA mappings of pure page lists. This initial commit just gets the basic API operational. The roadmap for future work includes things like better interactions with third-party XIOs, mapping user buffers into the kernel (extending the xio_init_ubuf() API into the MSFBUF API), and allowing higher level subsystems to pass previously released MSFBUFs as a hint to speed-up regeneration. We also need to come up with a way to overload additional sets of MSFBUFs representing smaller chunks of memory on top of the same KVA space in order to efficiently use our KVA reservation when dealing with subsystems like the buffer cache. MSFBUFs will eventually replace the KVA management in the BUF/BIO, PIPE, and other subsystems which create fake linear mappings with pbufs. The general idea for BUF/BIO will be to use XIO and MSFBUFs to avoid KVA mapping file data through the nominal I/O path. XIO will be the primary I/O buffer mechanism while MSFBUFs will be used when things like UFS decide they need a temporary mapping. This is a collaborative work between Hiten Pandya and Matthew Dillon . --- sys/conf/files | 3 +- sys/kern/kern_msfbuf.c | 282 +++++++++++++++++++++++++++++++++++++++++ sys/kern/kern_xio.c | 16 ++- sys/sys/msfbuf.h | 113 +++++++++++++++++ 4 files changed, 412 insertions(+), 2 deletions(-) create mode 100644 sys/kern/kern_msfbuf.c create mode 100644 sys/sys/msfbuf.h diff --git a/sys/conf/files b/sys/conf/files index 40151a664f..7ffce95020 100644 --- a/sys/conf/files +++ b/sys/conf/files @@ -1,5 +1,5 @@ # $FreeBSD: src/sys/conf/files,v 1.340.2.137 2003/06/04 17:10:30 sam Exp $ -# $DragonFly: src/sys/conf/files,v 1.62 2004/06/04 20:35:35 dillon Exp $ +# $DragonFly: src/sys/conf/files,v 1.63 2004/06/05 19:57:40 dillon Exp $ # # The long compile-with and dependency lines are required because of # limitations in config: backslash-newline doesn't work in strings, and @@ -647,6 +647,7 @@ kern/kern_sig.c standard kern/kern_sysmsg.c standard kern/kern_upcall.c standard kern/kern_sfbuf.c standard +kern/kern_msfbuf.c standard kern/kern_subr.c standard kern/kern_switch.c standard kern/lwkt_thread.c standard diff --git a/sys/kern/kern_msfbuf.c b/sys/kern/kern_msfbuf.c new file mode 100644 index 0000000000..3f097367c9 --- /dev/null +++ b/sys/kern/kern_msfbuf.c @@ -0,0 +1,282 @@ +/*- + * Copyright (c) 2004 Matthew Dillon . + * Copyright (c) 2004 Hiten Pandya . + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDERS, CONTRIBUTORS OR VOICES IN THE AUTHOR'S HEAD + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY + * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT + * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The MSF_BUF API was augmented from the SFBUF API: + * Copyright (c) 1998 David Greenman. All rights reserved. + * src/sys/kern/kern_sfbuf.c,v 1.7 2004/05/13 19:46:18 dillon + * + * $DragonFly: src/sys/kern/kern_msfbuf.c,v 1.1 2004/06/05 19:57:35 dillon Exp $ + */ +/* + * MSFBUFs cache linear multi-page ephermal mappings and operate similar + * to SFBUFs. MSFBUFs use XIO's internally to hold the page list and can + * be considered to be a KVA wrapper around an XIO. + * + * Like the SFBUF subsystem, the locking and validation of the page array + * is the responsibility of the caller. Also like the SFBUF subsystem, + * MSFBUFs are SMP-friendly, cache the mappings, and will avoid unnecessary + * page invalidations when possible. + * + * MSFBUFs are primarily designed to be used in subsystems that manipulate + * XIOs. The DEV and BUF subsystems are a good example. + * + * TODO LIST: + * - Implement the FREEQ optimization that exists in the SFBUF code. + * - Allow allocation (aka mapping) based on an XIO instead of a pglist. + * - Overload XIOs representitive of smaller chunks of memory onto the + * same KVA space to efficiently cache smaller mappings (filesystem + * blocks / buffer cache related). + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +MALLOC_DECLARE(M_MSFBUF); +MALLOC_DEFINE(M_MSFBUF, "MSFBUF", "direct-copy buffers"); + +/* lists and queues associated with msf_bufs */ +LIST_HEAD(msf_buf_list, msf_buf); + +TAILQ_HEAD(, msf_buf) msf_buf_freelist; + +/* hash table for tracking msf_bufs */ +static struct msf_buf_list *msf_buf_hashtable; +static u_long msf_buf_hashmask; + +/* indicate shortage of available msf_bufs */ +static u_int msf_buf_alloc_want; + +/* base of the msf_buf map */ +static vm_offset_t msf_base; +static struct msf_buf *msf_bufs; + +static int num_msf_bufs = 256; /* magic value */ +SYSCTL_INT(_kern_ipc, OID_AUTO, msfbufs, CTLFLAG_RD, &num_msf_bufs, + 0, "number of direct-copy buffers available"); + +static void +msf_buf_init(void *__dummy) +{ + int i; + + msf_buf_alloc_want = 0; + TUNABLE_INT_FETCH("kern.ipc.msfbufs", &num_msf_bufs); + + msf_buf_hashtable = hashinit(num_msf_bufs, M_TEMP, &msf_buf_hashmask); + TAILQ_INIT(&msf_buf_freelist); + + msf_base = kmem_alloc_nofault(kernel_map, + num_msf_bufs * XIO_INTERNAL_SIZE); + + /* + * Use contig. memory for the maps, so it is quicker to access + * linear maps, and they can also be passed to device + * buffers (in the future). + */ + msf_bufs = malloc(num_msf_bufs * sizeof(struct msf_buf), M_MSFBUF, + M_WAITOK|M_ZERO); + + /* Initialize the free list with necessary information. */ + for (i = 0; i < num_msf_bufs; i++) { + msf_bufs[i].m_kva = msf_base + i * XIO_INTERNAL_SIZE; + msf_bufs[i].m_flags |= SFBA_ONFREEQ; + xio_init(&msf_bufs[i].m_xio); + TAILQ_INSERT_TAIL(&msf_buf_freelist, &msf_bufs[i], free_list); + } +} +SYSINIT(msf_buf, SI_SUB_MBUF, SI_ORDER_ANY, msf_buf_init, NULL); + +/* + * Hash the base page of an MSF's array of pages. + */ +static __inline +int +msf_buf_hash(vm_page_t base_m) +{ + int hv; + + hv = ((int)base_m / sizeof(vm_page_t)) + ((int)base_m >> 12); + return(hv & msf_buf_hashmask); +} + +/* + * Get an msf_buf from the freelist; if none are available + * than it will block. + * + * If SFBA_PCATCH was specified in 'flags' than the sleep is + * block is interruptable by signals etc; this flag is normally + * use for system calls. + * + */ +struct msf_buf * +msf_buf_alloc(vm_page_t *pg_ary, int npages, int flags) +{ + struct msf_buf_list *hash_chain; + struct msf_buf *msf; + globaldata_t gd; + int error, pflags; + int i; + + KKASSERT(npages != 0 && npages <= XIO_INTERNAL_SIZE); + + gd = mycpu; + crit_enter(); + hash_chain = &msf_buf_hashtable[msf_buf_hash(*pg_ary)]; + LIST_FOREACH(msf, hash_chain, active_list) { + if (msf->m_xio.xio_npages == npages) { + for (i = npages -1; i >= 0; --i) { + if (msf->m_xio.xio_pages[i] != pg_ary[i]) + break; + } + if (i >= 0) + continue; + /* + * found existing mapping + */ + goto done; + } + } + + /* + * Didn't find old mapping. Get a buffer off the freelist. We + * may have to remove and skip buffers with non-zero ref counts + * that were lazily allocated. + * + * If the freelist is empty, we block until something becomes + * available. This usually happens pretty quickly because sf_bufs + * and msf_bufs are supposed to be temporary mappings. + */ + while ((msf = TAILQ_FIRST(&msf_buf_freelist)) == NULL) { + pflags = (flags & SFBA_PCATCH) ? PCATCH : 0; + ++msf_buf_alloc_want; + error = tsleep(&msf_buf_freelist, pflags, "msfbuf", 0); + --msf_buf_alloc_want; + if (error) + goto done2; + } + + /* + * We are finished when we find an msf_buf with ref. count of + * 0. Theoretically, we do not have to remove it from the + * freelist but it's a good idea to do so to preserve LRU + * operation for the (1) never seen before case and + * (2) accidently recycled due to prior cached uses not removing + * the buffer case. + */ + KKASSERT(msf->m_flags & SFBA_ONFREEQ); + TAILQ_REMOVE(&msf_buf_freelist, msf, free_list); + msf->m_flags &= ~SFBA_ONFREEQ; + + /* Remove previous mapping from hash table and overwrite new one */ + if (msf->m_xio.xio_pages[0] != NULL) + LIST_REMOVE(msf, active_list); + + LIST_INSERT_HEAD(hash_chain, msf, active_list); + + for (i = 0; i < npages; ++i) + msf->m_xio.xio_pages[i] = pg_ary[i]; + + msf->m_xio.xio_npages = npages; + msf->m_xio.xio_bytes = npages * PAGE_SIZE; + + /* + * Successful MSF setup, bump the ref count and enter the pages. + */ +done: + ++msf->m_refcnt; + if ((flags & SFBA_QUICK)) { + pmap_qenter2(msf->m_kva, msf->m_xio.xio_pages, + msf->m_xio.xio_npages, &msf->m_cpumask); + } else { + pmap_qenter(msf->m_kva, msf->m_xio.xio_pages, + msf->m_xio.xio_npages); + msf->m_cpumask = (cpumask_t)-1; + } +done2: + crit_exit(); + return (msf); +} + +#if 0 +/* + * Add a reference to a buffer (currently unused) + */ +void +msf_buf_ref(struct msf_buf *msf) +{ + if (msf->m_refcnt == 0) + panic("msf_buf_ref: referencing a free msf_buf"); + crit_enter(); + ++msf->m_refcnt; + crit_exit(); +} +#endif + +/* + * Lose a reference to an msf_buf. When none left, detach mapped page + * and release resources back to the system. Note that the sfbuf's + * removal from the freelist is delayed, so it may in fact already be + * on the free list. This is the optimal (and most likely) scenario. + * + * Must be called at splimp. + */ +void +msf_buf_free(struct msf_buf *msf) +{ + crit_enter(); + KKASSERT(msf->m_refcnt > 0); + + msf->m_refcnt--; + if (msf->m_refcnt == 0) { + KKASSERT((msf->m_flags & SFBA_ONFREEQ) == 0); + TAILQ_INSERT_TAIL(&msf_buf_freelist, msf, free_list); + msf->m_flags |= SFBA_ONFREEQ; + if (msf_buf_alloc_want > 0) + wakeup_one(&msf_buf_freelist); + } + crit_exit(); +} diff --git a/sys/kern/kern_xio.c b/sys/kern/kern_xio.c index c52ddea161..b09a86f3cf 100644 --- a/sys/kern/kern_xio.c +++ b/sys/kern/kern_xio.c @@ -23,7 +23,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sys/kern/kern_xio.c,v 1.5 2004/05/13 17:40:15 dillon Exp $ + * $DragonFly: src/sys/kern/kern_xio.c,v 1.6 2004/06/05 19:57:35 dillon Exp $ */ /* * Kernel XIO interface. An initialized XIO is basically a collection of @@ -61,6 +61,20 @@ #include #include +/* + * Just do basic initialization of an empty XIO + */ +void +xio_init(xio_t xio) +{ + xio->xio_flags = 0; + xio->xio_bytes = 0; + xio->xio_error = 0; + xio->xio_offset = 0; + xio->xio_npages = 0; + xio->xio_pages = xio->xio_internal_pages; +} + /* * Initialize an XIO given a userspace buffer. 0 is returned on success, * an error code on failure. The actual number of bytes that could be diff --git a/sys/sys/msfbuf.h b/sys/sys/msfbuf.h new file mode 100644 index 0000000000..033e81e54f --- /dev/null +++ b/sys/sys/msfbuf.h @@ -0,0 +1,113 @@ +/*- + * Copyright (c) 2004 Matthew Dillon . + * Copyright (c) 2004 Hiten Pandya . + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDERS, CONTRIBUTORS OR VOICES IN THE AUTHOR'S HEAD + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY + * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT + * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The MSF_BUF API is an augmentation similar to the SFBUF APIs and uses XIO + * page array handling. The SFBUF API originally came from: + * + * Copyright (c) 2003 Alan L. Cox . All rights reserved. + * Copyright (c) 1998 David Greenman. All rights reserved. + * src/sys/sys/sfbuf.h,v 1.4 2004/04/01 17:58:06 dillon + * + * $DragonFly: src/sys/sys/msfbuf.h,v 1.1 2004/06/05 19:57:37 dillon Exp $ + */ +#ifndef _SYS_MSFBUF_H_ +#define _SYS_MSFBUF_H_ + +/* + * MSF_BUFs are used for caching ephermal mappings that span more than + * one page. + * + * The interface returns an msf_buf data structure which has information + * about managing the ephermal mapping, its KVA pointer and an embedded + * XIO structure which describes the mapping. + * + * The embedded XIO structure be passed around to the DEV system because + * it is ref-counted; thus making it perfectly usable by anything that + * can accept an XIO as a transfer unit, most notably the buffer-cache + * and the XIO API. + * + */ + +struct msf_buf { + LIST_ENTRY(msf_buf) active_list; /* active list of buffers */ + TAILQ_ENTRY(msf_buf) free_list; /* free list of buffers */ + vm_offset_t m_kva; /* KVA offset */ + cpumask_t m_cpumask; /* CPU mask for synchronization */ + struct xio m_xio; /* xio embedded */ + int m_refcnt; /* map usage tracking */ + int m_flags; /* control flags */ +}; + +#if defined(_KERNEL) + +/* + * Return a KVA offset to the client + */ +static __inline +vm_offset_t +msf_buf_kva(struct msf_buf *msf) +{ + return (msf->m_kva); +} + +/* + * Return a reference to the underlying pages of an MSF_BUF + */ +static __inline +vm_page_t * +msf_buf_pages(struct msf_buf *msf) +{ + return (msf->m_xio.xio_pages); +} + +/* + * Return an XIO reference and bump reference count. + */ +static __inline +struct xio * +msf_buf_xio(struct msf_buf *msf) +{ + /* bump ref. count and return XIO reference */ + crit_enter(); + ++msf->m_refcnt; + crit_exit(); + + return (&msf->m_xio); +} + +/* API function prototypes */ +struct msf_buf *msf_buf_alloc(vm_page_t *pg_ary, int npages, int flags); +void msf_buf_free(struct msf_buf *); +void msf_buf_ref(struct msf_buf *); + +#endif /* _KERNEL */ + +#endif -- 2.41.0