2 * Copyright (c) 2004 Matthew Dillon <dillon@backplane.com>.
3 * Copyright (c) 2004 Hiten Pandya <hmp@backplane.com>.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in
16 * the documentation and/or other materials provided with the
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
22 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
23 * COPYRIGHT HOLDERS, CONTRIBUTORS OR VOICES IN THE AUTHOR'S HEAD
24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY
25 * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
26 * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
27 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
28 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
29 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
30 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 * The MSF_BUF API was augmented from the SFBUF API:
33 * Copyright (c) 1998 David Greenman. All rights reserved.
34 * src/sys/kern/kern_sfbuf.c,v 1.7 2004/05/13 19:46:18 dillon
36 * $DragonFly: src/sys/kern/kern_msfbuf.c,v 1.3 2004/06/14 01:35:36 hmp Exp $
39 * MSFBUFs cache linear multi-page ephermal mappings and operate similar
40 * to SFBUFs. MSFBUFs use XIO's internally to hold the page list and can
41 * be considered to be a KVA wrapper around an XIO.
43 * Like the SFBUF subsystem, the locking and validation of the page array
44 * is the responsibility of the caller. Also like the SFBUF subsystem,
45 * MSFBUFs are SMP-friendly, cache the mappings, and will avoid unnecessary
46 * page invalidations when possible.
48 * MSFBUFs are primarily designed to be used in subsystems that manipulate
49 * XIOs. The DEV and BUF subsystems are a good example.
52 * - Implement the FREEQ optimization that exists in the SFBUF code.
53 * - Allow allocation (aka mapping) based on an XIO instead of a pglist.
54 * - Overload XIOs representitive of smaller chunks of memory onto the
55 * same KVA space to efficiently cache smaller mappings (filesystem
56 * blocks / buffer cache related).
59 #include <sys/param.h>
60 #include <sys/systm.h>
61 #include <sys/globaldata.h>
62 #include <sys/kernel.h>
63 #include <sys/malloc.h>
64 #include <sys/queue.h>
65 #include <sys/sfbuf.h>
66 #include <sys/sysctl.h>
67 #include <sys/thread.h>
68 #include <sys/thread2.h>
70 #include <sys/msfbuf.h>
73 #include <vm/vm_extern.h>
74 #include <vm/vm_kern.h>
75 #include <vm/vm_page.h>
78 MALLOC_DECLARE(M_MSFBUF);
79 MALLOC_DEFINE(M_MSFBUF, "MSFBUF", "direct-copy buffers");
81 /* lists and queues associated with msf_bufs */
82 LIST_HEAD(msf_buf_list, msf_buf);
84 TAILQ_HEAD(, msf_buf) msf_buf_freelist;
86 /* hash table for tracking msf_bufs */
87 static struct msf_buf_list *msf_buf_hashtable;
88 static u_long msf_buf_hashmask;
90 /* indicate shortage of available msf_bufs */
91 static u_int msf_buf_alloc_want;
93 /* base of the msf_buf map */
94 static vm_offset_t msf_base;
95 static struct msf_buf *msf_bufs;
97 static int num_msf_bufs = 256; /* magic value */
98 SYSCTL_INT(_kern_ipc, OID_AUTO, msfbufs, CTLFLAG_RD, &num_msf_bufs,
99 0, "number of direct-copy buffers available");
102 msf_buf_init(void *__dummy)
106 msf_buf_alloc_want = 0;
107 TUNABLE_INT_FETCH("kern.ipc.msfbufs", &num_msf_bufs);
109 msf_buf_hashtable = hashinit(num_msf_bufs, M_TEMP, &msf_buf_hashmask);
110 TAILQ_INIT(&msf_buf_freelist);
112 msf_base = kmem_alloc_nofault(kernel_map,
113 num_msf_bufs * XIO_INTERNAL_SIZE);
116 * Use contig. memory for the maps, so it is quicker to access
117 * linear maps, and they can also be passed to device
118 * buffers (in the future).
120 msf_bufs = malloc(num_msf_bufs * sizeof(struct msf_buf), M_MSFBUF,
123 /* Initialize the free list with necessary information. */
124 for (i = 0; i < num_msf_bufs; i++) {
125 msf_bufs[i].m_kva = msf_base + i * XIO_INTERNAL_SIZE;
126 msf_bufs[i].m_flags |= SFBA_ONFREEQ;
127 xio_init(&msf_bufs[i].m_xio);
128 TAILQ_INSERT_TAIL(&msf_buf_freelist, &msf_bufs[i], free_list);
131 SYSINIT(msf_buf, SI_SUB_MBUF, SI_ORDER_ANY, msf_buf_init, NULL);
134 * Hash the base page of an MSF's array of pages.
138 msf_buf_hash(vm_page_t base_m)
142 hv = ((int)base_m / sizeof(vm_page_t)) + ((int)base_m >> 12);
143 return(hv & msf_buf_hashmask);
147 * Get an msf_buf from the freelist; if none are available
148 * than it will block.
150 * If SFBA_PCATCH was specified in 'flags' than the sleep is
151 * block is interruptable by signals etc; this flag is normally
152 * use for system calls.
156 msf_buf_alloc(vm_page_t *pg_ary, int npages, int flags)
158 struct msf_buf_list *hash_chain;
164 KKASSERT(npages != 0 && npages <= XIO_INTERNAL_SIZE);
168 hash_chain = &msf_buf_hashtable[msf_buf_hash(*pg_ary)];
169 LIST_FOREACH(msf, hash_chain, active_list) {
170 if (msf->m_xio.xio_npages == npages) {
171 for (i = npages - 1; i >= 0; --i) {
172 if (msf->m_xio.xio_pages[i] != pg_ary[i])
178 * found existing mapping
180 if (msf->m_flags & SFBA_ONFREEQ) {
181 TAILQ_REMOVE(&msf_buf_freelist, msf, free_list);
182 msf->m_flags &= ~SFBA_ONFREEQ;
190 * Didn't find old mapping. Get a buffer off the freelist. We
191 * may have to remove and skip buffers with non-zero ref counts
192 * that were lazily allocated.
194 * If the freelist is empty, we block until something becomes
195 * available. This usually happens pretty quickly because sf_bufs
196 * and msf_bufs are supposed to be temporary mappings.
198 while ((msf = TAILQ_FIRST(&msf_buf_freelist)) == NULL) {
199 pflags = (flags & SFBA_PCATCH) ? PCATCH : 0;
200 ++msf_buf_alloc_want;
201 error = tsleep(&msf_buf_freelist, pflags, "msfbuf", 0);
202 --msf_buf_alloc_want;
208 * We are finished when we find an msf_buf with ref. count of
209 * 0. Theoretically, we do not have to remove it from the
210 * freelist but it's a good idea to do so to preserve LRU
211 * operation for the (1) never seen before case and
212 * (2) accidently recycled due to prior cached uses not removing
215 KKASSERT(msf->m_flags & SFBA_ONFREEQ);
216 TAILQ_REMOVE(&msf_buf_freelist, msf, free_list);
217 msf->m_flags &= ~SFBA_ONFREEQ;
219 /* Remove previous mapping from hash table and overwrite new one */
220 if (msf->m_xio.xio_pages[0] != NULL)
221 LIST_REMOVE(msf, active_list);
223 LIST_INSERT_HEAD(hash_chain, msf, active_list);
225 for (i = 0; i < npages; ++i)
226 msf->m_xio.xio_pages[i] = pg_ary[i];
228 msf->m_xio.xio_npages = npages;
229 msf->m_xio.xio_bytes = npages * PAGE_SIZE;
232 * Successful MSF setup, bump the ref count and enter the pages.
236 if ((flags & SFBA_QUICK)) {
237 pmap_qenter2(msf->m_kva, msf->m_xio.xio_pages,
238 msf->m_xio.xio_npages, &msf->m_cpumask);
240 pmap_qenter(msf->m_kva, msf->m_xio.xio_pages,
241 msf->m_xio.xio_npages);
242 msf->m_cpumask = (cpumask_t)-1;
251 * Add a reference to a buffer (currently unused)
254 msf_buf_ref(struct msf_buf *msf)
256 if (msf->m_refcnt == 0)
257 panic("msf_buf_ref: referencing a free msf_buf");
265 * Lose a reference to an msf_buf. When none left, detach mapped page
266 * and release resources back to the system. Note that the sfbuf's
267 * removal from the freelist is delayed, so it may in fact already be
268 * on the free list. This is the optimal (and most likely) scenario.
270 * Must be called at splimp.
273 msf_buf_free(struct msf_buf *msf)
276 KKASSERT(msf->m_refcnt > 0);
279 if (msf->m_refcnt == 0) {
280 KKASSERT((msf->m_flags & SFBA_ONFREEQ) == 0);
281 TAILQ_INSERT_TAIL(&msf_buf_freelist, msf, free_list);
282 msf->m_flags |= SFBA_ONFREEQ;
283 if (msf_buf_alloc_want > 0)
284 wakeup_one(&msf_buf_freelist);