[dragonfly.git] / sys / kern / kern_sfbuf.c

/*
 * Copyright (c) 1998 David Greenman.  All rights reserved. 
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * $DragonFly: src/sys/kern/kern_sfbuf.c,v 1.15 2007/04/30 07:18:53 dillon Exp $
 */

#include <sys/param.h>
#include <sys/types.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/queue.h>
#include <sys/sfbuf.h>
#include <sys/globaldata.h>
#include <sys/thread.h>
#include <sys/sysctl.h>
#include <vm/vm.h>
#include <vm/vm_extern.h>
#include <vm/vm_kern.h>
#include <vm/vm_page.h>
#include <vm/pmap.h>
#include <sys/thread2.h>

static void sf_buf_init(void *arg);
SYSINIT(sock_sf, SI_BOOT2_MACHDEP, SI_ORDER_ANY, sf_buf_init, NULL)

LIST_HEAD(sf_buf_list, sf_buf);

SYSCTL_INT(_kern_ipc, OID_AUTO, nsfbufs, CTLFLAG_RD, &nsfbufs, 0,
	"Maximum number of sf_bufs available to the system");

/*
 * A hash table of active sendfile(2) buffers
 */
static struct sf_buf_list *sf_buf_hashtable;
static u_long sf_buf_hashmask;

static TAILQ_HEAD(, sf_buf) sf_buf_freelist;
static u_int sf_buf_alloc_want;

static vm_offset_t sf_base;
static struct sf_buf *sf_bufs;

static int sfbuf_quick = 1;
SYSCTL_INT(_debug, OID_AUTO, sfbuf_quick, CTLFLAG_RW, &sfbuf_quick, 0, "");
static int nsffree;
SYSCTL_INT(_kern_ipc, OID_AUTO, nsffree, CTLFLAG_RD, &nsffree, 0,
	"Number of free sf_bufs available to the system");

static __inline
int
sf_buf_hash(vm_page_t m)
{
    int hv;

    hv = ((int)(intptr_t)m / sizeof(vm_page_t)) + ((int)(intptr_t)m >> 12);
    return(hv & sf_buf_hashmask);
}

/*
 * Allocate a pool of sf_bufs (sendfile(2) or "super-fast" if you prefer. :-))
 */
static void
sf_buf_init(void *arg)
{
	int i;

	sf_buf_hashtable = hashinit(nsfbufs, M_TEMP, &sf_buf_hashmask);
	TAILQ_INIT(&sf_buf_freelist);
	sf_base = kmem_alloc_nofault(&kernel_map, nsfbufs * PAGE_SIZE);
	sf_bufs = kmalloc(nsfbufs * sizeof(struct sf_buf), M_TEMP,
			    M_WAITOK | M_ZERO);
	for (i = 0; i < nsfbufs; i++) {
		sf_bufs[i].kva = sf_base + i * PAGE_SIZE;
		sf_bufs[i].flags |= SFBA_ONFREEQ;
		TAILQ_INSERT_TAIL(&sf_buf_freelist, &sf_bufs[i], free_entry);
		++nsffree;
	}
}

/*
 * Get an sf_buf from the freelist. Will block if none are available.
 */
struct sf_buf *
sf_buf_alloc(struct vm_page *m, int flags)
{
	struct sf_buf_list *hash_chain;
	struct sf_buf *sf;
	globaldata_t gd;
	int error;
	int pflags;

	gd = mycpu;
	crit_enter();
	hash_chain = &sf_buf_hashtable[sf_buf_hash(m)];
	LIST_FOREACH(sf, hash_chain, list_entry) {
		if (sf->m == m) {
			/*
			 * cache hit
			 *
			 * We must invalidate the TLB entry based on whether
			 * it need only be valid on the local cpu (SFB_CPUPRIVATE),
			 * or on all cpus.  This is conditionalized and in
			 * most cases no system-wide invalidation should be
			 * needed.
			 *
			 * Note: we do not remove the entry from the freelist
			 * on the 0->1 transition. 
			 */
			++sf->refcnt;
			if ((flags & SFB_CPUPRIVATE) && sfbuf_quick) {
				if ((sf->cpumask & gd->gd_cpumask) == 0) {
					pmap_kenter_sync_quick(sf->kva);
					sf->cpumask |= gd->gd_cpumask;
				}
			} else {
				if (sf->cpumask != (cpumask_t)-1) {
					pmap_kenter_sync(sf->kva);
					sf->cpumask = (cpumask_t)-1;
				}
			}
			goto done;	/* found existing mapping */
		}
	}

	/*
	 * Didn't find old mapping.  Get a buffer off the freelist.  We
	 * may have to remove and skip buffers with non-zero ref counts 
	 * that were lazily allocated.
	 */
	for (;;) {
		if ((sf = TAILQ_FIRST(&sf_buf_freelist)) == NULL) {
			pflags = (flags & SFB_CATCH) ? PCATCH : 0;
			++sf_buf_alloc_want;
			error = tsleep(&sf_buf_freelist, pflags, "sfbufa", 0);
			--sf_buf_alloc_want;
			if (error)
				goto done;
		} else {
			/*
			 * We may have to do delayed removals for referenced
			 * sf_buf's here in addition to locating a sf_buf
			 * to reuse.  The sf_bufs must be removed.
			 *
			 * We are finished when we find an sf_buf with a
			 * refcnt of 0.  We theoretically do not have to
			 * remove it from the freelist but it's a good idea
			 * to do so to preserve LRU operation for the
			 * (1) never before seen before case and (2) 
			 * accidently recycled due to prior cached uses not
			 * removing the buffer case.
			 */
			KKASSERT(sf->flags & SFBA_ONFREEQ);
			TAILQ_REMOVE(&sf_buf_freelist, sf, free_entry);
			--nsffree;
			sf->flags &= ~SFBA_ONFREEQ;
			if (sf->refcnt == 0)
				break;
		}
	}
	if (sf->m != NULL)	/* remove previous mapping from hash table */
		LIST_REMOVE(sf, list_entry);
	LIST_INSERT_HEAD(hash_chain, sf, list_entry);
	sf->refcnt = 1;
	sf->m = m;
	if ((flags & SFB_CPUPRIVATE) && sfbuf_quick) {
		pmap_kenter_quick(sf->kva, sf->m->phys_addr);
		sf->cpumask = gd->gd_cpumask;
	} else {
		pmap_kenter(sf->kva, sf->m->phys_addr);
		sf->cpumask = (cpumask_t)-1;
	}
done:
	crit_exit();
	return (sf);
}

#if 0

/*
 * Add a reference to a buffer (currently unused)
 */
void
sf_buf_ref(struct sf_buf *sf)
{
	if (sf->refcnt == 0)
		panic("sf_buf_ref: referencing a free sf_buf");
	crit_enter();
	sf->refcnt++;
	crit_exit();
}

#endif

/*
 * Lose a reference to an sf_buf. When none left, detach mapped page
 * and release resources back to the system.  Note that the sfbuf's
 * removal from the freelist is delayed, so it may in fact already be
 * on the free list.  This is the optimal (and most likely) scenario.
 *
 * Must be called at splimp.
 */
void
sf_buf_free(struct sf_buf *sf)
{
	if (sf->refcnt == 0)
		panic("sf_buf_free: freeing free sf_buf");
	crit_enter();
	sf->refcnt--;
	if (sf->refcnt == 0 && (sf->flags & SFBA_ONFREEQ) == 0) {
		TAILQ_INSERT_TAIL(&sf_buf_freelist, sf, free_entry);
		++nsffree;
		sf->flags |= SFBA_ONFREEQ;
		if (sf_buf_alloc_want > 0)
			wakeup_one(&sf_buf_freelist);
	}
	crit_exit();
}
Commit	Line	Data
4860553a JH	1	/*
	2	* Copyright (c) 1998 David Greenman. All rights reserved.
	3	*
	4	* Redistribution and use in source and binary forms, with or without
	5	* modification, are permitted provided that the following conditions
	6	* are met:
	7	* 1. Redistributions of source code must retain the above copyright
	8	* notice, this list of conditions and the following disclaimer.
	9	* 2. Redistributions in binary form must reproduce the above copyright
	10	* notice, this list of conditions and the following disclaimer in the
	11	* documentation and/or other materials provided with the distribution.
	12	*
	13	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	14	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	15	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	16	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	17	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	18	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	19	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	20	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	21	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	22	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	23	* SUCH DAMAGE.
	24	*
ba39e2e0	25	* $DragonFly: src/sys/kern/kern_sfbuf.c,v 1.15 2007/04/30 07:18:53 dillon Exp $
4860553a JH	26	*/
	27
	28	#include <sys/param.h>
	29	#include <sys/types.h>
	30	#include <sys/systm.h>
	31	#include <sys/kernel.h>
	32	#include <sys/malloc.h>
	33	#include <sys/queue.h>
	34	#include <sys/sfbuf.h>
24712b90 MD	35	#include <sys/globaldata.h>
	36	#include <sys/thread.h>
	37	#include <sys/sysctl.h>
4860553a JH	38	#include <vm/vm.h>
	39	#include <vm/vm_extern.h>
	40	#include <vm/vm_kern.h>
	41	#include <vm/vm_page.h>
	42	#include <vm/pmap.h>
24712b90	43	#include <sys/thread2.h>
4860553a JH	44
4860553a JH	45	static void sf_buf_init(void *arg);
ba39e2e0	46	SYSINIT(sock_sf, SI_BOOT2_MACHDEP, SI_ORDER_ANY, sf_buf_init, NULL)
4860553a JH	47
	48	LIST_HEAD(sf_buf_list, sf_buf);
	49
ad9e1baa HP	50	SYSCTL_INT(_kern_ipc, OID_AUTO, nsfbufs, CTLFLAG_RD, &nsfbufs, 0,
	51	"Maximum number of sf_bufs available to the system");
	52
4860553a JH	53	/*
	54	* A hash table of active sendfile(2) buffers
	55	*/
	56	static struct sf_buf_list *sf_buf_hashtable;
	57	static u_long sf_buf_hashmask;
	58
4860553a JH	59	static TAILQ_HEAD(, sf_buf) sf_buf_freelist;
	60	static u_int sf_buf_alloc_want;
	61
	62	static vm_offset_t sf_base;
	63	static struct sf_buf *sf_bufs;
	64
24712b90 MD	65	static int sfbuf_quick = 1;
24712b90 MD	66	SYSCTL_INT(_debug, OID_AUTO, sfbuf_quick, CTLFLAG_RW, &sfbuf_quick, 0, "");
e66bab2b MD	67	static int nsffree;
	68	SYSCTL_INT(_kern_ipc, OID_AUTO, nsffree, CTLFLAG_RD, &nsffree, 0,
	69	"Number of free sf_bufs available to the system");
24712b90	70
b89e3239 MD	71	static __inline
	72	int
	73	sf_buf_hash(vm_page_t m)
	74	{
	75	int hv;
	76
973c11b9	77	hv = ((int)(intptr_t)m / sizeof(vm_page_t)) + ((int)(intptr_t)m >> 12);
b89e3239 MD	78	return(hv & sf_buf_hashmask);
	79	}
	80
4860553a JH	81	/*
	82	* Allocate a pool of sf_bufs (sendfile(2) or "super-fast" if you prefer. :-))
	83	*/
	84	static void
	85	sf_buf_init(void *arg)
	86	{
	87	int i;
	88
	89	sf_buf_hashtable = hashinit(nsfbufs, M_TEMP, &sf_buf_hashmask);
	90	TAILQ_INIT(&sf_buf_freelist);
e4846942	91	sf_base = kmem_alloc_nofault(&kernel_map, nsfbufs * PAGE_SIZE);
efda3bd0	92	sf_bufs = kmalloc(nsfbufs * sizeof(struct sf_buf), M_TEMP,
6ddd9284	93	M_WAITOK \| M_ZERO);
4860553a JH	94	for (i = 0; i < nsfbufs; i++) {
4860553a JH	95	sf_bufs[i].kva = sf_base + i * PAGE_SIZE;
24712b90	96	sf_bufs[i].flags \|= SFBA_ONFREEQ;
4860553a	97	TAILQ_INSERT_TAIL(&sf_buf_freelist, &sf_bufs[i], free_entry);
e66bab2b	98	++nsffree;
4860553a JH	99	}
	100	}
	101
	102	/*
	103	* Get an sf_buf from the freelist. Will block if none are available.
	104	*/
	105	struct sf_buf *
24712b90	106	sf_buf_alloc(struct vm_page *m, int flags)
4860553a JH	107	{
	108	struct sf_buf_list *hash_chain;
	109	struct sf_buf *sf;
24712b90	110	globaldata_t gd;
4860553a	111	int error;
24712b90	112	int pflags;
4860553a	113
24712b90 MD	114	gd = mycpu;
24712b90 MD	115	crit_enter();
b89e3239	116	hash_chain = &sf_buf_hashtable[sf_buf_hash(m)];
4860553a JH	117	LIST_FOREACH(sf, hash_chain, list_entry) {
4860553a JH	118	if (sf->m == m) {
24712b90 MD	119	/*
	120	* cache hit
	121	*
	122	* We must invalidate the TLB entry based on whether
4f1640d6	123	* it need only be valid on the local cpu (SFB_CPUPRIVATE),
24712b90 MD	124	* or on all cpus. This is conditionalized and in
	125	* most cases no system-wide invalidation should be
	126	* needed.
	127	*
	128	* Note: we do not remove the entry from the freelist
	129	* on the 0->1 transition.
	130	*/
4860553a	131	++sf->refcnt;
4f1640d6	132	if ((flags & SFB_CPUPRIVATE) && sfbuf_quick) {
24712b90 MD	133	if ((sf->cpumask & gd->gd_cpumask) == 0) {
	134	pmap_kenter_sync_quick(sf->kva);
	135	sf->cpumask \|= gd->gd_cpumask;
	136	}
	137	} else {
	138	if (sf->cpumask != (cpumask_t)-1) {
	139	pmap_kenter_sync(sf->kva);
	140	sf->cpumask = (cpumask_t)-1;
	141	}
	142	}
4860553a JH	143	goto done; /* found existing mapping */
	144	}
	145	}
	146
	147	/*
24712b90 MD	148	* Didn't find old mapping. Get a buffer off the freelist. We
	149	* may have to remove and skip buffers with non-zero ref counts
	150	* that were lazily allocated.
4860553a	151	*/
24712b90 MD	152	for (;;) {
24712b90 MD	153	if ((sf = TAILQ_FIRST(&sf_buf_freelist)) == NULL) {
4f1640d6	154	pflags = (flags & SFB_CATCH) ? PCATCH : 0;
24712b90 MD	155	++sf_buf_alloc_want;
	156	error = tsleep(&sf_buf_freelist, pflags, "sfbufa", 0);
	157	--sf_buf_alloc_want;
	158	if (error)
	159	goto done;
	160	} else {
	161	/*
	162	* We may have to do delayed removals for referenced
	163	* sf_buf's here in addition to locating a sf_buf
	164	* to reuse. The sf_bufs must be removed.
	165	*
	166	* We are finished when we find an sf_buf with a
	167	* refcnt of 0. We theoretically do not have to
	168	* remove it from the freelist but it's a good idea
	169	* to do so to preserve LRU operation for the
	170	* (1) never before seen before case and (2)
	171	* accidently recycled due to prior cached uses not
	172	* removing the buffer case.
	173	*/
	174	KKASSERT(sf->flags & SFBA_ONFREEQ);
	175	TAILQ_REMOVE(&sf_buf_freelist, sf, free_entry);
e66bab2b	176	--nsffree;
24712b90 MD	177	sf->flags &= ~SFBA_ONFREEQ;
	178	if (sf->refcnt == 0)
	179	break;
	180	}
4860553a	181	}
4860553a JH	182	if (sf->m != NULL) /* remove previous mapping from hash table */
	183	LIST_REMOVE(sf, list_entry);
	184	LIST_INSERT_HEAD(hash_chain, sf, list_entry);
	185	sf->refcnt = 1;
	186	sf->m = m;
4f1640d6	187	if ((flags & SFB_CPUPRIVATE) && sfbuf_quick) {
24712b90 MD	188	pmap_kenter_quick(sf->kva, sf->m->phys_addr);
	189	sf->cpumask = gd->gd_cpumask;
	190	} else {
	191	pmap_kenter(sf->kva, sf->m->phys_addr);
	192	sf->cpumask = (cpumask_t)-1;
	193	}
4860553a	194	done:
24712b90	195	crit_exit();
4860553a JH	196	return (sf);
	197	}
	198
83befd2d MD	199	#if 0
	200
	201	/*
	202	* Add a reference to a buffer (currently unused)
	203	*/
b4caac98 MD	204	void
	205	sf_buf_ref(struct sf_buf *sf)
	206	{
4860553a JH	207	if (sf->refcnt == 0)
4860553a JH	208	panic("sf_buf_ref: referencing a free sf_buf");
83befd2d	209	crit_enter();
4860553a	210	sf->refcnt++;
83befd2d	211	crit_exit();
4860553a JH	212	}
4860553a JH	213
83befd2d MD	214	#endif
83befd2d MD	215
4860553a JH	216	/*
4860553a JH	217	* Lose a reference to an sf_buf. When none left, detach mapped page
24712b90 MD	218	* and release resources back to the system. Note that the sfbuf's
	219	* removal from the freelist is delayed, so it may in fact already be
	220	* on the free list. This is the optimal (and most likely) scenario.
4860553a JH	221	*
	222	* Must be called at splimp.
	223	*/
	224	void
b4caac98	225	sf_buf_free(struct sf_buf *sf)
4860553a	226	{
4860553a JH	227	if (sf->refcnt == 0)
4860553a JH	228	panic("sf_buf_free: freeing free sf_buf");
1d5af025	229	crit_enter();
4860553a	230	sf->refcnt--;
24712b90	231	if (sf->refcnt == 0 && (sf->flags & SFBA_ONFREEQ) == 0) {
4860553a	232	TAILQ_INSERT_TAIL(&sf_buf_freelist, sf, free_entry);
e66bab2b	233	++nsffree;
24712b90	234	sf->flags \|= SFBA_ONFREEQ;
4860553a JH	235	if (sf_buf_alloc_want > 0)
	236	wakeup_one(&sf_buf_freelist);
	237	}
1d5af025	238	crit_exit();
4860553a	239	}
b4caac98	240