X-Git-Url: https://gitweb.dragonflybsd.org/dragonfly.git/blobdiff_plain/f74c30ab893b0293a36c7c90ef18aa73d1bbb438..5c5185ae1c44d1e9a2eb9d6d55dd551a57aa9ef1:/sys/kern/kern_sfbuf.c diff --git a/sys/kern/kern_sfbuf.c b/sys/kern/kern_sfbuf.c index 487a0915d7..b09c5c4747 100644 --- a/sys/kern/kern_sfbuf.c +++ b/sys/kern/kern_sfbuf.c @@ -30,211 +30,103 @@ #include #include #include -#include #include -#include -#include -#include +#include + +#include + #include #include #include #include #include -#include static void sf_buf_init(void *arg); SYSINIT(sock_sf, SI_BOOT2_MACHDEP, SI_ORDER_ANY, sf_buf_init, NULL) LIST_HEAD(sf_buf_list, sf_buf); -SYSCTL_INT(_kern_ipc, OID_AUTO, nsfbufs, CTLFLAG_RD, &nsfbufs, 0, - "Maximum number of sf_bufs available to the system"); +static struct objcache *sf_buf_cache; -/* - * A hash table of active sendfile(2) buffers - */ -static struct sf_buf_list *sf_buf_hashtable; -static u_long sf_buf_hashmask; +MALLOC_DEFINE(M_SFBUF, "sfbuf", "Sendfile buffer structures"); +struct objcache_malloc_args sf_buf_malloc_args = { sizeof(struct sf_buf), M_SFBUF }; -static TAILQ_HEAD(, sf_buf) sf_buf_freelist; -static u_int sf_buf_alloc_want; -static vm_offset_t sf_base; -static struct sf_buf *sf_bufs; - -static int sfbuf_quick = 1; -SYSCTL_INT(_debug, OID_AUTO, sfbuf_quick, CTLFLAG_RW, &sfbuf_quick, 0, ""); -static int nsffree; -SYSCTL_INT(_kern_ipc, OID_AUTO, nsffree, CTLFLAG_RD, &nsffree, 0, - "Number of free sf_bufs available to the system"); - -static __inline -int -sf_buf_hash(vm_page_t m) +static boolean_t +sf_buf_cache_ctor(void *obj, void *pdata, int ocflags) { - int hv; + struct sf_buf *sf = (struct sf_buf *)obj; - hv = ((int)(intptr_t)m / sizeof(vm_page_t)) + ((int)(intptr_t)m >> 12); - return(hv & sf_buf_hashmask); + sf->lwbuf = NULL; + sf->refs = 0; + + return (TRUE); } /* - * Allocate a pool of sf_bufs (sendfile(2) or "super-fast" if you prefer. :-)) + * Init objcache of sf_bufs (sendfile(2) or "super-fast" if you prefer. :-)) */ static void sf_buf_init(void *arg) { - int i; - - sf_buf_hashtable = hashinit(nsfbufs, M_TEMP, &sf_buf_hashmask); - TAILQ_INIT(&sf_buf_freelist); - sf_base = kmem_alloc_nofault(&kernel_map, nsfbufs * PAGE_SIZE); - sf_bufs = kmalloc(nsfbufs * sizeof(struct sf_buf), M_TEMP, - M_WAITOK | M_ZERO); - for (i = 0; i < nsfbufs; i++) { - sf_bufs[i].kva = sf_base + i * PAGE_SIZE; - sf_bufs[i].flags |= SFBA_ONFREEQ; - TAILQ_INSERT_TAIL(&sf_buf_freelist, &sf_bufs[i], free_entry); - ++nsffree; - } + sf_buf_cache = objcache_create("sf_buf", 0, 0, + sf_buf_cache_ctor, NULL, NULL, + objcache_malloc_alloc, objcache_malloc_free, + &sf_buf_malloc_args); } /* - * Get an sf_buf from the freelist. Will block if none are available. + * Acquire an sf_buf reference for a vm_page. */ struct sf_buf * -sf_buf_alloc(struct vm_page *m, int flags) +sf_buf_alloc(struct vm_page *m) { - struct sf_buf_list *hash_chain; struct sf_buf *sf; - globaldata_t gd; - int error; - int pflags; - - gd = mycpu; - crit_enter(); - hash_chain = &sf_buf_hashtable[sf_buf_hash(m)]; - LIST_FOREACH(sf, hash_chain, list_entry) { - if (sf->m == m) { - /* - * cache hit - * - * We must invalidate the TLB entry based on whether - * it need only be valid on the local cpu (SFB_CPUPRIVATE), - * or on all cpus. This is conditionalized and in - * most cases no system-wide invalidation should be - * needed. - * - * Note: we do not remove the entry from the freelist - * on the 0->1 transition. - */ - ++sf->refcnt; - if ((flags & SFB_CPUPRIVATE) && sfbuf_quick) { - if ((sf->cpumask & gd->gd_cpumask) == 0) { - pmap_kenter_sync_quick(sf->kva); - sf->cpumask |= gd->gd_cpumask; - } - } else { - if (sf->cpumask != (cpumask_t)-1) { - pmap_kenter_sync(sf->kva); - sf->cpumask = (cpumask_t)-1; - } - } - goto done; /* found existing mapping */ - } + + if ((sf = objcache_get(sf_buf_cache, M_WAITOK)) == NULL) + goto done; + + if ((sf->lwbuf = lwbuf_alloc(m)) == NULL) { + objcache_put(sf_buf_cache, sf); + sf = NULL; + goto done; } /* - * Didn't find old mapping. Get a buffer off the freelist. We - * may have to remove and skip buffers with non-zero ref counts - * that were lazily allocated. + * Force invalidation of the TLB entry on all CPU's */ - for (;;) { - if ((sf = TAILQ_FIRST(&sf_buf_freelist)) == NULL) { - pflags = (flags & SFB_CATCH) ? PCATCH : 0; - ++sf_buf_alloc_want; - error = tsleep(&sf_buf_freelist, pflags, "sfbufa", 0); - --sf_buf_alloc_want; - if (error) - goto done; - } else { - /* - * We may have to do delayed removals for referenced - * sf_buf's here in addition to locating a sf_buf - * to reuse. The sf_bufs must be removed. - * - * We are finished when we find an sf_buf with a - * refcnt of 0. We theoretically do not have to - * remove it from the freelist but it's a good idea - * to do so to preserve LRU operation for the - * (1) never before seen before case and (2) - * accidently recycled due to prior cached uses not - * removing the buffer case. - */ - KKASSERT(sf->flags & SFBA_ONFREEQ); - TAILQ_REMOVE(&sf_buf_freelist, sf, free_entry); - --nsffree; - sf->flags &= ~SFBA_ONFREEQ; - if (sf->refcnt == 0) - break; - } - } - if (sf->m != NULL) /* remove previous mapping from hash table */ - LIST_REMOVE(sf, list_entry); - LIST_INSERT_HEAD(hash_chain, sf, list_entry); - sf->refcnt = 1; - sf->m = m; - if ((flags & SFB_CPUPRIVATE) && sfbuf_quick) { - pmap_kenter_quick(sf->kva, sf->m->phys_addr); - sf->cpumask = gd->gd_cpumask; - } else { - pmap_kenter(sf->kva, sf->m->phys_addr); - sf->cpumask = (cpumask_t)-1; - } + lwbuf_set_global(sf->lwbuf); + + sf->refs = 1; + done: - crit_exit(); return (sf); } -#if 0 - -/* - * Add a reference to a buffer (currently unused) - */ void -sf_buf_ref(struct sf_buf *sf) +sf_buf_ref(void *arg) { - if (sf->refcnt == 0) - panic("sf_buf_ref: referencing a free sf_buf"); - crit_enter(); - sf->refcnt++; - crit_exit(); -} + struct sf_buf *sf = arg; -#endif + atomic_add_int(&sf->refs, 1); +} /* - * Lose a reference to an sf_buf. When none left, detach mapped page - * and release resources back to the system. Note that the sfbuf's - * removal from the freelist is delayed, so it may in fact already be - * on the free list. This is the optimal (and most likely) scenario. + * Detach mapped page and release resources back to the system. * * Must be called at splimp. */ -void -sf_buf_free(struct sf_buf *sf) +int +sf_buf_free(void *arg) { - if (sf->refcnt == 0) - panic("sf_buf_free: freeing free sf_buf"); - crit_enter(); - sf->refcnt--; - if (sf->refcnt == 0 && (sf->flags & SFBA_ONFREEQ) == 0) { - TAILQ_INSERT_TAIL(&sf_buf_freelist, sf, free_entry); - ++nsffree; - sf->flags |= SFBA_ONFREEQ; - if (sf_buf_alloc_want > 0) - wakeup_one(&sf_buf_freelist); + struct sf_buf *sf = arg; + + KKASSERT(sf->refs > 0); + if (atomic_fetchadd_int(&sf->refs, -1) == 1) { + lwbuf_free(sf->lwbuf); + objcache_put(sf_buf_cache, sf); + return (0); } - crit_exit(); -} + return (1); +}