From: Matthew Dillon Date: Fri, 28 Oct 2011 16:29:28 +0000 (-0700) Subject: kernel - Clean up spinlock code, add more lwkt_yield()s X-Git-Url: https://gitweb.dragonflybsd.org/~lentferj/dragonfly.git/commitdiff_plain/fc9ed34d0c9497fde8cf25ed5043846ebf55e8bf kernel - Clean up spinlock code, add more lwkt_yield()s * Clean up some of the critical path in the spin_unlock() API * Add a few more lwkt_yield()s in the buffer cache and vm_object cleaning code. --- diff --git a/sys/kern/kern_spinlock.c b/sys/kern/kern_spinlock.c index a736eedc18..d836833514 100644 --- a/sys/kern/kern_spinlock.c +++ b/sys/kern/kern_spinlock.c @@ -149,17 +149,17 @@ spin_trylock_contested(struct spinlock *spin) * (for intel/amd anyhow) is not strictly needed as cache bus resource use * is governed by the lazy update. * - * WARNING!!!! Performance matters here, by a huge margin. There are still - * a few bottlenecks in the kernel (e.g. the PQ_INACTIVE - * vm_page_queue) where things like parallel compiles hit up - * against full all-cores contention right here. + * WARNING!!!! Performance matters here, by a huge margin. * - * 48-core test with pre-read / -j 48 no-modules kernel compile - * came in at 75 seconds. Without pre-read it came in at 170 seconds. + * 48-core test with pre-read / -j 48 no-modules kernel compile + * with fanned-out inactive and active queues came in at 55 seconds. * - * 4-core test with pre-read / -j 48 no-modules kernel compile - * came in at 83 seconds. Without pre-read it came in at 83 seconds - * as well (no difference). + * 48-core test with pre-read / -j 48 no-modules kernel compile + * came in at 75 seconds. Without pre-read it came in at 170 seconds. + * + * 4-core test with pre-read / -j 48 no-modules kernel compile + * came in at 83 seconds. Without pre-read it came in at 83 seconds + * as well (no difference). */ void spin_lock_contested(struct spinlock *spin) diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c index 0e133b7572..2c82b0ef74 100644 --- a/sys/kern/vfs_bio.c +++ b/sys/kern/vfs_bio.c @@ -2694,6 +2694,7 @@ flushbufqueues(bufq_type_t q) } spin_unlock(&bufqspin); + lwkt_yield(); spun = 0; if (LIST_FIRST(&bp->b_dep) != NULL && diff --git a/sys/platform/pc64/x86_64/pmap.c b/sys/platform/pc64/x86_64/pmap.c index ffda71f158..2bed171199 100644 --- a/sys/platform/pc64/x86_64/pmap.c +++ b/sys/platform/pc64/x86_64/pmap.c @@ -2739,6 +2739,7 @@ kernel_skip: * the potentially conflicting pv and * re-checking. */ + lwkt_yield(); if (*ptep == 0) { pte_pv = pv_find(pmap, pmap_pte_pindex(sva)); if (pte_pv == NULL) { @@ -3349,6 +3350,7 @@ pmap_object_init_pt_callback(vm_page_t p, void *data) info->addr + x86_64_ptob(rel_index), p); } vm_page_wakeup(p); + lwkt_yield(); return(0); } diff --git a/sys/sys/spinlock2.h b/sys/sys/spinlock2.h index 85811d5b96..f4ad337148 100644 --- a/sys/sys/spinlock2.h +++ b/sys/sys/spinlock2.h @@ -172,18 +172,24 @@ spin_unlock_quick(globaldata_t gd, struct spinlock *spin) } #endif /* - * Don't use a locked instruction here. + * Don't use a locked instruction here. To reduce latency we avoid + * reading spin->counta prior to writing to it. */ +#ifdef DEBUG_LOCKS KKASSERT(spin->counta != 0); +#endif cpu_sfence(); spin->counta = 0; cpu_sfence(); #endif +#ifdef DEBUG_LOCKS KKASSERT(gd->gd_spinlocks_wr > 0); +#endif --gd->gd_spinlocks_wr; cpu_ccfence(); --gd->gd_curthread->td_critcount; #if 0 + /* FUTURE */ if (__predict_false(gd->gd_reqflags & RQF_IDLECHECK_MASK)) lwkt_maybe_splz(gd->gd_curthread); #endif diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c index 7487d6f6e9..cc2fd003a0 100644 --- a/sys/vm/vm_object.c +++ b/sys/vm/vm_object.c @@ -1013,6 +1013,7 @@ vm_object_page_clean_pass1(struct vm_page *p, void *data) } else { info->error = 1; } + lwkt_yield(); return(0); } @@ -1031,7 +1032,7 @@ vm_object_page_clean_pass2(struct vm_page *p, void *data) * the cleaning pass. */ if ((p->flags & PG_CLEANCHK) == 0) - return(0); + goto done; generation = info->object->generation; vm_page_busy_wait(p, TRUE, "vpcwai"); @@ -1039,7 +1040,7 @@ vm_object_page_clean_pass2(struct vm_page *p, void *data) info->object->generation != generation) { info->error = 1; vm_page_wakeup(p); - return(0); + goto done; } /* @@ -1049,7 +1050,7 @@ vm_object_page_clean_pass2(struct vm_page *p, void *data) if (p->valid == 0 || (p->queue - p->pc) == PQ_CACHE) { KKASSERT((p->dirty & p->valid) == 0); vm_page_wakeup(p); - return(0); + goto done; } /* @@ -1061,7 +1062,7 @@ vm_object_page_clean_pass2(struct vm_page *p, void *data) if ((p->dirty & p->valid) == 0) { vm_page_flag_clear(p, PG_CLEANCHK); vm_page_wakeup(p); - return(0); + goto done; } /* @@ -1073,7 +1074,7 @@ vm_object_page_clean_pass2(struct vm_page *p, void *data) if ((info->limit & OBJPC_NOSYNC) && (p->flags & PG_NOSYNC)) { vm_page_flag_clear(p, PG_CLEANCHK); vm_page_wakeup(p); - return(0); + goto done; } /* @@ -1082,6 +1083,8 @@ vm_object_page_clean_pass2(struct vm_page *p, void *data) * we raced an object modification. */ vm_object_page_collect_flush(info->object, p, info->pagerflags); +done: + lwkt_yield(); return(0); } diff --git a/sys/vm/vnode_pager.c b/sys/vm/vnode_pager.c index 9a589d4da1..040f4d6a61 100644 --- a/sys/vm/vnode_pager.c +++ b/sys/vm/vnode_pager.c @@ -661,7 +661,9 @@ vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *mpp, int bytecount, } } else if (mt->valid == 0) { if (error == 0) { - kprintf("page failed but no I/O error page %p object %p pindex %d\n", mt, mt->object, (int) mt->pindex); + kprintf("page failed but no I/O error page " + "%p object %p pindex %d\n", + mt, mt->object, (int) mt->pindex); /* whoops, something happened */ error = EINVAL; }