From: Matthew Dillon <dillon@apollo.backplane.com>
Date: Thu, 2 Nov 2017 23:21:13 +0000 (-0700)
Subject: pthreads - Fix rtld-elf and libthread_xu
X-Git-Tag: v5.3.0~923
X-Git-Url: https://gitweb.dragonflybsd.org/dragonfly.git/commitdiff_plain/982472833bfaf7f80422693d9d6752ca14334a18

pthreads - Fix rtld-elf and libthread_xu

* Fixes chrome, thunderbird, and multiple other issues with recent
  libpthreads work.

Testing-by: mneumann, dillon
---

diff --git a/lib/libthread_xu/thread/thr_barrier.c b/lib/libthread_xu/thread/thr_barrier.c
index 7f00ed1bde..d592176dcd 100644
--- a/lib/libthread_xu/thread/thr_barrier.c
+++ b/lib/libthread_xu/thread/thr_barrier.c
@@ -91,7 +91,7 @@ _pthread_barrier_wait(pthread_barrier_t *barrier)
 		/* Current thread is lastest thread */
 		bar->b_waiters = 0;
 		bar->b_cycle++;
-		_thr_umtx_wake(&bar->b_cycle, bar->b_count);
+		_thr_umtx_wake(&bar->b_cycle, 0);
 		THR_UMTX_UNLOCK(curthread, &bar->b_lock);
 		ret = PTHREAD_BARRIER_SERIAL_THREAD;
 	} else {
diff --git a/lib/libthread_xu/thread/thr_create.c b/lib/libthread_xu/thread/thr_create.c
index 18d9a1ab5f..2ddfa2405a 100644
--- a/lib/libthread_xu/thread/thr_create.c
+++ b/lib/libthread_xu/thread/thr_create.c
@@ -178,7 +178,7 @@ _pthread_create(pthread_t * thread, const pthread_attr_t * attr,
 		new_thread->terminated = 1;
 		if (new_thread->flags & THR_FLAGS_NEED_SUSPEND) {
 			new_thread->cycle++;
-			_thr_umtx_wake(&new_thread->cycle, INT_MAX);
+			_thr_umtx_wake(&new_thread->cycle, 0);
 		}
 		THR_THREAD_UNLOCK(curthread, new_thread);
 		THREAD_LIST_LOCK(curthread);
diff --git a/lib/libthread_xu/thread/thr_exit.c b/lib/libthread_xu/thread/thr_exit.c
index 28a7a820dd..37a2f27f6a 100644
--- a/lib/libthread_xu/thread/thr_exit.c
+++ b/lib/libthread_xu/thread/thr_exit.c
@@ -152,7 +152,7 @@ exit_thread(void)
 		THR_GCLIST_ADD(curthread);
 	THREAD_LIST_UNLOCK(curthread);
 	if (curthread->joiner)
-		_thr_umtx_wake(&curthread->state, INT_MAX);
+		_thr_umtx_wake(&curthread->state, 0);
 	if (SHOULD_REPORT_EVENT(curthread, TD_DEATH))
 		_thr_report_death(curthread);
 	/* Exit and set terminated to once we're really dead. */
diff --git a/lib/libthread_xu/thread/thr_fork.c b/lib/libthread_xu/thread/thr_fork.c
index 25033c6845..0391a3b1c9 100644
--- a/lib/libthread_xu/thread/thr_fork.c
+++ b/lib/libthread_xu/thread/thr_fork.c
@@ -68,6 +68,7 @@
 #include <fcntl.h>
 #include <pthread.h>
 #include <spinlock.h>
+#include <sys/file.h>
 #include "un-namespace.h"
 
 #include "libc_private.h"
@@ -174,6 +175,7 @@ _fork(void)
 	if (!_thr_is_inited())
 		return (__syscall(SYS_fork));
 
+	errsave = errno;
 	curthread = tls_get_curthread();
 
 	THR_UMTX_LOCK(curthread, &_thr_atfork_lock);
@@ -194,10 +196,6 @@ _fork(void)
 		if (af->prepare != NULL)
 			af->prepare();
 	}
-	TAILQ_FOREACH_REVERSE(af, &_thr_atfork_kern_list, atfork_head, qe) {
-		if (af->prepare != NULL)
-			af->prepare();
-	}
 
 #ifndef __DragonFly__
 	/*
@@ -213,19 +211,38 @@ _fork(void)
 	}
 #endif
 
-	/*
-	 * Block all signals until we reach a safe point.
-	 */
-	_thr_signal_block(curthread);
 #ifdef _PTHREADS_DEBUGGING
 	_thr_log("fork-parent\n", 12);
 #endif
 
+	_thr_signal_block(curthread);
+
+	/*
+	 * Must be executed Just before the fork.
+	 */
+	TAILQ_FOREACH_REVERSE(af, &_thr_atfork_kern_list, atfork_head, qe) {
+		if (af->prepare != NULL)
+			af->prepare();
+	}
+
 	/* Fork a new process: */
 	if ((ret = __syscall(SYS_fork)) == 0) {
-		/* Child process */
-		errsave = errno;
+		/*
+		 * Child process.
+		 *
+		 * NOTE: We are using the saved errno from above.  Do not
+		 *	 reload errno here.
+		 */
 		inprogress = 0;
+
+		/*
+		 * Internal child fork handlers must be run immediately.
+		 */
+		TAILQ_FOREACH(af, &_thr_atfork_kern_list, qe) {
+			if (af->child != NULL)
+				af->child();
+		}
+
 		curthread->cancelflags &= ~THR_CANCEL_NEEDED;
 		/*
 		 * Thread list will be reinitialized, and later we call
@@ -255,10 +272,6 @@ _fork(void)
 		_thr_signal_unblock(curthread);
 
 		/* Run down atfork child handlers. */
-		TAILQ_FOREACH(af, &_thr_atfork_kern_list, qe) {
-			if (af->child != NULL)
-				af->child();
-		}
 		TAILQ_FOREACH(af, &_thr_atfork_list, qe) {
 			if (af->child != NULL)
 				af->child();
@@ -274,14 +287,15 @@ _fork(void)
 #ifdef _PTHREADS_DEBUGGING
 		_thr_log("fork-done\n", 10);
 #endif
-		/* Ready to continue, unblock signals. */
-		_thr_signal_unblock(curthread);
-
 		/* Run down atfork parent handlers. */
 		TAILQ_FOREACH(af, &_thr_atfork_kern_list, qe) {
 			if (af->parent != NULL)
 				af->parent();
 		}
+
+		/* Ready to continue, unblock signals. */
+		_thr_signal_unblock(curthread);
+
 		TAILQ_FOREACH(af, &_thr_atfork_list, qe) {
 			if (af->parent != NULL)
 				af->parent();
@@ -290,7 +304,7 @@ _fork(void)
 		THR_UMTX_LOCK(curthread, &_thr_atfork_lock);
 		inprogress = 0;
 		if (waiters)
-			_thr_umtx_wake(&inprogress, waiters);
+			_thr_umtx_wake(&inprogress, 0);
 		THR_UMTX_UNLOCK(curthread, &_thr_atfork_lock);
 	}
 	errno = errsave;
diff --git a/lib/libthread_xu/thread/thr_init.c b/lib/libthread_xu/thread/thr_init.c
index 3ac3de0d64..6bb6730b7d 100644
--- a/lib/libthread_xu/thread/thr_init.c
+++ b/lib/libthread_xu/thread/thr_init.c
@@ -286,6 +286,8 @@ _libpthread_init(struct pthread *curthread)
 		__sys_sigprocmask(SIG_SETMASK, &oldset, NULL);
 		if (td_eventismember(&_thread_event_mask, TD_CREATE))
 			_thr_report_creation(curthread, curthread);
+		_thr_rtld_init();
+		_thr_sem_init();
 	}
 }
 
diff --git a/lib/libthread_xu/thread/thr_kern.c b/lib/libthread_xu/thread/thr_kern.c
index 5a92ddeea8..c15c2a39cd 100644
--- a/lib/libthread_xu/thread/thr_kern.c
+++ b/lib/libthread_xu/thread/thr_kern.c
@@ -43,19 +43,21 @@
  * This is called when the first thread (other than the initial
  * thread) is created.
  *
- * NOTE: Once rtld is initialized for threading, we never de-initialize
- *	 it (we do not call _thr_rtld_fini()).
+ * NOTE: we no longer call _thrd_rtld_fini here.
  */
 int
 _thr_setthreaded(int threaded)
 {
 	if (((threaded == 0) ^ (__isthreaded == 0)) == 0)
 		return (0);
-
 	__isthreaded = threaded;
+#if 0
+	/* save for later. */
 	if (threaded != 0)
-		_thr_rtld_init();
-
+		/* blah */ ;
+	else
+		/* blah */ ;
+#endif
 	return (0);
 }
 
diff --git a/lib/libthread_xu/thread/thr_private.h b/lib/libthread_xu/thread/thr_private.h
index efad533aad..e47acaee64 100644
--- a/lib/libthread_xu/thread/thr_private.h
+++ b/lib/libthread_xu/thread/thr_private.h
@@ -681,6 +681,7 @@ void	_thr_rtld_fini(void);
 int	_thr_stack_alloc(struct pthread_attr *);
 void	_thr_stack_free(struct pthread_attr *);
 void	_thr_stack_cleanup(void);
+void	_thr_sem_init(void);
 void	_thr_free(struct pthread *, struct pthread *);
 void	_thr_gc(struct pthread *);
 void	_thread_cleanupspecific(void);
diff --git a/lib/libthread_xu/thread/thr_rtld.c b/lib/libthread_xu/thread/thr_rtld.c
index 3ab9fb710f..dfc5fd3b5e 100644
--- a/lib/libthread_xu/thread/thr_rtld.c
+++ b/lib/libthread_xu/thread/thr_rtld.c
@@ -36,112 +36,12 @@
 #include "rtld_lock.h"
 #include "thr_private.h"
 
-static int	_thr_rtld_clr_flag(int);
-static void	*_thr_rtld_lock_create(void);
-static void	_thr_rtld_lock_destroy(void *);
-static void	_thr_rtld_lock_release(void *);
-static void	_thr_rtld_rlock_acquire(void *);
-static int	_thr_rtld_set_flag(int);
-static void	_thr_rtld_wlock_acquire(void *);
-
-static void *
-_thr_rtld_lock_create(void)
-{
-	pthread_rwlock_t prwlock;
-	if (_pthread_rwlock_init(&prwlock, NULL))
-		return (NULL);
-	return (prwlock);
-}
-
-static void
-_thr_rtld_lock_destroy(void *lock)
-{
-	pthread_rwlock_t prwlock;
-
-	prwlock = (pthread_rwlock_t)lock;
-	if (prwlock != NULL)
-		_pthread_rwlock_destroy(&prwlock);
-}
-
-static void
-_thr_rtld_rlock_acquire(void *lock)
-{
-	pthread_rwlock_t prwlock;
-
-	prwlock = (pthread_rwlock_t)lock;
-	_pthread_rwlock_rdlock(&prwlock);
-}
-
-static void
-_thr_rtld_wlock_acquire(void *lock)
-{
-	pthread_rwlock_t prwlock;
-
-	prwlock = (pthread_rwlock_t)lock;
-	_pthread_rwlock_wrlock(&prwlock);
-}
-
-static void
-_thr_rtld_lock_release(void *lock)
-{
-	pthread_rwlock_t prwlock;
-
-	prwlock = (pthread_rwlock_t)lock;
-	_pthread_rwlock_unlock(&prwlock);
-}
-
-
-static int
-_thr_rtld_set_flag(int mask)
-{
-	struct pthread *curthread;
-	int bits;
-
-	curthread = tls_get_curthread();
-	if (curthread != NULL) {
-		bits = curthread->rtld_bits;
-		curthread->rtld_bits |= mask;
-	} else {
-		bits = 0;
-		PANIC("No current thread in rtld call");
-	}
-
-	return (bits);
-}
-
-static int
-_thr_rtld_clr_flag(int mask)
-{
-	struct pthread *curthread;
-	int bits;
-
-	curthread = tls_get_curthread();
-	if (curthread != NULL) {
-		bits = curthread->rtld_bits;
-		curthread->rtld_bits &= ~mask;
-	} else {
-		bits = 0;
-		PANIC("No current thread in rtld call");
-	}
-	return (bits);
-}
-
 void
 _thr_rtld_init(void)
 {
-	struct RtldLockInfo li;
 	static int once = 0;
 
-	memset(&li, 0, sizeof(li));
-	li.lock_create  = _thr_rtld_lock_create;
-	li.lock_destroy = _thr_rtld_lock_destroy;
-	li.rlock_acquire = _thr_rtld_rlock_acquire;
-	li.wlock_acquire = _thr_rtld_wlock_acquire;
-	li.lock_release  = _thr_rtld_lock_release;
-	li.thread_set_flag = _thr_rtld_set_flag;
-	li.thread_clr_flag = _thr_rtld_clr_flag;
-	li.at_fork = NULL;
-	_rtld_thread_init(&li);
+	_rtld_thread_init(NULL);
 	if (once == 0) {
 		once = 1;
 		_thr_atfork_kern(_rtld_thread_prefork,
@@ -153,5 +53,5 @@ _thr_rtld_init(void)
 void
 _thr_rtld_fini(void)
 {
-	_rtld_thread_init(NULL);
+	/* _rtld_thread_init(NULL); */
 }
diff --git a/lib/libthread_xu/thread/thr_sem.c b/lib/libthread_xu/thread/thr_sem.c
index c7bb843d1d..a079ace3ba 100644
--- a/lib/libthread_xu/thread/thr_sem.c
+++ b/lib/libthread_xu/thread/thr_sem.c
@@ -44,10 +44,15 @@
 #include <string.h>
 #include <time.h>
 #include <unistd.h>
+#ifdef _PTHREADS_DEBUGGING
+#include <stdio.h>
+#endif
 #include "un-namespace.h"
 
 #include "thr_private.h"
 
+#define cpu_ccfence()        __asm __volatile("" : : : "memory")
+
 #define container_of(ptr, type, member)				\
 ({								\
 	__typeof(((type *)0)->member) *_p = (ptr);		\
@@ -58,11 +63,11 @@
  * Semaphore definitions.
  */
 struct sem {
-	u_int32_t		magic;
 	volatile umtx_t		count;
+	u_int32_t		magic;
 	int			semid;
 	int			unused; /* pad */
-};
+} __cachealign;
 
 #define	SEM_MAGIC	((u_int32_t) 0x09fa4012)
 
@@ -98,12 +103,34 @@ struct sem_info {
 	LIST_ENTRY(sem_info) next;
 };
 
-
-
-static pthread_once_t once = PTHREAD_ONCE_INIT;
 static pthread_mutex_t sem_lock;
 static LIST_HEAD(,sem_info) sem_list = LIST_HEAD_INITIALIZER(sem_list);
 
+#ifdef _PTHREADS_DEBUGGING
+
+static
+void
+sem_log(const char *ctl, ...)
+{
+        char buf[256];
+        va_list va;
+        size_t len;
+
+        va_start(va, ctl);
+        len = vsnprintf(buf, sizeof(buf), ctl, va);
+        va_end(va);
+        _thr_log(buf, len);
+}
+
+#else
+
+static __inline
+void
+sem_log(const char *ctl __unused, ...)
+{
+}
+
+#endif
 
 #define SEMID_LWP	0
 #define SEMID_FORK	1
@@ -127,8 +154,8 @@ sem_child_postfork(void)
 	_pthread_mutex_unlock(&sem_lock);
 }
 
-static void
-sem_module_init(void)
+void
+_thr_sem_init(void)
 {
 	pthread_mutexattr_t ma;
 
@@ -187,6 +214,9 @@ sem_alloc(unsigned int value, int pshared)
 	sem->magic = SEM_MAGIC;
 	sem->count = (u_int32_t)value;
 	sem->semid = semid;
+
+	sem_log("sem_alloc %p (%d)\n", sem, value);
+
 	return (sem);
 }
 
@@ -235,8 +265,8 @@ _sem_getvalue(sem_t * __restrict sem, int * __restrict sval)
 		errno = EINVAL;
 		return (-1);
 	}
-
 	*sval = (*sem)->count;
+
 	return (0);
 }
 
@@ -250,11 +280,16 @@ _sem_trywait(sem_t *sem)
 		return (-1);
 	}
 
+	sem_log("sem_trywait %p %d\n", *sem, (*sem)->count);
 	while ((val = (*sem)->count) > 0) {
-		if (atomic_cmpset_int(&(*sem)->count, val, val - 1))
+		cpu_ccfence();
+		if (atomic_cmpset_int(&(*sem)->count, val, val - 1)) {
+			sem_log("sem_trywait %p %d (success)\n", *sem, val - 1);
 			return (0);
+		}
 	}
 	errno = EAGAIN;
+	sem_log("sem_trywait %p %d (failure)\n", *sem, val);
 	return (-1);
 }
 
@@ -272,17 +307,28 @@ _sem_wait(sem_t *sem)
 	curthread = tls_get_curthread();
 	_pthread_testcancel();
 
-	for (;;) {
+	sem_log("sem_wait %p %d (begin)\n", *sem, (*sem)->count);
+
+	do {
+		cpu_ccfence();
 		while ((val = (*sem)->count) > 0) {
-			if (atomic_cmpset_acq_int(&(*sem)->count, val, val - 1))
+			cpu_ccfence();
+			if (atomic_cmpset_acq_int(&(*sem)->count, val, val - 1)) {
+				sem_log("sem_wait %p %d (success)\n",
+					*sem, val - 1);
 				return (0);
+			}
 		}
 		oldcancel = _thr_cancel_enter(curthread);
-		retval = _thr_umtx_wait(&(*sem)->count, 0, NULL, 0);
+		sem_log("sem_wait %p %d (wait)\n", *sem, val);
+		retval = _thr_umtx_wait_intr(&(*sem)->count, 0);
+		sem_log("sem_wait %p %d (wait return %d)\n",
+			*sem, (*sem)->count, retval);
 		_thr_cancel_leave(curthread, oldcancel);
 		/* ignore retval */
-	}
+	} while (retval != EINTR);
 
+	sem_log("sem_wait %p %d (error %d)\n", *sem, retval);
 	errno = retval;
 
 	return (-1);
@@ -300,6 +346,7 @@ _sem_timedwait(sem_t * __restrict sem, const struct timespec * __restrict abstim
 
 	curthread = tls_get_curthread();
 	_pthread_testcancel();
+	sem_log("sem_timedwait %p %d (begin)\n", *sem, (*sem)->count);
 
 	/*
 	 * The timeout argument is only supposed to
@@ -307,23 +354,32 @@ _sem_timedwait(sem_t * __restrict sem, const struct timespec * __restrict abstim
 	 */
 	do {
 		while ((val = (*sem)->count) > 0) {
-			if (atomic_cmpset_acq_int(&(*sem)->count, val, val - 1))
+			cpu_ccfence();
+			if (atomic_cmpset_acq_int(&(*sem)->count, val, val - 1)) {
+				sem_log("sem_wait %p %d (success)\n",
+					*sem, val - 1);
 				return (0);
+			}
 		}
 		if (abstime == NULL ||
 		    abstime->tv_nsec >= 1000000000 ||
 		    abstime->tv_nsec < 0) {
+			sem_log("sem_wait %p %d (bad abstime)\n", *sem, val);
 			errno = EINVAL;
 			return (-1);
 		}
 		clock_gettime(CLOCK_REALTIME, &ts);
 		TIMESPEC_SUB(&ts2, abstime, &ts);
 		oldcancel = _thr_cancel_enter(curthread);
+		sem_log("sem_wait %p %d (wait)\n", *sem, val);
 		retval = _thr_umtx_wait(&(*sem)->count, 0, &ts2,
 					CLOCK_REALTIME);
+		sem_log("sem_wait %p %d (wait return %d)\n",
+			*sem, (*sem)->count, retval);
 		_thr_cancel_leave(curthread, oldcancel);
 	} while (retval != ETIMEDOUT && retval != EINTR);
 
+	sem_log("sem_wait %p %d (error %d)\n", *sem, retval);
 	errno = retval;
 
 	return (-1);
@@ -341,10 +397,10 @@ _sem_post(sem_t *sem)
 	 * sem_post() is required to be safe to call from within
 	 * signal handlers, these code should work as that.
 	 */
-	do {
-		val = (*sem)->count;
-	} while (!atomic_cmpset_acq_int(&(*sem)->count, val, val + 1));
-	_thr_umtx_wake(&(*sem)->count, val + 1);
+	val = atomic_fetchadd_int(&(*sem)->count, 1) + 1;
+	sem_log("sem_post %p %d\n", *sem, val);
+	_thr_umtx_wake(&(*sem)->count, 0);
+
 	return (0);
 }
 
@@ -486,8 +542,6 @@ _sem_open(const char *name, int oflag, ...)
 		return (SEM_FAILED);
 	}
 
-	_pthread_once(&once, sem_module_init);
-
 	_pthread_mutex_lock(&sem_lock);
 
 	error = get_path(name, path, PATH_MAX, &prefix);
@@ -649,8 +703,6 @@ error:
 int
 _sem_close(sem_t *sem)
 {
-	_pthread_once(&once, sem_module_init);
-
 	_pthread_mutex_lock(&sem_lock);
 
 	if (sem_check_validity(sem)) {
diff --git a/lib/libthread_xu/thread/thr_sig.c b/lib/libthread_xu/thread/thr_sig.c
index 06356d576c..3a19ab5e03 100644
--- a/lib/libthread_xu/thread/thr_sig.c
+++ b/lib/libthread_xu/thread/thr_sig.c
@@ -102,7 +102,7 @@ _thr_suspend_check(struct pthread *curthread)
 		cycle = curthread->cycle;
 
 		/* Wake the thread suspending us. */
-		_thr_umtx_wake(&curthread->cycle, INT_MAX);
+		_thr_umtx_wake(&curthread->cycle, 0);
 
 		/*
 		 * if we are from pthread_exit, we don't want to
diff --git a/lib/libthread_xu/thread/thr_umtx.c b/lib/libthread_xu/thread/thr_umtx.c
index 22166bd83a..9376add8e7 100644
--- a/lib/libthread_xu/thread/thr_umtx.c
+++ b/lib/libthread_xu/thread/thr_umtx.c
@@ -47,19 +47,18 @@ __thr_umtx_lock(volatile umtx_t *mtx, int id, int timo)
 {
 	int v, errval, ret = 0;
 
+	v = *mtx;
+	cpu_ccfence();
 	id &= 0x3FFFFFFF;
-	/* contested */
+
 	for (;;) {
-		v = *mtx;
-		cpu_ccfence();
 		if (v == 0) {
 			if (atomic_cmpset_acq_int(mtx, 0, id)) {
 				break;
 			}
 			continue;
 		}
-		if ((v & 0x40000000) ||
-		    atomic_cmpset_acq_int(mtx, v, v|0x40000000)) {
+		if (atomic_fcmpset_int(mtx, &v, v|0x40000000)) {
 			if (timo == 0) {
 				_umtx_sleep_err(mtx, v|0x40000000, timo);
 			} else if ((errval = _umtx_sleep_err(mtx, v|0x40000000, timo)) > 0) {
@@ -85,13 +84,14 @@ __thr_umtx_unlock(volatile umtx_t *mtx, int id)
 {
 	int v;
 
+	v = *mtx;
+	cpu_ccfence();
 	id &= 0x3FFFFFFF;
+
 	for (;;) {
-		v = *mtx;
-		cpu_ccfence();
-		if (atomic_cmpset_acq_int(mtx, v, 0)) {
+		if (atomic_fcmpset_int(mtx, &v, 0)) {
 			if (v & 0x40000000)
-				_umtx_wakeup_err(mtx, 1);
+				_umtx_wakeup_err(mtx, 0);
 			THR_ASSERT((v & 0x3FFFFFFF) == id,
 				   "thr_umtx_unlock: wrong owner");
 			break;
@@ -144,6 +144,9 @@ __thr_umtx_timedlock(volatile umtx_t *mtx, int id,
 	return (ret);
 }
 
+/*
+ * Regular umtx wait that cannot return EINTR
+ */
 int
 _thr_umtx_wait(volatile umtx_t *mtx, int exp, const struct timespec *timeout,
 	       int clockid)
@@ -220,6 +223,33 @@ _thr_umtx_wait(volatile umtx_t *mtx, int exp, const struct timespec *timeout,
 	return (ret);
 }
 
+/*
+ * Simple version without a timeout which can also return EINTR
+ */
+int
+_thr_umtx_wait_intr(volatile umtx_t *mtx, int exp)
+{
+	int ret = 0;
+	int errval;
+
+	cpu_ccfence();
+	for (;;) {
+		if (*mtx != exp)
+			return (0);
+		errval = _umtx_sleep_err(mtx, exp, 10000000);
+		if (errval == 0)
+			break;
+		if (errval == EBUSY)
+			break;
+		if (errval == EINTR) {
+			ret = errval;
+			break;
+		}
+		cpu_ccfence();
+	}
+	return (ret);
+}
+
 void
 _thr_umtx_wake(volatile umtx_t *mtx, int count)
 {
diff --git a/lib/libthread_xu/thread/thr_umtx.h b/lib/libthread_xu/thread/thr_umtx.h
index 2c210d33fa..36ba7697d7 100644
--- a/lib/libthread_xu/thread/thr_umtx.h
+++ b/lib/libthread_xu/thread/thr_umtx.h
@@ -85,5 +85,6 @@ _thr_umtx_unlock(volatile umtx_t *mtx, int id)
 
 int _thr_umtx_wait(volatile umtx_t *mtx, umtx_t exp,
 		   const struct timespec *timeout, int clockid);
+int _thr_umtx_wait_intr(volatile umtx_t *mtx, umtx_t exp);
 void _thr_umtx_wake(volatile umtx_t *mtx, int count);
 #endif
diff --git a/libexec/rtld-elf/rtld_lock.c b/libexec/rtld-elf/rtld_lock.c
index 25bca852b8..124b45a862 100644
--- a/libexec/rtld-elf/rtld_lock.c
+++ b/libexec/rtld-elf/rtld_lock.c
@@ -47,214 +47,142 @@
 #include <stdlib.h>
 #include <time.h>
 
+#include <stdio.h>
+#include <sys/file.h>
+
 #include "debug.h"
 #include "rtld.h"
 #include "rtld_machdep.h"
 
+extern pid_t __sys_getpid(void);
+
 #define WAFLAG		0x1	/* A writer holds the lock */
 #define RC_INCR		0x2	/* Adjusts count of readers desiring lock */
 
-typedef struct Struct_Lock {
+struct Struct_Lock {
 	volatile u_int lock;
-	void *base;
-} Lock;
-
-static sigset_t fullsigmask, oldsigmask;
-static int thread_flag;
-
-static void *
-def_lock_create(void)
-{
-    void *base;
-    char *p;
-    uintptr_t r;
-    Lock *l;
-
-    /*
-     * Arrange for the lock to occupy its own cache line.  First, we
-     * optimistically allocate just a cache line, hoping that malloc
-     * will give us a well-aligned block of memory.  If that doesn't
-     * work, we allocate a larger block and take a well-aligned cache
-     * line from it.
-     */
-    base = xmalloc(CACHE_LINE_SIZE);
-    p = (char *)base;
-    if ((uintptr_t)p % CACHE_LINE_SIZE != 0) {
-	free(base);
-	base = xmalloc(2 * CACHE_LINE_SIZE);
-	p = (char *)base;
-	if ((r = (uintptr_t)p % CACHE_LINE_SIZE) != 0)
-	    p += CACHE_LINE_SIZE - r;
-    }
-    l = (Lock *)p;
-    l->base = base;
-    l->lock = 0;
-    return l;
-}
-
-static void
-def_lock_destroy(void *lock)
-{
-    Lock *l = (Lock *)lock;
-
-    free(l->base);
-}
-
-static void
-def_rlock_acquire(void *lock)
-{
-    Lock *l = (Lock *)lock;
-
-    atomic_add_acq_int(&l->lock, RC_INCR);
-    while (l->lock & WAFLAG)
-	    ;	/* Spin */
-}
-
-static void
-def_wlock_acquire(void *lock)
-{
-    Lock *l = (Lock *)lock;
-    sigset_t tmp_oldsigmask;
-
-    for ( ; ; ) {
-	sigprocmask(SIG_BLOCK, &fullsigmask, &tmp_oldsigmask);
-	if (atomic_cmpset_acq_int(&l->lock, 0, WAFLAG))
-	    break;
-	sigprocmask(SIG_SETMASK, &tmp_oldsigmask, NULL);
-    }
-    oldsigmask = tmp_oldsigmask;
-}
+	int tid;		/* owner (exclusive) */
+	int count;		/* recursion (exclusive) */
+	sigset_t savesigmask;	/* first exclusive owner sets mask */
+} __cachealign;
 
-static void
-def_lock_release(void *lock)
-{
-    Lock *l = (Lock *)lock;
+#define cpu_ccfence()	__asm __volatile("" : : : "memory")
 
-    if ((l->lock & WAFLAG) == 0)
-	atomic_add_rel_int(&l->lock, -RC_INCR);
-    else {
-	atomic_add_rel_int(&l->lock, -WAFLAG);
-	sigprocmask(SIG_SETMASK, &oldsigmask, NULL);
-    }
-}
+static sigset_t fullsigmask;
 
-static int
-def_thread_set_flag(int mask)
-{
-	int old_val = thread_flag;
-	thread_flag |= mask;
-	return (old_val);
-}
+struct Struct_Lock phdr_lock;
+struct Struct_Lock bind_lock;
+struct Struct_Lock libc_lock;
 
-static int
-def_thread_clr_flag(int mask)
-{
-	int old_val = thread_flag;
-	thread_flag &= ~mask;
-	return (old_val);
-}
+rtld_lock_t	rtld_phdr_lock = &phdr_lock;
+rtld_lock_t	rtld_bind_lock = &bind_lock;
+rtld_lock_t	rtld_libc_lock = &libc_lock;
 
-/*
- * Public interface exposed to the rest of the dynamic linker.
- */
-static struct RtldLockInfo lockinfo;
-static struct RtldLockInfo deflockinfo;
-
-static __inline int
-thread_mask_set(int mask)
-{
-	return lockinfo.thread_set_flag(mask);
-}
-
-static __inline void
-thread_mask_clear(int mask)
-{
-	lockinfo.thread_clr_flag(mask);
+void
+rlock_acquire(rtld_lock_t lock, RtldLockState *state)
+{
+	int v;
+	int tid = 0;
+
+	v = lock->lock;
+	cpu_ccfence();
+	for (;;) {
+		if ((v & WAFLAG) == 0) {
+			if (atomic_fcmpset_int(&lock->lock, &v, v + RC_INCR)) {
+				state->lockstate = RTLD_LOCK_RLOCKED;
+				break;
+			}
+		} else {
+			if (tid == 0)
+				tid = lwp_gettid();
+			if (lock->tid == tid) {
+				++lock->count;
+				state->lockstate = RTLD_LOCK_WLOCKED;
+				break;
+			}
+			umtx_sleep(&lock->lock, v, 0);
+			v = lock->lock;
+			cpu_ccfence();
+		}
+	}
 }
 
-#define	RTLD_LOCK_CNT	3
-struct rtld_lock {
-	void	*handle;
-	int	 mask;
-	RtldLockState forkstate;
-} rtld_locks[RTLD_LOCK_CNT];
-
-rtld_lock_t	rtld_bind_lock = &rtld_locks[0];
-rtld_lock_t	rtld_libc_lock = &rtld_locks[1];
-rtld_lock_t	rtld_phdr_lock = &rtld_locks[2];
-
 void
-rlock_acquire(rtld_lock_t lock, RtldLockState *lockstate)
+wlock_acquire(rtld_lock_t lock, RtldLockState *state)
 {
+	sigset_t tmp_oldsigmask;
+	int tid = lwp_gettid();
 
-	if (lockstate == NULL)
+	if (lock->tid == tid) {
+		++lock->count;
+		state->lockstate = RTLD_LOCK_WLOCKED;
 		return;
+	}
 
-	if (thread_mask_set(lock->mask) & lock->mask) {
-		dbg("rlock_acquire: recursed");
-		lockstate->lockstate = RTLD_LOCK_UNLOCKED;
-		return;
+	sigprocmask(SIG_BLOCK, &fullsigmask, &tmp_oldsigmask);
+	for (;;) {
+		if (atomic_cmpset_acq_int(&lock->lock, 0, WAFLAG))
+			break;
+		umtx_sleep(&lock->lock, 0, 0);
 	}
-	lockinfo.rlock_acquire(lock->handle);
-	lockstate->lockstate = RTLD_LOCK_RLOCKED;
+	lock->tid = tid;
+	lock->count = 1;
+	lock->savesigmask = tmp_oldsigmask;
+	state->lockstate = RTLD_LOCK_WLOCKED;
 }
 
 void
-wlock_acquire(rtld_lock_t lock, RtldLockState *lockstate)
+lock_release(rtld_lock_t lock, RtldLockState *state)
 {
-	if (lockstate == NULL)
-		return;
+	sigset_t tmp_oldsigmask;
+	int v;
 
-	if (thread_mask_set(lock->mask) & lock->mask) {
-		dbg("wlock_acquire: recursed");
-		lockstate->lockstate = RTLD_LOCK_UNLOCKED;
+	if (state->lockstate == RTLD_LOCK_UNLOCKED)
 		return;
+	if ((lock->lock & WAFLAG) == 0) {
+		v = atomic_fetchadd_int(&lock->lock, -RC_INCR) - RC_INCR;
+		if (v == 0)
+			umtx_wakeup(&lock->lock, 0);
+	} else if (--lock->count == 0) {
+		tmp_oldsigmask = lock->savesigmask;
+		lock->tid = 0;
+		v = atomic_fetchadd_int(&lock->lock, -WAFLAG) - WAFLAG;
+		if (v == 0)
+			umtx_wakeup(&lock->lock, 0);
+		sigprocmask(SIG_SETMASK, &tmp_oldsigmask, NULL);
 	}
-	lockinfo.wlock_acquire(lock->handle);
-	lockstate->lockstate = RTLD_LOCK_WLOCKED;
+	state->lockstate = RTLD_LOCK_UNLOCKED;
 }
 
+static
 void
-lock_release(rtld_lock_t lock, RtldLockState *lockstate)
+lock_reset(rtld_lock_t lock)
 {
-	if (lockstate == NULL)
-		return;
-
-	switch (lockstate->lockstate) {
-	case RTLD_LOCK_UNLOCKED:
-		break;
-	case RTLD_LOCK_RLOCKED:
-	case RTLD_LOCK_WLOCKED:
-		lockinfo.lock_release(lock->handle);
-		thread_mask_clear(lock->mask);
-		break;
-	default:
-		assert(0);
-	}
+	memset(lock, 0, sizeof(*lock));
 }
 
 void
-lock_upgrade(rtld_lock_t lock, RtldLockState *lockstate)
+lock_upgrade(rtld_lock_t lock, RtldLockState *state)
 {
-	if (lockstate == NULL)
+	if (state == NULL)
 		return;
-
-	lock_release(lock, lockstate);
-	wlock_acquire(lock, lockstate);
+	if (state->lockstate == RTLD_LOCK_RLOCKED) {
+		lock_release(lock, state);
+		wlock_acquire(lock, state);
+	}
 }
 
 void
-lock_restart_for_upgrade(RtldLockState *lockstate)
+lock_restart_for_upgrade(RtldLockState *state)
 {
-	if (lockstate == NULL)
+	if (state == NULL)
 		return;
-
-	switch (lockstate->lockstate) {
+	switch (state->lockstate) {
 	case RTLD_LOCK_UNLOCKED:
 	case RTLD_LOCK_WLOCKED:
 		break;
 	case RTLD_LOCK_RLOCKED:
-		siglongjmp(lockstate->env, 1);
+		siglongjmp(state->env, 1);
 		break;
 	default:
 		assert(0);
@@ -264,130 +192,60 @@ lock_restart_for_upgrade(RtldLockState *lockstate)
 void
 lockdflt_init(void)
 {
-    int i;
-
-    deflockinfo.rtli_version  = RTLI_VERSION;
-    deflockinfo.lock_create   = def_lock_create;
-    deflockinfo.lock_destroy  = def_lock_destroy;
-    deflockinfo.rlock_acquire = def_rlock_acquire;
-    deflockinfo.wlock_acquire = def_wlock_acquire;
-    deflockinfo.lock_release  = def_lock_release;
-    deflockinfo.thread_set_flag = def_thread_set_flag;
-    deflockinfo.thread_clr_flag = def_thread_clr_flag;
-    deflockinfo.at_fork = NULL;
+	/*
+	 * Construct a mask to block all signals except traps which might
+	 * conceivably be generated within the dynamic linker itself.
+	 */
+	sigfillset(&fullsigmask);
+	sigdelset(&fullsigmask, SIGILL);
+	sigdelset(&fullsigmask, SIGTRAP);
+	sigdelset(&fullsigmask, SIGABRT);
+	sigdelset(&fullsigmask, SIGEMT);
+	sigdelset(&fullsigmask, SIGFPE);
+	sigdelset(&fullsigmask, SIGBUS);
+	sigdelset(&fullsigmask, SIGSEGV);
+	sigdelset(&fullsigmask, SIGSYS);
 
-    for (i = 0; i < RTLD_LOCK_CNT; i++) {
-	    rtld_locks[i].mask   = (1 << i);
-	    rtld_locks[i].handle = NULL;
-    }
-
-    memcpy(&lockinfo, &deflockinfo, sizeof(lockinfo));
-    _rtld_thread_init(NULL);
-    /*
-     * Construct a mask to block all signals except traps which might
-     * conceivably be generated within the dynamic linker itself.
-     */
-    sigfillset(&fullsigmask);
-    sigdelset(&fullsigmask, SIGILL);
-    sigdelset(&fullsigmask, SIGTRAP);
-    sigdelset(&fullsigmask, SIGABRT);
-    sigdelset(&fullsigmask, SIGEMT);
-    sigdelset(&fullsigmask, SIGFPE);
-    sigdelset(&fullsigmask, SIGBUS);
-    sigdelset(&fullsigmask, SIGSEGV);
-    sigdelset(&fullsigmask, SIGSYS);
+	_rtld_thread_init(NULL);
 }
 
 /*
- * Callback function to allow threads implementation to
- * register their own locking primitives if the default
- * one is not suitable.
- * The current context should be the only context
- * executing at the invocation time.
+ * (also called by pthreads)
  */
 void
-_rtld_thread_init(struct RtldLockInfo *pli)
+_rtld_thread_init(void *dummy __unused)
 {
-	int flags, i;
-	void *locks[RTLD_LOCK_CNT];
-
-	/* disable all locking while this function is running */
-	flags =	thread_mask_set(~0);
-
-	if (pli == NULL)
-		pli = &deflockinfo;
-
-
-	for (i = 0; i < RTLD_LOCK_CNT; i++) {
-		if ((locks[i] = pli->lock_create()) == NULL)
-			break;
-	}
-
-	if (i < RTLD_LOCK_CNT) {
-		while (--i >= 0) {
-			pli->lock_destroy(locks[i]);
-		}
-		abort();
-	}
-
-	for (i = 0; i < RTLD_LOCK_CNT; i++) {
-		if (rtld_locks[i].handle == NULL)
-			continue;
-		if (flags & rtld_locks[i].mask)
-			lockinfo.lock_release(rtld_locks[i].handle);
-		lockinfo.lock_destroy(rtld_locks[i].handle);
-	}
-
-	for (i = 0; i < RTLD_LOCK_CNT; i++) {
-		rtld_locks[i].handle = locks[i];
-		if (flags & rtld_locks[i].mask)
-			pli->wlock_acquire(rtld_locks[i].handle);
-	}
-
-	lockinfo.lock_create = pli->lock_create;
-	lockinfo.lock_destroy = pli->lock_destroy;
-	lockinfo.rlock_acquire = pli->rlock_acquire;
-	lockinfo.wlock_acquire = pli->wlock_acquire;
-	lockinfo.lock_release  = pli->lock_release;
-	lockinfo.thread_set_flag = pli->thread_set_flag;
-	lockinfo.thread_clr_flag = pli->thread_clr_flag;
-	lockinfo.at_fork = pli->at_fork;
-
-	/* restore thread locking state, this time with new locks */
-	thread_mask_clear(~0);
-	thread_mask_set(flags);
-	dbg("_rtld_thread_init: done");
+	lock_reset(rtld_phdr_lock);
+	lock_reset(rtld_bind_lock);
+	lock_reset(rtld_libc_lock);
 }
 
+static RtldLockState fork_states[3];
+
 void
 _rtld_thread_prefork(void)
 {
-	int i;
-
-	for (i = 0; i < RTLD_LOCK_CNT; i++) {
-		if (rtld_locks[i].handle == NULL)
-			continue;
-		wlock_acquire(&rtld_locks[i], &rtld_locks[i].forkstate);
-	}
+	wlock_acquire(rtld_phdr_lock, &fork_states[0]);
+	wlock_acquire(rtld_bind_lock, &fork_states[1]);
+	wlock_acquire(rtld_libc_lock, &fork_states[2]);
 }
 
 void
 _rtld_thread_postfork(void)
 {
-	int i;
-
-	for (i = 0; i < RTLD_LOCK_CNT; i++) {
-		if (rtld_locks[i].handle == NULL)
-			continue;
-		lock_release(&rtld_locks[i], &rtld_locks[i].forkstate);
-	}
+	lock_release(rtld_libc_lock, &fork_states[2]);
+	lock_release(rtld_bind_lock, &fork_states[1]);
+	lock_release(rtld_phdr_lock, &fork_states[0]);
 }
 
-/*
- * pthreads already called _rtld_thread_init(NULL) in the child, so there
- * is nothing for us to do here.
- */
 void
 _rtld_thread_childfork(void)
 {
+	sigset_t tmp_oldsigmask;
+
+	lock_reset(rtld_phdr_lock);
+	lock_reset(rtld_bind_lock);
+	tmp_oldsigmask = rtld_libc_lock->savesigmask;
+	lock_reset(rtld_libc_lock);
+	sigprocmask(SIG_SETMASK, &tmp_oldsigmask, NULL);
 }
diff --git a/libexec/rtld-elf/rtld_lock.h b/libexec/rtld-elf/rtld_lock.h
index 3096f1f377..e9993e7d8e 100644
--- a/libexec/rtld-elf/rtld_lock.h
+++ b/libexec/rtld-elf/rtld_lock.h
@@ -28,7 +28,8 @@
 #ifndef _RTLD_LOCK_H_
 #define	_RTLD_LOCK_H_
 
-#define	RTLI_VERSION	0x01
+#define	RTLI_VERSION		0x02
+#define	RTLI_VERSION_FORK	0x02
 
 struct RtldLockInfo
 {
@@ -41,22 +42,23 @@ struct RtldLockInfo
 	int   (*thread_set_flag)(int);
 	int   (*thread_clr_flag)(int);
 	void  (*at_fork)(void *handle);
+	void  (*fork_acquire)(void *);
+	void  (*fork_release)(void *);
 };
 
-extern void _rtld_thread_init(struct RtldLockInfo *);
+extern void _rtld_thread_init(void *);
 extern void _rtld_thread_prefork(void);
 extern void _rtld_thread_postfork(void);
 extern void _rtld_thread_childfork(void);
 
-
 #ifdef IN_RTLD
 
-struct rtld_lock;
-typedef struct rtld_lock *rtld_lock_t;
+struct Struct_Lock;
+typedef struct Struct_Lock *rtld_lock_t;
 
-extern rtld_lock_t	rtld_bind_lock;
-extern rtld_lock_t	rtld_libc_lock;
-extern rtld_lock_t	rtld_phdr_lock;
+rtld_lock_t rtld_bind_lock;
+rtld_lock_t rtld_libc_lock;
+rtld_lock_t rtld_phdr_lock;
 
 #define	RTLD_LOCK_UNLOCKED	0
 #define	RTLD_LOCK_RLOCKED	1
diff --git a/libexec/rtld-elf/x86_64/rtld_machdep.h b/libexec/rtld-elf/x86_64/rtld_machdep.h
index 7c873401f6..2fefa88011 100644
--- a/libexec/rtld-elf/x86_64/rtld_machdep.h
+++ b/libexec/rtld-elf/x86_64/rtld_machdep.h
@@ -32,8 +32,6 @@
 #include <sys/types.h>
 #include <machine/atomic.h>
 
-#define	CACHE_LINE_SIZE		64
-
 struct Struct_Obj_Entry;
 
 /* Return the address of the .dynamic section in the dynamic linker. */