pthreads - Fix rtld-elf and libthread_xu
authorMatthew Dillon <dillon@apollo.backplane.com>
Thu, 2 Nov 2017 23:21:13 +0000 (16:21 -0700)
committerMatthew Dillon <dillon@apollo.backplane.com>
Thu, 2 Nov 2017 23:22:38 +0000 (16:22 -0700)
* Fixes chrome, thunderbird, and multiple other issues with recent
  libpthreads work.

Testing-by: mneumann, dillon
15 files changed:
lib/libthread_xu/thread/thr_barrier.c
lib/libthread_xu/thread/thr_create.c
lib/libthread_xu/thread/thr_exit.c
lib/libthread_xu/thread/thr_fork.c
lib/libthread_xu/thread/thr_init.c
lib/libthread_xu/thread/thr_kern.c
lib/libthread_xu/thread/thr_private.h
lib/libthread_xu/thread/thr_rtld.c
lib/libthread_xu/thread/thr_sem.c
lib/libthread_xu/thread/thr_sig.c
lib/libthread_xu/thread/thr_umtx.c
lib/libthread_xu/thread/thr_umtx.h
libexec/rtld-elf/rtld_lock.c
libexec/rtld-elf/rtld_lock.h
libexec/rtld-elf/x86_64/rtld_machdep.h

index 7f00ed1..d592176 100644 (file)
@@ -91,7 +91,7 @@ _pthread_barrier_wait(pthread_barrier_t *barrier)
                /* Current thread is lastest thread */
                bar->b_waiters = 0;
                bar->b_cycle++;
-               _thr_umtx_wake(&bar->b_cycle, bar->b_count);
+               _thr_umtx_wake(&bar->b_cycle, 0);
                THR_UMTX_UNLOCK(curthread, &bar->b_lock);
                ret = PTHREAD_BARRIER_SERIAL_THREAD;
        } else {
index 18d9a1a..2ddfa24 100644 (file)
@@ -178,7 +178,7 @@ _pthread_create(pthread_t * thread, const pthread_attr_t * attr,
                new_thread->terminated = 1;
                if (new_thread->flags & THR_FLAGS_NEED_SUSPEND) {
                        new_thread->cycle++;
-                       _thr_umtx_wake(&new_thread->cycle, INT_MAX);
+                       _thr_umtx_wake(&new_thread->cycle, 0);
                }
                THR_THREAD_UNLOCK(curthread, new_thread);
                THREAD_LIST_LOCK(curthread);
index 28a7a82..37a2f27 100644 (file)
@@ -152,7 +152,7 @@ exit_thread(void)
                THR_GCLIST_ADD(curthread);
        THREAD_LIST_UNLOCK(curthread);
        if (curthread->joiner)
-               _thr_umtx_wake(&curthread->state, INT_MAX);
+               _thr_umtx_wake(&curthread->state, 0);
        if (SHOULD_REPORT_EVENT(curthread, TD_DEATH))
                _thr_report_death(curthread);
        /* Exit and set terminated to once we're really dead. */
index 25033c6..0391a3b 100644 (file)
@@ -68,6 +68,7 @@
 #include <fcntl.h>
 #include <pthread.h>
 #include <spinlock.h>
+#include <sys/file.h>
 #include "un-namespace.h"
 
 #include "libc_private.h"
@@ -174,6 +175,7 @@ _fork(void)
        if (!_thr_is_inited())
                return (__syscall(SYS_fork));
 
+       errsave = errno;
        curthread = tls_get_curthread();
 
        THR_UMTX_LOCK(curthread, &_thr_atfork_lock);
@@ -194,10 +196,6 @@ _fork(void)
                if (af->prepare != NULL)
                        af->prepare();
        }
-       TAILQ_FOREACH_REVERSE(af, &_thr_atfork_kern_list, atfork_head, qe) {
-               if (af->prepare != NULL)
-                       af->prepare();
-       }
 
 #ifndef __DragonFly__
        /*
@@ -213,19 +211,38 @@ _fork(void)
        }
 #endif
 
-       /*
-        * Block all signals until we reach a safe point.
-        */
-       _thr_signal_block(curthread);
 #ifdef _PTHREADS_DEBUGGING
        _thr_log("fork-parent\n", 12);
 #endif
 
+       _thr_signal_block(curthread);
+
+       /*
+        * Must be executed Just before the fork.
+        */
+       TAILQ_FOREACH_REVERSE(af, &_thr_atfork_kern_list, atfork_head, qe) {
+               if (af->prepare != NULL)
+                       af->prepare();
+       }
+
        /* Fork a new process: */
        if ((ret = __syscall(SYS_fork)) == 0) {
-               /* Child process */
-               errsave = errno;
+               /*
+                * Child process.
+                *
+                * NOTE: We are using the saved errno from above.  Do not
+                *       reload errno here.
+                */
                inprogress = 0;
+
+               /*
+                * Internal child fork handlers must be run immediately.
+                */
+               TAILQ_FOREACH(af, &_thr_atfork_kern_list, qe) {
+                       if (af->child != NULL)
+                               af->child();
+               }
+
                curthread->cancelflags &= ~THR_CANCEL_NEEDED;
                /*
                 * Thread list will be reinitialized, and later we call
@@ -255,10 +272,6 @@ _fork(void)
                _thr_signal_unblock(curthread);
 
                /* Run down atfork child handlers. */
-               TAILQ_FOREACH(af, &_thr_atfork_kern_list, qe) {
-                       if (af->child != NULL)
-                               af->child();
-               }
                TAILQ_FOREACH(af, &_thr_atfork_list, qe) {
                        if (af->child != NULL)
                                af->child();
@@ -274,14 +287,15 @@ _fork(void)
 #ifdef _PTHREADS_DEBUGGING
                _thr_log("fork-done\n", 10);
 #endif
-               /* Ready to continue, unblock signals. */
-               _thr_signal_unblock(curthread);
-
                /* Run down atfork parent handlers. */
                TAILQ_FOREACH(af, &_thr_atfork_kern_list, qe) {
                        if (af->parent != NULL)
                                af->parent();
                }
+
+               /* Ready to continue, unblock signals. */
+               _thr_signal_unblock(curthread);
+
                TAILQ_FOREACH(af, &_thr_atfork_list, qe) {
                        if (af->parent != NULL)
                                af->parent();
@@ -290,7 +304,7 @@ _fork(void)
                THR_UMTX_LOCK(curthread, &_thr_atfork_lock);
                inprogress = 0;
                if (waiters)
-                       _thr_umtx_wake(&inprogress, waiters);
+                       _thr_umtx_wake(&inprogress, 0);
                THR_UMTX_UNLOCK(curthread, &_thr_atfork_lock);
        }
        errno = errsave;
index 3ac3de0..6bb6730 100644 (file)
@@ -286,6 +286,8 @@ _libpthread_init(struct pthread *curthread)
                __sys_sigprocmask(SIG_SETMASK, &oldset, NULL);
                if (td_eventismember(&_thread_event_mask, TD_CREATE))
                        _thr_report_creation(curthread, curthread);
+               _thr_rtld_init();
+               _thr_sem_init();
        }
 }
 
index 5a92dde..c15c2a3 100644 (file)
  * This is called when the first thread (other than the initial
  * thread) is created.
  *
- * NOTE: Once rtld is initialized for threading, we never de-initialize
- *      it (we do not call _thr_rtld_fini()).
+ * NOTE: we no longer call _thrd_rtld_fini here.
  */
 int
 _thr_setthreaded(int threaded)
 {
        if (((threaded == 0) ^ (__isthreaded == 0)) == 0)
                return (0);
-
        __isthreaded = threaded;
+#if 0
+       /* save for later. */
        if (threaded != 0)
-               _thr_rtld_init();
-
+               /* blah */ ;
+       else
+               /* blah */ ;
+#endif
        return (0);
 }
 
index efad533..e47acae 100644 (file)
@@ -681,6 +681,7 @@ void        _thr_rtld_fini(void);
 int    _thr_stack_alloc(struct pthread_attr *);
 void   _thr_stack_free(struct pthread_attr *);
 void   _thr_stack_cleanup(void);
+void   _thr_sem_init(void);
 void   _thr_free(struct pthread *, struct pthread *);
 void   _thr_gc(struct pthread *);
 void   _thread_cleanupspecific(void);
index 3ab9fb7..dfc5fd3 100644 (file)
 #include "rtld_lock.h"
 #include "thr_private.h"
 
-static int     _thr_rtld_clr_flag(int);
-static void    *_thr_rtld_lock_create(void);
-static void    _thr_rtld_lock_destroy(void *);
-static void    _thr_rtld_lock_release(void *);
-static void    _thr_rtld_rlock_acquire(void *);
-static int     _thr_rtld_set_flag(int);
-static void    _thr_rtld_wlock_acquire(void *);
-
-static void *
-_thr_rtld_lock_create(void)
-{
-       pthread_rwlock_t prwlock;
-       if (_pthread_rwlock_init(&prwlock, NULL))
-               return (NULL);
-       return (prwlock);
-}
-
-static void
-_thr_rtld_lock_destroy(void *lock)
-{
-       pthread_rwlock_t prwlock;
-
-       prwlock = (pthread_rwlock_t)lock;
-       if (prwlock != NULL)
-               _pthread_rwlock_destroy(&prwlock);
-}
-
-static void
-_thr_rtld_rlock_acquire(void *lock)
-{
-       pthread_rwlock_t prwlock;
-
-       prwlock = (pthread_rwlock_t)lock;
-       _pthread_rwlock_rdlock(&prwlock);
-}
-
-static void
-_thr_rtld_wlock_acquire(void *lock)
-{
-       pthread_rwlock_t prwlock;
-
-       prwlock = (pthread_rwlock_t)lock;
-       _pthread_rwlock_wrlock(&prwlock);
-}
-
-static void
-_thr_rtld_lock_release(void *lock)
-{
-       pthread_rwlock_t prwlock;
-
-       prwlock = (pthread_rwlock_t)lock;
-       _pthread_rwlock_unlock(&prwlock);
-}
-
-
-static int
-_thr_rtld_set_flag(int mask)
-{
-       struct pthread *curthread;
-       int bits;
-
-       curthread = tls_get_curthread();
-       if (curthread != NULL) {
-               bits = curthread->rtld_bits;
-               curthread->rtld_bits |= mask;
-       } else {
-               bits = 0;
-               PANIC("No current thread in rtld call");
-       }
-
-       return (bits);
-}
-
-static int
-_thr_rtld_clr_flag(int mask)
-{
-       struct pthread *curthread;
-       int bits;
-
-       curthread = tls_get_curthread();
-       if (curthread != NULL) {
-               bits = curthread->rtld_bits;
-               curthread->rtld_bits &= ~mask;
-       } else {
-               bits = 0;
-               PANIC("No current thread in rtld call");
-       }
-       return (bits);
-}
-
 void
 _thr_rtld_init(void)
 {
-       struct RtldLockInfo li;
        static int once = 0;
 
-       memset(&li, 0, sizeof(li));
-       li.lock_create  = _thr_rtld_lock_create;
-       li.lock_destroy = _thr_rtld_lock_destroy;
-       li.rlock_acquire = _thr_rtld_rlock_acquire;
-       li.wlock_acquire = _thr_rtld_wlock_acquire;
-       li.lock_release  = _thr_rtld_lock_release;
-       li.thread_set_flag = _thr_rtld_set_flag;
-       li.thread_clr_flag = _thr_rtld_clr_flag;
-       li.at_fork = NULL;
-       _rtld_thread_init(&li);
+       _rtld_thread_init(NULL);
        if (once == 0) {
                once = 1;
                _thr_atfork_kern(_rtld_thread_prefork,
@@ -153,5 +53,5 @@ _thr_rtld_init(void)
 void
 _thr_rtld_fini(void)
 {
-       _rtld_thread_init(NULL);
+       /* _rtld_thread_init(NULL); */
 }
index c7bb843..a079ace 100644 (file)
 #include <string.h>
 #include <time.h>
 #include <unistd.h>
+#ifdef _PTHREADS_DEBUGGING
+#include <stdio.h>
+#endif
 #include "un-namespace.h"
 
 #include "thr_private.h"
 
+#define cpu_ccfence()        __asm __volatile("" : : : "memory")
+
 #define container_of(ptr, type, member)                                \
 ({                                                             \
        __typeof(((type *)0)->member) *_p = (ptr);              \
  * Semaphore definitions.
  */
 struct sem {
-       u_int32_t               magic;
        volatile umtx_t         count;
+       u_int32_t               magic;
        int                     semid;
        int                     unused; /* pad */
-};
+} __cachealign;
 
 #define        SEM_MAGIC       ((u_int32_t) 0x09fa4012)
 
@@ -98,12 +103,34 @@ struct sem_info {
        LIST_ENTRY(sem_info) next;
 };
 
-
-
-static pthread_once_t once = PTHREAD_ONCE_INIT;
 static pthread_mutex_t sem_lock;
 static LIST_HEAD(,sem_info) sem_list = LIST_HEAD_INITIALIZER(sem_list);
 
+#ifdef _PTHREADS_DEBUGGING
+
+static
+void
+sem_log(const char *ctl, ...)
+{
+        char buf[256];
+        va_list va;
+        size_t len;
+
+        va_start(va, ctl);
+        len = vsnprintf(buf, sizeof(buf), ctl, va);
+        va_end(va);
+        _thr_log(buf, len);
+}
+
+#else
+
+static __inline
+void
+sem_log(const char *ctl __unused, ...)
+{
+}
+
+#endif
 
 #define SEMID_LWP      0
 #define SEMID_FORK     1
@@ -127,8 +154,8 @@ sem_child_postfork(void)
        _pthread_mutex_unlock(&sem_lock);
 }
 
-static void
-sem_module_init(void)
+void
+_thr_sem_init(void)
 {
        pthread_mutexattr_t ma;
 
@@ -187,6 +214,9 @@ sem_alloc(unsigned int value, int pshared)
        sem->magic = SEM_MAGIC;
        sem->count = (u_int32_t)value;
        sem->semid = semid;
+
+       sem_log("sem_alloc %p (%d)\n", sem, value);
+
        return (sem);
 }
 
@@ -235,8 +265,8 @@ _sem_getvalue(sem_t * __restrict sem, int * __restrict sval)
                errno = EINVAL;
                return (-1);
        }
-
        *sval = (*sem)->count;
+
        return (0);
 }
 
@@ -250,11 +280,16 @@ _sem_trywait(sem_t *sem)
                return (-1);
        }
 
+       sem_log("sem_trywait %p %d\n", *sem, (*sem)->count);
        while ((val = (*sem)->count) > 0) {
-               if (atomic_cmpset_int(&(*sem)->count, val, val - 1))
+               cpu_ccfence();
+               if (atomic_cmpset_int(&(*sem)->count, val, val - 1)) {
+                       sem_log("sem_trywait %p %d (success)\n", *sem, val - 1);
                        return (0);
+               }
        }
        errno = EAGAIN;
+       sem_log("sem_trywait %p %d (failure)\n", *sem, val);
        return (-1);
 }
 
@@ -272,17 +307,28 @@ _sem_wait(sem_t *sem)
        curthread = tls_get_curthread();
        _pthread_testcancel();
 
-       for (;;) {
+       sem_log("sem_wait %p %d (begin)\n", *sem, (*sem)->count);
+
+       do {
+               cpu_ccfence();
                while ((val = (*sem)->count) > 0) {
-                       if (atomic_cmpset_acq_int(&(*sem)->count, val, val - 1))
+                       cpu_ccfence();
+                       if (atomic_cmpset_acq_int(&(*sem)->count, val, val - 1)) {
+                               sem_log("sem_wait %p %d (success)\n",
+                                       *sem, val - 1);
                                return (0);
+                       }
                }
                oldcancel = _thr_cancel_enter(curthread);
-               retval = _thr_umtx_wait(&(*sem)->count, 0, NULL, 0);
+               sem_log("sem_wait %p %d (wait)\n", *sem, val);
+               retval = _thr_umtx_wait_intr(&(*sem)->count, 0);
+               sem_log("sem_wait %p %d (wait return %d)\n",
+                       *sem, (*sem)->count, retval);
                _thr_cancel_leave(curthread, oldcancel);
                /* ignore retval */
-       }
+       } while (retval != EINTR);
 
+       sem_log("sem_wait %p %d (error %d)\n", *sem, retval);
        errno = retval;
 
        return (-1);
@@ -300,6 +346,7 @@ _sem_timedwait(sem_t * __restrict sem, const struct timespec * __restrict abstim
 
        curthread = tls_get_curthread();
        _pthread_testcancel();
+       sem_log("sem_timedwait %p %d (begin)\n", *sem, (*sem)->count);
 
        /*
         * The timeout argument is only supposed to
@@ -307,23 +354,32 @@ _sem_timedwait(sem_t * __restrict sem, const struct timespec * __restrict abstim
         */
        do {
                while ((val = (*sem)->count) > 0) {
-                       if (atomic_cmpset_acq_int(&(*sem)->count, val, val - 1))
+                       cpu_ccfence();
+                       if (atomic_cmpset_acq_int(&(*sem)->count, val, val - 1)) {
+                               sem_log("sem_wait %p %d (success)\n",
+                                       *sem, val - 1);
                                return (0);
+                       }
                }
                if (abstime == NULL ||
                    abstime->tv_nsec >= 1000000000 ||
                    abstime->tv_nsec < 0) {
+                       sem_log("sem_wait %p %d (bad abstime)\n", *sem, val);
                        errno = EINVAL;
                        return (-1);
                }
                clock_gettime(CLOCK_REALTIME, &ts);
                TIMESPEC_SUB(&ts2, abstime, &ts);
                oldcancel = _thr_cancel_enter(curthread);
+               sem_log("sem_wait %p %d (wait)\n", *sem, val);
                retval = _thr_umtx_wait(&(*sem)->count, 0, &ts2,
                                        CLOCK_REALTIME);
+               sem_log("sem_wait %p %d (wait return %d)\n",
+                       *sem, (*sem)->count, retval);
                _thr_cancel_leave(curthread, oldcancel);
        } while (retval != ETIMEDOUT && retval != EINTR);
 
+       sem_log("sem_wait %p %d (error %d)\n", *sem, retval);
        errno = retval;
 
        return (-1);
@@ -341,10 +397,10 @@ _sem_post(sem_t *sem)
         * sem_post() is required to be safe to call from within
         * signal handlers, these code should work as that.
         */
-       do {
-               val = (*sem)->count;
-       } while (!atomic_cmpset_acq_int(&(*sem)->count, val, val + 1));
-       _thr_umtx_wake(&(*sem)->count, val + 1);
+       val = atomic_fetchadd_int(&(*sem)->count, 1) + 1;
+       sem_log("sem_post %p %d\n", *sem, val);
+       _thr_umtx_wake(&(*sem)->count, 0);
+
        return (0);
 }
 
@@ -486,8 +542,6 @@ _sem_open(const char *name, int oflag, ...)
                return (SEM_FAILED);
        }
 
-       _pthread_once(&once, sem_module_init);
-
        _pthread_mutex_lock(&sem_lock);
 
        error = get_path(name, path, PATH_MAX, &prefix);
@@ -649,8 +703,6 @@ error:
 int
 _sem_close(sem_t *sem)
 {
-       _pthread_once(&once, sem_module_init);
-
        _pthread_mutex_lock(&sem_lock);
 
        if (sem_check_validity(sem)) {
index 06356d5..3a19ab5 100644 (file)
@@ -102,7 +102,7 @@ _thr_suspend_check(struct pthread *curthread)
                cycle = curthread->cycle;
 
                /* Wake the thread suspending us. */
-               _thr_umtx_wake(&curthread->cycle, INT_MAX);
+               _thr_umtx_wake(&curthread->cycle, 0);
 
                /*
                 * if we are from pthread_exit, we don't want to
index 22166bd..9376add 100644 (file)
@@ -47,19 +47,18 @@ __thr_umtx_lock(volatile umtx_t *mtx, int id, int timo)
 {
        int v, errval, ret = 0;
 
+       v = *mtx;
+       cpu_ccfence();
        id &= 0x3FFFFFFF;
-       /* contested */
+
        for (;;) {
-               v = *mtx;
-               cpu_ccfence();
                if (v == 0) {
                        if (atomic_cmpset_acq_int(mtx, 0, id)) {
                                break;
                        }
                        continue;
                }
-               if ((v & 0x40000000) ||
-                   atomic_cmpset_acq_int(mtx, v, v|0x40000000)) {
+               if (atomic_fcmpset_int(mtx, &v, v|0x40000000)) {
                        if (timo == 0) {
                                _umtx_sleep_err(mtx, v|0x40000000, timo);
                        } else if ((errval = _umtx_sleep_err(mtx, v|0x40000000, timo)) > 0) {
@@ -85,13 +84,14 @@ __thr_umtx_unlock(volatile umtx_t *mtx, int id)
 {
        int v;
 
+       v = *mtx;
+       cpu_ccfence();
        id &= 0x3FFFFFFF;
+
        for (;;) {
-               v = *mtx;
-               cpu_ccfence();
-               if (atomic_cmpset_acq_int(mtx, v, 0)) {
+               if (atomic_fcmpset_int(mtx, &v, 0)) {
                        if (v & 0x40000000)
-                               _umtx_wakeup_err(mtx, 1);
+                               _umtx_wakeup_err(mtx, 0);
                        THR_ASSERT((v & 0x3FFFFFFF) == id,
                                   "thr_umtx_unlock: wrong owner");
                        break;
@@ -144,6 +144,9 @@ __thr_umtx_timedlock(volatile umtx_t *mtx, int id,
        return (ret);
 }
 
+/*
+ * Regular umtx wait that cannot return EINTR
+ */
 int
 _thr_umtx_wait(volatile umtx_t *mtx, int exp, const struct timespec *timeout,
               int clockid)
@@ -220,6 +223,33 @@ _thr_umtx_wait(volatile umtx_t *mtx, int exp, const struct timespec *timeout,
        return (ret);
 }
 
+/*
+ * Simple version without a timeout which can also return EINTR
+ */
+int
+_thr_umtx_wait_intr(volatile umtx_t *mtx, int exp)
+{
+       int ret = 0;
+       int errval;
+
+       cpu_ccfence();
+       for (;;) {
+               if (*mtx != exp)
+                       return (0);
+               errval = _umtx_sleep_err(mtx, exp, 10000000);
+               if (errval == 0)
+                       break;
+               if (errval == EBUSY)
+                       break;
+               if (errval == EINTR) {
+                       ret = errval;
+                       break;
+               }
+               cpu_ccfence();
+       }
+       return (ret);
+}
+
 void
 _thr_umtx_wake(volatile umtx_t *mtx, int count)
 {
index 2c210d3..36ba769 100644 (file)
@@ -85,5 +85,6 @@ _thr_umtx_unlock(volatile umtx_t *mtx, int id)
 
 int _thr_umtx_wait(volatile umtx_t *mtx, umtx_t exp,
                   const struct timespec *timeout, int clockid);
+int _thr_umtx_wait_intr(volatile umtx_t *mtx, umtx_t exp);
 void _thr_umtx_wake(volatile umtx_t *mtx, int count);
 #endif
index 25bca85..124b45a 100644 (file)
 #include <stdlib.h>
 #include <time.h>
 
+#include <stdio.h>
+#include <sys/file.h>
+
 #include "debug.h"
 #include "rtld.h"
 #include "rtld_machdep.h"
 
+extern pid_t __sys_getpid(void);
+
 #define WAFLAG         0x1     /* A writer holds the lock */
 #define RC_INCR                0x2     /* Adjusts count of readers desiring lock */
 
-typedef struct Struct_Lock {
+struct Struct_Lock {
        volatile u_int lock;
-       void *base;
-} Lock;
-
-static sigset_t fullsigmask, oldsigmask;
-static int thread_flag;
-
-static void *
-def_lock_create(void)
-{
-    void *base;
-    char *p;
-    uintptr_t r;
-    Lock *l;
-
-    /*
-     * Arrange for the lock to occupy its own cache line.  First, we
-     * optimistically allocate just a cache line, hoping that malloc
-     * will give us a well-aligned block of memory.  If that doesn't
-     * work, we allocate a larger block and take a well-aligned cache
-     * line from it.
-     */
-    base = xmalloc(CACHE_LINE_SIZE);
-    p = (char *)base;
-    if ((uintptr_t)p % CACHE_LINE_SIZE != 0) {
-       free(base);
-       base = xmalloc(2 * CACHE_LINE_SIZE);
-       p = (char *)base;
-       if ((r = (uintptr_t)p % CACHE_LINE_SIZE) != 0)
-           p += CACHE_LINE_SIZE - r;
-    }
-    l = (Lock *)p;
-    l->base = base;
-    l->lock = 0;
-    return l;
-}
-
-static void
-def_lock_destroy(void *lock)
-{
-    Lock *l = (Lock *)lock;
-
-    free(l->base);
-}
-
-static void
-def_rlock_acquire(void *lock)
-{
-    Lock *l = (Lock *)lock;
-
-    atomic_add_acq_int(&l->lock, RC_INCR);
-    while (l->lock & WAFLAG)
-           ;   /* Spin */
-}
-
-static void
-def_wlock_acquire(void *lock)
-{
-    Lock *l = (Lock *)lock;
-    sigset_t tmp_oldsigmask;
-
-    for ( ; ; ) {
-       sigprocmask(SIG_BLOCK, &fullsigmask, &tmp_oldsigmask);
-       if (atomic_cmpset_acq_int(&l->lock, 0, WAFLAG))
-           break;
-       sigprocmask(SIG_SETMASK, &tmp_oldsigmask, NULL);
-    }
-    oldsigmask = tmp_oldsigmask;
-}
+       int tid;                /* owner (exclusive) */
+       int count;              /* recursion (exclusive) */
+       sigset_t savesigmask;   /* first exclusive owner sets mask */
+} __cachealign;
 
-static void
-def_lock_release(void *lock)
-{
-    Lock *l = (Lock *)lock;
+#define cpu_ccfence()  __asm __volatile("" : : : "memory")
 
-    if ((l->lock & WAFLAG) == 0)
-       atomic_add_rel_int(&l->lock, -RC_INCR);
-    else {
-       atomic_add_rel_int(&l->lock, -WAFLAG);
-       sigprocmask(SIG_SETMASK, &oldsigmask, NULL);
-    }
-}
+static sigset_t fullsigmask;
 
-static int
-def_thread_set_flag(int mask)
-{
-       int old_val = thread_flag;
-       thread_flag |= mask;
-       return (old_val);
-}
+struct Struct_Lock phdr_lock;
+struct Struct_Lock bind_lock;
+struct Struct_Lock libc_lock;
 
-static int
-def_thread_clr_flag(int mask)
-{
-       int old_val = thread_flag;
-       thread_flag &= ~mask;
-       return (old_val);
-}
+rtld_lock_t    rtld_phdr_lock = &phdr_lock;
+rtld_lock_t    rtld_bind_lock = &bind_lock;
+rtld_lock_t    rtld_libc_lock = &libc_lock;
 
-/*
- * Public interface exposed to the rest of the dynamic linker.
- */
-static struct RtldLockInfo lockinfo;
-static struct RtldLockInfo deflockinfo;
-
-static __inline int
-thread_mask_set(int mask)
-{
-       return lockinfo.thread_set_flag(mask);
-}
-
-static __inline void
-thread_mask_clear(int mask)
-{
-       lockinfo.thread_clr_flag(mask);
+void
+rlock_acquire(rtld_lock_t lock, RtldLockState *state)
+{
+       int v;
+       int tid = 0;
+
+       v = lock->lock;
+       cpu_ccfence();
+       for (;;) {
+               if ((v & WAFLAG) == 0) {
+                       if (atomic_fcmpset_int(&lock->lock, &v, v + RC_INCR)) {
+                               state->lockstate = RTLD_LOCK_RLOCKED;
+                               break;
+                       }
+               } else {
+                       if (tid == 0)
+                               tid = lwp_gettid();
+                       if (lock->tid == tid) {
+                               ++lock->count;
+                               state->lockstate = RTLD_LOCK_WLOCKED;
+                               break;
+                       }
+                       umtx_sleep(&lock->lock, v, 0);
+                       v = lock->lock;
+                       cpu_ccfence();
+               }
+       }
 }
 
-#define        RTLD_LOCK_CNT   3
-struct rtld_lock {
-       void    *handle;
-       int      mask;
-       RtldLockState forkstate;
-} rtld_locks[RTLD_LOCK_CNT];
-
-rtld_lock_t    rtld_bind_lock = &rtld_locks[0];
-rtld_lock_t    rtld_libc_lock = &rtld_locks[1];
-rtld_lock_t    rtld_phdr_lock = &rtld_locks[2];
-
 void
-rlock_acquire(rtld_lock_t lock, RtldLockState *lockstate)
+wlock_acquire(rtld_lock_t lock, RtldLockState *state)
 {
+       sigset_t tmp_oldsigmask;
+       int tid = lwp_gettid();
 
-       if (lockstate == NULL)
+       if (lock->tid == tid) {
+               ++lock->count;
+               state->lockstate = RTLD_LOCK_WLOCKED;
                return;
+       }
 
-       if (thread_mask_set(lock->mask) & lock->mask) {
-               dbg("rlock_acquire: recursed");
-               lockstate->lockstate = RTLD_LOCK_UNLOCKED;
-               return;
+       sigprocmask(SIG_BLOCK, &fullsigmask, &tmp_oldsigmask);
+       for (;;) {
+               if (atomic_cmpset_acq_int(&lock->lock, 0, WAFLAG))
+                       break;
+               umtx_sleep(&lock->lock, 0, 0);
        }
-       lockinfo.rlock_acquire(lock->handle);
-       lockstate->lockstate = RTLD_LOCK_RLOCKED;
+       lock->tid = tid;
+       lock->count = 1;
+       lock->savesigmask = tmp_oldsigmask;
+       state->lockstate = RTLD_LOCK_WLOCKED;
 }
 
 void
-wlock_acquire(rtld_lock_t lock, RtldLockState *lockstate)
+lock_release(rtld_lock_t lock, RtldLockState *state)
 {
-       if (lockstate == NULL)
-               return;
+       sigset_t tmp_oldsigmask;
+       int v;
 
-       if (thread_mask_set(lock->mask) & lock->mask) {
-               dbg("wlock_acquire: recursed");
-               lockstate->lockstate = RTLD_LOCK_UNLOCKED;
+       if (state->lockstate == RTLD_LOCK_UNLOCKED)
                return;
+       if ((lock->lock & WAFLAG) == 0) {
+               v = atomic_fetchadd_int(&lock->lock, -RC_INCR) - RC_INCR;
+               if (v == 0)
+                       umtx_wakeup(&lock->lock, 0);
+       } else if (--lock->count == 0) {
+               tmp_oldsigmask = lock->savesigmask;
+               lock->tid = 0;
+               v = atomic_fetchadd_int(&lock->lock, -WAFLAG) - WAFLAG;
+               if (v == 0)
+                       umtx_wakeup(&lock->lock, 0);
+               sigprocmask(SIG_SETMASK, &tmp_oldsigmask, NULL);
        }
-       lockinfo.wlock_acquire(lock->handle);
-       lockstate->lockstate = RTLD_LOCK_WLOCKED;
+       state->lockstate = RTLD_LOCK_UNLOCKED;
 }
 
+static
 void
-lock_release(rtld_lock_t lock, RtldLockState *lockstate)
+lock_reset(rtld_lock_t lock)
 {
-       if (lockstate == NULL)
-               return;
-
-       switch (lockstate->lockstate) {
-       case RTLD_LOCK_UNLOCKED:
-               break;
-       case RTLD_LOCK_RLOCKED:
-       case RTLD_LOCK_WLOCKED:
-               lockinfo.lock_release(lock->handle);
-               thread_mask_clear(lock->mask);
-               break;
-       default:
-               assert(0);
-       }
+       memset(lock, 0, sizeof(*lock));
 }
 
 void
-lock_upgrade(rtld_lock_t lock, RtldLockState *lockstate)
+lock_upgrade(rtld_lock_t lock, RtldLockState *state)
 {
-       if (lockstate == NULL)
+       if (state == NULL)
                return;
-
-       lock_release(lock, lockstate);
-       wlock_acquire(lock, lockstate);
+       if (state->lockstate == RTLD_LOCK_RLOCKED) {
+               lock_release(lock, state);
+               wlock_acquire(lock, state);
+       }
 }
 
 void
-lock_restart_for_upgrade(RtldLockState *lockstate)
+lock_restart_for_upgrade(RtldLockState *state)
 {
-       if (lockstate == NULL)
+       if (state == NULL)
                return;
-
-       switch (lockstate->lockstate) {
+       switch (state->lockstate) {
        case RTLD_LOCK_UNLOCKED:
        case RTLD_LOCK_WLOCKED:
                break;
        case RTLD_LOCK_RLOCKED:
-               siglongjmp(lockstate->env, 1);
+               siglongjmp(state->env, 1);
                break;
        default:
                assert(0);
@@ -264,130 +192,60 @@ lock_restart_for_upgrade(RtldLockState *lockstate)
 void
 lockdflt_init(void)
 {
-    int i;
-
-    deflockinfo.rtli_version  = RTLI_VERSION;
-    deflockinfo.lock_create   = def_lock_create;
-    deflockinfo.lock_destroy  = def_lock_destroy;
-    deflockinfo.rlock_acquire = def_rlock_acquire;
-    deflockinfo.wlock_acquire = def_wlock_acquire;
-    deflockinfo.lock_release  = def_lock_release;
-    deflockinfo.thread_set_flag = def_thread_set_flag;
-    deflockinfo.thread_clr_flag = def_thread_clr_flag;
-    deflockinfo.at_fork = NULL;
+       /*
+        * Construct a mask to block all signals except traps which might
+        * conceivably be generated within the dynamic linker itself.
+        */
+       sigfillset(&fullsigmask);
+       sigdelset(&fullsigmask, SIGILL);
+       sigdelset(&fullsigmask, SIGTRAP);
+       sigdelset(&fullsigmask, SIGABRT);
+       sigdelset(&fullsigmask, SIGEMT);
+       sigdelset(&fullsigmask, SIGFPE);
+       sigdelset(&fullsigmask, SIGBUS);
+       sigdelset(&fullsigmask, SIGSEGV);
+       sigdelset(&fullsigmask, SIGSYS);
 
-    for (i = 0; i < RTLD_LOCK_CNT; i++) {
-           rtld_locks[i].mask   = (1 << i);
-           rtld_locks[i].handle = NULL;
-    }
-
-    memcpy(&lockinfo, &deflockinfo, sizeof(lockinfo));
-    _rtld_thread_init(NULL);
-    /*
-     * Construct a mask to block all signals except traps which might
-     * conceivably be generated within the dynamic linker itself.
-     */
-    sigfillset(&fullsigmask);
-    sigdelset(&fullsigmask, SIGILL);
-    sigdelset(&fullsigmask, SIGTRAP);
-    sigdelset(&fullsigmask, SIGABRT);
-    sigdelset(&fullsigmask, SIGEMT);
-    sigdelset(&fullsigmask, SIGFPE);
-    sigdelset(&fullsigmask, SIGBUS);
-    sigdelset(&fullsigmask, SIGSEGV);
-    sigdelset(&fullsigmask, SIGSYS);
+       _rtld_thread_init(NULL);
 }
 
 /*
- * Callback function to allow threads implementation to
- * register their own locking primitives if the default
- * one is not suitable.
- * The current context should be the only context
- * executing at the invocation time.
+ * (also called by pthreads)
  */
 void
-_rtld_thread_init(struct RtldLockInfo *pli)
+_rtld_thread_init(void *dummy __unused)
 {
-       int flags, i;
-       void *locks[RTLD_LOCK_CNT];
-
-       /* disable all locking while this function is running */
-       flags = thread_mask_set(~0);
-
-       if (pli == NULL)
-               pli = &deflockinfo;
-
-
-       for (i = 0; i < RTLD_LOCK_CNT; i++) {
-               if ((locks[i] = pli->lock_create()) == NULL)
-                       break;
-       }
-
-       if (i < RTLD_LOCK_CNT) {
-               while (--i >= 0) {
-                       pli->lock_destroy(locks[i]);
-               }
-               abort();
-       }
-
-       for (i = 0; i < RTLD_LOCK_CNT; i++) {
-               if (rtld_locks[i].handle == NULL)
-                       continue;
-               if (flags & rtld_locks[i].mask)
-                       lockinfo.lock_release(rtld_locks[i].handle);
-               lockinfo.lock_destroy(rtld_locks[i].handle);
-       }
-
-       for (i = 0; i < RTLD_LOCK_CNT; i++) {
-               rtld_locks[i].handle = locks[i];
-               if (flags & rtld_locks[i].mask)
-                       pli->wlock_acquire(rtld_locks[i].handle);
-       }
-
-       lockinfo.lock_create = pli->lock_create;
-       lockinfo.lock_destroy = pli->lock_destroy;
-       lockinfo.rlock_acquire = pli->rlock_acquire;
-       lockinfo.wlock_acquire = pli->wlock_acquire;
-       lockinfo.lock_release  = pli->lock_release;
-       lockinfo.thread_set_flag = pli->thread_set_flag;
-       lockinfo.thread_clr_flag = pli->thread_clr_flag;
-       lockinfo.at_fork = pli->at_fork;
-
-       /* restore thread locking state, this time with new locks */
-       thread_mask_clear(~0);
-       thread_mask_set(flags);
-       dbg("_rtld_thread_init: done");
+       lock_reset(rtld_phdr_lock);
+       lock_reset(rtld_bind_lock);
+       lock_reset(rtld_libc_lock);
 }
 
+static RtldLockState fork_states[3];
+
 void
 _rtld_thread_prefork(void)
 {
-       int i;
-
-       for (i = 0; i < RTLD_LOCK_CNT; i++) {
-               if (rtld_locks[i].handle == NULL)
-                       continue;
-               wlock_acquire(&rtld_locks[i], &rtld_locks[i].forkstate);
-       }
+       wlock_acquire(rtld_phdr_lock, &fork_states[0]);
+       wlock_acquire(rtld_bind_lock, &fork_states[1]);
+       wlock_acquire(rtld_libc_lock, &fork_states[2]);
 }
 
 void
 _rtld_thread_postfork(void)
 {
-       int i;
-
-       for (i = 0; i < RTLD_LOCK_CNT; i++) {
-               if (rtld_locks[i].handle == NULL)
-                       continue;
-               lock_release(&rtld_locks[i], &rtld_locks[i].forkstate);
-       }
+       lock_release(rtld_libc_lock, &fork_states[2]);
+       lock_release(rtld_bind_lock, &fork_states[1]);
+       lock_release(rtld_phdr_lock, &fork_states[0]);
 }
 
-/*
- * pthreads already called _rtld_thread_init(NULL) in the child, so there
- * is nothing for us to do here.
- */
 void
 _rtld_thread_childfork(void)
 {
+       sigset_t tmp_oldsigmask;
+
+       lock_reset(rtld_phdr_lock);
+       lock_reset(rtld_bind_lock);
+       tmp_oldsigmask = rtld_libc_lock->savesigmask;
+       lock_reset(rtld_libc_lock);
+       sigprocmask(SIG_SETMASK, &tmp_oldsigmask, NULL);
 }
index 3096f1f..e9993e7 100644 (file)
@@ -28,7 +28,8 @@
 #ifndef _RTLD_LOCK_H_
 #define        _RTLD_LOCK_H_
 
-#define        RTLI_VERSION    0x01
+#define        RTLI_VERSION            0x02
+#define        RTLI_VERSION_FORK       0x02
 
 struct RtldLockInfo
 {
@@ -41,22 +42,23 @@ struct RtldLockInfo
        int   (*thread_set_flag)(int);
        int   (*thread_clr_flag)(int);
        void  (*at_fork)(void *handle);
+       void  (*fork_acquire)(void *);
+       void  (*fork_release)(void *);
 };
 
-extern void _rtld_thread_init(struct RtldLockInfo *);
+extern void _rtld_thread_init(void *);
 extern void _rtld_thread_prefork(void);
 extern void _rtld_thread_postfork(void);
 extern void _rtld_thread_childfork(void);
 
-
 #ifdef IN_RTLD
 
-struct rtld_lock;
-typedef struct rtld_lock *rtld_lock_t;
+struct Struct_Lock;
+typedef struct Struct_Lock *rtld_lock_t;
 
-extern rtld_lock_t     rtld_bind_lock;
-extern rtld_lock_t     rtld_libc_lock;
-extern rtld_lock_t     rtld_phdr_lock;
+rtld_lock_t rtld_bind_lock;
+rtld_lock_t rtld_libc_lock;
+rtld_lock_t rtld_phdr_lock;
 
 #define        RTLD_LOCK_UNLOCKED      0
 #define        RTLD_LOCK_RLOCKED       1
index 7c87340..2fefa88 100644 (file)
@@ -32,8 +32,6 @@
 #include <sys/types.h>
 #include <machine/atomic.h>
 
-#define        CACHE_LINE_SIZE         64
-
 struct Struct_Obj_Entry;
 
 /* Return the address of the .dynamic section in the dynamic linker. */