libc - Implement sigblockall() and sigunblockall()
authorMatthew Dillon <dillon@apollo.backplane.com>
Tue, 12 Nov 2019 01:35:42 +0000 (17:35 -0800)
committerMatthew Dillon <dillon@apollo.backplane.com>
Tue, 12 Nov 2019 01:46:17 +0000 (17:46 -0800)
* Signal safety is becoming a defacto requirement for most of libc and
  pthreads.  In particular, the memory allocator.  Given the chances of
  teaching tens of thousands of programmers about signal safety, and
  just making it work in libc and pthreads, only one of these two
  possibilities is actually realizable.

  In particular, high-level languages have become so complex, and some
  applications (chrome, firefox, etc) have become so complex, that the
  code is regularly tripping over signal safety issues.

  However, implementing signal safety with current mechanisms is extremely
  expensive due to the need for multiple system calls.  To whit,
  DragonFlyBSD now has a mechanism that does not require system calls
  in the critical path.

* Implement sigblockall() and sigunblockall().  These functions
  leverage the new /dev/lpmap per-thread shared page mechanism
  to provide a way to temporary block the dispatch of all maskable
  signals without having to make any system calls.

  These are extremely fast routines.

  - Reentrant / Recursable

  - Temporarily blocks any dispatch of a maskable asynchronous signal
    to the calling thread.  Other threads are not affected... this is
    a per-thread mechanism.

  - The last sigunblockall() will immediately dispatch any blocked
    signals.

  - The normal signal mask is not affected by these routines.

  - Does not block signals caused by synchronous traps.

  - The current recursion count is retained on [v]fork() to ease
    coding and to also allow signals to be temporarily blocked across a
    fork until the child process is ready to deal with them, if desired.

* Implement signal safety for most of pthreads.  All temporary internal
  mutexes are now wrapped with sigblockall() and sigunblockall().

* Implement signal safety for the malloc subsystem.  All functions
  are wrawpped with sigblockall() and sigunblockall().

  These implementations make lang/mono and lang/rust far more reliable
  than they were before.  Where 9 out of 10 builds used to fail, now
  they succeed.

18 files changed:
lib/libc/gen/_thread_init.c
lib/libc/include/libc_private.h
lib/libc/stdlib/Symbol.map
lib/libc/stdlib/nmalloc.c
lib/libc/upmap/Makefile.inc
lib/libc/upmap/Symbol.map
lib/libc/upmap/ukp_blocksigs.c [new file with mode: 0644]
lib/libc/upmap/upmap.c
lib/libc/upmap/upmap.h
lib/libstand/Makefile
lib/libthread_xu/thread/thr_create.c
lib/libthread_xu/thread/thr_init.c
lib/libthread_xu/thread/thr_mutex.c
lib/libthread_xu/thread/thr_private.h
lib/libthread_xu/thread/thr_pspinlock.c
lib/libthread_xu/thread/thr_spinlock.c
lib/libthread_xu/thread/thr_umtx.h
libexec/rtld-elf/rtld.c

index 95ebbd2..513bdc8 100644 (file)
 
 #include <sys/cdefs.h>
 #include <sys/types.h>
+#include "libc_private.h"
 
 void   _thread_init_stub(void);
+void   _nmalloc_thr_init(void);
+void   _upmap_thr_init(void);
 
 int    _thread_autoinit_dummy_decl_stub = 0;
 
+/*
+ * This stub is overridden when libpthreads is linked in.  However,
+ * we can apply the constructor to the weak reference.  Instead the
+ * constructor is applied to the stub and we do a run-time check to
+ * see if the stub has been overridden by pthreads.
+ */
+void _thread_init_stub(void) __attribute__ ((constructor));
+__weak_reference(_thread_init_stub, _thread_init);
+
 void
 _thread_init_stub(void)
 {
-       /* This is just a stub; there is nothing to do. */
+       /*
+        * Only run libc related pthreads initialization from here
+        * if pthreads did not override the weak reference.  Otherwise
+        * pthreads will do it after setting up a real thread context.
+        */
+       if (_thread_init == _thread_init_stub) {
+               _libc_thr_init();
+       }
+}
+
+/*
+ * Pre-initialization prior to thread startup to avoid certain
+ * chicken-and-egg issues with low-level allocations.
+ */
+void
+_libc_thr_init(void)
+{
+       _nmalloc_thr_init();
+       _upmap_thr_init();
 }
 
-__weak_reference(_thread_init_stub, _thread_init);
 __weak_reference(_thread_autoinit_dummy_decl_stub, _thread_autoinit_dummy_decl);
index c3459e2..e5de2b2 100644 (file)
 #include <machine/types.h>
 #include <sys/_pthreadtypes.h>
 
+/*
+ * With this attribute set, do not require a function call for accessing
+ * this variable when the code is compiled -fPIC.  Use in combination
+ * with __thread.
+ *
+ * Must be empty for libc_rtld.
+ */
+#ifdef __LIBC_RTLD
+#define TLS_ATTRIBUTE
+#else
+#define TLS_ATTRIBUTE __attribute__ ((tls_model ("initial-exec")))
+#endif
+
 /*
  * This global flag is non-zero when a process has created one
  * or more threads. It is used to avoid calling locking functions
@@ -100,7 +113,9 @@ extern void (*__cleanup)(void);
 
 /* execve() with PATH processing to implement posix_spawnp() */
 int _execvpe(const char *, char * const *, char * const *);
+void _libc_thr_init(void);
 void _nmalloc_thr_init(void);
+void _upmap_thr_init(void);
 void _nmalloc_thr_prepfork(void);
 void _nmalloc_thr_parentfork(void);
 void _nmalloc_thr_childfork(void);
index 6a12311..151a6d7 100644 (file)
@@ -2,7 +2,7 @@ DF404.0 {
     __cxa_atexit;
     __cxa_finalize;
     _Exit;
-    _nmalloc_thr_init;
+    _libc_thr_init;
     _nmalloc_thr_prepfork;
     _nmalloc_thr_parentfork;
     _nmalloc_thr_childfork;
index 4ef9ac3..7218e51 100644 (file)
@@ -246,10 +246,26 @@ typedef struct slglobaldata {
 
 #define arysize(ary)   (sizeof(ary)/sizeof((ary)[0]))
 
-#define MASSERT(exp)   do { if (__predict_false(!(exp)))       \
-                               _mpanic("assertion: %s in %s",  \
-                               #exp, __func__);                \
-                           } while (0)
+/*
+ * The assertion macros try to pretty-print assertion failures
+ * which can be caused by corruption.  If a lock is held, we
+ * provide a macro that attempts to release it before asserting
+ * in order to prevent (e.g.) a reentrant SIGABRT calling malloc
+ * and deadlocking, resulting in the program freezing up.
+ */
+#define MASSERT(exp)                           \
+       do { if (__predict_false(!(exp)))       \
+           _mpanic("assertion: %s in %s",      \
+                   #exp, __func__);            \
+       } while (0)
+
+#define MASSERT_WTHUNLK(exp, unlk)             \
+       do { if (__predict_false(!(exp))) {     \
+           unlk;                               \
+           _mpanic("assertion: %s in %s",      \
+                   #exp, __func__);            \
+         }                                     \
+       } while (0)
 
 /*
  * Magazines
@@ -316,18 +332,6 @@ typedef struct thr_mags {
        int             init;
 } thr_mags;
 
-/*
- * With this attribute set, do not require a function call for accessing
- * this variable when the code is compiled -fPIC.
- *
- * Must be empty for libc_rtld (similar to __thread).
- */
-#ifdef __LIBC_RTLD
-#define TLS_ATTRIBUTE
-#else
-#define TLS_ATTRIBUTE __attribute__ ((tls_model ("initial-exec")))
-#endif
-
 static __thread thr_mags thread_mags TLS_ATTRIBUTE;
 static pthread_key_t thread_mags_key;
 static pthread_once_t thread_mags_once = PTHREAD_ONCE_INIT;
@@ -421,7 +425,6 @@ malloc_init(void)
 void
 _nmalloc_thr_init(void)
 {
-       static int init_once;
        thr_mags *tp;
 
        /*
@@ -431,10 +434,7 @@ _nmalloc_thr_init(void)
        tp = &thread_mags;
        tp->init = -1;
 
-       if (init_once == 0) {
-               init_once = 1;
-               _pthread_once(&thread_mags_once, mtmagazine_init);
-       }
+       _pthread_once(&thread_mags_once, mtmagazine_init);
        _pthread_setspecific(thread_mags_key, tp);
        tp->init = 1;
 }
@@ -466,6 +466,28 @@ _nmalloc_thr_childfork(void)
        }
 }
 
+/*
+ * Handle signal reentrancy safely whether we are threaded or not.
+ * This improves the stability for mono and will probably improve
+ * stability for other high-level languages which are becoming increasingly
+ * sophisticated.
+ *
+ * The sigblockall()/sigunblockall() implementation uses a counter on
+ * a per-thread shared user/kernel page, avoids system calls, and is thus
+ *  very fast.
+ */
+static __inline void
+nmalloc_sigblockall(void)
+{
+       sigblockall();
+}
+
+static __inline void
+nmalloc_sigunblockall(void)
+{
+       sigunblockall();
+}
+
 /*
  * Thread locks.
  */
@@ -474,6 +496,8 @@ slgd_lock(slglobaldata_t slgd)
 {
        if (__isthreaded)
                _SPINLOCK(&slgd->Spinlock);
+       else
+               sigblockall();
 }
 
 static __inline void
@@ -481,6 +505,8 @@ slgd_unlock(slglobaldata_t slgd)
 {
        if (__isthreaded)
                _SPINUNLOCK(&slgd->Spinlock);
+       else
+               sigunblockall();
 }
 
 static __inline void
@@ -488,6 +514,8 @@ depot_lock(magazine_depot *dp __unused)
 {
        if (__isthreaded)
                _SPINLOCK(&depot_spinlock);
+       else
+               sigblockall();
 #if 0
        if (__isthreaded)
                _SPINLOCK(&dp->lock);
@@ -499,6 +527,8 @@ depot_unlock(magazine_depot *dp __unused)
 {
        if (__isthreaded)
                _SPINUNLOCK(&depot_spinlock);
+       else
+               sigunblockall();
 #if 0
        if (__isthreaded)
                _SPINUNLOCK(&dp->lock);
@@ -510,6 +540,8 @@ zone_magazine_lock(void)
 {
        if (__isthreaded)
                _SPINLOCK(&zone_mag_lock);
+       else
+               sigblockall();
 }
 
 static __inline void
@@ -517,6 +549,8 @@ zone_magazine_unlock(void)
 {
        if (__isthreaded)
                _SPINUNLOCK(&zone_mag_lock);
+       else
+               sigunblockall();
 }
 
 static __inline void
@@ -666,8 +700,10 @@ handle_excess_big(void)
                        _SPINLOCK(&bigspin_array[i & BIGXMASK]);
                for (big = *bigp; big; big = big->next) {
                        if (big->active < big->bytes) {
-                               MASSERT((big->active & PAGE_MASK) == 0);
-                               MASSERT((big->bytes & PAGE_MASK) == 0);
+                               MASSERT_WTHUNLK((big->active & PAGE_MASK) == 0,
+                                   _SPINUNLOCK(&bigspin_array[i & BIGXMASK]));
+                               MASSERT_WTHUNLK((big->bytes & PAGE_MASK) == 0,
+                                   _SPINUNLOCK(&bigspin_array[i & BIGXMASK]));
                                munmap((char *)big->base + big->active,
                                       big->bytes - big->active);
                                atomic_add_long(&excess_alloc,
@@ -767,11 +803,14 @@ __malloc(size_t size)
 {
        void *ptr;
 
+       nmalloc_sigblockall();
        ptr = _slaballoc(size, 0);
        if (ptr == NULL)
                errno = ENOMEM;
        else
                UTRACE(0, size, ptr);
+       nmalloc_sigunblockall();
+
        return(ptr);
 }
 
@@ -791,11 +830,14 @@ __calloc(size_t number, size_t size)
                return(NULL);
        }
 
+       nmalloc_sigblockall();
        ptr = _slaballoc(number * size, SAFLAG_ZERO);
        if (ptr == NULL)
                errno = ENOMEM;
        else
                UTRACE(0, number * size, ptr);
+       nmalloc_sigunblockall();
+
        return(ptr);
 }
 
@@ -810,11 +852,15 @@ void *
 __realloc(void *ptr, size_t size)
 {
        void *ret;
+
+       nmalloc_sigblockall();
        ret = _slabrealloc(ptr, size);
        if (ret == NULL)
                errno = ENOMEM;
        else
                UTRACE(ptr, size, ret);
+       nmalloc_sigunblockall();
+
        return(ret);
 }
 
@@ -829,10 +875,12 @@ __aligned_alloc(size_t alignment, size_t size)
        void *ptr;
        int rc;
 
+       nmalloc_sigblockall();
        ptr = NULL;
        rc = _slabmemalign(&ptr, alignment, size);
        if (rc)
                errno = rc;
+       nmalloc_sigunblockall();
 
        return (ptr);
 }
@@ -856,7 +904,9 @@ __posix_memalign(void **memptr, size_t alignment, size_t size)
                return(EINVAL);
        }
 
+       nmalloc_sigblockall();
        rc = _slabmemalign(memptr, alignment, size);
+       nmalloc_sigunblockall();
 
        return (rc);
 }
@@ -1006,7 +1056,9 @@ void
 __free(void *ptr)
 {
        UTRACE(ptr, 0, 0);
+       nmalloc_sigblockall();
        _slabfree(ptr, 0, NULL);
+       nmalloc_sigunblockall();
 }
 
 /*
@@ -1209,7 +1261,7 @@ _slaballoc(size_t size, int flags)
         *
         * Remove us from the ZoneAry[] when we become empty
         */
-       MASSERT(z->z_NFree > 0);
+       MASSERT_WTHUNLK(z->z_NFree > 0, slgd_unlock(slgd));
 
        if (--z->z_NFree == 0) {
                slgd->ZoneAry[zi] = z->z_Next;
@@ -1223,7 +1275,10 @@ _slaballoc(size_t size, int flags)
         */
        while (z->z_FirstFreePg < ZonePageCount) {
                if ((chunk = z->z_PageAry[z->z_FirstFreePg]) != NULL) {
-                       MASSERT((uintptr_t)chunk & ZoneMask);
+                       if (((uintptr_t)chunk & ZoneMask) == 0) {
+                               slgd_unlock(slgd);
+                               _mpanic("assertion: corrupt malloc zone");
+                       }
                        z->z_PageAry[z->z_FirstFreePg] = chunk->c_Next;
                        goto done;
                }
@@ -1352,7 +1407,9 @@ _slabrealloc(void *ptr, size_t size)
 
                                                return(ptr);
                                        }
-                                       MASSERT((void *)addr == MAP_FAILED);
+                                       MASSERT_WTHUNLK(
+                                               (void *)addr == MAP_FAILED,
+                                               bigalloc_unlock(ptr));
                                }
 
                                /*
@@ -1614,6 +1671,7 @@ mtmagazine_alloc(int zi)
        /*
         * Primary per-thread allocation loop
         */
+       nmalloc_sigblockall();
        for (;;) {
                /*
                 * If the loaded magazine has rounds, allocate and return
@@ -1652,13 +1710,14 @@ mtmagazine_alloc(int zi)
                tp->mags[zi].loaded = mp;
                if (mp) {
                        SLIST_REMOVE_HEAD(&d->full, nextmagazine);
-                       MASSERT(MAGAZINE_NOTEMPTY(mp));
                        depot_unlock(d);
+                       MASSERT(MAGAZINE_NOTEMPTY(mp));
                        continue;
                }
                depot_unlock(d);
                break;
        }
+       nmalloc_sigunblockall();
 
        return (obj);
 }
@@ -1682,6 +1741,7 @@ mtmagazine_free(int zi, void *ptr)
        /*
         * Primary per-thread freeing loop
         */
+       nmalloc_sigblockall();
        for (;;) {
                /*
                 * Make sure a new magazine is available in case we have
@@ -1737,6 +1797,7 @@ mtmagazine_free(int zi, void *ptr)
                if (mp) {
                        tp->mags[zi].loaded = mp;
                        SLIST_REMOVE_HEAD(&d->empty, nextmagazine);
+                       depot_unlock(d);
                        MASSERT(MAGAZINE_NOTFULL(mp));
                } else {
                        mp = tp->newmag;
@@ -1745,9 +1806,10 @@ mtmagazine_free(int zi, void *ptr)
                        mp->rounds = 0;
                        mp->flags = 0;
                        tp->mags[zi].loaded = mp;
+                       depot_unlock(d);
                }
-               depot_unlock(d);
        }
+       nmalloc_sigunblockall();
 
        return rc;
 }
@@ -1852,7 +1914,7 @@ zone_alloc(int flags)
                        for (i = 1; i < burst; i++) {
                                j = magazine_free(&zone_magazine,
                                                  (char *) z + (ZoneSize * i));
-                               MASSERT(j == 0);
+                               MASSERT_WTHUNLK(j == 0, zone_magazine_unlock());
                        }
                }
                zone_magazine_unlock();
@@ -1895,7 +1957,8 @@ zone_free(void *z)
                j = zone_magazine.rounds - zone_magazine.low_factor;
                for (i = 0; i < j; i++) {
                        excess[i] = magazine_alloc(&zone_magazine, NULL);
-                       MASSERT(excess[i] !=  NULL);
+                       MASSERT_WTHUNLK(excess[i] !=  NULL,
+                                       zone_magazine_unlock());
                }
 
                zone_magazine_unlock();
@@ -1922,39 +1985,52 @@ zone_free(void *z)
 static void *
 _vmem_alloc(size_t size, size_t align, int flags)
 {
+       static char *addr_hint;
+       static int reset_hint = 16;
        char *addr;
        char *save;
-       size_t excess;
+
+       if (--reset_hint <= 0) {
+               addr_hint = NULL;
+               reset_hint = 16;
+       }
 
        /*
         * Map anonymous private memory.
         */
-       addr = mmap(NULL, size, PROT_READ|PROT_WRITE,
+       save = mmap(addr_hint, size, PROT_READ|PROT_WRITE,
                    MAP_PRIVATE|MAP_ANON, -1, 0);
-       if (addr == MAP_FAILED)
-               return(NULL);
+       if (save == MAP_FAILED)
+               goto worst_case;
+       if (((uintptr_t)save & (align - 1)) == 0)
+               return((void *)save);
 
-       /*
-        * Check alignment.  The misaligned offset is also the excess
-        * amount.  If misaligned unmap the excess so we have a chance of
-        * mapping at the next alignment point and recursively try again.
-        *
-        * BBBBBBBBBBB BBBBBBBBBBB BBBBBBBBBBB  block alignment
-        *   aaaaaaaaa aaaaaaaaaaa aa           mis-aligned allocation
-        *   xxxxxxxxx                          final excess calculation
-        *   ^ returned address
-        */
-       excess = (uintptr_t)addr & (align - 1);
+       addr_hint = (char *)(((size_t)save + (align - 1)) & ~(align - 1));
+       munmap(save, size);
 
-       if (excess) {
-               excess = align - excess;
-               save = addr;
+       save = mmap(addr_hint, size, PROT_READ|PROT_WRITE,
+                   MAP_PRIVATE|MAP_ANON, -1, 0);
+       if (save == MAP_FAILED)
+               goto worst_case;
+       if (((size_t)save & (align - 1)) == 0)
+               return((void *)save);
+       munmap(save, size);
+
+worst_case:
+       save = mmap(NULL, size + align, PROT_READ|PROT_WRITE,
+                   MAP_PRIVATE|MAP_ANON, -1, 0);
+       if (save == MAP_FAILED)
+               return NULL;
 
-               munmap(save + excess, size - excess);
-               addr = _vmem_alloc(size, align, flags);
-               munmap(save, excess);
-       }
-       return((void *)addr);
+       addr = (char *)(((size_t)save + (align - 1)) & ~(align - 1));
+       if (save != addr)
+               munmap(save, addr - save);
+       if (addr + size != save + size + align)
+               munmap(addr + size, save + align - addr);
+
+       addr_hint = addr + size;
+
+       return ((void *)addr);
 }
 
 /*
index 412e155..da1de81 100644 (file)
@@ -7,4 +7,6 @@ CFLAGS+=-I${.CURDIR}/../libc/sysvipc
 
 CMAPS+=        ${.CURDIR}/upmap/Symbol.map
 
-SRCS+= upmap.c ukp_clock.c ukp_getpid.c ukp_setproctitle.c ukp_gettimeofday.c
+SRCS+= upmap.c
+SRCS+= ukp_clock.c ukp_getpid.c ukp_setproctitle.c
+SRCS+= ukp_gettimeofday.c ukp_blocksigs.c
index 35e5086..4188bd8 100644 (file)
@@ -1,5 +1,13 @@
+DF508.0 {
+       sigblockall;
+       sigunblockall;
+};
+
 DFprivate_1.0 {
        __kpmap_map;
        __upmap_map;
+       __lpmap_map;
+       __lpmap_headers;
+       __lpmap_blockallsigs;
        __ukp_spt;
 };
diff --git a/lib/libc/upmap/ukp_blocksigs.c b/lib/libc/upmap/ukp_blocksigs.c
new file mode 100644 (file)
index 0000000..968863c
--- /dev/null
@@ -0,0 +1,100 @@
+/*
+ * Copyright (c) 2019 The DragonFly Project.  All rights reserved.
+ *
+ * This code is derived from software contributed to The DragonFly Project
+ * by Matthew Dillon <dillon@backplane.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ * 3. Neither the name of The DragonFly Project nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific, prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "namespace.h"
+#include <sys/cdefs.h>
+#include <sys/types.h>
+#include <sys/syscall.h>
+#include <sys/time.h>
+#include <machine/cpufunc.h>
+#include <machine/atomic.h>
+#include <errno.h>
+#include <time.h>
+#include <stdio.h>
+#include <unistd.h>
+#include "un-namespace.h"
+#include "libc_private.h"
+#include "upmap.h"
+
+int _sigblockall(void);
+int _sigunblockall(void);
+int __getpid(void);
+pid_t __sys_getpid(void);
+
+/*
+ * Block all maskable signals to this thread.  Does not prevent signal
+ * delivery to the thread's pending signal mask, but does prevent signal
+ * dispatch.
+ *
+ * Reentrant, returns the reentrancy count.  First call will thus return
+ * 1.  -1 is returned if the feature is not supported.
+ *
+ * The pointer is set-up on program startup and will remain valid on fork()
+ * (the kernel will re-fault the proper page).  On vfork(), however, the
+ * pointer points to the parent thread's mapping.
+ */
+int
+_sigblockall(void)
+{
+       if (__lpmap_blockallsigs)
+               return atomic_fetchadd_int(__lpmap_blockallsigs, 1) + 1;
+       return -1;
+}
+
+/*
+ * Reverse the effects of sigblockall().  Returns the reentrancy count
+ * after decrement.  -1 is returned if not supported.
+ *
+ * Bit 31 is set on return if signals were received while blocked.  Upon
+ * last unblock this bit also indicates that pending service routines ran.
+ * The bit is cumulative until the last unblock occurs.
+ */
+int
+_sigunblockall(void)
+{
+       uint32_t bas;
+
+       if (__lpmap_blockallsigs) {
+               bas = atomic_fetchadd_int(__lpmap_blockallsigs, -1) - 1;
+               if (bas == 0x80000000U) {
+                       (void)__sys_getpid();   /* force signal processing */
+                       atomic_clear_int(__lpmap_blockallsigs, 0x80000000U);
+               }
+               return (int)bas;
+       }
+       return -1;
+}
+
+__weak_reference(_sigblockall, sigblockall);
+__weak_reference(_sigunblockall, sigunblockall);
index 7bc9762..263e62c 100644 (file)
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014 The DragonFly Project.  All rights reserved.
+ * Copyright (c) 2014,2019 The DragonFly Project.  All rights reserved.
  *
  * This code is derived from software contributed to The DragonFly Project
  * by Matthew Dillon <dillon@backplane.com>
 #include "libc_private.h"
 #include "upmap.h"
 
+/*
+ * kpmap - Global user/kernel shared map (RO)
+ * upmap - Per-process user/kernel shared map (RW)
+ * lpmap - Per-thread user/kernel shared map (RW)
+ */
 static pthread_mutex_t ukpmap_lock;
-static ukpheader_t *kpmap_headers;
-static ukpheader_t *upmap_headers;
+static ukpheader_t *__kpmap_headers;
+static ukpheader_t *__upmap_headers;
+__thread ukpheader_t *__lpmap_headers TLS_ATTRIBUTE;
+__thread uint32_t *__lpmap_blockallsigs TLS_ATTRIBUTE;
+
+static __thread int lpmap_ok;
 static int kpmap_ok;
 static int upmap_ok;
+static pthread_once_t upmap_once = PTHREAD_ONCE_INIT;
+
 
 /*
  * Map the requested data item from the user-kernel global shared mmap
@@ -80,11 +91,11 @@ __kpmap_map(void *datap, int *state, uint16_t type)
                        kpmap_ok = -1;
                        goto failed;
                }
-               kpmap_headers = mmap(NULL, KPMAP_MAPSIZE,
-                                    PROT_READ, MAP_SHARED,
-                                    fd, 0);
+               __kpmap_headers = mmap(NULL, KPMAP_MAPSIZE,
+                                      PROT_READ, MAP_SHARED | MAP_FILE,
+                                      fd, 0);
                _close(fd);
-               if ((void *)kpmap_headers == MAP_FAILED) {
+               if ((void *)__kpmap_headers == MAP_FAILED) {
                        kpmap_ok = -1;
                        goto failed;
                }
@@ -105,9 +116,10 @@ __kpmap_map(void *datap, int *state, uint16_t type)
        /*
         * Look for type.
         */
-       for (head = kpmap_headers; head->type; ++head) {
+       for (head = __kpmap_headers; head->type; ++head) {
                if (head->type == type) {
-                       *(void **)datap = (char *)kpmap_headers + head->offset;
+                       *(void **)datap = (char *)__kpmap_headers +
+                                         head->offset;
                        if (__isthreaded)
                                _pthread_mutex_unlock(&ukpmap_lock);
                        return;
@@ -144,11 +156,12 @@ __upmap_map(void *datap, int *state, uint16_t type)
                        upmap_ok = -1;
                        goto failed;
                }
-               upmap_headers = mmap(NULL, UPMAP_MAPSIZE,
-                                    PROT_READ | PROT_WRITE, MAP_SHARED,
-                                    fd, 0);
+               __upmap_headers = mmap(NULL, UPMAP_MAPSIZE,
+                                      PROT_READ | PROT_WRITE,
+                                      MAP_SHARED | MAP_FILE,
+                                      fd, 0);
                _close(fd);
-               if ((void *)upmap_headers == MAP_FAILED) {
+               if ((void *)__upmap_headers == MAP_FAILED) {
                        upmap_ok = -1;
                        goto failed;
                }
@@ -169,9 +182,10 @@ __upmap_map(void *datap, int *state, uint16_t type)
        /*
         * Look for type.
         */
-       for (head = upmap_headers; head->type; ++head) {
+       for (head = __upmap_headers; head->type; ++head) {
                if (head->type == type) {
-                       *(void **)datap = (char *)upmap_headers + head->offset;
+                       *(void **)datap = (char *)__upmap_headers +
+                                         head->offset;
                        if (__isthreaded)
                                _pthread_mutex_unlock(&ukpmap_lock);
                        return;
@@ -182,3 +196,113 @@ failed:
        if (__isthreaded)
                _pthread_mutex_unlock(&ukpmap_lock);
 }
+
+/*
+ * Map the requested data item from the user-kernel per-thread shared mmap
+ *
+ * *state is set to -1 on failure, else it is left alone.
+ * *datap is set to a pointer to the item on success, else it is left alone.
+ * If type == 0 this function finalizes state, setting it to 1 if it is 0.
+ *
+ * WARNING!  This code is used all over pthreads and must NOT make any
+ *          reentrant pthreads calls until after the mapping has been
+ *          set up.
+ */
+static pthread_key_t lpmap_key;
+
+static void lpmap_unmap(void **datap);
+
+void
+__lpmap_map(void *datap, int *state, uint16_t type)
+{
+       ukpheader_t *head;
+
+       if (lpmap_ok <= 0) {
+               int fd;
+
+               if (lpmap_ok < 0)
+                       goto failed;
+               fd = _open("/dev/lpmap", O_RDWR);
+               if (fd < 0) {
+                       lpmap_ok = -1;
+                       goto failed;
+               }
+               __lpmap_headers = mmap(NULL, LPMAP_MAPSIZE,
+                                      PROT_READ | PROT_WRITE,
+                                      MAP_SHARED | MAP_FILE,
+                                      fd, 0);
+               _close(fd);
+               if ((void *)__lpmap_headers == MAP_FAILED) {
+                       lpmap_ok = -1;
+                       goto failed;
+               }
+               lpmap_ok = 1;
+               _pthread_setspecific(lpmap_key, &__lpmap_headers);
+       }
+
+       /*
+        * Special case to finalize state
+        */
+       if (type == 0) {
+               if (*state == 0)
+                       *state = 1;
+               return;
+       }
+
+       /*
+        * Look for type.
+        */
+       for (head = __lpmap_headers; head->type; ++head) {
+               if (head->type == type) {
+                       *(void **)datap = (char *)__lpmap_headers +
+                                         head->offset;
+                       return;
+               }
+       }
+failed:
+       *state = -1;
+}
+
+/*
+ * Cleanup thread state
+ */
+static void
+lpmap_unmap(void **datap)
+{
+       ukpheader_t *lpmap = *datap;
+
+       lpmap_ok = -1;
+       if (lpmap) {
+               __lpmap_blockallsigs = NULL;
+               *datap = NULL;
+               munmap(lpmap, LPMAP_MAPSIZE);
+       }
+}
+
+/*
+ * upmap initialization code, _upmap_thr_init() is called for the initial
+ * main thread by libc or pthreads, and on every thread create.  We need
+ * the __lpmap_blockallsigs pointer ASAP because it is used everywhere in
+ * pthreads.
+ *
+ * If pthreads is not linked in, _pthread_once() still runs via a stub in
+ * libc, and _pthread_key_create() is a NOP.
+ *
+ * NOTE: These pthreads calls are stubs when pthreads is not linked in.
+ *      The once routine will still be run once regardless.
+ */
+static
+void
+_upmap_init_once(void)
+{
+       _pthread_key_create(&lpmap_key, (void (*)(void *))lpmap_unmap);
+}
+
+void
+_upmap_thr_init(void)
+{
+       int dummy_state = 0;
+
+        _pthread_once(&upmap_once, _upmap_init_once);
+       __lpmap_map(&__lpmap_blockallsigs, &dummy_state, LPTYPE_BLOCKALLSIGS);
+}
index 99d2cc5..5691ada 100644 (file)
 #ifndef _SYS_TYPES_H_
 #include <sys/types.h>
 #endif
+#ifndef _SYS_UPMAP_H_
+#include <sys/upmap.h>
+#endif
 
 void __kpmap_map(void *datap, int *state, uint16_t type);
 void __upmap_map(void *datap, int *state, uint16_t type);
+void __lpmap_map(void *datap, int *state, uint16_t type);
 int __ukp_spt(const char *fmt, va_list ap);
-
+extern __thread ukpheader_t *__lpmap_headers TLS_ATTRIBUTE;
+extern __thread uint32_t *__lpmap_blockallsigs TLS_ATTRIBUTE;
 
 #endif
index eb09d6f..8aef675 100644 (file)
@@ -20,6 +20,8 @@ LIBSTAND_SRC?=        ${.CURDIR}
 LIBSTAND_ARCH?=        ${MACHINE_ARCH}
 LIBC_SRC=      ${LIBSTAND_SRC}/../libc
 
+CFLAGS+=-D_LIBSTAND_
+
 # Mostly OK, some of the libc imports are a bit noisy
 CFLAGS+=       -ffreestanding
 
index 271d589..00b0451 100644 (file)
@@ -227,7 +227,7 @@ thread_start(void *arg)
 
        if (curthread->flags & THR_FLAGS_NEED_SUSPEND)
                _thr_suspend_check(curthread);
-       _nmalloc_thr_init();
+       _libc_thr_init();
 
        /* Run the current thread's start routine with argument: */
        _pthread_exit(curthread->start_routine(curthread->arg));
index c505a34..636ecc9 100644 (file)
@@ -194,6 +194,7 @@ void
 _thread_init(void)
 {
        _libpthread_init(NULL);
+       _libc_thr_init();
 }
 
 void   _thread_uninit(void) __attribute__ ((destructor));
index d7c9b2c..3ed9d08 100644 (file)
@@ -302,7 +302,7 @@ _pthread_mutex_destroy(pthread_mutex_t *mutex)
                 * Try to lock the mutex structure, we only need to
                 * try once, if failed, the mutex is in use.
                 */
-               ret = THR_UMTX_TRYLOCK(curthread, &(*mutex)->m_lock);
+               ret = THR_UMTX_TRYLOCK_PERSIST(curthread, &(*mutex)->m_lock);
                if (ret)
                        return (ret);
 
@@ -314,7 +314,7 @@ _pthread_mutex_destroy(pthread_mutex_t *mutex)
                if (((*mutex)->m_owner != NULL) ||
                    (TAILQ_FIRST(&(*mutex)->m_queue) != NULL) ||
                    ((*mutex)->m_refcount != 0)) {
-                       THR_UMTX_UNLOCK(curthread, &(*mutex)->m_lock);
+                       THR_UMTX_UNLOCK_PERSIST(curthread, &(*mutex)->m_lock);
                        ret = EBUSY;
                } else {
                        /*
@@ -325,7 +325,7 @@ _pthread_mutex_destroy(pthread_mutex_t *mutex)
                        *mutex = NULL;
 
                        /* Unlock the mutex structure: */
-                       THR_UMTX_UNLOCK(curthread, &m->m_lock);
+                       THR_UMTX_UNLOCK_PERSIST(curthread, &m->m_lock);
 
                        /*
                         * Free the memory allocated for the mutex
@@ -348,7 +348,7 @@ mutex_trylock_common(struct pthread *curthread, pthread_mutex_t *mutex)
 
        m = *mutex;
        mutex_log("mutex_lock_trylock_common %p\n", m);
-       ret = THR_UMTX_TRYLOCK(curthread, &m->m_lock);
+       ret = THR_UMTX_TRYLOCK_PERSIST(curthread, &m->m_lock);
        if (ret == 0) {
                mutex_log2(curthread, m, 1);
                m->m_owner = curthread;
@@ -412,7 +412,7 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *mutex,
 
        m = *mutex;
        mutex_log("mutex_lock_common %p\n", m);
-       ret = THR_UMTX_TRYLOCK(curthread, &m->m_lock);
+       ret = THR_UMTX_TRYLOCK_PERSIST(curthread, &m->m_lock);
        if (ret == 0) {
                mutex_log2(curthread, m, 3);
                m->m_owner = curthread;
@@ -423,7 +423,7 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *mutex,
                ret = mutex_self_lock(m, abstime);
        } else {
                if (abstime == NULL) {
-                       THR_UMTX_LOCK(curthread, &m->m_lock);
+                       THR_UMTX_LOCK_PERSIST(curthread, &m->m_lock);
                        ret = 0;
                } else if (__predict_false(
                        abstime->tv_sec < 0 || abstime->tv_nsec < 0 ||
@@ -432,7 +432,8 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *mutex,
                } else {
                        clock_gettime(CLOCK_REALTIME, &ts);
                        timespecsub(abstime, &ts, &ts2);
-                       ret = THR_UMTX_TIMEDLOCK(curthread, &m->m_lock, &ts2);
+                       ret = THR_UMTX_TIMEDLOCK_PERSIST(curthread,
+                                                        &m->m_lock, &ts2);
                }
                if (ret == 0) {
                        mutex_log2(curthread, m, 4);
@@ -677,7 +678,7 @@ mutex_unlock_common(pthread_mutex_t *mutex)
                 */
                mutex_log("mutex_unlock_common %p (returns 0) lock %d\n",
                          m, m->m_lock);
-               THR_UMTX_UNLOCK(curthread, &m->m_lock);
+               THR_UMTX_UNLOCK_PERSIST(curthread, &m->m_lock);
                mutex_log2(tls_get_curthread(), m, 37);
                mutex_log2(curthread, m, 255);
        }
@@ -750,7 +751,7 @@ _mutex_cv_unlock(pthread_mutex_t *mutex, int *count)
        MUTEX_ASSERT_IS_OWNED(m);
        TAILQ_REMOVE(&curthread->mutexq, m, m_qe);
        MUTEX_INIT_LINK(m);
-       THR_UMTX_UNLOCK(curthread, &m->m_lock);
+       THR_UMTX_UNLOCK_PERSIST(curthread, &m->m_lock);
        mutex_log2(curthread, m, 250);
        return (0);
 }
index 72e5afa..afa4e8e 100644 (file)
@@ -486,22 +486,45 @@ struct pthread {
        (((thrd)->locklevel > 0) ||                     \
        ((thrd)->critical_count > 0))
 
+/*
+ * Internal temporary locks without suspend check
+ */
 #define THR_UMTX_TRYLOCK(thrd, lck)                    \
-       _thr_umtx_trylock((lck), (thrd)->tid)
+       _thr_umtx_trylock((lck), (thrd)->tid, 1)
 
 #define        THR_UMTX_LOCK(thrd, lck)                        \
-       _thr_umtx_lock((lck), (thrd)->tid)
+       _thr_umtx_lock((lck), (thrd)->tid, 1)
 
 #define        THR_UMTX_TIMEDLOCK(thrd, lck, timo)             \
-       _thr_umtx_timedlock((lck), (thrd)->tid, (timo))
+       _thr_umtx_timedlock((lck), (thrd)->tid, (timo), 1)
 
 #define        THR_UMTX_UNLOCK(thrd, lck)                      \
-       _thr_umtx_unlock((lck), (thrd)->tid)
+       _thr_umtx_unlock((lck), (thrd)->tid, 1)
+
+/*
+ * Interal locks without suspend check, used when the lock
+ * state needs to persist (i.e. to help implement things
+ * like pthread_mutex_lock()).  Non-temporary.
+ */
+#define THR_UMTX_TRYLOCK_PERSIST(thrd, lck)            \
+       _thr_umtx_trylock((lck), (thrd)->tid, 0)
 
+#define        THR_UMTX_LOCK_PERSIST(thrd, lck)                \
+       _thr_umtx_lock((lck), (thrd)->tid, 0)
+
+#define        THR_UMTX_TIMEDLOCK_PERSIST(thrd, lck, timo)     \
+       _thr_umtx_timedlock((lck), (thrd)->tid, (timo), 0)
+
+#define        THR_UMTX_UNLOCK_PERSIST(thrd, lck)              \
+       _thr_umtx_unlock((lck), (thrd)->tid, 0)
+
+/*
+ * Internal temporary locks with suspend check
+ */
 #define        THR_LOCK_ACQUIRE(thrd, lck)                     \
 do {                                                   \
        (thrd)->locklevel++;                            \
-       _thr_umtx_lock((lck), (thrd)->tid);             \
+       _thr_umtx_lock((lck), (thrd)->tid, 1);          \
 } while (0)
 
 #ifdef _PTHREADS_INVARIANTS
@@ -517,7 +540,7 @@ do {                                                        \
 #define        THR_LOCK_RELEASE(thrd, lck)                     \
 do {                                                   \
        THR_ASSERT_LOCKLEVEL(thrd);                     \
-       _thr_umtx_unlock((lck), (thrd)->tid);           \
+       _thr_umtx_unlock((lck), (thrd)->tid, 1);        \
        (thrd)->locklevel--;                            \
        _thr_ast(thrd);                                 \
 } while (0)
index 062abfe..d1002e1 100644 (file)
@@ -77,7 +77,7 @@ _pthread_spin_trylock(pthread_spinlock_t *lock)
 
        if (lock == NULL || (lck = *lock) == NULL)
                return (EINVAL);
-       return (THR_UMTX_TRYLOCK(curthread, &lck->s_lock));
+       return (THR_UMTX_TRYLOCK_PERSIST(curthread, &lck->s_lock));
 }
 
 int
@@ -92,7 +92,7 @@ _pthread_spin_lock(pthread_spinlock_t *lock)
 
        curthread = tls_get_curthread();
        count = SPIN_COUNT;
-       while (THR_UMTX_TRYLOCK(curthread, &lck->s_lock) != 0) {
+       while (THR_UMTX_TRYLOCK_PERSIST(curthread, &lck->s_lock) != 0) {
                while (lck->s_lock) {
                        CPU_SPINWAIT;   /* tell cpu we are spinning */
                        if (--count <= 0) {
@@ -113,7 +113,7 @@ _pthread_spin_unlock(pthread_spinlock_t *lock)
        if (lock == NULL || (lck = *lock) == NULL)
                return (EINVAL);
        /* XXX: shouldn't return status? */
-       THR_UMTX_UNLOCK(curthread, &lck->s_lock);
+       THR_UMTX_UNLOCK_PERSIST(curthread, &lck->s_lock);
        return (0);
 }
 
index 7493092..92c5b11 100644 (file)
@@ -55,10 +55,10 @@ static int                  initialized;
 static void    init_spinlock(spinlock_t *lck);
 
 /*
- * These are for compatibility only.  Spinlocks of this type
- * are deprecated.
+ * These functions implement temporary spinlocks as used by libc.
+ * these are not persistent spinlocks, so we use the API for
+ * temporary locks which block all signals for the duration.
  */
-
 void
 _spinunlock(spinlock_t *lck)
 {
index f048d1c..5b8d7bd 100644 (file)
@@ -49,8 +49,10 @@ _thr_umtx_init(volatile umtx_t *mtx)
 }
 
 static inline int
-_thr_umtx_trylock(volatile umtx_t *mtx, int id)
+_thr_umtx_trylock(volatile umtx_t *mtx, int id, int temporary)
 {
+       if (temporary)
+               sigblockall();
        if (atomic_cmpset_acq_int(mtx, 0, id))
                return (0);
        cpu_pause();
@@ -59,35 +61,53 @@ _thr_umtx_trylock(volatile umtx_t *mtx, int id)
        cpu_pause();
        if (atomic_cmpset_acq_int(mtx, 0, id))
                return (0);
+       if (temporary)
+               sigunblockall();
        return (EBUSY);
 }
 
 static inline int
-_thr_umtx_lock(volatile umtx_t *mtx, int id)
+_thr_umtx_lock(volatile umtx_t *mtx, int id, int temporary)
 {
+       int res;
+
+       if (temporary)
+               sigblockall();
        if (atomic_cmpset_acq_int(mtx, 0, id))
                return (0);
-       return (__thr_umtx_lock(mtx, id, 0));
+       res = __thr_umtx_lock(mtx, id, 0);
+       if (res && temporary)
+               sigunblockall();
+       return res;
 }
 
 static inline int
 _thr_umtx_timedlock(volatile umtx_t *mtx, int id,
-    const struct timespec *timeout)
+    const struct timespec *timeout, int temporary)
 {
+       int res;
+
+       if (temporary)
+               sigblockall();
        if (atomic_cmpset_acq_int(mtx, 0, id)) {
                return (0);
        }
-       return (__thr_umtx_timedlock(mtx, id, timeout));
+       res = __thr_umtx_timedlock(mtx, id, timeout);
+       if (res && temporary)
+               sigunblockall();
+       return res;
 }
 
 static inline void
-_thr_umtx_unlock(volatile umtx_t *mtx, int id)
+_thr_umtx_unlock(volatile umtx_t *mtx, int id, int temporary)
 {
        int v;
 
        v = atomic_swap_int(mtx, 0);
        if (v != id)
                __thr_umtx_unlock(mtx, v, id);
+       if (temporary)
+               sigunblockall();
 }
 
 int _thr_umtx_wait(volatile umtx_t *mtx, umtx_t exp,
index 1c237bd..207a741 100644 (file)
@@ -2894,6 +2894,8 @@ search_library_pathfds(const char *name, const char *path, int *fdp)
         * Use strtok_r() to walk the FD:FD:FD list.  This requires a local
         * copy of the path, as strtok_r rewrites separator tokens
         * with '\0'.
+        *
+        * NOTE: strtok() uses a __thread static and cannot be used by rtld.
         */
        found = NULL;
        envcopy = xstrdup(path);