From 721505dec240e78696660384d988da78813a33bd Mon Sep 17 00:00:00 2001 From: Matthew Dillon Date: Mon, 11 Nov 2019 17:35:42 -0800 Subject: [PATCH] libc - Implement sigblockall() and sigunblockall() * Signal safety is becoming a defacto requirement for most of libc and pthreads. In particular, the memory allocator. Given the chances of teaching tens of thousands of programmers about signal safety, and just making it work in libc and pthreads, only one of these two possibilities is actually realizable. In particular, high-level languages have become so complex, and some applications (chrome, firefox, etc) have become so complex, that the code is regularly tripping over signal safety issues. However, implementing signal safety with current mechanisms is extremely expensive due to the need for multiple system calls. To whit, DragonFlyBSD now has a mechanism that does not require system calls in the critical path. * Implement sigblockall() and sigunblockall(). These functions leverage the new /dev/lpmap per-thread shared page mechanism to provide a way to temporary block the dispatch of all maskable signals without having to make any system calls. These are extremely fast routines. - Reentrant / Recursable - Temporarily blocks any dispatch of a maskable asynchronous signal to the calling thread. Other threads are not affected... this is a per-thread mechanism. - The last sigunblockall() will immediately dispatch any blocked signals. - The normal signal mask is not affected by these routines. - Does not block signals caused by synchronous traps. - The current recursion count is retained on [v]fork() to ease coding and to also allow signals to be temporarily blocked across a fork until the child process is ready to deal with them, if desired. * Implement signal safety for most of pthreads. All temporary internal mutexes are now wrapped with sigblockall() and sigunblockall(). * Implement signal safety for the malloc subsystem. All functions are wrawpped with sigblockall() and sigunblockall(). These implementations make lang/mono and lang/rust far more reliable than they were before. Where 9 out of 10 builds used to fail, now they succeed. --- lib/libc/gen/_thread_init.c | 33 ++++- lib/libc/include/libc_private.h | 15 ++ lib/libc/stdlib/Symbol.map | 2 +- lib/libc/stdlib/nmalloc.c | 182 +++++++++++++++++------- lib/libc/upmap/Makefile.inc | 4 +- lib/libc/upmap/Symbol.map | 8 ++ lib/libc/upmap/ukp_blocksigs.c | 100 +++++++++++++ lib/libc/upmap/upmap.c | 154 ++++++++++++++++++-- lib/libc/upmap/upmap.h | 7 +- lib/libstand/Makefile | 2 + lib/libthread_xu/thread/thr_create.c | 2 +- lib/libthread_xu/thread/thr_init.c | 1 + lib/libthread_xu/thread/thr_mutex.c | 19 +-- lib/libthread_xu/thread/thr_private.h | 35 ++++- lib/libthread_xu/thread/thr_pspinlock.c | 6 +- lib/libthread_xu/thread/thr_spinlock.c | 6 +- lib/libthread_xu/thread/thr_umtx.h | 32 ++++- libexec/rtld-elf/rtld.c | 2 + 18 files changed, 509 insertions(+), 101 deletions(-) create mode 100644 lib/libc/upmap/ukp_blocksigs.c diff --git a/lib/libc/gen/_thread_init.c b/lib/libc/gen/_thread_init.c index 95ebbd23e6..513bdc856a 100644 --- a/lib/libc/gen/_thread_init.c +++ b/lib/libc/gen/_thread_init.c @@ -27,16 +27,45 @@ #include #include +#include "libc_private.h" void _thread_init_stub(void); +void _nmalloc_thr_init(void); +void _upmap_thr_init(void); int _thread_autoinit_dummy_decl_stub = 0; +/* + * This stub is overridden when libpthreads is linked in. However, + * we can apply the constructor to the weak reference. Instead the + * constructor is applied to the stub and we do a run-time check to + * see if the stub has been overridden by pthreads. + */ +void _thread_init_stub(void) __attribute__ ((constructor)); +__weak_reference(_thread_init_stub, _thread_init); + void _thread_init_stub(void) { - /* This is just a stub; there is nothing to do. */ + /* + * Only run libc related pthreads initialization from here + * if pthreads did not override the weak reference. Otherwise + * pthreads will do it after setting up a real thread context. + */ + if (_thread_init == _thread_init_stub) { + _libc_thr_init(); + } +} + +/* + * Pre-initialization prior to thread startup to avoid certain + * chicken-and-egg issues with low-level allocations. + */ +void +_libc_thr_init(void) +{ + _nmalloc_thr_init(); + _upmap_thr_init(); } -__weak_reference(_thread_init_stub, _thread_init); __weak_reference(_thread_autoinit_dummy_decl_stub, _thread_autoinit_dummy_decl); diff --git a/lib/libc/include/libc_private.h b/lib/libc/include/libc_private.h index c3459e2a57..e5de2b237c 100644 --- a/lib/libc/include/libc_private.h +++ b/lib/libc/include/libc_private.h @@ -38,6 +38,19 @@ #include #include +/* + * With this attribute set, do not require a function call for accessing + * this variable when the code is compiled -fPIC. Use in combination + * with __thread. + * + * Must be empty for libc_rtld. + */ +#ifdef __LIBC_RTLD +#define TLS_ATTRIBUTE +#else +#define TLS_ATTRIBUTE __attribute__ ((tls_model ("initial-exec"))) +#endif + /* * This global flag is non-zero when a process has created one * or more threads. It is used to avoid calling locking functions @@ -100,7 +113,9 @@ extern void (*__cleanup)(void); /* execve() with PATH processing to implement posix_spawnp() */ int _execvpe(const char *, char * const *, char * const *); +void _libc_thr_init(void); void _nmalloc_thr_init(void); +void _upmap_thr_init(void); void _nmalloc_thr_prepfork(void); void _nmalloc_thr_parentfork(void); void _nmalloc_thr_childfork(void); diff --git a/lib/libc/stdlib/Symbol.map b/lib/libc/stdlib/Symbol.map index 6a123112da..151a6d7217 100644 --- a/lib/libc/stdlib/Symbol.map +++ b/lib/libc/stdlib/Symbol.map @@ -2,7 +2,7 @@ DF404.0 { __cxa_atexit; __cxa_finalize; _Exit; - _nmalloc_thr_init; + _libc_thr_init; _nmalloc_thr_prepfork; _nmalloc_thr_parentfork; _nmalloc_thr_childfork; diff --git a/lib/libc/stdlib/nmalloc.c b/lib/libc/stdlib/nmalloc.c index 4ef9ac3094..7218e51298 100644 --- a/lib/libc/stdlib/nmalloc.c +++ b/lib/libc/stdlib/nmalloc.c @@ -246,10 +246,26 @@ typedef struct slglobaldata { #define arysize(ary) (sizeof(ary)/sizeof((ary)[0])) -#define MASSERT(exp) do { if (__predict_false(!(exp))) \ - _mpanic("assertion: %s in %s", \ - #exp, __func__); \ - } while (0) +/* + * The assertion macros try to pretty-print assertion failures + * which can be caused by corruption. If a lock is held, we + * provide a macro that attempts to release it before asserting + * in order to prevent (e.g.) a reentrant SIGABRT calling malloc + * and deadlocking, resulting in the program freezing up. + */ +#define MASSERT(exp) \ + do { if (__predict_false(!(exp))) \ + _mpanic("assertion: %s in %s", \ + #exp, __func__); \ + } while (0) + +#define MASSERT_WTHUNLK(exp, unlk) \ + do { if (__predict_false(!(exp))) { \ + unlk; \ + _mpanic("assertion: %s in %s", \ + #exp, __func__); \ + } \ + } while (0) /* * Magazines @@ -316,18 +332,6 @@ typedef struct thr_mags { int init; } thr_mags; -/* - * With this attribute set, do not require a function call for accessing - * this variable when the code is compiled -fPIC. - * - * Must be empty for libc_rtld (similar to __thread). - */ -#ifdef __LIBC_RTLD -#define TLS_ATTRIBUTE -#else -#define TLS_ATTRIBUTE __attribute__ ((tls_model ("initial-exec"))) -#endif - static __thread thr_mags thread_mags TLS_ATTRIBUTE; static pthread_key_t thread_mags_key; static pthread_once_t thread_mags_once = PTHREAD_ONCE_INIT; @@ -421,7 +425,6 @@ malloc_init(void) void _nmalloc_thr_init(void) { - static int init_once; thr_mags *tp; /* @@ -431,10 +434,7 @@ _nmalloc_thr_init(void) tp = &thread_mags; tp->init = -1; - if (init_once == 0) { - init_once = 1; - _pthread_once(&thread_mags_once, mtmagazine_init); - } + _pthread_once(&thread_mags_once, mtmagazine_init); _pthread_setspecific(thread_mags_key, tp); tp->init = 1; } @@ -466,6 +466,28 @@ _nmalloc_thr_childfork(void) } } +/* + * Handle signal reentrancy safely whether we are threaded or not. + * This improves the stability for mono and will probably improve + * stability for other high-level languages which are becoming increasingly + * sophisticated. + * + * The sigblockall()/sigunblockall() implementation uses a counter on + * a per-thread shared user/kernel page, avoids system calls, and is thus + * very fast. + */ +static __inline void +nmalloc_sigblockall(void) +{ + sigblockall(); +} + +static __inline void +nmalloc_sigunblockall(void) +{ + sigunblockall(); +} + /* * Thread locks. */ @@ -474,6 +496,8 @@ slgd_lock(slglobaldata_t slgd) { if (__isthreaded) _SPINLOCK(&slgd->Spinlock); + else + sigblockall(); } static __inline void @@ -481,6 +505,8 @@ slgd_unlock(slglobaldata_t slgd) { if (__isthreaded) _SPINUNLOCK(&slgd->Spinlock); + else + sigunblockall(); } static __inline void @@ -488,6 +514,8 @@ depot_lock(magazine_depot *dp __unused) { if (__isthreaded) _SPINLOCK(&depot_spinlock); + else + sigblockall(); #if 0 if (__isthreaded) _SPINLOCK(&dp->lock); @@ -499,6 +527,8 @@ depot_unlock(magazine_depot *dp __unused) { if (__isthreaded) _SPINUNLOCK(&depot_spinlock); + else + sigunblockall(); #if 0 if (__isthreaded) _SPINUNLOCK(&dp->lock); @@ -510,6 +540,8 @@ zone_magazine_lock(void) { if (__isthreaded) _SPINLOCK(&zone_mag_lock); + else + sigblockall(); } static __inline void @@ -517,6 +549,8 @@ zone_magazine_unlock(void) { if (__isthreaded) _SPINUNLOCK(&zone_mag_lock); + else + sigunblockall(); } static __inline void @@ -666,8 +700,10 @@ handle_excess_big(void) _SPINLOCK(&bigspin_array[i & BIGXMASK]); for (big = *bigp; big; big = big->next) { if (big->active < big->bytes) { - MASSERT((big->active & PAGE_MASK) == 0); - MASSERT((big->bytes & PAGE_MASK) == 0); + MASSERT_WTHUNLK((big->active & PAGE_MASK) == 0, + _SPINUNLOCK(&bigspin_array[i & BIGXMASK])); + MASSERT_WTHUNLK((big->bytes & PAGE_MASK) == 0, + _SPINUNLOCK(&bigspin_array[i & BIGXMASK])); munmap((char *)big->base + big->active, big->bytes - big->active); atomic_add_long(&excess_alloc, @@ -767,11 +803,14 @@ __malloc(size_t size) { void *ptr; + nmalloc_sigblockall(); ptr = _slaballoc(size, 0); if (ptr == NULL) errno = ENOMEM; else UTRACE(0, size, ptr); + nmalloc_sigunblockall(); + return(ptr); } @@ -791,11 +830,14 @@ __calloc(size_t number, size_t size) return(NULL); } + nmalloc_sigblockall(); ptr = _slaballoc(number * size, SAFLAG_ZERO); if (ptr == NULL) errno = ENOMEM; else UTRACE(0, number * size, ptr); + nmalloc_sigunblockall(); + return(ptr); } @@ -810,11 +852,15 @@ void * __realloc(void *ptr, size_t size) { void *ret; + + nmalloc_sigblockall(); ret = _slabrealloc(ptr, size); if (ret == NULL) errno = ENOMEM; else UTRACE(ptr, size, ret); + nmalloc_sigunblockall(); + return(ret); } @@ -829,10 +875,12 @@ __aligned_alloc(size_t alignment, size_t size) void *ptr; int rc; + nmalloc_sigblockall(); ptr = NULL; rc = _slabmemalign(&ptr, alignment, size); if (rc) errno = rc; + nmalloc_sigunblockall(); return (ptr); } @@ -856,7 +904,9 @@ __posix_memalign(void **memptr, size_t alignment, size_t size) return(EINVAL); } + nmalloc_sigblockall(); rc = _slabmemalign(memptr, alignment, size); + nmalloc_sigunblockall(); return (rc); } @@ -1006,7 +1056,9 @@ void __free(void *ptr) { UTRACE(ptr, 0, 0); + nmalloc_sigblockall(); _slabfree(ptr, 0, NULL); + nmalloc_sigunblockall(); } /* @@ -1209,7 +1261,7 @@ _slaballoc(size_t size, int flags) * * Remove us from the ZoneAry[] when we become empty */ - MASSERT(z->z_NFree > 0); + MASSERT_WTHUNLK(z->z_NFree > 0, slgd_unlock(slgd)); if (--z->z_NFree == 0) { slgd->ZoneAry[zi] = z->z_Next; @@ -1223,7 +1275,10 @@ _slaballoc(size_t size, int flags) */ while (z->z_FirstFreePg < ZonePageCount) { if ((chunk = z->z_PageAry[z->z_FirstFreePg]) != NULL) { - MASSERT((uintptr_t)chunk & ZoneMask); + if (((uintptr_t)chunk & ZoneMask) == 0) { + slgd_unlock(slgd); + _mpanic("assertion: corrupt malloc zone"); + } z->z_PageAry[z->z_FirstFreePg] = chunk->c_Next; goto done; } @@ -1352,7 +1407,9 @@ _slabrealloc(void *ptr, size_t size) return(ptr); } - MASSERT((void *)addr == MAP_FAILED); + MASSERT_WTHUNLK( + (void *)addr == MAP_FAILED, + bigalloc_unlock(ptr)); } /* @@ -1614,6 +1671,7 @@ mtmagazine_alloc(int zi) /* * Primary per-thread allocation loop */ + nmalloc_sigblockall(); for (;;) { /* * If the loaded magazine has rounds, allocate and return @@ -1652,13 +1710,14 @@ mtmagazine_alloc(int zi) tp->mags[zi].loaded = mp; if (mp) { SLIST_REMOVE_HEAD(&d->full, nextmagazine); - MASSERT(MAGAZINE_NOTEMPTY(mp)); depot_unlock(d); + MASSERT(MAGAZINE_NOTEMPTY(mp)); continue; } depot_unlock(d); break; } + nmalloc_sigunblockall(); return (obj); } @@ -1682,6 +1741,7 @@ mtmagazine_free(int zi, void *ptr) /* * Primary per-thread freeing loop */ + nmalloc_sigblockall(); for (;;) { /* * Make sure a new magazine is available in case we have @@ -1737,6 +1797,7 @@ mtmagazine_free(int zi, void *ptr) if (mp) { tp->mags[zi].loaded = mp; SLIST_REMOVE_HEAD(&d->empty, nextmagazine); + depot_unlock(d); MASSERT(MAGAZINE_NOTFULL(mp)); } else { mp = tp->newmag; @@ -1745,9 +1806,10 @@ mtmagazine_free(int zi, void *ptr) mp->rounds = 0; mp->flags = 0; tp->mags[zi].loaded = mp; + depot_unlock(d); } - depot_unlock(d); } + nmalloc_sigunblockall(); return rc; } @@ -1852,7 +1914,7 @@ zone_alloc(int flags) for (i = 1; i < burst; i++) { j = magazine_free(&zone_magazine, (char *) z + (ZoneSize * i)); - MASSERT(j == 0); + MASSERT_WTHUNLK(j == 0, zone_magazine_unlock()); } } zone_magazine_unlock(); @@ -1895,7 +1957,8 @@ zone_free(void *z) j = zone_magazine.rounds - zone_magazine.low_factor; for (i = 0; i < j; i++) { excess[i] = magazine_alloc(&zone_magazine, NULL); - MASSERT(excess[i] != NULL); + MASSERT_WTHUNLK(excess[i] != NULL, + zone_magazine_unlock()); } zone_magazine_unlock(); @@ -1922,39 +1985,52 @@ zone_free(void *z) static void * _vmem_alloc(size_t size, size_t align, int flags) { + static char *addr_hint; + static int reset_hint = 16; char *addr; char *save; - size_t excess; + + if (--reset_hint <= 0) { + addr_hint = NULL; + reset_hint = 16; + } /* * Map anonymous private memory. */ - addr = mmap(NULL, size, PROT_READ|PROT_WRITE, + save = mmap(addr_hint, size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, 0); - if (addr == MAP_FAILED) - return(NULL); + if (save == MAP_FAILED) + goto worst_case; + if (((uintptr_t)save & (align - 1)) == 0) + return((void *)save); - /* - * Check alignment. The misaligned offset is also the excess - * amount. If misaligned unmap the excess so we have a chance of - * mapping at the next alignment point and recursively try again. - * - * BBBBBBBBBBB BBBBBBBBBBB BBBBBBBBBBB block alignment - * aaaaaaaaa aaaaaaaaaaa aa mis-aligned allocation - * xxxxxxxxx final excess calculation - * ^ returned address - */ - excess = (uintptr_t)addr & (align - 1); + addr_hint = (char *)(((size_t)save + (align - 1)) & ~(align - 1)); + munmap(save, size); - if (excess) { - excess = align - excess; - save = addr; + save = mmap(addr_hint, size, PROT_READ|PROT_WRITE, + MAP_PRIVATE|MAP_ANON, -1, 0); + if (save == MAP_FAILED) + goto worst_case; + if (((size_t)save & (align - 1)) == 0) + return((void *)save); + munmap(save, size); + +worst_case: + save = mmap(NULL, size + align, PROT_READ|PROT_WRITE, + MAP_PRIVATE|MAP_ANON, -1, 0); + if (save == MAP_FAILED) + return NULL; - munmap(save + excess, size - excess); - addr = _vmem_alloc(size, align, flags); - munmap(save, excess); - } - return((void *)addr); + addr = (char *)(((size_t)save + (align - 1)) & ~(align - 1)); + if (save != addr) + munmap(save, addr - save); + if (addr + size != save + size + align) + munmap(addr + size, save + align - addr); + + addr_hint = addr + size; + + return ((void *)addr); } /* diff --git a/lib/libc/upmap/Makefile.inc b/lib/libc/upmap/Makefile.inc index 412e155a5a..da1de81608 100644 --- a/lib/libc/upmap/Makefile.inc +++ b/lib/libc/upmap/Makefile.inc @@ -7,4 +7,6 @@ CFLAGS+=-I${.CURDIR}/../libc/sysvipc CMAPS+= ${.CURDIR}/upmap/Symbol.map -SRCS+= upmap.c ukp_clock.c ukp_getpid.c ukp_setproctitle.c ukp_gettimeofday.c +SRCS+= upmap.c +SRCS+= ukp_clock.c ukp_getpid.c ukp_setproctitle.c +SRCS+= ukp_gettimeofday.c ukp_blocksigs.c diff --git a/lib/libc/upmap/Symbol.map b/lib/libc/upmap/Symbol.map index 35e5086c90..4188bd8b88 100644 --- a/lib/libc/upmap/Symbol.map +++ b/lib/libc/upmap/Symbol.map @@ -1,5 +1,13 @@ +DF508.0 { + sigblockall; + sigunblockall; +}; + DFprivate_1.0 { __kpmap_map; __upmap_map; + __lpmap_map; + __lpmap_headers; + __lpmap_blockallsigs; __ukp_spt; }; diff --git a/lib/libc/upmap/ukp_blocksigs.c b/lib/libc/upmap/ukp_blocksigs.c new file mode 100644 index 0000000000..968863cc10 --- /dev/null +++ b/lib/libc/upmap/ukp_blocksigs.c @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2019 The DragonFly Project. All rights reserved. + * + * This code is derived from software contributed to The DragonFly Project + * by Matthew Dillon + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of The DragonFly Project nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific, prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "namespace.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "un-namespace.h" +#include "libc_private.h" +#include "upmap.h" + +int _sigblockall(void); +int _sigunblockall(void); +int __getpid(void); +pid_t __sys_getpid(void); + +/* + * Block all maskable signals to this thread. Does not prevent signal + * delivery to the thread's pending signal mask, but does prevent signal + * dispatch. + * + * Reentrant, returns the reentrancy count. First call will thus return + * 1. -1 is returned if the feature is not supported. + * + * The pointer is set-up on program startup and will remain valid on fork() + * (the kernel will re-fault the proper page). On vfork(), however, the + * pointer points to the parent thread's mapping. + */ +int +_sigblockall(void) +{ + if (__lpmap_blockallsigs) + return atomic_fetchadd_int(__lpmap_blockallsigs, 1) + 1; + return -1; +} + +/* + * Reverse the effects of sigblockall(). Returns the reentrancy count + * after decrement. -1 is returned if not supported. + * + * Bit 31 is set on return if signals were received while blocked. Upon + * last unblock this bit also indicates that pending service routines ran. + * The bit is cumulative until the last unblock occurs. + */ +int +_sigunblockall(void) +{ + uint32_t bas; + + if (__lpmap_blockallsigs) { + bas = atomic_fetchadd_int(__lpmap_blockallsigs, -1) - 1; + if (bas == 0x80000000U) { + (void)__sys_getpid(); /* force signal processing */ + atomic_clear_int(__lpmap_blockallsigs, 0x80000000U); + } + return (int)bas; + } + return -1; +} + +__weak_reference(_sigblockall, sigblockall); +__weak_reference(_sigunblockall, sigunblockall); diff --git a/lib/libc/upmap/upmap.c b/lib/libc/upmap/upmap.c index 7bc9762210..263e62c0af 100644 --- a/lib/libc/upmap/upmap.c +++ b/lib/libc/upmap/upmap.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 The DragonFly Project. All rights reserved. + * Copyright (c) 2014,2019 The DragonFly Project. All rights reserved. * * This code is derived from software contributed to The DragonFly Project * by Matthew Dillon @@ -49,11 +49,22 @@ #include "libc_private.h" #include "upmap.h" +/* + * kpmap - Global user/kernel shared map (RO) + * upmap - Per-process user/kernel shared map (RW) + * lpmap - Per-thread user/kernel shared map (RW) + */ static pthread_mutex_t ukpmap_lock; -static ukpheader_t *kpmap_headers; -static ukpheader_t *upmap_headers; +static ukpheader_t *__kpmap_headers; +static ukpheader_t *__upmap_headers; +__thread ukpheader_t *__lpmap_headers TLS_ATTRIBUTE; +__thread uint32_t *__lpmap_blockallsigs TLS_ATTRIBUTE; + +static __thread int lpmap_ok; static int kpmap_ok; static int upmap_ok; +static pthread_once_t upmap_once = PTHREAD_ONCE_INIT; + /* * Map the requested data item from the user-kernel global shared mmap @@ -80,11 +91,11 @@ __kpmap_map(void *datap, int *state, uint16_t type) kpmap_ok = -1; goto failed; } - kpmap_headers = mmap(NULL, KPMAP_MAPSIZE, - PROT_READ, MAP_SHARED, - fd, 0); + __kpmap_headers = mmap(NULL, KPMAP_MAPSIZE, + PROT_READ, MAP_SHARED | MAP_FILE, + fd, 0); _close(fd); - if ((void *)kpmap_headers == MAP_FAILED) { + if ((void *)__kpmap_headers == MAP_FAILED) { kpmap_ok = -1; goto failed; } @@ -105,9 +116,10 @@ __kpmap_map(void *datap, int *state, uint16_t type) /* * Look for type. */ - for (head = kpmap_headers; head->type; ++head) { + for (head = __kpmap_headers; head->type; ++head) { if (head->type == type) { - *(void **)datap = (char *)kpmap_headers + head->offset; + *(void **)datap = (char *)__kpmap_headers + + head->offset; if (__isthreaded) _pthread_mutex_unlock(&ukpmap_lock); return; @@ -144,11 +156,12 @@ __upmap_map(void *datap, int *state, uint16_t type) upmap_ok = -1; goto failed; } - upmap_headers = mmap(NULL, UPMAP_MAPSIZE, - PROT_READ | PROT_WRITE, MAP_SHARED, - fd, 0); + __upmap_headers = mmap(NULL, UPMAP_MAPSIZE, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FILE, + fd, 0); _close(fd); - if ((void *)upmap_headers == MAP_FAILED) { + if ((void *)__upmap_headers == MAP_FAILED) { upmap_ok = -1; goto failed; } @@ -169,9 +182,10 @@ __upmap_map(void *datap, int *state, uint16_t type) /* * Look for type. */ - for (head = upmap_headers; head->type; ++head) { + for (head = __upmap_headers; head->type; ++head) { if (head->type == type) { - *(void **)datap = (char *)upmap_headers + head->offset; + *(void **)datap = (char *)__upmap_headers + + head->offset; if (__isthreaded) _pthread_mutex_unlock(&ukpmap_lock); return; @@ -182,3 +196,113 @@ failed: if (__isthreaded) _pthread_mutex_unlock(&ukpmap_lock); } + +/* + * Map the requested data item from the user-kernel per-thread shared mmap + * + * *state is set to -1 on failure, else it is left alone. + * *datap is set to a pointer to the item on success, else it is left alone. + * If type == 0 this function finalizes state, setting it to 1 if it is 0. + * + * WARNING! This code is used all over pthreads and must NOT make any + * reentrant pthreads calls until after the mapping has been + * set up. + */ +static pthread_key_t lpmap_key; + +static void lpmap_unmap(void **datap); + +void +__lpmap_map(void *datap, int *state, uint16_t type) +{ + ukpheader_t *head; + + if (lpmap_ok <= 0) { + int fd; + + if (lpmap_ok < 0) + goto failed; + fd = _open("/dev/lpmap", O_RDWR); + if (fd < 0) { + lpmap_ok = -1; + goto failed; + } + __lpmap_headers = mmap(NULL, LPMAP_MAPSIZE, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FILE, + fd, 0); + _close(fd); + if ((void *)__lpmap_headers == MAP_FAILED) { + lpmap_ok = -1; + goto failed; + } + lpmap_ok = 1; + _pthread_setspecific(lpmap_key, &__lpmap_headers); + } + + /* + * Special case to finalize state + */ + if (type == 0) { + if (*state == 0) + *state = 1; + return; + } + + /* + * Look for type. + */ + for (head = __lpmap_headers; head->type; ++head) { + if (head->type == type) { + *(void **)datap = (char *)__lpmap_headers + + head->offset; + return; + } + } +failed: + *state = -1; +} + +/* + * Cleanup thread state + */ +static void +lpmap_unmap(void **datap) +{ + ukpheader_t *lpmap = *datap; + + lpmap_ok = -1; + if (lpmap) { + __lpmap_blockallsigs = NULL; + *datap = NULL; + munmap(lpmap, LPMAP_MAPSIZE); + } +} + +/* + * upmap initialization code, _upmap_thr_init() is called for the initial + * main thread by libc or pthreads, and on every thread create. We need + * the __lpmap_blockallsigs pointer ASAP because it is used everywhere in + * pthreads. + * + * If pthreads is not linked in, _pthread_once() still runs via a stub in + * libc, and _pthread_key_create() is a NOP. + * + * NOTE: These pthreads calls are stubs when pthreads is not linked in. + * The once routine will still be run once regardless. + */ +static +void +_upmap_init_once(void) +{ + _pthread_key_create(&lpmap_key, (void (*)(void *))lpmap_unmap); +} + +void +_upmap_thr_init(void) +{ + int dummy_state = 0; + + _pthread_once(&upmap_once, _upmap_init_once); + __lpmap_map(&__lpmap_blockallsigs, &dummy_state, LPTYPE_BLOCKALLSIGS); +} diff --git a/lib/libc/upmap/upmap.h b/lib/libc/upmap/upmap.h index 99d2cc5b8a..5691ada46c 100644 --- a/lib/libc/upmap/upmap.h +++ b/lib/libc/upmap/upmap.h @@ -38,10 +38,15 @@ #ifndef _SYS_TYPES_H_ #include #endif +#ifndef _SYS_UPMAP_H_ +#include +#endif void __kpmap_map(void *datap, int *state, uint16_t type); void __upmap_map(void *datap, int *state, uint16_t type); +void __lpmap_map(void *datap, int *state, uint16_t type); int __ukp_spt(const char *fmt, va_list ap); - +extern __thread ukpheader_t *__lpmap_headers TLS_ATTRIBUTE; +extern __thread uint32_t *__lpmap_blockallsigs TLS_ATTRIBUTE; #endif diff --git a/lib/libstand/Makefile b/lib/libstand/Makefile index eb09d6f3bf..8aef675588 100644 --- a/lib/libstand/Makefile +++ b/lib/libstand/Makefile @@ -20,6 +20,8 @@ LIBSTAND_SRC?= ${.CURDIR} LIBSTAND_ARCH?= ${MACHINE_ARCH} LIBC_SRC= ${LIBSTAND_SRC}/../libc +CFLAGS+=-D_LIBSTAND_ + # Mostly OK, some of the libc imports are a bit noisy CFLAGS+= -ffreestanding diff --git a/lib/libthread_xu/thread/thr_create.c b/lib/libthread_xu/thread/thr_create.c index 271d589c6c..00b0451cb3 100644 --- a/lib/libthread_xu/thread/thr_create.c +++ b/lib/libthread_xu/thread/thr_create.c @@ -227,7 +227,7 @@ thread_start(void *arg) if (curthread->flags & THR_FLAGS_NEED_SUSPEND) _thr_suspend_check(curthread); - _nmalloc_thr_init(); + _libc_thr_init(); /* Run the current thread's start routine with argument: */ _pthread_exit(curthread->start_routine(curthread->arg)); diff --git a/lib/libthread_xu/thread/thr_init.c b/lib/libthread_xu/thread/thr_init.c index c505a3406c..636ecc946e 100644 --- a/lib/libthread_xu/thread/thr_init.c +++ b/lib/libthread_xu/thread/thr_init.c @@ -194,6 +194,7 @@ void _thread_init(void) { _libpthread_init(NULL); + _libc_thr_init(); } void _thread_uninit(void) __attribute__ ((destructor)); diff --git a/lib/libthread_xu/thread/thr_mutex.c b/lib/libthread_xu/thread/thr_mutex.c index d7c9b2c1d0..3ed9d08f69 100644 --- a/lib/libthread_xu/thread/thr_mutex.c +++ b/lib/libthread_xu/thread/thr_mutex.c @@ -302,7 +302,7 @@ _pthread_mutex_destroy(pthread_mutex_t *mutex) * Try to lock the mutex structure, we only need to * try once, if failed, the mutex is in use. */ - ret = THR_UMTX_TRYLOCK(curthread, &(*mutex)->m_lock); + ret = THR_UMTX_TRYLOCK_PERSIST(curthread, &(*mutex)->m_lock); if (ret) return (ret); @@ -314,7 +314,7 @@ _pthread_mutex_destroy(pthread_mutex_t *mutex) if (((*mutex)->m_owner != NULL) || (TAILQ_FIRST(&(*mutex)->m_queue) != NULL) || ((*mutex)->m_refcount != 0)) { - THR_UMTX_UNLOCK(curthread, &(*mutex)->m_lock); + THR_UMTX_UNLOCK_PERSIST(curthread, &(*mutex)->m_lock); ret = EBUSY; } else { /* @@ -325,7 +325,7 @@ _pthread_mutex_destroy(pthread_mutex_t *mutex) *mutex = NULL; /* Unlock the mutex structure: */ - THR_UMTX_UNLOCK(curthread, &m->m_lock); + THR_UMTX_UNLOCK_PERSIST(curthread, &m->m_lock); /* * Free the memory allocated for the mutex @@ -348,7 +348,7 @@ mutex_trylock_common(struct pthread *curthread, pthread_mutex_t *mutex) m = *mutex; mutex_log("mutex_lock_trylock_common %p\n", m); - ret = THR_UMTX_TRYLOCK(curthread, &m->m_lock); + ret = THR_UMTX_TRYLOCK_PERSIST(curthread, &m->m_lock); if (ret == 0) { mutex_log2(curthread, m, 1); m->m_owner = curthread; @@ -412,7 +412,7 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *mutex, m = *mutex; mutex_log("mutex_lock_common %p\n", m); - ret = THR_UMTX_TRYLOCK(curthread, &m->m_lock); + ret = THR_UMTX_TRYLOCK_PERSIST(curthread, &m->m_lock); if (ret == 0) { mutex_log2(curthread, m, 3); m->m_owner = curthread; @@ -423,7 +423,7 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *mutex, ret = mutex_self_lock(m, abstime); } else { if (abstime == NULL) { - THR_UMTX_LOCK(curthread, &m->m_lock); + THR_UMTX_LOCK_PERSIST(curthread, &m->m_lock); ret = 0; } else if (__predict_false( abstime->tv_sec < 0 || abstime->tv_nsec < 0 || @@ -432,7 +432,8 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *mutex, } else { clock_gettime(CLOCK_REALTIME, &ts); timespecsub(abstime, &ts, &ts2); - ret = THR_UMTX_TIMEDLOCK(curthread, &m->m_lock, &ts2); + ret = THR_UMTX_TIMEDLOCK_PERSIST(curthread, + &m->m_lock, &ts2); } if (ret == 0) { mutex_log2(curthread, m, 4); @@ -677,7 +678,7 @@ mutex_unlock_common(pthread_mutex_t *mutex) */ mutex_log("mutex_unlock_common %p (returns 0) lock %d\n", m, m->m_lock); - THR_UMTX_UNLOCK(curthread, &m->m_lock); + THR_UMTX_UNLOCK_PERSIST(curthread, &m->m_lock); mutex_log2(tls_get_curthread(), m, 37); mutex_log2(curthread, m, 255); } @@ -750,7 +751,7 @@ _mutex_cv_unlock(pthread_mutex_t *mutex, int *count) MUTEX_ASSERT_IS_OWNED(m); TAILQ_REMOVE(&curthread->mutexq, m, m_qe); MUTEX_INIT_LINK(m); - THR_UMTX_UNLOCK(curthread, &m->m_lock); + THR_UMTX_UNLOCK_PERSIST(curthread, &m->m_lock); mutex_log2(curthread, m, 250); return (0); } diff --git a/lib/libthread_xu/thread/thr_private.h b/lib/libthread_xu/thread/thr_private.h index 72e5afa9dc..afa4e8e5c3 100644 --- a/lib/libthread_xu/thread/thr_private.h +++ b/lib/libthread_xu/thread/thr_private.h @@ -486,22 +486,45 @@ struct pthread { (((thrd)->locklevel > 0) || \ ((thrd)->critical_count > 0)) +/* + * Internal temporary locks without suspend check + */ #define THR_UMTX_TRYLOCK(thrd, lck) \ - _thr_umtx_trylock((lck), (thrd)->tid) + _thr_umtx_trylock((lck), (thrd)->tid, 1) #define THR_UMTX_LOCK(thrd, lck) \ - _thr_umtx_lock((lck), (thrd)->tid) + _thr_umtx_lock((lck), (thrd)->tid, 1) #define THR_UMTX_TIMEDLOCK(thrd, lck, timo) \ - _thr_umtx_timedlock((lck), (thrd)->tid, (timo)) + _thr_umtx_timedlock((lck), (thrd)->tid, (timo), 1) #define THR_UMTX_UNLOCK(thrd, lck) \ - _thr_umtx_unlock((lck), (thrd)->tid) + _thr_umtx_unlock((lck), (thrd)->tid, 1) + +/* + * Interal locks without suspend check, used when the lock + * state needs to persist (i.e. to help implement things + * like pthread_mutex_lock()). Non-temporary. + */ +#define THR_UMTX_TRYLOCK_PERSIST(thrd, lck) \ + _thr_umtx_trylock((lck), (thrd)->tid, 0) +#define THR_UMTX_LOCK_PERSIST(thrd, lck) \ + _thr_umtx_lock((lck), (thrd)->tid, 0) + +#define THR_UMTX_TIMEDLOCK_PERSIST(thrd, lck, timo) \ + _thr_umtx_timedlock((lck), (thrd)->tid, (timo), 0) + +#define THR_UMTX_UNLOCK_PERSIST(thrd, lck) \ + _thr_umtx_unlock((lck), (thrd)->tid, 0) + +/* + * Internal temporary locks with suspend check + */ #define THR_LOCK_ACQUIRE(thrd, lck) \ do { \ (thrd)->locklevel++; \ - _thr_umtx_lock((lck), (thrd)->tid); \ + _thr_umtx_lock((lck), (thrd)->tid, 1); \ } while (0) #ifdef _PTHREADS_INVARIANTS @@ -517,7 +540,7 @@ do { \ #define THR_LOCK_RELEASE(thrd, lck) \ do { \ THR_ASSERT_LOCKLEVEL(thrd); \ - _thr_umtx_unlock((lck), (thrd)->tid); \ + _thr_umtx_unlock((lck), (thrd)->tid, 1); \ (thrd)->locklevel--; \ _thr_ast(thrd); \ } while (0) diff --git a/lib/libthread_xu/thread/thr_pspinlock.c b/lib/libthread_xu/thread/thr_pspinlock.c index 062abfe708..d1002e1f82 100644 --- a/lib/libthread_xu/thread/thr_pspinlock.c +++ b/lib/libthread_xu/thread/thr_pspinlock.c @@ -77,7 +77,7 @@ _pthread_spin_trylock(pthread_spinlock_t *lock) if (lock == NULL || (lck = *lock) == NULL) return (EINVAL); - return (THR_UMTX_TRYLOCK(curthread, &lck->s_lock)); + return (THR_UMTX_TRYLOCK_PERSIST(curthread, &lck->s_lock)); } int @@ -92,7 +92,7 @@ _pthread_spin_lock(pthread_spinlock_t *lock) curthread = tls_get_curthread(); count = SPIN_COUNT; - while (THR_UMTX_TRYLOCK(curthread, &lck->s_lock) != 0) { + while (THR_UMTX_TRYLOCK_PERSIST(curthread, &lck->s_lock) != 0) { while (lck->s_lock) { CPU_SPINWAIT; /* tell cpu we are spinning */ if (--count <= 0) { @@ -113,7 +113,7 @@ _pthread_spin_unlock(pthread_spinlock_t *lock) if (lock == NULL || (lck = *lock) == NULL) return (EINVAL); /* XXX: shouldn't return status? */ - THR_UMTX_UNLOCK(curthread, &lck->s_lock); + THR_UMTX_UNLOCK_PERSIST(curthread, &lck->s_lock); return (0); } diff --git a/lib/libthread_xu/thread/thr_spinlock.c b/lib/libthread_xu/thread/thr_spinlock.c index 7493092320..92c5b119dc 100644 --- a/lib/libthread_xu/thread/thr_spinlock.c +++ b/lib/libthread_xu/thread/thr_spinlock.c @@ -55,10 +55,10 @@ static int initialized; static void init_spinlock(spinlock_t *lck); /* - * These are for compatibility only. Spinlocks of this type - * are deprecated. + * These functions implement temporary spinlocks as used by libc. + * these are not persistent spinlocks, so we use the API for + * temporary locks which block all signals for the duration. */ - void _spinunlock(spinlock_t *lck) { diff --git a/lib/libthread_xu/thread/thr_umtx.h b/lib/libthread_xu/thread/thr_umtx.h index f048d1cbbb..5b8d7bdff1 100644 --- a/lib/libthread_xu/thread/thr_umtx.h +++ b/lib/libthread_xu/thread/thr_umtx.h @@ -49,8 +49,10 @@ _thr_umtx_init(volatile umtx_t *mtx) } static inline int -_thr_umtx_trylock(volatile umtx_t *mtx, int id) +_thr_umtx_trylock(volatile umtx_t *mtx, int id, int temporary) { + if (temporary) + sigblockall(); if (atomic_cmpset_acq_int(mtx, 0, id)) return (0); cpu_pause(); @@ -59,35 +61,53 @@ _thr_umtx_trylock(volatile umtx_t *mtx, int id) cpu_pause(); if (atomic_cmpset_acq_int(mtx, 0, id)) return (0); + if (temporary) + sigunblockall(); return (EBUSY); } static inline int -_thr_umtx_lock(volatile umtx_t *mtx, int id) +_thr_umtx_lock(volatile umtx_t *mtx, int id, int temporary) { + int res; + + if (temporary) + sigblockall(); if (atomic_cmpset_acq_int(mtx, 0, id)) return (0); - return (__thr_umtx_lock(mtx, id, 0)); + res = __thr_umtx_lock(mtx, id, 0); + if (res && temporary) + sigunblockall(); + return res; } static inline int _thr_umtx_timedlock(volatile umtx_t *mtx, int id, - const struct timespec *timeout) + const struct timespec *timeout, int temporary) { + int res; + + if (temporary) + sigblockall(); if (atomic_cmpset_acq_int(mtx, 0, id)) { return (0); } - return (__thr_umtx_timedlock(mtx, id, timeout)); + res = __thr_umtx_timedlock(mtx, id, timeout); + if (res && temporary) + sigunblockall(); + return res; } static inline void -_thr_umtx_unlock(volatile umtx_t *mtx, int id) +_thr_umtx_unlock(volatile umtx_t *mtx, int id, int temporary) { int v; v = atomic_swap_int(mtx, 0); if (v != id) __thr_umtx_unlock(mtx, v, id); + if (temporary) + sigunblockall(); } int _thr_umtx_wait(volatile umtx_t *mtx, umtx_t exp, diff --git a/libexec/rtld-elf/rtld.c b/libexec/rtld-elf/rtld.c index 1c237bdc06..207a741ed5 100644 --- a/libexec/rtld-elf/rtld.c +++ b/libexec/rtld-elf/rtld.c @@ -2894,6 +2894,8 @@ search_library_pathfds(const char *name, const char *path, int *fdp) * Use strtok_r() to walk the FD:FD:FD list. This requires a local * copy of the path, as strtok_r rewrites separator tokens * with '\0'. + * + * NOTE: strtok() uses a __thread static and cannot be used by rtld. */ found = NULL; envcopy = xstrdup(path); -- 2.41.0