From e2b148c60d04976ee2f7c03874a91216c8c5b4c1 Mon Sep 17 00:00:00 2001 From: Matthew Dillon Date: Tue, 1 Nov 2011 23:38:42 -0700 Subject: [PATCH] kernel - Major MP work on kq and signal processing * Remove the global kq_token and move to a per-kq and per-kqlist pool token. This greatly improves postgresql and mysql performance on MP systems. * Adjust signal processing tokens to be per-LWP instead of per-PROC. Signal delivery still utilizes a per-proc token but signal distribution now uses a per-LWP token, which allows the trap code to only lock the LWP when checking for pending signals. This also significantly improves database performance. * The socket code also now uses only its per-socket pool token instead of kq_token for its kq operations. kq handles its own tokens. --- sys/kern/init_main.c | 1 + sys/kern/kern_event.c | 132 +++++++++++++++++++++++++++++++++------- sys/kern/kern_exit.c | 2 + sys/kern/kern_fork.c | 1 + sys/kern/kern_sig.c | 120 +++++++++++++++++++++++++++++++----- sys/kern/kern_synch.c | 19 +++--- sys/kern/uipc_msg.c | 16 +++-- sys/kern/uipc_socket2.c | 2 - sys/sys/event.h | 2 +- sys/sys/eventvar.h | 3 + sys/sys/proc.h | 1 + 11 files changed, 249 insertions(+), 50 deletions(-) diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c index 3452b44f8f..0b9d6e0921 100644 --- a/sys/kern/init_main.c +++ b/sys/kern/init_main.c @@ -176,6 +176,7 @@ mi_proc0init(struct globaldata *gd, struct user *proc0paddr) lwp0.lwp_proc = &proc0; proc0.p_usched = usched_init(); lwp0.lwp_cpumask = (cpumask_t)-1; + lwkt_token_init(&lwp0.lwp_token, "lwp_token"); varsymset_init(&proc0.p_varsymset, NULL); thread0.td_flags |= TDF_RUNNING; thread0.td_proc = &proc0; diff --git a/sys/kern/kern_event.c b/sys/kern/kern_event.c index f475cf13a1..63aa5d19a0 100644 --- a/sys/kern/kern_event.c +++ b/sys/kern/kern_event.c @@ -58,10 +58,12 @@ /* * Global token for kqueue subsystem */ +#if 0 struct lwkt_token kq_token = LWKT_TOKEN_INITIALIZER(kq_token); SYSCTL_LONG(_lwkt, OID_AUTO, kq_collisions, CTLFLAG_RW, &kq_token.t_collisions, 0, "Collision counter of kq_token"); +#endif MALLOC_DEFINE(M_KQUEUE, "kqueue", "memory for kqueue system"); @@ -338,12 +340,14 @@ filt_proc(struct knote *kn, long hint) static void filt_timerexpire(void *knx) { + struct lwkt_token *tok; struct knote *kn = knx; struct callout *calloutp; struct timeval tv; int tticks; - lwkt_gettoken(&kq_token); + tok = lwkt_token_pool_lookup(kn->kn_kq); + lwkt_gettoken(tok); if ((kn->kn_status & KN_DELETING) == 0) { kn->kn_data++; KNOTE_ACTIVATE(kn); @@ -356,7 +360,7 @@ filt_timerexpire(void *knx) callout_reset(calloutp, tticks, filt_timerexpire, kn); } } - lwkt_reltoken(&kq_token); + lwkt_reltoken(tok); } /* @@ -419,6 +423,8 @@ filt_timer(struct knote *kn, long hint) * If we cannot acquire the knote we sleep and return 0. The knote * may be stale on return in this case and the caller must restart * whatever loop they are in. + * + * Related kq token must be held. */ static __inline int @@ -438,6 +444,8 @@ knote_acquire(struct knote *kn) * Release an acquired knote, clearing KN_PROCESSING and handling any * KN_REPROCESS events. * + * Caller must be holding the related kq token + * * Non-zero is returned if the knote is destroyed. */ static __inline @@ -489,9 +497,11 @@ kqueue_init(struct kqueue *kq, struct filedesc *fdp) void kqueue_terminate(struct kqueue *kq) { + struct lwkt_token *tok; struct knote *kn; - lwkt_gettoken(&kq_token); + tok = lwkt_token_pool_lookup(kq); + lwkt_gettoken(tok); while ((kn = TAILQ_FIRST(&kq->kq_knlist)) != NULL) { if (knote_acquire(kn)) knote_detach_and_drop(kn); @@ -501,7 +511,7 @@ kqueue_terminate(struct kqueue *kq) kq->kq_knhash = NULL; kq->kq_knhashmask = 0; } - lwkt_reltoken(&kq_token); + lwkt_reltoken(tok); } /* @@ -592,11 +602,13 @@ kern_kevent(struct kqueue *kq, int nevents, int *res, void *uap, int limit = kq_checkloop; struct kevent kev[KQ_NEVENTS]; struct knote marker; + struct lwkt_token *tok; tsp = tsp_in; *res = 0; - lwkt_gettoken(&kq_token); + tok = lwkt_token_pool_lookup(kq); + lwkt_gettoken(tok); for ( ;; ) { n = 0; error = kevent_copyinfn(uap, kev, KQ_NEVENTS, &n); @@ -741,7 +753,7 @@ kern_kevent(struct kqueue *kq, int nevents, int *res, void *uap, error = 0; done: - lwkt_reltoken(&kq_token); + lwkt_reltoken(tok); return (error); } @@ -790,9 +802,13 @@ sys_kevent(struct kevent_args *uap) return (error); } +/* + * Caller must be holding the kq token + */ int kqueue_register(struct kqueue *kq, struct kevent *kev) { + struct lwkt_token *tok; struct filedesc *fdp = kq->kq_fdp; struct filterops *fops; struct file *fp = NULL; @@ -813,15 +829,16 @@ kqueue_register(struct kqueue *kq, struct kevent *kev) return (EINVAL); } - lwkt_gettoken(&kq_token); + tok = lwkt_token_pool_lookup(kq); + lwkt_gettoken(tok); if (fops->f_flags & FILTEROP_ISFD) { /* validate descriptor */ fp = holdfp(fdp, kev->ident, -1); if (fp == NULL) { - lwkt_reltoken(&kq_token); + lwkt_reltoken(tok); return (EBADF); } - + lwkt_getpooltoken(&fp->f_klist); again1: SLIST_FOREACH(kn, &fp->f_klist, kn_link) { if (kn->kn_kq == kq && @@ -832,12 +849,14 @@ again1: break; } } + lwkt_relpooltoken(&fp->f_klist); } else { if (kq->kq_knhashmask) { struct klist *list; list = &kq->kq_knhash[ KN_HASH((u_long)kev->ident, kq->kq_knhashmask)]; + lwkt_getpooltoken(list); again2: SLIST_FOREACH(kn, list, kn_link) { if (kn->kn_id == kev->ident && @@ -847,6 +866,7 @@ again2: break; } } + lwkt_relpooltoken(list); } } @@ -965,7 +985,7 @@ again2: /* kn may be invalid now */ done: - lwkt_reltoken(&kq_token); + lwkt_reltoken(tok); if (fp != NULL) fdrop(fp); return (error); @@ -975,6 +995,8 @@ done: * Block as necessary until the target time is reached. * If tsp is NULL we block indefinitely. If tsp->ts_secs/nsecs are both * 0 we do not block at all. + * + * Caller must be holding the kq token. */ static int kqueue_sleep(struct kqueue *kq, struct timespec *tsp) @@ -1016,6 +1038,8 @@ kqueue_sleep(struct kqueue *kq, struct timespec *tsp) * * Continuous mode events may get recycled, do not continue scanning past * marker unless no events have been collected. + * + * Caller must be holding the kq token */ static int kqueue_scan(struct kqueue *kq, struct kevent *kevp, int count, @@ -1159,11 +1183,13 @@ static int kqueue_ioctl(struct file *fp, u_long com, caddr_t data, struct ucred *cred, struct sysmsg *msg) { + struct lwkt_token *tok; struct kqueue *kq; int error; - lwkt_gettoken(&kq_token); kq = (struct kqueue *)fp->f_data; + tok = lwkt_token_pool_lookup(kq); + lwkt_gettoken(tok); switch(com) { case FIOASYNC: @@ -1180,7 +1206,7 @@ kqueue_ioctl(struct file *fp, u_long com, caddr_t data, error = ENOTTY; break; } - lwkt_reltoken(&kq_token); + lwkt_reltoken(tok); return (error); } @@ -1229,6 +1255,8 @@ kqueue_wakeup(struct kqueue *kq) /* * Calls filterops f_attach function, acquiring mplock if filter is not * marked as FILTEROP_MPSAFE. + * + * Caller must be holding the related kq token */ static int filter_attach(struct knote *kn) @@ -1251,6 +1279,8 @@ filter_attach(struct knote *kn) * * Calls filterops f_detach function, acquiring mplock if filter is not * marked as FILTEROP_MPSAFE. + * + * Caller must be holding the related kq token */ static void knote_detach_and_drop(struct knote *kn) @@ -1272,6 +1302,8 @@ knote_detach_and_drop(struct knote *kn) * * If the knote is in the middle of being created or deleted we cannot * safely call the filter op. + * + * Caller must be holding the related kq token */ static int filter_event(struct knote *kn, long hint) @@ -1299,11 +1331,26 @@ filter_event(struct knote *kn, long hint) void knote(struct klist *list, long hint) { + struct kqueue *kq; struct knote *kn; + struct knote *kntmp; - lwkt_gettoken(&kq_token); + lwkt_getpooltoken(list); restart: SLIST_FOREACH(kn, list, kn_next) { + kq = kn->kn_kq; + lwkt_getpooltoken(kq); + + /* temporary verification hack */ + SLIST_FOREACH(kntmp, list, kn_next) { + if (kn == kntmp) + break; + } + if (kn != kntmp || kn->kn_kq != kq) { + lwkt_relpooltoken(kq); + goto restart; + } + if (kn->kn_status & KN_PROCESSING) { /* * Someone else is processing the knote, ask the @@ -1312,6 +1359,7 @@ restart: */ if (hint == 0) { kn->kn_status |= KN_REPROCESS; + lwkt_relpooltoken(kq); continue; } @@ -1326,6 +1374,7 @@ restart: */ kn->kn_status |= KN_WAITING | KN_REPROCESS; tsleep(kn, 0, "knotec", hz); + lwkt_relpooltoken(kq); goto restart; } @@ -1341,10 +1390,13 @@ restart: if (filter_event(kn, hint)) KNOTE_ACTIVATE(kn); } - if (knote_release(kn)) + if (knote_release(kn)) { + lwkt_relpooltoken(kq); goto restart; + } + lwkt_relpooltoken(kq); } - lwkt_reltoken(&kq_token); + lwkt_relpooltoken(list); } /* @@ -1356,9 +1408,10 @@ restart: void knote_insert(struct klist *klist, struct knote *kn) { + lwkt_getpooltoken(klist); KKASSERT(kn->kn_status & KN_PROCESSING); - ASSERT_LWKT_TOKEN_HELD(&kq_token); SLIST_INSERT_HEAD(klist, kn, kn_next); + lwkt_relpooltoken(klist); } /* @@ -1370,11 +1423,13 @@ knote_insert(struct klist *klist, struct knote *kn) void knote_remove(struct klist *klist, struct knote *kn) { + lwkt_getpooltoken(klist); KKASSERT(kn->kn_status & KN_PROCESSING); - ASSERT_LWKT_TOKEN_HELD(&kq_token); SLIST_REMOVE(klist, kn, knote, kn_next); + lwkt_relpooltoken(klist); } +#if 0 /* * Remove all knotes from a specified klist * @@ -1392,15 +1447,24 @@ knote_empty(struct klist *list) } lwkt_reltoken(&kq_token); } +#endif void knote_assume_knotes(struct kqinfo *src, struct kqinfo *dst, struct filterops *ops, void *hook) { + struct kqueue *kq; struct knote *kn; - lwkt_gettoken(&kq_token); + lwkt_getpooltoken(&src->ki_note); + lwkt_getpooltoken(&dst->ki_note); while ((kn = SLIST_FIRST(&src->ki_note)) != NULL) { + kq = kn->kn_kq; + lwkt_getpooltoken(kq); + if (SLIST_FIRST(&src->ki_note) != kn || kn->kn_kq != kq) { + lwkt_relpooltoken(kq); + continue; + } if (knote_acquire(kn)) { knote_remove(&src->ki_note, kn); kn->kn_fop = ops; @@ -1409,8 +1473,10 @@ knote_assume_knotes(struct kqinfo *src, struct kqinfo *dst, knote_release(kn); /* kn may be invalid now */ } + lwkt_relpooltoken(kq); } - lwkt_reltoken(&kq_token); + lwkt_relpooltoken(&dst->ki_note); + lwkt_relpooltoken(&src->ki_note); } /* @@ -1419,24 +1485,41 @@ knote_assume_knotes(struct kqinfo *src, struct kqinfo *dst, void knote_fdclose(struct file *fp, struct filedesc *fdp, int fd) { + struct kqueue *kq; struct knote *kn; + struct knote *kntmp; - lwkt_gettoken(&kq_token); + lwkt_getpooltoken(&fp->f_klist); restart: SLIST_FOREACH(kn, &fp->f_klist, kn_link) { if (kn->kn_kq->kq_fdp == fdp && kn->kn_id == fd) { + kq = kn->kn_kq; + lwkt_getpooltoken(kq); + + /* temporary verification hack */ + SLIST_FOREACH(kntmp, &fp->f_klist, kn_link) { + if (kn == kntmp) + break; + } + if (kn != kntmp || kn->kn_kq->kq_fdp != fdp || + kn->kn_id != fd || kn->kn_kq != kq) { + lwkt_relpooltoken(kq); + goto restart; + } if (knote_acquire(kn)) knote_detach_and_drop(kn); + lwkt_relpooltoken(kq); goto restart; } } - lwkt_reltoken(&kq_token); + lwkt_relpooltoken(&fp->f_klist); } /* * Low level attach function. * * The knote should already be marked for processing. + * Caller must hold the related kq token. */ static void knote_attach(struct knote *kn) @@ -1453,14 +1536,17 @@ knote_attach(struct knote *kn) &kq->kq_knhashmask); list = &kq->kq_knhash[KN_HASH(kn->kn_id, kq->kq_knhashmask)]; } + lwkt_getpooltoken(list); SLIST_INSERT_HEAD(list, kn, kn_link); TAILQ_INSERT_HEAD(&kq->kq_knlist, kn, kn_kqlink); + lwkt_relpooltoken(list); } /* * Low level drop function. * * The knote should already be marked for processing. + * Caller must hold the related kq token. */ static void knote_drop(struct knote *kn) @@ -1475,6 +1561,7 @@ knote_drop(struct knote *kn) else list = &kq->kq_knhash[KN_HASH(kn->kn_id, kq->kq_knhashmask)]; + lwkt_getpooltoken(list); SLIST_REMOVE(list, kn, knote, kn_link); TAILQ_REMOVE(&kq->kq_knlist, kn, kn_kqlink); if (kn->kn_status & KN_QUEUED) @@ -1484,12 +1571,14 @@ knote_drop(struct knote *kn) kn->kn_fp = NULL; } knote_free(kn); + lwkt_relpooltoken(list); } /* * Low level enqueue function. * * The knote should already be marked for processing. + * Caller must be holding the kq token */ static void knote_enqueue(struct knote *kn) @@ -1514,6 +1603,7 @@ knote_enqueue(struct knote *kn) * Low level dequeue function. * * The knote should already be marked for processing. + * Caller must be holding the kq token */ static void knote_dequeue(struct knote *kn) diff --git a/sys/kern/kern_exit.c b/sys/kern/kern_exit.c index f3916580ca..053711f9db 100644 --- a/sys/kern/kern_exit.c +++ b/sys/kern/kern_exit.c @@ -246,10 +246,12 @@ killlwps(struct lwp *lp) */ FOREACH_LWP_IN_PROC(tlp, p) { LWPHOLD(tlp); + lwkt_gettoken(&tlp->lwp_token); if ((tlp->lwp_flag & LWP_WEXIT) == 0) { lwpsignal(p, tlp, SIGKILL); tlp->lwp_flag |= LWP_WEXIT; } + lwkt_reltoken(&tlp->lwp_token); LWPRELE(tlp); } diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c index cc0edff7a3..89a2bd1e54 100644 --- a/sys/kern/kern_fork.c +++ b/sys/kern/kern_fork.c @@ -632,6 +632,7 @@ lwp_fork(struct lwp *origlp, struct proc *destproc, int flags) destproc->p_usched->heuristic_forking(origlp, lp); crit_exit(); lp->lwp_cpumask &= usched_mastermask; + lwkt_token_init(&lp->lwp_token, "lwp_token"); /* * Assign the thread to the current cpu to begin with so we diff --git a/sys/kern/kern_sig.c b/sys/kern/kern_sig.c index b1c1c5efbd..3ff4400b1d 100644 --- a/sys/kern/kern_sig.c +++ b/sys/kern/kern_sig.c @@ -351,7 +351,11 @@ kern_sigaction(int sig, struct sigaction *act, struct sigaction *oact) * Remove the signal also from the thread lists. */ FOREACH_LWP_IN_PROC(lp, p) { + LWPHOLD(lp); + lwkt_gettoken(&lp->lwp_token); SIGDELSET(lp->lwp_siglist, sig); + lwkt_reltoken(&lp->lwp_token); + LWPRELE(lp); } if (sig != SIGCONT) { /* easier in ksignal */ @@ -950,6 +954,9 @@ trapsignal(struct lwp *lp, int sig, u_long code) * lwps hold the signal blocked. * * Caller must hold p->p_token. + * + * Returns a lp or NULL. If non-NULL the lp is held and its token is + * acquired. */ static struct lwp * find_lwp_for_signal(struct proc *p, int sig) @@ -965,9 +972,15 @@ find_lwp_for_signal(struct proc *p, int sig) * soon anyways. */ lp = lwkt_preempted_proc(); - if (lp != NULL && lp->lwp_proc == p && - !SIGISMEMBER(lp->lwp_sigmask, sig)) { - return (lp); + if (lp != NULL && lp->lwp_proc == p) { + LWPHOLD(lp); + lwkt_gettoken(&lp->lwp_token); + if (!SIGISMEMBER(lp->lwp_sigmask, sig)) { + /* return w/ token held */ + return (lp); + } + lwkt_reltoken(&lp->lwp_token); + LWPRELE(lp); } run = sleep = stop = NULL; @@ -976,23 +989,66 @@ find_lwp_for_signal(struct proc *p, int sig) * If the signal is being blocked by the lwp, then this * lwp is not eligible for receiving the signal. */ - if (SIGISMEMBER(lp->lwp_sigmask, sig)) + LWPHOLD(lp); + lwkt_gettoken(&lp->lwp_token); + + if (SIGISMEMBER(lp->lwp_sigmask, sig)) { + lwkt_reltoken(&lp->lwp_token); + LWPRELE(lp); continue; + } switch (lp->lwp_stat) { case LSRUN: - run = lp; - break; - - case LSSTOP: - stop = lp; + if (sleep) { + lwkt_token_swap(); + lwkt_reltoken(&sleep->lwp_token); + LWPRELE(sleep); + sleep = NULL; + run = lp; + } else if (stop) { + lwkt_token_swap(); + lwkt_reltoken(&stop->lwp_token); + LWPRELE(stop); + stop = NULL; + run = lp; + } else { + run = lp; + } break; - case LSSLEEP: - if (lp->lwp_flag & LWP_SINTR) - sleep = lp; + if (lp->lwp_flag & LWP_SINTR) { + if (sleep) { + lwkt_reltoken(&lp->lwp_token); + LWPRELE(lp); + } else if (stop) { + lwkt_token_swap(); + lwkt_reltoken(&stop->lwp_token); + LWPRELE(stop); + stop = NULL; + sleep = lp; + } else { + sleep = lp; + } + } else { + lwkt_reltoken(&lp->lwp_token); + LWPRELE(lp); + } + break; + case LSSTOP: + if (sleep) { + lwkt_reltoken(&lp->lwp_token); + LWPRELE(lp); + } else if (stop) { + lwkt_reltoken(&lp->lwp_token); + LWPRELE(lp); + } else { + stop = lp; + } break; } + if (run) + break; } if (run != NULL) @@ -1052,6 +1108,10 @@ lwpsignal(struct proc *p, struct lwp *lp, int sig) PHOLD(p); lwkt_gettoken(&p->p_token); + if (lp) { + LWPHOLD(lp); + lwkt_gettoken(&lp->lwp_token); + } prop = sigprop(sig); @@ -1069,6 +1129,10 @@ lwpsignal(struct proc *p, struct lwp *lp, int sig) * in the process flags. */ if (lp && (lp->lwp_flag & LWP_WEXIT)) { + if (lp) { + lwkt_reltoken(&lp->lwp_token); + LWPRELE(lp); + } lwkt_reltoken(&p->p_token); PRELE(p); return; @@ -1085,6 +1149,10 @@ lwpsignal(struct proc *p, struct lwp *lp, int sig) * lurking in a kqueue. */ KNOTE(&p->p_klist, NOTE_SIGNAL | sig); + if (lp) { + lwkt_reltoken(&lp->lwp_token); + LWPRELE(lp); + } lwkt_reltoken(&p->p_token); PRELE(p); return; @@ -1204,8 +1272,10 @@ lwpsignal(struct proc *p, struct lwp *lp, int sig) * so that the current signal will break the sleep * as soon as a SA_CONT signal will unstop the process. */ - if (lp == NULL) + if (lp == NULL) { + /* NOTE: returns lp w/ token held */ lp = find_lwp_for_signal(p, sig); + } if (lp != NULL && (lp->lwp_stat == LSSLEEP || lp->lwp_stat == LSSTOP)) lp->lwp_flag |= LWP_BREAKTSLEEP; @@ -1220,9 +1290,15 @@ active_process: * Never deliver a lwp-specific signal to a random lwp. */ if (lp == NULL) { + /* NOTE: returns lp w/ token held */ lp = find_lwp_for_signal(p, sig); - if (lp && SIGISMEMBER(lp->lwp_sigmask, sig)) - lp = NULL; + if (lp) { + if (SIGISMEMBER(lp->lwp_sigmask, sig)) { + lwkt_reltoken(&lp->lwp_token); + LWPRELE(lp); + lp = NULL; + } + } } /* @@ -1283,6 +1359,10 @@ active_process: lwp_signotify(lp); out: + if (lp) { + lwkt_reltoken(&lp->lwp_token); + LWPRELE(lp); + } lwkt_reltoken(&p->p_token); PRELE(p); crit_exit(); @@ -1428,6 +1508,9 @@ proc_stop(struct proc *p) p->p_stat = SSTOP; FOREACH_LWP_IN_PROC(lp, p) { + LWPHOLD(lp); + lwkt_gettoken(&lp->lwp_token); + switch (lp->lwp_stat) { case LSSTOP: /* @@ -1458,6 +1541,8 @@ proc_stop(struct proc *p) lwp_signotify(lp); break; } + lwkt_reltoken(&lp->lwp_token); + LWPRELE(lp); } if (p->p_nstopped == p->p_nthreads) { @@ -1497,6 +1582,9 @@ proc_unstop(struct proc *p) p->p_stat = SACTIVE; FOREACH_LWP_IN_PROC(lp, p) { + LWPHOLD(lp); + lwkt_gettoken(&lp->lwp_token); + switch (lp->lwp_stat) { case LSRUN: /* @@ -1532,6 +1620,8 @@ proc_unstop(struct proc *p) break; } + lwkt_reltoken(&lp->lwp_token); + LWPRELE(lp); } crit_exit(); } diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c index 30e4c07fb6..b5e495592c 100644 --- a/sys/kern/kern_synch.c +++ b/sys/kern/kern_synch.c @@ -573,7 +573,7 @@ tsleep(const volatile void *ident, int flags, const char *wmesg, int timo) * Early termination only occurs when tsleep() is * entered while in a normal LSRUN state. */ - lwkt_gettoken(&p->p_token); + lwkt_gettoken(&lp->lwp_token); if ((sig = CURSIG(lp)) != 0) goto resume; @@ -715,7 +715,7 @@ tsleep(const volatile void *ident, int flags, const char *wmesg, int timo) * that it wishes to interlock a mailbox signal against since * the flag is cleared on *any* system call that sleeps. * - * p->p_token is held in the p && catch case. + * lp->lwp_token is held in the lwp && catch case. */ resume: if (p) { @@ -729,10 +729,15 @@ resume: error = ERESTART; } } + lp->lwp_flag &= ~(LWP_BREAKTSLEEP | LWP_SINTR); if (catch) + lwkt_reltoken(&lp->lwp_token); + if (p->p_flag & P_MAILBOX) { + lwkt_gettoken(&p->p_token); + p->p_flag &= ~P_MAILBOX; lwkt_reltoken(&p->p_token); - lp->lwp_flag &= ~(LWP_BREAKTSLEEP | LWP_SINTR); - p->p_flag &= ~P_MAILBOX; + } + } logtsleep1(tsleep_end); crit_exit_quick(td); @@ -889,7 +894,7 @@ endtsleep(void *arg) * This can block */ if ((lp = td->td_lwp) != NULL) - lwkt_gettoken(&lp->lwp_proc->p_token); + lwkt_gettoken(&lp->lwp_token); /* * Only do nominal wakeup processing if TDF_TIMEOUT and @@ -908,7 +913,7 @@ endtsleep(void *arg) } } if (lp) - lwkt_reltoken(&lp->lwp_proc->p_token); + lwkt_reltoken(&lp->lwp_token); lwkt_rele(td); crit_exit(); } @@ -1103,7 +1108,7 @@ wakeup_domain_one(const volatile void *ident, int domain) void setrunnable(struct lwp *lp) { - ASSERT_LWKT_TOKEN_HELD(&lp->lwp_proc->p_token); + ASSERT_LWKT_TOKEN_HELD(&lp->lwp_token); crit_enter(); if (lp->lwp_stat == LSSTOP) lp->lwp_stat = LSSLEEP; diff --git a/sys/kern/uipc_msg.c b/sys/kern/uipc_msg.c index a81285fe4d..791c42b4f3 100644 --- a/sys/kern/uipc_msg.c +++ b/sys/kern/uipc_msg.c @@ -430,6 +430,7 @@ so_pru_ctlinput(struct protosw *pr, int cmd, struct sockaddr *arg, void *extra) void netmsg_so_notify(netmsg_t msg) { + struct lwkt_token *tok; struct signalsockbuf *ssb; ssb = (msg->notify.nm_etype & NM_REVENT) ? @@ -439,15 +440,20 @@ netmsg_so_notify(netmsg_t msg) /* * Reply immediately if the event has occured, otherwise queue the * request. + * + * NOTE: Socket can change if this is an accept predicate so cache + * the token. */ + tok = lwkt_token_pool_lookup(msg->base.nm_so); + lwkt_gettoken(tok); if (msg->notify.nm_predicate(&msg->notify)) { + lwkt_reltoken(tok); lwkt_replymsg(&msg->base.lmsg, msg->base.lmsg.ms_error); } else { - lwkt_gettoken(&kq_token); TAILQ_INSERT_TAIL(&ssb->ssb_kq.ki_mlist, &msg->notify, nm_list); atomic_set_int(&ssb->ssb_flags, SSB_MEVENT); - lwkt_reltoken(&kq_token); + lwkt_reltoken(tok); } } @@ -504,14 +510,16 @@ netmsg_so_notify_abort(netmsg_t msg) * The original notify message is not destroyed until after the * abort request is returned, so we can check its state. */ + lwkt_getpooltoken(nmsg->base.nm_so); if ((nmsg->base.lmsg.ms_flags & (MSGF_DONE | MSGF_REPLY)) == 0) { ssb = (nmsg->nm_etype & NM_REVENT) ? &nmsg->base.nm_so->so_rcv : &nmsg->base.nm_so->so_snd; - lwkt_gettoken(&kq_token); TAILQ_REMOVE(&ssb->ssb_kq.ki_mlist, nmsg, nm_list); - lwkt_reltoken(&kq_token); + lwkt_relpooltoken(nmsg->base.nm_so); lwkt_replymsg(&nmsg->base.lmsg, EINTR); + } else { + lwkt_relpooltoken(nmsg->base.nm_so); } /* diff --git a/sys/kern/uipc_socket2.c b/sys/kern/uipc_socket2.c index e546b91780..d0d7cdf122 100644 --- a/sys/kern/uipc_socket2.c +++ b/sys/kern/uipc_socket2.c @@ -507,7 +507,6 @@ sowakeup(struct socket *so, struct signalsockbuf *ssb) if (ssb->ssb_flags & SSB_MEVENT) { struct netmsg_so_notify *msg, *nmsg; - lwkt_gettoken(&kq_token); lwkt_getpooltoken(so); TAILQ_FOREACH_MUTABLE(msg, &kqinfo->ki_mlist, nm_list, nmsg) { if (msg->nm_predicate(msg)) { @@ -519,7 +518,6 @@ sowakeup(struct socket *so, struct signalsockbuf *ssb) if (TAILQ_EMPTY(&ssb->ssb_kq.ki_mlist)) atomic_clear_int(&ssb->ssb_flags, SSB_MEVENT); lwkt_relpooltoken(so); - lwkt_reltoken(&kq_token); } } diff --git a/sys/sys/event.h b/sys/sys/event.h index c39a6e7155..334d414173 100644 --- a/sys/sys/event.h +++ b/sys/sys/event.h @@ -229,7 +229,7 @@ int kern_kevent(struct kqueue *kq, int nevents, int *res, void *uap, extern void knote(struct klist *list, long hint); extern void knote_insert(struct klist *klist, struct knote *kn); extern void knote_remove(struct klist *klist, struct knote *kn); -extern void knote_empty(struct klist *list); +/*extern void knote_empty(struct klist *list);*/ extern void knote_assume_knotes(struct kqinfo *, struct kqinfo *, struct filterops *, void *); extern void knote_fdclose(struct file *fp, struct filedesc *fdp, int fd); diff --git a/sys/sys/eventvar.h b/sys/sys/eventvar.h index bba0d27703..50e7e9bf7a 100644 --- a/sys/sys/eventvar.h +++ b/sys/sys/eventvar.h @@ -42,6 +42,9 @@ #ifndef _SYS_EVENT_H_ #include #endif +#ifndef _SYS_THREAD_H_ +#include +#endif #ifndef _SYS_FILEDESC_H_ #include #endif diff --git a/sys/sys/proc.h b/sys/sys/proc.h index b82beca6cf..af776c097b 100644 --- a/sys/sys/proc.h +++ b/sys/sys/proc.h @@ -220,6 +220,7 @@ struct lwp { struct upcall *lwp_upcall; /* REGISTERED USERLAND POINTER! */ struct kqueue lwp_kqueue; /* for select/poll */ u_int lwp_kqueue_serial; + struct lwkt_token lwp_token; /* per-lwp token for signal/state */ }; struct proc { -- 2.41.0