X-Git-Url: https://gitweb.dragonflybsd.org/dragonfly.git/blobdiff_plain/002c1265bf9676475d08e73e8d873cea9ee4a82e..3e184884618d66845f8b90e6dae483155da6dce6:/sys/net/netisr.c diff --git a/sys/net/netisr.c b/sys/net/netisr.c index 911aba6366..03d53dc598 100644 --- a/sys/net/netisr.c +++ b/sys/net/netisr.c @@ -34,8 +34,6 @@ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. - * - * $DragonFly: src/sys/net/netisr.c,v 1.49 2008/11/01 10:29:31 sephe Exp $ */ #include @@ -52,15 +50,16 @@ #include #include #include +#include #include #include #include #include -static void netmsg_sync_func(netmsg_t msg); static void netmsg_service_loop(void *arg); static void cpu0_cpufn(struct mbuf **mp, int hoff); +static void netisr_nohashck(struct mbuf *, const struct pktinfo *); struct netmsg_port_registration { TAILQ_ENTRY(netmsg_port_registration) npr_entry; @@ -72,6 +71,20 @@ struct netmsg_rollup { netisr_ru_t ru_func; }; +struct netmsg_barrier { + struct netmsg_base base; + volatile cpumask_t *br_cpumask; + volatile uint32_t br_done; +}; + +#define NETISR_BR_NOTDONE 0x1 +#define NETISR_BR_WAITDONE 0x80000000 + +struct netisr_barrier { + struct netmsg_barrier *br_msgs[MAXCPU]; + int br_isset; +}; + static struct netisr netisrs[NETISR_MAX]; static TAILQ_HEAD(,netmsg_port_registration) netreglist; static TAILQ_HEAD(,netmsg_rollup) netrulist; @@ -79,6 +92,7 @@ static TAILQ_HEAD(,netmsg_rollup) netrulist; /* Per-CPU thread to handle any protocol. */ static struct thread netisr_cpu[MAXCPU]; lwkt_port netisr_afree_rport; +lwkt_port netisr_afree_free_so_rport; lwkt_port netisr_adone_rport; lwkt_port netisr_apanic_rport; lwkt_port netisr_sync_port; @@ -97,6 +111,13 @@ netisr_autofree_reply(lwkt_port_t port, lwkt_msg_t msg) kfree(msg, M_LWKTMSG); } +static void +netisr_autofree_free_so_reply(lwkt_port_t port, lwkt_msg_t msg) +{ + sofree(((netmsg_t)msg)->base.nm_so); + kfree(msg, M_LWKTMSG); +} + /* * We need a custom putport function to handle the case where the * message target is the current thread's message port. This case @@ -159,8 +180,8 @@ netisr_init(void) */ for (i = 0; i < ncpus; ++i) { lwkt_create(netmsg_service_loop, NULL, NULL, - &netisr_cpu[i], TDF_STOPREQ, i, - "netisr_cpu %d", i); + &netisr_cpu[i], TDF_NOSTART|TDF_FORCE_SPINPORT, + i, "netisr_cpu %d", i); netmsg_service_port_init(&netisr_cpu[i].td_msgport); lwkt_schedule(&netisr_cpu[i]); } @@ -172,6 +193,8 @@ netisr_init(void) * the message is replied to. */ lwkt_initport_replyonly(&netisr_afree_rport, netisr_autofree_reply); + lwkt_initport_replyonly(&netisr_afree_free_so_rport, + netisr_autofree_free_so_reply); lwkt_initport_replyonly_null(&netisr_adone_rport); lwkt_initport_panic(&netisr_apanic_rport); @@ -228,7 +251,7 @@ netmsg_service_sync(void) struct netmsg_port_registration *reg; struct netmsg_base smsg; - netmsg_init(&smsg, NULL, &curthread->td_msgport, 0, netmsg_sync_func); + netmsg_init(&smsg, NULL, &curthread->td_msgport, 0, netmsg_sync_handler); TAILQ_FOREACH(reg, &netreglist, npr_entry) { lwkt_domsg(reg->npr_port, &smsg.lmsg, 0); @@ -239,8 +262,8 @@ netmsg_service_sync(void) * The netmsg function simply replies the message. API semantics require * EASYNC to be returned if the netmsg function disposes of the message. */ -static void -netmsg_sync_func(netmsg_t msg) +void +netmsg_sync_handler(netmsg_t msg) { lwkt_replymsg(&msg->lmsg, 0); } @@ -311,7 +334,7 @@ netisr_queue(int num, struct mbuf *m) struct netmsg_packet *pmsg; lwkt_port_t port; - KASSERT((num > 0 && num <= (sizeof(netisrs)/sizeof(netisrs[0]))), + KASSERT((num > 0 && num <= NELEM(netisrs)), ("Bad isr %d", num)); ni = &netisrs[num]; @@ -355,6 +378,46 @@ netisr_queue(int num, struct mbuf *m) return (0); } +/* + * Run a netisr service function on the packet. + * + * The packet must have been correctly characterized! + */ +int +netisr_handle(int num, struct mbuf *m) +{ + struct netisr *ni; + struct netmsg_packet *pmsg; + lwkt_port_t port; + + /* + * Get the protocol port based on the packet hash + */ + KASSERT((m->m_flags & M_HASH), ("packet not characterized\n")); + port = cpu_portfn(m->m_pkthdr.hash); + KASSERT(&curthread->td_msgport == port, ("wrong msgport\n")); + + KASSERT((num > 0 && num <= NELEM(netisrs)), ("bad isr %d", num)); + ni = &netisrs[num]; + if (ni->ni_handler == NULL) { + kprintf("unregistered isr %d\n", num); + m_freem(m); + return EIO; + } + + /* + * Initialize the netmsg, and run the handler directly. + */ + pmsg = &m->m_hdr.mh_netmsg; + netmsg_init(&pmsg->base, NULL, &netisr_apanic_rport, + 0, ni->ni_handler); + pmsg->nm_packet = m; + pmsg->base.lmsg.u.ms_result = num; + ni->ni_handler((netmsg_t)&pmsg->base); + + return 0; +} + /* * Pre-characterization of a deeper portion of the packet for the * requested isr. @@ -410,7 +473,7 @@ netisr_register(int num, netisr_fn_t handler, netisr_cpufn_t cpufn) { struct netisr *ni; - KASSERT((num > 0 && num <= (sizeof(netisrs)/sizeof(netisrs[0]))), + KASSERT((num > 0 && num <= NELEM(netisrs)), ("netisr_register: bad isr %d", num)); KKASSERT(handler != NULL); @@ -420,10 +483,23 @@ netisr_register(int num, netisr_fn_t handler, netisr_cpufn_t cpufn) ni = &netisrs[num]; ni->ni_handler = handler; + ni->ni_hashck = netisr_nohashck; ni->ni_cpufn = cpufn; netmsg_init(&ni->ni_netmsg, NULL, &netisr_adone_rport, 0, NULL); } +void +netisr_register_hashcheck(int num, netisr_hashck_t hashck) +{ + struct netisr *ni; + + KASSERT((num > 0 && num <= NELEM(netisrs)), + ("netisr_register: bad isr %d", num)); + + ni = &netisrs[num]; + ni->ni_hashck = hashck; +} + void netisr_register_rollup(netisr_ru_t ru_func) { @@ -511,7 +587,7 @@ schednetisr_remote(void *data) void schednetisr(int num) { - KASSERT((num > 0 && num <= (sizeof(netisrs)/sizeof(netisrs[0]))), + KASSERT((num > 0 && num <= NELEM(netisrs)), ("schednetisr: bad isr %d", num)); KKASSERT(netisrs[num].ni_handler != NULL); #ifdef SMP @@ -529,3 +605,138 @@ schednetisr(int num) crit_exit(); #endif } + +#ifdef SMP + +static void +netisr_barrier_dispatch(netmsg_t nmsg) +{ + struct netmsg_barrier *msg = (struct netmsg_barrier *)nmsg; + + atomic_clear_cpumask(msg->br_cpumask, mycpu->gd_cpumask); + if (*msg->br_cpumask == 0) + wakeup(msg->br_cpumask); + + for (;;) { + uint32_t done = msg->br_done; + + cpu_ccfence(); + if ((done & NETISR_BR_NOTDONE) == 0) + break; + + tsleep_interlock(&msg->br_done, 0); + if (atomic_cmpset_int(&msg->br_done, + done, done | NETISR_BR_WAITDONE)) + tsleep(&msg->br_done, PINTERLOCKED, "nbrdsp", 0); + } + + lwkt_replymsg(&nmsg->lmsg, 0); +} + +#endif + +struct netisr_barrier * +netisr_barrier_create(void) +{ + struct netisr_barrier *br; + + br = kmalloc(sizeof(*br), M_LWKTMSG, M_WAITOK | M_ZERO); + return br; +} + +void +netisr_barrier_set(struct netisr_barrier *br) +{ +#ifdef SMP + volatile cpumask_t other_cpumask; + int i, cur_cpuid; + + KKASSERT(&curthread->td_msgport == cpu_portfn(0)); + KKASSERT(!br->br_isset); + + other_cpumask = mycpu->gd_other_cpus & smp_active_mask; + cur_cpuid = mycpuid; + + for (i = 0; i < ncpus; ++i) { + struct netmsg_barrier *msg; + + if (i == cur_cpuid) + continue; + + msg = kmalloc(sizeof(struct netmsg_barrier), + M_LWKTMSG, M_WAITOK); + netmsg_init(&msg->base, NULL, &netisr_afree_rport, + MSGF_PRIORITY, netisr_barrier_dispatch); + msg->br_cpumask = &other_cpumask; + msg->br_done = NETISR_BR_NOTDONE; + + KKASSERT(br->br_msgs[i] == NULL); + br->br_msgs[i] = msg; + } + + for (i = 0; i < ncpus; ++i) { + if (i == cur_cpuid) + continue; + lwkt_sendmsg(cpu_portfn(i), &br->br_msgs[i]->base.lmsg); + } + + while (other_cpumask != 0) { + tsleep_interlock(&other_cpumask, 0); + if (other_cpumask != 0) + tsleep(&other_cpumask, PINTERLOCKED, "nbrset", 0); + } +#endif + br->br_isset = 1; +} + +void +netisr_barrier_rem(struct netisr_barrier *br) +{ +#ifdef SMP + int i, cur_cpuid; + + KKASSERT(&curthread->td_msgport == cpu_portfn(0)); + KKASSERT(br->br_isset); + + cur_cpuid = mycpuid; + for (i = 0; i < ncpus; ++i) { + struct netmsg_barrier *msg = br->br_msgs[i]; + uint32_t done; + + msg = br->br_msgs[i]; + br->br_msgs[i] = NULL; + + if (i == cur_cpuid) + continue; + + done = atomic_swap_int(&msg->br_done, 0); + if (done & NETISR_BR_WAITDONE) + wakeup(&msg->br_done); + } +#endif + br->br_isset = 0; +} + +static void +netisr_nohashck(struct mbuf *m, const struct pktinfo *pi __unused) +{ + m->m_flags &= ~M_HASH; +} + +void +netisr_hashcheck(int num, struct mbuf *m, const struct pktinfo *pi) +{ + struct netisr *ni; + + if (num < 0 || num >= NETISR_MAX) + panic("Bad isr %d", num); + + /* + * Valid netisr? + */ + ni = &netisrs[num]; + if (ni->ni_handler == NULL) + panic("Unregistered isr %d\n", num); + + ni->ni_hashck(m, pi); +}