From e57af3ca8a579c30015773a73fe470940d28fb0a Mon Sep 17 00:00:00 2001
From: Sepherosa Ziehau <sephe@dragonflybsd.org>
Date: Sat, 7 Apr 2012 22:17:44 +0800
Subject: [PATCH] carp: Lockless MPSAFE step 9 of many

Carry out timeout processing in netisr0
---
 sys/netinet/ip_carp.c | 90 +++++++++++++++++++++++++++++++++++++------
 1 file changed, 79 insertions(+), 11 deletions(-)

diff --git a/sys/netinet/ip_carp.c b/sys/netinet/ip_carp.c
index 03f73946f3..132ca11d1b 100644
--- a/sys/netinet/ip_carp.c
+++ b/sys/netinet/ip_carp.c
@@ -85,6 +85,8 @@
 #define CARP_IS_RUNNING(ifp)	\
 	(((ifp)->if_flags & (IFF_UP | IFF_RUNNING)) == (IFF_UP | IFF_RUNNING))
 
+struct carp_softc;
+
 struct carp_vhaddr {
 	uint32_t		vha_flags;	/* CARP_VHAF_ */
 	struct in_ifaddr	*vha_ia;	/* carp address */
@@ -93,6 +95,14 @@ struct carp_vhaddr {
 };
 TAILQ_HEAD(carp_vhaddr_list, carp_vhaddr);
 
+struct netmsg_carp {
+	struct netmsg_base	base;
+	struct ifnet		*nc_carpdev;
+	struct carp_softc	*nc_softc;
+	void			*nc_data;
+	size_t			nc_datalen;
+};
+
 struct carp_softc {
 	struct arpcom		 arpcom;
 	struct ifnet		*sc_carpdev;	/* parent interface */
@@ -133,8 +143,10 @@ struct carp_softc {
 	SHA1_CTX		 sc_sha1;
 
 	struct callout		 sc_ad_tmo;	/* advertisement timeout */
-	struct callout		 sc_md_tmo;	/* master down timeout */
-	struct callout 		 sc_md6_tmo;	/* master down timeout */
+	struct netmsg_carp	 sc_ad_msg;	/* adv timeout netmsg */
+	struct callout		 sc_md_tmo;	/* ip4 master down timeout */
+	struct callout 		 sc_md6_tmo;	/* ip6 master down timeout */
+	struct netmsg_carp	 sc_md_msg;	/* master down timeout netmsg */
 
 	LIST_ENTRY(carp_softc)	 sc_next;	/* Interface clue */
 };
@@ -145,14 +157,6 @@ struct carp_if {
 	TAILQ_HEAD(, carp_softc) vhif_vrs;
 };
 
-struct netmsg_carp {
-	struct netmsg_base	base;
-	struct ifnet		*nc_carpdev;
-	struct carp_softc	*nc_softc;
-	void			*nc_data;
-	size_t			nc_datalen;
-};
-
 SYSCTL_DECL(_net_inet_carp);
 
 static int carp_opts[CARPCTL_MAXID] = { 0, 1, 0, 1, 0, 0 }; /* XXX for now */
@@ -270,6 +274,8 @@ static void	carp_ioctl_setvh_dispatch(netmsg_t);
 static void	carp_ioctl_getvh_dispatch(netmsg_t);
 static void	carp_ioctl_getdevname_dispatch(netmsg_t);
 static void	carp_ioctl_getvhaddr_dispatch(netmsg_t);
+static void	carp_send_ad_timeout_dispatch(netmsg_t);
+static void	carp_master_down_timeout_dispatch(netmsg_t);
 
 static MALLOC_DEFINE(M_CARP, "CARP", "CARP interfaces");
 
@@ -456,8 +462,15 @@ carp_clone_create(struct if_clone *ifc, int unit, caddr_t param __unused)
 #endif
 
 	callout_init_mp(&sc->sc_ad_tmo);
+	netmsg_init(&sc->sc_ad_msg.base, NULL, &netisr_adone_rport,
+	    MSGF_DROPABLE | MSGF_PRIORITY, carp_send_ad_timeout_dispatch);
+	sc->sc_ad_msg.nc_softc = sc;
+
 	callout_init_mp(&sc->sc_md_tmo);
 	callout_init_mp(&sc->sc_md6_tmo);
+	netmsg_init(&sc->sc_md_msg.base, NULL, &netisr_adone_rport,
+	    MSGF_DROPABLE | MSGF_PRIORITY, carp_master_down_timeout_dispatch);
+	sc->sc_md_msg.nc_softc = sc;
 
 	if_initname(ifp, CARP_IFNAME, unit);
 	ifp->if_softc = sc;
@@ -499,6 +512,17 @@ carp_clone_destroy_dispatch(netmsg_t msg)
 
 	carp_reltok();
 
+	callout_stop_sync(&sc->sc_ad_tmo);
+	callout_stop_sync(&sc->sc_md_tmo);
+	callout_stop_sync(&sc->sc_md6_tmo);
+
+	crit_enter();
+	if ((sc->sc_ad_msg.base.lmsg.ms_flags & MSGF_DONE) == 0)
+		lwkt_dropmsg(&sc->sc_ad_msg.base.lmsg);
+	if ((sc->sc_md_msg.base.lmsg.ms_flags & MSGF_DONE) == 0)
+		lwkt_dropmsg(&sc->sc_md_msg.base.lmsg);
+	crit_exit();
+
 	lwkt_replymsg(&cmsg->base.lmsg, 0);
 }
 
@@ -973,8 +997,31 @@ carp_send_ad_all(void)
 static void
 carp_send_ad_timeout(void *xsc)
 {
+	struct carp_softc *sc = xsc;
+	struct netmsg_carp *cmsg = &sc->sc_ad_msg;
+
+	KASSERT(mycpuid == 0, ("%s not on cpu0 but on cpu%d\n",
+	    __func__, mycpuid));
+
+	crit_enter();
+	if (cmsg->base.lmsg.ms_flags & MSGF_DONE)
+		lwkt_sendmsg(cpu_portfn(0), &cmsg->base.lmsg);
+	crit_exit();
+}
+
+static void
+carp_send_ad_timeout_dispatch(netmsg_t msg)
+{
+	struct netmsg_carp *cmsg = (struct netmsg_carp *)msg;
+	struct carp_softc *sc = cmsg->nc_softc;
+
+	/* Reply ASAP */
+	crit_enter();
+	lwkt_replymsg(&cmsg->base.lmsg, 0);
+	crit_exit();
+
 	carp_gettok();
-	carp_send_ad(xsc);
+	carp_send_ad(sc);
 	carp_reltok();
 }
 
@@ -1369,6 +1416,27 @@ static void
 carp_master_down_timeout(void *xsc)
 {
 	struct carp_softc *sc = xsc;
+	struct netmsg_carp *cmsg = &sc->sc_md_msg;
+
+	KASSERT(mycpuid == 0, ("%s not on cpu0 but on cpu%d\n",
+	    __func__, mycpuid));
+
+	crit_enter();
+	if (cmsg->base.lmsg.ms_flags & MSGF_DONE)
+		lwkt_sendmsg(cpu_portfn(0), &cmsg->base.lmsg);
+	crit_exit();
+}
+
+static void
+carp_master_down_timeout_dispatch(netmsg_t msg)
+{
+	struct netmsg_carp *cmsg = (struct netmsg_carp *)msg;
+	struct carp_softc *sc = cmsg->nc_softc;
+
+	/* Reply ASAP */
+	crit_enter();
+	lwkt_replymsg(&cmsg->base.lmsg, 0);
+	crit_exit();
 
 	CARP_DEBUG("%s: BACKUP -> MASTER (master timed out)\n",
 		   sc->sc_if.if_xname);
-- 
2.41.0