From 33dbeae810812d056db4c1a65d540b9dceeccede Mon Sep 17 00:00:00 2001 From: Joe Talbott Date: Fri, 4 Mar 2011 07:28:29 -0500 Subject: [PATCH] mbuf - Add MJUMPAGESIZE mbuf cluster support. This is needed by Intel WiFi chipsets which require larger than usual clusters. Revert MCLSHIFT back to 11 to reduce mbuf cluster space usage and make non-Intel Wifi NICs work without needing to adjust MCLSHIFT. --- sys/kern/uipc_mbuf.c | 180 +++++++++++++++++++++++++++++++++++++++---- sys/sys/mbuf.h | 2 + sys/sys/param.h | 2 +- 3 files changed, 167 insertions(+), 17 deletions(-) diff --git a/sys/kern/uipc_mbuf.c b/sys/kern/uipc_mbuf.c index 415c578d49..22323f0003 100644 --- a/sys/kern/uipc_mbuf.c +++ b/sys/kern/uipc_mbuf.c @@ -253,8 +253,9 @@ int m_defragrandomfailures; #endif struct objcache *mbuf_cache, *mbufphdr_cache; -struct objcache *mclmeta_cache; +struct objcache *mclmeta_cache, *mjclmeta_cache; struct objcache *mbufcluster_cache, *mbufphdrcluster_cache; +struct objcache *mbufjcluster_cache, *mbufphdrjcluster_cache; int nmbclusters; int nmbufs; @@ -358,7 +359,9 @@ SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragrandomfailures, CTLFLAG_RW, static MALLOC_DEFINE(M_MBUF, "mbuf", "mbuf"); static MALLOC_DEFINE(M_MBUFCL, "mbufcl", "mbufcl"); +static MALLOC_DEFINE(M_MJBUFCL, "mbufcl", "mbufcl"); static MALLOC_DEFINE(M_MCLMETA, "mclmeta", "mclmeta"); +static MALLOC_DEFINE(M_MJCLMETA, "mjclmeta", "mjclmeta"); static void m_reclaim (void); static void m_mclref(void *arg); @@ -461,6 +464,23 @@ mclmeta_ctor(void *obj, void *private, int ocflags) return (TRUE); } +static boolean_t +mjclmeta_ctor(void *obj, void *private, int ocflags) +{ + struct mbcluster *cl = obj; + void *buf; + + if (ocflags & M_NOWAIT) + buf = kmalloc(MJUMPAGESIZE, M_MBUFCL, M_NOWAIT | M_ZERO); + else + buf = kmalloc(MJUMPAGESIZE, M_MBUFCL, M_INTWAIT | M_ZERO); + if (buf == NULL) + return (FALSE); + cl->mcl_refs = 0; + cl->mcl_data = buf; + return (TRUE); +} + static void mclmeta_dtor(void *obj, void *private) { @@ -471,7 +491,7 @@ mclmeta_dtor(void *obj, void *private) } static void -linkcluster(struct mbuf *m, struct mbcluster *cl) +linkjcluster(struct mbuf *m, struct mbcluster *cl, uint size) { /* * Add the cluster to the mbuf. The caller will detect that the @@ -481,13 +501,19 @@ linkcluster(struct mbuf *m, struct mbcluster *cl) m->m_ext.ext_buf = cl->mcl_data; m->m_ext.ext_ref = m_mclref; m->m_ext.ext_free = m_mclfree; - m->m_ext.ext_size = MCLBYTES; + m->m_ext.ext_size = size; atomic_add_int(&cl->mcl_refs, 1); m->m_data = m->m_ext.ext_buf; m->m_flags |= M_EXT | M_EXT_CLUSTER; } +static void +linkcluster(struct mbuf *m, struct mbcluster *cl) +{ + linkjcluster(m, cl, MCLBYTES); +} + static boolean_t mbufphdrcluster_ctor(void *obj, void *private, int ocflags) { @@ -505,6 +531,23 @@ mbufphdrcluster_ctor(void *obj, void *private, int ocflags) return (TRUE); } +static boolean_t +mbufphdrjcluster_ctor(void *obj, void *private, int ocflags) +{ + struct mbuf *m = obj; + struct mbcluster *cl; + + mbufphdr_ctor(obj, private, ocflags); + cl = objcache_get(mjclmeta_cache, ocflags); + if (cl == NULL) { + ++mbstat[mycpu->gd_cpuid].m_drops; + return (FALSE); + } + m->m_flags |= M_CLCACHE; + linkjcluster(m, cl, MJUMPAGESIZE); + return (TRUE); +} + static boolean_t mbufcluster_ctor(void *obj, void *private, int ocflags) { @@ -522,6 +565,23 @@ mbufcluster_ctor(void *obj, void *private, int ocflags) return (TRUE); } +static boolean_t +mbufjcluster_ctor(void *obj, void *private, int ocflags) +{ + struct mbuf *m = obj; + struct mbcluster *cl; + + mbuf_ctor(obj, private, ocflags); + cl = objcache_get(mjclmeta_cache, ocflags); + if (cl == NULL) { + ++mbstat[mycpu->gd_cpuid].m_drops; + return (FALSE); + } + m->m_flags |= M_CLCACHE; + linkjcluster(m, cl, MJUMPAGESIZE); + return (TRUE); +} + /* * Used for both the cluster and cluster PHDR caches. * @@ -539,7 +599,10 @@ mbufcluster_dtor(void *obj, void *private) mcl = m->m_ext.ext_arg; KKASSERT(mcl->mcl_refs == 1); mcl->mcl_refs = 0; - objcache_put(mclmeta_cache, mcl); + if (m->m_flags & M_EXT && m->m_ext.ext_size != MCLBYTES) + objcache_put(mjclmeta_cache, mcl); + else + objcache_put(mclmeta_cache, mcl); } } @@ -561,6 +624,7 @@ mbinit(void *dummy) for (i = 0; i < ncpus; i++) { atomic_set_long_nonlocked(&mbstat[i].m_msize, MSIZE); atomic_set_long_nonlocked(&mbstat[i].m_mclbytes, MCLBYTES); + atomic_set_long_nonlocked(&mbstat[i].m_mjumpagesize, MJUMPAGESIZE); atomic_set_long_nonlocked(&mbstat[i].m_minclsize, MINCLSIZE); atomic_set_long_nonlocked(&mbstat[i].m_mlen, MLEN); atomic_set_long_nonlocked(&mbstat[i].m_mhlen, MHLEN); @@ -590,6 +654,11 @@ mbinit(void *dummy) mclmeta_ctor, mclmeta_dtor, NULL, objcache_malloc_alloc, objcache_malloc_free, &mclmeta_malloc_args); + cl_limit = nmbclusters; + mjclmeta_cache = objcache_create("jcluster mbuf", &cl_limit, 0, + mjclmeta_ctor, mclmeta_dtor, NULL, + objcache_malloc_alloc, objcache_malloc_free, &mclmeta_malloc_args); + limit = nmbclusters; mbufcluster_cache = objcache_create("mbuf + cluster", &limit, 0, mbufcluster_ctor, mbufcluster_dtor, NULL, @@ -602,6 +671,18 @@ mbinit(void *dummy) objcache_malloc_alloc, objcache_malloc_free, &mbuf_malloc_args); mb_limit += limit; + limit = nmbclusters; + mbufjcluster_cache = objcache_create("mbuf + jcluster", &limit, 0, + mbufjcluster_ctor, mbufcluster_dtor, NULL, + objcache_malloc_alloc, objcache_malloc_free, &mbuf_malloc_args); + mb_limit += limit; + + limit = nmbclusters; + mbufphdrjcluster_cache = objcache_create("mbuf pkt hdr + jcluster", + &limit, 64, mbufphdrjcluster_ctor, mbufcluster_dtor, NULL, + objcache_malloc_alloc, objcache_malloc_free, &mbuf_malloc_args); + mb_limit += limit; + /* * Adjust backing kmalloc pools' limit * @@ -611,7 +692,8 @@ mbinit(void *dummy) cl_limit += cl_limit / 8; kmalloc_raise_limit(mclmeta_malloc_args.mtype, mclmeta_malloc_args.objsize * cl_limit); - kmalloc_raise_limit(M_MBUFCL, MCLBYTES * cl_limit); + kmalloc_raise_limit(M_MBUFCL, MCLBYTES * cl_limit * 3/4 + MJUMPAGESIZE * cl_limit / 4); + /*kmalloc_raise_limit(M_MBUFCL, MCLBYTES * cl_limit);*/ mb_limit += mb_limit / 8; kmalloc_raise_limit(mbuf_malloc_args.mtype, @@ -704,7 +786,9 @@ retryonce: struct objcache *reclaimlist[] = { mbufphdr_cache, mbufcluster_cache, - mbufphdrcluster_cache + mbufphdrcluster_cache, + mbufjcluster_cache, + mbufphdrjcluster_cache }; const int nreclaims = NELEM(reclaimlist); @@ -739,7 +823,8 @@ retryonce: if ((how & MB_TRYWAIT) && ntries++ == 0) { struct objcache *reclaimlist[] = { mbuf_cache, - mbufcluster_cache, mbufphdrcluster_cache + mbufcluster_cache, mbufphdrcluster_cache, + mbufjcluster_cache, mbufphdrjcluster_cache }; const int nreclaims = NELEM(reclaimlist); @@ -775,6 +860,51 @@ m_getclr(int how, int type) return (m); } +struct mbuf * +m_getjcl(int how, short type, int flags, size_t size) +{ + struct mbuf *m = NULL; + int ocflags = MBTOM(how); + int ntries = 0; + +retryonce: + + if (flags & M_PKTHDR) + m = objcache_get(mbufphdrjcluster_cache, ocflags); + else + m = objcache_get(mbufjcluster_cache, ocflags); + + if (m == NULL) { + if ((how & MB_TRYWAIT) && ntries++ == 0) { + struct objcache *reclaimlist[1]; + + if (flags & M_PKTHDR) + reclaimlist[0] = mbufjcluster_cache; + else + reclaimlist[0] = mbufphdrjcluster_cache; + if (!objcache_reclaimlist(reclaimlist, 1, ocflags)) + m_reclaim(); + goto retryonce; + } + ++mbstat[mycpu->gd_cpuid].m_drops; + return (NULL); + } + +#ifdef MBUF_DEBUG + KASSERT(m->m_data == m->m_ext.ext_buf, + ("mbuf %p: bad m_data in get", m)); +#endif + m->m_type = type; + m->m_len = 0; + m->m_pkthdr.len = 0; /* just do it unconditonally */ + + mbuftrack(m); + + atomic_add_long_nonlocked(&mbtypes[mycpu->gd_cpuid][type], 1); + atomic_add_long_nonlocked(&mbstat[mycpu->gd_cpuid].m_clusters, 1); + return (m); +} + /* * Returns an mbuf with an attached cluster. * Because many network drivers use this kind of buffers a lot, it is @@ -1024,10 +1154,17 @@ m_free(struct mbuf *m) * an mbuf). */ m->m_data = m->m_ext.ext_buf; - if (m->m_flags & M_PHCACHE) - objcache_put(mbufphdrcluster_cache, m); - else - objcache_put(mbufcluster_cache, m); + if (m->m_flags & M_EXT && m->m_ext.ext_size != MCLBYTES) { + if (m->m_flags & M_PHCACHE) + objcache_put(mbufphdrjcluster_cache, m); + else + objcache_put(mbufjcluster_cache, m); + } else { + if (m->m_flags & M_PHCACHE) + objcache_put(mbufphdrcluster_cache, m); + else + objcache_put(mbufcluster_cache, m); + } atomic_subtract_long_nonlocked(&mbstat[mycpu->gd_cpuid].m_clusters, 1); } else { /* @@ -1042,12 +1179,20 @@ m_free(struct mbuf *m) * XXX we could try to connect another cluster to * it. */ + m->m_ext.ext_free(m->m_ext.ext_arg); m->m_flags &= ~(M_EXT | M_EXT_CLUSTER); - if (m->m_flags & M_PHCACHE) - objcache_dtor(mbufphdrcluster_cache, m); - else - objcache_dtor(mbufcluster_cache, m); + if (m->m_ext.ext_size == MCLBYTES) { + if (m->m_flags & M_PHCACHE) + objcache_dtor(mbufphdrcluster_cache, m); + else + objcache_dtor(mbufcluster_cache, m); + } else { + if (m->m_flags & M_PHCACHE) + objcache_dtor(mbufphdrjcluster_cache, m); + else + objcache_dtor(mbufjcluster_cache, m); + } } break; case M_EXT | M_EXT_CLUSTER: @@ -1403,7 +1548,10 @@ m_dup_data(struct mbuf *m, int how) * Optimize the mbuf allocation but do not get too carried away. */ if (m->m_next || m->m_len > MLEN) - gsize = MCLBYTES; + if (m->m_flags & M_EXT && m->m_ext.ext_size == MCLBYTES) + gsize = MCLBYTES; + else + gsize = MJUMPAGESIZE; else gsize = MLEN; diff --git a/sys/sys/mbuf.h b/sys/sys/mbuf.h index a78eecb010..689c159628 100644 --- a/sys/sys/mbuf.h +++ b/sys/sys/mbuf.h @@ -324,6 +324,7 @@ struct mbstat { u_long m_mpfail; /* times m_pullup failed */ u_long m_msize; /* length of an mbuf */ u_long m_mclbytes; /* length of an mbuf cluster */ + u_long m_mjumpagesize; /* length of a jumbo mbuf cluster */ u_long m_minclsize; /* min length of data to allocate a cluster */ u_long m_mlen; /* length of data in an mbuf */ u_long m_mhlen; /* length of data in a header mbuf */ @@ -508,6 +509,7 @@ void m_freem(struct mbuf *); struct mbuf *m_get(int, int); struct mbuf *m_getc(int len, int how, int type); struct mbuf *m_getcl(int how, short type, int flags); +struct mbuf *m_getjcl(int how, short type, int flags, uint size); struct mbuf *m_getclr(int, int); struct mbuf *m_gethdr(int, int); struct mbuf *m_getm(struct mbuf *, int, int, int); diff --git a/sys/sys/param.h b/sys/sys/param.h index b729ed0938..7dd7cad148 100644 --- a/sys/sys/param.h +++ b/sys/sys/param.h @@ -289,7 +289,7 @@ #endif #ifndef MCLSHIFT -#define MCLSHIFT 12 /* convert bytes to m_buf clusters */ +#define MCLSHIFT 11 /* convert bytes to m_buf clusters */ #endif #define MCLBYTES (1 << MCLSHIFT) /* size of an m_buf cluster */ #define MCLOFSET (MCLBYTES - 1) /* offset within an m_buf cluster */ -- 2.41.0