kernel - Remove PG_ZERO and zeroidle (page-zeroing) entirely
authorMatthew Dillon <dillon@apollo.backplane.com>
Wed, 3 Aug 2016 00:41:08 +0000 (17:41 -0700)
committerMatthew Dillon <dillon@apollo.backplane.com>
Wed, 3 Aug 2016 00:41:08 +0000 (17:41 -0700)
* Remove the PG_ZERO flag and remove all page-zeroing optimizations,
  entirely.  Aftering doing a substantial amount of testing, these
  optimizations, which existed all the way back to CSRG BSD, no longer
  provide any benefit on a modern system.

  - Pre-zeroing a page only takes 80ns on a modern cpu.  vm_fault overhead
    in general is ~at least 1 microscond.

  - Pre-zeroing a page leads to a cold-cache case on-use, forcing the fault
    source (e.g. a userland program) to actually get the data from main
    memory in its likely immediate use of the faulted page, reducing
    performance.

  - Zeroing the page at fault-time is actually more optimal because it does
    not require any reading of dynamic ram and leaves the cache hot.

  - Multiple synth and build tests show that active idle-time zeroing of
    pages actually reduces performance somewhat and incidental allocations
    of already-zerod pages (from page-table tear-downs) do not affect
    performance in any meaningful way.

* Remove bcopyi() and obbcopy() -> collapse into bcopy().  These other
  versions existed because bcopy() used to be specially-optimized and
  could not be used in all situations.  That is no longer true.

* Remove bcopy function pointer argument to m_devget().  It is no longer
  used.  This function existed to help support ancient drivers which might
  have needed a special memory copy to read and write mapped data.  It has
  long been supplanted by BUSDMA.

66 files changed:
sys/ddb/db_input.c
sys/dev/netif/ae/if_ae.c
sys/dev/netif/ale/if_ale.c
sys/dev/netif/dc/if_dc.c
sys/dev/netif/ic/if_ic.c
sys/dev/netif/lge/if_lge.c
sys/dev/netif/my/if_my.c
sys/dev/netif/nge/if_nge.c
sys/dev/netif/plip/if_plip.c
sys/dev/netif/ral/rt2860.c
sys/dev/netif/rl/if_rl.c
sys/dev/netif/sf/if_sf.c
sys/dev/netif/sln/if_sln.c
sys/dev/netif/vr/if_vr.c
sys/dev/netif/wb/if_wb.c
sys/dev/video/fb/fbreg.h
sys/kern/kern_slaballoc.c
sys/kern/uipc_mbuf.c
sys/kern/uipc_syscalls.c
sys/kern/vfs_bio.c
sys/net/bpf.c
sys/net/net_osdep.h
sys/net/netmap/netmap.c
sys/net/ppp_layer/slcompress.c
sys/netgraph/async/ng_async.c
sys/netgraph/l2tp/ng_l2tp.c
sys/netgraph/mppc/ng_mppc.c
sys/netgraph/ng_device.c
sys/netgraph/pptpgre/ng_pptpgre.c
sys/netgraph/socket/ng_socket.c
sys/netgraph7/async/ng_async.c
sys/netgraph7/deflate/ng_deflate.c
sys/netgraph7/mppc/ng_mppc.c
sys/netgraph7/ng_pred1.c
sys/netgraph7/socket/ng_socket.c
sys/netinet/ip_output.c
sys/netinet6/ah_input.c
sys/netinet6/esp_input.c
sys/netinet6/frag6.c
sys/netinet6/ipsec.c
sys/netproto/802_11/wlan/ieee80211_input.c
sys/netproto/802_11/wlan/ieee80211_output.c
sys/netproto/802_11/wlan_ccmp/ieee80211_crypto_ccmp.c
sys/netproto/802_11/wlan_wep/ieee80211_crypto_wep.c
sys/netproto/ipsec/ipsec_mbuf.c
sys/netproto/ipsec/ipsec_output.c
sys/platform/pc64/include/md_var.h
sys/platform/pc64/x86_64/pmap.c
sys/platform/pc64/x86_64/support.s
sys/platform/vkernel64/platform/copyio.c
sys/platform/vkernel64/platform/pmap.c
sys/sys/mbuf.h
sys/sys/systm.h
sys/vfs/devfs/devfs_vnops.c
sys/vfs/smbfs/smbfs_io.c
sys/vm/phys_pager.c
sys/vm/pmap.h
sys/vm/swap_pager.c
sys/vm/vm_contig.c
sys/vm/vm_fault.c
sys/vm/vm_kern.c
sys/vm/vm_page.c
sys/vm/vm_page.h
sys/vm/vm_page2.h
sys/vm/vm_pager.c
sys/vm/vm_zeroidle.c

index 86ccc2d..1dfb834 100644 (file)
@@ -320,8 +320,8 @@ db_readline(char *lstart, int lsize)
            /* Maintain input line history for non-empty lines. */
            if (++db_lhistidx == db_lhist_nlines) {
                /* Rotate history. */
-               ovbcopy(db_lhistory + db_lhistlsize, db_lhistory,
-                       db_lhistlsize * (db_lhist_nlines - 1));
+               bcopy(db_lhistory + db_lhistlsize, db_lhistory,
+                     db_lhistlsize * (db_lhist_nlines - 1));
                db_lhistidx--;
            }
            bcopy(lstart, db_lhistory + db_lhistidx * db_lhistlsize,
index e453d0d..eb37fb6 100644 (file)
@@ -614,7 +614,7 @@ ae_rxeof(struct ae_softc *sc, struct ae_rxd *rxd)
                return (EIO);
        }
 
-       m = m_devget(&rxd->data[0], size, ETHER_ALIGN, ifp, NULL);
+       m = m_devget(&rxd->data[0], size, ETHER_ALIGN, ifp);
        if (m == NULL)
                return (ENOBUFS);
 
index d324cbd..d00a879 100644 (file)
@@ -2227,7 +2227,7 @@ ale_rxeof(struct ale_softc *sc)
                        }
                }
                /*
-                * m_devget(9) is major bottle-neck of ale(4)(It comes
+                * m_devget(9) is major bottle-neck of ale(4) (It comes
                 * from hardware limitation). For jumbo frames we could
                 * get a slightly better performance if driver use
                 * m_getjcl(9) with proper buffer size argument. However
@@ -2236,7 +2236,7 @@ ale_rxeof(struct ale_softc *sc)
                 * on these low-end consumer ethernet controller.
                 */
                m = m_devget((char *)(rs + 1), length - ETHER_CRC_LEN,
-                   ETHER_ALIGN, ifp, NULL);
+                            ETHER_ALIGN, ifp);
                if (m == NULL) {
                        IFNET_STAT_INC(ifp, iqdrops, 1);
                        ale_rx_update_page(sc, &rx_page, length, &prod);
index 95b307e..d2591cd 100644 (file)
@@ -2573,6 +2573,7 @@ dc_rxeof(struct dc_softc *sc)
                 * allocate a new buffer for the receive ring, and pass up
                 * the one where the packet is already, saving the expensive
                 * copy done in m_devget().
+                *
                 * If we are on an architecture with alignment problems, or
                 * if the allocation fails, then use m_devget and leave the
                 * existing buffer in the receive ring.
@@ -2587,7 +2588,7 @@ dc_rxeof(struct dc_softc *sc)
                        struct mbuf *m0;
 
                        m0 = m_devget(mtod(m, char *) - ETHER_ALIGN,
-                           total_len + ETHER_ALIGN, 0, ifp, NULL);
+                                     total_len + ETHER_ALIGN, 0, ifp);
                        dc_newbuf(sc, i, m);
                        DC_INC(i, DC_RX_LIST_CNT);
                        if (m0 == NULL) {
index 91b89df..fcb5b7f 100644 (file)
@@ -291,7 +291,7 @@ icintr (device_t dev, int event, char *ptr)
 
          BPF_TAP(&sc->ic_if, sc->ic_ifbuf, len + ICHDRLEN);
 
-         top = m_devget(sc->ic_ifbuf + ICHDRLEN, len, 0, &sc->ic_if, 0);
+         top = m_devget(sc->ic_ifbuf + ICHDRLEN, len, 0, &sc->ic_if);
 
          if (top)
            netisr_queue(NETISR_IP, top);
index d2c9262..affd44c 100644 (file)
@@ -885,7 +885,7 @@ lge_rxeof(struct lge_softc *sc, int cnt)
 
                if (lge_newbuf(sc, &LGE_RXTAIL(sc), NULL) == ENOBUFS) {
                        m0 = m_devget(mtod(m, char *) - ETHER_ALIGN,
-                           total_len + ETHER_ALIGN, 0, ifp, NULL);
+                                     total_len + ETHER_ALIGN, 0, ifp);
                        lge_newbuf(sc, &LGE_RXTAIL(sc), m);
                        if (m0 == NULL) {
                                kprintf("lge%d: no receive buffers "
index 1830f2a..b2cec09 100644 (file)
@@ -1107,7 +1107,7 @@ my_rxeof(struct my_softc * sc)
 
                if (total_len < MINCLSIZE) {
                        m = m_devget(mtod(cur_rx->my_mbuf, char *),
-                           total_len, 0, ifp, NULL);
+                                    total_len, 0, ifp);
                        cur_rx->my_ptr->my_status = MY_OWNByNIC;
                        if (m == NULL) {
                                IFNET_STAT_INC(ifp, ierrors, 1);
index 4fc949e..de86b61 100644 (file)
@@ -1253,7 +1253,7 @@ nge_rxeof(struct nge_softc *sc)
                if (nge_newbuf(sc, cur_rx, NULL) == ENOBUFS) {
 #endif
                        m0 = m_devget(mtod(m, char *) - ETHER_ALIGN,
-                           total_len + ETHER_ALIGN, 0, ifp, NULL);
+                                     total_len + ETHER_ALIGN, 0, ifp);
                        nge_newbuf(sc, cur_rx, m);
                        if (m0 == NULL) {
                                kprintf("nge%d: no receive buffers "
index 52d2fe5..bba63dc 100644 (file)
@@ -494,7 +494,7 @@ lp_intr (void *arg)
            len -= CLPIPHDRLEN;
            IFNET_STAT_INC(&sc->sc_if, ipackets, 1);
            IFNET_STAT_INC(&sc->sc_if, ibytes, len);
-           top = m_devget(sc->sc_ifbuf + CLPIPHDRLEN, len, 0, &sc->sc_if, 0);
+           top = m_devget(sc->sc_ifbuf + CLPIPHDRLEN, len, 0, &sc->sc_if);
            if (top) {
                if (sc->sc_if.if_bpf)
                    lptap(&sc->sc_if, top);
@@ -539,7 +539,7 @@ lp_intr (void *arg)
            len -= LPIPHDRLEN;
            IFNET_STAT_INC(&sc->sc_if, ipackets, 1);
            IFNET_STAT_INC(&sc->sc_if, ibytes, len);
-           top = m_devget(sc->sc_ifbuf + LPIPHDRLEN, len, 0, &sc->sc_if, 0);
+           top = m_devget(sc->sc_ifbuf + LPIPHDRLEN, len, 0, &sc->sc_if);
            if (top) {
                if (sc->sc_if.if_bpf)
                    lptap(&sc->sc_if, top);
index 40f1f60..888ed50 100644 (file)
@@ -1336,7 +1336,7 @@ rt2860_rx_intr(struct rt2860_softc *sc)
                /* HW may insert 2 padding bytes after 802.11 header */
                if (rxd->flags & htole32(RT2860_RX_L2PAD)) {
                        u_int hdrlen = ieee80211_hdrsize(wh);
-                       ovbcopy(wh, (caddr_t)wh + 2, hdrlen);
+                       bcopy(wh, (caddr_t)wh + 2, hdrlen);
                        m->m_data += 2;
                        wh = mtod(m, struct ieee80211_frame *);
                }
index b95aa26..93649f5 100644 (file)
@@ -1066,7 +1066,7 @@ rl_rxeof(struct rl_softc *sc)
                         * the data.
                         */
                        m = m_devget(rxbufpos - RL_ETHER_ALIGN,
-                           wrap + RL_ETHER_ALIGN, 0, ifp, NULL);
+                                    wrap + RL_ETHER_ALIGN, 0, ifp);
                        if (m == NULL) {
                                IFNET_STAT_INC(ifp, ierrors, 1);
                        } else {
@@ -1077,7 +1077,7 @@ rl_rxeof(struct rl_softc *sc)
                        cur_rx = (total_len - wrap + ETHER_CRC_LEN);
                } else {
                        m = m_devget(rxbufpos - RL_ETHER_ALIGN,
-                           total_len + RL_ETHER_ALIGN, 0, ifp, NULL);
+                                    total_len + RL_ETHER_ALIGN, 0, ifp);
                        if (m == NULL) {
                                IFNET_STAT_INC(ifp, ierrors, 1);
                        } else
index ad44580..6ff7e8a 100644 (file)
@@ -949,7 +949,7 @@ sf_rxeof(struct sf_softc *sc)
                }
 
                m0 = m_devget(mtod(m, char *) - ETHER_ALIGN,
-                   cur_rx->sf_len + ETHER_ALIGN, 0, ifp, NULL);
+                             cur_rx->sf_len + ETHER_ALIGN, 0, ifp);
                sf_newbuf(sc, desc, m);
                if (m0 == NULL) {
                        IFNET_STAT_INC(ifp, ierrors, 1);
index 5086e07..b70ab74 100644 (file)
@@ -909,8 +909,8 @@ sln_rx(struct sln_softc *sc)
                rx_space = (u_long)((sc->sln_bufdata.sln_rx_buf + SL_RX_BUFLEN) - rx_bufpos);
 
                if (pkt_size > rx_space) {
-                       m = m_devget(rx_bufpos - 2, pkt_size + 2, 0, ifp, NULL);        /* 2 for etherer head
-                                                                                        * align */
+                       /* 2 for etherer head align */
+                       m = m_devget(rx_bufpos - 2, pkt_size + 2, 0, ifp);
 
                        if (m == NULL) {
                                IFNET_STAT_INC(ifp, ierrors, 1);
@@ -922,7 +922,7 @@ sln_rx(struct sln_softc *sc)
                                m_copyback(m, rx_space, pkt_size - rx_space, sc->sln_bufdata.sln_rx_buf);
                        }
                } else {
-                       m = m_devget(rx_bufpos - 2, pkt_size + 2, 0, ifp, NULL);
+                       m = m_devget(rx_bufpos - 2, pkt_size + 2, 0, ifp);
 
                        if (m == NULL) {
                                u_long ierr;
index 4e8da76..c78058e 100644 (file)
@@ -1014,7 +1014,7 @@ vr_rxeof(struct vr_softc *sc)
                total_len -= ETHER_CRC_LEN;
 
                m0 = m_devget(mtod(m, char *) - ETHER_ALIGN,
-                   total_len + ETHER_ALIGN, 0, ifp, NULL);
+                             total_len + ETHER_ALIGN, 0, ifp);
                vr_newbuf(sc, cur_rx, m);
                if (m0 == NULL) {
                        IFNET_STAT_INC(ifp, ierrors, 1);
index 71081aa..5621bed 100644 (file)
@@ -1012,7 +1012,7 @@ wb_rxeof(struct wb_softc *sc)
                total_len -= ETHER_CRC_LEN;
 
                m0 = m_devget(mtod(m, char *) - ETHER_ALIGN,
-                    total_len + ETHER_ALIGN, 0, ifp, NULL);
+                             total_len + ETHER_ALIGN, 0, ifp);
                wb_newbuf(sc, cur_rx, m);
                if (m0 == NULL) {
                        IFNET_STAT_INC(ifp, ierrors, 1);
index e493ada..232960d 100644 (file)
 #define V_MAX_ADAPTERS         8               /* XXX */
 
 /* some macros */
-#if defined(__x86_64__)
-#define bcopy_io(s, d, c)      generic_bcopy((void *)(s), (void *)(d), (c))
-#define bcopy_toio(s, d, c)    generic_bcopy((void *)(s), (void *)(d), (c))
-#define bcopy_fromio(s, d, c)  generic_bcopy((void *)(s), (void *)(d), (c))
+#define bcopy_io(s, d, c)      bcopy((void *)(s), (void *)(d), (c))
+#define bcopy_toio(s, d, c)    bcopy((void *)(s), (void *)(d), (c))
+#define bcopy_fromio(s, d, c)  bcopy((void *)(s), (void *)(d), (c))
 #define bzero_io(d, c)         bzero((void *)(d), (c))
 #define fill_io(p, d, c)       fill((p), (void *)(d), (c))
 #define fillw_io(p, d, c)      fillw((p), (void *)(d), (c))
-void generic_bcopy(const void *s, void *d, size_t c);
-#else /* !__x86_64__ */
-#define bcopy_io(s, d, c)      memcpy_io((d), (s), (c))
-#define bcopy_toio(s, d, c)    memcpy_toio((d), (void *)(s), (c))
-#define bcopy_fromio(s, d, c)  memcpy_fromio((void *)(d), (s), (c))
-#define bzero_io(d, c)         memset_io((d), 0, (c))
-#define fill_io(p, d, c)       memset_io((d), (p), (c))
-#define fillw(p, d, c)         memsetw((d), (p), (c))
-#define fillw_io(p, d, c)      memsetw_io((d), (p), (c))
-#endif /* !__x86_64__ */
 
 /* video function table */
 typedef int vi_probe_t(int unit, video_adapter_t **adpp, void *arg, int flags);
index c6ee3b1..39c7aaa 100644 (file)
@@ -1559,8 +1559,8 @@ kmem_slab_alloc(vm_size_t size, vm_offset_t align, int flags)
     }
 
     /*
-     * Allocate the pages.  Do not mess with the PG_ZERO flag or map
-     * them yet.  VM_ALLOC_NORMAL can only be set if we are not preempting.
+     * Allocate the pages.  Do not map them yet.  VM_ALLOC_NORMAL can only
+     * be set if we are not preempting.
      *
      * VM_ALLOC_SYSTEM is automatically set if we are preempting and
      * M_WAITOK was specified as an alternative (i.e. M_USE_RESERVE is
@@ -1634,7 +1634,7 @@ kmem_slab_alloc(vm_size_t size, vm_offset_t align, int flags)
     crit_exit();
 
     /*
-     * Enter the pages into the pmap and deal with PG_ZERO and M_ZERO.
+     * Enter the pages into the pmap and deal with M_ZERO.
      */
     m = mbase;
     i = 0;
@@ -1647,9 +1647,8 @@ kmem_slab_alloc(vm_size_t size, vm_offset_t align, int flags)
        vm_page_wire(m);
        pmap_enter(&kernel_pmap, addr + i, m, VM_PROT_ALL | VM_PROT_NOSYNC,
                   1, NULL);
-       if ((m->flags & PG_ZERO) == 0 && (flags & M_ZERO))
-           bzero((char *)addr + i, PAGE_SIZE);
-       vm_page_flag_clear(m, PG_ZERO);
+       if (flags & M_ZERO)
+               pagezero((char *)addr + i);
        KKASSERT(m->flags & (PG_WRITEABLE | PG_MAPPED));
        vm_page_flag_set(m, PG_REFERENCED);
        vm_page_wakeup(m);
index 9499072..e73260a 100644 (file)
@@ -2243,14 +2243,11 @@ extpacket:
  * Note: "offset" is ill-defined and always called as 0, so ignore it.
  */
 struct mbuf *
-m_devget(char *buf, int len, int offset, struct ifnet *ifp,
-    void (*copy)(volatile const void *from, volatile void *to, size_t length))
+m_devget(char *buf, int len, int offset, struct ifnet *ifp)
 {
        struct mbuf *m, *mfirst = NULL, **mtail;
        int nsize, flags;
 
-       if (copy == NULL)
-               copy = bcopy;
        mtail = &mfirst;
        flags = M_PKTHDR;
 
@@ -2270,7 +2267,7 @@ m_devget(char *buf, int len, int offset, struct ifnet *ifp,
                        flags = 0;
                }
 
-               copy(buf, m->m_data, (unsigned)m->m_len);
+               bcopy(buf, m->m_data, (unsigned)m->m_len);
                buf += m->m_len;
                len -= m->m_len;
                *mtail = m;
index 74820d4..d091bf5 100644 (file)
@@ -1809,7 +1809,6 @@ retry_lookup:
                                    IO_VMIO | ((MAXBSIZE / bsize) << 16),
                                    td->td_ucred);
                        vn_unlock(vp);
-                       vm_page_flag_clear(pg, PG_ZERO);
                        vm_page_busy_wait(pg, FALSE, "sockpg");
                        /*vm_page_io_finish(pg);*/
                        if (error) {
index 5156164..1ead0d3 100644 (file)
@@ -1448,7 +1448,7 @@ brelse(struct buf *bp)
 
                for (i = 0; i < bp->b_xio.xio_npages; i++) {
                        m = bp->b_xio.xio_pages[i];
-                       vm_page_flag_clear(m, PG_ZERO);
+
                        /*
                         * If we hit a bogus page, fixup *all* of them
                         * now.  Note that we left these pages wired
@@ -1844,8 +1844,6 @@ vfs_vmio_release(struct buf *bp)
                 * WARNING: vm_page_try_*() also checks PG_NEED_COMMIT for us.
                 */
                if (m->wire_count == 0) {
-                       vm_page_flag_clear(m, PG_ZERO);
-
                        if (bp->b_flags & B_DIRECT) {
                                /*
                                 * Attempt to free the page if B_DIRECT is
@@ -3330,7 +3328,6 @@ allocbuf(struct buf *bp, int size)
                                        m = bio_page_alloc(bp, obj, pi, desiredpages - bp->b_xio.xio_npages);
                                        if (m) {
                                                vm_page_wire(m);
-                                               vm_page_flag_clear(m, PG_ZERO);
                                                vm_page_wakeup(m);
                                                bp->b_flags &= ~B_CACHE;
                                                bp->b_xio.xio_pages[bp->b_xio.xio_npages] = m;
@@ -3342,7 +3339,6 @@ allocbuf(struct buf *bp, int size)
                                /*
                                 * We found a page and were able to busy it.
                                 */
-                               vm_page_flag_clear(m, PG_ZERO);
                                vm_page_wire(m);
                                vm_page_wakeup(m);
                                bp->b_xio.xio_pages[bp->b_xio.xio_npages] = m;
@@ -3516,7 +3512,6 @@ repurposebuf(struct buf *bp, int size)
                        m = bio_page_alloc(bp, obj, pi, desiredpages - i);
                        if (m) {
                                vm_page_wire(m);
-                               vm_page_flag_clear(m, PG_ZERO);
                                vm_page_wakeup(m);
                                bp->b_flags &= ~B_CACHE;
                                bp->b_xio.xio_pages[i] = m;
@@ -3533,7 +3528,6 @@ repurposebuf(struct buf *bp, int size)
                /*
                 * We found a page and were able to busy it.
                 */
-               vm_page_flag_clear(m, PG_ZERO);
                if (!iswired)
                        vm_page_wire(m);
                vm_page_wakeup(m);
@@ -4029,7 +4023,6 @@ bpdone(struct buf *bp, int elseit)
                        vm_page_busy_wait(m, FALSE, "bpdpgw");
                        if (cmd == BUF_CMD_READ && isbogus == 0 && resid > 0)
                                vfs_clean_one_page(bp, i, m);
-                       vm_page_flag_clear(m, PG_ZERO);
 
                        /*
                         * when debugging new filesystems or buffer I/O
@@ -4199,7 +4192,6 @@ vfs_unbusy_pages(struct buf *bp)
                                bp->b_xio.xio_pages[i] = m;
                        }
                        vm_page_busy_wait(m, FALSE, "bpdpgw");
-                       vm_page_flag_clear(m, PG_ZERO);
                        vm_page_io_finish(m);
                        vm_page_wakeup(m);
                        vm_object_pip_wakeup(obj);
@@ -4271,7 +4263,6 @@ retry:
                for (i = 0; i < bp->b_xio.xio_npages; i++) {
                        vm_page_t m = bp->b_xio.xio_pages[i];
 
-                       vm_page_flag_clear(m, PG_ZERO);
                        if ((bp->b_flags & B_CLUSTER) == 0) {
                                vm_object_pip_add(obj, 1);
                                vm_page_io_start(m);
@@ -4299,7 +4290,6 @@ retry:
                for (i = 0; i < bp->b_xio.xio_npages; i++) {
                        vm_page_t m = bp->b_xio.xio_pages[i];
 
-                       vm_page_flag_clear(m, PG_ZERO); /* XXX */
                        if (bp->b_cmd == BUF_CMD_WRITE) {
                                /*
                                 * When readying a vnode-backed buffer for
@@ -4580,8 +4570,7 @@ vfs_bio_clrbuf(struct buf *bp)
                                bp->b_resid = 0;
                                return;
                        }
-                       if (((bp->b_xio.xio_pages[0]->flags & PG_ZERO) == 0) &&
-                           ((bp->b_xio.xio_pages[0]->valid & mask) == 0)) {
+                       if ((bp->b_xio.xio_pages[0]->valid & mask) == 0) {
                                bzero(bp->b_data, bp->b_bufsize);
                                bp->b_xio.xio_pages[0]->valid |= mask;
                                bp->b_resid = 0;
@@ -4599,18 +4588,16 @@ vfs_bio_clrbuf(struct buf *bp)
                        if ((bp->b_xio.xio_pages[i]->valid & mask) == mask)
                                continue;
                        if ((bp->b_xio.xio_pages[i]->valid & mask) == 0) {
-                               if ((bp->b_xio.xio_pages[i]->flags & PG_ZERO) == 0) {
-                                       bzero(sa, ea - sa);
-                               }
+                               bzero(sa, ea - sa);
                        } else {
                                for (; sa < ea; sa += DEV_BSIZE, j++) {
-                                       if (((bp->b_xio.xio_pages[i]->flags & PG_ZERO) == 0) &&
-                                               (bp->b_xio.xio_pages[i]->valid & (1<<j)) == 0)
+                                       if ((bp->b_xio.xio_pages[i]->valid &
+                                           (1<<j)) == 0) {
                                                bzero(sa, DEV_BSIZE);
+                                       }
                                }
                        }
                        bp->b_xio.xio_pages[i]->valid |= mask;
-                       vm_page_flag_clear(bp->b_xio.xio_pages[i], PG_ZERO);
                }
                bp->b_resid = 0;
        } else {
@@ -4654,7 +4641,6 @@ vm_hold_load_pages(struct buf *bp, vm_offset_t from, vm_offset_t to)
                if (p) {
                        vm_page_wire(p);
                        p->valid = VM_PAGE_BITS_ALL;
-                       vm_page_flag_clear(p, PG_ZERO);
                        pmap_kenter_noinval(pg, VM_PAGE_TO_PHYS(p));
                        bp->b_xio.xio_pages[index] = p;
                        vm_page_wakeup(p);
index efa9979..f30875d 100644 (file)
@@ -114,14 +114,15 @@ static void       bpf_attachd(struct bpf_d *d, struct bpf_if *bp);
 static void    bpf_detachd(struct bpf_d *d);
 static void    bpf_resetd(struct bpf_d *);
 static void    bpf_freed(struct bpf_d *);
-static void    bpf_mcopy(const void *, void *, size_t);
+static void    bpf_mcopy(volatile const void *, volatile void *, size_t);
 static int     bpf_movein(struct uio *, int, struct mbuf **,
                           struct sockaddr *, int *, struct bpf_insn *);
 static int     bpf_setif(struct bpf_d *, struct ifreq *);
 static void    bpf_timed_out(void *);
 static void    bpf_wakeup(struct bpf_d *);
 static void    catchpacket(struct bpf_d *, u_char *, u_int, u_int,
-                           void (*)(const void *, void *, size_t),
+                           void (*)(volatile const void *,
+                                    volatile void *, size_t),
                            const struct timeval *);
 static int     bpf_setf(struct bpf_d *, struct bpf_program *, u_long cmd);
 static int     bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *);
@@ -1219,7 +1220,7 @@ bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen)
                                microtime(&tv);
                                gottime = 1;
                        }
-                       catchpacket(d, pkt, pktlen, slen, ovbcopy, &tv);
+                       catchpacket(d, pkt, pktlen, slen, bcopy, &tv);
                }
        }
        lwkt_reltoken(&bpf_token);
@@ -1230,11 +1231,11 @@ bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen)
  * from m_copydata in sys/uipc_mbuf.c.
  */
 static void
-bpf_mcopy(const void *src_arg, void *dst_arg, size_t len)
+bpf_mcopy(volatile const void *src_arg, volatile void *dst_arg, size_t len)
 {
-       const struct mbuf *m;
+       volatile const struct mbuf *m;
        u_int count;
-       u_char *dst;
+       volatile u_char *dst;
 
        m = src_arg;
        dst = dst_arg;
@@ -1361,7 +1362,7 @@ bpf_ptap(struct bpf_if *bp, struct mbuf *m, const void *data, u_int dlen)
  */
 static void
 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen,
-           void (*cpfn)(const void *, void *, size_t),
+           void (*cpfn)(volatile const void *, volatile void *, size_t),
            const struct timeval *tv)
 {
        struct bpf_hdr *hp;
index 41e9534..2265e92 100644 (file)
  *             void xxattach(void *);
  *             PSEUDO_SET(xxattach, if_xx);
  *
- * - ovbcopy()
- *     in NetBSD 1.4 or later, ovbcopy() is not supplied in the kernel.
- *     we have updated sys/systm.h to include declaration.
- *
- * - splnet()
- *     NetBSD 1.4 or later requires splsoftnet().
- *     other operating systems use splnet().
- *
- * - splimp()
- *     NetBSD-current (2001/4/13): use splnet() in network, splvm() in vm.
- *     other operating systems: use splimp().
- *
- * - struct ifnet for loopback interface
- *     BSDI3: struct ifnet loif;
- *     BSDI4: struct ifnet *loifp;
- *     NetBSD, OpenBSD 2.8, FreeBSD2: struct ifnet loif[NLOOP];
- *     OpenBSD 2.9: struct ifnet *lo0ifp;
- *
- *     odd thing is that many of them refers loif as ifnet *loif,
- *     not loif[NLOOP], from outside of if_loop.c.
- *
- * - number of bpf pseudo devices
- *     others: bpfilter.h, NBPFILTER
- *     FreeBSD4: use_bpf.h, NBPF
- *     solution:
- *             #if defined(__FreeBSD__) && __FreeBSD__ >= 4
- *             #include "use_bpf.h"
- *             #define NBPFILTER       NBPF
- *             #else
- *             #include "bpfilter.h"
- *             #endif
- *
  * - protosw for IPv4 (sys/netinet)
  *     FreeBSD4: struct ipprotosw in netinet/ipprotosw.h
  *     others: struct protosw in sys/protosw.h
index 2b956fc..6b6a70e 100644 (file)
@@ -751,7 +751,7 @@ netmap_grab_packets(struct netmap_kring *kring, struct mbq *q, int force)
                }
                slot->flags &= ~NS_FORWARD; // XXX needed ?
                /* XXX adapt to the case of a multisegment packet */
-               m = m_devget(BDG_NMB(na, slot), slot->len, 0, na->ifp, NULL);
+               m = m_devget(BDG_NMB(na, slot), slot->len, 0, na->ifp);
 
                if (m == NULL)
                        break;
index 901e884..0192764 100644 (file)
@@ -59,9 +59,6 @@
 
 #define BCMP(p1, p2, n) bcmp((char *)(p1), (char *)(p2), (int)(n))
 #define BCOPY(p1, p2, n) bcopy((char *)(p1), (char *)(p2), (int)(n))
-#ifndef _KERNEL
-#define ovbcopy bcopy
-#endif
 
 void
 sl_compress_init(struct slcompress *comp, int max_state)
@@ -436,7 +433,7 @@ sl_uncompress_tcp(u_char **bufp, int len, u_int type, struct slcompress *comp)
         */
        if ((intptr_t)cp & 3) {
                if (len > 0)
-                       ovbcopy(cp, (caddr_t)((intptr_t)cp &~ 3), len);
+                       bcopy(cp, (caddr_t)((intptr_t)cp &~ 3), len);
                cp = (u_char *)((intptr_t)cp &~ 3);
        }
        cp -= hlen;
index 9327139..fcdac71 100644 (file)
@@ -501,11 +501,12 @@ nga_rcv_sync(const sc_p sc, struct mbuf *m, meta_p meta)
        sc->abuf[alen++] = PPP_FLAG;
 
        /* Put frame in an mbuf and ship it off */
-       if (!(m = m_devget(sc->abuf, alen, 0, rcvif, NULL))) {
+       if (!(m = m_devget(sc->abuf, alen, 0, rcvif))) {
                NG_FREE_META(meta);
                error = ENOBUFS;
-       } else
+       } else {
                NG_SEND_DATA(error, sc->async, m, meta);
+       }
        return (error);
 }
 
@@ -563,8 +564,10 @@ nga_rcv_async(const sc_p sc, struct mbuf * m, meta_p meta)
 
                                /* OK, ship it out */
                                if ((n = m_devget(sc->sbuf + skip,
-                                          sc->slen - skip, 0, rcvif, NULL)))
+                                                 sc->slen - skip,
+                                                 0, rcvif))) {
                                        NG_SEND_DATA(error, sc->sync, n, meta);
+                               }
                                sc->stats.asyncFrames++;
 reset:
                                sc->amode = MODE_NORMAL;
index 3ba96a2..2061146 100644 (file)
@@ -297,7 +297,7 @@ NETGRAPH_INIT(l2tp, &ng_l2tp_typestruct);
 #endif
 
 /* memmove macro */
-#define memmove(d, s, l)       ovbcopy(s, d, l)
+#define memmove(d, s, l)       bcopy(s, d, l)
 
 /* Whether to use m_copypacket() or m_dup() */
 #define L2TP_COPY_MBUF         m_copypacket
index 83296d8..c9c0344 100644 (file)
@@ -565,7 +565,7 @@ ng_mppc_compress(node_p node, struct mbuf *m, struct mbuf **resultp)
        *((u_int16_t *)outbuf) = htons(header);
 
        /* Return packet in an mbuf */
-       *resultp = m_devget((caddr_t)outbuf, outlen, 0, NULL, NULL);
+       *resultp = m_devget((caddr_t)outbuf, outlen, 0, NULL);
        kfree(outbuf, M_NETGRAPH);
        return (*resultp == NULL ? ENOBUFS : 0);
 }
@@ -737,7 +737,7 @@ failed:
 #endif
 
        /* Return result in an mbuf */
-       *resultp = m_devget((caddr_t)buf, len, 0, NULL, NULL);
+       *resultp = m_devget((caddr_t)buf, len, 0, NULL);
        kfree(buf, M_NETGRAPH);
        return (*resultp == NULL ? ENOBUFS : 0);
 }
index 9959272..a71fe75 100644 (file)
@@ -586,7 +586,7 @@ ngdwrite(cdev_t dev, struct uio *uio, int flag)
        } else
                kprintf("%s(): len <= 0 : supposed to happen?!\n", __func__);
 
-       m = m_devget(buffer, len, 0, NULL, NULL);
+       m = m_devget(buffer, len, 0, NULL);
 
        NG_SEND_DATA_ONLY(error, connection->active_hook, m);
 
index dd14c1f..e08f94f 100644 (file)
@@ -665,8 +665,8 @@ bad:
                        a->ato = PPTP_MIN_TIMEOUT;
 
                /* Shift packet transmit times in our transmit window */
-               ovbcopy(a->timeSent + index + 1, a->timeSent,
-                   sizeof(*a->timeSent) * (PPTP_XMIT_WIN - (index + 1)));
+               bcopy(a->timeSent + index + 1, a->timeSent,
+                     sizeof(*a->timeSent) * (PPTP_XMIT_WIN - (index + 1)));
 
                /* If we sent an entire window, increase window size by one */
                if (PPTP_SEQ_DIFF(ack, a->winAck) >= 0
index 835164b..c546bde 100644 (file)
@@ -732,7 +732,7 @@ ship_msg(struct ngpcb *pcbp, struct ng_mesg *msg, struct sockaddr_ng *addr)
 
        /* Copy the message itself into an mbuf chain */
        msglen = sizeof(struct ng_mesg) + msg->header.arglen;
-       mdata = m_devget((caddr_t) msg, msglen, 0, NULL, NULL);
+       mdata = m_devget((caddr_t) msg, msglen, 0, NULL);
 
        /* Here we free the message, as we are the end of the line.
         * We need to do that regardless of whether we got mbufs. */
index a07bd04..280156c 100644 (file)
@@ -525,7 +525,7 @@ nga_rcv_sync(const sc_p sc, item_p item)
        sc->abuf[alen++] = PPP_FLAG;
 
        /* Put frame in an mbuf and ship it off */
-       if (!(m = m_devget(sc->abuf, alen, 0, rcvif, NULL))) {
+       if (!(m = m_devget(sc->abuf, alen, 0, rcvif))) {
                NG_FREE_ITEM(item);
                error = ENOBUFS;
        } else {
@@ -590,7 +590,7 @@ nga_rcv_async(const sc_p sc, item_p item)
 
                                /* OK, ship it out */
                                if ((n = m_devget(sc->sbuf + skip,
-                                          sc->slen - skip, 0, rcvif, NULL))) {
+                                          sc->slen - skip, 0, rcvif))) {
                                        if (item) { /* sets NULL -> item */
                                                NG_FWD_NEW_DATA(error, item,
                                                        sc->sync, n);
index 1492f73..f6bf746 100644 (file)
@@ -504,8 +504,7 @@ ng_deflate_compress(node_p node, struct mbuf *m, struct mbuf **resultp)
                ((u_int16_t *)priv->outbuf)[1] = htons(priv->seqnum);
 
                /* Return packet in an mbuf. */
-               *resultp = m_devget((caddr_t)priv->outbuf, outlen, 0, NULL,
-                   NULL);
+               *resultp = m_devget((caddr_t)priv->outbuf, outlen, 0, NULL);
                if (*resultp == NULL) {
                        priv->stats.Errors++;
                        return (ENOMEM);
@@ -617,12 +616,12 @@ ng_deflate_decompress(node_p node, struct mbuf *m, struct mbuf **resultp)
                        priv->outbuf[0] = 0;
                        /* Return packet in an mbuf. */
                        *resultp = m_devget((caddr_t)priv->outbuf, outlen, 0,
-                           NULL, NULL);
+                                           NULL);
                } else {
                        outlen--;
                        /* Return packet in an mbuf. */
                        *resultp = m_devget((caddr_t)(priv->outbuf + 1),
-                           outlen, 0, NULL, NULL);
+                                           outlen, 0, NULL);
                }
                if (*resultp == NULL) {
                        priv->stats.Errors++;
index d57a23e..4a5947b 100644 (file)
@@ -529,7 +529,7 @@ ng_mppc_compress(node_p node, struct mbuf **datap)
                                
                        /* Replace m by the compresed one. */
                        m_freem(m);
-                       m = m_devget((caddr_t)outbuf, outlen, 0, NULL, NULL);
+                       m = m_devget((caddr_t)outbuf, outlen, 0, NULL);
                }
                d->flushed = (rtn & MPPC_EXPANDED) != 0
                    || (flags & MPPC_SAVE_HISTORY) == 0;
@@ -774,7 +774,7 @@ failed:
                len = decomplen - destCnt;
        
                m_freem(m);
-               m = m_devget((caddr_t)decompbuf, len, 0, NULL, NULL);
+               m = m_devget((caddr_t)decompbuf, len, 0, NULL);
                kfree(decompbuf, M_NETGRAPH_MPPC);
        }
 #endif
index 15ce13b..071dea6 100644 (file)
@@ -437,7 +437,7 @@ ng_pred1_compress(node_p node, struct mbuf *m, struct mbuf **resultp)
        outlen += 2;
 
        /* Return packet in an mbuf. */
-       *resultp = m_devget((caddr_t)out, outlen, 0, NULL, NULL);
+       *resultp = m_devget((caddr_t)out, outlen, 0, NULL);
        if (*resultp == NULL) {
            priv->stats.Errors++;
            return (ENOMEM);
@@ -516,7 +516,7 @@ ng_pred1_decompress(node_p node, struct mbuf *m, struct mbuf **resultp)
                }
 
                /* Return packet in an mbuf. */
-               *resultp = m_devget((caddr_t)priv->outbuf, len, 0, NULL, NULL);
+               *resultp = m_devget((caddr_t)priv->outbuf, len, 0, NULL);
                if (*resultp == NULL) {
                        priv->stats.Errors++;
                        return (ENOMEM);
index 77537ed..65b69ae 100644 (file)
@@ -940,7 +940,7 @@ ngs_rcvmsg(node_p node, item_p item, hook_p lasthook)
 
        /* Copy the message itself into an mbuf chain. */
        m = m_devget((caddr_t)msg, sizeof(struct ng_mesg) + msg->header.arglen,
-           0, NULL, NULL);
+                    0, NULL);
 
        /*
         * Here we free the message. We need to do that
index 58fcedd..00d8717 100644 (file)
@@ -1285,7 +1285,7 @@ ip_insertoptions(struct mbuf *m, struct mbuf *opt, int *phlen)
                m->m_data -= optlen;
                m->m_len += optlen;
                m->m_pkthdr.len += optlen;
-               ovbcopy(ip, mtod(m, caddr_t), sizeof(struct ip));
+               bcopy(ip, mtod(m, caddr_t), sizeof(struct ip));
        }
        ip = mtod(m, struct ip *);
        bcopy(p->ipopt_list, ip + 1, optlen);
@@ -1685,7 +1685,7 @@ ip_pcbopts(int optname, struct mbuf **pcbopt, struct mbuf *m)
        cnt = m->m_len;
        m->m_len += sizeof(struct in_addr);
        cp = mtod(m, u_char *) + sizeof(struct in_addr);
-       ovbcopy(mtod(m, caddr_t), cp, cnt);
+       bcopy(mtod(m, caddr_t), cp, cnt);
        bzero(mtod(m, caddr_t), sizeof(struct in_addr));
 
        for (; cnt > 0; cnt -= optlen, cp += optlen) {
@@ -1731,9 +1731,9 @@ ip_pcbopts(int optname, struct mbuf **pcbopt, struct mbuf *m)
                         * Then copy rest of options back
                         * to close up the deleted entry.
                         */
-                       ovbcopy(&cp[IPOPT_OFFSET+1] + sizeof(struct in_addr),
-                               &cp[IPOPT_OFFSET+1],
-                               cnt - (IPOPT_MINOFF - 1));
+                       bcopy(&cp[IPOPT_OFFSET+1] + sizeof(struct in_addr),
+                             &cp[IPOPT_OFFSET+1],
+                             cnt - (IPOPT_MINOFF - 1));
                        break;
                }
        }
index c4ab73b..2797b77 100644 (file)
@@ -471,7 +471,7 @@ ah4_input(struct mbuf **mp, int *offp, int proto)
                 * We do deep-copy since KAME requires that
                 * the packet is placed in a single external mbuf.
                 */
-               ovbcopy((caddr_t)ip, (caddr_t)(((u_char *)ip) + stripsiz), off);
+               bcopy((caddr_t)ip, (caddr_t)(((u_char *)ip) + stripsiz), off);
                m->m_data += stripsiz;
                m->m_len -= stripsiz;
                m->m_pkthdr.len -= stripsiz;
@@ -481,7 +481,7 @@ ah4_input(struct mbuf **mp, int *offp, int proto)
                 * we can compute checksum for multiple AH correctly.
                 */
                if (m->m_len >= stripsiz + off) {
-                       ovbcopy((caddr_t)ip, ((caddr_t)ip) + stripsiz, off);
+                       bcopy((caddr_t)ip, ((caddr_t)ip) + stripsiz, off);
                        m->m_data += stripsiz;
                        m->m_len -= stripsiz;
                        m->m_pkthdr.len -= stripsiz;
@@ -882,7 +882,7 @@ ah6_input(struct mbuf **mp, int *offp, int proto)
                 * We do deep-copy since KAME requires that
                 * the packet is placed in a single mbuf.
                 */
-               ovbcopy((caddr_t)ip6, ((caddr_t)ip6) + stripsiz, off);
+               bcopy((caddr_t)ip6, ((caddr_t)ip6) + stripsiz, off);
                m->m_data += stripsiz;
                m->m_len -= stripsiz;
                m->m_pkthdr.len -= stripsiz;
@@ -892,7 +892,7 @@ ah6_input(struct mbuf **mp, int *offp, int proto)
                 * we can compute checksum for multiple AH correctly.
                 */
                if (m->m_len >= stripsiz + off) {
-                       ovbcopy((caddr_t)ip6, ((caddr_t)ip6) + stripsiz, off);
+                       bcopy((caddr_t)ip6, ((caddr_t)ip6) + stripsiz, off);
                        m->m_data += stripsiz;
                        m->m_len -= stripsiz;
                        m->m_pkthdr.len -= stripsiz;
index 94bdf7f..4a620e4 100644 (file)
@@ -404,7 +404,7 @@ noreplaycheck:
                stripsiz = esplen + ivlen;
 
                ip = mtod(m, struct ip *);
-               ovbcopy((caddr_t)ip, (caddr_t)(((u_char *)ip) + stripsiz), off);
+               bcopy((caddr_t)ip, (caddr_t)(((u_char *)ip) + stripsiz), off);
                m->m_data += stripsiz;
                m->m_len -= stripsiz;
                m->m_pkthdr.len -= stripsiz;
@@ -778,7 +778,7 @@ noreplaycheck:
 
                ip6 = mtod(m, struct ip6_hdr *);
                if (m->m_len >= stripsiz + off) {
-                       ovbcopy((caddr_t)ip6, ((caddr_t)ip6) + stripsiz, off);
+                       bcopy((caddr_t)ip6, ((caddr_t)ip6) + stripsiz, off);
                        m->m_data += stripsiz;
                        m->m_len -= stripsiz;
                        m->m_pkthdr.len -= stripsiz;
index 8a070c2..4a874c6 100644 (file)
@@ -446,8 +446,8 @@ insert:
         * Delete frag6 header with as a few cost as possible.
         */
        if (offset < m->m_len) {
-               ovbcopy((caddr_t)ip6, (caddr_t)ip6 + sizeof(struct ip6_frag),
-                       offset);
+               bcopy((caddr_t)ip6, (caddr_t)ip6 + sizeof(struct ip6_frag),
+                     offset);
                m->m_data += sizeof(struct ip6_frag);
                m->m_len -= sizeof(struct ip6_frag);
        } else {
index aa085ba..f9aa5c4 100644 (file)
@@ -2028,7 +2028,7 @@ ipsec4_encapsulate(struct mbuf *m, struct secasvar *sav)
                oip = mtod(m->m_next, struct ip *);
        }
        ip = mtod(m, struct ip *);
-       ovbcopy((caddr_t)ip, (caddr_t)oip, hlen);
+       bcopy((caddr_t)ip, (caddr_t)oip, hlen);
        m->m_len = sizeof(struct ip);
        m->m_pkthdr.len -= (hlen - sizeof(struct ip));
 
@@ -2125,7 +2125,7 @@ ipsec6_encapsulate(struct mbuf *m, struct secasvar *sav)
                oip6 = mtod(m->m_next, struct ip6_hdr *);
        }
        ip6 = mtod(m, struct ip6_hdr *);
-       ovbcopy((caddr_t)ip6, (caddr_t)oip6, sizeof(struct ip6_hdr));
+       bcopy((caddr_t)ip6, (caddr_t)oip6, sizeof(struct ip6_hdr));
 
        /* XXX: Fake scoped addresses */
        in6_clearscope(&oip6->ip6_src);
index 4e88dd9..9ab30ac 100644 (file)
@@ -404,8 +404,8 @@ ieee80211_decap1(struct mbuf *m, int *framelen)
        *framelen = ntohs(eh->ether_type)       /* encap'd frame size */
                  + sizeof(struct ether_header) - sizeof(struct llc);
        eh->ether_type = llc->llc_un.type_snap.ether_type;
-       ovbcopy(eh, mtod(m, uint8_t *) + sizeof(struct llc),
-               sizeof(struct ether_header));
+       bcopy(eh, mtod(m, uint8_t *) + sizeof(struct llc),
+             sizeof(struct ether_header));
        m_adj(m, sizeof(struct llc));
        return m;
 #undef FF_LLC_SIZE
index ad4a04e..8ed74d5 100644 (file)
@@ -3451,9 +3451,9 @@ ieee80211_beacon_update(struct ieee80211_node *ni, struct mbuf *m, int mcast)
                                /* copy up/down trailer */
                                int adjust = tie->tim_bitmap+timlen
                                           - bo->bo_tim_trailer;
-                               ovbcopy(bo->bo_tim_trailer,
-                                   bo->bo_tim_trailer+adjust,
-                                   bo->bo_tim_trailer_len);
+                               bcopy(bo->bo_tim_trailer,
+                                     bo->bo_tim_trailer+adjust,
+                                     bo->bo_tim_trailer_len);
                                bo->bo_tim_trailer += adjust;
                                bo->bo_erp += adjust;
                                bo->bo_htinfo += adjust;
@@ -3564,8 +3564,8 @@ ieee80211_beacon_update(struct ieee80211_node *ni, struct mbuf *m, int mcast)
                if (aielen != bo->bo_appie_len) {
                        /* copy up/down trailer */
                        int adjust = aielen - bo->bo_appie_len;
-                       ovbcopy(bo->bo_tim_trailer, bo->bo_tim_trailer+adjust,
-                               bo->bo_tim_trailer_len);
+                       bcopy(bo->bo_tim_trailer, bo->bo_tim_trailer+adjust,
+                             bo->bo_tim_trailer_len);
                        bo->bo_tim_trailer += adjust;
                        bo->bo_appie += adjust;
                        bo->bo_appie_len = aielen;
index a8a4aa2..d95e949 100644 (file)
@@ -181,7 +181,7 @@ ccmp_encap(struct ieee80211_key *k, struct mbuf *m)
        if (m == NULL)
                return 0;
        ivp = mtod(m, uint8_t *);
-       ovbcopy(ivp + ccmp.ic_header, ivp, hdrlen);
+       bcopy(ivp + ccmp.ic_header, ivp, hdrlen);
        ivp += hdrlen;
 
        ccmp_setiv(k, ivp);
@@ -269,7 +269,7 @@ ccmp_decap(struct ieee80211_key *k, struct mbuf *m, int hdrlen)
        /*
         * Copy up 802.11 header and strip crypto bits.
         */
-       ovbcopy(mtod(m, void *), mtod(m, uint8_t *) + ccmp.ic_header, hdrlen);
+       bcopy(mtod(m, void *), mtod(m, uint8_t *) + ccmp.ic_header, hdrlen);
        m_adj(m, ccmp.ic_header);
        m_adj(m, -ccmp.ic_trailer);
 
index e339b2d..547eb4c 100644 (file)
@@ -197,7 +197,7 @@ wep_encap(struct ieee80211_key *k, struct mbuf *m)
        if (m == NULL)
                return 0;
        ivp = mtod(m, uint8_t *);
-       ovbcopy(ivp + wep.ic_header, ivp, hdrlen);
+       bcopy(ivp + wep.ic_header, ivp, hdrlen);
        ivp += hdrlen;
 
        wep_setiv(k, ivp);
@@ -252,7 +252,7 @@ wep_decap(struct ieee80211_key *k, struct mbuf *m, int hdrlen)
        /*
         * Copy up 802.11 header and strip crypto bits.
         */
-       ovbcopy(mtod(m, void *), mtod(m, uint8_t *) + wep.ic_header, hdrlen);
+       bcopy(mtod(m, void *), mtod(m, uint8_t *) + wep.ic_header, hdrlen);
        m_adj(m, wep.ic_header);
        m_adj(m, -wep.ic_trailer);
 
index 501bd97..4c04299 100644 (file)
@@ -293,8 +293,8 @@ m_makespace(struct mbuf *m0, int skip, int hlen, int *off)
                 * so there's space to write the new header.
                 */
                /* XXX can this be memcpy? does it handle overlap? */
-               ovbcopy(mtod(m, caddr_t) + skip,
-                       mtod(m, caddr_t) + skip + hlen, remain);
+               bcopy(mtod(m, caddr_t) + skip,
+                     mtod(m, caddr_t) + skip + hlen, remain);
                m->m_len += hlen;
                *off = skip;
        }
index 68d1c3e..b7f54cd 100644 (file)
@@ -614,7 +614,7 @@ ipsec6_encapsulate(struct mbuf *m, struct secasvar *sav)
                oip6 = mtod(m->m_next, struct ip6_hdr *);
        }
        ip6 = mtod(m, struct ip6_hdr *);
-       ovbcopy((caddr_t)ip6, (caddr_t)oip6, sizeof(struct ip6_hdr));
+       bcopy((caddr_t)ip6, (caddr_t)oip6, sizeof(struct ip6_hdr));
 
        /* Fake link-local scope-class addresses */
        if (IN6_IS_SCOPE_LINKLOCAL(&oip6->ip6_src))
index 91c7cfa..0991f37 100644 (file)
@@ -85,7 +85,6 @@ void  doreti_iret_fault(void) __asm(__STRING(doreti_iret_fault));
 void   enable_sse(void);
 void   fillw(int /*u_short*/ pat, void *base, size_t cnt);
 void   pagezero(void *addr);
-void   pagecopy(void *from, void *to);
 void   setidt_global(int idx, alias_for_inthand_t *func,
            int typ, int dpl, int ist);
 void   setidt(int idx, alias_for_inthand_t *func,
index 998ef4a..89e23f1 100644 (file)
@@ -2023,16 +2023,8 @@ pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, pv_entry_t *pvpp)
        vm_page_spin_unlock(m);
        vm_page_unmanage(m);    /* m must be spinunlocked */
 
-       if ((m->flags & PG_ZERO) == 0) {
-               pmap_zero_page(VM_PAGE_TO_PHYS(m));
-       }
-#ifdef PMAP_DEBUG
-       else {
-               pmap_page_assertzero(VM_PAGE_TO_PHYS(m));
-       }
-#endif
+       pmap_zero_page(VM_PAGE_TO_PHYS(m));
        m->valid = VM_PAGE_BITS_ALL;
-       vm_page_flag_clear(m, PG_ZERO);
        vm_page_wire(m);        /* wire for mapping in parent */
 
        /*
@@ -2483,11 +2475,6 @@ pmap_release_pv(pv_entry_t pv, pv_entry_t pvp, pmap_inval_bulk_t *bulk)
        vm_page_unwire(p, 0);
        KKASSERT(p->wire_count == 0);
 
-       /*
-        * Theoretically this page, if not the pml4 page, should contain
-        * all-zeros.  But its just too dangerous to mark it PG_ZERO.  Free
-        * normally.
-        */
        vm_page_free(p);
 skip:
        pv_free(pv);
@@ -2773,9 +2760,7 @@ pmap_growkernel(vm_offset_t kstart, vm_offset_t kend)
                                      "kernel");
                        }
                        paddr = VM_PAGE_TO_PHYS(nkpg);
-                       if ((nkpg->flags & PG_ZERO) == 0)
-                               pmap_zero_page(paddr);
-                       vm_page_flag_clear(nkpg, PG_ZERO);
+                       pmap_zero_page(paddr);
                        newpd = (pdp_entry_t)
                            (paddr |
                            kernel_pmap.pmap_bits[PG_V_IDX] |
@@ -2809,7 +2794,6 @@ pmap_growkernel(vm_offset_t kstart, vm_offset_t kend)
                vm_page_wire(nkpg);
                ptppaddr = VM_PAGE_TO_PHYS(nkpg);
                pmap_zero_page(ptppaddr);
-               vm_page_flag_clear(nkpg, PG_ZERO);
                newpt = (pd_entry_t) (ptppaddr |
                    kernel_pmap.pmap_bits[PG_V_IDX] |
                    kernel_pmap.pmap_bits[PG_RW_IDX] |
@@ -4579,25 +4563,6 @@ pmap_zero_page(vm_paddr_t phys)
        pagezero((void *)va);
 }
 
-/*
- * pmap_page_assertzero:
- *
- *     Assert that a page is empty, panic if it isn't.
- */
-void
-pmap_page_assertzero(vm_paddr_t phys)
-{
-       vm_offset_t va = PHYS_TO_DMAP(phys);
-       size_t i;
-
-       for (i = 0; i < PAGE_SIZE; i += sizeof(long)) {
-               if (*(long *)((char *)va + i) != 0) {
-                       panic("pmap_page_assertzero() @ %p not zero!",
-                             (void *)(intptr_t)va);
-               }
-       }
-}
-
 /*
  * pmap_zero_page:
  *
index 0d890e3..651166e 100644 (file)
        .text
 
 /*
- * bcopy family
- * void bzero(void *buf, size_t len)
+ * bzero(ptr:%rdi, bytes:%rsi)
+ *
+ * Using rep stosq is 70% faster than a %rax loop and almost as fast as
+ * a %xmm0 loop on a modern intel cpu.
+ *
+ * Do not use non-termportal instructions here as we do not know the caller's
+ * intent.
  */
-
-/* done */
 ENTRY(bzero)
        movq    %rsi,%rcx
        xorl    %eax,%eax
@@ -59,21 +62,26 @@ ENTRY(bzero)
        stosb
        ret
 
-/* Address: %rdi */
+/*
+ * pagezero(ptr:%rdi)
+ *
+ * Using rep stosq is nearly as fast as using %xmm0 on a modern intel cpu,
+ * and about 70% faster than a %rax loop.
+ *
+ * Do not use non-termportal instructions here as we do not know the caller's
+ * intent.
+ */
 ENTRY(pagezero)
-       movq    $-PAGE_SIZE,%rdx
-       subq    %rdx,%rdi
+       movq    $PAGE_SIZE>>3,%rcx
        xorl    %eax,%eax
-1:
-       movq    %rax,(%rdi,%rdx)        /* movnti */
-       movq    %rax,8(%rdi,%rdx)       /* movnti */
-       movq    %rax,16(%rdi,%rdx)      /* movnti */
-       movq    %rax,24(%rdi,%rdx)      /* movnti */
-       addq    $32,%rdx
-       jne     1b
-       /*sfence*/
+       cld
+       rep
+       stosq
        ret
 
+/*
+ * bcmp(ptr:%rdi, ptr:%rsi, bytes:%rdx)
+ */
 ENTRY(bcmp)
        movq    %rdx,%rcx
        shrq    $3,%rcx
@@ -92,12 +100,10 @@ ENTRY(bcmp)
        ret
 
 /*
- * bcopy(src, dst, cnt)
- *       rdi, rsi, rdx
- *  ws@tools.de     (Wolfgang Solfrank, TooLs GmbH) +49-228-985800
+ * bcopy(src:%rdi, dst:%rsi, cnt:%rdx)
+ *
+ * ws@tools.de (Wolfgang Solfrank, TooLs GmbH) +49-228-985800
  */
-ENTRY(generic_bcopy)   /* generic_bcopy is bcopy without FPU */
-ENTRY(ovbcopy) /* our bcopy doesn't use the FPU, so ovbcopy is the same */
 ENTRY(bcopy)
        xchgq   %rsi,%rdi
        movq    %rdx,%rcx
@@ -117,7 +123,7 @@ ENTRY(bcopy)
        movsb
        ret
 
-       /* ALIGN_TEXT */
+       ALIGN_TEXT
 1:
        addq    %rcx,%rdi                       /* copy backwards */
        addq    %rcx,%rsi
@@ -127,7 +133,7 @@ ENTRY(bcopy)
        std
        rep
        movsb
-       movq    %rdx,%rcx                       /* copy remainder by 32-bit words */
+       movq    %rdx,%rcx                       /* copy by 32-bit words */
        shrq    $3,%rcx
        subq    $7,%rsi
        subq    $7,%rdi
@@ -135,6 +141,7 @@ ENTRY(bcopy)
        movsq
        cld
        ret
+
 ENTRY(reset_dbregs)
        movq    $0x200,%rax   /* the manual says that bit 10 must be set to 1 */
        movq    %rax,%dr7     /* disable all breapoints first */
@@ -161,32 +168,6 @@ ENTRY(memcpy)
        movsb
        ret
 
-/*
- * pagecopy(%rdi=from, %rsi=to)
- */
-ENTRY(pagecopy)
-       movq    $-PAGE_SIZE,%rax
-       movq    %rax,%rdx
-       subq    %rax,%rdi
-       subq    %rax,%rsi
-1:
-       /*prefetchnta (%rdi,%rax)*/
-       /*addq  $64,%rax*/
-       /*jne   1b*/
-2:
-       movq    (%rdi,%rdx),%rax
-       movq    %rax,(%rsi,%rdx)        /* movnti */
-       movq    8(%rdi,%rdx),%rax
-       movq    %rax,8(%rsi,%rdx)       /* movnti */
-       movq    16(%rdi,%rdx),%rax
-       movq    %rax,16(%rsi,%rdx)      /* movnti */
-       movq    24(%rdi,%rdx),%rax
-       movq    %rax,24(%rsi,%rdx)      /* movnti */
-       addq    $32,%rdx
-       jne     2b
-       /*sfence*/
-       ret
-
 /* fillw(pat, base, cnt) */  
 /*       %rdi,%rsi, %rdx */
 ENTRY(fillw)
index 475941d..0f2cc5f 100644 (file)
 /*
  * A bcopy that works dring low level boot, before FP is working
  */
-void
-ovbcopy(const void *src, void *dst, size_t len)
-{
-       bcopy(src, dst, len);
-}
-
-void
-bcopyi(const void *src, void *dst, size_t len)
-{
-       bcopy(src, dst, len);
-}
-
 u_long
 casuword(volatile u_long *p, u_long oldval, u_long newval)
 {
index b2a1534..c5071ad 100644 (file)
@@ -1380,11 +1380,10 @@ pmap_release_free_page(struct pmap *pmap, vm_page_t p)
        /*
         * We leave the top-level page table page cached, wired, and mapped in
         * the pmap until the dtor function (pmap_puninit()) gets called.
-        * However, still clean it up so we can set PG_ZERO.
+        * However, still clean it up.
         */
        if (p->pindex == NUPDE + NUPDPE + PML4PML4I) {
                bzero(pmap->pm_pml4, PAGE_SIZE);
-               vm_page_flag_set(p, PG_ZERO);
                vm_page_wakeup(p);
        } else {
                abort();
@@ -1710,9 +1709,7 @@ pmap_growkernel(vm_offset_t kstart, vm_offset_t kend)
                                      "grow kernel");
                        }
                        paddr = VM_PAGE_TO_PHYS(nkpg);
-                       if ((nkpg->flags & PG_ZERO) == 0)
-                               pmap_zero_page(paddr);
-                       vm_page_flag_clear(nkpg, PG_ZERO);
+                       pmap_zero_page(paddr);
                        newpdp = (pdp_entry_t)(paddr |
                            VPTE_V | VPTE_RW | VPTE_U |
                            VPTE_A | VPTE_M);
@@ -1743,7 +1740,6 @@ pmap_growkernel(vm_offset_t kstart, vm_offset_t kend)
                vm_page_wire(nkpg);
                ptppaddr = VM_PAGE_TO_PHYS(nkpg);
                pmap_zero_page(ptppaddr);
-               vm_page_flag_clear(nkpg, PG_ZERO);
                newpdir = (pd_entry_t)(ptppaddr |
                    VPTE_V | VPTE_RW | VPTE_U |
                    VPTE_A | VPTE_M);
@@ -2829,28 +2825,6 @@ pmap_zero_page(vm_paddr_t phys)
        bzero((void *)va, PAGE_SIZE);
 }
 
-/*
- * pmap_page_assertzero:
- *
- *     Assert that a page is empty, panic if it isn't.
- */
-void
-pmap_page_assertzero(vm_paddr_t phys)
-{
-       int i;
-
-       crit_enter();
-       vm_offset_t virt = PHYS_TO_DMAP(phys);
-
-       for (i = 0; i < PAGE_SIZE; i += sizeof(int)) {
-           if (*(int *)((char *)virt + i) != 0) {
-               panic("pmap_page_assertzero() @ %p not zero!",
-                   (void *)virt);
-           }
-       }
-       crit_exit();
-}
-
 /*
  * pmap_zero_page:
  *
index e26f9b0..95cd7e9 100644 (file)
@@ -506,8 +506,7 @@ struct      mbuf    *m_copym(const struct mbuf *, int, int, int);
 struct mbuf    *m_copypacket(struct mbuf *, int);
 struct mbuf    *m_defrag(struct mbuf *, int);
 struct mbuf    *m_defrag_nofree(struct mbuf *, int);
-struct mbuf    *m_devget(char *, int, int, struct ifnet *,
-                 void (*copy)(volatile const void *, volatile void *, size_t));
+struct mbuf    *m_devget(char *, int, int, struct ifnet *);
 struct mbuf    *m_dup(struct mbuf *, int);
 struct mbuf    *m_dup_data(struct mbuf *, int);
 int             m_dup_pkthdr(struct mbuf *, const struct mbuf *, int);
index 195e025..2ad9de1 100644 (file)
@@ -242,17 +242,11 @@ u_quad_t strtouq(const char *, char **, int) __nonnull(1);
  *
  * NOTE: bcopyb() - is a dumb byte-granular bcopy.  This routine is
  *                 explicitly not meant to be sophisticated.
- * NOTE: bcopyi() - is a dumb int-granular bcopy (len is still in bytes).
- *                 This routine is explicitly not meant to be sophisticated.
  */
 void   bcopyb(const void *from, void *to, size_t len)
            __nonnull(1) __nonnull(2);
-void   bcopyi(const void *from, void *to, size_t len)
-           __nonnull(1) __nonnull(2);
 void   bcopy(volatile const void *from, volatile void *to, size_t len)
            __nonnull(1) __nonnull(2);
-void   ovbcopy(const void *from, void *to, size_t len)
-           __nonnull(1) __nonnull(2);
 void   bzero(volatile void *buf, size_t len) __nonnull(1);
 void   bzeront(volatile void *buf, size_t len) __nonnull(1);
 void   *memcpy(void *to, const void *from, size_t len)
index 338d9ab..de42687 100644 (file)
@@ -2051,8 +2051,6 @@ devfs_spec_getpages(struct vop_getpages_args *ap)
                nextoff = toff + PAGE_SIZE;
                m = ap->a_m[i];
 
-               m->flags &= ~PG_ZERO;
-
                /*
                 * NOTE: vm_page_undirty/clear_dirty etc do not clear the
                 *       pmap modified bit.  pmap modified bit should have
index 1231685..fcee3de 100644 (file)
@@ -481,8 +481,6 @@ smbfs_getpages(struct vop_getpages_args *ap)
                nextoff = toff + PAGE_SIZE;
                m = pages[i];
 
-               m->flags &= ~PG_ZERO;
-
                /*
                 * NOTE: pmap dirty bit should have already been cleared.
                 *       We do not clear it here.
index 708676a..84c8ed2 100644 (file)
@@ -82,9 +82,7 @@ phys_pager_getpage(vm_object_t object, vm_page_t *mpp, int seqaccess)
 {
        vm_page_t m = *mpp;
 
-       if ((m->flags & PG_ZERO) == 0)
-               vm_page_zero_fill(m);
-       vm_page_flag_set(m, PG_ZERO);
+       vm_page_zero_fill(m);
        /* Switch off pv_entries */
        vm_page_unmanage(m);
        m->valid = VM_PAGE_BITS_ALL;
index fe4f4bc..7d534c7 100644 (file)
@@ -186,7 +186,6 @@ void                 pmap_reference (pmap_t);
 void            pmap_remove (pmap_t, vm_offset_t, vm_offset_t);
 void            pmap_remove_pages (pmap_t, vm_offset_t, vm_offset_t);
 void            pmap_zero_page (vm_paddr_t);
-void            pmap_page_assertzero (vm_paddr_t);
 void            pmap_zero_page_area (vm_paddr_t, int off, int size);
 int             pmap_prefault_ok (pmap_t, vm_offset_t);
 void            pmap_change_attr(vm_offset_t va, vm_size_t count, int mode);
index 3ca1eeb..3f4a9e6 100644 (file)
@@ -1796,7 +1796,6 @@ swp_pager_async_iodone(struct bio *bio)
                                 */
 
                                m->valid = 0;
-                               vm_page_flag_clear(m, PG_ZERO);
                                vm_page_flag_clear(m, PG_SWAPINPROG);
 
                                /*
@@ -1850,8 +1849,6 @@ swp_pager_async_iodone(struct bio *bio)
                         * that existed in the old swapper for a time before
                         * it got ripped out due to precisely this problem.
                         *
-                        * clear PG_ZERO in page.
-                        *
                         * If not the requested page then deactivate it.
                         *
                         * Note that the requested page, reqpage, is left
@@ -1869,7 +1866,7 @@ swp_pager_async_iodone(struct bio *bio)
                        /*pmap_clear_modify(m);*/
                        m->valid = VM_PAGE_BITS_ALL;
                        vm_page_undirty(m);
-                       vm_page_flag_clear(m, PG_ZERO | PG_SWAPINPROG);
+                       vm_page_flag_clear(m, PG_SWAPINPROG);
                        vm_page_flag_set(m, PG_SWAPPED);
 
                        /*
index f4bea30..1f54b57 100644 (file)
@@ -412,11 +412,12 @@ again:
                        KKASSERT(m->busy == 0);
 
                        /*
-                        * Clear all flags except PG_BUSY, PG_ZERO, and
-                        * PG_WANTED, then unbusy the now allocated page.
+                        * Clear all flags except PG_[S]BUSY and PG_WANTED,
+                        * then unbusy the now allocated page.
                         */
-                       vm_page_flag_clear(m, ~(PG_BUSY | PG_SBUSY |
-                                               PG_ZERO | PG_WANTED));
+                       vm_page_flag_clear(m, ~(PG_BUSY |
+                                               PG_SBUSY |
+                                               PG_WANTED));
                        vm_page_wire(m);
                        vm_page_wakeup(m);
                }
index 606bb7c..46ea82f 100644 (file)
@@ -1743,17 +1743,9 @@ readrest:
                        fs->first_m = NULL;
 
                        /*
-                        * Zero the page if necessary and mark it valid.
+                        * Zero the page and mark it valid.
                         */
-                       if ((fs->m->flags & PG_ZERO) == 0) {
-                               vm_page_zero_fill(fs->m);
-                       } else {
-#ifdef PMAP_DEBUG
-                               pmap_page_assertzero(VM_PAGE_TO_PHYS(fs->m));
-#endif
-                               vm_page_flag_clear(fs->m, PG_ZERO);
-                               mycpu->gd_cnt.v_ozfod++;
-                       }
+                       vm_page_zero_fill(fs->m);
                        mycpu->gd_cnt.v_zfod++;
                        fs->m->valid = VM_PAGE_BITS_ALL;
                        break;  /* break to PAGE HAS BEEN FOUND */
@@ -2000,7 +1992,6 @@ readrest:
                vm_page_zero_invalid(fs->m, TRUE);
                kprintf("Warning: page %p partially invalid on fault\n", fs->m);
        }
-       vm_page_flag_clear(fs->m, PG_ZERO);
 
        return (KERN_SUCCESS);
 }
@@ -2247,8 +2238,6 @@ vm_fault_copy_entry(vm_map_t dst_map, vm_map_t src_map,
                /*
                 * Enter it in the pmap...
                 */
-
-               vm_page_flag_clear(dst_m, PG_ZERO);
                pmap_enter(dst_map->pmap, vaddr, dst_m, prot, FALSE, dst_entry);
 
                /*
@@ -2699,16 +2688,7 @@ vm_prefault(pmap_t pmap, vm_offset_t addra, vm_map_entry_t entry, int prot,
                        /*
                         * Page must be zerod.
                         */
-                       if ((m->flags & PG_ZERO) == 0) {
-                               vm_page_zero_fill(m);
-                       } else {
-#ifdef PMAP_DEBUG
-                               pmap_page_assertzero(
-                                               VM_PAGE_TO_PHYS(m));
-#endif
-                               vm_page_flag_clear(m, PG_ZERO);
-                               mycpu->gd_cnt.v_ozfod++;
-                       }
+                       vm_page_zero_fill(m);
                        mycpu->gd_cnt.v_zfod++;
                        m->valid = VM_PAGE_BITS_ALL;
 
index 311463d..e764360 100644 (file)
@@ -399,7 +399,7 @@ kmem_alloc_wait(vm_map_t map, vm_size_t size)
  */
 vm_offset_t
 kmem_alloc_attr(vm_map_t map, vm_size_t size, int flags, vm_paddr_t low,
-    vm_paddr_t high, vm_memattr_t memattr)
+               vm_paddr_t high, vm_memattr_t memattr)
 {
        vm_offset_t addr, i, offset;
        vm_page_t m;
@@ -433,7 +433,7 @@ kmem_alloc_attr(vm_map_t map, vm_size_t size, int flags, vm_paddr_t low,
                vm_object_hold(&kernel_object);
                vm_page_insert(m, &kernel_object, OFF_TO_IDX(offset + i));
                vm_object_drop(&kernel_object);
-               if ((flags & M_ZERO) && (m->flags & PG_ZERO) == 0)
+               if (flags & M_ZERO)
                        pmap_zero_page(VM_PAGE_TO_PHYS(m));
                m->valid = VM_PAGE_BITS_ALL;
        }
index 932d367..a0bd752 100644 (file)
@@ -251,20 +251,7 @@ vm_add_new_page(vm_paddr_t pa)
        atomic_add_int(&vmstats.v_page_count, 1);
        atomic_add_int(&vmstats.v_free_count, 1);
        vpq = &vm_page_queues[m->queue];
-#if 0
-       /* too expensive time-wise in large-mem configurations */
-       if ((vpq->flipflop & 15) == 0) {
-               pmap_zero_page(VM_PAGE_TO_PHYS(m));
-               m->flags |= PG_ZERO;
-               TAILQ_INSERT_TAIL(&vpq->pl, m, pageq);
-               ++vpq->zero_count;
-       } else {
-#endif
-               TAILQ_INSERT_HEAD(&vpq->pl, m, pageq);
-#if 0
-       }
-       ++vpq->flipflop;
-#endif
+       TAILQ_INSERT_HEAD(&vpq->pl, m, pageq);
        ++vpq->lcnt;
 }
 
@@ -687,8 +674,6 @@ _vm_page_rem_queue_spinlocked(vm_page_t m)
                pq->lcnt--;
                m->queue = PQ_NONE;
                oqueue = queue;
-               if ((queue - m->pc) == PQ_FREE && (m->flags & PG_ZERO))
-                       --pq->zero_count;
                if ((queue - m->pc) == PQ_CACHE || (queue - m->pc) == PQ_FREE)
                        queue -= m->pc;
                vm_page_queues_spin_unlock(oqueue);     /* intended */
@@ -717,16 +702,11 @@ _vm_page_add_queue_spinlocked(vm_page_t m, u_short queue, int athead)
                m->queue = queue;
 
                /*
-                * Put zero'd pages on the end ( where we look for zero'd pages
-                * first ) and non-zerod pages at the head.
+                * PQ_FREE is always handled LIFO style to try to provide
+                * cache-hot pages to programs.
                 */
                if (queue - m->pc == PQ_FREE) {
-                       if (m->flags & PG_ZERO) {
-                               TAILQ_INSERT_TAIL(&pq->pl, m, pageq);
-                               ++pq->zero_count;
-                       } else {
-                               TAILQ_INSERT_HEAD(&pq->pl, m, pageq);
-                       }
+                       TAILQ_INSERT_HEAD(&pq->pl, m, pageq);
                } else if (athead) {
                        TAILQ_INSERT_HEAD(&pq->pl, m, pageq);
                } else {
@@ -1787,7 +1767,7 @@ done:
         * Initialize the structure, inheriting some flags but clearing
         * all the rest.  The page has already been busied for us.
         */
-       vm_page_flag_clear(m, ~(PG_ZERO | PG_BUSY | PG_SBUSY));
+       vm_page_flag_clear(m, ~(PG_BUSY | PG_SBUSY));
        KKASSERT(m->wire_count == 0);
        KKASSERT(m->busy == 0);
        m->act_count = 0;
@@ -2203,7 +2183,6 @@ vm_page_free_toq(vm_page_t m)
                vm_page_flag_clear(m, PG_NEED_COMMIT);
 
        if (m->hold_count != 0) {
-               vm_page_flag_clear(m, PG_ZERO);
                _vm_page_add_queue_spinlocked(m, PQ_HOLD + m->pc, 0);
        } else {
                _vm_page_add_queue_spinlocked(m, PQ_FREE + m->pc, 0);
@@ -2225,76 +2204,6 @@ vm_page_free_toq(vm_page_t m)
        vm_page_free_wakeup();
 }
 
-/*
- * vm_page_free_fromq_fast()
- *
- * Remove a non-zero page from one of the free queues; the page is removed for
- * zeroing, so do not issue a wakeup.
- *
- * Our zeroidle code is now per-cpu so only do a limited scan.  We try to
- * stay within a single cpu's domain but we do a little statistical
- * improvement by encompassing two cpu's domains worst-case.
- */
-vm_page_t
-vm_page_free_fromq_fast(void)
-{
-       globaldata_t gd = mycpu;
-       vm_page_t m;
-       int i;
-       int qi;
-
-       m = NULL;
-       qi = vm_get_pg_color(gd, NULL, ++gd->gd_quick_color);
-       qi = qi & PQ_L2_MASK;
-
-       /*
-        * 16 = one cpu's domain
-        * 32 = two cpu's domains
-        * (note masking at bottom of loop!)
-        */
-       for (i = 0; i < 10; ++i) {
-               m = vm_page_list_find(PQ_FREE, qi, FALSE);
-               /* page is returned spinlocked and removed from its queue */
-               if (m) {
-                       if (vm_page_busy_try(m, TRUE)) {
-                               /*
-                                * We were unable to busy the page, deactivate
-                                * it and loop.
-                                */
-                               _vm_page_deactivate_locked(m, 0);
-                               vm_page_spin_unlock(m);
-                       } else if (m->flags & PG_ZERO) {
-                               /*
-                                * The page is already PG_ZERO, requeue it
-                                * and loop.
-                                */
-                               _vm_page_add_queue_spinlocked(m,
-                                                             PQ_FREE + m->pc,
-                                                             0);
-                               vm_page_queue_spin_unlock(m);
-                               if (_vm_page_wakeup(m)) {
-                                       vm_page_spin_unlock(m);
-                                       wakeup(m);
-                               } else {
-                                       vm_page_spin_unlock(m);
-                               }
-                       } else {
-                               /*
-                                * The page is not PG_ZERO'd so return it.
-                                */
-                               KKASSERT((m->flags & (PG_UNMANAGED |
-                                                     PG_NEED_COMMIT)) == 0);
-                               KKASSERT(m->hold_count == 0);
-                               KKASSERT(m->wire_count == 0);
-                               vm_page_spin_unlock(m);
-                               break;
-                       }
-                       m = NULL;
-               }
-       }
-       return (m);
-}
-
 /*
  * vm_page_unmanage()
  *
@@ -2769,8 +2678,6 @@ vm_page_clear_commit(vm_page_t m)
  *
  * This routine may not be called from an interrupt.
  *
- * PG_ZERO is *ALWAYS* cleared by this routine.
- *
  * No other requirements.
  */
 vm_page_t
@@ -2817,18 +2724,26 @@ vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags)
         *
         * If VM_ALLOC_FORCE_ZERO the page is unconditionally zero'd and set
         * valid even if already valid.
+        *
+        * NOTE!  We have removed all of the PG_ZERO optimizations and also
+        *        removed the idle zeroing code.  These optimizations actually
+        *        slow things down on modern cpus because the zerod area is
+        *        likely uncached, placing a memory-access burden on the
+        *        accesors taking the fault.
+        *
+        *        By always zeroing the page in-line with the fault, no
+        *        dynamic ram reads are needed and the caches are hot, ready
+        *        for userland to access the memory.
         */
        if (m->valid == 0) {
                if (allocflags & (VM_ALLOC_ZERO | VM_ALLOC_FORCE_ZERO)) {
-                       if ((m->flags & PG_ZERO) == 0)
-                               pmap_zero_page(VM_PAGE_TO_PHYS(m));
+                       pmap_zero_page(VM_PAGE_TO_PHYS(m));
                        m->valid = VM_PAGE_BITS_ALL;
                }
        } else if (allocflags & VM_ALLOC_FORCE_ZERO) {
                pmap_zero_page(VM_PAGE_TO_PHYS(m));
                m->valid = VM_PAGE_BITS_ALL;
        }
-       vm_page_flag_clear(m, PG_ZERO);
 failed:
        vm_object_drop(object);
        return(m);
index dd42fec..6ee8ca4 100644 (file)
@@ -287,7 +287,7 @@ extern struct vpgqueues vm_page_queues[PQ_COUNT];
 #define        PG_FICTITIOUS   0x00000008      /* physical page doesn't exist (O) */
 #define        PG_WRITEABLE    0x00000010      /* page is writeable */
 #define PG_MAPPED      0x00000020      /* page is mapped (managed) */
-#define        PG_ZERO         0x00000040      /* page is zeroed */
+#define        PG_UNUSED0040   0x00000040
 #define PG_REFERENCED  0x00000080      /* page has been referenced */
 #define PG_CLEANCHK    0x00000100      /* page will be checked for cleaning */
 #define PG_SWAPINPROG  0x00000200      /* swap I/O in progress on page      */
index 935ed74..435664f 100644 (file)
@@ -340,16 +340,10 @@ vm_page_copy(vm_page_t src_m, vm_page_t dest_m)
 
 /*
  * Free a page.  The page must be marked BUSY.
- *
- * Always clear PG_ZERO when freeing a page, which ensures the flag is not
- * set unless we are absolutely certain the page is zerod.  This is
- * particularly important when the vm_page_alloc*() code moves pages from
- * PQ_CACHE to PQ_FREE.
  */
 static __inline void
 vm_page_free(vm_page_t m)
 {
-       vm_page_flag_clear(m, PG_ZERO);
        vm_page_free_toq(m);
 }
 
@@ -372,7 +366,6 @@ vm_page_free_zero(vm_page_t m)
        }
 #endif
 #endif
-       vm_page_flag_set(m, PG_ZERO);
        vm_page_free_toq(m);
 }
 
index 91188bf..e03dcd6 100644 (file)
@@ -280,7 +280,6 @@ vm_pager_bufferinit(void *dummy __unused)
                        KKASSERT(m != NULL);
                        bp->b_xio.xio_internal_pages[j] = m;
                        vm_page_wire(m);
-                       vm_page_flag_clear(m, PG_ZERO);
                        /* early boot, no other cpus running yet */
                        pmap_kenter_noinval(pg * PAGE_SIZE, VM_PAGE_TO_PHYS(m));
                        cpu_invlpg((void *)(pg * PAGE_SIZE));
index 1b2e6cb..da07165 100644 (file)
 #include <sys/thread2.h>
 #include <vm/vm_page2.h>
 
+#if 0
+/*
+ * Remove this file in 2017.
+ *
+ * REMOVED -  Basically does not provide any performance benefit and instead
+ *           appears to cause a performance detriment.  I surmise the issue
+ *           is simply that it takes such an enormous amount of time to read
+ *           data from dynamic ram, what really matters for a page-fault is
+ *           not that the page is zerod but instead that its cache is hot.
+ *
+ *           Zeroing the page during idle periods means the page is likely
+ *           to be cold in the cache when it actually gets used.  Zeroing the
+ *           page in-line with the VM-fault, on the other-hand, not only
+ *           ensures that the memory will be hot in the cache, the zeroing
+ *           operation itself does not actually have to read the dynamic ram,
+ *           it really only writes into the cache (for a 4K page), so the
+ *           page is already hot when the user program then accesses it.
+ */
+
 /*
  * Implement the pre-zeroed page mechanism.
  */
@@ -266,3 +285,5 @@ pagezero_start(void __unused *arg)
 }
 
 SYSINIT(pagezero, SI_SUB_KTHREAD_VM, SI_ORDER_ANY, pagezero_start, NULL);
+
+#endif