tcp/sack: Further optimize scoreboard block allocation

Use one slot freed SACK scoreboard cache.

30minutes tests are conducted on a heavy congested real-life network path;
the new statistics:

  31254 SACK scoreboard updates
      2 overflows
      0 failures
      13392 records reused
      12703 records fast allocated

Before this commit, ~42% allocations are avoided (reused); after this commit,
~83% allocations are avoided (reused + fast allocated).

diff --git a/sys/netinet/tcp_sack.c b/sys/netinet/tcp_sack.c
index 4bac1c9..6e52b4b 100644 (file)
--- a/sys/netinet/tcp_sack.c
+++ b/sys/netinet/tcp_sack.c
@@ -148,17 +148,23 @@ sack_block_lookup(struct scoreboard *scb, tcp_seq seq, struct sackblock **sb)
  * Allocate a SACK block.
  */
 static __inline struct sackblock *
-alloc_sackblock(const struct raw_sackblock *raw_sb)
+alloc_sackblock(struct scoreboard *scb, const struct raw_sackblock *raw_sb)
 {
        struct sackblock *sb;
 
-       sb = kmalloc(sizeof(struct sackblock), M_SACKBLOCK, M_NOWAIT);
-       if (sb != NULL) {
-               sb->sblk_start = raw_sb->rblk_start;
-               sb->sblk_end = raw_sb->rblk_end;
+       if (scb->freecache != NULL) {
+               sb = scb->freecache;
+               scb->freecache = NULL;
+               tcpstat.tcps_sacksbfast++;
        } else {
-               tcpstat.tcps_sacksbfailed++;
+               sb = kmalloc(sizeof(struct sackblock), M_SACKBLOCK, M_NOWAIT);
+               if (sb == NULL) {
+                       tcpstat.tcps_sacksbfailed++;
+                       return NULL;
+               }
        }
+       sb->sblk_start = raw_sb->rblk_start;
+       sb->sblk_end = raw_sb->rblk_end;
        return sb;
 }
 
@@ -176,15 +182,20 @@ alloc_sackblock_limit(struct scoreboard *scb,
                tcpstat.tcps_sacksboverflow++;
                return NULL;
        }
-       return alloc_sackblock(raw_sb);
+       return alloc_sackblock(scb, raw_sb);
 }
 
 /*
  * Free a SACK block.
  */
 static __inline void
-free_sackblock(struct sackblock *s)
+free_sackblock(struct scoreboard *scb, struct sackblock *s)
 {
+       if (scb->freecache == NULL) {
+               /* YYY Maybe use the latest freed block? */
+               scb->freecache = s;
+               return;
+       }
        kfree(s, M_SACKBLOCK);
 }
 
@@ -202,7 +213,7 @@ tcp_sack_ack_blocks(struct scoreboard *scb, tcp_seq th_ack)
                if (scb->lastfound == sb)
                        scb->lastfound = NULL;
                TAILQ_REMOVE(&scb->sackblocks, sb, sblk_list);
-               free_sackblock(sb);
+               free_sackblock(scb, sb);
                --scb->nblocks;
                KASSERT(scb->nblocks >= 0,
                    ("SACK block count underflow: %d < 0", scb->nblocks));
@@ -221,7 +232,7 @@ tcp_sack_cleanup(struct scoreboard *scb)
        struct sackblock *sb, *nb;
 
        TAILQ_FOREACH_MUTABLE(sb, &scb->sackblocks, sblk_list, nb) {
-               free_sackblock(sb);
+               free_sackblock(scb, sb);
                --scb->nblocks;
        }
        KASSERT(scb->nblocks == 0,
@@ -231,6 +242,20 @@ tcp_sack_cleanup(struct scoreboard *scb)
 }
 
 /*
+ * Delete and free SACK blocks saved in scoreboard.
+ * Delete the one slot block cache.
+ */
+void
+tcp_sack_destroy(struct scoreboard *scb)
+{
+       tcp_sack_cleanup(scb);
+       if (scb->freecache != NULL) {
+               kfree(scb->freecache, M_SACKBLOCK);
+               scb->freecache = NULL;
+       }
+}
+
+/*
  * Cleanup the reported SACK block information
  */
 void
@@ -322,7 +347,7 @@ insert_block(struct scoreboard *scb, const struct raw_sackblock *raw_sb)
 
                KASSERT(scb->nblocks == 0, ("emply scb w/ blocks"));
 
-               newblock = alloc_sackblock(raw_sb);
+               newblock = alloc_sackblock(scb, raw_sb);
                if (newblock == NULL)
                        return ENOMEM;
                TAILQ_INSERT_HEAD(&scb->sackblocks, newblock, sblk_list);
@@ -370,7 +395,7 @@ insert_block(struct scoreboard *scb, const struct raw_sackblock *raw_sb)
                        scb->lastfound = NULL;
                /* Remove completely overlapped block */
                TAILQ_REMOVE(&scb->sackblocks, sb, sblk_list);
-               free_sackblock(sb);
+               free_sackblock(scb, sb);
                --scb->nblocks;
                KASSERT(scb->nblocks > 0,
                    ("removed overlapped block: %d blocks left", scb->nblocks));
@@ -383,7 +408,7 @@ insert_block(struct scoreboard *scb, const struct raw_sackblock *raw_sb)
                if (scb->lastfound == sb)
                        scb->lastfound = NULL;
                TAILQ_REMOVE(&scb->sackblocks, sb, sblk_list);
-               free_sackblock(sb);
+               free_sackblock(scb, sb);
                --scb->nblocks;
                KASSERT(scb->nblocks > 0,
                    ("removed partial right: %d blocks left", scb->nblocks));

diff --git a/sys/netinet/tcp_subr.c b/sys/netinet/tcp_subr.c
index a40ae29..6976735 100644 (file)
--- a/sys/netinet/tcp_subr.c
+++ b/sys/netinet/tcp_subr.c
@@ -982,7 +982,7 @@ no_valid_rt:
        }
        /* throw away SACK blocks in scoreboard*/
        if (TCP_DO_SACK(tp))
-               tcp_sack_cleanup(&tp->scb);
+               tcp_sack_destroy(&tp->scb);
 
        inp->inp_ppcb = NULL;
        soisdisconnected(so);

diff --git a/sys/netinet/tcp_var.h b/sys/netinet/tcp_var.h
index 4253462..01f30b8 100644 (file)
--- a/sys/netinet/tcp_var.h
+++ b/sys/netinet/tcp_var.h
@@ -123,6 +123,7 @@ struct scoreboard {
        struct sackblock_list sackblocks;
        tcp_seq lostseq;                        /* passed SACK lost test */
        struct sackblock *lastfound;            /* search hint */
+       struct sackblock *freecache;            /* one slot free block cache */
 };
 
 struct netmsg_tcp_timer;
@@ -391,6 +392,7 @@ struct tcp_stats {
        u_long  tcps_sacksboverflow;    /* times SACK scoreboard overflowed */
        u_long  tcps_sacksbreused;      /* times SACK sb-block reused */
        u_long  tcps_sacksbfailed;      /* times SACK sb update failed */
+       u_long  tcps_sacksbfast;        /* timee SACK sb-block uses cache */
 
        u_long  tcps_sc_added;          /* entry added to syncache */
        u_long  tcps_sc_retransmitted;  /* syncache entry was retransmitted */
@@ -622,6 +624,7 @@ void         tcp_respond (struct tcpcb *, void *,
 struct rtentry *
         tcp_rtlookup (struct in_conninfo *);
 int     tcp_sack_bytes_below(struct scoreboard *scb, tcp_seq seq);
+void    tcp_sack_destroy(struct scoreboard *scb);
 void    tcp_sack_cleanup(struct scoreboard *scb);
 void    tcp_sack_report_cleanup(struct tcpcb *tp);
 int     tcp_sack_ndsack_blocks(struct raw_sackblock *blocks,

diff --git a/usr.bin/netstat/inet.c b/usr.bin/netstat/inet.c
index 1cc0ea9..83569db 100644 (file)
--- a/usr.bin/netstat/inet.c
+++ b/usr.bin/netstat/inet.c
@@ -535,6 +535,7 @@ tcp_stats(u_long off __unused, const char *name, int af1 __unused)
        p(tcps_sacksboverflow, "\t\t%lu overflow%s\n");
        p(tcps_sacksbfailed, "\t\t%lu failure%s\n");
        p(tcps_sacksbreused, "\t\t%lu record%s reused\n");
+       p(tcps_sacksbfast, "\t\t%lu record%s fast allocated\n");
 
        free(stattmp);
 #undef p