From 8c72e3d5eb98215e26f57511bee19cbaa1c44dd1 Mon Sep 17 00:00:00 2001 From: Alex Hornung Date: Thu, 25 Feb 2010 08:54:32 +0000 Subject: [PATCH] dsched - Tie the dsched framework into the system * tie the dsched framework into the system by binding it into the build, adding all the necessary hooks and other calls. --- sys/conf/files | 4 ++++ sys/kern/kern_device.c | 5 +++++ sys/kern/kern_fork.c | 4 ++++ sys/kern/kern_proc.c | 2 ++ sys/kern/lwkt_thread.c | 6 ++++++ sys/kern/subr_disk.c | 5 +++-- sys/kern/vfs_bio.c | 12 ++++++++++++ sys/sys/bio.h | 10 +++++++++- sys/sys/buf.h | 3 ++- sys/sys/disk.h | 3 +++ sys/sys/proc.h | 7 +++++-- sys/sys/thread.h | 1 + sys/vm/vm_pager.c | 3 +++ 13 files changed, 59 insertions(+), 6 deletions(-) diff --git a/sys/conf/files b/sys/conf/files index e61eaf662a..afdff4b403 100644 --- a/sys/conf/files +++ b/sys/conf/files @@ -717,6 +717,10 @@ kern/subr_autoconf.c standard kern/subr_bus.c standard kern/subr_busdma.c standard kern/subr_devstat.c standard +kern/subr_dsched.c standard +dsched/fq/dsched_fq_core.c standard +dsched/fq/dsched_fq_diskops.c standard +dsched/fq/dsched_fq_procops.c standard kern/subr_disk.c standard kern/subr_disklabel32.c standard kern/subr_disklabel64.c standard diff --git a/sys/kern/kern_device.c b/sys/kern/kern_device.c index f50d864955..197622fb75 100644 --- a/sys/kern/kern_device.c +++ b/sys/kern/kern_device.c @@ -48,6 +48,7 @@ #include #include #include +#include /* * system link descriptors identify the command in the @@ -250,6 +251,10 @@ dev_dstrategy(cdev_t dev, struct bio *bio) track = &dev->si_track_write; bio_track_ref(track); bio->bio_track = track; + + if (dsched_is_clear_buf_priv(bio->bio_buf)) + dsched_new_buf(bio->bio_buf); + KKASSERT((bio->bio_flags & BIO_DONE) == 0); (void)dev->si_ops->d_strategy(&ap); } diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c index 8c98d9265c..4920d35541 100644 --- a/sys/kern/kern_fork.c +++ b/sys/kern/kern_fork.c @@ -70,6 +70,8 @@ #include #include +#include + static MALLOC_DEFINE(M_ATFORK, "atfork", "atfork callback"); /* @@ -399,6 +401,8 @@ fork1(struct lwp *lp1, int flags, struct proc **procp) p2->p_args->ar_ref++; p2->p_usched = p1->p_usched; + /* XXX: verify copy of the secondary iosched stuff */ + dsched_new_proc(p2); if (flags & RFSIGSHARE) { p2->p_sigacts = p1->p_sigacts; diff --git a/sys/kern/kern_proc.c b/sys/kern/kern_proc.c index 46997aa322..cce2b3fc5b 100644 --- a/sys/kern/kern_proc.c +++ b/sys/kern/kern_proc.c @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -492,6 +493,7 @@ proc_move_allproc_zombie(struct proc *p) LIST_REMOVE(p, p_hash); p->p_stat = SZOMB; spin_unlock_wr(&allproc_spin); + dsched_exit_proc(p); } /* diff --git a/sys/kern/lwkt_thread.c b/sys/kern/lwkt_thread.c index 288b152e94..edd1507740 100644 --- a/sys/kern/lwkt_thread.c +++ b/sys/kern/lwkt_thread.c @@ -57,6 +57,8 @@ #include #include +#include + #include #include #include @@ -380,6 +382,8 @@ lwkt_init_thread(thread_t td, void *stack, int stksize, int flags, TAILQ_INSERT_TAIL(&gd->gd_tdallq, td, td_allq); crit_exit_gd(mygd); #endif + + dsched_new_thread(td); } void @@ -419,6 +423,8 @@ lwkt_free_thread(thread_t td) KASSERT((td->td_flags & TDF_RUNNING) == 0, ("lwkt_free_thread: did not exit! %p", td)); + dsched_exit_thread(td); + if (td->td_flags & TDF_ALLOCATED_THREAD) { objcache_put(thread_cache, td); } else if (td->td_flags & TDF_ALLOCATED_STACK) { diff --git a/sys/kern/subr_disk.c b/sys/kern/subr_disk.c index 1c253e3f5d..be467c26c6 100644 --- a/sys/kern/subr_disk.c +++ b/sys/kern/subr_disk.c @@ -106,7 +106,7 @@ #include #include #include - +#include #include #include @@ -531,6 +531,7 @@ disk_create(int unit, struct disk *dp, struct dev_ops *raw_ops) "disk_create (end): %s%d\n", raw_ops->head.name, unit); + dsched_create(dp, raw_ops->head.name, unit); return (dp->d_rawdev); } @@ -903,7 +904,7 @@ diskstrategy(struct dev_strategy_args *ap) * or error due to being beyond the device size). */ if ((nbio = dscheck(dev, bio, dp->d_slice)) != NULL) { - dev_dstrategy(dp->d_rawdev, nbio); + dsched_queue(dp, nbio); } else { biodone(bio); } diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c index 5fd005f3ca..ddeaf0f294 100644 --- a/sys/kern/vfs_bio.c +++ b/sys/kern/vfs_bio.c @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -1049,6 +1050,9 @@ bdwrite(struct buf *bp) } bdirty(bp); + if (dsched_is_clear_buf_priv(bp)) + dsched_new_buf(bp); + /* * Set B_CACHE, indicating that the buffer is fully valid. This is * true even of NFS now. @@ -1320,6 +1324,7 @@ brelse(struct buf *bp) * or B_RELBUF flags. */ bp->b_cmd = BUF_CMD_DONE; + dsched_clr_buf_priv(bp); /* * VMIO buffer rundown. Make sure the VM page array is restored @@ -1652,6 +1657,7 @@ bqrelse(struct buf *bp) * buffer is actively locked. */ bp->b_flags &= ~(B_ORDERED | B_NOCACHE | B_RELBUF); + dsched_clr_buf_priv(bp); BUF_UNLOCK(bp); } @@ -2959,6 +2965,7 @@ loop: allocbuf(bp, size); rel_mplock(); } + KKASSERT(dsched_is_clear_buf_priv(bp)); return (bp); } @@ -3007,6 +3014,7 @@ geteblk(int size) allocbuf(bp, size); rel_mplock(); bp->b_flags |= B_INVAL; /* b_dep cleared by getnewbuf() */ + KKASSERT(dsched_is_clear_buf_priv(bp)); return (bp); } @@ -3396,6 +3404,8 @@ void bio_start_transaction(struct bio *bio, struct bio_track *track) { bio->bio_track = track; + if (dsched_is_clear_buf_priv(bio->bio_buf)) + dsched_new_buf(bio->bio_buf); bio_track_ref(track); } @@ -3449,6 +3459,8 @@ vn_strategy(struct vnode *vp, struct bio *bio) track = &vp->v_track_write; KKASSERT((bio->bio_flags & BIO_DONE) == 0); bio->bio_track = track; + if (dsched_is_clear_buf_priv(bio->bio_buf)) + dsched_new_buf(bio->bio_buf); bio_track_ref(track); vop_strategy(*vp->v_ops, vp, bio); } diff --git a/sys/sys/bio.h b/sys/sys/bio.h index eb21b8b262..b01491a8e1 100644 --- a/sys/sys/bio.h +++ b/sys/sys/bio.h @@ -46,7 +46,7 @@ struct bio; struct bio_track; - +struct disk; typedef void biodone_t(struct bio *); /* @@ -60,7 +60,9 @@ typedef void biodone_t(struct bio *); */ struct bio { TAILQ_ENTRY(bio) bio_act; /* driver queue when active */ + TAILQ_ENTRY(bio) link; struct bio_track *bio_track; /* BIO tracking structure */ + struct disk *bio_disk; struct bio *bio_prev; /* BIO stack */ struct bio *bio_next; /* BIO stack / cached translations */ struct buf *bio_buf; /* High-level buffer back-pointer. */ @@ -82,6 +84,12 @@ struct bio { int index; struct buf *cluster_tail; } bio_caller_info2; + union { + void *ptr; + int value; + long lvalue; + struct timeval tv; + } bio_caller_info3; }; /* diff --git a/sys/sys/buf.h b/sys/sys/buf.h index f2815077c6..4c4cc55d6e 100644 --- a/sys/sys/buf.h +++ b/sys/sys/buf.h @@ -74,7 +74,7 @@ struct mount; struct vnode; struct xio; -#define NBUF_BIO 4 +#define NBUF_BIO 6 struct buf_rb_tree; struct buf_rb_hash; @@ -165,6 +165,7 @@ struct buf { unsigned char b_act_count; /* similar to vm_page act_count */ unsigned char b_unused01; struct lock b_lock; /* Buffer lock */ + void *b_iosched; /* I/O scheduler priv data */ buf_cmd_t b_cmd; /* I/O command */ int b_bufsize; /* Allocated buffer size. */ int b_runningbufspace; /* when I/O is running, pipelining */ diff --git a/sys/sys/disk.h b/sys/sys/disk.h index 257a43db7f..3afb317fba 100644 --- a/sys/sys/disk.h +++ b/sys/sys/disk.h @@ -127,6 +127,9 @@ struct disk { cdev_t d_cdev; /* special whole-disk part */ struct diskslices *d_slice; struct disk_info d_info; /* info structure for media */ + void *d_dsched_priv1;/* I/O scheduler priv. data */ + void *d_dsched_priv2;/* I/O scheduler priv. data */ + struct dsched_ops *d_sched_ops; /* I/O scheduler ops */ LIST_ENTRY(disk) d_list; }; diff --git a/sys/sys/proc.h b/sys/sys/proc.h index da916d84b8..b4d8046695 100644 --- a/sys/sys/proc.h +++ b/sys/sys/proc.h @@ -280,6 +280,7 @@ struct proc { struct rusage p_ru; /* stats for this proc */ struct rusage p_cru; /* sum of stats for reaped children */ + void *p_dsched_priv1; /* The following fields are all copied upon creation in fork. */ #define p_startcopy p_comm @@ -297,8 +298,10 @@ struct proc { struct rtprio p_rtprio; /* Realtime priority. */ struct pargs *p_args; u_short p_xstat; /* Exit status or last stop signal */ + + void *p_dsched_priv2; /* End area that is copied on creation. */ -#define p_endcopy p_xstat +#define p_endcopy p_dsched_priv2 u_short p_acflag; /* Accounting flags. */ int p_lock; /* Prevent proc destruction */ @@ -453,7 +456,7 @@ extern u_long pidhash; extern LIST_HEAD(pgrphashhead, pgrp) *pgrphashtbl; extern u_long pgrphash; -#if 0 +#if 0 #ifndef SET_CURPROC #define SET_CURPROC(p) (curproc = (p)) #endif diff --git a/sys/sys/thread.h b/sys/sys/thread.h index f8a2e9cc50..474db4eba7 100644 --- a/sys/sys/thread.h +++ b/sys/sys/thread.h @@ -214,6 +214,7 @@ struct thread { __uint64_t td_iticks; /* Statclock hits processing intr (uS) */ int td_locks; /* lockmgr lock debugging */ int td_unused01; + void *td_dsched_priv1; /* priv data for I/O schedulers */ int td_refs; /* hold position in gd_tdallq / hold free */ int td_nest_count; /* prevent splz nesting */ #ifdef SMP diff --git a/sys/vm/vm_pager.c b/sys/vm/vm_pager.c index 4c93a07bd5..a323a832a9 100644 --- a/sys/vm/vm_pager.c +++ b/sys/vm/vm_pager.c @@ -77,6 +77,7 @@ #include #include #include +#include #include #include @@ -307,6 +308,7 @@ getpbuf(int *pfreecnt) spin_unlock_wr(&bswspin); initpbuf(bp); + KKASSERT(dsched_is_clear_buf_priv(bp)); return bp; } @@ -354,6 +356,7 @@ relpbuf(struct buf *bp, int *pfreecnt) int wake_freecnt = 0; KKASSERT(bp->b_flags & B_PAGING); + dsched_clr_buf_priv(bp); spin_lock_wr(&bswspin); -- 2.41.0