3 #include <sys/kernel.h>
5 #include <sys/sysctl.h>
8 #include <sys/diskslice.h>
10 #include <sys/malloc.h>
11 #include <machine/md_var.h>
12 #include <sys/ctype.h>
13 #include <sys/syslog.h>
14 #include <sys/device.h>
15 #include <sys/msgport.h>
16 #include <sys/msgport2.h>
18 #include <sys/dsched.h>
19 #include <sys/fcntl.h>
20 #include <machine/varargs.h>
23 * A simple anticipatory scheduler
26 struct dsched_disk_ctx head;
27 TAILQ_HEAD(, bio) as_queue_rd;
28 TAILQ_HEAD(, bio) as_queue_wr;
30 * TODO: lockmgr may be too heavy here,
31 * use spinlock instead!
33 struct lock as_queue_rd_lock;
34 struct lock as_queue_wr_lock;
38 struct callout as_callout;
44 struct dsched_as_stats{
48 static dsched_prepare_t as_prepare;
49 static dsched_teardown_t as_teardown;
50 static dsched_cancel_t as_cancel;
51 static dsched_queue_t as_queue;
52 static dsched_polling_func_t as_dequeue;
54 static struct dsched_policy dsched_as_policy = {
57 * field need_request_polling
58 * is removed from struct dsched_policy
60 //.need_request_polling = 1,
61 .prepare = as_prepare,
62 .teardown = as_teardown,
63 .cancel_all = as_cancel,
64 .bio_queue = as_queue,
65 .polling_func = as_dequeue
68 static int dsched_as_version_maj = 1;
69 static int dsched_as_version_min = 0;
72 as_prepare(struct dsched_disk_ctx *diskctx)
74 struct as_disk_ctx *as_diskctx = (struct as_disk_ctx *)diskctx;
75 TAILQ_INIT(&as_diskctx->as_queue_wr);
76 as_diskctx->queue_wr_size = 0;
77 TAILQ_INIT(&as_diskctx->as_queue_rd);
78 as_diskctx->queue_rd_size = 0;
79 lockinit(&as_diskctx->as_queue_rd_lock, "as_queue_rd", 0, LK_CANRECURSE);
80 lockinit(&as_diskctx->as_queue_wr_lock, "as_queue_wr", 0, LK_CANRECURSE);
81 callout_init(&as_diskctx->as_callout);
82 as_diskctx->as_blockall = 0;
83 as_diskctx->as_blockon = NO_PID;
88 as_teardown(struct dsched_disk_ctx *diskctx)
95 as_cancel(struct dsched_disk_ctx *diskctx)
97 struct as_disk_ctx *as_diskctx = (struct as_disk_ctx *)diskctx;
98 struct bio *bio, *bio2;
99 struct dsched_thread_io *tdio;
100 DSCHED_DISK_CTX_LOCK(&as_diskctx->head);
101 lockmgr(&as_diskctx->as_queue_rd_lock, LK_EXCLUSIVE);
102 TAILQ_FOREACH_MUTABLE(bio, &as_diskctx->as_queue_rd, link, bio2){
103 TAILQ_REMOVE(&as_diskctx->as_queue_rd, bio, link);
104 tdio = dsched_get_bio_tdio(bio);
105 dsched_cancel_bio(bio);
106 dsched_thread_io_unref(tdio);
108 lockmgr(&as_diskctx->as_queue_rd_lock, LK_RELEASE);
110 lockmgr(&as_diskctx->as_queue_wr_lock, LK_EXCLUSIVE);
111 TAILQ_FOREACH_MUTABLE(bio, &as_diskctx->as_queue_wr, link, bio2){
112 TAILQ_REMOVE(&as_diskctx->as_queue_wr, bio, link);
113 tdio = dsched_get_bio_tdio(bio);
114 dsched_cancel_bio(bio);
115 dsched_thread_io_unref(tdio);
117 lockmgr(&as_diskctx->as_queue_wr_lock, LK_RELEASE);
119 DSCHED_DISK_CTX_UNLOCK(&as_diskctx->head);
126 struct as_disk_ctx *as_diskctx = (struct as_disk_ctx *)p;
127 DSCHED_DISK_CTX_LOCK(&as_diskctx->head);
128 as_diskctx->as_blockall = 0;
129 last_blockon = as_diskctx->as_blockon;
130 as_diskctx->as_blockon = NO_PID;
131 DSCHED_DISK_CTX_UNLOCK(&as_diskctx->head);
132 //dsched_debug(0, "dsched: as, timeout %d\n", last_blockon);
133 as_dequeue((struct dsched_disk_ctx *)as_diskctx);
137 as_queue(struct dsched_disk_ctx *diskctx, struct dsched_thread_io *tdio,
140 struct as_disk_ctx *as_diskctx = (struct as_disk_ctx *)diskctx;
141 //if (tdio->p && (uint32_t)tdio->p != ~0)
142 // dsched_debug(0, "dsched: user process bio from %d\n", tdio->p->p_pid);
143 /*save tdio for each bio*/
144 dsched_set_bio_priv(bio, tdio);
145 dsched_set_bio_tdio(bio, tdio);
146 /* will be unreferenced in bio_done function */
147 dsched_thread_io_ref(tdio);
148 DSCHED_DISK_CTX_LOCK(&as_diskctx->head);
150 * if current bio is from as_blockon, insert it at head
152 if (bio->bio_buf->b_cmd == BUF_CMD_READ){
153 lockmgr(&as_diskctx->as_queue_rd_lock, LK_EXCLUSIVE);
154 if (as_diskctx->as_blockall && tdio->p && as_diskctx->as_blockon == tdio->p->p_pid)
155 TAILQ_INSERT_HEAD(&as_diskctx->as_queue_rd, bio, link);
157 TAILQ_INSERT_TAIL(&as_diskctx->as_queue_rd, bio, link);
158 atomic_add_int(&as_diskctx->queue_rd_size, 1);
159 lockmgr(&as_diskctx->as_queue_rd_lock, LK_RELEASE);
161 lockmgr(&as_diskctx->as_queue_wr_lock, LK_EXCLUSIVE);
162 TAILQ_INSERT_TAIL(&as_diskctx->as_queue_wr, bio, link);
163 atomic_add_int(&as_diskctx->queue_wr_size, 1);
164 lockmgr(&as_diskctx->as_queue_wr_lock, LK_RELEASE);
166 DSCHED_DISK_CTX_UNLOCK(&as_diskctx->head);
172 as_dequeue(struct dsched_disk_ctx *diskctx)
175 struct as_disk_ctx *as_diskctx = (struct as_disk_ctx *)diskctx;
177 struct dsched_thread_io *tdio;
178 /*Lock the diskctx for the whole dispatching process,
179 * to ensure atomic change to current_tab_queue_depth*/
181 DSCHED_DISK_CTX_LOCK(&as_diskctx->head);
182 /* if blocking all dispatching for anticipatory scheduling
185 if (as_diskctx->as_blockall){
186 // dsched_debug(0, "dsched: as, dequeue blocked! %d\n", as_diskctx->as_blockon);
189 free_slots = as_diskctx->head.max_tag_queue_depth - as_diskctx->head.current_tag_queue_depth;
190 KKASSERT(free_slots>=0 && free_slots <=64);
191 lockmgr(&as_diskctx->as_queue_rd_lock, LK_EXCLUSIVE);
192 while (free_slots > 0){
193 if (TAILQ_EMPTY(&as_diskctx->as_queue_rd))
195 bio = TAILQ_FIRST(&as_diskctx->as_queue_rd);
196 tdio = dsched_get_bio_priv(bio);
198 if (!tdio->p || (uint32_t)(tdio->p) == ~0){
199 TAILQ_REMOVE(&as_diskctx->as_queue_rd, bio, link);
200 dsched_strategy_request_polling(as_diskctx->head.dp, bio, diskctx);
202 //user process, continue
203 // dsched_debug(0, "dsched: as, user process bio\n");
204 if (as_diskctx->as_blockon == NO_PID || as_diskctx->as_blockon == tdio->p->p_pid){
205 as_diskctx->as_blockon = tdio->p->p_pid;
206 TAILQ_REMOVE(&as_diskctx->as_queue_rd, bio, link);
207 dsched_strategy_request_polling(as_diskctx->head.dp, bio, diskctx);
209 //user process, before switching, as!
210 as_diskctx->as_blockall = 1;
211 // dsched_debug(0, "dsched: as, block on %d\n", as_diskctx->as_blockon);
212 callout_reset(&as_diskctx->as_callout, 10, as_timeout, as_diskctx);
219 lockmgr(&as_diskctx->as_queue_rd_lock, LK_RELEASE);
220 lockmgr(&as_diskctx->as_queue_wr_lock, LK_EXCLUSIVE);
221 while (free_slots > 0){
222 if (TAILQ_EMPTY(&as_diskctx->as_queue_wr))
224 bio = TAILQ_FIRST(&as_diskctx->as_queue_wr);
225 TAILQ_REMOVE(&as_diskctx->as_queue_wr, bio, link);
226 dsched_strategy_request_polling(as_diskctx->head.dp, bio, diskctx);
229 lockmgr(&as_diskctx->as_queue_wr_lock, LK_RELEASE);
232 DSCHED_DISK_CTX_UNLOCK(&as_diskctx->head);
236 do_asstats(SYSCTL_HANDLER_ARGS)
238 return (sysctl_handle_opaque(oidp, &as_stats, sizeof(struct dsched_as_stats), req));
242 as_mod_handler(module_t mod, int type, void *unused)
244 static struct sysctl_ctx_list sysctl_ctx;
245 static struct sysctl_oid *oid;
246 static char version[16];
249 ksnprintf(version, sizeof(version), "%d.%d",
250 dsched_as_version_maj, dsched_as_version_min);
254 bzero(&as_stats, sizeof(struct dsched_as_stats));
255 if ((error = dsched_register(&dsched_as_policy)))
258 sysctl_ctx_init(&sysctl_ctx);
259 oid = SYSCTL_ADD_NODE(&sysctl_ctx,
260 SYSCTL_STATIC_CHILDREN(_dsched),
265 SYSCTL_ADD_PROC(&sysctl_ctx, SYSCTL_CHILDREN(oid),
266 OID_AUTO, "stats", CTLTYPE_OPAQUE|CTLFLAG_RD,
267 0, 0, do_asstats, "S,dsched_as_stats", "as statistics");
269 SYSCTL_ADD_STRING(&sysctl_ctx, SYSCTL_CHILDREN(oid),
270 OID_AUTO, "version", CTLFLAG_RD, version, 0, "as version");
272 kprintf("AS scheduler policy version %d.%d loaded\n",
273 dsched_as_version_maj, dsched_as_version_min);
277 if ((error = dsched_unregister(&dsched_as_policy)))
279 sysctl_ctx_free(&sysctl_ctx);
280 kprintf("AS scheduler policy unloaded\n");
290 DSCHED_POLICY_MODULE(dsched_as, as_mod_handler);