[dragonfly.git] / sys / kern / dsched / as / as.c

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/proc.h>
#include <sys/sysctl.h>
#include <sys/buf.h>
#include <sys/conf.h>
#include <sys/diskslice.h>
#include <sys/disk.h>
#include <sys/malloc.h>
#include <machine/md_var.h>
#include <sys/ctype.h>
#include <sys/syslog.h>
#include <sys/device.h>
#include <sys/msgport.h>
#include <sys/msgport2.h>
#include <sys/buf2.h>
#include <sys/dsched.h>
#include <sys/fcntl.h>
#include <machine/varargs.h>

/*
 * A simple anticipatory scheduler
 */
struct as_disk_ctx{
	struct dsched_disk_ctx head;
	TAILQ_HEAD(, bio) as_queue_rd;
	TAILQ_HEAD(, bio) as_queue_wr;
	/*
	 * TODO: lockmgr may be too heavy here,
	 * use spinlock instead!
	 */
	struct lock as_queue_rd_lock;
	struct lock as_queue_wr_lock;
	int queue_rd_size;
	int queue_wr_size;

	struct callout as_callout;

	int as_blockall;
	pid_t as_blockon;
};

struct dsched_as_stats{
	int32_t unused;
}as_stats;

static dsched_prepare_t		as_prepare;
static dsched_teardown_t	as_teardown;
static dsched_cancel_t		as_cancel;
static dsched_queue_t		as_queue;
static dsched_polling_func_t	as_dequeue;

static struct dsched_policy dsched_as_policy = {
	.name ="as",
/*
 * field need_request_polling
 * is removed from struct dsched_policy
 */
	//.need_request_polling = 1,
	.prepare = as_prepare,
	.teardown = as_teardown,
	.cancel_all = as_cancel,
	.bio_queue = as_queue,
	.polling_func = as_dequeue
};

static int dsched_as_version_maj = 1;
static int dsched_as_version_min = 0;

static int
as_prepare(struct dsched_disk_ctx *diskctx)
{
	struct as_disk_ctx *as_diskctx = (struct as_disk_ctx *)diskctx;
	TAILQ_INIT(&as_diskctx->as_queue_wr);
	as_diskctx->queue_wr_size = 0;
	TAILQ_INIT(&as_diskctx->as_queue_rd);
	as_diskctx->queue_rd_size = 0;
	lockinit(&as_diskctx->as_queue_rd_lock, "as_queue_rd", 0, LK_CANRECURSE);
	lockinit(&as_diskctx->as_queue_wr_lock, "as_queue_wr", 0, LK_CANRECURSE);
	callout_init(&as_diskctx->as_callout);
	as_diskctx->as_blockall = 0;
	as_diskctx->as_blockon = NO_PID;
	return 0;
}

static void
as_teardown(struct dsched_disk_ctx *diskctx)
{


}

static void
as_cancel(struct dsched_disk_ctx *diskctx)
{
	struct as_disk_ctx *as_diskctx = (struct as_disk_ctx *)diskctx;
	struct bio *bio, *bio2;
	struct dsched_thread_io *tdio;
	DSCHED_DISK_CTX_LOCK(&as_diskctx->head);
	lockmgr(&as_diskctx->as_queue_rd_lock, LK_EXCLUSIVE);
	TAILQ_FOREACH_MUTABLE(bio, &as_diskctx->as_queue_rd, link, bio2){
		TAILQ_REMOVE(&as_diskctx->as_queue_rd, bio, link);
		tdio = dsched_get_bio_tdio(bio);
		dsched_cancel_bio(bio);
		dsched_thread_io_unref(tdio);
	}
	lockmgr(&as_diskctx->as_queue_rd_lock, LK_RELEASE);

	lockmgr(&as_diskctx->as_queue_wr_lock, LK_EXCLUSIVE);
	TAILQ_FOREACH_MUTABLE(bio, &as_diskctx->as_queue_wr, link, bio2){
		TAILQ_REMOVE(&as_diskctx->as_queue_wr, bio, link);
		tdio = dsched_get_bio_tdio(bio);
		dsched_cancel_bio(bio);
		dsched_thread_io_unref(tdio);
	}
	lockmgr(&as_diskctx->as_queue_wr_lock, LK_RELEASE);

	DSCHED_DISK_CTX_UNLOCK(&as_diskctx->head);
}

static void
as_timeout(void *p)
{
	pid_t last_blockon;
	struct as_disk_ctx *as_diskctx = (struct as_disk_ctx *)p;
	DSCHED_DISK_CTX_LOCK(&as_diskctx->head);
	as_diskctx->as_blockall = 0;
	last_blockon = as_diskctx->as_blockon;
	as_diskctx->as_blockon = NO_PID;
	DSCHED_DISK_CTX_UNLOCK(&as_diskctx->head);
	//dsched_debug(0, "dsched: as, timeout %d\n", last_blockon);
	as_dequeue((struct dsched_disk_ctx *)as_diskctx);
}

static int
as_queue(struct dsched_disk_ctx *diskctx, struct dsched_thread_io *tdio,
		struct  bio *bio)
{
	struct as_disk_ctx *as_diskctx = (struct as_disk_ctx *)diskctx;
	//if (tdio->p && (uint32_t)tdio->p != ~0)
	//	dsched_debug(0, "dsched: user process bio from %d\n", tdio->p->p_pid);
	/*save tdio for each bio*/
	dsched_set_bio_priv(bio, tdio);
	dsched_set_bio_tdio(bio, tdio);
	/* will be unreferenced in bio_done function */
	dsched_thread_io_ref(tdio);
	DSCHED_DISK_CTX_LOCK(&as_diskctx->head);
	/* blocking for as,
	 * if current bio is from as_blockon, insert it at head
	 */
	if (bio->bio_buf->b_cmd == BUF_CMD_READ){
		lockmgr(&as_diskctx->as_queue_rd_lock, LK_EXCLUSIVE);
		if (as_diskctx->as_blockall && tdio->p && as_diskctx->as_blockon == tdio->p->p_pid)
			TAILQ_INSERT_HEAD(&as_diskctx->as_queue_rd, bio, link);
		else
			TAILQ_INSERT_TAIL(&as_diskctx->as_queue_rd, bio, link);
		atomic_add_int(&as_diskctx->queue_rd_size, 1);
		lockmgr(&as_diskctx->as_queue_rd_lock, LK_RELEASE);
	} else {
		lockmgr(&as_diskctx->as_queue_wr_lock, LK_EXCLUSIVE);
		TAILQ_INSERT_TAIL(&as_diskctx->as_queue_wr, bio, link);
		atomic_add_int(&as_diskctx->queue_wr_size, 1);
		lockmgr(&as_diskctx->as_queue_wr_lock, LK_RELEASE);
	}
	DSCHED_DISK_CTX_UNLOCK(&as_diskctx->head);
	as_dequeue(diskctx);
	return 0;

}
static void
as_dequeue(struct dsched_disk_ctx *diskctx)
{
	int free_slots = 0;
	struct as_disk_ctx *as_diskctx = (struct as_disk_ctx *)diskctx;
	struct bio *bio;
	struct dsched_thread_io *tdio;
	/*Lock the diskctx for the whole dispatching process,
	 * to ensure atomic change to current_tab_queue_depth*/

	DSCHED_DISK_CTX_LOCK(&as_diskctx->head);
	/* if blocking all dispatching for anticipatory scheduling
	 * return directly
	 */
	if (as_diskctx->as_blockall){
	//	dsched_debug(0, "dsched: as, dequeue blocked! %d\n", as_diskctx->as_blockon);
		goto rtn;
	}
	free_slots = as_diskctx->head.max_tag_queue_depth - as_diskctx->head.current_tag_queue_depth;
	KKASSERT(free_slots>=0 && free_slots <=64);
	lockmgr(&as_diskctx->as_queue_rd_lock, LK_EXCLUSIVE);
	while (free_slots > 0){
		if (TAILQ_EMPTY(&as_diskctx->as_queue_rd))
			break;
		bio = TAILQ_FIRST(&as_diskctx->as_queue_rd);
		tdio = dsched_get_bio_priv(bio);
		//kernel thread
		if (!tdio->p) {
			TAILQ_REMOVE(&as_diskctx->as_queue_rd, bio, link);
			dsched_strategy_request_polling(as_diskctx->head.dp, bio, diskctx);
		} else {
			//user process, continue
//			dsched_debug(0, "dsched: as, user process bio\n");
			if (as_diskctx->as_blockon == NO_PID || as_diskctx->as_blockon == tdio->p->p_pid){
				as_diskctx->as_blockon = tdio->p->p_pid;
				TAILQ_REMOVE(&as_diskctx->as_queue_rd, bio, link);
				dsched_strategy_request_polling(as_diskctx->head.dp, bio, diskctx);
			} else {
				//user process, before switching, as!
				as_diskctx->as_blockall = 1;
//				dsched_debug(0, "dsched: as, block on %d\n", as_diskctx->as_blockon);
				callout_reset(&as_diskctx->as_callout, 10, as_timeout, as_diskctx);
				break;
			}

		}
		free_slots --;
	}
	lockmgr(&as_diskctx->as_queue_rd_lock, LK_RELEASE);
	lockmgr(&as_diskctx->as_queue_wr_lock, LK_EXCLUSIVE);
	while (free_slots > 0){
		if (TAILQ_EMPTY(&as_diskctx->as_queue_wr))
			break;
		bio = TAILQ_FIRST(&as_diskctx->as_queue_wr);
		TAILQ_REMOVE(&as_diskctx->as_queue_wr, bio, link);
		dsched_strategy_request_polling(as_diskctx->head.dp, bio, diskctx);
		free_slots --;
	}
	lockmgr(&as_diskctx->as_queue_wr_lock, LK_RELEASE);

rtn:
	DSCHED_DISK_CTX_UNLOCK(&as_diskctx->head);
}

static int
do_asstats(SYSCTL_HANDLER_ARGS)
{
	return (sysctl_handle_opaque(oidp, &as_stats, sizeof(struct dsched_as_stats), req));
}

static int
as_mod_handler(module_t mod, int type, void *unused)
{
	static struct sysctl_ctx_list sysctl_ctx;
	static struct sysctl_oid *oid;
	static char version[16];
	int error;

	ksnprintf(version, sizeof(version), "%d.%d",
	    dsched_as_version_maj, dsched_as_version_min);

	switch (type) {
	case MOD_LOAD:
		bzero(&as_stats, sizeof(struct dsched_as_stats));
		if ((error = dsched_register(&dsched_as_policy)))
			return (error);

		sysctl_ctx_init(&sysctl_ctx);
		oid = SYSCTL_ADD_NODE(&sysctl_ctx,
		    SYSCTL_STATIC_CHILDREN(_dsched),
		    OID_AUTO,
		    "as",
		    CTLFLAG_RD, 0, "");

		SYSCTL_ADD_PROC(&sysctl_ctx, SYSCTL_CHILDREN(oid),
		    OID_AUTO, "stats", CTLTYPE_OPAQUE|CTLFLAG_RD,
		    0, 0, do_asstats, "S,dsched_as_stats", "as statistics");

		SYSCTL_ADD_STRING(&sysctl_ctx, SYSCTL_CHILDREN(oid),
		    OID_AUTO, "version", CTLFLAG_RD, version, 0, "as version");

		kprintf("AS scheduler policy version %d.%d loaded\n",
		    dsched_as_version_maj, dsched_as_version_min);
		break;

	case MOD_UNLOAD:
		if ((error = dsched_unregister(&dsched_as_policy)))
			return (error);
		sysctl_ctx_free(&sysctl_ctx);
		kprintf("AS scheduler policy unloaded\n");
		break;

	default:
		break;
	}

	return 0;

}
DSCHED_POLICY_MODULE(dsched_as, as_mod_handler, 1);
Commit	Line	Data
e8e9bef2 BP	1	#include <sys/param.h>
	2	#include <sys/systm.h>
	3	#include <sys/kernel.h>
	4	#include <sys/proc.h>
	5	#include <sys/sysctl.h>
	6	#include <sys/buf.h>
	7	#include <sys/conf.h>
	8	#include <sys/diskslice.h>
	9	#include <sys/disk.h>
	10	#include <sys/malloc.h>
	11	#include <machine/md_var.h>
	12	#include <sys/ctype.h>
	13	#include <sys/syslog.h>
	14	#include <sys/device.h>
	15	#include <sys/msgport.h>
	16	#include <sys/msgport2.h>
	17	#include <sys/buf2.h>
	18	#include <sys/dsched.h>
	19	#include <sys/fcntl.h>
	20	#include <machine/varargs.h>
	21
	22	/*
	23	* A simple anticipatory scheduler
	24	*/
	25	struct as_disk_ctx{
	26	struct dsched_disk_ctx head;
	27	TAILQ_HEAD(, bio) as_queue_rd;
	28	TAILQ_HEAD(, bio) as_queue_wr;
	29	/*
	30	* TODO: lockmgr may be too heavy here,
	31	* use spinlock instead!
	32	*/
	33	struct lock as_queue_rd_lock;
	34	struct lock as_queue_wr_lock;
	35	int queue_rd_size;
	36	int queue_wr_size;
	37
	38	struct callout as_callout;
	39
	40	int as_blockall;
	41	pid_t as_blockon;
	42	};
	43
	44	struct dsched_as_stats{
	45	int32_t unused;
	46	}as_stats;
	47
	48	static dsched_prepare_t as_prepare;
	49	static dsched_teardown_t as_teardown;
	50	static dsched_cancel_t as_cancel;
	51	static dsched_queue_t as_queue;
	52	static dsched_polling_func_t as_dequeue;
	53
	54	static struct dsched_policy dsched_as_policy = {
	55	.name ="as",
	56	/*
	57	* field need_request_polling
	58	* is removed from struct dsched_policy
	59	*/
	60	//.need_request_polling = 1,
	61	.prepare = as_prepare,
	62	.teardown = as_teardown,
	63	.cancel_all = as_cancel,
	64	.bio_queue = as_queue,
65	.polling_func = as_dequeue
66	};
67
68	static int dsched_as_version_maj = 1;
69	static int dsched_as_version_min = 0;
70
71	static int
72	as_prepare(struct dsched_disk_ctx *diskctx)
73	{
74	struct as_disk_ctx as_diskctx = (struct as_disk_ctx )diskctx;
75	TAILQ_INIT(&as_diskctx->as_queue_wr);
76	as_diskctx->queue_wr_size = 0;
77	TAILQ_INIT(&as_diskctx->as_queue_rd);
78	as_diskctx->queue_rd_size = 0;
79	lockinit(&as_diskctx->as_queue_rd_lock, "as_queue_rd", 0, LK_CANRECURSE);
80	lockinit(&as_diskctx->as_queue_wr_lock, "as_queue_wr", 0, LK_CANRECURSE);
81	callout_init(&as_diskctx->as_callout);
82	as_diskctx->as_blockall = 0;
83	as_diskctx->as_blockon = NO_PID;
84	return 0;
85	}
86
87	static void
88	as_teardown(struct dsched_disk_ctx *diskctx)
89	{
90
91
92	}
93
94	static void
95	as_cancel(struct dsched_disk_ctx *diskctx)
96	{
97	struct as_disk_ctx as_diskctx = (struct as_disk_ctx )diskctx;
98	struct bio bio, bio2;
99	struct dsched_thread_io *tdio;
100	DSCHED_DISK_CTX_LOCK(&as_diskctx->head);
101	lockmgr(&as_diskctx->as_queue_rd_lock, LK_EXCLUSIVE);
102	TAILQ_FOREACH_MUTABLE(bio, &as_diskctx->as_queue_rd, link, bio2){
103	TAILQ_REMOVE(&as_diskctx->as_queue_rd, bio, link);
104	tdio = dsched_get_bio_tdio(bio);
105	dsched_cancel_bio(bio);
106	dsched_thread_io_unref(tdio);
107	}
108	lockmgr(&as_diskctx->as_queue_rd_lock, LK_RELEASE);
109
110	lockmgr(&as_diskctx->as_queue_wr_lock, LK_EXCLUSIVE);
111	TAILQ_FOREACH_MUTABLE(bio, &as_diskctx->as_queue_wr, link, bio2){
112	TAILQ_REMOVE(&as_diskctx->as_queue_wr, bio, link);
113	tdio = dsched_get_bio_tdio(bio);
114	dsched_cancel_bio(bio);
115	dsched_thread_io_unref(tdio);
116	}
117	lockmgr(&as_diskctx->as_queue_wr_lock, LK_RELEASE);
118
119	DSCHED_DISK_CTX_UNLOCK(&as_diskctx->head);
120	}
121
122	static void
123	as_timeout(void *p)
124	{
125	pid_t last_blockon;
126	struct as_disk_ctx as_diskctx = (struct as_disk_ctx )p;
127	DSCHED_DISK_CTX_LOCK(&as_diskctx->head);
128	as_diskctx->as_blockall = 0;
129	last_blockon = as_diskctx->as_blockon;
130	as_diskctx->as_blockon = NO_PID;
131	DSCHED_DISK_CTX_UNLOCK(&as_diskctx->head);
132	//dsched_debug(0, "dsched: as, timeout %d\n", last_blockon);
133	as_dequeue((struct dsched_disk_ctx *)as_diskctx);
134	}
135
136	static int
137	as_queue(struct dsched_disk_ctx diskctx, struct dsched_thread_io tdio,
138	struct bio *bio)
139	{
140	struct as_disk_ctx as_diskctx = (struct as_disk_ctx )diskctx;
141	//if (tdio->p && (uint32_t)tdio->p != ~0)
142	// dsched_debug(0, "dsched: user process bio from %d\n", tdio->p->p_pid);
143	/save tdio for each bio/
144	dsched_set_bio_priv(bio, tdio);
145	dsched_set_bio_tdio(bio, tdio);
146	/* will be unreferenced in bio_done function */
147	dsched_thread_io_ref(tdio);
148	DSCHED_DISK_CTX_LOCK(&as_diskctx->head);
149	/* blocking for as,
150	* if current bio is from as_blockon, insert it at head
151	*/
152	if (bio->bio_buf->b_cmd == BUF_CMD_READ){
153	lockmgr(&as_diskctx->as_queue_rd_lock, LK_EXCLUSIVE);
154	if (as_diskctx->as_blockall && tdio->p && as_diskctx->as_blockon == tdio->p->p_pid)
155	TAILQ_INSERT_HEAD(&as_diskctx->as_queue_rd, bio, link);
156	else
157	TAILQ_INSERT_TAIL(&as_diskctx->as_queue_rd, bio, link);
158	atomic_add_int(&as_diskctx->queue_rd_size, 1);
159	lockmgr(&as_diskctx->as_queue_rd_lock, LK_RELEASE);
160	} else {
161	lockmgr(&as_diskctx->as_queue_wr_lock, LK_EXCLUSIVE);
162	TAILQ_INSERT_TAIL(&as_diskctx->as_queue_wr, bio, link);
163	atomic_add_int(&as_diskctx->queue_wr_size, 1);
164	lockmgr(&as_diskctx->as_queue_wr_lock, LK_RELEASE);
165	}
166	DSCHED_DISK_CTX_UNLOCK(&as_diskctx->head);
167	as_dequeue(diskctx);
168	return 0;
169
170	}
171	static void
172	as_dequeue(struct dsched_disk_ctx *diskctx)
173	{
174	int free_slots = 0;
175	struct as_disk_ctx as_diskctx = (struct as_disk_ctx )diskctx;
176	struct bio *bio;
177	struct dsched_thread_io *tdio;
178	/*Lock the diskctx for the whole dispatching process,
179	* to ensure atomic change to current_tab_queue_depth*/
180
181	DSCHED_DISK_CTX_LOCK(&as_diskctx->head);
182	/* if blocking all dispatching for anticipatory scheduling
183	* return directly
184	*/
185	if (as_diskctx->as_blockall){
186	// dsched_debug(0, "dsched: as, dequeue blocked! %d\n", as_diskctx->as_blockon);
187	goto rtn;
188	}
189	free_slots = as_diskctx->head.max_tag_queue_depth - as_diskctx->head.current_tag_queue_depth;
190	KKASSERT(free_slots>=0 && free_slots <=64);
191	lockmgr(&as_diskctx->as_queue_rd_lock, LK_EXCLUSIVE);
192	while (free_slots > 0){
193	if (TAILQ_EMPTY(&as_diskctx->as_queue_rd))
194	break;
195	bio = TAILQ_FIRST(&as_diskctx->as_queue_rd);
196	tdio = dsched_get_bio_priv(bio);
197	//kernel thread
b3fc94f8	198	if (!tdio->p) {
e8e9bef2 BP	199	TAILQ_REMOVE(&as_diskctx->as_queue_rd, bio, link);
	200	dsched_strategy_request_polling(as_diskctx->head.dp, bio, diskctx);
	201	} else {
	202	//user process, continue
	203	// dsched_debug(0, "dsched: as, user process bio\n");
	204	if (as_diskctx->as_blockon == NO_PID \|\| as_diskctx->as_blockon == tdio->p->p_pid){
	205	as_diskctx->as_blockon = tdio->p->p_pid;
	206	TAILQ_REMOVE(&as_diskctx->as_queue_rd, bio, link);
	207	dsched_strategy_request_polling(as_diskctx->head.dp, bio, diskctx);
	208	} else {
	209	//user process, before switching, as!
	210	as_diskctx->as_blockall = 1;
	211	// dsched_debug(0, "dsched: as, block on %d\n", as_diskctx->as_blockon);
	212	callout_reset(&as_diskctx->as_callout, 10, as_timeout, as_diskctx);
	213	break;
	214	}
	215
	216	}
	217	free_slots --;
	218	}
	219	lockmgr(&as_diskctx->as_queue_rd_lock, LK_RELEASE);
	220	lockmgr(&as_diskctx->as_queue_wr_lock, LK_EXCLUSIVE);
	221	while (free_slots > 0){
	222	if (TAILQ_EMPTY(&as_diskctx->as_queue_wr))
	223	break;
	224	bio = TAILQ_FIRST(&as_diskctx->as_queue_wr);
	225	TAILQ_REMOVE(&as_diskctx->as_queue_wr, bio, link);
	226	dsched_strategy_request_polling(as_diskctx->head.dp, bio, diskctx);
	227	free_slots --;
	228	}
	229	lockmgr(&as_diskctx->as_queue_wr_lock, LK_RELEASE);
	230
	231	rtn:
	232	DSCHED_DISK_CTX_UNLOCK(&as_diskctx->head);
	233	}
	234
	235	static int
	236	do_asstats(SYSCTL_HANDLER_ARGS)
	237	{
	238	return (sysctl_handle_opaque(oidp, &as_stats, sizeof(struct dsched_as_stats), req));
	239	}
	240
	241	static int
	242	as_mod_handler(module_t mod, int type, void *unused)
	243	{
	244	static struct sysctl_ctx_list sysctl_ctx;
	245	static struct sysctl_oid *oid;
	246	static char version[16];
	247	int error;
	248
	249	ksnprintf(version, sizeof(version), "%d.%d",
	250	dsched_as_version_maj, dsched_as_version_min);
	251
	252	switch (type) {
	253	case MOD_LOAD:
	254	bzero(&as_stats, sizeof(struct dsched_as_stats));
	255	if ((error = dsched_register(&dsched_as_policy)))
	256	return (error);
	257
	258	sysctl_ctx_init(&sysctl_ctx);
	259	oid = SYSCTL_ADD_NODE(&sysctl_ctx,
	260	SYSCTL_STATIC_CHILDREN(_dsched),
	261	OID_AUTO,
	262	"as",
263	CTLFLAG_RD, 0, "");
264
265	SYSCTL_ADD_PROC(&sysctl_ctx, SYSCTL_CHILDREN(oid),
266	OID_AUTO, "stats", CTLTYPE_OPAQUE\|CTLFLAG_RD,
267	0, 0, do_asstats, "S,dsched_as_stats", "as statistics");
268
269	SYSCTL_ADD_STRING(&sysctl_ctx, SYSCTL_CHILDREN(oid),
270	OID_AUTO, "version", CTLFLAG_RD, version, 0, "as version");
271
272	kprintf("AS scheduler policy version %d.%d loaded\n",
273	dsched_as_version_maj, dsched_as_version_min);
274	break;
275
276	case MOD_UNLOAD:
277	if ((error = dsched_unregister(&dsched_as_policy)))
278	return (error);
279	sysctl_ctx_free(&sysctl_ctx);
280	kprintf("AS scheduler policy unloaded\n");
281	break;
282
283	default:
284	break;
285	}
286
287	return 0;
288
289	}
1f509c0d	290	DSCHED_POLICY_MODULE(dsched_as, as_mod_handler, 1);