Commit | Line | Data |
---|---|---|
b80a9543 AH |
1 | /* |
2 | * Copyright (c) 2009 The DragonFly Project. All rights reserved. | |
3 | * | |
4 | * This code is derived from software contributed to The DragonFly Project | |
5 | * by Alex Hornung <ahornung@gmail.com> | |
6 | * | |
7 | * Redistribution and use in source and binary forms, with or without | |
8 | * modification, are permitted provided that the following conditions | |
9 | * are met: | |
10 | * | |
11 | * 1. Redistributions of source code must retain the above copyright | |
12 | * notice, this list of conditions and the following disclaimer. | |
13 | * 2. Redistributions in binary form must reproduce the above copyright | |
14 | * notice, this list of conditions and the following disclaimer in | |
15 | * the documentation and/or other materials provided with the | |
16 | * distribution. | |
17 | * 3. Neither the name of The DragonFly Project nor the names of its | |
18 | * contributors may be used to endorse or promote products derived | |
19 | * from this software without specific, prior written permission. | |
20 | * | |
21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
22 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
23 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS | |
24 | * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE | |
25 | * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, | |
26 | * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, | |
27 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; | |
28 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED | |
29 | * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, | |
30 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT | |
31 | * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | |
32 | * SUCH DAMAGE. | |
33 | */ | |
34 | #ifndef _SYS_DSCHED_H_ | |
35 | #define _SYS_DSCHED_H_ | |
36 | ||
22befd90 | 37 | #if defined(_KERNEL) |
b80a9543 AH |
38 | |
39 | #ifndef _SYS_QUEUE_H_ | |
40 | #include <sys/queue.h> | |
41 | #endif | |
42 | #ifndef _SYS_BIO_H_ | |
43 | #include <sys/bio.h> | |
44 | #endif | |
45 | #ifndef _SYS_BIOTRACK_H_ | |
46 | #include <sys/biotrack.h> | |
47 | #endif | |
48 | #ifndef _SYS_LOCK_H_ | |
49 | #include <sys/lock.h> | |
50 | #endif | |
51 | #ifndef _SYS_CONF_H_ | |
52 | #include <sys/conf.h> | |
53 | #endif | |
54 | #ifndef _SYS_MSGPORT_H_ | |
55 | #include <sys/msgport.h> | |
56 | #endif | |
e02e815e AH |
57 | #ifndef _SYS_SYSCTL_H_ |
58 | #include <sys/sysctl.h> | |
59 | #endif | |
279e9fd5 AH |
60 | #ifndef SYS_DISK_H_ |
61 | #include <sys/disk.h> | |
62 | #endif | |
b80a9543 AH |
63 | |
64 | #define DSCHED_POLICY_NAME_LENGTH 64 | |
65 | ||
66 | #define dsched_set_disk_priv(dp, x) ((dp)->d_dsched_priv1 = (x)) | |
67 | #define dsched_get_disk_priv(dp) ((dp)?((dp)->d_dsched_priv1):NULL) | |
68 | #define dsched_set_proc_priv(pp, x) ((pp)->p_dsched_priv1 = (x)) | |
69 | #define dsched_get_proc_priv(pp) ((pp)?((pp)->p_dsched_priv1):NULL) | |
70 | ||
71 | #define dsched_set_thread_priv(td, x) ((td)->td_dsched_priv1 = (x)) | |
72 | #define dsched_get_thread_priv(td) ((td)?((td)->td_dsched_priv1):NULL) | |
73 | ||
74 | #define dsched_set_buf_priv(bp, x) ((bp)->b_iosched = (x)) | |
75 | #define dsched_get_buf_priv(bp) ((bp)?((bp)->b_iosched):NULL) | |
76 | #define dsched_clr_buf_priv(bp) ((bp)->b_iosched = NULL) | |
77 | #define dsched_is_clear_buf_priv(bp) ((bp)->b_iosched == NULL) | |
78 | ||
79 | ||
80 | #define dsched_set_bio_dp(bio, x) ((bio)->bio_caller_info1.ptr = (x)) | |
81 | #define dsched_get_bio_dp(bio) ((bio)?((bio)->bio_caller_info1.ptr):NULL) | |
82 | #define dsched_set_bio_priv(bio, x) ((bio)->bio_caller_info2.ptr = (x)) | |
83 | #define dsched_get_bio_priv(bio) ((bio)?((bio)->bio_caller_info2.ptr):NULL) | |
84 | #define dsched_set_bio_stime(bio, x) ((bio)->bio_caller_info3.lvalue = (x)) | |
85 | #define dsched_get_bio_stime(bio) ((bio)?((bio)->bio_caller_info3.lvalue):0) | |
c7a0a046 AH |
86 | #define dsched_set_bio_tdio(bio, x) ((bio)->bio_caller_info3.ptr = (x)) |
87 | #define dsched_get_bio_tdio(bio) ((bio)?((bio)->bio_caller_info3.ptr):0) | |
b80a9543 AH |
88 | |
89 | ||
e02e815e AH |
90 | struct dsched_thread_ctx { |
91 | TAILQ_ENTRY(dsched_thread_ctx) link; | |
92 | ||
93 | TAILQ_HEAD(, dsched_thread_io) tdio_list; /* list of thread_io */ | |
94 | struct lock lock; | |
95 | ||
96 | int32_t refcount; | |
97 | ||
98 | struct proc *p; | |
99 | struct thread *td; | |
100 | int32_t dead; | |
101 | }; | |
102 | ||
103 | struct dsched_disk_ctx { | |
104 | TAILQ_ENTRY(dsched_disk_ctx) link; | |
105 | ||
106 | TAILQ_HEAD(, dsched_thread_io) tdio_list; /* list of thread_io of disk */ | |
107 | struct lock lock; | |
108 | ||
109 | int32_t refcount; | |
bc3c9325 | 110 | int32_t flags; |
e02e815e | 111 | |
09f2bfe9 BP |
112 | int max_tag_queue_depth; /* estimated max tag queue depth */ |
113 | int current_tag_queue_depth; /* estimated current tag queue depth */ | |
114 | ||
e02e815e | 115 | struct disk *dp; /* back pointer to disk struct */ |
bc3c9325 AH |
116 | |
117 | struct sysctl_ctx_list sysctl_ctx; | |
e02e815e AH |
118 | }; |
119 | ||
c7a0a046 AH |
120 | struct dsched_policy; |
121 | ||
e02e815e AH |
122 | struct dsched_thread_io { |
123 | TAILQ_ENTRY(dsched_thread_io) link; | |
124 | TAILQ_ENTRY(dsched_thread_io) dlink; | |
125 | ||
126 | TAILQ_HEAD(, bio) queue; /* IO queue (bio) */ | |
127 | struct lock lock; | |
128 | int32_t qlength;/* IO queue length */ | |
129 | ||
130 | int32_t refcount; | |
131 | ||
132 | int32_t flags; | |
133 | ||
134 | struct disk *dp; | |
135 | struct dsched_disk_ctx *diskctx; | |
136 | struct dsched_thread_ctx *tdctx; | |
137 | struct proc *p; | |
c7a0a046 AH |
138 | struct dsched_policy *debug_policy; |
139 | int debug_inited; | |
140 | int debug_priv; | |
e02e815e AH |
141 | }; |
142 | ||
143 | typedef int dsched_prepare_t(struct dsched_disk_ctx *diskctx); | |
144 | typedef void dsched_teardown_t(struct dsched_disk_ctx *diskctx); | |
145 | typedef void dsched_cancel_t(struct dsched_disk_ctx *diskctx); | |
146 | typedef int dsched_queue_t(struct dsched_disk_ctx *diskctx, | |
147 | struct dsched_thread_io *tdio, struct bio *bio); | |
09f2bfe9 | 148 | typedef void dsched_dequeue_t(struct dsched_disk_ctx *diskctx); |
e02e815e AH |
149 | |
150 | typedef void dsched_new_tdio_t(struct dsched_thread_io *tdio); | |
151 | typedef void dsched_new_diskctx_t(struct dsched_disk_ctx *diskctx); | |
152 | typedef void dsched_destroy_tdio_t(struct dsched_thread_io *tdio); | |
153 | typedef void dsched_destroy_diskctx_t(struct dsched_disk_ctx *diskctx); | |
09f2bfe9 BP |
154 | typedef void dsched_bio_done_t(struct bio *bio); |
155 | typedef void dsched_polling_func_t(struct dsched_disk_ctx *diskctx); | |
0160356d AH |
156 | |
157 | struct dsched_policy { | |
158 | char name[DSCHED_POLICY_NAME_LENGTH]; | |
159 | uint64_t uniq_id; | |
160 | int ref_count; | |
161 | ||
162 | TAILQ_ENTRY(dsched_policy) link; | |
b80a9543 AH |
163 | |
164 | dsched_prepare_t *prepare; | |
165 | dsched_teardown_t *teardown; | |
b80a9543 AH |
166 | dsched_cancel_t *cancel_all; |
167 | dsched_queue_t *bio_queue; | |
168 | ||
e02e815e AH |
169 | dsched_new_tdio_t *new_tdio; |
170 | dsched_new_diskctx_t *new_diskctx; | |
171 | dsched_destroy_tdio_t *destroy_tdio; | |
172 | dsched_destroy_diskctx_t *destroy_diskctx; | |
09f2bfe9 BP |
173 | |
174 | dsched_bio_done_t *bio_done; /* call back when a bio dispatched by dsched_strategy_request_polling() is done */ | |
175 | dsched_polling_func_t *polling_func; /* it gets called when the disk is idle or about to idle */ | |
b80a9543 AH |
176 | }; |
177 | ||
b80a9543 AH |
178 | TAILQ_HEAD(dsched_policy_head, dsched_policy); |
179 | ||
e02e815e | 180 | |
fd87e9e4 MD |
181 | #define DSCHED_THREAD_IO_LOCKINIT(x) \ |
182 | lockinit(&(x)->lock, "tdiobioq", 0, LK_CANRECURSE) | |
183 | ||
184 | #define DSCHED_THREAD_IO_LOCK(x) do { \ | |
185 | dsched_thread_io_ref((x)); \ | |
186 | lockmgr(&(x)->lock, LK_EXCLUSIVE); \ | |
187 | } while(0) | |
188 | ||
189 | #define DSCHED_THREAD_IO_UNLOCK(x) do { \ | |
190 | lockmgr(&(x)->lock, LK_RELEASE); \ | |
191 | dsched_thread_io_unref((x)); \ | |
192 | } while(0) | |
193 | ||
194 | #define DSCHED_DISK_CTX_LOCKINIT(x) \ | |
195 | lockinit(&(x)->lock, "tdiodiskq", 0, LK_CANRECURSE) | |
196 | ||
197 | #define DSCHED_DISK_CTX_LOCK(x) do { \ | |
198 | dsched_disk_ctx_ref((x)); \ | |
199 | lockmgr(&(x)->lock, LK_EXCLUSIVE); \ | |
200 | } while(0) | |
201 | ||
202 | #define DSCHED_DISK_CTX_UNLOCK(x) do { \ | |
203 | lockmgr(&(x)->lock, LK_RELEASE); \ | |
204 | dsched_disk_ctx_unref((x)); \ | |
205 | } while(0) | |
206 | ||
207 | #define DSCHED_DISK_CTX_LOCK_ASSERT(x) \ | |
208 | KKASSERT(lockstatus(&(x)->lock, curthread) == LK_EXCLUSIVE) | |
209 | ||
210 | #define DSCHED_GLOBAL_THREAD_CTX_LOCKINIT(x) \ | |
211 | lockinit(&dsched_tdctx_lock, "tdctxglob", 0, LK_CANRECURSE) | |
212 | #define DSCHED_GLOBAL_THREAD_CTX_LOCK(x) \ | |
213 | lockmgr(&dsched_tdctx_lock, LK_EXCLUSIVE) | |
214 | #define DSCHED_GLOBAL_THREAD_CTX_UNLOCK(x) \ | |
215 | lockmgr(&dsched_tdctx_lock, LK_RELEASE) | |
216 | ||
217 | #define DSCHED_THREAD_CTX_LOCKINIT(x) \ | |
218 | lockinit(&(x)->lock, "tdctx", 0, LK_CANRECURSE) | |
219 | ||
220 | #define DSCHED_THREAD_CTX_LOCK(x) do { \ | |
221 | dsched_thread_ctx_ref((x)); \ | |
222 | lockmgr(&(x)->lock, LK_EXCLUSIVE); \ | |
223 | } while(0) | |
224 | ||
225 | #define DSCHED_THREAD_CTX_UNLOCK(x) do { \ | |
226 | lockmgr(&(x)->lock, LK_RELEASE); \ | |
227 | dsched_thread_ctx_unref((x)); \ | |
228 | } while(0) | |
e02e815e | 229 | |
bc3c9325 | 230 | /* flags for thread_io */ |
e02e815e AH |
231 | #define DSCHED_LINKED_DISK_CTX 0x01 |
232 | #define DSCHED_LINKED_THREAD_CTX 0x02 | |
bc3c9325 AH |
233 | /* flags for disk_ctx */ |
234 | #define DSCHED_SYSCTL_CTX_INITED 0x01 | |
e02e815e AH |
235 | |
236 | #define DSCHED_THREAD_CTX_MAX_SZ sizeof(struct dsched_thread_ctx) | |
237 | #define DSCHED_THREAD_IO_MAX_SZ 256 | |
209d9536 | 238 | #define DSCHED_DISK_CTX_MAX_SZ 512 |
e02e815e AH |
239 | |
240 | #define DSCHED_POLICY_MODULE(name, evh) \ | |
241 | static moduledata_t name##_mod = { \ | |
242 | #name, \ | |
243 | evh, \ | |
244 | NULL \ | |
245 | }; \ | |
246 | DECLARE_MODULE(name, name##_mod, SI_SUB_PRE_DRIVERS, SI_ORDER_MIDDLE) | |
247 | ||
0160356d | 248 | void dsched_disk_create_callback(struct disk *dp, const char *head_name, int unit); |
279e9fd5 | 249 | void dsched_disk_update_callback(struct disk *dp, struct disk_info *info); |
0160356d | 250 | void dsched_disk_destroy_callback(struct disk *dp); |
b80a9543 | 251 | void dsched_queue(struct disk *dp, struct bio *bio); |
0160356d AH |
252 | int dsched_register(struct dsched_policy *d_policy); |
253 | int dsched_unregister(struct dsched_policy *d_policy); | |
254 | int dsched_switch(struct disk *dp, struct dsched_policy *new_policy); | |
255 | void dsched_set_policy(struct disk *dp, struct dsched_policy *new_policy); | |
b80a9543 AH |
256 | struct dsched_policy *dsched_find_policy(char *search); |
257 | struct disk *dsched_find_disk(char *search); | |
258 | struct dsched_policy *dsched_policy_enumerate(struct dsched_policy *pol); | |
0160356d | 259 | struct disk *dsched_disk_enumerate(struct disk *dp, struct dsched_policy *policy); |
b80a9543 AH |
260 | void dsched_cancel_bio(struct bio *bp); |
261 | void dsched_strategy_raw(struct disk *dp, struct bio *bp); | |
262 | void dsched_strategy_sync(struct disk *dp, struct bio *bp); | |
263 | void dsched_strategy_async(struct disk *dp, struct bio *bp, biodone_t *done, void *priv); | |
09f2bfe9 | 264 | void dsched_strategy_request_polling(struct disk *bp, struct bio *bio, struct dsched_disk_ctx *diskctx); |
fcefa6f2 | 265 | int dsched_debug(int level, char *fmt, ...) __printflike(2, 3); |
e02e815e AH |
266 | |
267 | void policy_new(struct disk *dp, struct dsched_policy *pol); | |
268 | void policy_destroy(struct disk *dp); | |
269 | ||
270 | void dsched_disk_ctx_ref(struct dsched_disk_ctx *diskctx); | |
271 | void dsched_thread_io_ref(struct dsched_thread_io *tdio); | |
272 | void dsched_thread_ctx_ref(struct dsched_thread_ctx *tdctx); | |
273 | void dsched_disk_ctx_unref(struct dsched_disk_ctx *diskctx); | |
274 | void dsched_thread_io_unref(struct dsched_thread_io *tdio); | |
275 | void dsched_thread_ctx_unref(struct dsched_thread_ctx *tdctx); | |
276 | ||
89dabacd AH |
277 | struct dsched_thread_io *dsched_new_policy_thread_tdio(struct dsched_disk_ctx *diskctx, |
278 | struct dsched_policy *pol); | |
e02e815e AH |
279 | struct dsched_thread_io *dsched_thread_io_alloc(struct disk *dp, |
280 | struct dsched_thread_ctx *tdctx, struct dsched_policy *pol); | |
281 | struct dsched_disk_ctx *dsched_disk_ctx_alloc(struct disk *dp, | |
282 | struct dsched_policy *pol); | |
283 | struct dsched_thread_ctx *dsched_thread_ctx_alloc(struct proc *p); | |
284 | ||
285 | typedef void dsched_new_buf_t(struct buf *bp); | |
286 | typedef void dsched_new_proc_t(struct proc *p); | |
287 | typedef void dsched_new_thread_t(struct thread *td); | |
288 | typedef void dsched_exit_buf_t(struct buf *bp); | |
289 | typedef void dsched_exit_proc_t(struct proc *p); | |
290 | typedef void dsched_exit_thread_t(struct thread *td); | |
291 | ||
b80a9543 AH |
292 | dsched_new_buf_t dsched_new_buf; |
293 | dsched_new_proc_t dsched_new_proc; | |
294 | dsched_new_thread_t dsched_new_thread; | |
aa166ad1 | 295 | dsched_exit_buf_t dsched_exit_buf; |
b80a9543 AH |
296 | dsched_exit_proc_t dsched_exit_proc; |
297 | dsched_exit_thread_t dsched_exit_thread; | |
298 | ||
22befd90 | 299 | #endif /* _KERNEL */ |
b80a9543 AH |
300 | |
301 | ||
302 | #define DSCHED_NAME_LENGTH 64 | |
303 | #define DSCHED_SET_DEVICE_POLICY _IOWR('d', 1, struct dsched_ioctl) | |
304 | #define DSCHED_LIST_DISKS _IOWR('d', 2, struct dsched_ioctl) | |
305 | #define DSCHED_LIST_DISK _IOWR('d', 3, struct dsched_ioctl) | |
306 | #define DSCHED_LIST_POLICIES _IOWR('d', 4, struct dsched_ioctl) | |
307 | ||
308 | struct dsched_ioctl { | |
309 | uint16_t num_elem; | |
310 | char dev_name[DSCHED_NAME_LENGTH]; | |
311 | char pol_name[DSCHED_NAME_LENGTH]; | |
312 | }; | |
313 | ||
e02e815e AH |
314 | struct dsched_stats { |
315 | int32_t tdctx_allocations; | |
316 | int32_t tdio_allocations; | |
317 | int32_t diskctx_allocations; | |
318 | ||
319 | int32_t no_tdctx; | |
320 | ||
321 | int32_t nthreads; | |
322 | int32_t nprocs; | |
323 | }; | |
324 | ||
b80a9543 | 325 | #endif /* _SYS_DSCHED_H_ */ |