From 5374d04ff57a66cc5c140a32249db93546195130 Mon Sep 17 00:00:00 2001 From: Matthew Dillon Date: Fri, 30 Nov 2012 11:17:50 -0800 Subject: [PATCH] kernel - Fix races in disk iteration and diskctx handling * Add disk->d_refs to prevent a disk structure from being destroyed out from under an iteration. * Redo the disk_enumeration() API to use markers and d_refs. * Make adjustments to the dsched API. In particular, do not return unreferenced tdio pointers in situations where they aren't used by the caller. * Properly implement the ref count on the tdio's, one for each of the two lists the tdio belongs to, and ensure that dsched_thread_io_alloc() keeps an extra ref on the tdio after releasing the diskctx lock to prevent it from being ripped out while the code is pondering whether to place the tdio on the tdctx list. * When deleting the tdio's for a tdctx try to destroy the diskctx. That is, simply dereferencing it from the thread is not sufficient. * When deleting the tdio's for a diskctx try to destroy the tdctx. That is, simply dereferencing it from the diskctx is not sufficient. * Handle destroy/ref races. --- sys/kern/dsched/bfq/bfq_helper_thread.c | 4 +- sys/kern/dsched/fq/fq_core.c | 8 +- sys/kern/kern_dsched.c | 192 ++++++++++++++++++------ sys/kern/subr_disk.c | 86 ++++++++--- sys/sys/disk.h | 12 +- sys/sys/dsched.h | 16 +- 6 files changed, 229 insertions(+), 89 deletions(-) diff --git a/sys/kern/dsched/bfq/bfq_helper_thread.c b/sys/kern/dsched/bfq/bfq_helper_thread.c index 75d8a0429d..6a5bffe53e 100644 --- a/sys/kern/dsched/bfq/bfq_helper_thread.c +++ b/sys/kern/dsched/bfq/bfq_helper_thread.c @@ -333,12 +333,10 @@ helper_sysctl_init(struct bfq_disk_ctx *bfq_diskctx) static void helper_thread(struct bfq_disk_ctx *bfq_diskctx) { - struct dsched_thread_io *tdio; - int r; helper_msg_t msg; - tdio = dsched_new_policy_thread_tdio(&bfq_diskctx->head, &dsched_bfq_policy); + dsched_new_policy_thread_tdio(&bfq_diskctx->head, &dsched_bfq_policy); lwkt_initport_thread(&bfq_diskctx->helper_msg_port, curthread); dsched_disk_ctx_ref(&bfq_diskctx->head); diff --git a/sys/kern/dsched/fq/fq_core.c b/sys/kern/dsched/fq/fq_core.c index 01657f67f5..2bd45ff15f 100644 --- a/sys/kern/dsched/fq/fq_core.c +++ b/sys/kern/dsched/fq/fq_core.c @@ -67,7 +67,7 @@ fq_dispatcher(struct fq_disk_ctx *diskctx) { struct dispatch_prep *dispatch_ary; struct dsched_thread_io *ds_tdio, *ds_tdio2; - struct fq_thread_io *tdio; + struct fq_thread_io *tdio; struct bio *bio, *bio2; int idle; int i, prepd_io; @@ -84,8 +84,7 @@ fq_dispatcher(struct fq_disk_ctx *diskctx) * since it isn't assigned one during fq_prepare, as the disk * is not set up yet. */ - tdio = (struct fq_thread_io *)dsched_new_policy_thread_tdio(&diskctx->head, - &dsched_fq_policy); + dsched_new_policy_thread_tdio(&diskctx->head, &dsched_fq_policy); DSCHED_DISK_CTX_LOCK(&diskctx->head); for(;;) { @@ -122,7 +121,8 @@ fq_dispatcher(struct fq_disk_ctx *diskctx) * to dispatch a few requests from each tdio as to ensure * real fairness. */ - TAILQ_FOREACH_MUTABLE(ds_tdio, &diskctx->head.tdio_list, dlink, ds_tdio2) { + TAILQ_FOREACH_MUTABLE(ds_tdio, &diskctx->head.tdio_list, + dlink, ds_tdio2) { tdio = (struct fq_thread_io *)ds_tdio; if (tdio->head.qlength == 0) continue; diff --git a/sys/kern/kern_dsched.c b/sys/kern/kern_dsched.c index c3e779100b..2bcedc155c 100644 --- a/sys/kern/kern_dsched.c +++ b/sys/kern/kern_dsched.c @@ -61,6 +61,7 @@ static dsched_teardown_t noop_teardown; static dsched_cancel_t noop_cancel; static dsched_queue_t noop_queue; +static void dsched_thread_io_unref_destroy(struct dsched_thread_io *tdio); static void dsched_sysctl_add_disk(struct dsched_disk_ctx *diskctx, char *name); static void dsched_disk_ctx_destroy(struct dsched_disk_ctx *diskctx); static void dsched_thread_io_destroy(struct dsched_thread_io *tdio); @@ -135,18 +136,18 @@ dsched_disk_create_callback(struct disk *dp, const char *head_name, int unit) struct dsched_policy *policy = NULL; /* Also look for serno stuff? */ - /* kprintf("dsched_disk_create_callback() for disk %s%d\n", head_name, unit); */ lockmgr(&dsched_lock, LK_EXCLUSIVE); - ksnprintf(tunable_key, sizeof(tunable_key), "dsched.policy.%s%d", - head_name, unit); + ksnprintf(tunable_key, sizeof(tunable_key), + "dsched.policy.%s%d", head_name, unit); if (TUNABLE_STR_FETCH(tunable_key, sched_policy, sizeof(sched_policy)) != 0) { policy = dsched_find_policy(sched_policy); } - ksnprintf(tunable_key, sizeof(tunable_key), "dsched.policy.%s", - head_name); + ksnprintf(tunable_key, sizeof(tunable_key), + "dsched.policy.%s", head_name); + for (ptr = tunable_key; *ptr; ptr++) { if (*ptr == '/') *ptr = '-'; @@ -157,8 +158,9 @@ dsched_disk_create_callback(struct disk *dp, const char *head_name, int unit) } ksnprintf(tunable_key, sizeof(tunable_key), "dsched.policy.default"); - if (!policy && !default_set && (TUNABLE_STR_FETCH(tunable_key, sched_policy, - sizeof(sched_policy)) != 0)) { + if (!policy && !default_set && + (TUNABLE_STR_FETCH(tunable_key, sched_policy, + sizeof(sched_policy)) != 0)) { policy = dsched_find_policy(sched_policy); } @@ -456,30 +458,33 @@ dsched_find_policy(char *search) return policy_found; } -struct disk* +/* + * Returns ref'd disk + */ +struct disk * dsched_find_disk(char *search) { - struct disk *dp_found = NULL; + struct disk marker; struct disk *dp = NULL; - while((dp = disk_enumerate(dp))) { - if (!strcmp(dp->d_cdev->si_name, search)) { - dp_found = dp; + while ((dp = disk_enumerate(&marker, dp)) != NULL) { + if (strcmp(dp->d_cdev->si_name, search) == 0) { + disk_enumerate_stop(&marker, NULL); + /* leave ref on dp */ break; } } - - return dp_found; + return dp; } -struct disk* -dsched_disk_enumerate(struct disk *dp, struct dsched_policy *policy) +struct disk * +dsched_disk_enumerate(struct disk *marker, struct disk *dp, + struct dsched_policy *policy) { - while ((dp = disk_enumerate(dp))) { + while ((dp = disk_enumerate(marker, dp)) != NULL) { if (dp->d_sched_policy == policy) - return dp; + break; } - return NULL; } @@ -711,6 +716,8 @@ void dsched_disk_ctx_destroy(struct dsched_disk_ctx *diskctx) { struct dsched_thread_io *tdio; + int refs; + int nrefs; #if 0 kprintf("diskctx (%p) destruction started, trace:\n", diskctx); @@ -723,12 +730,33 @@ dsched_disk_ctx_destroy(struct dsched_disk_ctx *diskctx) atomic_clear_int(&tdio->flags, DSCHED_LINKED_DISK_CTX); tdio->diskctx = NULL; /* XXX tdio->diskctx->dp->d_sched_policy->destroy_tdio(tdio);*/ - dsched_thread_io_unref(tdio); + lockmgr(&diskctx->lock, LK_RELEASE); + dsched_thread_io_unref_destroy(tdio); + lockmgr(&diskctx->lock, LK_EXCLUSIVE); } lockmgr(&diskctx->lock, LK_RELEASE); + + /* + * Expect diskctx->refcount to be 0x80000000. If it isn't someone + * else still has a temporary ref on the diskctx and we have to + * transition it back to an undestroyed-state (albeit without any + * associations), so the other user destroys it properly when the + * ref is released. + */ + while ((refs = diskctx->refcount) != 0x80000000) { + kprintf("dsched_thread_io: destroy race diskctx=%p\n", diskctx); + cpu_ccfence(); + KKASSERT(refs & 0x80000000); + nrefs = refs & 0x7FFFFFFF; + if (atomic_cmpset_int(&diskctx->refcount, refs, nrefs)) + return; + } + + /* + * Really for sure now. + */ if (diskctx->dp->d_sched_policy->destroy_diskctx) diskctx->dp->d_sched_policy->destroy_diskctx(diskctx); - KKASSERT(diskctx->refcount == 0x80000000); objcache_put(dsched_diskctx_cache, diskctx); atomic_subtract_int(&dsched_stats.diskctx_allocations, 1); } @@ -766,11 +794,46 @@ dsched_thread_io_unref(struct dsched_thread_io *tdio) } } +/* + * Unref and destroy the tdio even if additional refs are present. + */ +static +void +dsched_thread_io_unref_destroy(struct dsched_thread_io *tdio) +{ + int refs; + int nrefs; + + /* + * If not already transitioned to destroy-in-progress we transition + * to destroy-in-progress, cleanup our ref, and destroy the tdio. + */ + for (;;) { + refs = tdio->refcount; + cpu_ccfence(); + nrefs = refs - 1; + + KKASSERT(((refs ^ nrefs) & 0x80000000) == 0); + if (nrefs & 0x80000000) { + if (atomic_cmpset_int(&tdio->refcount, refs, nrefs)) + break; + continue; + } + nrefs |= 0x80000000; + if (atomic_cmpset_int(&tdio->refcount, refs, nrefs)) { + dsched_thread_io_destroy(tdio); + break; + } + } +} + static void dsched_thread_io_destroy(struct dsched_thread_io *tdio) { struct dsched_thread_ctx *tdctx; struct dsched_disk_ctx *diskctx; + int refs; + int nrefs; #if 0 kprintf("tdio (%p) destruction started, trace:\n", tdio); @@ -792,6 +855,7 @@ dsched_thread_io_destroy(struct dsched_thread_io *tdio) TAILQ_REMOVE(&diskctx->tdio_list, tdio, dlink); atomic_clear_int(&tdio->flags, DSCHED_LINKED_DISK_CTX); tdio->diskctx = NULL; + dsched_thread_io_unref(tdio); lockmgr(&diskctx->lock, LK_RELEASE); dsched_disk_ctx_unref(diskctx); } @@ -807,15 +871,31 @@ dsched_thread_io_destroy(struct dsched_thread_io *tdio) TAILQ_REMOVE(&tdctx->tdio_list, tdio, link); atomic_clear_int(&tdio->flags, DSCHED_LINKED_THREAD_CTX); tdio->tdctx = NULL; + dsched_thread_io_unref(tdio); lockmgr(&tdctx->lock, LK_RELEASE); dsched_thread_ctx_unref(tdctx); } - KKASSERT(tdio->refcount == 0x80000000); + + /* + * Expect tdio->refcount to be 0x80000000. If it isn't someone else + * still has a temporary ref on the tdio and we have to transition + * it back to an undestroyed-state (albeit without any associations) + * so the other user destroys it properly when the ref is released. + */ + while ((refs = tdio->refcount) != 0x80000000) { + kprintf("dsched_thread_io: destroy race tdio=%p\n", tdio); + cpu_ccfence(); + KKASSERT(refs & 0x80000000); + nrefs = refs & 0x7FFFFFFF; + if (atomic_cmpset_int(&tdio->refcount, refs, nrefs)) + return; + } + + /* + * Really for sure now. + */ objcache_put(dsched_tdio_cache, tdio); atomic_subtract_int(&dsched_stats.tdio_allocations, 1); -#if 0 - dsched_disk_ctx_unref(diskctx); -#endif } void @@ -869,7 +949,9 @@ dsched_thread_ctx_destroy(struct dsched_thread_ctx *tdctx) TAILQ_REMOVE(&tdctx->tdio_list, tdio, link); atomic_clear_int(&tdio->flags, DSCHED_LINKED_THREAD_CTX); tdio->tdctx = NULL; - dsched_thread_io_unref(tdio); + lockmgr(&tdctx->lock, LK_RELEASE); /* avoid deadlock */ + dsched_thread_io_unref_destroy(tdio); + lockmgr(&tdctx->lock, LK_EXCLUSIVE); } KKASSERT(tdctx->refcount == 0x80000000); TAILQ_REMOVE(&dsched_tdctx_list, tdctx, link); @@ -882,9 +964,12 @@ dsched_thread_ctx_destroy(struct dsched_thread_ctx *tdctx) atomic_subtract_int(&dsched_stats.tdctx_allocations, 1); } -struct dsched_thread_io * +/* + * Ensures that a tdio is assigned to tdctx and disk. + */ +void dsched_thread_io_alloc(struct disk *dp, struct dsched_thread_ctx *tdctx, - struct dsched_policy *pol) + struct dsched_policy *pol) { struct dsched_thread_io *tdio; #if 0 @@ -893,8 +978,8 @@ dsched_thread_io_alloc(struct disk *dp, struct dsched_thread_ctx *tdctx, tdio = objcache_get(dsched_tdio_cache, M_WAITOK); bzero(tdio, DSCHED_THREAD_IO_MAX_SZ); - /* XXX: maybe we do need another ref for the disk list for tdio */ - dsched_thread_io_ref(tdio); + dsched_thread_io_ref(tdio); /* prevent ripout */ + dsched_thread_io_ref(tdio); /* for diskctx ref */ DSCHED_THREAD_IO_LOCKINIT(tdio); tdio->dp = dp; @@ -911,21 +996,24 @@ dsched_thread_io_alloc(struct disk *dp, struct dsched_thread_ctx *tdctx, lockmgr(&tdio->diskctx->lock, LK_RELEASE); if (tdctx) { + /* + * Put the tdio in the tdctx list. Inherit the temporary + * ref (one ref for each list). + */ + DSCHED_THREAD_CTX_LOCK(tdctx); tdio->tdctx = tdctx; tdio->p = tdctx->p; - - /* Put the tdio in the tdctx list */ - DSCHED_THREAD_CTX_LOCK(tdctx); TAILQ_INSERT_TAIL(&tdctx->tdio_list, tdio, link); - DSCHED_THREAD_CTX_UNLOCK(tdctx); atomic_set_int(&tdio->flags, DSCHED_LINKED_THREAD_CTX); + DSCHED_THREAD_CTX_UNLOCK(tdctx); + } else { + dsched_thread_io_unref(tdio); } tdio->debug_policy = pol; tdio->debug_inited = 0xF00F1234; atomic_add_int(&dsched_stats.tdio_allocations, 1); - return tdio; } @@ -959,8 +1047,8 @@ struct dsched_thread_ctx * dsched_thread_ctx_alloc(struct proc *p) { struct dsched_thread_ctx *tdctx; - struct dsched_thread_io *tdio; - struct disk *dp = NULL; + struct disk marker; + struct disk *dp; tdctx = objcache_get(dsched_tdctx_cache, M_WAITOK); bzero(tdctx, DSCHED_THREAD_CTX_MAX_SZ); @@ -973,9 +1061,9 @@ dsched_thread_ctx_alloc(struct proc *p) tdctx->p = p; DSCHED_GLOBAL_THREAD_CTX_LOCK(); - while ((dp = disk_enumerate(dp))) { - tdio = dsched_thread_io_alloc(dp, tdctx, dp->d_sched_policy); - } + dp = NULL; + while ((dp = disk_enumerate(&marker, dp)) != NULL) + dsched_thread_io_alloc(dp, tdctx, dp->d_sched_policy); TAILQ_INSERT_TAIL(&dsched_tdctx_list, tdctx, link); DSCHED_GLOBAL_THREAD_CTX_UNLOCK(); @@ -989,15 +1077,16 @@ void policy_new(struct disk *dp, struct dsched_policy *pol) { struct dsched_thread_ctx *tdctx; struct dsched_disk_ctx *diskctx; - struct dsched_thread_io *tdio; diskctx = dsched_disk_ctx_alloc(dp, pol); dsched_disk_ctx_ref(diskctx); dsched_set_disk_priv(dp, diskctx); - TAILQ_FOREACH(tdctx, &dsched_tdctx_list, link) { - tdio = dsched_thread_io_alloc(dp, tdctx, pol); - } + /* + * XXX this is really really expensive! + */ + TAILQ_FOREACH(tdctx, &dsched_tdctx_list, link) + dsched_thread_io_alloc(dp, tdctx, pol); } void @@ -1136,21 +1225,24 @@ dsched_exit_thread(struct thread *td) atomic_subtract_int(&dsched_stats.nthreads, 1); } -struct dsched_thread_io * +/* + * Returns ref'd tdio. + * + * tdio may have additional refs for the diskctx and tdctx it resides on. + */ +void dsched_new_policy_thread_tdio(struct dsched_disk_ctx *diskctx, - struct dsched_policy *pol) { + struct dsched_policy *pol) +{ struct dsched_thread_ctx *tdctx; - struct dsched_thread_io *tdio; DSCHED_GLOBAL_THREAD_CTX_LOCK(); tdctx = dsched_get_thread_priv(curthread); KKASSERT(tdctx != NULL); - tdio = dsched_thread_io_alloc(diskctx->dp, tdctx, pol); + dsched_thread_io_alloc(diskctx->dp, tdctx, pol); DSCHED_GLOBAL_THREAD_CTX_UNLOCK(); - - return tdio; } /* DEFAULT NOOP POLICY */ diff --git a/sys/kern/subr_disk.c b/sys/kern/subr_disk.c index a83832f483..6167227ddd 100644 --- a/sys/kern/subr_disk.c +++ b/sys/kern/subr_disk.c @@ -486,9 +486,13 @@ disk_msg_core(void *arg) devfs_destroy_related(dp->d_cdev); destroy_dev(dp->d_cdev); destroy_only_dev(dp->d_rawdev); + lwkt_gettoken(&disklist_token); + while (dp->d_refs) + tsleep(&dp->d_refs, 0, "diskdel", hz / 10); LIST_REMOVE(dp, d_list); lwkt_reltoken(&disklist_token); + if (dp->d_info.d_serialno) { kfree(dp->d_info.d_serialno, M_TEMP); dp->d_info.d_serialno = NULL; @@ -886,45 +890,90 @@ disk_invalidate (struct disk *disk) dsgone(&disk->d_slice); } +/* + * Enumerate disks, pass a marker and an initial NULL dp to initialize, + * then loop with the previously returned dp. + * + * The returned dp will be referenced, preventing its destruction. When + * you pass the returned dp back into the loop the ref is dropped. + * + * WARNING: If terminating your loop early you must call + * disk_enumerate_stop(). + */ struct disk * -disk_enumerate(struct disk *disk) +disk_enumerate(struct disk *marker, struct disk *dp) { - struct disk *dp; - lwkt_gettoken(&disklist_token); - if (!disk) - dp = (LIST_FIRST(&disklist)); - else - dp = (LIST_NEXT(disk, d_list)); + if (dp) { + --dp->d_refs; + dp = LIST_NEXT(marker, d_list); + LIST_REMOVE(marker, d_list); + } else { + bzero(marker, sizeof(*marker)); + marker->d_flags = DISKFLAG_MARKER; + dp = LIST_FIRST(&disklist); + } + while (dp) { + if ((dp->d_flags & DISKFLAG_MARKER) == 0) + break; + dp = LIST_NEXT(dp, d_list); + } + if (dp) { + ++dp->d_refs; + LIST_INSERT_AFTER(dp, marker, d_list); + } lwkt_reltoken(&disklist_token); + return (dp); +} - return dp; +/* + * Terminate an enumeration early. Do not call this function if the + * enumeration ended normally. dp can be NULL, indicating that you + * wish to retain the ref count on dp. + * + * This function removes the marker. + */ +void +disk_enumerate_stop(struct disk *marker, struct disk *dp) +{ + lwkt_gettoken(&disklist_token); + LIST_REMOVE(marker, d_list); + if (dp) + --dp->d_refs; + lwkt_reltoken(&disklist_token); } static int sysctl_disks(SYSCTL_HANDLER_ARGS) { - struct disk *disk; + struct disk marker; + struct disk *dp; int error, first; - disk = NULL; first = 1; + error = 0; + dp = NULL; - while ((disk = disk_enumerate(disk))) { + while ((dp = disk_enumerate(&marker, dp))) { if (!first) { error = SYSCTL_OUT(req, " ", 1); - if (error) - return error; + if (error) { + disk_enumerate_stop(&marker, dp); + break; + } } else { first = 0; } - error = SYSCTL_OUT(req, disk->d_rawdev->si_name, - strlen(disk->d_rawdev->si_name)); - if (error) - return error; + error = SYSCTL_OUT(req, dp->d_rawdev->si_name, + strlen(dp->d_rawdev->si_name)); + if (error) { + disk_enumerate_stop(&marker, dp); + break; + } } - error = SYSCTL_OUT(req, "", 1); + if (error == 0) + error = SYSCTL_OUT(req, "", 1); return error; } @@ -1069,7 +1118,6 @@ diskioctl(struct dev_ioctl_args *ap) } if (ap->a_cmd == DIOCRECLUSTER && dev == dp->d_cdev) { - kprintf("RECLUSTER\n"); error = disk_iocom_ioctl(dp, ap->a_cmd, ap->a_data); return error; } diff --git a/sys/sys/disk.h b/sys/sys/disk.h index 28588c4aac..761aa7bc98 100644 --- a/sys/sys/disk.h +++ b/sys/sys/disk.h @@ -148,6 +148,7 @@ struct disk { const char *d_disktype; /* Disk type information */ LIST_ENTRY(disk) d_list; kdmsg_iocom_t d_iocom; /* cluster import/export */ + int d_refs; /* interlock destruction */ }; #endif @@ -155,8 +156,9 @@ struct disk { /* * d_flags */ -#define DISKFLAG_LOCK 0x1 -#define DISKFLAG_WANTED 0x2 +#define DISKFLAG_LOCK 0x0001 +#define DISKFLAG_WANTED 0x0002 +#define DISKFLAG_MARKER 0x0004 #ifdef _KERNEL cdev_t disk_create (int unit, struct disk *disk, struct dev_ops *raw_ops); @@ -171,7 +173,8 @@ int disk_getopencount(struct disk *disk); void disk_setdiskinfo_sync(struct disk *disk, struct disk_info *info); int disk_dumpcheck (cdev_t dev, u_int64_t *count, u_int64_t *blkno, u_int *secsize); int disk_dumpconf(cdev_t dev, u_int onoff); -struct disk *disk_enumerate (struct disk *disk); +struct disk *disk_enumerate (struct disk *marker, struct disk *dp); +void disk_enumerate_stop (struct disk *marker, struct disk *dp); void disk_invalidate (struct disk *disk); void disk_unprobe(struct disk *disk); @@ -186,9 +189,6 @@ void disk_iocom_update(struct disk *dp); void disk_iocom_uninit(struct disk *dp); int disk_iocom_ioctl(struct disk *dp, int cmd, void *data); void disk_clusterctl_wakeup(kdmsg_iocom_t *iocom); -int disk_lnk_rcvmsg(kdmsg_msg_t *msg); -int disk_dbg_rcvmsg(kdmsg_msg_t *msg); -int disk_adhoc_input(kdmsg_msg_t *msg); typedef struct disk_msg { struct lwkt_msg hdr; diff --git a/sys/sys/dsched.h b/sys/sys/dsched.h index e254a31aaf..b2825bfa19 100644 --- a/sys/sys/dsched.h +++ b/sys/sys/dsched.h @@ -255,9 +255,10 @@ int dsched_unregister(struct dsched_policy *d_policy); int dsched_switch(struct disk *dp, struct dsched_policy *new_policy); void dsched_set_policy(struct disk *dp, struct dsched_policy *new_policy); struct dsched_policy *dsched_find_policy(char *search); -struct disk *dsched_find_disk(char *search); +struct disk *dsched_find_disk(char *search); struct dsched_policy *dsched_policy_enumerate(struct dsched_policy *pol); -struct disk *dsched_disk_enumerate(struct disk *dp, struct dsched_policy *policy); +struct disk *dsched_disk_enumerate(struct disk *marker, struct disk *dp, + struct dsched_policy *policy); void dsched_cancel_bio(struct bio *bp); void dsched_strategy_raw(struct disk *dp, struct bio *bp); void dsched_strategy_sync(struct disk *dp, struct bio *bp); @@ -275,12 +276,13 @@ void dsched_disk_ctx_unref(struct dsched_disk_ctx *diskctx); void dsched_thread_io_unref(struct dsched_thread_io *tdio); void dsched_thread_ctx_unref(struct dsched_thread_ctx *tdctx); -struct dsched_thread_io *dsched_new_policy_thread_tdio(struct dsched_disk_ctx *diskctx, - struct dsched_policy *pol); -struct dsched_thread_io *dsched_thread_io_alloc(struct disk *dp, - struct dsched_thread_ctx *tdctx, struct dsched_policy *pol); +void dsched_new_policy_thread_tdio(struct dsched_disk_ctx *diskctx, + struct dsched_policy *pol); +void dsched_thread_io_alloc(struct disk *dp, + struct dsched_thread_ctx *tdctx, + struct dsched_policy *pol); struct dsched_disk_ctx *dsched_disk_ctx_alloc(struct disk *dp, - struct dsched_policy *pol); + struct dsched_policy *pol); struct dsched_thread_ctx *dsched_thread_ctx_alloc(struct proc *p); typedef void dsched_new_buf_t(struct buf *bp); -- 2.41.0