| 1 | /* |
| 2 | * Copyright (c) 1997, 1998 Justin T. Gibbs. |
| 3 | * All rights reserved. |
| 4 | * |
| 5 | * Redistribution and use in source and binary forms, with or without |
| 6 | * modification, are permitted provided that the following conditions |
| 7 | * are met: |
| 8 | * 1. Redistributions of source code must retain the above copyright |
| 9 | * notice, this list of conditions, and the following disclaimer, |
| 10 | * without modification, immediately at the beginning of the file. |
| 11 | * 2. The name of the author may not be used to endorse or promote products |
| 12 | * derived from this software without specific prior written permission. |
| 13 | * |
| 14 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND |
| 15 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
| 16 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
| 17 | * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR |
| 18 | * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
| 19 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS |
| 20 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
| 21 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
| 22 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY |
| 23 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
| 24 | * SUCH DAMAGE. |
| 25 | * |
| 26 | * $FreeBSD: src/sys/i386/i386/busdma_machdep.c,v 1.94 2008/08/15 20:51:31 kmacy Exp $ |
| 27 | */ |
| 28 | |
| 29 | #include <sys/param.h> |
| 30 | #include <sys/systm.h> |
| 31 | #include <sys/malloc.h> |
| 32 | #include <sys/mbuf.h> |
| 33 | #include <sys/uio.h> |
| 34 | #include <sys/bus_dma.h> |
| 35 | #include <sys/kernel.h> |
| 36 | #include <sys/sysctl.h> |
| 37 | #include <sys/lock.h> |
| 38 | |
| 39 | #include <sys/thread2.h> |
| 40 | #include <sys/spinlock2.h> |
| 41 | #include <sys/mplock2.h> |
| 42 | |
| 43 | #include <vm/vm.h> |
| 44 | #include <vm/vm_page.h> |
| 45 | |
| 46 | /* XXX needed for to access pmap to convert per-proc virtual to physical */ |
| 47 | #include <sys/proc.h> |
| 48 | #include <sys/lock.h> |
| 49 | #include <vm/vm_map.h> |
| 50 | |
| 51 | #include <machine/md_var.h> |
| 52 | |
| 53 | #define MAX_BPAGES 1024 |
| 54 | |
| 55 | /* |
| 56 | * 16 x N declared on stack. |
| 57 | */ |
| 58 | #define BUS_DMA_CACHE_SEGMENTS 8 |
| 59 | |
| 60 | struct bounce_zone; |
| 61 | struct bus_dmamap; |
| 62 | |
| 63 | struct bus_dma_tag { |
| 64 | bus_dma_tag_t parent; |
| 65 | bus_size_t alignment; |
| 66 | bus_size_t boundary; |
| 67 | bus_addr_t lowaddr; |
| 68 | bus_addr_t highaddr; |
| 69 | bus_dma_filter_t *filter; |
| 70 | void *filterarg; |
| 71 | bus_size_t maxsize; |
| 72 | u_int nsegments; |
| 73 | bus_size_t maxsegsz; |
| 74 | int flags; |
| 75 | int ref_count; |
| 76 | int map_count; |
| 77 | bus_dma_segment_t *segments; |
| 78 | struct bounce_zone *bounce_zone; |
| 79 | #ifdef SMP |
| 80 | struct spinlock spin; |
| 81 | #else |
| 82 | int unused0; |
| 83 | #endif |
| 84 | }; |
| 85 | |
| 86 | /* |
| 87 | * bus_dma_tag private flags |
| 88 | */ |
| 89 | #define BUS_DMA_BOUNCE_ALIGN BUS_DMA_BUS2 |
| 90 | #define BUS_DMA_BOUNCE_LOWADDR BUS_DMA_BUS3 |
| 91 | #define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 |
| 92 | |
| 93 | #define BUS_DMA_COULD_BOUNCE (BUS_DMA_BOUNCE_LOWADDR | BUS_DMA_BOUNCE_ALIGN) |
| 94 | |
| 95 | #define BUS_DMAMEM_KMALLOC(dmat) \ |
| 96 | ((dmat)->maxsize <= PAGE_SIZE && \ |
| 97 | (dmat)->alignment <= PAGE_SIZE && \ |
| 98 | (dmat)->lowaddr >= ptoa(Maxmem)) |
| 99 | |
| 100 | struct bounce_page { |
| 101 | vm_offset_t vaddr; /* kva of bounce buffer */ |
| 102 | bus_addr_t busaddr; /* Physical address */ |
| 103 | vm_offset_t datavaddr; /* kva of client data */ |
| 104 | bus_size_t datacount; /* client data count */ |
| 105 | STAILQ_ENTRY(bounce_page) links; |
| 106 | }; |
| 107 | |
| 108 | struct bounce_zone { |
| 109 | STAILQ_ENTRY(bounce_zone) links; |
| 110 | STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; |
| 111 | STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; |
| 112 | #ifdef SMP |
| 113 | struct spinlock spin; |
| 114 | #else |
| 115 | int unused0; |
| 116 | #endif |
| 117 | int total_bpages; |
| 118 | int free_bpages; |
| 119 | int reserved_bpages; |
| 120 | int active_bpages; |
| 121 | int total_bounced; |
| 122 | int total_deferred; |
| 123 | int reserve_failed; |
| 124 | bus_size_t alignment; |
| 125 | bus_addr_t lowaddr; |
| 126 | char zoneid[8]; |
| 127 | char lowaddrid[20]; |
| 128 | struct sysctl_ctx_list sysctl_ctx; |
| 129 | struct sysctl_oid *sysctl_tree; |
| 130 | }; |
| 131 | |
| 132 | #ifdef SMP |
| 133 | #define BZ_LOCK(bz) spin_lock(&(bz)->spin) |
| 134 | #define BZ_UNLOCK(bz) spin_unlock(&(bz)->spin) |
| 135 | #else |
| 136 | #define BZ_LOCK(bz) crit_enter() |
| 137 | #define BZ_UNLOCK(bz) crit_exit() |
| 138 | #endif |
| 139 | |
| 140 | static struct lwkt_token bounce_zone_tok = |
| 141 | LWKT_TOKEN_INITIALIZER(bounce_zone_tok); |
| 142 | static int busdma_zonecount; |
| 143 | static STAILQ_HEAD(, bounce_zone) bounce_zone_list = |
| 144 | STAILQ_HEAD_INITIALIZER(bounce_zone_list); |
| 145 | |
| 146 | static int busdma_priv_zonecount = -1; |
| 147 | |
| 148 | int busdma_swi_pending; |
| 149 | static int total_bounce_pages; |
| 150 | static int max_bounce_pages = MAX_BPAGES; |
| 151 | static int bounce_alignment = 1; /* XXX temporary */ |
| 152 | |
| 153 | TUNABLE_INT("hw.busdma.max_bpages", &max_bounce_pages); |
| 154 | TUNABLE_INT("hw.busdma.bounce_alignment", &bounce_alignment); |
| 155 | |
| 156 | struct bus_dmamap { |
| 157 | struct bp_list bpages; |
| 158 | int pagesneeded; |
| 159 | int pagesreserved; |
| 160 | bus_dma_tag_t dmat; |
| 161 | void *buf; /* unmapped buffer pointer */ |
| 162 | bus_size_t buflen; /* unmapped buffer length */ |
| 163 | bus_dmamap_callback_t *callback; |
| 164 | void *callback_arg; |
| 165 | STAILQ_ENTRY(bus_dmamap) links; |
| 166 | }; |
| 167 | |
| 168 | static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist = |
| 169 | STAILQ_HEAD_INITIALIZER(bounce_map_callbacklist); |
| 170 | static struct spinlock bounce_map_list_spin = |
| 171 | SPINLOCK_INITIALIZER(&bounce_map_list_spin); |
| 172 | |
| 173 | static struct bus_dmamap nobounce_dmamap; |
| 174 | |
| 175 | static int alloc_bounce_zone(bus_dma_tag_t); |
| 176 | static int alloc_bounce_pages(bus_dma_tag_t, u_int, int); |
| 177 | static void free_bounce_pages_all(bus_dma_tag_t); |
| 178 | static void free_bounce_zone(bus_dma_tag_t); |
| 179 | static int reserve_bounce_pages(bus_dma_tag_t, bus_dmamap_t, int); |
| 180 | static void return_bounce_pages(bus_dma_tag_t, bus_dmamap_t); |
| 181 | static bus_addr_t add_bounce_page(bus_dma_tag_t, bus_dmamap_t, |
| 182 | vm_offset_t, bus_size_t); |
| 183 | static void free_bounce_page(bus_dma_tag_t, struct bounce_page *); |
| 184 | |
| 185 | static bus_dmamap_t get_map_waiting(bus_dma_tag_t); |
| 186 | static void add_map_callback(bus_dmamap_t); |
| 187 | |
| 188 | SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters"); |
| 189 | SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bounce_pages, |
| 190 | 0, "Total bounce pages"); |
| 191 | SYSCTL_INT(_hw_busdma, OID_AUTO, max_bpages, CTLFLAG_RD, &max_bounce_pages, |
| 192 | 0, "Max bounce pages per bounce zone"); |
| 193 | SYSCTL_INT(_hw_busdma, OID_AUTO, bounce_alignment, CTLFLAG_RD, |
| 194 | &bounce_alignment, 0, "Obey alignment constraint"); |
| 195 | |
| 196 | static __inline int |
| 197 | run_filter(bus_dma_tag_t dmat, bus_addr_t paddr) |
| 198 | { |
| 199 | int retval; |
| 200 | |
| 201 | retval = 0; |
| 202 | do { |
| 203 | if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr) || |
| 204 | (bounce_alignment && (paddr & (dmat->alignment - 1)) != 0)) |
| 205 | && (dmat->filter == NULL || |
| 206 | dmat->filter(dmat->filterarg, paddr) != 0)) |
| 207 | retval = 1; |
| 208 | |
| 209 | dmat = dmat->parent; |
| 210 | } while (retval == 0 && dmat != NULL); |
| 211 | return (retval); |
| 212 | } |
| 213 | |
| 214 | static __inline |
| 215 | bus_dma_segment_t * |
| 216 | bus_dma_tag_lock(bus_dma_tag_t tag, bus_dma_segment_t *cache) |
| 217 | { |
| 218 | if (tag->flags & BUS_DMA_PROTECTED) |
| 219 | return(tag->segments); |
| 220 | |
| 221 | if (tag->nsegments <= BUS_DMA_CACHE_SEGMENTS) |
| 222 | return(cache); |
| 223 | #ifdef SMP |
| 224 | spin_lock(&tag->spin); |
| 225 | #endif |
| 226 | return(tag->segments); |
| 227 | } |
| 228 | |
| 229 | static __inline |
| 230 | void |
| 231 | bus_dma_tag_unlock(bus_dma_tag_t tag) |
| 232 | { |
| 233 | #ifdef SMP |
| 234 | if (tag->flags & BUS_DMA_PROTECTED) |
| 235 | return; |
| 236 | |
| 237 | if (tag->nsegments > BUS_DMA_CACHE_SEGMENTS) |
| 238 | spin_unlock(&tag->spin); |
| 239 | #endif |
| 240 | } |
| 241 | |
| 242 | /* |
| 243 | * Allocate a device specific dma_tag. |
| 244 | */ |
| 245 | int |
| 246 | bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, |
| 247 | bus_size_t boundary, bus_addr_t lowaddr, |
| 248 | bus_addr_t highaddr, bus_dma_filter_t *filter, |
| 249 | void *filterarg, bus_size_t maxsize, int nsegments, |
| 250 | bus_size_t maxsegsz, int flags, bus_dma_tag_t *dmat) |
| 251 | { |
| 252 | bus_dma_tag_t newtag; |
| 253 | int error = 0; |
| 254 | |
| 255 | /* |
| 256 | * Sanity checks |
| 257 | */ |
| 258 | |
| 259 | if (alignment == 0) |
| 260 | alignment = 1; |
| 261 | if (alignment & (alignment - 1)) |
| 262 | panic("alignment must be power of 2"); |
| 263 | |
| 264 | if (boundary != 0) { |
| 265 | if (boundary & (boundary - 1)) |
| 266 | panic("boundary must be power of 2"); |
| 267 | if (boundary < maxsegsz) { |
| 268 | kprintf("boundary < maxsegsz:\n"); |
| 269 | print_backtrace(-1); |
| 270 | maxsegsz = boundary; |
| 271 | } |
| 272 | } |
| 273 | |
| 274 | /* Return a NULL tag on failure */ |
| 275 | *dmat = NULL; |
| 276 | |
| 277 | newtag = kmalloc(sizeof(*newtag), M_DEVBUF, M_INTWAIT | M_ZERO); |
| 278 | |
| 279 | #ifdef SMP |
| 280 | spin_init(&newtag->spin); |
| 281 | #endif |
| 282 | newtag->parent = parent; |
| 283 | newtag->alignment = alignment; |
| 284 | newtag->boundary = boundary; |
| 285 | newtag->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1); |
| 286 | newtag->highaddr = trunc_page((vm_paddr_t)highaddr) + (PAGE_SIZE - 1); |
| 287 | newtag->filter = filter; |
| 288 | newtag->filterarg = filterarg; |
| 289 | newtag->maxsize = maxsize; |
| 290 | newtag->nsegments = nsegments; |
| 291 | newtag->maxsegsz = maxsegsz; |
| 292 | newtag->flags = flags; |
| 293 | newtag->ref_count = 1; /* Count ourself */ |
| 294 | newtag->map_count = 0; |
| 295 | newtag->segments = NULL; |
| 296 | newtag->bounce_zone = NULL; |
| 297 | |
| 298 | /* Take into account any restrictions imposed by our parent tag */ |
| 299 | if (parent != NULL) { |
| 300 | newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); |
| 301 | newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); |
| 302 | |
| 303 | if (newtag->boundary == 0) { |
| 304 | newtag->boundary = parent->boundary; |
| 305 | } else if (parent->boundary != 0) { |
| 306 | newtag->boundary = MIN(parent->boundary, |
| 307 | newtag->boundary); |
| 308 | } |
| 309 | |
| 310 | #ifdef notyet |
| 311 | newtag->alignment = MAX(parent->alignment, newtag->alignment); |
| 312 | #endif |
| 313 | |
| 314 | if (newtag->filter == NULL) { |
| 315 | /* |
| 316 | * Short circuit looking at our parent directly |
| 317 | * since we have encapsulated all of its information |
| 318 | */ |
| 319 | newtag->filter = parent->filter; |
| 320 | newtag->filterarg = parent->filterarg; |
| 321 | newtag->parent = parent->parent; |
| 322 | } |
| 323 | if (newtag->parent != NULL) |
| 324 | parent->ref_count++; |
| 325 | } |
| 326 | |
| 327 | if (newtag->lowaddr < ptoa(Maxmem)) |
| 328 | newtag->flags |= BUS_DMA_BOUNCE_LOWADDR; |
| 329 | if (bounce_alignment && newtag->alignment > 1 && |
| 330 | !(newtag->flags & BUS_DMA_ALIGNED)) |
| 331 | newtag->flags |= BUS_DMA_BOUNCE_ALIGN; |
| 332 | |
| 333 | if ((newtag->flags & BUS_DMA_COULD_BOUNCE) && |
| 334 | (flags & BUS_DMA_ALLOCNOW) != 0) { |
| 335 | struct bounce_zone *bz; |
| 336 | |
| 337 | /* Must bounce */ |
| 338 | |
| 339 | error = alloc_bounce_zone(newtag); |
| 340 | if (error) |
| 341 | goto back; |
| 342 | bz = newtag->bounce_zone; |
| 343 | |
| 344 | if ((newtag->flags & BUS_DMA_ALLOCALL) == 0 && |
| 345 | ptoa(bz->total_bpages) < maxsize) { |
| 346 | int pages; |
| 347 | |
| 348 | if (flags & BUS_DMA_ONEBPAGE) { |
| 349 | pages = 1; |
| 350 | } else { |
| 351 | pages = atop(round_page(maxsize)) - |
| 352 | bz->total_bpages; |
| 353 | pages = MAX(pages, 1); |
| 354 | } |
| 355 | |
| 356 | /* Add pages to our bounce pool */ |
| 357 | if (alloc_bounce_pages(newtag, pages, flags) < pages) |
| 358 | error = ENOMEM; |
| 359 | |
| 360 | /* Performed initial allocation */ |
| 361 | newtag->flags |= BUS_DMA_MIN_ALLOC_COMP; |
| 362 | } |
| 363 | } |
| 364 | back: |
| 365 | if (error) { |
| 366 | free_bounce_zone(newtag); |
| 367 | kfree(newtag, M_DEVBUF); |
| 368 | } else { |
| 369 | *dmat = newtag; |
| 370 | } |
| 371 | return error; |
| 372 | } |
| 373 | |
| 374 | int |
| 375 | bus_dma_tag_destroy(bus_dma_tag_t dmat) |
| 376 | { |
| 377 | if (dmat != NULL) { |
| 378 | if (dmat->map_count != 0) |
| 379 | return (EBUSY); |
| 380 | |
| 381 | while (dmat != NULL) { |
| 382 | bus_dma_tag_t parent; |
| 383 | |
| 384 | parent = dmat->parent; |
| 385 | dmat->ref_count--; |
| 386 | if (dmat->ref_count == 0) { |
| 387 | free_bounce_zone(dmat); |
| 388 | if (dmat->segments != NULL) |
| 389 | kfree(dmat->segments, M_DEVBUF); |
| 390 | kfree(dmat, M_DEVBUF); |
| 391 | /* |
| 392 | * Last reference count, so |
| 393 | * release our reference |
| 394 | * count on our parent. |
| 395 | */ |
| 396 | dmat = parent; |
| 397 | } else |
| 398 | dmat = NULL; |
| 399 | } |
| 400 | } |
| 401 | return (0); |
| 402 | } |
| 403 | |
| 404 | bus_size_t |
| 405 | bus_dma_tag_getmaxsize(bus_dma_tag_t tag) |
| 406 | { |
| 407 | return(tag->maxsize); |
| 408 | } |
| 409 | |
| 410 | /* |
| 411 | * Allocate a handle for mapping from kva/uva/physical |
| 412 | * address space into bus device space. |
| 413 | */ |
| 414 | int |
| 415 | bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) |
| 416 | { |
| 417 | int error; |
| 418 | |
| 419 | error = 0; |
| 420 | |
| 421 | if (dmat->segments == NULL) { |
| 422 | KKASSERT(dmat->nsegments && dmat->nsegments < 16384); |
| 423 | dmat->segments = kmalloc(sizeof(bus_dma_segment_t) * |
| 424 | dmat->nsegments, M_DEVBUF, M_INTWAIT); |
| 425 | } |
| 426 | |
| 427 | if (dmat->flags & BUS_DMA_COULD_BOUNCE) { |
| 428 | struct bounce_zone *bz; |
| 429 | int maxpages; |
| 430 | |
| 431 | /* Must bounce */ |
| 432 | |
| 433 | if (dmat->bounce_zone == NULL) { |
| 434 | error = alloc_bounce_zone(dmat); |
| 435 | if (error) |
| 436 | return error; |
| 437 | } |
| 438 | bz = dmat->bounce_zone; |
| 439 | |
| 440 | *mapp = kmalloc(sizeof(**mapp), M_DEVBUF, M_INTWAIT | M_ZERO); |
| 441 | |
| 442 | /* Initialize the new map */ |
| 443 | STAILQ_INIT(&((*mapp)->bpages)); |
| 444 | |
| 445 | /* |
| 446 | * Attempt to add pages to our pool on a per-instance |
| 447 | * basis up to a sane limit. |
| 448 | */ |
| 449 | if (dmat->flags & BUS_DMA_ALLOCALL) { |
| 450 | maxpages = Maxmem - atop(dmat->lowaddr); |
| 451 | } else if (dmat->flags & BUS_DMA_BOUNCE_ALIGN) { |
| 452 | maxpages = max_bounce_pages; |
| 453 | } else { |
| 454 | maxpages = MIN(max_bounce_pages, |
| 455 | Maxmem - atop(dmat->lowaddr)); |
| 456 | } |
| 457 | if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 || |
| 458 | (dmat->map_count > 0 && bz->total_bpages < maxpages)) { |
| 459 | int pages; |
| 460 | |
| 461 | if (flags & BUS_DMA_ONEBPAGE) { |
| 462 | pages = 1; |
| 463 | } else { |
| 464 | pages = atop(round_page(dmat->maxsize)); |
| 465 | pages = MIN(maxpages - bz->total_bpages, pages); |
| 466 | pages = MAX(pages, 1); |
| 467 | } |
| 468 | if (alloc_bounce_pages(dmat, pages, flags) < pages) |
| 469 | error = ENOMEM; |
| 470 | |
| 471 | if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) { |
| 472 | if (!error && |
| 473 | (dmat->flags & BUS_DMA_ALLOCALL) == 0) |
| 474 | dmat->flags |= BUS_DMA_MIN_ALLOC_COMP; |
| 475 | } else { |
| 476 | error = 0; |
| 477 | } |
| 478 | } |
| 479 | } else { |
| 480 | *mapp = NULL; |
| 481 | } |
| 482 | if (!error) { |
| 483 | dmat->map_count++; |
| 484 | } else { |
| 485 | kfree(*mapp, M_DEVBUF); |
| 486 | *mapp = NULL; |
| 487 | } |
| 488 | return error; |
| 489 | } |
| 490 | |
| 491 | /* |
| 492 | * Destroy a handle for mapping from kva/uva/physical |
| 493 | * address space into bus device space. |
| 494 | */ |
| 495 | int |
| 496 | bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) |
| 497 | { |
| 498 | if (map != NULL) { |
| 499 | if (STAILQ_FIRST(&map->bpages) != NULL) |
| 500 | return (EBUSY); |
| 501 | kfree(map, M_DEVBUF); |
| 502 | } |
| 503 | dmat->map_count--; |
| 504 | return (0); |
| 505 | } |
| 506 | |
| 507 | static __inline bus_size_t |
| 508 | check_kmalloc(bus_dma_tag_t dmat, const void *vaddr0, int verify) |
| 509 | { |
| 510 | bus_size_t maxsize = 0; |
| 511 | uintptr_t vaddr = (uintptr_t)vaddr0; |
| 512 | |
| 513 | if ((vaddr ^ (vaddr + dmat->maxsize - 1)) & ~PAGE_MASK) { |
| 514 | if (verify || bootverbose) |
| 515 | kprintf("boundary check failed\n"); |
| 516 | if (verify) |
| 517 | print_backtrace(-1); /* XXX panic */ |
| 518 | maxsize = dmat->maxsize; |
| 519 | } |
| 520 | if (vaddr & (dmat->alignment - 1)) { |
| 521 | if (verify || bootverbose) |
| 522 | kprintf("alignment check failed\n"); |
| 523 | if (verify) |
| 524 | print_backtrace(-1); /* XXX panic */ |
| 525 | if (dmat->maxsize < dmat->alignment) |
| 526 | maxsize = dmat->alignment; |
| 527 | else |
| 528 | maxsize = dmat->maxsize; |
| 529 | } |
| 530 | return maxsize; |
| 531 | } |
| 532 | |
| 533 | /* |
| 534 | * Allocate a piece of memory that can be efficiently mapped into |
| 535 | * bus device space based on the constraints lited in the dma tag. |
| 536 | * |
| 537 | * mapp is degenerate. By definition this allocation should not require |
| 538 | * bounce buffers so do not allocate a dma map. |
| 539 | */ |
| 540 | int |
| 541 | bus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddr, int flags, |
| 542 | bus_dmamap_t *mapp) |
| 543 | { |
| 544 | int mflags; |
| 545 | |
| 546 | /* If we succeed, no mapping/bouncing will be required */ |
| 547 | *mapp = NULL; |
| 548 | |
| 549 | if (dmat->segments == NULL) { |
| 550 | KKASSERT(dmat->nsegments < 16384); |
| 551 | dmat->segments = kmalloc(sizeof(bus_dma_segment_t) * |
| 552 | dmat->nsegments, M_DEVBUF, M_INTWAIT); |
| 553 | } |
| 554 | |
| 555 | if (flags & BUS_DMA_NOWAIT) |
| 556 | mflags = M_NOWAIT; |
| 557 | else |
| 558 | mflags = M_WAITOK; |
| 559 | if (flags & BUS_DMA_ZERO) |
| 560 | mflags |= M_ZERO; |
| 561 | |
| 562 | if (BUS_DMAMEM_KMALLOC(dmat)) { |
| 563 | bus_size_t maxsize; |
| 564 | |
| 565 | *vaddr = kmalloc(dmat->maxsize, M_DEVBUF, mflags); |
| 566 | |
| 567 | /* |
| 568 | * XXX |
| 569 | * Check whether the allocation |
| 570 | * - crossed a page boundary |
| 571 | * - was not aligned |
| 572 | * Retry with power-of-2 alignment in the above cases. |
| 573 | */ |
| 574 | maxsize = check_kmalloc(dmat, *vaddr, 0); |
| 575 | if (maxsize) { |
| 576 | size_t size; |
| 577 | |
| 578 | kfree(*vaddr, M_DEVBUF); |
| 579 | /* XXX check for overflow? */ |
| 580 | for (size = 1; size <= maxsize; size <<= 1) |
| 581 | ; |
| 582 | *vaddr = kmalloc(size, M_DEVBUF, mflags); |
| 583 | check_kmalloc(dmat, *vaddr, 1); |
| 584 | } |
| 585 | } else { |
| 586 | /* |
| 587 | * XXX Use Contigmalloc until it is merged into this facility |
| 588 | * and handles multi-seg allocations. Nobody is doing |
| 589 | * multi-seg allocations yet though. |
| 590 | */ |
| 591 | *vaddr = contigmalloc(dmat->maxsize, M_DEVBUF, mflags, |
| 592 | 0ul, dmat->lowaddr, dmat->alignment, dmat->boundary); |
| 593 | } |
| 594 | if (*vaddr == NULL) |
| 595 | return (ENOMEM); |
| 596 | return (0); |
| 597 | } |
| 598 | |
| 599 | /* |
| 600 | * Free a piece of memory and it's allociated dmamap, that was allocated |
| 601 | * via bus_dmamem_alloc. Make the same choice for free/contigfree. |
| 602 | */ |
| 603 | void |
| 604 | bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) |
| 605 | { |
| 606 | /* |
| 607 | * dmamem does not need to be bounced, so the map should be |
| 608 | * NULL |
| 609 | */ |
| 610 | if (map != NULL) |
| 611 | panic("bus_dmamem_free: Invalid map freed"); |
| 612 | if (BUS_DMAMEM_KMALLOC(dmat)) |
| 613 | kfree(vaddr, M_DEVBUF); |
| 614 | else |
| 615 | contigfree(vaddr, dmat->maxsize, M_DEVBUF); |
| 616 | } |
| 617 | |
| 618 | static __inline vm_paddr_t |
| 619 | _bus_dma_extract(pmap_t pmap, vm_offset_t vaddr) |
| 620 | { |
| 621 | if (pmap) |
| 622 | return pmap_extract(pmap, vaddr); |
| 623 | else |
| 624 | return pmap_kextract(vaddr); |
| 625 | } |
| 626 | |
| 627 | /* |
| 628 | * Utility function to load a linear buffer. lastaddrp holds state |
| 629 | * between invocations (for multiple-buffer loads). segp contains |
| 630 | * the segment following the starting one on entrace, and the ending |
| 631 | * segment on exit. first indicates if this is the first invocation |
| 632 | * of this function. |
| 633 | */ |
| 634 | static int |
| 635 | _bus_dmamap_load_buffer(bus_dma_tag_t dmat, |
| 636 | bus_dmamap_t map, |
| 637 | void *buf, bus_size_t buflen, |
| 638 | bus_dma_segment_t *segments, |
| 639 | int nsegments, |
| 640 | pmap_t pmap, |
| 641 | int flags, |
| 642 | vm_paddr_t *lastpaddrp, |
| 643 | int *segp, |
| 644 | int first) |
| 645 | { |
| 646 | vm_offset_t vaddr; |
| 647 | vm_paddr_t paddr, nextpaddr; |
| 648 | bus_dma_segment_t *sg; |
| 649 | bus_addr_t bmask; |
| 650 | int seg, error = 0; |
| 651 | |
| 652 | if (map == NULL) |
| 653 | map = &nobounce_dmamap; |
| 654 | |
| 655 | #ifdef INVARIANTS |
| 656 | if (dmat->flags & BUS_DMA_ALIGNED) |
| 657 | KKASSERT(((uintptr_t)buf & (dmat->alignment - 1)) == 0); |
| 658 | #endif |
| 659 | |
| 660 | /* |
| 661 | * If we are being called during a callback, pagesneeded will |
| 662 | * be non-zero, so we can avoid doing the work twice. |
| 663 | */ |
| 664 | if ((dmat->flags & BUS_DMA_COULD_BOUNCE) && |
| 665 | map != &nobounce_dmamap && map->pagesneeded == 0) { |
| 666 | vm_offset_t vendaddr; |
| 667 | |
| 668 | /* |
| 669 | * Count the number of bounce pages |
| 670 | * needed in order to complete this transfer |
| 671 | */ |
| 672 | vaddr = (vm_offset_t)buf; |
| 673 | vendaddr = (vm_offset_t)buf + buflen; |
| 674 | |
| 675 | while (vaddr < vendaddr) { |
| 676 | paddr = _bus_dma_extract(pmap, vaddr); |
| 677 | if (run_filter(dmat, paddr) != 0) |
| 678 | map->pagesneeded++; |
| 679 | vaddr += (PAGE_SIZE - (vaddr & PAGE_MASK)); |
| 680 | } |
| 681 | } |
| 682 | |
| 683 | /* Reserve Necessary Bounce Pages */ |
| 684 | if (map->pagesneeded != 0) { |
| 685 | struct bounce_zone *bz; |
| 686 | |
| 687 | bz = dmat->bounce_zone; |
| 688 | BZ_LOCK(bz); |
| 689 | if (flags & BUS_DMA_NOWAIT) { |
| 690 | if (reserve_bounce_pages(dmat, map, 0) != 0) { |
| 691 | BZ_UNLOCK(bz); |
| 692 | error = ENOMEM; |
| 693 | goto free_bounce; |
| 694 | } |
| 695 | } else { |
| 696 | if (reserve_bounce_pages(dmat, map, 1) != 0) { |
| 697 | /* Queue us for resources */ |
| 698 | map->dmat = dmat; |
| 699 | map->buf = buf; |
| 700 | map->buflen = buflen; |
| 701 | |
| 702 | STAILQ_INSERT_TAIL( |
| 703 | &dmat->bounce_zone->bounce_map_waitinglist, |
| 704 | map, links); |
| 705 | BZ_UNLOCK(bz); |
| 706 | |
| 707 | return (EINPROGRESS); |
| 708 | } |
| 709 | } |
| 710 | BZ_UNLOCK(bz); |
| 711 | } |
| 712 | |
| 713 | KKASSERT(*segp >= 1 && *segp <= nsegments); |
| 714 | seg = *segp; |
| 715 | sg = &segments[seg - 1]; |
| 716 | |
| 717 | vaddr = (vm_offset_t)buf; |
| 718 | nextpaddr = *lastpaddrp; |
| 719 | bmask = ~(dmat->boundary - 1); /* note: will be 0 if boundary is 0 */ |
| 720 | |
| 721 | /* force at least one segment */ |
| 722 | do { |
| 723 | bus_size_t size; |
| 724 | |
| 725 | /* |
| 726 | * Per-page main loop |
| 727 | */ |
| 728 | paddr = _bus_dma_extract(pmap, vaddr); |
| 729 | size = PAGE_SIZE - (paddr & PAGE_MASK); |
| 730 | if (size > buflen) |
| 731 | size = buflen; |
| 732 | if (map->pagesneeded != 0 && run_filter(dmat, paddr)) { |
| 733 | /* |
| 734 | * note: this paddr has the same in-page offset |
| 735 | * as vaddr and thus the paddr above, so the |
| 736 | * size does not have to be recalculated |
| 737 | */ |
| 738 | paddr = add_bounce_page(dmat, map, vaddr, size); |
| 739 | } |
| 740 | |
| 741 | /* |
| 742 | * Fill in the bus_dma_segment |
| 743 | */ |
| 744 | if (first) { |
| 745 | sg->ds_addr = paddr; |
| 746 | sg->ds_len = size; |
| 747 | first = 0; |
| 748 | } else if (paddr == nextpaddr) { |
| 749 | sg->ds_len += size; |
| 750 | } else { |
| 751 | sg++; |
| 752 | seg++; |
| 753 | if (seg > nsegments) |
| 754 | break; |
| 755 | sg->ds_addr = paddr; |
| 756 | sg->ds_len = size; |
| 757 | } |
| 758 | nextpaddr = paddr + size; |
| 759 | |
| 760 | /* |
| 761 | * Handle maxsegsz and boundary issues with a nested loop |
| 762 | */ |
| 763 | for (;;) { |
| 764 | bus_size_t tmpsize; |
| 765 | |
| 766 | /* |
| 767 | * Limit to the boundary and maximum segment size |
| 768 | */ |
| 769 | if (((nextpaddr - 1) ^ sg->ds_addr) & bmask) { |
| 770 | tmpsize = dmat->boundary - |
| 771 | (sg->ds_addr & ~bmask); |
| 772 | if (tmpsize > dmat->maxsegsz) |
| 773 | tmpsize = dmat->maxsegsz; |
| 774 | KKASSERT(tmpsize < sg->ds_len); |
| 775 | } else if (sg->ds_len > dmat->maxsegsz) { |
| 776 | tmpsize = dmat->maxsegsz; |
| 777 | } else { |
| 778 | break; |
| 779 | } |
| 780 | |
| 781 | /* |
| 782 | * Futz, split the data into a new segment. |
| 783 | */ |
| 784 | if (seg >= nsegments) |
| 785 | goto fail; |
| 786 | sg[1].ds_len = sg[0].ds_len - tmpsize; |
| 787 | sg[1].ds_addr = sg[0].ds_addr + tmpsize; |
| 788 | sg[0].ds_len = tmpsize; |
| 789 | sg++; |
| 790 | seg++; |
| 791 | } |
| 792 | |
| 793 | /* |
| 794 | * Adjust for loop |
| 795 | */ |
| 796 | buflen -= size; |
| 797 | vaddr += size; |
| 798 | } while (buflen > 0); |
| 799 | fail: |
| 800 | if (buflen != 0) |
| 801 | error = EFBIG; |
| 802 | |
| 803 | *segp = seg; |
| 804 | *lastpaddrp = nextpaddr; |
| 805 | |
| 806 | free_bounce: |
| 807 | if (error && (dmat->flags & BUS_DMA_COULD_BOUNCE) && |
| 808 | map != &nobounce_dmamap) { |
| 809 | _bus_dmamap_unload(dmat, map); |
| 810 | return_bounce_pages(dmat, map); |
| 811 | } |
| 812 | return error; |
| 813 | } |
| 814 | |
| 815 | /* |
| 816 | * Map the buffer buf into bus space using the dmamap map. |
| 817 | */ |
| 818 | int |
| 819 | bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, |
| 820 | bus_size_t buflen, bus_dmamap_callback_t *callback, |
| 821 | void *callback_arg, int flags) |
| 822 | { |
| 823 | bus_dma_segment_t cache_segments[BUS_DMA_CACHE_SEGMENTS]; |
| 824 | bus_dma_segment_t *segments; |
| 825 | vm_paddr_t lastaddr = 0; |
| 826 | int error, nsegs = 1; |
| 827 | |
| 828 | if (map != NULL) { |
| 829 | /* |
| 830 | * XXX |
| 831 | * Follow old semantics. Once all of the callers are fixed, |
| 832 | * we should get rid of these internal flag "adjustment". |
| 833 | */ |
| 834 | flags &= ~BUS_DMA_NOWAIT; |
| 835 | flags |= BUS_DMA_WAITOK; |
| 836 | |
| 837 | map->callback = callback; |
| 838 | map->callback_arg = callback_arg; |
| 839 | } |
| 840 | |
| 841 | segments = bus_dma_tag_lock(dmat, cache_segments); |
| 842 | error = _bus_dmamap_load_buffer(dmat, map, buf, buflen, |
| 843 | segments, dmat->nsegments, |
| 844 | NULL, flags, &lastaddr, &nsegs, 1); |
| 845 | if (error == EINPROGRESS) { |
| 846 | KKASSERT((dmat->flags & |
| 847 | (BUS_DMA_PRIVBZONE | BUS_DMA_ALLOCALL)) != |
| 848 | (BUS_DMA_PRIVBZONE | BUS_DMA_ALLOCALL)); |
| 849 | |
| 850 | if (dmat->flags & BUS_DMA_PROTECTED) |
| 851 | panic("protected dmamap callback will be defered"); |
| 852 | |
| 853 | bus_dma_tag_unlock(dmat); |
| 854 | return error; |
| 855 | } |
| 856 | callback(callback_arg, segments, nsegs, error); |
| 857 | bus_dma_tag_unlock(dmat); |
| 858 | return 0; |
| 859 | } |
| 860 | |
| 861 | /* |
| 862 | * Like _bus_dmamap_load(), but for mbufs. |
| 863 | */ |
| 864 | int |
| 865 | bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, |
| 866 | struct mbuf *m0, |
| 867 | bus_dmamap_callback2_t *callback, void *callback_arg, |
| 868 | int flags) |
| 869 | { |
| 870 | bus_dma_segment_t cache_segments[BUS_DMA_CACHE_SEGMENTS]; |
| 871 | bus_dma_segment_t *segments; |
| 872 | int nsegs, error; |
| 873 | |
| 874 | /* |
| 875 | * XXX |
| 876 | * Follow old semantics. Once all of the callers are fixed, |
| 877 | * we should get rid of these internal flag "adjustment". |
| 878 | */ |
| 879 | flags &= ~BUS_DMA_WAITOK; |
| 880 | flags |= BUS_DMA_NOWAIT; |
| 881 | |
| 882 | segments = bus_dma_tag_lock(dmat, cache_segments); |
| 883 | error = bus_dmamap_load_mbuf_segment(dmat, map, m0, |
| 884 | segments, dmat->nsegments, &nsegs, flags); |
| 885 | if (error) { |
| 886 | /* force "no valid mappings" in callback */ |
| 887 | callback(callback_arg, segments, 0, |
| 888 | 0, error); |
| 889 | } else { |
| 890 | callback(callback_arg, segments, nsegs, |
| 891 | m0->m_pkthdr.len, error); |
| 892 | } |
| 893 | bus_dma_tag_unlock(dmat); |
| 894 | return error; |
| 895 | } |
| 896 | |
| 897 | int |
| 898 | bus_dmamap_load_mbuf_segment(bus_dma_tag_t dmat, bus_dmamap_t map, |
| 899 | struct mbuf *m0, |
| 900 | bus_dma_segment_t *segs, int maxsegs, |
| 901 | int *nsegs, int flags) |
| 902 | { |
| 903 | int error; |
| 904 | |
| 905 | M_ASSERTPKTHDR(m0); |
| 906 | |
| 907 | KASSERT(maxsegs >= 1, ("invalid maxsegs %d", maxsegs)); |
| 908 | KASSERT(maxsegs <= dmat->nsegments, |
| 909 | ("%d too many segments, dmat only supports %d segments", |
| 910 | maxsegs, dmat->nsegments)); |
| 911 | KASSERT(flags & BUS_DMA_NOWAIT, |
| 912 | ("only BUS_DMA_NOWAIT is supported")); |
| 913 | |
| 914 | if (m0->m_pkthdr.len <= dmat->maxsize) { |
| 915 | int first = 1; |
| 916 | vm_paddr_t lastaddr = 0; |
| 917 | struct mbuf *m; |
| 918 | |
| 919 | *nsegs = 1; |
| 920 | error = 0; |
| 921 | for (m = m0; m != NULL && error == 0; m = m->m_next) { |
| 922 | if (m->m_len == 0) |
| 923 | continue; |
| 924 | |
| 925 | error = _bus_dmamap_load_buffer(dmat, map, |
| 926 | m->m_data, m->m_len, |
| 927 | segs, maxsegs, |
| 928 | NULL, flags, &lastaddr, |
| 929 | nsegs, first); |
| 930 | if (error == ENOMEM && !first) { |
| 931 | /* |
| 932 | * Out of bounce pages due to too many |
| 933 | * fragments in the mbuf chain; return |
| 934 | * EFBIG instead. |
| 935 | */ |
| 936 | error = EFBIG; |
| 937 | } |
| 938 | first = 0; |
| 939 | } |
| 940 | #ifdef INVARIANTS |
| 941 | if (!error) |
| 942 | KKASSERT(*nsegs <= maxsegs && *nsegs >= 1); |
| 943 | #endif |
| 944 | } else { |
| 945 | *nsegs = 0; |
| 946 | error = EINVAL; |
| 947 | } |
| 948 | KKASSERT(error != EINPROGRESS); |
| 949 | return error; |
| 950 | } |
| 951 | |
| 952 | /* |
| 953 | * Like _bus_dmamap_load(), but for uios. |
| 954 | */ |
| 955 | int |
| 956 | bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, |
| 957 | struct uio *uio, |
| 958 | bus_dmamap_callback2_t *callback, void *callback_arg, |
| 959 | int flags) |
| 960 | { |
| 961 | vm_paddr_t lastaddr; |
| 962 | int nsegs, error, first, i; |
| 963 | bus_size_t resid; |
| 964 | struct iovec *iov; |
| 965 | pmap_t pmap; |
| 966 | bus_dma_segment_t cache_segments[BUS_DMA_CACHE_SEGMENTS]; |
| 967 | bus_dma_segment_t *segments; |
| 968 | bus_dma_segment_t *segs; |
| 969 | int nsegs_left; |
| 970 | |
| 971 | if (dmat->nsegments <= BUS_DMA_CACHE_SEGMENTS) |
| 972 | segments = cache_segments; |
| 973 | else |
| 974 | segments = kmalloc(sizeof(bus_dma_segment_t) * dmat->nsegments, |
| 975 | M_DEVBUF, M_WAITOK | M_ZERO); |
| 976 | |
| 977 | /* |
| 978 | * XXX |
| 979 | * Follow old semantics. Once all of the callers are fixed, |
| 980 | * we should get rid of these internal flag "adjustment". |
| 981 | */ |
| 982 | flags &= ~BUS_DMA_WAITOK; |
| 983 | flags |= BUS_DMA_NOWAIT; |
| 984 | |
| 985 | resid = (bus_size_t)uio->uio_resid; |
| 986 | iov = uio->uio_iov; |
| 987 | |
| 988 | segs = segments; |
| 989 | nsegs_left = dmat->nsegments; |
| 990 | |
| 991 | if (uio->uio_segflg == UIO_USERSPACE) { |
| 992 | struct thread *td; |
| 993 | |
| 994 | td = uio->uio_td; |
| 995 | KASSERT(td != NULL && td->td_proc != NULL, |
| 996 | ("bus_dmamap_load_uio: USERSPACE but no proc")); |
| 997 | pmap = vmspace_pmap(td->td_proc->p_vmspace); |
| 998 | } else { |
| 999 | pmap = NULL; |
| 1000 | } |
| 1001 | |
| 1002 | error = 0; |
| 1003 | nsegs = 1; |
| 1004 | first = 1; |
| 1005 | lastaddr = 0; |
| 1006 | for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) { |
| 1007 | /* |
| 1008 | * Now at the first iovec to load. Load each iovec |
| 1009 | * until we have exhausted the residual count. |
| 1010 | */ |
| 1011 | bus_size_t minlen = |
| 1012 | resid < iov[i].iov_len ? resid : iov[i].iov_len; |
| 1013 | caddr_t addr = (caddr_t) iov[i].iov_base; |
| 1014 | |
| 1015 | error = _bus_dmamap_load_buffer(dmat, map, addr, minlen, |
| 1016 | segs, nsegs_left, |
| 1017 | pmap, flags, &lastaddr, &nsegs, first); |
| 1018 | first = 0; |
| 1019 | |
| 1020 | resid -= minlen; |
| 1021 | if (error == 0) { |
| 1022 | nsegs_left -= nsegs; |
| 1023 | segs += nsegs; |
| 1024 | } |
| 1025 | } |
| 1026 | |
| 1027 | /* |
| 1028 | * Minimum one DMA segment, even if 0-length buffer. |
| 1029 | */ |
| 1030 | if (nsegs_left == dmat->nsegments) |
| 1031 | --nsegs_left; |
| 1032 | |
| 1033 | if (error) { |
| 1034 | /* force "no valid mappings" in callback */ |
| 1035 | callback(callback_arg, segments, 0, |
| 1036 | 0, error); |
| 1037 | } else { |
| 1038 | callback(callback_arg, segments, dmat->nsegments - nsegs_left, |
| 1039 | (bus_size_t)uio->uio_resid, error); |
| 1040 | } |
| 1041 | if (dmat->nsegments > BUS_DMA_CACHE_SEGMENTS) |
| 1042 | kfree(segments, M_DEVBUF); |
| 1043 | return error; |
| 1044 | } |
| 1045 | |
| 1046 | /* |
| 1047 | * Release the mapping held by map. |
| 1048 | */ |
| 1049 | void |
| 1050 | _bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) |
| 1051 | { |
| 1052 | struct bounce_page *bpage; |
| 1053 | |
| 1054 | while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { |
| 1055 | STAILQ_REMOVE_HEAD(&map->bpages, links); |
| 1056 | free_bounce_page(dmat, bpage); |
| 1057 | } |
| 1058 | } |
| 1059 | |
| 1060 | void |
| 1061 | _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) |
| 1062 | { |
| 1063 | struct bounce_page *bpage; |
| 1064 | |
| 1065 | if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { |
| 1066 | /* |
| 1067 | * Handle data bouncing. We might also |
| 1068 | * want to add support for invalidating |
| 1069 | * the caches on broken hardware |
| 1070 | */ |
| 1071 | switch (op) { |
| 1072 | case BUS_DMASYNC_PREWRITE: |
| 1073 | while (bpage != NULL) { |
| 1074 | bcopy((void *)bpage->datavaddr, |
| 1075 | (void *)bpage->vaddr, |
| 1076 | bpage->datacount); |
| 1077 | bpage = STAILQ_NEXT(bpage, links); |
| 1078 | } |
| 1079 | dmat->bounce_zone->total_bounced++; |
| 1080 | break; |
| 1081 | |
| 1082 | case BUS_DMASYNC_POSTREAD: |
| 1083 | while (bpage != NULL) { |
| 1084 | bcopy((void *)bpage->vaddr, |
| 1085 | (void *)bpage->datavaddr, |
| 1086 | bpage->datacount); |
| 1087 | bpage = STAILQ_NEXT(bpage, links); |
| 1088 | } |
| 1089 | dmat->bounce_zone->total_bounced++; |
| 1090 | break; |
| 1091 | |
| 1092 | case BUS_DMASYNC_PREREAD: |
| 1093 | case BUS_DMASYNC_POSTWRITE: |
| 1094 | /* No-ops */ |
| 1095 | break; |
| 1096 | } |
| 1097 | } |
| 1098 | } |
| 1099 | |
| 1100 | static int |
| 1101 | alloc_bounce_zone(bus_dma_tag_t dmat) |
| 1102 | { |
| 1103 | struct bounce_zone *bz, *new_bz; |
| 1104 | |
| 1105 | KASSERT(dmat->bounce_zone == NULL, |
| 1106 | ("bounce zone was already assigned")); |
| 1107 | |
| 1108 | new_bz = kmalloc(sizeof(*new_bz), M_DEVBUF, M_INTWAIT | M_ZERO); |
| 1109 | |
| 1110 | lwkt_gettoken(&bounce_zone_tok); |
| 1111 | |
| 1112 | if ((dmat->flags & BUS_DMA_PRIVBZONE) == 0) { |
| 1113 | /* |
| 1114 | * For shared bounce zone, check to see |
| 1115 | * if we already have a suitable zone |
| 1116 | */ |
| 1117 | STAILQ_FOREACH(bz, &bounce_zone_list, links) { |
| 1118 | if (dmat->alignment <= bz->alignment && |
| 1119 | dmat->lowaddr >= bz->lowaddr) { |
| 1120 | lwkt_reltoken(&bounce_zone_tok); |
| 1121 | |
| 1122 | dmat->bounce_zone = bz; |
| 1123 | kfree(new_bz, M_DEVBUF); |
| 1124 | return 0; |
| 1125 | } |
| 1126 | } |
| 1127 | } |
| 1128 | bz = new_bz; |
| 1129 | |
| 1130 | #ifdef SMP |
| 1131 | spin_init(&bz->spin); |
| 1132 | #endif |
| 1133 | STAILQ_INIT(&bz->bounce_page_list); |
| 1134 | STAILQ_INIT(&bz->bounce_map_waitinglist); |
| 1135 | bz->free_bpages = 0; |
| 1136 | bz->reserved_bpages = 0; |
| 1137 | bz->active_bpages = 0; |
| 1138 | bz->lowaddr = dmat->lowaddr; |
| 1139 | bz->alignment = round_page(dmat->alignment); |
| 1140 | ksnprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr); |
| 1141 | |
| 1142 | if ((dmat->flags & BUS_DMA_PRIVBZONE) == 0) { |
| 1143 | ksnprintf(bz->zoneid, 8, "zone%d", busdma_zonecount); |
| 1144 | busdma_zonecount++; |
| 1145 | STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links); |
| 1146 | } else { |
| 1147 | ksnprintf(bz->zoneid, 8, "zone%d", busdma_priv_zonecount); |
| 1148 | busdma_priv_zonecount--; |
| 1149 | } |
| 1150 | |
| 1151 | lwkt_reltoken(&bounce_zone_tok); |
| 1152 | |
| 1153 | dmat->bounce_zone = bz; |
| 1154 | |
| 1155 | sysctl_ctx_init(&bz->sysctl_ctx); |
| 1156 | bz->sysctl_tree = SYSCTL_ADD_NODE(&bz->sysctl_ctx, |
| 1157 | SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid, |
| 1158 | CTLFLAG_RD, 0, ""); |
| 1159 | if (bz->sysctl_tree == NULL) { |
| 1160 | sysctl_ctx_free(&bz->sysctl_ctx); |
| 1161 | return 0; /* XXX error code? */ |
| 1162 | } |
| 1163 | |
| 1164 | SYSCTL_ADD_INT(&bz->sysctl_ctx, |
| 1165 | SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO, |
| 1166 | "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0, |
| 1167 | "Total bounce pages"); |
| 1168 | SYSCTL_ADD_INT(&bz->sysctl_ctx, |
| 1169 | SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO, |
| 1170 | "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0, |
| 1171 | "Free bounce pages"); |
| 1172 | SYSCTL_ADD_INT(&bz->sysctl_ctx, |
| 1173 | SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO, |
| 1174 | "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0, |
| 1175 | "Reserved bounce pages"); |
| 1176 | SYSCTL_ADD_INT(&bz->sysctl_ctx, |
| 1177 | SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO, |
| 1178 | "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0, |
| 1179 | "Active bounce pages"); |
| 1180 | SYSCTL_ADD_INT(&bz->sysctl_ctx, |
| 1181 | SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO, |
| 1182 | "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0, |
| 1183 | "Total bounce requests"); |
| 1184 | SYSCTL_ADD_INT(&bz->sysctl_ctx, |
| 1185 | SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO, |
| 1186 | "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0, |
| 1187 | "Total bounce requests that were deferred"); |
| 1188 | SYSCTL_ADD_INT(&bz->sysctl_ctx, |
| 1189 | SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO, |
| 1190 | "reserve_failed", CTLFLAG_RD, &bz->reserve_failed, 0, |
| 1191 | "Total bounce page reservations that were failed"); |
| 1192 | SYSCTL_ADD_STRING(&bz->sysctl_ctx, |
| 1193 | SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO, |
| 1194 | "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, ""); |
| 1195 | SYSCTL_ADD_INT(&bz->sysctl_ctx, |
| 1196 | SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO, |
| 1197 | "alignment", CTLFLAG_RD, &bz->alignment, 0, ""); |
| 1198 | |
| 1199 | return 0; |
| 1200 | } |
| 1201 | |
| 1202 | static int |
| 1203 | alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages, int flags) |
| 1204 | { |
| 1205 | struct bounce_zone *bz = dmat->bounce_zone; |
| 1206 | int count = 0, mflags; |
| 1207 | |
| 1208 | if (flags & BUS_DMA_NOWAIT) |
| 1209 | mflags = M_NOWAIT; |
| 1210 | else |
| 1211 | mflags = M_WAITOK; |
| 1212 | |
| 1213 | while (numpages > 0) { |
| 1214 | struct bounce_page *bpage; |
| 1215 | |
| 1216 | bpage = kmalloc(sizeof(*bpage), M_DEVBUF, M_INTWAIT | M_ZERO); |
| 1217 | |
| 1218 | bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF, |
| 1219 | mflags, 0ul, |
| 1220 | bz->lowaddr, |
| 1221 | bz->alignment, 0); |
| 1222 | if (bpage->vaddr == 0) { |
| 1223 | kfree(bpage, M_DEVBUF); |
| 1224 | break; |
| 1225 | } |
| 1226 | bpage->busaddr = pmap_kextract(bpage->vaddr); |
| 1227 | |
| 1228 | BZ_LOCK(bz); |
| 1229 | STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links); |
| 1230 | total_bounce_pages++; |
| 1231 | bz->total_bpages++; |
| 1232 | bz->free_bpages++; |
| 1233 | BZ_UNLOCK(bz); |
| 1234 | |
| 1235 | count++; |
| 1236 | numpages--; |
| 1237 | } |
| 1238 | return count; |
| 1239 | } |
| 1240 | |
| 1241 | static void |
| 1242 | free_bounce_pages_all(bus_dma_tag_t dmat) |
| 1243 | { |
| 1244 | struct bounce_zone *bz = dmat->bounce_zone; |
| 1245 | struct bounce_page *bpage; |
| 1246 | |
| 1247 | BZ_LOCK(bz); |
| 1248 | |
| 1249 | while ((bpage = STAILQ_FIRST(&bz->bounce_page_list)) != NULL) { |
| 1250 | STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links); |
| 1251 | |
| 1252 | KKASSERT(total_bounce_pages > 0); |
| 1253 | total_bounce_pages--; |
| 1254 | |
| 1255 | KKASSERT(bz->total_bpages > 0); |
| 1256 | bz->total_bpages--; |
| 1257 | |
| 1258 | KKASSERT(bz->free_bpages > 0); |
| 1259 | bz->free_bpages--; |
| 1260 | |
| 1261 | BZ_UNLOCK(bz); |
| 1262 | contigfree((void *)bpage->vaddr, PAGE_SIZE, M_DEVBUF); |
| 1263 | kfree(bpage, M_DEVBUF); |
| 1264 | BZ_LOCK(bz); |
| 1265 | } |
| 1266 | if (bz->total_bpages) { |
| 1267 | kprintf("#%d bounce pages are still in use\n", |
| 1268 | bz->total_bpages); |
| 1269 | print_backtrace(-1); |
| 1270 | } |
| 1271 | |
| 1272 | BZ_UNLOCK(bz); |
| 1273 | } |
| 1274 | |
| 1275 | static void |
| 1276 | free_bounce_zone(bus_dma_tag_t dmat) |
| 1277 | { |
| 1278 | struct bounce_zone *bz = dmat->bounce_zone; |
| 1279 | |
| 1280 | if (bz == NULL) |
| 1281 | return; |
| 1282 | |
| 1283 | if ((dmat->flags & BUS_DMA_PRIVBZONE) == 0) |
| 1284 | return; |
| 1285 | |
| 1286 | free_bounce_pages_all(dmat); |
| 1287 | dmat->bounce_zone = NULL; |
| 1288 | |
| 1289 | if (bz->sysctl_tree != NULL) |
| 1290 | sysctl_ctx_free(&bz->sysctl_ctx); |
| 1291 | kfree(bz, M_DEVBUF); |
| 1292 | } |
| 1293 | |
| 1294 | /* Assume caller holds bounce zone spinlock */ |
| 1295 | static int |
| 1296 | reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit) |
| 1297 | { |
| 1298 | struct bounce_zone *bz = dmat->bounce_zone; |
| 1299 | int pages; |
| 1300 | |
| 1301 | pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved); |
| 1302 | if (!commit && map->pagesneeded > (map->pagesreserved + pages)) { |
| 1303 | bz->reserve_failed++; |
| 1304 | return (map->pagesneeded - (map->pagesreserved + pages)); |
| 1305 | } |
| 1306 | |
| 1307 | bz->free_bpages -= pages; |
| 1308 | |
| 1309 | bz->reserved_bpages += pages; |
| 1310 | KKASSERT(bz->reserved_bpages <= bz->total_bpages); |
| 1311 | |
| 1312 | map->pagesreserved += pages; |
| 1313 | pages = map->pagesneeded - map->pagesreserved; |
| 1314 | |
| 1315 | return pages; |
| 1316 | } |
| 1317 | |
| 1318 | static void |
| 1319 | return_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map) |
| 1320 | { |
| 1321 | struct bounce_zone *bz = dmat->bounce_zone; |
| 1322 | int reserved = map->pagesreserved; |
| 1323 | bus_dmamap_t wait_map; |
| 1324 | |
| 1325 | map->pagesreserved = 0; |
| 1326 | map->pagesneeded = 0; |
| 1327 | |
| 1328 | if (reserved == 0) |
| 1329 | return; |
| 1330 | |
| 1331 | BZ_LOCK(bz); |
| 1332 | |
| 1333 | bz->free_bpages += reserved; |
| 1334 | KKASSERT(bz->free_bpages <= bz->total_bpages); |
| 1335 | |
| 1336 | KKASSERT(bz->reserved_bpages >= reserved); |
| 1337 | bz->reserved_bpages -= reserved; |
| 1338 | |
| 1339 | wait_map = get_map_waiting(dmat); |
| 1340 | |
| 1341 | BZ_UNLOCK(bz); |
| 1342 | |
| 1343 | if (wait_map != NULL) |
| 1344 | add_map_callback(map); |
| 1345 | } |
| 1346 | |
| 1347 | static bus_addr_t |
| 1348 | add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, |
| 1349 | bus_size_t size) |
| 1350 | { |
| 1351 | struct bounce_zone *bz = dmat->bounce_zone; |
| 1352 | struct bounce_page *bpage; |
| 1353 | |
| 1354 | KASSERT(map->pagesneeded > 0, ("map doesn't need any pages")); |
| 1355 | map->pagesneeded--; |
| 1356 | |
| 1357 | KASSERT(map->pagesreserved > 0, ("map doesn't reserve any pages")); |
| 1358 | map->pagesreserved--; |
| 1359 | |
| 1360 | BZ_LOCK(bz); |
| 1361 | |
| 1362 | bpage = STAILQ_FIRST(&bz->bounce_page_list); |
| 1363 | KASSERT(bpage != NULL, ("free page list is empty")); |
| 1364 | STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links); |
| 1365 | |
| 1366 | KKASSERT(bz->reserved_bpages > 0); |
| 1367 | bz->reserved_bpages--; |
| 1368 | |
| 1369 | bz->active_bpages++; |
| 1370 | KKASSERT(bz->active_bpages <= bz->total_bpages); |
| 1371 | |
| 1372 | BZ_UNLOCK(bz); |
| 1373 | |
| 1374 | if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { |
| 1375 | /* Page offset needs to be preserved. */ |
| 1376 | bpage->vaddr |= vaddr & PAGE_MASK; |
| 1377 | bpage->busaddr |= vaddr & PAGE_MASK; |
| 1378 | } |
| 1379 | bpage->datavaddr = vaddr; |
| 1380 | bpage->datacount = size; |
| 1381 | STAILQ_INSERT_TAIL(&map->bpages, bpage, links); |
| 1382 | return bpage->busaddr; |
| 1383 | } |
| 1384 | |
| 1385 | static void |
| 1386 | free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage) |
| 1387 | { |
| 1388 | struct bounce_zone *bz = dmat->bounce_zone; |
| 1389 | bus_dmamap_t map; |
| 1390 | |
| 1391 | bpage->datavaddr = 0; |
| 1392 | bpage->datacount = 0; |
| 1393 | |
| 1394 | if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { |
| 1395 | /* |
| 1396 | * Reset the bounce page to start at offset 0. Other uses |
| 1397 | * of this bounce page may need to store a full page of |
| 1398 | * data and/or assume it starts on a page boundary. |
| 1399 | */ |
| 1400 | bpage->vaddr &= ~PAGE_MASK; |
| 1401 | bpage->busaddr &= ~PAGE_MASK; |
| 1402 | } |
| 1403 | |
| 1404 | BZ_LOCK(bz); |
| 1405 | |
| 1406 | STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links); |
| 1407 | |
| 1408 | bz->free_bpages++; |
| 1409 | KKASSERT(bz->free_bpages <= bz->total_bpages); |
| 1410 | |
| 1411 | KKASSERT(bz->active_bpages > 0); |
| 1412 | bz->active_bpages--; |
| 1413 | |
| 1414 | map = get_map_waiting(dmat); |
| 1415 | |
| 1416 | BZ_UNLOCK(bz); |
| 1417 | |
| 1418 | if (map != NULL) |
| 1419 | add_map_callback(map); |
| 1420 | } |
| 1421 | |
| 1422 | /* Assume caller holds bounce zone spinlock */ |
| 1423 | static bus_dmamap_t |
| 1424 | get_map_waiting(bus_dma_tag_t dmat) |
| 1425 | { |
| 1426 | struct bounce_zone *bz = dmat->bounce_zone; |
| 1427 | bus_dmamap_t map; |
| 1428 | |
| 1429 | map = STAILQ_FIRST(&bz->bounce_map_waitinglist); |
| 1430 | if (map != NULL) { |
| 1431 | if (reserve_bounce_pages(map->dmat, map, 1) == 0) { |
| 1432 | STAILQ_REMOVE_HEAD(&bz->bounce_map_waitinglist, links); |
| 1433 | bz->total_deferred++; |
| 1434 | } else { |
| 1435 | map = NULL; |
| 1436 | } |
| 1437 | } |
| 1438 | return map; |
| 1439 | } |
| 1440 | |
| 1441 | static void |
| 1442 | add_map_callback(bus_dmamap_t map) |
| 1443 | { |
| 1444 | spin_lock(&bounce_map_list_spin); |
| 1445 | STAILQ_INSERT_TAIL(&bounce_map_callbacklist, map, links); |
| 1446 | busdma_swi_pending = 1; |
| 1447 | setsoftvm(); |
| 1448 | spin_unlock(&bounce_map_list_spin); |
| 1449 | } |
| 1450 | |
| 1451 | void |
| 1452 | busdma_swi(void) |
| 1453 | { |
| 1454 | bus_dmamap_t map; |
| 1455 | |
| 1456 | spin_lock(&bounce_map_list_spin); |
| 1457 | while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) { |
| 1458 | STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links); |
| 1459 | spin_unlock(&bounce_map_list_spin); |
| 1460 | bus_dmamap_load(map->dmat, map, map->buf, map->buflen, |
| 1461 | map->callback, map->callback_arg, /*flags*/0); |
| 1462 | spin_lock(&bounce_map_list_spin); |
| 1463 | } |
| 1464 | spin_unlock(&bounce_map_list_spin); |
| 1465 | } |