kernel - lwkt_token revamp
[dragonfly.git] / sys / platform / vkernel / platform / busdma_machdep.c
1 /*
2  * Copyright (c) 1997, 1998 Justin T. Gibbs.
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions, and the following disclaimer,
10  *    without modification, immediately at the beginning of the file.
11  * 2. The name of the author may not be used to endorse or promote products
12  *    derived from this software without specific prior written permission.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
18  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  *
26  * $FreeBSD: src/sys/i386/i386/busdma_machdep.c,v 1.94 2008/08/15 20:51:31 kmacy Exp $
27  * $DragonFly: src/sys/platform/pc32/i386/busdma_machdep.c,v 1.23 2008/06/05 18:06:32 swildner Exp $
28  */
29
30 #include <sys/param.h>
31 #include <sys/systm.h>
32 #include <sys/malloc.h>
33 #include <sys/mbuf.h>
34 #include <sys/uio.h>
35 #include <sys/bus_dma.h>
36 #include <sys/kernel.h>
37 #include <sys/sysctl.h>
38 #include <sys/lock.h>
39
40 #include <sys/thread2.h>
41 #include <sys/spinlock2.h>
42 #include <sys/mplock2.h>
43
44 #include <vm/vm.h>
45 #include <vm/vm_page.h>
46
47 /* XXX needed for to access pmap to convert per-proc virtual to physical */
48 #include <sys/proc.h>
49 #include <sys/lock.h>
50 #include <vm/vm_map.h>
51
52 #include <machine/md_var.h>
53
54 #define MAX_BPAGES      1024
55
56 struct bounce_zone;
57 struct bus_dmamap;
58
59 struct bus_dma_tag {
60         bus_dma_tag_t   parent;
61         bus_size_t      alignment;
62         bus_size_t      boundary;
63         bus_addr_t      lowaddr;
64         bus_addr_t      highaddr;
65         bus_dma_filter_t *filter;
66         void            *filterarg;
67         bus_size_t      maxsize;
68         u_int           nsegments;
69         bus_size_t      maxsegsz;
70         int             flags;
71         int             ref_count;
72         int             map_count;
73         bus_dma_segment_t *segments;
74         struct bounce_zone *bounce_zone;
75 };
76
77 /*
78  * bus_dma_tag private flags
79  */
80 #define BUS_DMA_BOUNCE_ALIGN    BUS_DMA_BUS2
81 #define BUS_DMA_BOUNCE_LOWADDR  BUS_DMA_BUS3
82 #define BUS_DMA_MIN_ALLOC_COMP  BUS_DMA_BUS4
83
84 #define BUS_DMA_COULD_BOUNCE    (BUS_DMA_BOUNCE_LOWADDR | BUS_DMA_BOUNCE_ALIGN)
85
86 #define BUS_DMAMEM_KMALLOC(dmat) \
87         ((dmat)->maxsize <= PAGE_SIZE && \
88          (dmat)->alignment <= PAGE_SIZE && \
89          (dmat)->lowaddr >= ptoa(Maxmem))
90
91 struct bounce_page {
92         vm_offset_t     vaddr;          /* kva of bounce buffer */
93         bus_addr_t      busaddr;        /* Physical address */
94         vm_offset_t     datavaddr;      /* kva of client data */
95         bus_size_t      datacount;      /* client data count */
96         STAILQ_ENTRY(bounce_page) links;
97 };
98
99 struct bounce_zone {
100         STAILQ_ENTRY(bounce_zone) links;
101         STAILQ_HEAD(bp_list, bounce_page) bounce_page_list;
102         STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist;
103 #ifdef SMP
104         struct spinlock spin;
105 #else
106         int             unused0;
107 #endif
108         int             total_bpages;
109         int             free_bpages;
110         int             reserved_bpages;
111         int             active_bpages;
112         int             total_bounced;
113         int             total_deferred;
114         int             reserve_failed;
115         bus_size_t      alignment;
116         bus_addr_t      lowaddr;
117         char            zoneid[8];
118         char            lowaddrid[20];
119         struct sysctl_ctx_list sysctl_ctx;
120         struct sysctl_oid *sysctl_tree;
121 };
122
123 #ifdef SMP
124 #define BZ_LOCK(bz)     spin_lock_wr(&(bz)->spin)
125 #define BZ_UNLOCK(bz)   spin_unlock_wr(&(bz)->spin)
126 #else
127 #define BZ_LOCK(bz)     crit_enter()
128 #define BZ_UNLOCK(bz)   crit_exit()
129 #endif
130
131 static struct lwkt_token bounce_zone_tok =
132         LWKT_TOKEN_MP_INITIALIZER(bounce_zone_tok);
133 static int busdma_zonecount;
134 static STAILQ_HEAD(, bounce_zone) bounce_zone_list =
135         STAILQ_HEAD_INITIALIZER(bounce_zone_list);
136
137 int busdma_swi_pending;
138 static int total_bounce_pages;
139 static int max_bounce_pages = MAX_BPAGES;
140 static int bounce_alignment = 1; /* XXX temporary */
141
142 TUNABLE_INT("hw.busdma.max_bpages", &max_bounce_pages);
143 TUNABLE_INT("hw.busdma.bounce_alignment", &bounce_alignment);
144
145 struct bus_dmamap {
146         struct bp_list  bpages;
147         int             pagesneeded;
148         int             pagesreserved;
149         bus_dma_tag_t   dmat;
150         void            *buf;           /* unmapped buffer pointer */
151         bus_size_t      buflen;         /* unmapped buffer length */
152         bus_dmamap_callback_t *callback;
153         void            *callback_arg;
154         STAILQ_ENTRY(bus_dmamap) links;
155 };
156
157 static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist =
158         STAILQ_HEAD_INITIALIZER(bounce_map_callbacklist);
159
160 static struct bus_dmamap nobounce_dmamap;
161
162 static int              alloc_bounce_zone(bus_dma_tag_t);
163 static int              alloc_bounce_pages(bus_dma_tag_t, u_int, int);
164 static int              reserve_bounce_pages(bus_dma_tag_t, bus_dmamap_t, int);
165 static void             return_bounce_pages(bus_dma_tag_t, bus_dmamap_t);
166 static bus_addr_t       add_bounce_page(bus_dma_tag_t, bus_dmamap_t,
167                             vm_offset_t, bus_size_t);
168 static void             free_bounce_page(bus_dma_tag_t, struct bounce_page *);
169
170 static bus_dmamap_t     get_map_waiting(bus_dma_tag_t);
171 static void             add_map_callback(bus_dmamap_t);
172
173 SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters");
174 SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bounce_pages,
175            0, "Total bounce pages");
176 SYSCTL_INT(_hw_busdma, OID_AUTO, max_bpages, CTLFLAG_RD, &max_bounce_pages,
177            0, "Max bounce pages per bounce zone");
178 SYSCTL_INT(_hw_busdma, OID_AUTO, bounce_alignment, CTLFLAG_RD,
179            &bounce_alignment, 0, "Obey alignment constraint");
180
181 static __inline int
182 run_filter(bus_dma_tag_t dmat, bus_addr_t paddr)
183 {
184         int retval;
185
186         retval = 0;
187         do {
188                 if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr) ||
189                      (bounce_alignment && (paddr & (dmat->alignment - 1)) != 0))
190                  && (dmat->filter == NULL ||
191                      dmat->filter(dmat->filterarg, paddr) != 0))
192                         retval = 1;
193
194                 dmat = dmat->parent;
195         } while (retval == 0 && dmat != NULL);
196         return (retval);
197 }
198
199 /*
200  * Allocate a device specific dma_tag.
201  */
202 int
203 bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
204                    bus_size_t boundary, bus_addr_t lowaddr,
205                    bus_addr_t highaddr, bus_dma_filter_t *filter,
206                    void *filterarg, bus_size_t maxsize, int nsegments,
207                    bus_size_t maxsegsz, int flags, bus_dma_tag_t *dmat)
208 {
209         bus_dma_tag_t newtag;
210         int error = 0;
211
212         /*
213          * Sanity checks
214          */
215
216         if (alignment == 0)
217                 alignment = 1;
218         if (alignment & (alignment - 1))
219                 panic("alignment must be power of 2\n");
220
221         if (boundary != 0) {
222                 if (boundary & (boundary - 1))
223                         panic("boundary must be power of 2\n");
224                 if (boundary < maxsegsz) {
225                         kprintf("boundary < maxsegsz:\n");
226                         print_backtrace(-1);
227                         maxsegsz = boundary;
228                 }
229         }
230
231         /* Return a NULL tag on failure */
232         *dmat = NULL;
233
234         newtag = kmalloc(sizeof(*newtag), M_DEVBUF, M_INTWAIT);
235
236         newtag->parent = parent;
237         newtag->alignment = alignment;
238         newtag->boundary = boundary;
239         newtag->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1);
240         newtag->highaddr = trunc_page((vm_paddr_t)highaddr) + (PAGE_SIZE - 1);
241         newtag->filter = filter;
242         newtag->filterarg = filterarg;
243         newtag->maxsize = maxsize;
244         newtag->nsegments = nsegments;
245         newtag->maxsegsz = maxsegsz;
246         newtag->flags = flags;
247         newtag->ref_count = 1; /* Count ourself */
248         newtag->map_count = 0;
249         newtag->segments = NULL;
250         newtag->bounce_zone = NULL;
251
252         /* Take into account any restrictions imposed by our parent tag */
253         if (parent != NULL) {
254                 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr);
255                 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr);
256
257                 if (newtag->boundary == 0) {
258                         newtag->boundary = parent->boundary;
259                 } else if (parent->boundary != 0) {
260                         newtag->boundary = MIN(parent->boundary,
261                                                newtag->boundary);
262                 }
263
264 #ifdef notyet
265                 newtag->alignment = MAX(parent->alignment, newtag->alignment);
266 #endif
267
268                 if (newtag->filter == NULL) {
269                         /*
270                          * Short circuit looking at our parent directly
271                          * since we have encapsulated all of its information
272                          */
273                         newtag->filter = parent->filter;
274                         newtag->filterarg = parent->filterarg;
275                         newtag->parent = parent->parent;
276                 }
277                 if (newtag->parent != NULL)
278                         parent->ref_count++;
279         }
280
281         if (newtag->lowaddr < ptoa(Maxmem))
282                 newtag->flags |= BUS_DMA_BOUNCE_LOWADDR;
283         if (bounce_alignment && newtag->alignment > 1 &&
284             !(newtag->flags & BUS_DMA_ALIGNED))
285                 newtag->flags |= BUS_DMA_BOUNCE_ALIGN;
286
287         if ((newtag->flags & BUS_DMA_COULD_BOUNCE) &&
288             (flags & BUS_DMA_ALLOCNOW) != 0) {
289                 struct bounce_zone *bz;
290
291                 /* Must bounce */
292
293                 error = alloc_bounce_zone(newtag);
294                 if (error)
295                         goto back;
296                 bz = newtag->bounce_zone;
297
298                 if (ptoa(bz->total_bpages) < maxsize) {
299                         int pages;
300
301                         if (flags & BUS_DMA_ONEBPAGE) {
302                                 pages = 1;
303                         } else {
304                                 pages = atop(round_page(maxsize)) -
305                                         bz->total_bpages;
306                                 pages = MAX(pages, 1);
307                         }
308
309                         /* Add pages to our bounce pool */
310                         if (alloc_bounce_pages(newtag, pages, flags) < pages)
311                                 error = ENOMEM;
312
313                         /* Performed initial allocation */
314                         newtag->flags |= BUS_DMA_MIN_ALLOC_COMP;
315                 }
316         }
317 back:
318         if (error)
319                 kfree(newtag, M_DEVBUF);
320         else
321                 *dmat = newtag;
322         return error;
323 }
324
325 int
326 bus_dma_tag_destroy(bus_dma_tag_t dmat)
327 {
328         if (dmat != NULL) {
329                 if (dmat->map_count != 0)
330                         return (EBUSY);
331
332                 while (dmat != NULL) {
333                         bus_dma_tag_t parent;
334
335                         parent = dmat->parent;
336                         dmat->ref_count--;
337                         if (dmat->ref_count == 0) {
338                                 if (dmat->segments != NULL)
339                                         kfree(dmat->segments, M_DEVBUF);
340                                 kfree(dmat, M_DEVBUF);
341                                 /*
342                                  * Last reference count, so
343                                  * release our reference
344                                  * count on our parent.
345                                  */
346                                 dmat = parent;
347                         } else
348                                 dmat = NULL;
349                 }
350         }
351         return (0);
352 }
353
354 /*
355  * Allocate a handle for mapping from kva/uva/physical
356  * address space into bus device space.
357  */
358 int
359 bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
360 {
361         int error;
362
363         error = 0;
364
365         if (dmat->segments == NULL) {
366                 KKASSERT(dmat->nsegments && dmat->nsegments < 16384);
367                 dmat->segments = kmalloc(sizeof(bus_dma_segment_t) * 
368                                         dmat->nsegments, M_DEVBUF, M_INTWAIT);
369         }
370
371         if (dmat->flags & BUS_DMA_COULD_BOUNCE) {
372                 struct bounce_zone *bz;
373                 int maxpages;
374
375                 /* Must bounce */
376
377                 if (dmat->bounce_zone == NULL) {
378                         error = alloc_bounce_zone(dmat);
379                         if (error)
380                                 return error;
381                 }
382                 bz = dmat->bounce_zone;
383
384                 *mapp = kmalloc(sizeof(**mapp), M_DEVBUF, M_INTWAIT | M_ZERO);
385
386                 /* Initialize the new map */
387                 STAILQ_INIT(&((*mapp)->bpages));
388
389                 /*
390                  * Attempt to add pages to our pool on a per-instance
391                  * basis up to a sane limit.
392                  */
393                 if (dmat->flags & BUS_DMA_BOUNCE_ALIGN) {
394                         maxpages = max_bounce_pages;
395                 } else {
396                         maxpages = MIN(max_bounce_pages,
397                                        Maxmem - atop(dmat->lowaddr));
398                 }
399                 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0
400                  || (dmat->map_count > 0
401                   && bz->total_bpages < maxpages)) {
402                         int pages;
403
404                         if (flags & BUS_DMA_ONEBPAGE) {
405                                 pages = 1;
406                         } else {
407                                 pages = atop(round_page(dmat->maxsize));
408                                 pages = MIN(maxpages - bz->total_bpages, pages);
409                                 pages = MAX(pages, 1);
410                         }
411                         if (alloc_bounce_pages(dmat, pages, flags) < pages)
412                                 error = ENOMEM;
413
414                         if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) {
415                                 if (!error)
416                                         dmat->flags |= BUS_DMA_MIN_ALLOC_COMP;
417                         } else {
418                                 error = 0;
419                         }
420                 }
421         } else {
422                 *mapp = NULL;
423         }
424         if (!error)
425                 dmat->map_count++;
426         return error;
427 }
428
429 /*
430  * Destroy a handle for mapping from kva/uva/physical
431  * address space into bus device space.
432  */
433 int
434 bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
435 {
436         if (map != NULL) {
437                 if (STAILQ_FIRST(&map->bpages) != NULL)
438                         return (EBUSY);
439                 kfree(map, M_DEVBUF);
440         }
441         dmat->map_count--;
442         return (0);
443 }
444
445 static __inline bus_size_t
446 check_kmalloc(bus_dma_tag_t dmat, const void *vaddr0, int verify)
447 {
448         bus_size_t maxsize = 0;
449         uintptr_t vaddr = (uintptr_t)vaddr0;
450
451         if ((vaddr ^ (vaddr + dmat->maxsize - 1)) & ~PAGE_MASK) {
452                 if (verify || bootverbose)
453                         kprintf("boundary check failed\n");
454                 if (verify)
455                         print_backtrace(-1); /* XXX panic */
456                 maxsize = dmat->maxsize;
457         }
458         if (vaddr & (dmat->alignment - 1)) {
459                 if (verify || bootverbose)
460                         kprintf("alignment check failed\n");
461                 if (verify)
462                         print_backtrace(-1); /* XXX panic */
463                 if (dmat->maxsize < dmat->alignment)
464                         maxsize = dmat->alignment;
465                 else
466                         maxsize = dmat->maxsize;
467         }
468         return maxsize;
469 }
470
471 /*
472  * Allocate a piece of memory that can be efficiently mapped into
473  * bus device space based on the constraints lited in the dma tag.
474  *
475  * mapp is degenerate.  By definition this allocation should not require
476  * bounce buffers so do not allocate a dma map.
477  */
478 int
479 bus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddr, int flags,
480                  bus_dmamap_t *mapp)
481 {
482         int mflags;
483
484         /* If we succeed, no mapping/bouncing will be required */
485         *mapp = NULL;
486
487         if (dmat->segments == NULL) {
488                 KKASSERT(dmat->nsegments < 16384);
489                 dmat->segments = kmalloc(sizeof(bus_dma_segment_t) * 
490                                         dmat->nsegments, M_DEVBUF, M_INTWAIT);
491         }
492
493         if (flags & BUS_DMA_NOWAIT)
494                 mflags = M_NOWAIT;
495         else
496                 mflags = M_WAITOK;
497         if (flags & BUS_DMA_ZERO)
498                 mflags |= M_ZERO;
499
500         if (BUS_DMAMEM_KMALLOC(dmat)) {
501                 bus_size_t maxsize;
502
503                 *vaddr = kmalloc(dmat->maxsize, M_DEVBUF, mflags);
504
505                 /*
506                  * XXX
507                  * Check whether the allocation
508                  * - crossed a page boundary
509                  * - was not aligned
510                  * Retry with power-of-2 alignment in the above cases.
511                  */
512                 maxsize = check_kmalloc(dmat, *vaddr, 0);
513                 if (maxsize) {
514                         size_t size;
515
516                         kfree(*vaddr, M_DEVBUF);
517                         /* XXX check for overflow? */
518                         for (size = 1; size <= maxsize; size <<= 1)
519                                 ;
520                         *vaddr = kmalloc(size, M_DEVBUF, mflags);
521                         check_kmalloc(dmat, *vaddr, 1);
522                 }
523         } else {
524                 /*
525                  * XXX Use Contigmalloc until it is merged into this facility
526                  *     and handles multi-seg allocations.  Nobody is doing
527                  *     multi-seg allocations yet though.
528                  */
529                 *vaddr = contigmalloc(dmat->maxsize, M_DEVBUF, mflags,
530                     0ul, dmat->lowaddr, dmat->alignment, dmat->boundary);
531         }
532         if (*vaddr == NULL)
533                 return (ENOMEM);
534         return (0);
535 }
536
537 /*
538  * Free a piece of memory and it's allociated dmamap, that was allocated
539  * via bus_dmamem_alloc.  Make the same choice for free/contigfree.
540  */
541 void
542 bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map)
543 {
544         /*
545          * dmamem does not need to be bounced, so the map should be
546          * NULL
547          */
548         if (map != NULL)
549                 panic("bus_dmamem_free: Invalid map freed\n");
550         if (BUS_DMAMEM_KMALLOC(dmat))
551                 kfree(vaddr, M_DEVBUF);
552         else
553                 contigfree(vaddr, dmat->maxsize, M_DEVBUF);
554 }
555
556 static __inline vm_paddr_t
557 _bus_dma_extract(pmap_t pmap, vm_offset_t vaddr)
558 {
559         if (pmap)
560                 return pmap_extract(pmap, vaddr);
561         else
562                 return pmap_kextract(vaddr);
563 }
564
565 /*
566  * Utility function to load a linear buffer.  lastaddrp holds state
567  * between invocations (for multiple-buffer loads).  segp contains
568  * the segment following the starting one on entrace, and the ending
569  * segment on exit.  first indicates if this is the first invocation
570  * of this function.
571  */
572 static int
573 _bus_dmamap_load_buffer(bus_dma_tag_t dmat,
574                         bus_dmamap_t map,
575                         void *buf, bus_size_t buflen,
576                         bus_dma_segment_t *segments,
577                         int nsegments,
578                         pmap_t pmap,
579                         int flags,
580                         vm_paddr_t *lastpaddrp,
581                         int *segp,
582                         int first)
583 {
584         vm_offset_t vaddr;
585         vm_paddr_t paddr, nextpaddr;
586         bus_dma_segment_t *sg;
587         bus_addr_t bmask;
588         int seg, error = 0;
589
590         if (map == NULL)
591                 map = &nobounce_dmamap;
592
593 #ifdef INVARIANTS
594         if (dmat->flags & BUS_DMA_ALIGNED)
595                 KKASSERT(((uintptr_t)buf & (dmat->alignment - 1)) == 0);
596 #endif
597
598         /*
599          * If we are being called during a callback, pagesneeded will
600          * be non-zero, so we can avoid doing the work twice.
601          */
602         if ((dmat->flags & BUS_DMA_COULD_BOUNCE) &&
603             map != &nobounce_dmamap && map->pagesneeded == 0) {
604                 vm_offset_t vendaddr;
605
606                 /*
607                  * Count the number of bounce pages
608                  * needed in order to complete this transfer
609                  */
610                 vaddr = (vm_offset_t)buf;
611                 vendaddr = (vm_offset_t)buf + buflen;
612
613                 while (vaddr < vendaddr) {
614                         paddr = _bus_dma_extract(pmap, vaddr);
615                         if (run_filter(dmat, paddr) != 0)
616                                 map->pagesneeded++;
617                         vaddr += (PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK));
618                 }
619         }
620
621         /* Reserve Necessary Bounce Pages */
622         if (map->pagesneeded != 0) {
623                 struct bounce_zone *bz;
624
625                 bz = dmat->bounce_zone;
626                 BZ_LOCK(bz);
627                 if (flags & BUS_DMA_NOWAIT) {
628                         if (reserve_bounce_pages(dmat, map, 0) != 0) {
629                                 BZ_UNLOCK(bz);
630                                 error = ENOMEM;
631                                 goto free_bounce;
632                         }
633                 } else {
634                         if (reserve_bounce_pages(dmat, map, 1) != 0) {
635                                 /* Queue us for resources */
636                                 map->dmat = dmat;
637                                 map->buf = buf;
638                                 map->buflen = buflen;
639
640                                 STAILQ_INSERT_TAIL(
641                                     &dmat->bounce_zone->bounce_map_waitinglist,
642                                     map, links);
643                                 BZ_UNLOCK(bz);
644
645                                 return (EINPROGRESS);
646                         }
647                 }
648                 BZ_UNLOCK(bz);
649         }
650
651         KKASSERT(*segp >= 1 && *segp <= nsegments);
652         seg = *segp;
653         sg = &segments[seg - 1];
654
655         vaddr = (vm_offset_t)buf;
656         nextpaddr = *lastpaddrp;
657         bmask = ~(dmat->boundary - 1);  /* note: will be 0 if boundary is 0 */
658
659         /* force at least one segment */
660         do {
661                 bus_size_t size;
662
663                 /*
664                  * Per-page main loop
665                  */
666                 paddr = _bus_dma_extract(pmap, vaddr);
667                 size = PAGE_SIZE - (paddr & PAGE_MASK);
668                 if (size > buflen)
669                         size = buflen;
670                 if (map->pagesneeded != 0 && run_filter(dmat, paddr)) {
671                         /*
672                          * note: this paddr has the same in-page offset
673                          * as vaddr and thus the paddr above, so the
674                          * size does not have to be recalculated
675                          */
676                         paddr = add_bounce_page(dmat, map, vaddr, size);
677                 }
678
679                 /*
680                  * Fill in the bus_dma_segment
681                  */
682                 if (first) {
683                         sg->ds_addr = paddr;
684                         sg->ds_len = size;
685                         first = 0;
686                 } else if (paddr == nextpaddr) {
687                         sg->ds_len += size;
688                 } else {
689                         sg++;
690                         seg++;
691                         if (seg > nsegments)
692                                 break;
693                         sg->ds_addr = paddr;
694                         sg->ds_len = size;
695                 }
696                 nextpaddr = paddr + size;
697
698                 /*
699                  * Handle maxsegsz and boundary issues with a nested loop
700                  */
701                 for (;;) {
702                         bus_size_t tmpsize;
703
704                         /*
705                          * Limit to the boundary and maximum segment size
706                          */
707                         if (((nextpaddr - 1) ^ sg->ds_addr) & bmask) {
708                                 tmpsize = dmat->boundary -
709                                           (sg->ds_addr & ~bmask);
710                                 if (tmpsize > dmat->maxsegsz)
711                                         tmpsize = dmat->maxsegsz;
712                                 KKASSERT(tmpsize < sg->ds_len);
713                         } else if (sg->ds_len > dmat->maxsegsz) {
714                                 tmpsize = dmat->maxsegsz;
715                         } else {
716                                 break;
717                         }
718
719                         /*
720                          * Futz, split the data into a new segment.
721                          */
722                         if (seg >= nsegments)
723                                 goto fail;
724                         sg[1].ds_len = sg[0].ds_len - tmpsize;
725                         sg[1].ds_addr = sg[0].ds_addr + tmpsize;
726                         sg[0].ds_len = tmpsize;
727                         sg++;
728                         seg++;
729                 }
730
731                 /*
732                  * Adjust for loop
733                  */
734                 buflen -= size;
735                 vaddr += size;
736         } while (buflen > 0);
737 fail:
738         if (buflen != 0)
739                 error = EFBIG;
740
741         *segp = seg;
742         *lastpaddrp = nextpaddr;
743
744 free_bounce:
745         if (error && (dmat->flags & BUS_DMA_COULD_BOUNCE) &&
746             map != &nobounce_dmamap) {
747                 _bus_dmamap_unload(dmat, map);
748                 return_bounce_pages(dmat, map);
749         }
750         return error;
751 }
752
753 /*
754  * Map the buffer buf into bus space using the dmamap map.
755  */
756 int
757 bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
758                 bus_size_t buflen, bus_dmamap_callback_t *callback,
759                 void *callback_arg, int flags)
760 {
761         vm_paddr_t lastaddr = 0;
762         int error, nsegs = 1;
763
764         if (map != NULL) {
765                 /*
766                  * XXX
767                  * Follow old semantics.  Once all of the callers are fixed,
768                  * we should get rid of these internal flag "adjustment".
769                  */
770                 flags &= ~BUS_DMA_NOWAIT;
771                 flags |= BUS_DMA_WAITOK;
772
773                 map->callback = callback;
774                 map->callback_arg = callback_arg;
775         }
776
777         error = _bus_dmamap_load_buffer(dmat, map, buf, buflen,
778                         dmat->segments, dmat->nsegments,
779                         NULL, flags, &lastaddr, &nsegs, 1);
780         if (error == EINPROGRESS)
781                 return error;
782
783         callback(callback_arg, dmat->segments, nsegs, error);
784         return 0;
785 }
786
787 /*
788  * Like _bus_dmamap_load(), but for mbufs.
789  */
790 int
791 bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map,
792                      struct mbuf *m0,
793                      bus_dmamap_callback2_t *callback, void *callback_arg,
794                      int flags)
795 {
796         int nsegs, error;
797
798         /*
799          * XXX
800          * Follow old semantics.  Once all of the callers are fixed,
801          * we should get rid of these internal flag "adjustment".
802          */
803         flags &= ~BUS_DMA_WAITOK;
804         flags |= BUS_DMA_NOWAIT;
805
806         error = bus_dmamap_load_mbuf_segment(dmat, map, m0,
807                         dmat->segments, dmat->nsegments, &nsegs, flags);
808         if (error) {
809                 /* force "no valid mappings" in callback */
810                 callback(callback_arg, dmat->segments, 0, 0, error);
811         } else {
812                 callback(callback_arg, dmat->segments, nsegs,
813                          m0->m_pkthdr.len, error);
814         }
815         return error;
816 }
817
818 int
819 bus_dmamap_load_mbuf_segment(bus_dma_tag_t dmat, bus_dmamap_t map,
820                              struct mbuf *m0,
821                              bus_dma_segment_t *segs, int maxsegs,
822                              int *nsegs, int flags)
823 {
824         int error;
825
826         M_ASSERTPKTHDR(m0);
827
828         KASSERT(maxsegs >= 1, ("invalid maxsegs %d\n", maxsegs));
829         KASSERT(maxsegs <= dmat->nsegments,
830                 ("%d too many segments, dmat only support %d segments\n",
831                  maxsegs, dmat->nsegments));
832         KASSERT(flags & BUS_DMA_NOWAIT,
833                 ("only BUS_DMA_NOWAIT is supported\n"));
834
835         if (m0->m_pkthdr.len <= dmat->maxsize) {
836                 int first = 1;
837                 vm_paddr_t lastaddr = 0;
838                 struct mbuf *m;
839
840                 *nsegs = 1;
841                 error = 0;
842                 for (m = m0; m != NULL && error == 0; m = m->m_next) {
843                         if (m->m_len == 0)
844                                 continue;
845
846                         error = _bus_dmamap_load_buffer(dmat, map,
847                                         m->m_data, m->m_len,
848                                         segs, maxsegs,
849                                         NULL, flags, &lastaddr,
850                                         nsegs, first);
851                         if (error == ENOMEM && !first) {
852                                 /*
853                                  * Out of bounce pages due to too many
854                                  * fragments in the mbuf chain; return
855                                  * EFBIG instead.
856                                  */
857                                 error = EFBIG;
858                         }
859                         first = 0;
860                 }
861 #ifdef INVARIANTS
862                 if (!error)
863                         KKASSERT(*nsegs <= maxsegs && *nsegs >= 1);
864 #endif
865         } else {
866                 *nsegs = 0;
867                 error = EINVAL;
868         }
869         KKASSERT(error != EINPROGRESS);
870         return error;
871 }
872
873 /*
874  * Like _bus_dmamap_load(), but for uios.
875  */
876 int
877 bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map,
878                     struct uio *uio,
879                     bus_dmamap_callback2_t *callback, void *callback_arg,
880                     int flags)
881 {
882         vm_paddr_t lastaddr;
883         int nsegs, error, first, i;
884         bus_size_t resid;
885         struct iovec *iov;
886         pmap_t pmap;
887
888         /*
889          * XXX
890          * Follow old semantics.  Once all of the callers are fixed,
891          * we should get rid of these internal flag "adjustment".
892          */
893         flags &= ~BUS_DMA_WAITOK;
894         flags |= BUS_DMA_NOWAIT;
895
896         resid = (bus_size_t)uio->uio_resid;
897         iov = uio->uio_iov;
898
899         if (uio->uio_segflg == UIO_USERSPACE) {
900                 struct thread *td;
901
902                 td = uio->uio_td;
903                 KASSERT(td != NULL && td->td_proc != NULL,
904                         ("bus_dmamap_load_uio: USERSPACE but no proc"));
905                 pmap = vmspace_pmap(td->td_proc->p_vmspace);
906         } else {
907                 pmap = NULL;
908         }
909
910         error = 0;
911         nsegs = 1;
912         first = 1;
913         lastaddr = 0;
914         for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) {
915                 /*
916                  * Now at the first iovec to load.  Load each iovec
917                  * until we have exhausted the residual count.
918                  */
919                 bus_size_t minlen =
920                         resid < iov[i].iov_len ? resid : iov[i].iov_len;
921                 caddr_t addr = (caddr_t) iov[i].iov_base;
922
923                 error = _bus_dmamap_load_buffer(dmat, map, addr, minlen,
924                                 dmat->segments, dmat->nsegments,
925                                 pmap, flags, &lastaddr, &nsegs, first);
926                 first = 0;
927
928                 resid -= minlen;
929         }
930
931         if (error) {
932                 /* force "no valid mappings" in callback */
933                 callback(callback_arg, dmat->segments, 0, 0, error);
934         } else {
935                 callback(callback_arg, dmat->segments, nsegs,
936                          (bus_size_t)uio->uio_resid, error);
937         }
938         return error;
939 }
940
941 /*
942  * Release the mapping held by map.
943  */
944 void
945 _bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map)
946 {
947         struct bounce_page *bpage;
948
949         while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
950                 STAILQ_REMOVE_HEAD(&map->bpages, links);
951                 free_bounce_page(dmat, bpage);
952         }
953 }
954
955 void
956 _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
957 {
958         struct bounce_page *bpage;
959
960         if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
961                 /*
962                  * Handle data bouncing.  We might also
963                  * want to add support for invalidating
964                  * the caches on broken hardware
965                  */
966                 switch (op) {
967                 case BUS_DMASYNC_PREWRITE:
968                         while (bpage != NULL) {
969                                 bcopy((void *)bpage->datavaddr,
970                                       (void *)bpage->vaddr,
971                                       bpage->datacount);
972                                 bpage = STAILQ_NEXT(bpage, links);
973                         }
974                         dmat->bounce_zone->total_bounced++;
975                         break;
976
977                 case BUS_DMASYNC_POSTREAD:
978                         while (bpage != NULL) {
979                                 bcopy((void *)bpage->vaddr,
980                                       (void *)bpage->datavaddr,
981                                       bpage->datacount);
982                                 bpage = STAILQ_NEXT(bpage, links);
983                         }
984                         dmat->bounce_zone->total_bounced++;
985                         break;
986
987                 case BUS_DMASYNC_PREREAD:
988                 case BUS_DMASYNC_POSTWRITE:
989                         /* No-ops */
990                         break;
991                 }
992         }
993 }
994
995 static int
996 alloc_bounce_zone(bus_dma_tag_t dmat)
997 {
998         struct bounce_zone *bz, *new_bz;
999
1000         KASSERT(dmat->bounce_zone == NULL,
1001                 ("bounce zone was already assigned\n"));
1002
1003         new_bz = kmalloc(sizeof(*new_bz), M_DEVBUF, M_INTWAIT | M_ZERO);
1004
1005         lwkt_gettoken(&bounce_zone_tok);
1006
1007         /* Check to see if we already have a suitable zone */
1008         STAILQ_FOREACH(bz, &bounce_zone_list, links) {
1009                 if (dmat->alignment <= bz->alignment &&
1010                     dmat->lowaddr >= bz->lowaddr) {
1011                         lwkt_reltoken(&bounce_zone_tok);
1012
1013                         dmat->bounce_zone = bz;
1014                         kfree(new_bz, M_DEVBUF);
1015                         return 0;
1016                 }
1017         }
1018         bz = new_bz;
1019
1020 #ifdef SMP
1021         spin_init(&bz->spin);
1022 #endif
1023         STAILQ_INIT(&bz->bounce_page_list);
1024         STAILQ_INIT(&bz->bounce_map_waitinglist);
1025         bz->free_bpages = 0;
1026         bz->reserved_bpages = 0;
1027         bz->active_bpages = 0;
1028         bz->lowaddr = dmat->lowaddr;
1029         bz->alignment = round_page(dmat->alignment);
1030         ksnprintf(bz->zoneid, 8, "zone%d", busdma_zonecount);
1031         busdma_zonecount++;
1032         ksnprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr);
1033         STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links);
1034
1035         lwkt_reltoken(&bounce_zone_tok);
1036
1037         dmat->bounce_zone = bz;
1038
1039         sysctl_ctx_init(&bz->sysctl_ctx);
1040         bz->sysctl_tree = SYSCTL_ADD_NODE(&bz->sysctl_ctx,
1041             SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid,
1042             CTLFLAG_RD, 0, "");
1043         if (bz->sysctl_tree == NULL) {
1044                 sysctl_ctx_free(&bz->sysctl_ctx);
1045                 return 0;       /* XXX error code? */
1046         }
1047
1048         SYSCTL_ADD_INT(&bz->sysctl_ctx,
1049             SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO,
1050             "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0,
1051             "Total bounce pages");
1052         SYSCTL_ADD_INT(&bz->sysctl_ctx,
1053             SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO,
1054             "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0,
1055             "Free bounce pages");
1056         SYSCTL_ADD_INT(&bz->sysctl_ctx,
1057             SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO,
1058             "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0,
1059             "Reserved bounce pages");
1060         SYSCTL_ADD_INT(&bz->sysctl_ctx,
1061             SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO,
1062             "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0,
1063             "Active bounce pages");
1064         SYSCTL_ADD_INT(&bz->sysctl_ctx,
1065             SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO,
1066             "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0,
1067             "Total bounce requests");
1068         SYSCTL_ADD_INT(&bz->sysctl_ctx,
1069             SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO,
1070             "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0,
1071             "Total bounce requests that were deferred");
1072         SYSCTL_ADD_INT(&bz->sysctl_ctx,
1073             SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO,
1074             "reserve_failed", CTLFLAG_RD, &bz->reserve_failed, 0,
1075             "Total bounce page reservations that were failed");
1076         SYSCTL_ADD_STRING(&bz->sysctl_ctx,
1077             SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO,
1078             "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, "");
1079         SYSCTL_ADD_INT(&bz->sysctl_ctx,
1080             SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO,
1081             "alignment", CTLFLAG_RD, &bz->alignment, 0, "");
1082
1083         return 0;
1084 }
1085
1086 static int
1087 alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages, int flags)
1088 {
1089         struct bounce_zone *bz = dmat->bounce_zone;
1090         int count = 0, mflags;
1091
1092         if (flags & BUS_DMA_NOWAIT)
1093                 mflags = M_NOWAIT;
1094         else
1095                 mflags = M_WAITOK;
1096
1097         while (numpages > 0) {
1098                 struct bounce_page *bpage;
1099
1100                 bpage = kmalloc(sizeof(*bpage), M_DEVBUF, M_INTWAIT | M_ZERO);
1101
1102                 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF,
1103                                                          mflags, 0ul,
1104                                                          bz->lowaddr,
1105                                                          bz->alignment, 0);
1106                 if (bpage->vaddr == 0) {
1107                         kfree(bpage, M_DEVBUF);
1108                         break;
1109                 }
1110                 bpage->busaddr = pmap_kextract(bpage->vaddr);
1111
1112                 BZ_LOCK(bz);
1113                 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links);
1114                 total_bounce_pages++;
1115                 bz->total_bpages++;
1116                 bz->free_bpages++;
1117                 BZ_UNLOCK(bz);
1118
1119                 count++;
1120                 numpages--;
1121         }
1122         return count;
1123 }
1124
1125 /* Assume caller holds bounce zone spinlock */
1126 static int
1127 reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit)
1128 {
1129         struct bounce_zone *bz = dmat->bounce_zone;
1130         int pages;
1131
1132         pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved);
1133         if (!commit && map->pagesneeded > (map->pagesreserved + pages)) {
1134                 bz->reserve_failed++;
1135                 return (map->pagesneeded - (map->pagesreserved + pages));
1136         }
1137
1138         bz->free_bpages -= pages;
1139
1140         bz->reserved_bpages += pages;
1141         KKASSERT(bz->reserved_bpages <= bz->total_bpages);
1142
1143         map->pagesreserved += pages;
1144         pages = map->pagesneeded - map->pagesreserved;
1145
1146         return pages;
1147 }
1148
1149 static void
1150 return_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map)
1151 {
1152         struct bounce_zone *bz = dmat->bounce_zone;
1153         int reserved = map->pagesreserved;
1154         bus_dmamap_t wait_map;
1155
1156         map->pagesreserved = 0;
1157         map->pagesneeded = 0;
1158
1159         if (reserved == 0)
1160                 return;
1161
1162         BZ_LOCK(bz);
1163
1164         bz->free_bpages += reserved;
1165         KKASSERT(bz->free_bpages <= bz->total_bpages);
1166
1167         KKASSERT(bz->reserved_bpages >= reserved);
1168         bz->reserved_bpages -= reserved;
1169
1170         wait_map = get_map_waiting(dmat);
1171
1172         BZ_UNLOCK(bz);
1173
1174         if (wait_map != NULL)
1175                 add_map_callback(map);
1176 }
1177
1178 static bus_addr_t
1179 add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr,
1180                 bus_size_t size)
1181 {
1182         struct bounce_zone *bz = dmat->bounce_zone;
1183         struct bounce_page *bpage;
1184
1185         KASSERT(map->pagesneeded > 0, ("map doesn't need any pages"));
1186         map->pagesneeded--;
1187
1188         KASSERT(map->pagesreserved > 0, ("map doesn't reserve any pages"));
1189         map->pagesreserved--;
1190
1191         BZ_LOCK(bz);
1192
1193         bpage = STAILQ_FIRST(&bz->bounce_page_list);
1194         KASSERT(bpage != NULL, ("free page list is empty"));
1195         STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links);
1196
1197         KKASSERT(bz->reserved_bpages > 0);
1198         bz->reserved_bpages--;
1199
1200         bz->active_bpages++;
1201         KKASSERT(bz->active_bpages <= bz->total_bpages);
1202
1203         BZ_UNLOCK(bz);
1204
1205         bpage->datavaddr = vaddr;
1206         bpage->datacount = size;
1207         STAILQ_INSERT_TAIL(&map->bpages, bpage, links);
1208         return bpage->busaddr;
1209 }
1210
1211 static void
1212 free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage)
1213 {
1214         struct bounce_zone *bz = dmat->bounce_zone;
1215         bus_dmamap_t map;
1216
1217         bpage->datavaddr = 0;
1218         bpage->datacount = 0;
1219
1220         BZ_LOCK(bz);
1221
1222         STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links);
1223
1224         bz->free_bpages++;
1225         KKASSERT(bz->free_bpages <= bz->total_bpages);
1226
1227         KKASSERT(bz->active_bpages > 0);
1228         bz->active_bpages--;
1229
1230         map = get_map_waiting(dmat);
1231
1232         BZ_UNLOCK(bz);
1233
1234         if (map != NULL)
1235                 add_map_callback(map);
1236 }
1237
1238 /* Assume caller holds bounce zone spinlock */
1239 static bus_dmamap_t
1240 get_map_waiting(bus_dma_tag_t dmat)
1241 {
1242         struct bounce_zone *bz = dmat->bounce_zone;
1243         bus_dmamap_t map;
1244
1245         map = STAILQ_FIRST(&bz->bounce_map_waitinglist);
1246         if (map != NULL) {
1247                 if (reserve_bounce_pages(map->dmat, map, 1) == 0) {
1248                         STAILQ_REMOVE_HEAD(&bz->bounce_map_waitinglist, links);
1249                         bz->total_deferred++;
1250                 } else {
1251                         map = NULL;
1252                 }
1253         }
1254         return map;
1255 }
1256
1257 static void
1258 add_map_callback(bus_dmamap_t map)
1259 {
1260 #ifdef notyet
1261         /* XXX callbacklist is not MPSAFE */
1262         crit_enter();
1263         get_mplock();
1264         STAILQ_INSERT_TAIL(&bounce_map_callbacklist, map, links);
1265         busdma_swi_pending = 1;
1266         setsoftvm();
1267         rel_mplock();
1268         crit_exit();
1269 #else
1270         panic("%s uncoded\n", __func__);
1271 #endif
1272 }
1273
1274 #ifdef notyet
1275 void
1276 busdma_swi(void)
1277 {
1278         bus_dmamap_t map;
1279
1280         crit_enter();
1281         while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) {
1282                 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links);
1283                 crit_exit();
1284                 bus_dmamap_load(map->dmat, map, map->buf, map->buflen,
1285                                 map->callback, map->callback_arg, /*flags*/0);
1286                 crit_enter();
1287         }
1288         crit_exit();
1289 }
1290 #endif