4 * Copyright (c) 1997, 1998 John S. Dyson
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice immediately at the beginning of the file, without modification,
12 * this list of conditions, and the following disclaimer.
13 * 2. Absolutely no warranty of function or purpose is made by the author
16 * $FreeBSD: src/sys/vm/vm_zone.c,v 1.30.2.6 2002/10/10 19:50:16 dillon Exp $
17 * $DragonFly: src/sys/vm/vm_zone.c,v 1.28 2008/01/23 17:35:48 nth Exp $
20 #include <sys/param.h>
21 #include <sys/queue.h>
22 #include <sys/systm.h>
23 #include <sys/kernel.h>
25 #include <sys/malloc.h>
26 #include <sys/sysctl.h>
27 #include <sys/vmmeter.h>
30 #include <vm/vm_object.h>
31 #include <vm/vm_page.h>
32 #include <vm/vm_map.h>
33 #include <vm/vm_kern.h>
34 #include <vm/vm_extern.h>
35 #include <vm/vm_zone.h>
37 #include <sys/spinlock2.h>
38 #include <sys/mplock2.h>
40 static MALLOC_DEFINE(M_ZONE, "ZONE", "Zone header");
42 #define ZONE_ERROR_INVALID 0
43 #define ZONE_ERROR_NOTFREE 1
44 #define ZONE_ERROR_ALREADYFREE 2
46 #define ZONE_ROUNDING 32
48 #define ZENTRY_FREE 0x12342378
50 static void *zget(vm_zone_t z);
53 * Return an item from the specified zone. This function is non-blocking for
54 * ZONE_INTERRUPT zones.
65 zerror(ZONE_ERROR_INVALID);
68 if (z->zfreecnt > z->zfreemin) {
71 KASSERT(item != NULL, ("zitems unexpectedly NULL"));
72 if (((void **) item)[1] != (void *) ZENTRY_FREE)
73 zerror(ZONE_ERROR_NOTFREE);
74 ((void **) item)[1] = 0;
76 z->zitems = ((void **) item)[0];
79 spin_unlock(&z->zlock);
81 spin_unlock(&z->zlock);
84 * PANICFAIL allows the caller to assume that the zalloc()
85 * will always succeed. If it doesn't, we panic here.
87 if (item == NULL && (z->zflags & ZONE_PANICFAIL))
88 panic("zalloc(%s) failed", z->zname);
94 * Free an item to the specified zone.
99 zfree(vm_zone_t z, void *item)
101 spin_lock(&z->zlock);
102 ((void **) item)[0] = z->zitems;
104 if (((void **) item)[1] == (void *) ZENTRY_FREE)
105 zerror(ZONE_ERROR_ALREADYFREE);
106 ((void **) item)[1] = (void *) ZENTRY_FREE;
110 spin_unlock(&z->zlock);
114 * This file comprises a very simple zone allocator. This is used
115 * in lieu of the malloc allocator, where needed or more optimal.
117 * Note that the initial implementation of this had coloring, and
118 * absolutely no improvement (actually perf degradation) occurred.
120 * Note also that the zones are type stable. The only restriction is
121 * that the first two longwords of a data structure can be changed
122 * between allocations. Any data that must be stable between allocations
123 * must reside in areas after the first two longwords.
125 * zinitna, zinit, zbootinit are the initialization routines.
126 * zalloc, zfree, are the allocation/free routines.
129 LIST_HEAD(zlist, vm_zone) zlist = LIST_HEAD_INITIALIZER(zlist);
130 static int sysctl_vm_zone(SYSCTL_HANDLER_ARGS);
131 static int zone_kmem_pages, zone_kern_pages, zone_kmem_kvaspace;
134 * Create a zone, but don't allocate the zone structure. If the
135 * zone had been previously created by the zone boot code, initialize
136 * various parts of the zone code.
138 * If waits are not allowed during allocation (e.g. during interrupt
139 * code), a-priori allocate the kernel virtual space, and allocate
140 * only pages when needed.
143 * z pointer to zone structure.
144 * obj pointer to VM object (opt).
146 * size size of zone entries.
147 * nentries number of zone entries allocated (only ZONE_INTERRUPT.)
148 * flags ZONE_INTERRUPT -- items can be allocated at interrupt time.
149 * zalloc number of pages allocated when memory is needed.
151 * Note that when using ZONE_INTERRUPT, the size of the zone is limited
152 * by the nentries argument. The size of the memory allocatable is
153 * unlimited if ZONE_INTERRUPT is not set.
158 zinitna(vm_zone_t z, vm_object_t obj, char *name, int size,
159 int nentries, int flags, int zalloc)
164 * Only zones created with zinit() are destroyable.
166 if (z->zflags & ZONE_DESTROYABLE)
167 panic("zinitna: can't create destroyable zone");
170 * NOTE: We can only adjust zsize if we previously did not
173 if ((z->zflags & ZONE_BOOT) == 0) {
174 z->zsize = (size + ZONE_ROUNDING - 1) & ~(ZONE_ROUNDING - 1);
175 spin_init(&z->zlock);
183 lwkt_gettoken(&vm_token);
184 LIST_INSERT_HEAD(&zlist, z, zlink);
185 lwkt_reltoken(&vm_token);
189 z->zkmcur = z->zkmmax = 0;
193 * If we cannot wait, allocate KVA space up front, and we will fill
194 * in pages as needed. This is particularly required when creating
195 * an allocation space for map entries in kernel_map, because we
196 * do not want to go into a recursion deadlock with
197 * vm_map_entry_reserve().
199 if (z->zflags & ZONE_INTERRUPT) {
200 totsize = round_page(z->zsize * nentries);
201 zone_kmem_kvaspace += totsize;
203 z->zkva = kmem_alloc_pageable(&kernel_map, totsize);
205 LIST_REMOVE(z, zlink);
209 z->zpagemax = totsize / PAGE_SIZE;
211 z->zobj = vm_object_allocate(OBJT_DEFAULT, z->zpagemax);
214 _vm_object_allocate(OBJT_DEFAULT, z->zpagemax, obj);
216 z->zallocflag = VM_ALLOC_SYSTEM | VM_ALLOC_INTERRUPT;
219 z->zallocflag = VM_ALLOC_NORMAL | VM_ALLOC_SYSTEM;
224 if (z->zsize > PAGE_SIZE)
227 z->zfreemin = PAGE_SIZE / z->zsize;
236 * Populate the interrrupt zone at creation time rather than
237 * on first allocation, as this is a potentially long operation.
239 if (z->zflags & ZONE_INTERRUPT) {
250 * Subroutine same as zinitna, except zone data structure is allocated
251 * automatically by malloc. This routine should normally be used, except
252 * in certain tricky startup conditions in the VM system -- then
253 * zbootinit and zinitna can be used. Zinit is the standard zone
254 * initialization call.
259 zinit(char *name, int size, int nentries, int flags, int zalloc)
263 z = (vm_zone_t) kmalloc(sizeof (struct vm_zone), M_ZONE, M_NOWAIT);
268 if (zinitna(z, NULL, name, size, nentries,
269 flags & ~ZONE_DESTROYABLE, zalloc) == 0) {
274 if (flags & ZONE_DESTROYABLE)
275 z->zflags |= ZONE_DESTROYABLE;
281 * Initialize a zone before the system is fully up. This routine should
282 * only be called before full VM startup.
284 * Called from the low level boot code only.
287 zbootinit(vm_zone_t z, char *name, int size, void *item, int nitems)
295 z->zflags = ZONE_BOOT;
301 spin_init(&z->zlock);
303 bzero(item, nitems * z->zsize);
305 for (i = 0; i < nitems; i++) {
306 ((void **) item)[0] = z->zitems;
308 ((void **) item)[1] = (void *) ZENTRY_FREE;
311 item = (uint8_t *)item + z->zsize;
313 z->zfreecnt = nitems;
317 lwkt_gettoken(&vm_token);
318 LIST_INSERT_HEAD(&zlist, z, zlink);
319 lwkt_reltoken(&vm_token);
323 * Release all resources owned by zone created with zinit().
328 zdestroy(vm_zone_t z)
333 panic("zdestroy: null zone");
334 if ((z->zflags & ZONE_DESTROYABLE) == 0)
335 panic("zdestroy: undestroyable zone");
337 lwkt_gettoken(&vm_token);
338 LIST_REMOVE(z, zlink);
339 lwkt_reltoken(&vm_token);
342 * Release virtual mappings, physical memory and update sysctl stats.
344 if (z->zflags & ZONE_INTERRUPT) {
346 * Pages mapped via pmap_kenter() must be removed from the
347 * kernel_pmap() before calling kmem_free() to avoid issues
348 * with kernel_pmap.pm_stats.resident_count.
350 pmap_qremove(z->zkva, z->zpagemax);
355 kmem_free(&kernel_map, z->zkva, z->zpagemax*PAGE_SIZE);
356 atomic_subtract_int(&zone_kmem_kvaspace, z->zpagemax*PAGE_SIZE);
359 * Free the backing object and physical pages.
361 vm_object_deallocate(z->zobj);
362 atomic_subtract_int(&zone_kmem_pages, z->zpagecount);
364 for (i=0; i < z->zkmcur; i++) {
365 kmem_free(&kernel_map, z->zkmvec[i],
366 z->zalloc*PAGE_SIZE);
367 atomic_subtract_int(&zone_kern_pages, z->zalloc);
369 if (z->zkmvec != NULL)
370 kfree(z->zkmvec, M_ZONE);
373 spin_uninit(&z->zlock);
379 * void *zalloc(vm_zone_t zone) --
380 * Returns an item from a specified zone. May not be called from a
381 * FAST interrupt or IPI function.
383 * void zfree(vm_zone_t zone, void *item) --
384 * Frees an item back to a specified zone. May not be called from a
385 * FAST interrupt or IPI function.
389 * Internal zone routine. Not to be called from external (non vm_zone) code.
403 panic("zget: null zone");
405 if (z->zflags & ZONE_INTERRUPT) {
407 * Interrupt zones do not mess with the kernel_map, they
408 * simply populate an existing mapping.
410 lwkt_gettoken(&vm_token);
411 savezpc = z->zpagecount;
412 nbytes = z->zpagecount * PAGE_SIZE;
413 nbytes -= nbytes % z->zsize;
414 item = (char *) z->zkva + nbytes;
415 for (i = 0; ((i < z->zalloc) && (z->zpagecount < z->zpagemax));
419 m = vm_page_alloc(z->zobj, z->zpagecount,
421 /* note: z might be modified due to blocking */
426 * Unbusy page so it can freed in zdestroy(). Make
427 * sure it is not on any queue and so can not be
428 * recycled under our feet.
430 KKASSERT(m->queue == PQ_NONE);
431 vm_page_flag_clear(m, PG_BUSY);
433 zkva = z->zkva + z->zpagecount * PAGE_SIZE;
434 pmap_kenter(zkva, VM_PAGE_TO_PHYS(m)); /* YYY */
435 bzero((void *)zkva, PAGE_SIZE);
436 KKASSERT(savezpc == z->zpagecount);
440 vmstats.v_wire_count++;
442 nitems = ((z->zpagecount * PAGE_SIZE) - nbytes) / z->zsize;
443 lwkt_reltoken(&vm_token);
444 } else if (z->zflags & ZONE_SPECIAL) {
446 * The special zone is the one used for vm_map_entry_t's.
447 * We have to avoid an infinite recursion in
448 * vm_map_entry_reserve() by using vm_map_entry_kreserve()
449 * instead. The map entries are pre-reserved by the kernel
450 * by vm_map_entry_reserve_cpu_init().
452 nbytes = z->zalloc * PAGE_SIZE;
454 item = (void *)kmem_alloc3(&kernel_map, nbytes, KM_KRESERVE);
456 /* note: z might be modified due to blocking */
458 zone_kern_pages += z->zalloc; /* not MP-safe XXX */
463 nitems = nbytes / z->zsize;
466 * Otherwise allocate KVA from the kernel_map.
468 nbytes = z->zalloc * PAGE_SIZE;
470 item = (void *)kmem_alloc3(&kernel_map, nbytes, 0);
472 /* note: z might be modified due to blocking */
474 zone_kern_pages += z->zalloc; /* not MP-safe XXX */
477 if (z->zflags & ZONE_DESTROYABLE) {
478 if (z->zkmcur == z->zkmmax) {
480 z->zkmmax==0 ? 1 : z->zkmmax*2;
481 z->zkmvec = krealloc(z->zkmvec,
482 z->zkmmax * sizeof(z->zkmvec[0]),
485 z->zkmvec[z->zkmcur++] = (vm_offset_t)item;
490 nitems = nbytes / z->zsize;
493 spin_lock(&z->zlock);
496 * Save one for immediate allocation
500 for (i = 0; i < nitems; i++) {
501 ((void **) item)[0] = z->zitems;
503 ((void **) item)[1] = (void *) ZENTRY_FREE;
506 item = (uint8_t *)item + z->zsize;
508 z->zfreecnt += nitems;
510 } else if (z->zfreecnt > 0) {
512 z->zitems = ((void **) item)[0];
514 if (((void **) item)[1] != (void *) ZENTRY_FREE)
515 zerror(ZONE_ERROR_NOTFREE);
516 ((void **) item)[1] = 0;
523 spin_unlock(&z->zlock);
526 * A special zone may have used a kernel-reserved vm_map_entry. If
527 * so we have to be sure to recover our reserve so we don't run out.
528 * We will panic if we run out.
530 if (z->zflags & ZONE_SPECIAL)
531 vm_map_entry_reserve(0);
540 sysctl_vm_zone(SYSCTL_HANDLER_ARGS)
547 ksnprintf(tmpbuf, sizeof(tmpbuf),
548 "\nITEM SIZE LIMIT USED FREE REQUESTS\n");
549 error = SYSCTL_OUT(req, tmpbuf, strlen(tmpbuf));
553 lwkt_gettoken(&vm_token);
554 LIST_FOREACH(curzone, &zlist, zlink) {
559 len = strlen(curzone->zname);
560 if (len >= (sizeof(tmpname) - 1))
561 len = (sizeof(tmpname) - 1);
562 for(i = 0; i < sizeof(tmpname) - 1; i++)
565 memcpy(tmpname, curzone->zname, len);
568 if (curzone == LIST_FIRST(&zlist)) {
573 ksnprintf(tmpbuf + offset, sizeof(tmpbuf) - offset,
574 "%s %6.6u, %8.8u, %6.6u, %6.6u, %8.8u\n",
575 tmpname, curzone->zsize, curzone->zmax,
576 (curzone->ztotal - curzone->zfreecnt),
577 curzone->zfreecnt, curzone->znalloc);
579 len = strlen((char *)tmpbuf);
580 if (LIST_NEXT(curzone, zlink) == NULL)
583 error = SYSCTL_OUT(req, tmpbuf, len);
588 lwkt_reltoken(&vm_token);
592 #if defined(INVARIANTS)
603 case ZONE_ERROR_INVALID:
604 msg = "zone: invalid zone";
606 case ZONE_ERROR_NOTFREE:
607 msg = "zone: entry not free";
609 case ZONE_ERROR_ALREADYFREE:
610 msg = "zone: freeing free entry";
613 msg = "zone: invalid error";
620 SYSCTL_OID(_vm, OID_AUTO, zone, CTLTYPE_STRING|CTLFLAG_RD, \
621 NULL, 0, sysctl_vm_zone, "A", "Zone Info");
623 SYSCTL_INT(_vm, OID_AUTO, zone_kmem_pages,
624 CTLFLAG_RD, &zone_kmem_pages, 0, "Number of interrupt safe pages allocated by zone");
625 SYSCTL_INT(_vm, OID_AUTO, zone_kmem_kvaspace,
626 CTLFLAG_RD, &zone_kmem_kvaspace, 0, "KVA space allocated by zone");
627 SYSCTL_INT(_vm, OID_AUTO, zone_kern_pages,
628 CTLFLAG_RD, &zone_kern_pages, 0, "Number of non-interrupt safe pages allocated by zone");