2 * Copyright (c) 1997, 1998 John S. Dyson
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice immediately at the beginning of the file, without modification,
10 * this list of conditions, and the following disclaimer.
11 * 2. Absolutely no warranty of function or purpose is made by the author
14 * $FreeBSD: src/sys/vm/vm_zone.c,v 1.30.2.6 2002/10/10 19:50:16 dillon Exp $
17 #include <sys/param.h>
18 #include <sys/systm.h>
19 #include <sys/kernel.h>
21 #include <sys/malloc.h>
22 #include <sys/sysctl.h>
23 #include <sys/vmmeter.h>
26 #include <vm/vm_object.h>
27 #include <vm/vm_page.h>
28 #include <vm/vm_map.h>
29 #include <vm/vm_kern.h>
30 #include <vm/vm_extern.h>
31 #include <vm/vm_zone.h>
33 static MALLOC_DEFINE(M_ZONE, "ZONE", "Zone header");
35 #define ZONE_ERROR_INVALID 0
36 #define ZONE_ERROR_NOTFREE 1
37 #define ZONE_ERROR_ALREADYFREE 2
39 #define ZONE_ROUNDING 32
41 #define ZENTRY_FREE 0x12342378
43 * void *zalloc(vm_zone_t zone) --
44 * Returns an item from a specified zone.
46 * void zfree(vm_zone_t zone, void *item) --
47 * Frees an item back to a specified zone.
49 static __inline__ void *
56 zerror(ZONE_ERROR_INVALID);
59 if (z->zfreecnt <= z->zfreemin) {
62 * PANICFAIL allows the caller to assume that the zalloc()
63 * will always succeed. If it doesn't, we panic here.
65 if (item == NULL && (z->zflags & ZONE_PANICFAIL))
66 panic("zalloc(%s) failed", z->zname);
71 z->zitems = ((void **) item)[0];
73 KASSERT(item != NULL, ("zitems unexpectedly NULL"));
74 if (((void **) item)[1] != (void *) ZENTRY_FREE)
75 zerror(ZONE_ERROR_NOTFREE);
76 ((void **) item)[1] = 0;
84 static __inline__ void
85 _zfree(vm_zone_t z, void *item)
87 ((void **) item)[0] = z->zitems;
89 if (((void **) item)[1] == (void *) ZENTRY_FREE)
90 zerror(ZONE_ERROR_ALREADYFREE);
91 ((void **) item)[1] = (void *) ZENTRY_FREE;
98 * This file comprises a very simple zone allocator. This is used
99 * in lieu of the malloc allocator, where needed or more optimal.
101 * Note that the initial implementation of this had coloring, and
102 * absolutely no improvement (actually perf degradation) occurred.
104 * Note also that the zones are type stable. The only restriction is
105 * that the first two longwords of a data structure can be changed
106 * between allocations. Any data that must be stable between allocations
107 * must reside in areas after the first two longwords.
109 * zinitna, zinit, zbootinit are the initialization routines.
110 * zalloc, zfree, are the interrupt/lock unsafe allocation/free routines.
111 * zalloci, zfreei, are the interrupt/lock safe allocation/free routines.
114 static struct vm_zone *zlist;
115 static int sysctl_vm_zone(SYSCTL_HANDLER_ARGS);
116 static int zone_kmem_pages, zone_kern_pages, zone_kmem_kvaspace;
119 * Create a zone, but don't allocate the zone structure. If the
120 * zone had been previously created by the zone boot code, initialize
121 * various parts of the zone code.
123 * If waits are not allowed during allocation (e.g. during interrupt
124 * code), a-priori allocate the kernel virtual space, and allocate
125 * only pages when needed.
128 * z pointer to zone structure.
129 * obj pointer to VM object (opt).
131 * size size of zone entries.
132 * nentries number of zone entries allocated (only ZONE_INTERRUPT.)
133 * flags ZONE_INTERRUPT -- items can be allocated at interrupt time.
134 * zalloc number of pages allocated when memory is needed.
136 * Note that when using ZONE_INTERRUPT, the size of the zone is limited
137 * by the nentries argument. The size of the memory allocatable is
138 * unlimited if ZONE_INTERRUPT is not set.
142 zinitna(vm_zone_t z, vm_object_t obj, char *name, int size,
143 int nentries, int flags, int zalloc)
147 if ((z->zflags & ZONE_BOOT) == 0) {
148 z->zsize = (size + ZONE_ROUNDING - 1) & ~(ZONE_ROUNDING - 1);
149 simple_lock_init(&z->zlock);
164 * If we cannot wait, allocate KVA space up front, and we will fill
165 * in pages as needed.
167 if (z->zflags & ZONE_INTERRUPT) {
169 totsize = round_page(z->zsize * nentries);
170 zone_kmem_kvaspace += totsize;
172 z->zkva = kmem_alloc_pageable(kernel_map, totsize);
178 z->zpagemax = totsize / PAGE_SIZE;
180 z->zobj = vm_object_allocate(OBJT_DEFAULT, z->zpagemax);
183 _vm_object_allocate(OBJT_DEFAULT, z->zpagemax, obj);
185 z->zallocflag = VM_ALLOC_INTERRUPT;
188 z->zallocflag = VM_ALLOC_SYSTEM;
193 if (z->zsize > PAGE_SIZE)
196 z->zfreemin = PAGE_SIZE / z->zsize;
208 * Subroutine same as zinitna, except zone data structure is allocated
209 * automatically by malloc. This routine should normally be used, except
210 * in certain tricky startup conditions in the VM system -- then
211 * zbootinit and zinitna can be used. Zinit is the standard zone
212 * initialization call.
215 zinit(char *name, int size, int nentries, int flags, int zalloc)
219 z = (vm_zone_t) malloc(sizeof (struct vm_zone), M_ZONE, M_NOWAIT);
224 if (zinitna(z, NULL, name, size, nentries, flags, zalloc) == 0) {
233 * Initialize a zone before the system is fully up. This routine should
234 * only be called before full VM startup.
237 zbootinit(vm_zone_t z, char *name, int size, void *item, int nitems)
245 z->zflags = ZONE_BOOT;
251 simple_lock_init(&z->zlock);
253 bzero(item, nitems * z->zsize);
255 for (i = 0; i < nitems; i++) {
256 ((void **) item)[0] = z->zitems;
258 ((void **) item)[1] = (void *) ZENTRY_FREE;
261 (char *) item += z->zsize;
263 z->zfreecnt = nitems;
276 * Zone critical region locks.
284 simple_lock(&z->zlock);
289 zunlock(vm_zone_t z, int s)
291 simple_unlock(&z->zlock);
296 * void *zalloc(vm_zone_t zone) --
297 * Returns an item from a specified zone.
299 * void zfree(vm_zone_t zone, void *item) --
300 * Frees an item back to a specified zone.
302 * void *zalloci(vm_zone_t zone) --
303 * Returns an item from a specified zone, interrupt safe.
305 * void zfreei(vm_zone_t zone, void *item) --
306 * Frees an item back to a specified zone, interrupt safe.
321 zfree(vm_zone_t z, void *item)
331 * Zone allocator/deallocator. These are interrupt / (or potentially SMP)
332 * safe. The raw zalloc/zfree routines are not interrupt safe, but are fast.
347 zfreei(vm_zone_t z, void *item)
358 * Internal zone routine. Not to be called from external (non vm_zone) code.
369 panic("zget: null zone");
371 if (z->zflags & ZONE_INTERRUPT) {
372 nbytes = z->zpagecount * PAGE_SIZE;
373 nbytes -= nbytes % z->zsize;
374 item = (char *) z->zkva + nbytes;
375 for (i = 0; ((i < z->zalloc) && (z->zpagecount < z->zpagemax));
379 m = vm_page_alloc(z->zobj, z->zpagecount,
384 zkva = z->zkva + z->zpagecount * PAGE_SIZE;
385 pmap_kenter(zkva, VM_PAGE_TO_PHYS(m));
386 bzero((caddr_t) zkva, PAGE_SIZE);
391 nitems = ((z->zpagecount * PAGE_SIZE) - nbytes) / z->zsize;
393 nbytes = z->zalloc * PAGE_SIZE;
396 * Check to see if the kernel map is already locked. We could allow
397 * for recursive locks, but that eliminates a valuable debugging
398 * mechanism, and opens up the kernel map for potential corruption
399 * by inconsistent data structure manipulation. We could also use
400 * the interrupt allocation mechanism, but that has size limitations.
401 * Luckily, we have kmem_map that is a submap of kernel map available
402 * for memory allocation, and manipulation of that map doesn't affect
403 * the kernel map structures themselves.
405 * We can wait, so just do normal map allocation in the appropriate
408 if (lockstatus(&kernel_map->lock, NULL)) {
412 simple_unlock(&z->zlock);
414 item = (void *) kmem_malloc(kmem_map, nbytes, M_WAITOK);
416 simple_lock(&z->zlock);
419 zone_kmem_pages += z->zalloc;
423 simple_unlock(&z->zlock);
425 item = (void *) kmem_alloc(kernel_map, nbytes);
427 simple_lock(&z->zlock);
430 zone_kern_pages += z->zalloc;
437 nitems = nbytes / z->zsize;
442 * Save one for immediate allocation
446 for (i = 0; i < nitems; i++) {
447 ((void **) item)[0] = z->zitems;
449 ((void **) item)[1] = (void *) ZENTRY_FREE;
452 (char *) item += z->zsize;
454 z->zfreecnt += nitems;
456 } else if (z->zfreecnt > 0) {
458 z->zitems = ((void **) item)[0];
460 if (((void **) item)[1] != (void *) ZENTRY_FREE)
461 zerror(ZONE_ERROR_NOTFREE);
462 ((void **) item)[1] = 0;
474 sysctl_vm_zone(SYSCTL_HANDLER_ARGS)
477 vm_zone_t curzone, nextzone;
481 snprintf(tmpbuf, sizeof(tmpbuf),
482 "\nITEM SIZE LIMIT USED FREE REQUESTS\n");
483 error = SYSCTL_OUT(req, tmpbuf, strlen(tmpbuf));
487 for (curzone = zlist; curzone; curzone = nextzone) {
492 nextzone = curzone->znext;
493 len = strlen(curzone->zname);
494 if (len >= (sizeof(tmpname) - 1))
495 len = (sizeof(tmpname) - 1);
496 for(i = 0; i < sizeof(tmpname) - 1; i++)
499 memcpy(tmpname, curzone->zname, len);
502 if (curzone == zlist) {
507 snprintf(tmpbuf + offset, sizeof(tmpbuf) - offset,
508 "%s %6.6u, %8.8u, %6.6u, %6.6u, %8.8u\n",
509 tmpname, curzone->zsize, curzone->zmax,
510 (curzone->ztotal - curzone->zfreecnt),
511 curzone->zfreecnt, curzone->znalloc);
513 len = strlen((char *)tmpbuf);
514 if (nextzone == NULL)
517 error = SYSCTL_OUT(req, tmpbuf, len);
525 #ifdef INVARIANT_SUPPORT
532 case ZONE_ERROR_INVALID:
533 msg = "zone: invalid zone";
535 case ZONE_ERROR_NOTFREE:
536 msg = "zone: entry not free";
538 case ZONE_ERROR_ALREADYFREE:
539 msg = "zone: freeing free entry";
542 msg = "zone: invalid error";
549 SYSCTL_OID(_vm, OID_AUTO, zone, CTLTYPE_STRING|CTLFLAG_RD, \
550 NULL, 0, sysctl_vm_zone, "A", "Zone Info");
552 SYSCTL_INT(_vm, OID_AUTO, zone_kmem_pages,
553 CTLFLAG_RD, &zone_kmem_pages, 0, "Number of interrupt safe pages allocated by zone");
554 SYSCTL_INT(_vm, OID_AUTO, zone_kmem_kvaspace,
555 CTLFLAG_RD, &zone_kmem_kvaspace, 0, "KVA space allocated by zone");
556 SYSCTL_INT(_vm, OID_AUTO, zone_kern_pages,
557 CTLFLAG_RD, &zone_kern_pages, 0, "Number of non-interrupt safe pages allocated by zone");