4 * Copyright (c) 1991, 1993
5 * The Regents of the University of California. All rights reserved.
7 * This code is derived from software contributed to Berkeley by
8 * The Mach Operating System project at Carnegie-Mellon University.
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 * 3. Neither the name of the University nor the names of its contributors
19 * may be used to endorse or promote products derived from this software
20 * without specific prior written permission.
22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * from: @(#)vm_kern.c 8.3 (Berkeley) 1/12/94
37 * Copyright (c) 1987, 1990 Carnegie-Mellon University.
38 * All rights reserved.
40 * Authors: Avadis Tevanian, Jr., Michael Wayne Young
42 * Permission to use, copy, modify and distribute this software and
43 * its documentation is hereby granted, provided that both the copyright
44 * notice and this permission notice appear in all copies of the
45 * software, derivative works or modified versions, and any portions
46 * thereof, and that both notices appear in supporting documentation.
48 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
49 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
50 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
52 * Carnegie Mellon requests users of this software to return to
54 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
55 * School of Computer Science
56 * Carnegie Mellon University
57 * Pittsburgh PA 15213-3890
59 * any improvements or extensions that they make and grant Carnegie the
60 * rights to redistribute these changes.
62 * $FreeBSD: src/sys/vm/vm_kern.c,v 1.61.2.2 2002/03/12 18:25:26 tegge Exp $
66 * Kernel memory management.
69 #include <sys/param.h>
70 #include <sys/systm.h>
72 #include <sys/malloc.h>
73 #include <sys/kernel.h>
74 #include <sys/sysctl.h>
77 #include <vm/vm_param.h>
80 #include <vm/vm_map.h>
81 #include <vm/vm_object.h>
82 #include <vm/vm_page.h>
83 #include <vm/vm_pageout.h>
84 #include <vm/vm_kern.h>
85 #include <vm/vm_extern.h>
87 struct vm_map kernel_map;
88 struct vm_map clean_map;
89 struct vm_map buffer_map;
92 * Allocate pageable swap-backed anonymous memory
95 kmem_alloc_swapbacked(kmem_anon_desc_t *kp, vm_size_t size)
100 size = round_page(size);
101 npages = size / PAGE_SIZE;
104 kp->map = &kernel_map;
105 kp->data = vm_map_min(&kernel_map);
107 kp->object = vm_object_allocate(OBJT_DEFAULT, npages);
109 error = vm_map_find(kp->map, kp->object, NULL, 0,
112 1, VM_MAPTYPE_NORMAL,
113 VM_PROT_ALL, VM_PROT_ALL, 0);
115 kprintf("kmem_alloc_swapbacked: %zd bytes failed %d\n",
117 kp->data = (vm_offset_t)0;
118 kmem_free_swapbacked(kp);
121 return ((void *)(intptr_t)kp->data);
125 kmem_free_swapbacked(kmem_anon_desc_t *kp)
129 * The object will be deallocated by kmem_free().
131 kmem_free(kp->map, kp->data, kp->size);
132 kp->data = (vm_offset_t)0;
135 * Failure during allocation, object must be deallocated
138 vm_object_deallocate(kp->object);
144 * Allocate pageable memory to the kernel's address map. "map" must
145 * be kernel_map or a submap of kernel_map. Caller must adjust map or
146 * enter VM pages itself.
151 kmem_alloc_pageable(vm_map_t map, vm_size_t size)
156 size = round_page(size);
157 addr = vm_map_min(map);
158 result = vm_map_find(map, NULL, NULL,
159 (vm_offset_t) 0, &addr, size,
161 TRUE, VM_MAPTYPE_NORMAL,
162 VM_PROT_ALL, VM_PROT_ALL, 0);
163 if (result != KERN_SUCCESS)
169 * Same as kmem_alloc_pageable, except that it create a nofault entry.
174 kmem_alloc_nofault(vm_map_t map, vm_size_t size, vm_size_t align)
179 size = round_page(size);
180 addr = vm_map_min(map);
181 result = vm_map_find(map, NULL, NULL,
182 (vm_offset_t) 0, &addr, size,
184 TRUE, VM_MAPTYPE_NORMAL,
185 VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT);
186 if (result != KERN_SUCCESS)
192 * Allocate wired-down memory in the kernel's address map or a submap.
197 kmem_alloc3(vm_map_t map, vm_size_t size, int kmflags)
205 size = round_page(size);
207 if (kmflags & KM_KRESERVE)
208 count = vm_map_entry_kreserve(MAP_RESERVE_COUNT);
210 count = vm_map_entry_reserve(MAP_RESERVE_COUNT);
212 if (kmflags & KM_STACK) {
221 * Use the kernel object for wired-down kernel pages. Assume that no
222 * region of the kernel object is referenced more than once.
224 * Locate sufficient space in the map. This will give us the final
225 * virtual address for the new memory, and thus will tell us the
226 * offset within the kernel map.
229 if (vm_map_findspace(map, vm_map_min(map), size, PAGE_SIZE, 0, &addr)) {
231 if (kmflags & KM_KRESERVE)
232 vm_map_entry_krelease(count);
234 vm_map_entry_release(count);
237 vm_object_hold(&kernel_object);
238 vm_object_reference_locked(&kernel_object);
239 vm_map_insert(map, &count,
240 &kernel_object, NULL,
241 addr, addr, addr + size,
243 VM_PROT_ALL, VM_PROT_ALL, cow);
244 vm_object_drop(&kernel_object);
247 if (kmflags & KM_KRESERVE)
248 vm_map_entry_krelease(count);
250 vm_map_entry_release(count);
253 * Guarantee that there are pages already in this object before
254 * calling vm_map_wire. This is to prevent the following
257 * 1) Threads have swapped out, so that there is a pager for the
258 * kernel_object. 2) The kmsg zone is empty, and so we are
259 * kmem_allocing a new page for it. 3) vm_map_wire calls vm_fault;
260 * there is no page, but there is a pager, so we call
261 * pager_data_request. But the kmsg zone is empty, so we must
262 * kmem_alloc. 4) goto 1 5) Even if the kmsg zone is not empty: when
263 * we get the data back from the pager, it will be (very stale)
264 * non-zero data. kmem_alloc is defined to return zero-filled memory.
266 * We're intentionally not activating the pages we allocate to prevent a
267 * race with page-out. vm_map_wire will wire the pages.
269 vm_object_hold(&kernel_object);
270 for (i = gstart; i < size; i += PAGE_SIZE) {
273 mem = vm_page_grab(&kernel_object, OFF_TO_IDX(addr + i),
274 VM_ALLOC_FORCE_ZERO | VM_ALLOC_NORMAL |
276 vm_page_unqueue_nowakeup(mem);
279 vm_object_drop(&kernel_object);
282 * And finally, mark the data as non-pageable.
284 * NOTE: vm_map_wire() handles any kstack guard.
286 vm_map_wire(map, addr, addr + size, kmflags);
292 * Release a region of kernel virtual memory allocated with kmem_alloc,
293 * and return the physical pages associated with that region.
295 * WARNING! If the caller entered pages into the region using pmap_kenter()
296 * it must remove the pages using pmap_kremove[_quick]() before freeing the
297 * underlying kmem, otherwise resident_count will be mistabulated.
302 kmem_free(vm_map_t map, vm_offset_t addr, vm_size_t size)
304 vm_map_remove(map, trunc_page(addr), round_page(addr + size));
308 * Used to break a system map into smaller maps, usually to reduce
309 * contention and to provide large KVA spaces for subsystems like the
312 * parent Map to take range from
314 * size Size of range to find
315 * min, max Returned endpoints of map
316 * pageable Can the region be paged
321 kmem_suballoc(vm_map_t parent, vm_map_t result,
322 vm_offset_t *min, vm_offset_t *max, vm_size_t size)
326 size = round_page(size);
328 *min = (vm_offset_t) vm_map_min(parent);
329 ret = vm_map_find(parent, NULL, NULL,
330 (vm_offset_t) 0, min, size,
332 TRUE, VM_MAPTYPE_UNSPECIFIED,
333 VM_PROT_ALL, VM_PROT_ALL, 0);
334 if (ret != KERN_SUCCESS) {
335 kprintf("kmem_suballoc: bad status return of %d.\n", ret);
336 panic("kmem_suballoc");
339 pmap_reference(vm_map_pmap(parent));
340 vm_map_init(result, *min, *max, vm_map_pmap(parent));
341 if ((ret = vm_map_submap(parent, *min, *max, result)) != KERN_SUCCESS)
342 panic("kmem_suballoc: unable to change range to submap");
346 * Allocates pageable memory from a sub-map of the kernel. If the submap
347 * has no room, the caller sleeps waiting for more memory in the submap.
352 kmem_alloc_wait(vm_map_t map, vm_size_t size)
357 size = round_page(size);
359 count = vm_map_entry_reserve(MAP_RESERVE_COUNT);
363 * To make this work for more than one map, use the map's lock
364 * to lock out sleepers/wakers.
367 if (vm_map_findspace(map, vm_map_min(map),
368 size, PAGE_SIZE, 0, &addr) == 0) {
371 /* no space now; see if we can ever get space */
372 if (vm_map_max(map) - vm_map_min(map) < size) {
373 vm_map_entry_release(count);
378 tsleep(map, 0, "kmaw", 0);
380 vm_map_insert(map, &count,
382 (vm_offset_t) 0, addr, addr + size,
384 VM_PROT_ALL, VM_PROT_ALL,
387 vm_map_entry_release(count);
393 * Allocates a region from the kernel address map and physical pages
394 * within the specified address range to the kernel object. Creates a
395 * wired mapping from this region to these pages, and returns the
396 * region's starting virtual address. The allocated pages are not
397 * necessarily physically contiguous. If M_ZERO is specified through the
398 * given flags, then the pages are zeroed before they are mapped.
401 kmem_alloc_attr(vm_map_t map, vm_size_t size, int flags, vm_paddr_t low,
402 vm_paddr_t high, vm_memattr_t memattr)
404 vm_offset_t addr, i, offset;
408 size = round_page(size);
409 count = vm_map_entry_reserve(MAP_RESERVE_COUNT);
411 if (vm_map_findspace(map, vm_map_min(map), size, PAGE_SIZE,
414 vm_map_entry_release(count);
417 offset = addr - vm_map_min(&kernel_map);
418 vm_object_hold(&kernel_object);
419 vm_object_reference_locked(&kernel_object);
420 vm_map_insert(map, &count,
421 &kernel_object, NULL,
422 offset, addr, addr + size,
424 VM_PROT_ALL, VM_PROT_ALL, 0);
426 vm_map_entry_release(count);
427 vm_object_drop(&kernel_object);
428 for (i = 0; i < size; i += PAGE_SIZE) {
429 m = vm_page_alloc_contig(low, high, PAGE_SIZE, 0, PAGE_SIZE, memattr);
433 vm_object_hold(&kernel_object);
434 vm_page_insert(m, &kernel_object, OFF_TO_IDX(offset + i));
435 vm_object_drop(&kernel_object);
437 pmap_zero_page(VM_PAGE_TO_PHYS(m));
438 m->valid = VM_PAGE_BITS_ALL;
440 vm_map_wire(map, addr, addr + size, 0);
446 * Returns memory to a submap of the kernel, and wakes up any processes
447 * waiting for memory in that map.
452 kmem_free_wakeup(vm_map_t map, vm_offset_t addr, vm_size_t size)
456 count = vm_map_entry_reserve(MAP_RESERVE_COUNT);
458 vm_map_delete(map, trunc_page(addr), round_page(addr + size), &count);
461 vm_map_entry_release(count);
465 * Create the kernel_ma for (KvaStart,KvaEnd) and insert mappings to
466 * cover areas already allocated or reserved thus far.
468 * The areas (virtual_start, virtual_end) and (virtual2_start, virtual2_end)
469 * are available so the cutouts are the areas around these ranges between
470 * KvaStart and KvaEnd.
472 * Depend on the zalloc bootstrap cache to get our vm_map_entry_t.
473 * Called from the low level boot code only.
482 m = vm_map_create(&kernel_map, &kernel_pmap, KvaStart, KvaEnd);
484 /* N.B.: cannot use kgdb to debug, starting with this assignment ... */
486 count = vm_map_entry_reserve(MAP_RESERVE_COUNT);
488 if (virtual2_start) {
489 if (addr < virtual2_start) {
490 vm_map_insert(m, &count,
492 (vm_offset_t) 0, addr, virtual2_start,
494 VM_PROT_ALL, VM_PROT_ALL, 0);
498 if (addr < virtual_start) {
499 vm_map_insert(m, &count,
501 (vm_offset_t) 0, addr, virtual_start,
503 VM_PROT_ALL, VM_PROT_ALL, 0);
507 vm_map_insert(m, &count,
509 (vm_offset_t) 0, addr, KvaEnd,
511 VM_PROT_ALL, VM_PROT_ALL, 0);
513 /* ... and ending with the completion of the above `insert' */
515 vm_map_entry_release(count);
522 kvm_size(SYSCTL_HANDLER_ARGS)
524 unsigned long ksize = KvaSize;
526 return sysctl_handle_long(oidp, &ksize, 0, req);
528 SYSCTL_PROC(_vm, OID_AUTO, kvm_size, CTLTYPE_ULONG|CTLFLAG_RD,
529 0, 0, kvm_size, "LU", "Size of KVM");
535 kvm_free(SYSCTL_HANDLER_ARGS)
537 unsigned long kfree = virtual_end - kernel_vm_end;
539 return sysctl_handle_long(oidp, &kfree, 0, req);
541 SYSCTL_PROC(_vm, OID_AUTO, kvm_free, CTLTYPE_ULONG|CTLFLAG_RD,
542 0, 0, kvm_free, "LU", "Amount of KVM free");