4 * Copyright (c) 1991, 1993
5 * The Regents of the University of California. All rights reserved.
7 * This code is derived from software contributed to Berkeley by
8 * The Mach Operating System project at Carnegie-Mellon University.
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 * 3. All advertising materials mentioning features or use of this software
19 * must display the following acknowledgement:
20 * This product includes software developed by the University of
21 * California, Berkeley and its contributors.
22 * 4. Neither the name of the University nor the names of its contributors
23 * may be used to endorse or promote products derived from this software
24 * without specific prior written permission.
26 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
27 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
29 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
30 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
31 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
32 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
38 * from: @(#)vm_kern.c 8.3 (Berkeley) 1/12/94
41 * Copyright (c) 1987, 1990 Carnegie-Mellon University.
42 * All rights reserved.
44 * Authors: Avadis Tevanian, Jr., Michael Wayne Young
46 * Permission to use, copy, modify and distribute this software and
47 * its documentation is hereby granted, provided that both the copyright
48 * notice and this permission notice appear in all copies of the
49 * software, derivative works or modified versions, and any portions
50 * thereof, and that both notices appear in supporting documentation.
52 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
53 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
54 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
56 * Carnegie Mellon requests users of this software to return to
58 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
59 * School of Computer Science
60 * Carnegie Mellon University
61 * Pittsburgh PA 15213-3890
63 * any improvements or extensions that they make and grant Carnegie the
64 * rights to redistribute these changes.
66 * $FreeBSD: src/sys/vm/vm_kern.c,v 1.61.2.2 2002/03/12 18:25:26 tegge Exp $
67 * $DragonFly: src/sys/vm/vm_kern.c,v 1.29 2007/06/07 23:14:29 dillon Exp $
71 * Kernel memory management.
74 #include <sys/param.h>
75 #include <sys/systm.h>
77 #include <sys/malloc.h>
78 #include <sys/kernel.h>
79 #include <sys/sysctl.h>
82 #include <vm/vm_param.h>
85 #include <vm/vm_map.h>
86 #include <vm/vm_object.h>
87 #include <vm/vm_page.h>
88 #include <vm/vm_pageout.h>
89 #include <vm/vm_kern.h>
90 #include <vm/vm_extern.h>
92 struct vm_map kernel_map;
93 struct vm_map clean_map;
94 struct vm_map buffer_map;
97 * Allocate pageable memory to the kernel's address map. "map" must
98 * be kernel_map or a submap of kernel_map.
103 kmem_alloc_pageable(vm_map_t map, vm_size_t size)
108 size = round_page(size);
109 addr = vm_map_min(map);
110 result = vm_map_find(map, NULL, (vm_offset_t) 0,
111 &addr, size, PAGE_SIZE,
112 TRUE, VM_MAPTYPE_NORMAL,
113 VM_PROT_ALL, VM_PROT_ALL,
115 if (result != KERN_SUCCESS) {
122 * Same as kmem_alloc_pageable, except that it create a nofault entry.
127 kmem_alloc_nofault(vm_map_t map, vm_size_t size, vm_size_t align)
132 size = round_page(size);
133 addr = vm_map_min(map);
134 result = vm_map_find(map, NULL, (vm_offset_t) 0,
136 TRUE, VM_MAPTYPE_NORMAL,
137 VM_PROT_ALL, VM_PROT_ALL,
139 if (result != KERN_SUCCESS) {
146 * Allocate wired-down memory in the kernel's address map or a submap.
151 kmem_alloc3(vm_map_t map, vm_size_t size, int kmflags)
157 size = round_page(size);
159 if (kmflags & KM_KRESERVE)
160 count = vm_map_entry_kreserve(MAP_RESERVE_COUNT);
162 count = vm_map_entry_reserve(MAP_RESERVE_COUNT);
165 * Use the kernel object for wired-down kernel pages. Assume that no
166 * region of the kernel object is referenced more than once.
168 * Locate sufficient space in the map. This will give us the final
169 * virtual address for the new memory, and thus will tell us the
170 * offset within the kernel map.
173 if (vm_map_findspace(map, vm_map_min(map), size, PAGE_SIZE, 0, &addr)) {
175 if (kmflags & KM_KRESERVE)
176 vm_map_entry_krelease(count);
178 vm_map_entry_release(count);
181 vm_object_reference(&kernel_object);
182 vm_map_insert(map, &count,
183 &kernel_object, addr, addr, addr + size,
185 VM_PROT_ALL, VM_PROT_ALL,
188 if (kmflags & KM_KRESERVE)
189 vm_map_entry_krelease(count);
191 vm_map_entry_release(count);
194 * Guarantee that there are pages already in this object before
195 * calling vm_map_wire. This is to prevent the following
198 * 1) Threads have swapped out, so that there is a pager for the
199 * kernel_object. 2) The kmsg zone is empty, and so we are
200 * kmem_allocing a new page for it. 3) vm_map_wire calls vm_fault;
201 * there is no page, but there is a pager, so we call
202 * pager_data_request. But the kmsg zone is empty, so we must
203 * kmem_alloc. 4) goto 1 5) Even if the kmsg zone is not empty: when
204 * we get the data back from the pager, it will be (very stale)
205 * non-zero data. kmem_alloc is defined to return zero-filled memory.
207 * We're intentionally not activating the pages we allocate to prevent a
208 * race with page-out. vm_map_wire will wire the pages.
211 lwkt_gettoken(&vm_token);
212 for (i = 0; i < size; i += PAGE_SIZE) {
215 mem = vm_page_grab(&kernel_object, OFF_TO_IDX(addr + i),
216 VM_ALLOC_ZERO | VM_ALLOC_NORMAL | VM_ALLOC_RETRY);
217 if ((mem->flags & PG_ZERO) == 0)
218 vm_page_zero_fill(mem);
219 mem->valid = VM_PAGE_BITS_ALL;
220 vm_page_flag_clear(mem, PG_ZERO);
223 lwkt_reltoken(&vm_token);
226 * And finally, mark the data as non-pageable.
228 vm_map_wire(map, (vm_offset_t)addr, addr + size, kmflags);
234 * Release a region of kernel virtual memory allocated with kmem_alloc,
235 * and return the physical pages associated with that region.
237 * WARNING! If the caller entered pages into the region using pmap_kenter()
238 * it must remove the pages using pmap_kremove[_quick]() before freeing the
239 * underlying kmem, otherwise resident_count will be mistabulated.
244 kmem_free(vm_map_t map, vm_offset_t addr, vm_size_t size)
246 vm_map_remove(map, trunc_page(addr), round_page(addr + size));
250 * Used to break a system map into smaller maps, usually to reduce
251 * contention and to provide large KVA spaces for subsystems like the
254 * parent Map to take range from
256 * size Size of range to find
257 * min, max Returned endpoints of map
258 * pageable Can the region be paged
263 kmem_suballoc(vm_map_t parent, vm_map_t result,
264 vm_offset_t *min, vm_offset_t *max, vm_size_t size)
268 size = round_page(size);
270 lwkt_gettoken(&vm_token);
271 *min = (vm_offset_t) vm_map_min(parent);
272 ret = vm_map_find(parent, NULL, (vm_offset_t) 0,
273 min, size, PAGE_SIZE,
274 TRUE, VM_MAPTYPE_UNSPECIFIED,
275 VM_PROT_ALL, VM_PROT_ALL,
277 if (ret != KERN_SUCCESS) {
278 kprintf("kmem_suballoc: bad status return of %d.\n", ret);
279 panic("kmem_suballoc");
282 pmap_reference(vm_map_pmap(parent));
283 vm_map_init(result, *min, *max, vm_map_pmap(parent));
284 if ((ret = vm_map_submap(parent, *min, *max, result)) != KERN_SUCCESS)
285 panic("kmem_suballoc: unable to change range to submap");
286 lwkt_reltoken(&vm_token);
290 * Allocates pageable memory from a sub-map of the kernel. If the submap
291 * has no room, the caller sleeps waiting for more memory in the submap.
296 kmem_alloc_wait(vm_map_t map, vm_size_t size)
301 size = round_page(size);
303 count = vm_map_entry_reserve(MAP_RESERVE_COUNT);
307 * To make this work for more than one map, use the map's lock
308 * to lock out sleepers/wakers.
311 if (vm_map_findspace(map, vm_map_min(map),
312 size, PAGE_SIZE, 0, &addr) == 0) {
315 /* no space now; see if we can ever get space */
316 if (vm_map_max(map) - vm_map_min(map) < size) {
317 vm_map_entry_release(count);
322 tsleep(map, 0, "kmaw", 0);
324 vm_map_insert(map, &count,
325 NULL, (vm_offset_t) 0,
328 VM_PROT_ALL, VM_PROT_ALL,
331 vm_map_entry_release(count);
337 * Returns memory to a submap of the kernel, and wakes up any processes
338 * waiting for memory in that map.
343 kmem_free_wakeup(vm_map_t map, vm_offset_t addr, vm_size_t size)
347 count = vm_map_entry_reserve(MAP_RESERVE_COUNT);
349 vm_map_delete(map, trunc_page(addr), round_page(addr + size), &count);
352 vm_map_entry_release(count);
356 * Create the kernel_map and insert mappings to cover areas already
357 * allocated or reserved thus far. That is, the area (KvaStart,start)
358 * and (end,KvaEnd) must be marked as allocated.
360 * virtual2_start/end is a cutout Between KvaStart and start,
361 * for x86_64 due to the location of KERNBASE (at -2G).
363 * We could use a min_offset of 0 instead of KvaStart, but since the
364 * min_offset is not used for any calculations other then a bounds check
365 * it does not effect readability. KvaStart is more appropriate.
367 * Depend on the zalloc bootstrap cache to get our vm_map_entry_t.
368 * Called from the low level boot code only.
371 kmem_init(vm_offset_t start, vm_offset_t end)
377 m = vm_map_create(&kernel_map, &kernel_pmap, KvaStart, KvaEnd);
379 /* N.B.: cannot use kgdb to debug, starting with this assignment ... */
381 count = vm_map_entry_reserve(MAP_RESERVE_COUNT);
383 if (virtual2_start) {
384 if (addr < virtual2_start) {
385 vm_map_insert(m, &count, NULL, (vm_offset_t) 0,
386 addr, virtual2_start,
388 VM_PROT_ALL, VM_PROT_ALL,
394 vm_map_insert(m, &count, NULL, (vm_offset_t) 0,
397 VM_PROT_ALL, VM_PROT_ALL,
402 vm_map_insert(m, &count, NULL, (vm_offset_t) 0,
405 VM_PROT_ALL, VM_PROT_ALL,
408 /* ... and ending with the completion of the above `insert' */
410 vm_map_entry_release(count);
417 kvm_size(SYSCTL_HANDLER_ARGS)
419 unsigned long ksize = KvaSize;
421 return sysctl_handle_long(oidp, &ksize, 0, req);
423 SYSCTL_PROC(_vm, OID_AUTO, kvm_size, CTLTYPE_LONG|CTLFLAG_RD,
424 0, 0, kvm_size, "IU", "Size of KVM");
430 kvm_free(SYSCTL_HANDLER_ARGS)
432 unsigned long kfree = virtual_end - kernel_vm_end;
434 return sysctl_handle_long(oidp, &kfree, 0, req);
436 SYSCTL_PROC(_vm, OID_AUTO, kvm_free, CTLTYPE_LONG|CTLFLAG_RD,
437 0, 0, kvm_free, "IU", "Amount of KVM free");