4 * Copyright (c) 2006 The DragonFly Project. All rights reserved.
6 * This code is derived from software contributed to The DragonFly Project
7 * by Matthew Dillon <dillon@backplane.com>
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in
17 * the documentation and/or other materials provided with the
19 * 3. Neither the name of The DragonFly Project nor the names of its
20 * contributors may be used to endorse or promote products derived
21 * from this software without specific, prior written permission.
23 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
24 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
25 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
26 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
27 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
28 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
29 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
30 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
31 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
32 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
33 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
37 #include <sys/param.h>
38 #include <sys/kernel.h>
39 #include <sys/systm.h>
40 #include <sys/sysproto.h>
41 #include <sys/kern_syscall.h>
43 #include <sys/thread.h>
45 #include <sys/malloc.h>
46 #include <sys/sysctl.h>
47 #include <sys/vkernel.h>
48 #include <sys/vmspace.h>
50 #include <vm/vm_extern.h>
53 #include <machine/vmparam.h>
55 #include <sys/sysref2.h>
56 #include <sys/mplock2.h>
58 static struct vmspace_entry *vkernel_find_vmspace(struct vkernel_proc *vkp,
60 static void vmspace_entry_delete(struct vmspace_entry *ve,
61 struct vkernel_proc *vkp);
63 static MALLOC_DEFINE(M_VKERNEL, "vkernel", "VKernel structures");
66 * vmspace_create (void *id, int type, void *data)
68 * Create a VMSPACE under the control of the caller with the specified id.
69 * An id of NULL cannot be used. The type and data fields must currently
72 * The vmspace starts out completely empty. Memory may be mapped into the
73 * VMSPACE with vmspace_mmap() and MAP_VPAGETABLE section(s) controlled
74 * with vmspace_mcontrol().
79 sys_vmspace_create(struct vmspace_create_args *uap)
81 struct vmspace_entry *ve;
82 struct vkernel_proc *vkp;
83 struct proc *p = curproc;
86 if (vkernel_enable == 0)
90 * Create a virtual kernel side-structure for the process if one
93 * Implement a simple resolution for SMP races.
95 if ((vkp = p->p_vkernel) == NULL) {
96 vkp = kmalloc(sizeof(*vkp), M_VKERNEL, M_WAITOK|M_ZERO);
97 lwkt_gettoken(&proc_token);
98 if (p->p_vkernel == NULL) {
100 lwkt_token_init(&vkp->token, 1, "vkernel");
104 kfree(vkp, M_VKERNEL);
107 lwkt_reltoken(&proc_token);
113 * Create a new VMSPACE, disallow conflicting ids
115 ve = kmalloc(sizeof(struct vmspace_entry), M_VKERNEL, M_WAITOK|M_ZERO);
116 ve->vmspace = vmspace_alloc(VM_MIN_USER_ADDRESS, VM_MAX_USER_ADDRESS);
118 pmap_pinit2(vmspace_pmap(ve->vmspace));
120 lwkt_gettoken(&vkp->token);
121 if (RB_INSERT(vmspace_rb_tree, &vkp->root, ve)) {
122 sysref_put(&ve->vmspace->vm_sysref);
123 kfree(ve, M_VKERNEL);
128 lwkt_reltoken(&vkp->token);
134 * Destroy a VMSPACE given its identifier.
139 sys_vmspace_destroy(struct vmspace_destroy_args *uap)
141 struct vkernel_proc *vkp;
142 struct vmspace_entry *ve;
146 if ((vkp = curproc->p_vkernel) == NULL) {
150 lwkt_gettoken(&vkp->token);
151 if ((ve = vkernel_find_vmspace(vkp, uap->id)) == NULL) {
159 vmspace_entry_delete(ve, vkp);
162 lwkt_reltoken(&vkp->token);
169 * vmspace_ctl (void *id, int cmd, struct trapframe *tframe,
170 * struct vextframe *vframe);
172 * Transfer control to a VMSPACE. Control is returned after the specified
173 * number of microseconds or if a page fault, signal, trap, or system call
174 * occurs. The context is updated as appropriate.
179 sys_vmspace_ctl(struct vmspace_ctl_args *uap)
181 struct vkernel_proc *vkp;
182 struct vkernel_lwp *vklp;
183 struct vmspace_entry *ve;
189 lp = curthread->td_lwp;
192 if ((vkp = p->p_vkernel) == NULL)
196 lwkt_gettoken(&vkp->token);
197 if ((ve = vkernel_find_vmspace(vkp, uap->id)) == NULL) {
203 * Signal mailbox interlock
205 if (p->p_flag & P_MAILBOX) {
206 p->p_flag &= ~P_MAILBOX;
212 case VMSPACE_CTL_RUN:
214 * Save the caller's register context, swap VM spaces, and
215 * install the passed register context. Return with
216 * EJUSTRETURN so the syscall code doesn't adjust the context.
218 atomic_add_int(&ve->refs, 1);
219 framesz = sizeof(struct trapframe);
220 if ((vklp = lp->lwp_vkernel) == NULL) {
221 vklp = kmalloc(sizeof(*vklp), M_VKERNEL,
223 lp->lwp_vkernel = vklp;
225 vklp->user_trapframe = uap->tframe;
226 vklp->user_vextframe = uap->vframe;
227 bcopy(uap->sysmsg_frame, &vklp->save_trapframe, framesz);
228 bcopy(&curthread->td_tls, &vklp->save_vextframe.vx_tls,
229 sizeof(vklp->save_vextframe.vx_tls));
230 error = copyin(uap->tframe, uap->sysmsg_frame, framesz);
232 error = copyin(&uap->vframe->vx_tls,
234 sizeof(struct savetls));
237 error = cpu_sanitize_frame(uap->sysmsg_frame);
239 error = cpu_sanitize_tls(&curthread->td_tls);
241 bcopy(&vklp->save_trapframe, uap->sysmsg_frame,
243 bcopy(&vklp->save_vextframe.vx_tls, &curthread->td_tls,
244 sizeof(vklp->save_vextframe.vx_tls));
246 atomic_subtract_int(&ve->refs, 1);
249 pmap_setlwpvm(lp, ve->vmspace);
251 set_vkernel_fp(uap->sysmsg_frame);
260 lwkt_reltoken(&vkp->token);
266 * vmspace_mmap(id, addr, len, prot, flags, fd, offset)
268 * map memory within a VMSPACE. This function is just like a normal mmap()
269 * but operates on the vmspace's memory map. Most callers use this to create
270 * a MAP_VPAGETABLE mapping.
275 sys_vmspace_mmap(struct vmspace_mmap_args *uap)
277 struct vkernel_proc *vkp;
278 struct vmspace_entry *ve;
282 * We hold the vmspace token to serialize calls to vkernel_find_vmspace
283 * and the vm token to serialize calls to kern_mmap.
285 lwkt_gettoken(&vm_token);
286 lwkt_gettoken(&vmspace_token);
287 if ((vkp = curproc->p_vkernel) == NULL) {
293 * NOTE: kern_mmap() can block so we need to temporarily ref ve->refs.
295 lwkt_gettoken(&vkp->token);
296 if ((ve = vkernel_find_vmspace(vkp, uap->id)) != NULL) {
297 atomic_add_int(&ve->refs, 1);
298 error = kern_mmap(ve->vmspace, uap->addr, uap->len,
299 uap->prot, uap->flags,
300 uap->fd, uap->offset, &uap->sysmsg_resultp);
301 atomic_subtract_int(&ve->refs, 1);
305 lwkt_reltoken(&vkp->token);
307 lwkt_reltoken(&vmspace_token);
308 lwkt_reltoken(&vm_token);
313 * vmspace_munmap(id, addr, len)
315 * unmap memory within a VMSPACE.
320 sys_vmspace_munmap(struct vmspace_munmap_args *uap)
322 struct vkernel_proc *vkp;
323 struct vmspace_entry *ve;
326 vm_size_t size, pageoff;
331 if ((vkp = curproc->p_vkernel) == NULL) {
335 lwkt_gettoken(&vkp->token);
336 if ((ve = vkernel_find_vmspace(vkp, uap->id)) == NULL) {
342 * NOTE: kern_munmap() can block so we need to temporarily
345 atomic_add_int(&ve->refs, 1);
348 * Copied from sys_munmap()
350 addr = (vm_offset_t)uap->addr;
353 pageoff = (addr & PAGE_MASK);
356 size = (vm_size_t)round_page(size);
357 if (size < uap->len) { /* wrap */
361 tmpaddr = addr + size; /* workaround gcc4 opt */
362 if (tmpaddr < addr) { /* wrap */
371 if (VM_MAX_USER_ADDRESS > 0 && tmpaddr > VM_MAX_USER_ADDRESS) {
375 if (VM_MIN_USER_ADDRESS > 0 && addr < VM_MIN_USER_ADDRESS) {
379 map = &ve->vmspace->vm_map;
380 if (!vm_map_check_protection(map, addr, tmpaddr, VM_PROT_NONE, FALSE)) {
384 vm_map_remove(map, addr, addr + size);
387 atomic_subtract_int(&ve->refs, 1);
389 lwkt_reltoken(&vkp->token);
396 * vmspace_pread(id, buf, nbyte, flags, offset)
398 * Read data from a vmspace. The number of bytes read is returned or
399 * -1 if an unrecoverable error occured. If the number of bytes read is
400 * less then the request size, a page fault occured in the VMSPACE which
401 * the caller must resolve in order to proceed.
403 * (not implemented yet)
407 sys_vmspace_pread(struct vmspace_pread_args *uap)
409 struct vkernel_proc *vkp;
410 struct vmspace_entry *ve;
414 if ((vkp = curproc->p_vkernel) == NULL) {
418 lwkt_gettoken(&vkp->token);
419 if ((ve = vkernel_find_vmspace(vkp, uap->id)) == NULL) {
425 lwkt_reltoken(&vkp->token);
432 * vmspace_pwrite(id, buf, nbyte, flags, offset)
434 * Write data to a vmspace. The number of bytes written is returned or
435 * -1 if an unrecoverable error occured. If the number of bytes written is
436 * less then the request size, a page fault occured in the VMSPACE which
437 * the caller must resolve in order to proceed.
439 * (not implemented yet)
443 sys_vmspace_pwrite(struct vmspace_pwrite_args *uap)
445 struct vkernel_proc *vkp;
446 struct vmspace_entry *ve;
450 if ((vkp = curproc->p_vkernel) == NULL) {
454 lwkt_gettoken(&vkp->token);
455 if ((ve = vkernel_find_vmspace(vkp, uap->id)) == NULL) {
461 lwkt_reltoken(&vkp->token);
468 * vmspace_mcontrol(id, addr, len, behav, value)
470 * madvise/mcontrol support for a vmspace.
475 sys_vmspace_mcontrol(struct vmspace_mcontrol_args *uap)
477 struct vkernel_proc *vkp;
478 struct vmspace_entry *ve;
479 vm_offset_t start, end;
480 vm_offset_t tmpaddr = (vm_offset_t)uap->addr + uap->len;
484 if ((vkp = curproc->p_vkernel) == NULL) {
488 lwkt_gettoken(&vkp->token);
489 if ((ve = vkernel_find_vmspace(vkp, uap->id)) == NULL) {
495 * NOTE: kern_madvise() can block so we need to temporarily
498 atomic_add_int(&ve->refs, 1);
501 * This code is basically copied from sys_mcontrol()
503 if (uap->behav < 0 || uap->behav > MADV_CONTROL_END) {
508 if (tmpaddr < (vm_offset_t)uap->addr) {
512 if (VM_MAX_USER_ADDRESS > 0 && tmpaddr > VM_MAX_USER_ADDRESS) {
516 if (VM_MIN_USER_ADDRESS > 0 && uap->addr < VM_MIN_USER_ADDRESS) {
521 start = trunc_page((vm_offset_t) uap->addr);
522 end = round_page(tmpaddr);
524 error = vm_map_madvise(&ve->vmspace->vm_map, start, end,
525 uap->behav, uap->value);
527 atomic_subtract_int(&ve->refs, 1);
529 lwkt_reltoken(&vkp->token);
536 * Red black tree functions
538 static int rb_vmspace_compare(struct vmspace_entry *, struct vmspace_entry *);
539 RB_GENERATE(vmspace_rb_tree, vmspace_entry, rb_entry, rb_vmspace_compare);
542 * a->start is address, and the only field has to be initialized.
543 * The caller must hold vkp->token.
545 * The caller must hold vkp->token.
548 rb_vmspace_compare(struct vmspace_entry *a, struct vmspace_entry *b)
550 if ((char *)a->id < (char *)b->id)
552 else if ((char *)a->id > (char *)b->id)
558 * The caller must hold vkp->token.
562 rb_vmspace_delete(struct vmspace_entry *ve, void *data)
564 struct vkernel_proc *vkp = data;
566 KKASSERT(ve->refs == 0);
567 vmspace_entry_delete(ve, vkp);
572 * Remove a vmspace_entry from the RB tree and destroy it. We have to clean
573 * up the pmap, the vm_map, then destroy the vmspace.
575 * This function must remove the ve immediately before it might potentially
578 * The caller must hold vkp->token.
582 vmspace_entry_delete(struct vmspace_entry *ve, struct vkernel_proc *vkp)
584 RB_REMOVE(vmspace_rb_tree, &vkp->root, ve);
586 pmap_remove_pages(vmspace_pmap(ve->vmspace),
587 VM_MIN_USER_ADDRESS, VM_MAX_USER_ADDRESS);
588 vm_map_remove(&ve->vmspace->vm_map,
589 VM_MIN_USER_ADDRESS, VM_MAX_USER_ADDRESS);
590 sysref_put(&ve->vmspace->vm_sysref);
591 kfree(ve, M_VKERNEL);
595 * Locate the ve for (id), return the ve or NULL. If found this function
596 * will bump ve->refs which prevents the ve from being immediately destroyed
597 * (but it can still be removed).
599 * The caller must hold vkp->token.
602 struct vmspace_entry *
603 vkernel_find_vmspace(struct vkernel_proc *vkp, void *id)
605 struct vmspace_entry *ve;
606 struct vmspace_entry key;
609 ve = RB_FIND(vmspace_rb_tree, &vkp->root, &key);
614 * Manage vkernel refs, used by the kernel when fork()ing or exit()ing
620 vkernel_inherit(struct proc *p1, struct proc *p2)
622 struct vkernel_proc *vkp;
625 KKASSERT(vkp->refs > 0);
626 atomic_add_int(&vkp->refs, 1);
634 vkernel_exit(struct proc *p)
636 struct vkernel_proc *vkp;
642 * Restore the original VM context if we are killed while running
645 * This isn't supposed to happen. What is supposed to happen is
646 * that the process should enter vkernel_trap() before the handling
649 RB_FOREACH(lp, lwp_rb_tree, &p->p_lwp_tree) {
650 vkernel_lwp_exit(lp);
654 * Dereference the common area
657 KKASSERT(vkp->refs > 0);
659 if (atomic_fetchadd_int(&vkp->refs, -1) == 1) {
660 lwkt_gettoken(&vkp->token);
661 RB_SCAN(vmspace_rb_tree, &vkp->root, NULL,
662 rb_vmspace_delete, vkp);
663 lwkt_reltoken(&vkp->token);
664 kfree(vkp, M_VKERNEL);
672 vkernel_lwp_exit(struct lwp *lp)
674 struct vkernel_lwp *vklp;
675 struct vmspace_entry *ve;
677 if ((vklp = lp->lwp_vkernel) != NULL) {
678 if ((ve = vklp->ve) != NULL) {
679 kprintf("Warning, pid %d killed with "
680 "active VC!\n", lp->lwp_proc->p_pid);
682 pmap_setlwpvm(lp, lp->lwp_proc->p_vmspace);
684 KKASSERT(ve->refs > 0);
685 atomic_subtract_int(&ve->refs, 1);
687 lp->lwp_vkernel = NULL;
688 kfree(vklp, M_VKERNEL);
693 * A VM space under virtual kernel control trapped out or made a system call
694 * or otherwise needs to return control to the virtual kernel context.
699 vkernel_trap(struct lwp *lp, struct trapframe *frame)
701 struct proc *p = lp->lwp_proc;
702 struct vmspace_entry *ve;
703 struct vkernel_lwp *vklp;
707 * Which vmspace entry was running?
709 vklp = lp->lwp_vkernel;
712 KKASSERT(ve != NULL);
715 * Switch the LWP vmspace back to the virtual kernel's VM space.
718 pmap_setlwpvm(lp, p->p_vmspace);
719 KKASSERT(ve->refs > 0);
720 atomic_subtract_int(&ve->refs, 1);
721 /* ve is invalid once we kill our ref */
724 * Copy the emulated process frame to the virtual kernel process.
725 * The emulated process cannot change TLS descriptors so don't
726 * bother saving them, we already have a copy.
728 * Restore the virtual kernel's saved context so the virtual kernel
729 * process can resume.
731 error = copyout(frame, vklp->user_trapframe, sizeof(*frame));
732 bcopy(&vklp->save_trapframe, frame, sizeof(*frame));
733 bcopy(&vklp->save_vextframe.vx_tls, &curthread->td_tls,
734 sizeof(vklp->save_vextframe.vx_tls));
736 cpu_vkernel_trap(frame, error);