kernel: Make SMP support default (and non-optional).
[dragonfly.git] / sys / platform / pc64 / x86_64 / pmap_inval.c
CommitLineData
c8fe38ae 1/*
d5b2d319 2 * Copyright (c) 2003-2011 The DragonFly Project. All rights reserved.
c8fe38ae
MD
3 *
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
c8fe38ae
MD
33 */
34
35/*
36 * pmap invalidation support code. Certain hardware requirements must
37 * be dealt with when manipulating page table entries and page directory
38 * entries within a pmap. In particular, we cannot safely manipulate
39 * page tables which are in active use by another cpu (even if it is
40 * running in userland) for two reasons: First, TLB writebacks will
41 * race against our own modifications and tests. Second, even if we
42 * were to use bus-locked instruction we can still screw up the
43 * target cpu's instruction pipeline due to Intel cpu errata.
44 */
45
46#include <sys/param.h>
47#include <sys/systm.h>
48#include <sys/kernel.h>
49#include <sys/proc.h>
50#include <sys/vmmeter.h>
51#include <sys/thread2.h>
52
53#include <vm/vm.h>
54#include <vm/pmap.h>
55#include <vm/vm_object.h>
56
57#include <machine/cputypes.h>
58#include <machine/md_var.h>
59#include <machine/specialreg.h>
60#include <machine/smp.h>
61#include <machine/globaldata.h>
62#include <machine/pmap.h>
63#include <machine/pmap_inval.h>
64
d5b2d319 65static void pmap_inval_callback(void *arg);
c8fe38ae
MD
66
67/*
68 * Initialize for add or flush
d5b2d319
MD
69 *
70 * The critical section is required to prevent preemption, allowing us to
71 * set CPUMASK_LOCK on the pmap. The critical section is also assumed
72 * when lwkt_process_ipiq() is called.
c8fe38ae
MD
73 */
74void
75pmap_inval_init(pmap_inval_info_t info)
76{
77 info->pir_flags = 0;
c2fb025d 78 crit_enter_id("inval");
c8fe38ae
MD
79}
80
81/*
82 * Add a (pmap, va) pair to the invalidation list and protect access
83 * as appropriate.
c2fb025d 84 *
d5b2d319
MD
85 * CPUMASK_LOCK is used to interlock thread switchins, otherwise another
86 * cpu can switch in a pmap that we are unaware of and interfere with our
87 * pte operation.
c8fe38ae
MD
88 */
89void
c2fb025d 90pmap_inval_interlock(pmap_inval_info_t info, pmap_t pmap, vm_offset_t va)
c8fe38ae 91{
c2fb025d
MD
92 cpumask_t oactive;
93 cpumask_t nactive;
94
cfaeae2a 95 DEBUG_PUSH_INFO("pmap_inval_interlock");
c2fb025d 96 for (;;) {
54341a3b
MD
97 oactive = pmap->pm_active;
98 cpu_ccfence();
c2fb025d 99 nactive = oactive | CPUMASK_LOCK;
54341a3b
MD
100 if ((oactive & CPUMASK_LOCK) == 0 &&
101 atomic_cmpset_cpumask(&pmap->pm_active, oactive, nactive)) {
c2fb025d 102 break;
54341a3b 103 }
c2fb025d 104 lwkt_process_ipiq();
d5b2d319 105 cpu_pause();
c8fe38ae 106 }
cfaeae2a 107 DEBUG_POP_INFO();
d5b2d319
MD
108 KKASSERT((info->pir_flags & PIRF_CPUSYNC) == 0);
109 info->pir_va = va;
110 info->pir_flags = PIRF_CPUSYNC;
111 lwkt_cpusync_init(&info->pir_cpusync, oactive, pmap_inval_callback, info);
112 lwkt_cpusync_interlock(&info->pir_cpusync);
c8fe38ae
MD
113}
114
921c891e
MD
115void
116pmap_inval_invltlb(pmap_inval_info_t info)
117{
118 info->pir_va = (vm_offset_t)-1;
119}
120
c2fb025d
MD
121void
122pmap_inval_deinterlock(pmap_inval_info_t info, pmap_t pmap)
123{
d5b2d319 124 KKASSERT(info->pir_flags & PIRF_CPUSYNC);
d5b2d319 125 atomic_clear_cpumask(&pmap->pm_active, CPUMASK_LOCK);
d5b2d319
MD
126 lwkt_cpusync_deinterlock(&info->pir_cpusync);
127 info->pir_flags = 0;
c2fb025d
MD
128}
129
d5b2d319
MD
130static void
131pmap_inval_callback(void *arg)
c8fe38ae 132{
d5b2d319
MD
133 pmap_inval_info_t info = arg;
134
135 if (info->pir_va == (vm_offset_t)-1)
c8fe38ae 136 cpu_invltlb();
d5b2d319
MD
137 else
138 cpu_invlpg((void *)info->pir_va);
c8fe38ae
MD
139}
140
c2fb025d
MD
141void
142pmap_inval_done(pmap_inval_info_t info)
143{
d5b2d319 144 KKASSERT((info->pir_flags & PIRF_CPUSYNC) == 0);
c2fb025d
MD
145 crit_exit_id("inval");
146}
147