kernel: Make SMP support default (and non-optional).
[dragonfly.git] / sys / platform / pc32 / i386 / pmap_inval.c
CommitLineData
0f7a3396 1/*
d5b2d319 2 * Copyright (c) 2003-2011 The DragonFly Project. All rights reserved.
8c10bfcf
MD
3 *
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
6 *
0f7a3396
MD
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
8c10bfcf 10 *
0f7a3396
MD
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
8c10bfcf
MD
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
0f7a3396 32 * SUCH DAMAGE.
0f7a3396
MD
33 */
34
35/*
36 * pmap invalidation support code. Certain hardware requirements must
37 * be dealt with when manipulating page table entries and page directory
38 * entries within a pmap. In particular, we cannot safely manipulate
39 * page tables which are in active use by another cpu (even if it is
40 * running in userland) for two reasons: First, TLB writebacks will
41 * race against our own modifications and tests. Second, even if we
42 * were to use bus-locked instruction we can still screw up the
43 * target cpu's instruction pipeline due to Intel cpu errata.
44 */
45
46#include <sys/param.h>
47#include <sys/systm.h>
48#include <sys/kernel.h>
49#include <sys/proc.h>
50#include <sys/vmmeter.h>
51#include <sys/thread2.h>
52
53#include <vm/vm.h>
54#include <vm/pmap.h>
55#include <vm/vm_object.h>
56
57#include <machine/cputypes.h>
58#include <machine/md_var.h>
59#include <machine/specialreg.h>
0f7a3396 60#include <machine/smp.h>
0f7a3396
MD
61#include <machine/globaldata.h>
62#include <machine/pmap.h>
63#include <machine/pmap_inval.h>
64
d5b2d319 65static void pmap_inval_callback(void *arg);
0f7a3396
MD
66
67/*
68 * Initialize for add or flush
69 */
70void
71pmap_inval_init(pmap_inval_info_t info)
72{
73 info->pir_flags = 0;
c2fb025d 74 crit_enter_id("inval");
0f7a3396
MD
75}
76
77/*
78 * Add a (pmap, va) pair to the invalidation list and protect access
79 * as appropriate.
c2fb025d
MD
80 *
81 * CPUMASK_LOCK is used to interlock thread switchins
0f7a3396
MD
82 */
83void
c2fb025d 84pmap_inval_interlock(pmap_inval_info_t info, pmap_t pmap, vm_offset_t va)
0f7a3396 85{
c2fb025d
MD
86 cpumask_t oactive;
87 cpumask_t nactive;
88
cfaeae2a 89 DEBUG_PUSH_INFO("pmap_inval_interlock");
c2fb025d
MD
90 for (;;) {
91 oactive = pmap->pm_active & ~CPUMASK_LOCK;
92 nactive = oactive | CPUMASK_LOCK;
da23a592 93 if (atomic_cmpset_cpumask(&pmap->pm_active, oactive, nactive))
c2fb025d 94 break;
c2fb025d 95 lwkt_process_ipiq();
d5b2d319 96 cpu_pause();
0f7a3396 97 }
cfaeae2a 98 DEBUG_POP_INFO();
d5b2d319
MD
99 KKASSERT((info->pir_flags & PIRF_CPUSYNC) == 0);
100 info->pir_va = va;
101 info->pir_flags = PIRF_CPUSYNC;
102 lwkt_cpusync_init(&info->pir_cpusync, oactive, pmap_inval_callback, info);
103 lwkt_cpusync_interlock(&info->pir_cpusync);
0f7a3396
MD
104}
105
c2fb025d
MD
106void
107pmap_inval_deinterlock(pmap_inval_info_t info, pmap_t pmap)
108{
d5b2d319 109 KKASSERT(info->pir_flags & PIRF_CPUSYNC);
da23a592 110 atomic_clear_cpumask(&pmap->pm_active, CPUMASK_LOCK);
d5b2d319
MD
111 lwkt_cpusync_deinterlock(&info->pir_cpusync);
112 info->pir_flags = 0;
c2fb025d
MD
113}
114
d5b2d319
MD
115static void
116pmap_inval_callback(void *arg)
0f7a3396 117{
d5b2d319
MD
118 pmap_inval_info_t info = arg;
119
120 if (info->pir_va == (vm_offset_t)-1)
0f7a3396 121 cpu_invltlb();
d5b2d319
MD
122 else
123 cpu_invlpg((void *)info->pir_va);
0f7a3396
MD
124}
125
c2fb025d
MD
126void
127pmap_inval_done(pmap_inval_info_t info)
128{
d5b2d319
MD
129 KKASSERT((info->pir_flags & PIRF_CPUSYNC) == 0);
130 crit_exit_id("inval");
c2fb025d
MD
131}
132