3 * Bill Paul <wpaul@windriver.com>. All rights reserved.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 3. All advertising materials mentioning features or use of this software
14 * must display the following acknowledgement:
15 * This product includes software developed by Bill Paul.
16 * 4. Neither the name of the author nor the names of any co-contributors
17 * may be used to endorse or promote products derived from this software
18 * without specific prior written permission.
20 * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD
24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
30 * THE POSSIBILITY OF SUCH DAMAGE.
32 * $FreeBSD: src/sys/compat/ndis/subr_hal.c,v 1.34 2012/11/17 01:51:26 svnexp Exp $
35 #include <sys/param.h>
36 #include <sys/types.h>
37 #include <sys/errno.h>
39 #include <sys/callout.h>
40 #include <sys/kernel.h>
42 #include <sys/mutex.h>
44 #include <sys/sched.h>
45 #include <sys/module.h>
47 #include <sys/systm.h>
51 #include <sys/thread2.h>
53 #include <emulation/ndis/pe_var.h>
54 #include <emulation/ndis/resource_var.h>
55 #include <emulation/ndis/cfg_var.h>
56 #include <emulation/ndis/ntoskrnl_var.h>
57 #include <emulation/ndis/hal_var.h>
59 static void KeStallExecutionProcessor(uint32_t);
60 static void WRITE_PORT_BUFFER_ULONG(uint32_t *,
61 uint32_t *, uint32_t);
62 static void WRITE_PORT_BUFFER_USHORT(uint16_t *,
63 uint16_t *, uint32_t);
64 static void WRITE_PORT_BUFFER_UCHAR(uint8_t *,
66 static void WRITE_PORT_ULONG(uint32_t *, uint32_t);
67 static void WRITE_PORT_USHORT(uint16_t *, uint16_t);
68 static void WRITE_PORT_UCHAR(uint8_t *, uint8_t);
69 static uint32_t READ_PORT_ULONG(uint32_t *);
70 static uint16_t READ_PORT_USHORT(uint16_t *);
71 static uint8_t READ_PORT_UCHAR(uint8_t *);
72 static void READ_PORT_BUFFER_ULONG(uint32_t *,
73 uint32_t *, uint32_t);
74 static void READ_PORT_BUFFER_USHORT(uint16_t *,
75 uint16_t *, uint32_t);
76 static void READ_PORT_BUFFER_UCHAR(uint8_t *,
78 static uint64_t KeQueryPerformanceCounter(uint64_t *);
79 static void _KeLowerIrql(uint8_t);
80 static uint8_t KeRaiseIrqlToDpcLevel(void);
81 static void dummy (void);
83 #define NDIS_MAXCPUS 64
84 static struct lock disp_lock[NDIS_MAXCPUS];
89 image_patch_table *patch;
92 for (i = 0; i < NDIS_MAXCPUS; i++)
93 lockinit(&disp_lock[i], "HAL preemption lock", 0,
97 while (patch->ipt_func != NULL) {
98 windrv_wrap((funcptr)patch->ipt_func,
99 (funcptr *)&patch->ipt_wrap,
100 patch->ipt_argcnt, patch->ipt_ftype);
110 image_patch_table *patch;
113 for (i = 0; i < NDIS_MAXCPUS; i++)
114 lockuninit(&disp_lock[i]);
117 while (patch->ipt_func != NULL) {
118 windrv_unwrap(patch->ipt_wrap);
126 KeStallExecutionProcessor(uint32_t usecs)
132 WRITE_PORT_ULONG(uint32_t *port, uint32_t val)
134 bus_space_write_4(NDIS_BUS_SPACE_IO, 0x0, (bus_size_t)port, val);
138 WRITE_PORT_USHORT(uint16_t *port, uint16_t val)
140 bus_space_write_2(NDIS_BUS_SPACE_IO, 0x0, (bus_size_t)port, val);
144 WRITE_PORT_UCHAR(uint8_t *port, uint8_t val)
146 bus_space_write_1(NDIS_BUS_SPACE_IO, 0x0, (bus_size_t)port, val);
150 WRITE_PORT_BUFFER_ULONG(uint32_t *port, uint32_t *val, uint32_t cnt)
152 bus_space_write_multi_4(NDIS_BUS_SPACE_IO, 0x0,
153 (bus_size_t)port, val, cnt);
157 WRITE_PORT_BUFFER_USHORT(uint16_t *port, uint16_t *val, uint32_t cnt)
159 bus_space_write_multi_2(NDIS_BUS_SPACE_IO, 0x0,
160 (bus_size_t)port, val, cnt);
164 WRITE_PORT_BUFFER_UCHAR(uint8_t *port, uint8_t *val, uint32_t cnt)
166 bus_space_write_multi_1(NDIS_BUS_SPACE_IO, 0x0,
167 (bus_size_t)port, val, cnt);
171 READ_PORT_USHORT(uint16_t *port)
173 return (bus_space_read_2(NDIS_BUS_SPACE_IO, 0x0, (bus_size_t)port));
177 READ_PORT_ULONG(uint32_t *port)
179 return (bus_space_read_4(NDIS_BUS_SPACE_IO, 0x0, (bus_size_t)port));
183 READ_PORT_UCHAR(uint8_t *port)
185 return (bus_space_read_1(NDIS_BUS_SPACE_IO, 0x0, (bus_size_t)port));
189 READ_PORT_BUFFER_ULONG(uint32_t *port, uint32_t *val, uint32_t cnt)
191 bus_space_read_multi_4(NDIS_BUS_SPACE_IO, 0x0,
192 (bus_size_t)port, val, cnt);
196 READ_PORT_BUFFER_USHORT(uint16_t *port, uint16_t *val, uint32_t cnt)
198 bus_space_read_multi_2(NDIS_BUS_SPACE_IO, 0x0,
199 (bus_size_t)port, val, cnt);
203 READ_PORT_BUFFER_UCHAR(uint8_t *port, uint8_t *val, uint32_t cnt)
205 bus_space_read_multi_1(NDIS_BUS_SPACE_IO, 0x0,
206 (bus_size_t)port, val, cnt);
210 * The spinlock implementation in Windows differs from that of FreeBSD.
211 * The basic operation of spinlocks involves two steps: 1) spin in a
212 * tight loop while trying to acquire a lock, 2) after obtaining the
213 * lock, disable preemption. (Note that on uniprocessor systems, you're
214 * allowed to skip the first step and just lock out pre-emption, since
215 * it's not possible for you to be in contention with another running
216 * thread.) Later, you release the lock then re-enable preemption.
217 * The difference between Windows and FreeBSD lies in how preemption
218 * is disabled. In FreeBSD, it's done using critical_enter(), which on
219 * the x86 arch translates to a cli instruction. This masks off all
220 * interrupts, and effectively stops the scheduler from ever running
221 * so _nothing_ can execute except the current thread. In Windows,
222 * preemption is disabled by raising the processor IRQL to DISPATCH_LEVEL.
223 * This stops other threads from running, but does _not_ block device
224 * interrupts. This means ISRs can still run, and they can make other
225 * threads runable, but those other threads won't be able to execute
226 * until the current thread lowers the IRQL to something less than
229 * There's another commonly used IRQL in Windows, which is APC_LEVEL.
230 * An APC is an Asynchronous Procedure Call, which differs from a DPC
231 * (Defered Procedure Call) in that a DPC is queued up to run in
232 * another thread, while an APC runs in the thread that scheduled
233 * it (similar to a signal handler in a UNIX process). We don't
234 * actually support the notion of APCs in FreeBSD, so for now, the
235 * only IRQLs we're interested in are DISPATCH_LEVEL and PASSIVE_LEVEL.
237 * To simulate DISPATCH_LEVEL, we raise the current thread's priority
238 * to TDPRI_INT_HIGH, which is the highest we can give it. This should,
239 * if I understand things correctly, prevent anything except for an
240 * interrupt thread from preempting us. PASSIVE_LEVEL is basically
243 * Be aware that, at least on the x86 arch, the Windows spinlock
244 * functions are divided up in peculiar ways. The actual spinlock
245 * functions are KfAcquireSpinLock() and KfReleaseSpinLock(), and
246 * they live in HAL.dll. Meanwhile, KeInitializeSpinLock(),
247 * KefAcquireSpinLockAtDpcLevel() and KefReleaseSpinLockFromDpcLevel()
248 * live in ntoskrnl.exe. Most Windows source code will call
249 * KeAcquireSpinLock() and KeReleaseSpinLock(), but these are just
250 * macros that call KfAcquireSpinLock() and KfReleaseSpinLock().
251 * KefAcquireSpinLockAtDpcLevel() and KefReleaseSpinLockFromDpcLevel()
252 * perform the lock aquisition/release functions without doing the
253 * IRQL manipulation, and are used when one is already running at
254 * DISPATCH_LEVEL. Make sense? Good.
256 * According to the Microsoft documentation, any thread that calls
257 * KeAcquireSpinLock() must be running at IRQL <= DISPATCH_LEVEL. If
258 * we detect someone trying to acquire a spinlock from DEVICE_LEVEL
259 * or HIGH_LEVEL, we panic.
261 * Alternate sleep-lock-based spinlock implementation
262 * --------------------------------------------------
264 * The earlier spinlock implementation was arguably a bit of a hack
265 * and presented several problems. It was basically designed to provide
266 * the functionality of spinlocks without incurring the wrath of
267 * WITNESS. We could get away with using both our spinlock implementation
268 * and FreeBSD sleep locks at the same time, but if WITNESS knew what
269 * we were really up to, it would have spanked us rather severely.
271 * There's another method we can use based entirely on sleep locks.
272 * First, it's important to realize that everything we're locking
273 * resides inside Project Evil itself: any critical data being locked
274 * by drivers belongs to the drivers, and should not be referenced
275 * by any other OS code outside of the NDISulator. The priority-based
276 * locking scheme has system-wide effects, just like real spinlocks
277 * (blocking preemption affects the whole CPU), but since we keep all
278 * our critical data private, we can use a simpler mechanism that
279 * affects only code/threads directly related to Project Evil.
281 * The idea is to create a sleep lock mutex for each CPU in the system.
282 * When a CPU running in the NDISulator wants to acquire a spinlock, it
283 * does the following:
284 * - Pin ourselves to the current CPU
285 * - Acquire the mutex for the current CPU
286 * - Spin on the spinlock variable using atomic test and set, just like
288 * - Once we have the lock, we execute our critical code
290 * To give up the lock, we do:
291 * - Clear the spinlock variable with an atomic op
292 * - Release the per-CPU mutex
293 * - Unpin ourselves from the current CPU.
295 * On a uniprocessor system, this means all threads that access protected
296 * data are serialized through the per-CPU mutex. After one thread
297 * acquires the 'spinlock,' any other thread that uses a spinlock on the
298 * current CPU will block on the per-CPU mutex, which has the same general
299 * effect of blocking pre-emption, but _only_ for those threads that are
300 * running NDISulator code.
302 * On a multiprocessor system, threads on different CPUs all block on
303 * their respective per-CPU mutex, and the atomic test/set operation
304 * on the spinlock variable provides inter-CPU synchronization, though
305 * only for threads running NDISulator code.
307 * This method solves an important problem. In Windows, you're allowed
308 * to do an ExAllocatePoolWithTag() with a spinlock held, provided you
309 * allocate from NonPagedPool. This implies an atomic heap allocation
310 * that will not cause the current thread to sleep. (You can't sleep
311 * while holding real spinlock: clowns will eat you.) But in FreeBSD,
312 * malloc(9) _always_ triggers the acquisition of a sleep lock, even
313 * when you use M_NOWAIT. This is not a problem for FreeBSD native
314 * code: you're allowed to sleep in things like interrupt threads. But
315 * it is a problem with the old priority-based spinlock implementation:
316 * even though we get away with it most of the time, we really can't
317 * do a malloc(9) after doing a KeAcquireSpinLock() or KeRaiseIrql().
318 * With the new implementation, it's not a problem: you're allowed to
319 * acquire more than one sleep lock (as long as you avoid lock order
322 * The one drawback to this approach is that now we have a lot of
323 * contention on one per-CPU mutex within the NDISulator code. Whether
324 * or not this is preferable to the expected Windows spinlock behavior
325 * of blocking pre-emption is debatable.
329 KfAcquireSpinLock(kspin_lock *lock)
333 KeRaiseIrql(DISPATCH_LEVEL, &oldirql);
334 KeAcquireSpinLockAtDpcLevel(lock);
340 KfReleaseSpinLock(kspin_lock *lock, uint8_t newirql)
342 KeReleaseSpinLockFromDpcLevel(lock);
343 KeLowerIrql(newirql);
347 KeGetCurrentIrql(void)
349 if (lockstatus(&disp_lock[curthread->td_gd->gd_cpuid], curthread))
350 return (DISPATCH_LEVEL);
351 return (PASSIVE_LEVEL);
355 KeQueryPerformanceCounter(uint64_t *freq)
360 return ((uint64_t)ticks);
364 KfRaiseIrql(uint8_t irql)
368 #if 0 /* XXX swildner */
371 oldirql = KeGetCurrentIrql();
373 /* I am so going to hell for this. */
375 panic("IRQL_NOT_LESS_THAN_OR_EQUAL");
377 if (oldirql != DISPATCH_LEVEL)
378 lockmgr(&disp_lock[curthread->td_gd->gd_cpuid], LK_EXCLUSIVE);
379 #if 0 /* XXX swildner */
384 /*kprintf("RAISE IRQL: %d %d\n", irql, oldirql);*/
390 KfLowerIrql(uint8_t oldirql)
392 if (oldirql == DISPATCH_LEVEL)
395 if (KeGetCurrentIrql() != DISPATCH_LEVEL)
396 panic("IRQL_NOT_GREATER_THAN");
398 lockmgr(&disp_lock[curthread->td_gd->gd_cpuid], LK_RELEASE);
399 #if 0 /* XXX swildner */
405 KeRaiseIrqlToDpcLevel(void)
409 KeRaiseIrql(DISPATCH_LEVEL, &irql);
414 _KeLowerIrql(uint8_t oldirql)
416 KeLowerIrql(oldirql);
419 static void dummy(void)
421 kprintf("hal dummy called...\n");
424 image_patch_table hal_functbl[] = {
425 IMPORT_SFUNC(KeStallExecutionProcessor, 1),
426 IMPORT_SFUNC(WRITE_PORT_ULONG, 2),
427 IMPORT_SFUNC(WRITE_PORT_USHORT, 2),
428 IMPORT_SFUNC(WRITE_PORT_UCHAR, 2),
429 IMPORT_SFUNC(WRITE_PORT_BUFFER_ULONG, 3),
430 IMPORT_SFUNC(WRITE_PORT_BUFFER_USHORT, 3),
431 IMPORT_SFUNC(WRITE_PORT_BUFFER_UCHAR, 3),
432 IMPORT_SFUNC(READ_PORT_ULONG, 1),
433 IMPORT_SFUNC(READ_PORT_USHORT, 1),
434 IMPORT_SFUNC(READ_PORT_UCHAR, 1),
435 IMPORT_SFUNC(READ_PORT_BUFFER_ULONG, 3),
436 IMPORT_SFUNC(READ_PORT_BUFFER_USHORT, 3),
437 IMPORT_SFUNC(READ_PORT_BUFFER_UCHAR, 3),
438 IMPORT_FFUNC(KfAcquireSpinLock, 1),
439 IMPORT_FFUNC(KfReleaseSpinLock, 1),
440 IMPORT_SFUNC(KeGetCurrentIrql, 0),
441 IMPORT_SFUNC(KeQueryPerformanceCounter, 1),
442 IMPORT_FFUNC(KfLowerIrql, 1),
443 IMPORT_FFUNC(KfRaiseIrql, 1),
444 IMPORT_SFUNC(KeRaiseIrqlToDpcLevel, 0),
446 IMPORT_SFUNC_MAP(KeLowerIrql, _KeLowerIrql, 1),
449 * This last entry is a catch-all for any function we haven't
450 * implemented yet. The PE import list patching routine will
451 * use it for any function that doesn't have an explicit match
455 { NULL, (FUNC)dummy, NULL, 0, WINDRV_WRAP_STDCALL },