kernel - unwind kthread_create() mplock
[dragonfly.git] / sys / emulation / ndis / subr_ntoskrnl.c
1 /*
2  * Copyright (c) 2003
3  *      Bill Paul <wpaul@windriver.com>.  All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  * 3. All advertising materials mentioning features or use of this software
14  *    must display the following acknowledgement:
15  *      This product includes software developed by Bill Paul.
16  * 4. Neither the name of the author nor the names of any co-contributors
17  *    may be used to endorse or promote products derived from this software
18  *    without specific prior written permission.
19  *
20  * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND
21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23  * ARE DISCLAIMED.  IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD
24  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
30  * THE POSSIBILITY OF SUCH DAMAGE.
31  *
32  * $FreeBSD: src/sys/compat/ndis/subr_ntoskrnl.c,v 1.40 2004/07/20 20:28:57 wpaul Exp $
33  * $DragonFly: src/sys/emulation/ndis/subr_ntoskrnl.c,v 1.13 2006/12/23 00:27:02 swildner Exp $
34  */
35
36 #include <sys/ctype.h>
37 #include <sys/unistd.h>
38 #include <sys/param.h>
39 #include <sys/types.h>
40 #include <sys/errno.h>
41 #include <sys/systm.h>
42 #include <sys/malloc.h>
43 #include <sys/lock.h>
44
45 #include <sys/callout.h>
46 #if __FreeBSD_version > 502113
47 #include <sys/kdb.h>
48 #endif
49 #include <sys/kernel.h>
50 #include <sys/proc.h>
51 #include <sys/kthread.h>
52 #include <sys/bus.h>
53 #include <sys/rman.h>
54
55 #include <sys/mplock2.h>
56
57 #include <machine/atomic.h>
58 #include <machine/clock.h>
59 #include <machine/stdarg.h>
60
61 #include "regcall.h"
62 #include "pe_var.h"
63 #include "resource_var.h"
64 #include "ntoskrnl_var.h"
65 #include "ndis_var.h"
66 #include "hal_var.h"
67
68 #define __regparm __attribute__((regparm(3)))
69
70 #define FUNC void(*)(void)
71
72 __stdcall static uint8_t ntoskrnl_unicode_equal(ndis_unicode_string *,
73         ndis_unicode_string *, uint8_t);
74 __stdcall static void ntoskrnl_unicode_copy(ndis_unicode_string *,
75         ndis_unicode_string *);
76 __stdcall static ndis_status ntoskrnl_unicode_to_ansi(ndis_ansi_string *,
77         ndis_unicode_string *, uint8_t);
78 __stdcall static ndis_status ntoskrnl_ansi_to_unicode(ndis_unicode_string *,
79         ndis_ansi_string *, uint8_t);
80 __stdcall static void *ntoskrnl_iobuildsynchfsdreq(uint32_t, void *,
81         void *, uint32_t, uint32_t *, void *, void *);
82
83 /*
84  * registerized calls
85  */
86 __stdcall __regcall static uint32_t
87     ntoskrnl_iofcalldriver(REGARGS2(void *dobj, void *irp));
88 __stdcall __regcall static void
89     ntoskrnl_iofcompletereq(REGARGS2(void *irp, uint8_t prioboost));
90 __stdcall __regcall static slist_entry *
91     ntoskrnl_push_slist(REGARGS2(slist_header *head, slist_entry *entry));
92 __stdcall __regcall static slist_entry *
93     ntoskrnl_pop_slist(REGARGS1(slist_header *head));
94 __stdcall __regcall static slist_entry *
95     ntoskrnl_push_slist_ex(REGARGS2(slist_header *head, slist_entry *entry), kspin_lock *lock);
96 __stdcall __regcall static slist_entry *
97     ntoskrnl_pop_slist_ex(REGARGS2(slist_header *head, kspin_lock *lock));
98
99 __stdcall __regcall static uint32_t
100     ntoskrnl_interlock_inc(REGARGS1(volatile uint32_t *addend));
101 __stdcall __regcall static uint32_t
102     ntoskrnl_interlock_dec(REGARGS1(volatile uint32_t *addend));
103 __stdcall __regcall static void
104     ntoskrnl_interlock_addstat(REGARGS2(uint64_t *addend, uint32_t inc));
105 __stdcall __regcall static void
106     ntoskrnl_objderef(REGARGS1(void *object));
107
108 __stdcall static uint32_t ntoskrnl_waitforobjs(uint32_t,
109         nt_dispatch_header **, uint32_t, uint32_t, uint32_t, uint8_t,
110         int64_t *, wait_block *);
111 static void ntoskrnl_wakeup(void *);
112 static void ntoskrnl_timercall(void *);
113 static void ntoskrnl_run_dpc(void *);
114 __stdcall static void ntoskrnl_writereg_ushort(uint16_t *, uint16_t);
115 __stdcall static uint16_t ntoskrnl_readreg_ushort(uint16_t *);
116 __stdcall static void ntoskrnl_writereg_ulong(uint32_t *, uint32_t);
117 __stdcall static uint32_t ntoskrnl_readreg_ulong(uint32_t *);
118 __stdcall static void ntoskrnl_writereg_uchar(uint8_t *, uint8_t);
119 __stdcall static uint8_t ntoskrnl_readreg_uchar(uint8_t *);
120 __stdcall static int64_t _allmul(int64_t, int64_t);
121 __stdcall static int64_t _alldiv(int64_t, int64_t);
122 __stdcall static int64_t _allrem(int64_t, int64_t);
123 __regparm static int64_t _allshr(int64_t, uint8_t);
124 __regparm static int64_t _allshl(int64_t, uint8_t);
125 __stdcall static uint64_t _aullmul(uint64_t, uint64_t);
126 __stdcall static uint64_t _aulldiv(uint64_t, uint64_t);
127 __stdcall static uint64_t _aullrem(uint64_t, uint64_t);
128 __regparm static uint64_t _aullshr(uint64_t, uint8_t);
129 __regparm static uint64_t _aullshl(uint64_t, uint8_t);
130 __stdcall static void *ntoskrnl_allocfunc(uint32_t, size_t, uint32_t);
131 __stdcall static void ntoskrnl_freefunc(void *);
132 static slist_entry *ntoskrnl_pushsl(slist_header *, slist_entry *);
133 static slist_entry *ntoskrnl_popsl(slist_header *);
134 __stdcall static void ntoskrnl_init_lookaside(paged_lookaside_list *,
135         lookaside_alloc_func *, lookaside_free_func *,
136         uint32_t, size_t, uint32_t, uint16_t);
137 __stdcall static void ntoskrnl_delete_lookaside(paged_lookaside_list *);
138 __stdcall static void ntoskrnl_init_nplookaside(npaged_lookaside_list *,
139         lookaside_alloc_func *, lookaside_free_func *,
140         uint32_t, size_t, uint32_t, uint16_t);
141 __stdcall static void ntoskrnl_delete_nplookaside(npaged_lookaside_list *);
142 __stdcall static void ntoskrnl_freemdl(ndis_buffer *);
143 __stdcall static uint32_t ntoskrnl_sizeofmdl(void *, size_t);
144 __stdcall static void ntoskrnl_build_npaged_mdl(ndis_buffer *);
145 __stdcall static void *ntoskrnl_mmaplockedpages(ndis_buffer *, uint8_t);
146 __stdcall static void *ntoskrnl_mmaplockedpages_cache(ndis_buffer *,
147         uint8_t, uint32_t, void *, uint32_t, uint32_t);
148 __stdcall static void ntoskrnl_munmaplockedpages(void *, ndis_buffer *);
149 __stdcall static void ntoskrnl_init_lock(kspin_lock *);
150 __stdcall static size_t ntoskrnl_memcmp(const void *, const void *, size_t);
151 __stdcall static void ntoskrnl_init_ansi_string(ndis_ansi_string *, char *);
152 __stdcall static void ntoskrnl_init_unicode_string(ndis_unicode_string *,
153         uint16_t *);
154 __stdcall static void ntoskrnl_free_unicode_string(ndis_unicode_string *);
155 __stdcall static void ntoskrnl_free_ansi_string(ndis_ansi_string *);
156 __stdcall static ndis_status ntoskrnl_unicode_to_int(ndis_unicode_string *,
157         uint32_t, uint32_t *);
158 static int atoi (const char *);
159 static long atol (const char *);
160 static int rand(void);
161 static void ntoskrnl_time(uint64_t *);
162 __stdcall static uint8_t ntoskrnl_wdmver(uint8_t, uint8_t);
163 static void ntoskrnl_thrfunc(void *);
164 __stdcall static ndis_status ntoskrnl_create_thread(ndis_handle *,
165         uint32_t, void *, ndis_handle, void *, void *, void *);
166 __stdcall static ndis_status ntoskrnl_thread_exit(ndis_status);
167 __stdcall static ndis_status ntoskrnl_devprop(device_object *, uint32_t,
168         uint32_t, void *, uint32_t *);
169 __stdcall static void ntoskrnl_init_mutex(kmutant *, uint32_t);
170 __stdcall static uint32_t ntoskrnl_release_mutex(kmutant *, uint8_t);
171 __stdcall static uint32_t ntoskrnl_read_mutex(kmutant *);
172 __stdcall static ndis_status ntoskrnl_objref(ndis_handle, uint32_t, void *,
173     uint8_t, void **, void **);
174 __stdcall static uint32_t ntoskrnl_zwclose(ndis_handle);
175 static uint32_t ntoskrnl_dbgprint(char *, ...);
176 __stdcall static void ntoskrnl_debugger(void);
177 __stdcall static void dummy(void);
178
179 static struct lwkt_token ntoskrnl_dispatchtoken;
180 static kspin_lock ntoskrnl_global;
181 static int ntoskrnl_kth = 0;
182 static struct nt_objref_head ntoskrnl_reflist;
183
184 static MALLOC_DEFINE(M_NDIS, "ndis", "ndis emulation");
185
186 int
187 ntoskrnl_libinit(void)
188 {
189         lwkt_token_init(&ntoskrnl_dispatchtoken, 1, "ndiskrnl");
190         ntoskrnl_init_lock(&ntoskrnl_global);
191         TAILQ_INIT(&ntoskrnl_reflist);
192         return(0);
193 }
194
195 int
196 ntoskrnl_libfini(void)
197 {
198         lwkt_token_uninit(&ntoskrnl_dispatchtoken);
199         return(0);
200 }
201
202 __stdcall static uint8_t 
203 ntoskrnl_unicode_equal(ndis_unicode_string *str1,
204                        ndis_unicode_string *str2,
205                        uint8_t caseinsensitive)
206 {
207         int                     i;
208
209         if (str1->nus_len != str2->nus_len)
210                 return(FALSE);
211
212         for (i = 0; i < str1->nus_len; i++) {
213                 if (caseinsensitive == TRUE) {
214                         if (toupper((char)(str1->nus_buf[i] & 0xFF)) !=
215                             toupper((char)(str2->nus_buf[i] & 0xFF)))
216                                 return(FALSE);
217                 } else {
218                         if (str1->nus_buf[i] != str2->nus_buf[i])
219                                 return(FALSE);
220                 }
221         }
222
223         return(TRUE);
224 }
225
226 __stdcall static void
227 ntoskrnl_unicode_copy(ndis_unicode_string *dest,
228                       ndis_unicode_string *src)
229 {
230
231         if (dest->nus_maxlen >= src->nus_len)
232                 dest->nus_len = src->nus_len;
233         else
234                 dest->nus_len = dest->nus_maxlen;
235         memcpy(dest->nus_buf, src->nus_buf, dest->nus_len);
236         return;
237 }
238
239 __stdcall static ndis_status
240 ntoskrnl_unicode_to_ansi(ndis_ansi_string *dest,
241                          ndis_unicode_string *src,
242                          uint8_t allocate)
243 {
244         char                    *astr = NULL;
245
246         if (dest == NULL || src == NULL)
247                 return(NDIS_STATUS_FAILURE);
248
249         if (allocate == TRUE) {
250                 if (ndis_unicode_to_ascii(src->nus_buf, src->nus_len, &astr))
251                         return(NDIS_STATUS_FAILURE);
252                 dest->nas_buf = astr;
253                 dest->nas_len = dest->nas_maxlen = strlen(astr);
254         } else {
255                 dest->nas_len = src->nus_len / 2; /* XXX */
256                 if (dest->nas_maxlen < dest->nas_len)
257                         dest->nas_len = dest->nas_maxlen;
258                 ndis_unicode_to_ascii(src->nus_buf, dest->nas_len * 2,
259                     &dest->nas_buf);
260         }
261         return (NDIS_STATUS_SUCCESS);
262 }
263
264 __stdcall static ndis_status
265 ntoskrnl_ansi_to_unicode(ndis_unicode_string *dest,
266                          ndis_ansi_string *src,
267                          uint8_t allocate)
268 {
269         uint16_t                *ustr = NULL;
270
271         if (dest == NULL || src == NULL)
272                 return(NDIS_STATUS_FAILURE);
273
274         if (allocate == TRUE) {
275                 if (ndis_ascii_to_unicode(src->nas_buf, &ustr))
276                         return(NDIS_STATUS_FAILURE);
277                 dest->nus_buf = ustr;
278                 dest->nus_len = dest->nus_maxlen = strlen(src->nas_buf) * 2;
279         } else {
280                 dest->nus_len = src->nas_len * 2; /* XXX */
281                 if (dest->nus_maxlen < dest->nus_len)
282                         dest->nus_len = dest->nus_maxlen;
283                 ndis_ascii_to_unicode(src->nas_buf, &dest->nus_buf);
284         }
285         return (NDIS_STATUS_SUCCESS);
286 }
287
288 __stdcall static void *
289 ntoskrnl_iobuildsynchfsdreq(uint32_t func, void *dobj, void *buf,
290                             uint32_t len, uint32_t *off,
291                             void *event, void *status)
292 {
293         return(NULL);
294 }
295         
296 __stdcall __regcall static uint32_t
297 ntoskrnl_iofcalldriver(REGARGS2(void *dobj, void *irp))
298 {
299         return(0);
300 }
301
302 __stdcall __regcall static void
303 ntoskrnl_iofcompletereq(REGARGS2(void *irp, uint8_t prioboost))
304 {
305 }
306
307 static void
308 ntoskrnl_wakeup(void *arg)
309 {
310         nt_dispatch_header      *obj;
311         wait_block              *w;
312         list_entry              *e;
313         struct thread           *td;
314
315         obj = arg;
316
317         lwkt_gettoken(&ntoskrnl_dispatchtoken);
318         obj->dh_sigstate = TRUE;
319         e = obj->dh_waitlisthead.nle_flink;
320         while (e != &obj->dh_waitlisthead) {
321                 w = (wait_block *)e;
322                 td = w->wb_kthread;
323                 ndis_thresume(td);
324                 /*
325                  * For synchronization objects, only wake up
326                  * the first waiter.
327                  */
328                 if (obj->dh_type == EVENT_TYPE_SYNC)
329                         break;
330                 e = e->nle_flink;
331         }
332         lwkt_reltoken(&ntoskrnl_dispatchtoken);
333 }
334
335 static void 
336 ntoskrnl_time(uint64_t *tval)
337 {
338         struct timespec         ts;
339
340         nanotime(&ts);
341         *tval = (uint64_t)ts.tv_nsec / 100 + (uint64_t)ts.tv_sec * 10000000 +
342             11644473600LL;
343
344         return;
345 }
346
347 /*
348  * KeWaitForSingleObject() is a tricky beast, because it can be used
349  * with several different object types: semaphores, timers, events,
350  * mutexes and threads. Semaphores don't appear very often, but the
351  * other object types are quite common. KeWaitForSingleObject() is
352  * what's normally used to acquire a mutex, and it can be used to
353  * wait for a thread termination.
354  *
355  * The Windows NDIS API is implemented in terms of Windows kernel
356  * primitives, and some of the object manipulation is duplicated in
357  * NDIS. For example, NDIS has timers and events, which are actually
358  * Windows kevents and ktimers. Now, you're supposed to only use the
359  * NDIS variants of these objects within the confines of the NDIS API,
360  * but there are some naughty developers out there who will use
361  * KeWaitForSingleObject() on NDIS timer and event objects, so we
362  * have to support that as well. Conseqently, our NDIS timer and event
363  * code has to be closely tied into our ntoskrnl timer and event code,
364  * just as it is in Windows.
365  *
366  * KeWaitForSingleObject() may do different things for different kinds
367  * of objects:
368  *
369  * - For events, we check if the event has been signalled. If the
370  *   event is already in the signalled state, we just return immediately,
371  *   otherwise we wait for it to be set to the signalled state by someone
372  *   else calling KeSetEvent(). Events can be either synchronization or
373  *   notification events.
374  *
375  * - For timers, if the timer has already fired and the timer is in
376  *   the signalled state, we just return, otherwise we wait on the
377  *   timer. Unlike an event, timers get signalled automatically when
378  *   they expire rather than someone having to trip them manually.
379  *   Timers initialized with KeInitializeTimer() are always notification
380  *   events: KeInitializeTimerEx() lets you initialize a timer as
381  *   either a notification or synchronization event.
382  *
383  * - For mutexes, we try to acquire the mutex and if we can't, we wait
384  *   on the mutex until it's available and then grab it. When a mutex is
385  *   released, it enters the signaled state, which wakes up one of the
386  *   threads waiting to acquire it. Mutexes are always synchronization
387  *   events.
388  *
389  * - For threads, the only thing we do is wait until the thread object
390  *   enters a signalled state, which occurs when the thread terminates.
391  *   Threads are always notification events.
392  *
393  * A notification event wakes up all threads waiting on an object. A
394  * synchronization event wakes up just one. Also, a synchronization event
395  * is auto-clearing, which means we automatically set the event back to
396  * the non-signalled state once the wakeup is done.
397  */
398
399 __stdcall uint32_t
400 ntoskrnl_waitforobj(nt_dispatch_header *obj, uint32_t reason,
401                     uint32_t mode, uint8_t alertable, int64_t *duetime)
402 {
403         struct thread           *td = curthread;
404         kmutant                 *km;
405         wait_block              w;
406         struct timeval          tv;
407         int                     error = 0;
408         int                     ticks;
409         uint64_t                curtime;
410
411         if (obj == NULL)
412                 return(STATUS_INVALID_PARAMETER);
413
414         lwkt_gettoken(&ntoskrnl_dispatchtoken);
415
416         /*
417          * See if the object is a mutex. If so, and we already own
418          * it, then just increment the acquisition count and return.
419          *
420          * For any other kind of object, see if it's already in the
421          * signalled state, and if it is, just return. If the object
422          * is marked as a synchronization event, reset the state to
423          * unsignalled.
424          */
425
426         if (obj->dh_size == OTYPE_MUTEX) {
427                 km = (kmutant *)obj;
428                 if (km->km_ownerthread == NULL ||
429                     km->km_ownerthread == curthread->td_proc) {
430                         obj->dh_sigstate = FALSE;
431                         km->km_acquirecnt++;
432                         km->km_ownerthread = curthread->td_proc;
433                         lwkt_reltoken(&ntoskrnl_dispatchtoken);
434                         return (STATUS_SUCCESS);
435                 }
436         } else if (obj->dh_sigstate == TRUE) {
437                 if (obj->dh_type == EVENT_TYPE_SYNC)
438                         obj->dh_sigstate = FALSE;
439                 lwkt_reltoken(&ntoskrnl_dispatchtoken);
440                 return (STATUS_SUCCESS);
441         }
442
443         w.wb_object = obj;
444         w.wb_kthread = td;
445
446         INSERT_LIST_TAIL((&obj->dh_waitlisthead), (&w.wb_waitlist));
447
448         /*
449          * The timeout value is specified in 100 nanosecond units
450          * and can be a positive or negative number. If it's positive,
451          * then the duetime is absolute, and we need to convert it
452          * to an absolute offset relative to now in order to use it.
453          * If it's negative, then the duetime is relative and we
454          * just have to convert the units.
455          */
456
457         if (duetime != NULL) {
458                 if (*duetime < 0) {
459                         tv.tv_sec = - (*duetime) / 10000000;
460                         tv.tv_usec = (- (*duetime) / 10) -
461                             (tv.tv_sec * 1000000);
462                 } else {
463                         ntoskrnl_time(&curtime);
464                         if (*duetime < curtime)
465                                 tv.tv_sec = tv.tv_usec = 0;
466                         else {
467                                 tv.tv_sec = ((*duetime) - curtime) / 10000000;
468                                 tv.tv_usec = ((*duetime) - curtime) / 10 -
469                                     (tv.tv_sec * 1000000);
470                         }
471                 }
472         }
473
474         lwkt_reltoken(&ntoskrnl_dispatchtoken);
475
476         ticks = 1 + tv.tv_sec * hz + tv.tv_usec * hz / 1000000;
477         error = ndis_thsuspend(td, duetime == NULL ? 0 : ticks);
478
479         lwkt_gettoken(&tokref, &ntoskrnl_dispatchtoken);
480
481         /* We timed out. Leave the object alone and return status. */
482
483         if (error == EWOULDBLOCK) {
484                 REMOVE_LIST_ENTRY((&w.wb_waitlist));
485                 lwkt_reltoken(&ntoskrnl_dispatchtoken);
486                 return(STATUS_TIMEOUT);
487         }
488
489         /*
490          * Mutexes are always synchronization objects, which means
491          * if several threads are waiting to acquire it, only one will
492          * be woken up. If that one is us, and the mutex is up for grabs,
493          * grab it.
494          */
495
496         if (obj->dh_size == OTYPE_MUTEX) {
497                 km = (kmutant *)obj;
498                 if (km->km_ownerthread == NULL) {
499                         km->km_ownerthread = curthread->td_proc;
500                         km->km_acquirecnt++;
501                 }
502         }
503
504         if (obj->dh_type == EVENT_TYPE_SYNC)
505                 obj->dh_sigstate = FALSE;
506         REMOVE_LIST_ENTRY((&w.wb_waitlist));
507
508         lwkt_reltoken(&ntoskrnl_dispatchtoken);
509
510         return(STATUS_SUCCESS);
511 }
512
513 __stdcall static uint32_t
514 ntoskrnl_waitforobjs(uint32_t cnt, nt_dispatch_header *obj[],
515                      uint32_t wtype, uint32_t reason, uint32_t mode,
516                      uint8_t alertable, int64_t *duetime,
517                      wait_block *wb_array)
518 {
519         struct thread           *td = curthread;
520         kmutant                 *km;
521         wait_block              _wb_array[THREAD_WAIT_OBJECTS];
522         wait_block              *w;
523         struct timeval          tv;
524         int                     i, wcnt = 0, widx = 0, error = 0;
525         uint64_t                curtime;
526         struct timespec         t1, t2;
527
528         if (cnt > MAX_WAIT_OBJECTS)
529                 return(STATUS_INVALID_PARAMETER);
530         if (cnt > THREAD_WAIT_OBJECTS && wb_array == NULL)
531                 return(STATUS_INVALID_PARAMETER);
532
533         lwkt_gettoken(&ntoskrnl_dispatchtoken);
534
535         if (wb_array == NULL)
536                 w = &_wb_array[0];
537         else
538                 w = wb_array;
539
540         tv.tv_sec = 0;          /* fix compiler warning */
541         tv.tv_usec = 0;         /* fix compiler warning */
542
543         /* First pass: see if we can satisfy any waits immediately. */
544
545         for (i = 0; i < cnt; i++) {
546                 if (obj[i]->dh_size == OTYPE_MUTEX) {
547                         km = (kmutant *)obj[i];
548                         if (km->km_ownerthread == NULL ||
549                             km->km_ownerthread == curthread->td_proc) {
550                                 obj[i]->dh_sigstate = FALSE;
551                                 km->km_acquirecnt++;
552                                 km->km_ownerthread = curthread->td_proc;
553                                 if (wtype == WAITTYPE_ANY) {
554                                         lwkt_reltoken(&ntoskrnl_dispatchtoken);
555                                         return (STATUS_WAIT_0 + i);
556                                 }
557                         }
558                 } else if (obj[i]->dh_sigstate == TRUE) {
559                         if (obj[i]->dh_type == EVENT_TYPE_SYNC)
560                                 obj[i]->dh_sigstate = FALSE;
561                         if (wtype == WAITTYPE_ANY) {
562                                 lwkt_reltoken(&ntoskrnl_dispatchtoken);
563                                 return (STATUS_WAIT_0 + i);
564                         }
565                 }
566         }
567
568         /*
569          * Second pass: set up wait for anything we can't
570          * satisfy immediately.
571          */
572
573         for (i = 0; i < cnt; i++) {
574                 if (obj[i]->dh_sigstate == TRUE)
575                         continue;
576                 INSERT_LIST_TAIL((&obj[i]->dh_waitlisthead),
577                     (&w[i].wb_waitlist));
578                 w[i].wb_kthread = td;
579                 w[i].wb_object = obj[i];
580                 wcnt++;
581         }
582
583         if (duetime) {
584                 if (*duetime < 0) {
585                         tv.tv_sec = -*duetime / 10000000;
586                         tv.tv_usec = (-*duetime / 10) - (tv.tv_sec * 1000000);
587                 } else {
588                         ntoskrnl_time(&curtime);
589                         if (*duetime < curtime) {
590                                 tv.tv_sec = 0;
591                                 tv.tv_usec = 0;
592                         } else {
593                                 tv.tv_sec = ((*duetime) - curtime) / 10000000;
594                                 tv.tv_usec = ((*duetime) - curtime) / 10 -
595                                     (tv.tv_sec * 1000000);
596                         }
597                 }
598         }
599
600         while (wcnt) {
601                 nanotime(&t1);
602                 lwkt_reltoken(&ntoskrnl_dispatchtoken);
603
604                 if (duetime) {
605                         ticks = 1 + tv.tv_sec * hz + tv.tv_usec * hz / 1000000;
606                         error = ndis_thsuspend(td, ticks);
607                 } else {
608                         error = ndis_thsuspend(td, 0);
609                 }
610
611                 lwkt_gettoken(&ntoskrnl_dispatchtoken);
612                 nanotime(&t2);
613
614                 for (i = 0; i < cnt; i++) {
615                         if (obj[i]->dh_size == OTYPE_MUTEX) {
616                                 km = (kmutant *)obj;
617                                 if (km->km_ownerthread == NULL) {
618                                         km->km_ownerthread =
619                                             curthread->td_proc;
620                                         km->km_acquirecnt++;
621                                 }
622                         }
623                         if (obj[i]->dh_sigstate == TRUE) {
624                                 widx = i;
625                                 if (obj[i]->dh_type == EVENT_TYPE_SYNC)
626                                         obj[i]->dh_sigstate = FALSE;
627                                 REMOVE_LIST_ENTRY((&w[i].wb_waitlist));
628                                 wcnt--;
629                         }
630                 }
631
632                 if (error || wtype == WAITTYPE_ANY)
633                         break;
634
635                 if (duetime) {
636                         tv.tv_sec -= (t2.tv_sec - t1.tv_sec);
637                         tv.tv_usec -= (t2.tv_nsec - t1.tv_nsec) / 1000;
638                 }
639         }
640
641         if (wcnt) {
642                 for (i = 0; i < cnt; i++)
643                         REMOVE_LIST_ENTRY((&w[i].wb_waitlist));
644         }
645
646         if (error == EWOULDBLOCK) {
647                 lwkt_reltoken(&ntoskrnl_dispatchtoken);
648                 return(STATUS_TIMEOUT);
649         }
650
651         if (wtype == WAITTYPE_ANY && wcnt) {
652                 lwkt_reltoken(&ntoskrnl_dispatchtoken);
653                 return(STATUS_WAIT_0 + widx);
654         }
655
656         lwkt_reltoken(&ntoskrnl_dispatchtoken);
657
658         return(STATUS_SUCCESS);
659 }
660
661 __stdcall static void
662 ntoskrnl_writereg_ushort(uint16_t *reg, uint16_t val)
663 {
664         bus_space_write_2(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
665         return;
666 }
667
668 __stdcall static uint16_t
669 ntoskrnl_readreg_ushort(uint16_t *reg)
670 {
671         return(bus_space_read_2(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
672 }
673
674 __stdcall static void
675 ntoskrnl_writereg_ulong(uint32_t *reg, uint32_t val)
676 {
677         bus_space_write_4(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
678         return;
679 }
680
681 __stdcall static uint32_t
682 ntoskrnl_readreg_ulong(uint32_t *reg)
683 {
684         return(bus_space_read_4(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
685 }
686
687 __stdcall static uint8_t
688 ntoskrnl_readreg_uchar(uint8_t *reg)
689 {
690         return(bus_space_read_1(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
691 }
692
693 __stdcall static void
694 ntoskrnl_writereg_uchar(uint8_t *reg, uint8_t val)
695 {
696         bus_space_write_1(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
697         return;
698 }
699
700 __stdcall static int64_t
701 _allmul(int64_t a, int64_t b)
702 {
703         return (a * b);
704 }
705
706 __stdcall static int64_t
707 _alldiv(int64_t a, int64_t b)
708 {
709         return (a / b);
710 }
711
712 __stdcall static int64_t
713 _allrem(int64_t a, int64_t b)
714 {
715         return (a % b);
716 }
717
718 __stdcall static uint64_t
719 _aullmul(uint64_t a, uint64_t b)
720 {
721         return (a * b);
722 }
723
724 __stdcall static uint64_t
725 _aulldiv(uint64_t a, uint64_t b)
726 {
727         return (a / b);
728 }
729
730 __stdcall static uint64_t
731 _aullrem(uint64_t a, uint64_t b)
732 {
733         return (a % b);
734 }
735
736 __regparm static int64_t
737 _allshl(int64_t a, uint8_t b)
738 {
739         return (a << b);
740 }
741
742 __regparm static uint64_t
743 _aullshl(uint64_t a, uint8_t b)
744 {
745         return (a << b);
746 }
747
748 __regparm static int64_t
749 _allshr(int64_t a, uint8_t b)
750 {
751         return (a >> b);
752 }
753
754 __regparm static uint64_t
755 _aullshr(uint64_t a, uint8_t b)
756 {
757         return (a >> b);
758 }
759
760 static slist_entry *
761 ntoskrnl_pushsl(slist_header *head, slist_entry *entry)
762 {
763         slist_entry             *oldhead;
764
765         oldhead = head->slh_list.slh_next;
766         entry->sl_next = head->slh_list.slh_next;
767         head->slh_list.slh_next = entry;
768         head->slh_list.slh_depth++;
769         head->slh_list.slh_seq++;
770
771         return(oldhead);
772 }
773
774 static slist_entry *
775 ntoskrnl_popsl(slist_header *head)
776 {
777         slist_entry             *first;
778
779         first = head->slh_list.slh_next;
780         if (first != NULL) {
781                 head->slh_list.slh_next = first->sl_next;
782                 head->slh_list.slh_depth--;
783                 head->slh_list.slh_seq++;
784         }
785
786         return(first);
787 }
788
789 __stdcall static void *
790 ntoskrnl_allocfunc(uint32_t pooltype, size_t size, uint32_t tag)
791 {
792         return(kmalloc(size, M_DEVBUF, M_WAITOK));
793 }
794
795 __stdcall static void
796 ntoskrnl_freefunc(void *buf)
797 {
798         kfree(buf, M_DEVBUF);
799         return;
800 }
801
802 __stdcall static void
803 ntoskrnl_init_lookaside(paged_lookaside_list *lookaside,
804                         lookaside_alloc_func *allocfunc,
805                         lookaside_free_func *freefunc,
806                         uint32_t flags, size_t size,
807                         uint32_t tag, uint16_t depth)
808 {
809         bzero((char *)lookaside, sizeof(paged_lookaside_list));
810
811         if (size < sizeof(slist_entry))
812                 lookaside->nll_l.gl_size = sizeof(slist_entry);
813         else
814                 lookaside->nll_l.gl_size = size;
815         lookaside->nll_l.gl_tag = tag;
816         if (allocfunc == NULL)
817                 lookaside->nll_l.gl_allocfunc = ntoskrnl_allocfunc;
818         else
819                 lookaside->nll_l.gl_allocfunc = allocfunc;
820
821         if (freefunc == NULL)
822                 lookaside->nll_l.gl_freefunc = ntoskrnl_freefunc;
823         else
824                 lookaside->nll_l.gl_freefunc = freefunc;
825
826         ntoskrnl_init_lock(&lookaside->nll_obsoletelock);
827
828         lookaside->nll_l.gl_depth = LOOKASIDE_DEPTH;
829         lookaside->nll_l.gl_maxdepth = LOOKASIDE_DEPTH;
830
831         return;
832 }
833
834 __stdcall static void
835 ntoskrnl_delete_lookaside(paged_lookaside_list *lookaside)
836 {
837         void                    *buf;
838         __stdcall void          (*freefunc)(void *);
839
840         freefunc = lookaside->nll_l.gl_freefunc;
841         while((buf = ntoskrnl_popsl(&lookaside->nll_l.gl_listhead)) != NULL)
842                 freefunc(buf);
843
844         return;
845 }
846
847 __stdcall static void
848 ntoskrnl_init_nplookaside(npaged_lookaside_list *lookaside,
849                           lookaside_alloc_func *allocfunc,
850                           lookaside_free_func *freefunc,
851                           uint32_t flags, size_t size,
852                           uint32_t tag, uint16_t depth)
853 {
854         bzero((char *)lookaside, sizeof(npaged_lookaside_list));
855
856         if (size < sizeof(slist_entry))
857                 lookaside->nll_l.gl_size = sizeof(slist_entry);
858         else
859                 lookaside->nll_l.gl_size = size;
860         lookaside->nll_l.gl_tag = tag;
861         if (allocfunc == NULL)
862                 lookaside->nll_l.gl_allocfunc = ntoskrnl_allocfunc;
863         else
864                 lookaside->nll_l.gl_allocfunc = allocfunc;
865
866         if (freefunc == NULL)
867                 lookaside->nll_l.gl_freefunc = ntoskrnl_freefunc;
868         else
869                 lookaside->nll_l.gl_freefunc = freefunc;
870
871         ntoskrnl_init_lock(&lookaside->nll_obsoletelock);
872
873         lookaside->nll_l.gl_depth = LOOKASIDE_DEPTH;
874         lookaside->nll_l.gl_maxdepth = LOOKASIDE_DEPTH;
875
876         return;
877 }
878
879 __stdcall static void
880 ntoskrnl_delete_nplookaside(npaged_lookaside_list *lookaside)
881 {
882         void                    *buf;
883         __stdcall void          (*freefunc)(void *);
884
885         freefunc = lookaside->nll_l.gl_freefunc;
886         while((buf = ntoskrnl_popsl(&lookaside->nll_l.gl_listhead)) != NULL)
887                 freefunc(buf);
888
889         return;
890 }
891
892 /*
893  * Note: the interlocked slist push and pop routines are
894  * declared to be _fastcall in Windows. gcc 3.4 is supposed
895  * to have support for this calling convention, however we
896  * don't have that version available yet, so we kludge things
897  * up using some inline assembly.
898  */
899
900 __stdcall __regcall static slist_entry *
901 ntoskrnl_push_slist(REGARGS2(slist_header *head, slist_entry *entry))
902 {
903         slist_entry             *oldhead;
904
905         oldhead = (slist_entry *)FASTCALL3(ntoskrnl_push_slist_ex,
906             head, entry, &ntoskrnl_global);
907
908         return(oldhead);
909 }
910
911 __stdcall __regcall static slist_entry *
912 ntoskrnl_pop_slist(REGARGS1(slist_header *head))
913 {
914         slist_entry             *first;
915
916         first = (slist_entry *)FASTCALL2(ntoskrnl_pop_slist_ex,
917             head, &ntoskrnl_global);
918
919         return(first);
920 }
921
922 __stdcall __regcall static slist_entry *
923 ntoskrnl_push_slist_ex(REGARGS2(slist_header *head, slist_entry *entry), kspin_lock *lock)
924 {
925         slist_entry             *oldhead;
926         uint8_t                 irql;
927
928         irql = FASTCALL2(hal_lock, lock, DISPATCH_LEVEL);
929         oldhead = ntoskrnl_pushsl(head, entry);
930         FASTCALL2(hal_unlock, lock, irql);
931
932         return(oldhead);
933 }
934
935 __stdcall __regcall static slist_entry *
936 ntoskrnl_pop_slist_ex(REGARGS2(slist_header *head, kspin_lock *lock))
937 {
938         slist_entry             *first;
939         uint8_t                 irql;
940
941         irql = FASTCALL2(hal_lock, lock, DISPATCH_LEVEL);
942         first = ntoskrnl_popsl(head);
943         FASTCALL2(hal_unlock, lock, irql);
944
945         return(first);
946 }
947
948 __stdcall __regcall void
949 ntoskrnl_lock_dpc(REGARGS1(kspin_lock *lock))
950 {
951         while (atomic_poll_acquire_int((volatile u_int *)lock) == 0)
952                 /* sit and spin */;
953 }
954
955 __stdcall __regcall void
956 ntoskrnl_unlock_dpc(REGARGS1(kspin_lock *lock))
957 {
958         atomic_poll_release_int((volatile u_int *)lock);
959 }
960
961 __stdcall __regcall static uint32_t
962 ntoskrnl_interlock_inc(REGARGS1(volatile uint32_t *addend))
963 {
964         atomic_add_long((volatile u_long *)addend, 1);
965         return(*addend);
966 }
967
968 __stdcall __regcall static uint32_t
969 ntoskrnl_interlock_dec(REGARGS1(volatile uint32_t *addend))
970 {
971         atomic_subtract_long((volatile u_long *)addend, 1);
972         return(*addend);
973 }
974
975 __stdcall __regcall static void
976 ntoskrnl_interlock_addstat(REGARGS2(uint64_t *addend, uint32_t inc))
977 {
978         uint8_t                 irql;
979
980         irql = FASTCALL2(hal_lock, &ntoskrnl_global, DISPATCH_LEVEL);
981         *addend += inc;
982         FASTCALL2(hal_unlock, &ntoskrnl_global, irql);
983
984         return;
985 };
986
987 __stdcall static void
988 ntoskrnl_freemdl(ndis_buffer *mdl)
989 {
990         ndis_buffer             *head;
991
992         if (mdl == NULL || mdl->nb_process == NULL)
993                 return;
994
995         head = mdl->nb_process;
996
997         if (head->nb_flags != 0x1)
998                 return;
999
1000         mdl->nb_next = head->nb_next;
1001         head->nb_next = mdl;
1002
1003         /* Decrement count of busy buffers. */
1004
1005         head->nb_bytecount--;
1006
1007         /*
1008          * If the pool has been marked for deletion and there are
1009          * no more buffers outstanding, nuke the pool.
1010          */
1011
1012         if (head->nb_byteoffset && head->nb_bytecount == 0)
1013                 kfree(head, M_DEVBUF);
1014
1015         return;
1016 }
1017
1018 __stdcall static uint32_t
1019 ntoskrnl_sizeofmdl(void *vaddr, size_t len)
1020 {
1021         uint32_t                l;
1022
1023         l = sizeof(struct ndis_buffer) +
1024             (sizeof(uint32_t) * SPAN_PAGES(vaddr, len));
1025
1026         return(l);
1027 }
1028
1029 __stdcall static void
1030 ntoskrnl_build_npaged_mdl(ndis_buffer *mdl)
1031 {
1032         mdl->nb_mappedsystemva = (char *)mdl->nb_startva + mdl->nb_byteoffset;
1033         return;
1034 }
1035
1036 __stdcall static void *
1037 ntoskrnl_mmaplockedpages(ndis_buffer *buf, uint8_t accessmode)
1038 {
1039         return(MDL_VA(buf));
1040 }
1041
1042 __stdcall static void *
1043 ntoskrnl_mmaplockedpages_cache(ndis_buffer *buf, uint8_t accessmode,
1044                                uint32_t cachetype, void *vaddr,
1045                                uint32_t bugcheck, uint32_t prio)
1046 {
1047         return(MDL_VA(buf));
1048 }
1049
1050 __stdcall static void
1051 ntoskrnl_munmaplockedpages(void *vaddr, ndis_buffer *buf)
1052 {
1053         return;
1054 }
1055
1056 /*
1057  * The KeInitializeSpinLock(), KefAcquireSpinLockAtDpcLevel()
1058  * and KefReleaseSpinLockFromDpcLevel() appear to be analagous
1059  * to crit_enter()/crit_exit() in their use. We can't create a new mutex
1060  * lock here because there is no complimentary KeFreeSpinLock()
1061  * function. Instead, we grab a mutex from the mutex pool.
1062  */
1063 __stdcall static void
1064 ntoskrnl_init_lock(kspin_lock *lock)
1065 {
1066         *lock = 0;
1067
1068         return;
1069 }
1070
1071 __stdcall static size_t
1072 ntoskrnl_memcmp(const void *s1, const void *s2, size_t len)
1073 {
1074         size_t                  i, total = 0;
1075         uint8_t                 *m1, *m2;
1076
1077         m1 = __DECONST(char *, s1);
1078         m2 = __DECONST(char *, s2);
1079
1080         for (i = 0; i < len; i++) {
1081                 if (m1[i] == m2[i])
1082                         total++;
1083         }
1084         return(total);
1085 }
1086
1087 __stdcall static void
1088 ntoskrnl_init_ansi_string(ndis_ansi_string *dst, char *src)
1089 {
1090         ndis_ansi_string        *a;
1091
1092         a = dst;
1093         if (a == NULL)
1094                 return;
1095         if (src == NULL) {
1096                 a->nas_len = a->nas_maxlen = 0;
1097                 a->nas_buf = NULL;
1098         } else {
1099                 a->nas_buf = src;
1100                 a->nas_len = a->nas_maxlen = strlen(src);
1101         }
1102
1103         return;
1104 }
1105
1106 __stdcall static void
1107 ntoskrnl_init_unicode_string(ndis_unicode_string *dst, uint16_t *src)
1108 {
1109         ndis_unicode_string     *u;
1110         int                     i;
1111
1112         u = dst;
1113         if (u == NULL)
1114                 return;
1115         if (src == NULL) {
1116                 u->nus_len = u->nus_maxlen = 0;
1117                 u->nus_buf = NULL;
1118         } else {
1119                 i = 0;
1120                 while(src[i] != 0)
1121                         i++;
1122                 u->nus_buf = src;
1123                 u->nus_len = u->nus_maxlen = i * 2;
1124         }
1125
1126         return;
1127 }
1128
1129 __stdcall ndis_status
1130 ntoskrnl_unicode_to_int(ndis_unicode_string *ustr, uint32_t base,
1131                         uint32_t *val)
1132 {
1133         uint16_t                *uchr;
1134         int                     len, neg = 0;
1135         char                    abuf[64];
1136         char                    *astr;
1137
1138         uchr = ustr->nus_buf;
1139         len = ustr->nus_len;
1140         bzero(abuf, sizeof(abuf));
1141
1142         if ((char)((*uchr) & 0xFF) == '-') {
1143                 neg = 1;
1144                 uchr++;
1145                 len -= 2;
1146         } else if ((char)((*uchr) & 0xFF) == '+') {
1147                 neg = 0;
1148                 uchr++;
1149                 len -= 2;
1150         }
1151
1152         if (base == 0) {
1153                 if ((char)((*uchr) & 0xFF) == 'b') {
1154                         base = 2;
1155                         uchr++;
1156                         len -= 2;
1157                 } else if ((char)((*uchr) & 0xFF) == 'o') {
1158                         base = 8;
1159                         uchr++;
1160                         len -= 2;
1161                 } else if ((char)((*uchr) & 0xFF) == 'x') {
1162                         base = 16;
1163                         uchr++;
1164                         len -= 2;
1165                 } else
1166                         base = 10;
1167         }
1168
1169         astr = abuf;
1170         if (neg) {
1171                 strcpy(astr, "-");
1172                 astr++;
1173         }
1174
1175         ndis_unicode_to_ascii(uchr, len, &astr);
1176         *val = strtoul(abuf, NULL, base);
1177
1178         return(NDIS_STATUS_SUCCESS);
1179 }
1180
1181 __stdcall static void
1182 ntoskrnl_free_unicode_string(ndis_unicode_string *ustr)
1183 {
1184         if (ustr->nus_buf == NULL)
1185                 return;
1186         kfree(ustr->nus_buf, M_DEVBUF);
1187         ustr->nus_buf = NULL;
1188         return;
1189 }
1190
1191 __stdcall static void
1192 ntoskrnl_free_ansi_string(ndis_ansi_string *astr)
1193 {
1194         if (astr->nas_buf == NULL)
1195                 return;
1196         kfree(astr->nas_buf, M_DEVBUF);
1197         astr->nas_buf = NULL;
1198         return;
1199 }
1200
1201 static int
1202 atoi(const char *str)
1203 {
1204         return (int)strtol(str, NULL, 10);
1205 }
1206
1207 static long
1208 atol(const char *str)
1209 {
1210         return strtol(str, NULL, 10);
1211 }
1212
1213 static int
1214 rand(void)
1215 {
1216         struct timeval          tv;
1217
1218         microtime(&tv);
1219         skrandom(tv.tv_usec);
1220         return((int)krandom());
1221 }
1222
1223 __stdcall static uint8_t
1224 ntoskrnl_wdmver(uint8_t major, uint8_t minor)
1225 {
1226         if (major == WDM_MAJOR && minor == WDM_MINOR_WINXP)
1227                 return(TRUE);
1228         return(FALSE);
1229 }
1230
1231 __stdcall static ndis_status
1232 ntoskrnl_devprop(device_object *devobj, uint32_t regprop, uint32_t buflen,
1233                  void *prop, uint32_t *reslen)
1234 {
1235         ndis_miniport_block     *block;
1236
1237         block = devobj->do_rsvd;
1238
1239         switch (regprop) {
1240         case DEVPROP_DRIVER_KEYNAME:
1241                 ndis_ascii_to_unicode(__DECONST(char *,
1242                     device_get_nameunit(block->nmb_dev)), (uint16_t **)&prop);
1243                 *reslen = strlen(device_get_nameunit(block->nmb_dev)) * 2;
1244                 break;
1245         default:
1246                 return(STATUS_INVALID_PARAMETER_2);
1247                 break;
1248         }
1249
1250         return(STATUS_SUCCESS);
1251 }
1252
1253 __stdcall static void
1254 ntoskrnl_init_mutex(kmutant *kmutex, uint32_t level)
1255 {
1256         INIT_LIST_HEAD((&kmutex->km_header.dh_waitlisthead));
1257         kmutex->km_abandoned = FALSE;
1258         kmutex->km_apcdisable = 1;
1259         kmutex->km_header.dh_sigstate = TRUE;
1260         kmutex->km_header.dh_type = EVENT_TYPE_SYNC;
1261         kmutex->km_header.dh_size = OTYPE_MUTEX;
1262         kmutex->km_acquirecnt = 0;
1263         kmutex->km_ownerthread = NULL;
1264         return;
1265 }
1266
1267 __stdcall static uint32_t
1268 ntoskrnl_release_mutex(kmutant *kmutex, uint8_t kwait)
1269 {
1270         lwkt_gettoken(&ntoskrnl_dispatchtoken);
1271         if (kmutex->km_ownerthread != curthread->td_proc) {
1272                 lwkt_reltoken(&ntoskrnl_dispatchtoken);
1273                 return(STATUS_MUTANT_NOT_OWNED);
1274         }
1275         kmutex->km_acquirecnt--;
1276         if (kmutex->km_acquirecnt == 0) {
1277                 kmutex->km_ownerthread = NULL;
1278                 lwkt_reltoken(&ntoskrnl_dispatchtoken);
1279                 ntoskrnl_wakeup(&kmutex->km_header);
1280         } else {
1281                 lwkt_reltoken(&ntoskrnl_dispatchtoken);
1282         }
1283
1284         return(kmutex->km_acquirecnt);
1285 }
1286
1287 __stdcall static uint32_t
1288 ntoskrnl_read_mutex(kmutant *kmutex)
1289 {
1290         return(kmutex->km_header.dh_sigstate);
1291 }
1292
1293 __stdcall void
1294 ntoskrnl_init_event(nt_kevent *kevent, uint32_t type, uint8_t state)
1295 {
1296         INIT_LIST_HEAD((&kevent->k_header.dh_waitlisthead));
1297         kevent->k_header.dh_sigstate = state;
1298         kevent->k_header.dh_type = type;
1299         kevent->k_header.dh_size = OTYPE_EVENT;
1300         return;
1301 }
1302
1303 __stdcall uint32_t
1304 ntoskrnl_reset_event(nt_kevent *kevent)
1305 {
1306         uint32_t                prevstate;
1307
1308         lwkt_gettoken(&ntoskrnl_dispatchtoken);
1309         prevstate = kevent->k_header.dh_sigstate;
1310         kevent->k_header.dh_sigstate = FALSE;
1311         lwkt_reltoken(&ntoskrnl_dispatchtoken);
1312
1313         return(prevstate);
1314 }
1315
1316 __stdcall uint32_t
1317 ntoskrnl_set_event(nt_kevent *kevent, uint32_t increment, uint8_t kwait)
1318 {
1319         uint32_t                prevstate;
1320
1321         prevstate = kevent->k_header.dh_sigstate;
1322         ntoskrnl_wakeup(&kevent->k_header);
1323
1324         return(prevstate);
1325 }
1326
1327 __stdcall void
1328 ntoskrnl_clear_event(nt_kevent *kevent)
1329 {
1330         kevent->k_header.dh_sigstate = FALSE;
1331         return;
1332 }
1333
1334 __stdcall uint32_t
1335 ntoskrnl_read_event(nt_kevent *kevent)
1336 {
1337         return(kevent->k_header.dh_sigstate);
1338 }
1339
1340 __stdcall static ndis_status
1341 ntoskrnl_objref(ndis_handle handle, uint32_t reqaccess, void *otype,
1342                 uint8_t accessmode, void **object, void **handleinfo)
1343 {
1344         nt_objref               *nr;
1345
1346         nr = kmalloc(sizeof(nt_objref), M_DEVBUF, M_WAITOK|M_ZERO);
1347
1348         INIT_LIST_HEAD((&nr->no_dh.dh_waitlisthead));
1349         nr->no_obj = handle;
1350         nr->no_dh.dh_size = OTYPE_THREAD;
1351         TAILQ_INSERT_TAIL(&ntoskrnl_reflist, nr, link);
1352         *object = nr;
1353
1354         return(NDIS_STATUS_SUCCESS);
1355 }
1356
1357 __stdcall __regcall static void
1358 ntoskrnl_objderef(REGARGS1(void *object))
1359 {
1360         nt_objref               *nr;
1361
1362         nr = object;
1363         TAILQ_REMOVE(&ntoskrnl_reflist, nr, link);
1364         kfree(nr, M_DEVBUF);
1365
1366         return;
1367 }
1368
1369 __stdcall static uint32_t
1370 ntoskrnl_zwclose(ndis_handle handle)
1371 {
1372         return(STATUS_SUCCESS);
1373 }
1374
1375 /*
1376  * This is here just in case the thread returns without calling
1377  * PsTerminateSystemThread().
1378  */
1379 static void
1380 ntoskrnl_thrfunc(void *arg)
1381 {
1382         thread_context          *thrctx;
1383         __stdcall uint32_t (*tfunc)(void *);
1384         void                    *tctx;
1385         uint32_t                rval;
1386
1387         get_mplock();
1388
1389         thrctx = arg;
1390         tfunc = thrctx->tc_thrfunc;
1391         tctx = thrctx->tc_thrctx;
1392         kfree(thrctx, M_TEMP);
1393
1394         rval = tfunc(tctx);
1395         ntoskrnl_thread_exit(rval);
1396         /* not reached */
1397 }
1398
1399 __stdcall static ndis_status
1400 ntoskrnl_create_thread(ndis_handle *handle, uint32_t reqaccess,
1401                        void *objattrs, ndis_handle phandle,
1402                        void *clientid, void *thrfunc, void *thrctx)
1403 {
1404         int                     error;
1405         char                    tname[128];
1406         thread_context          *tc;
1407         thread_t                td;
1408
1409         tc = kmalloc(sizeof(thread_context), M_TEMP, M_WAITOK);
1410
1411         tc->tc_thrctx = thrctx;
1412         tc->tc_thrfunc = thrfunc;
1413
1414         ksprintf(tname, "windows kthread %d", ntoskrnl_kth);
1415         error = kthread_create_stk(ntoskrnl_thrfunc, tc, &td,
1416             NDIS_KSTACK_PAGES * PAGE_SIZE, tname);
1417         *handle = td;
1418
1419         ntoskrnl_kth++;
1420
1421         return(error);
1422 }
1423
1424 /*
1425  * In Windows, the exit of a thread is an event that you're allowed
1426  * to wait on, assuming you've obtained a reference to the thread using
1427  * ObReferenceObjectByHandle(). Unfortunately, the only way we can
1428  * simulate this behavior is to register each thread we create in a
1429  * reference list, and if someone holds a reference to us, we poke
1430  * them.
1431  */
1432 __stdcall static ndis_status
1433 ntoskrnl_thread_exit(ndis_status status)
1434 {
1435         struct nt_objref        *nr;
1436
1437         TAILQ_FOREACH(nr, &ntoskrnl_reflist, link) {
1438                 if (nr->no_obj != curthread)
1439                         continue;
1440                 ntoskrnl_wakeup(&nr->no_dh);
1441                 break;
1442         }
1443
1444         ntoskrnl_kth--;
1445
1446         rel_mplock();
1447         kthread_exit(); /* call explicitly */
1448
1449         return(0);      /* notreached */
1450 }
1451
1452 static uint32_t
1453 ntoskrnl_dbgprint(char *fmt, ...)
1454 {
1455         __va_list                       ap;
1456
1457         if (bootverbose) {
1458                 __va_start(ap, fmt);
1459                 kvprintf(fmt, ap);
1460         }
1461
1462         return(STATUS_SUCCESS);
1463 }
1464
1465 __stdcall static void
1466 ntoskrnl_debugger(void)
1467 {
1468
1469 #if __FreeBSD_version < 502113
1470         Debugger("ntoskrnl_debugger(): breakpoint");
1471 #else
1472         kdb_enter("ntoskrnl_debugger(): breakpoint");
1473 #endif
1474 }
1475
1476 static void
1477 ntoskrnl_timercall(void *arg)
1478 {
1479         ktimer                  *timer;
1480
1481         timer = arg;
1482
1483         timer->k_header.dh_inserted = FALSE;
1484
1485         /*
1486          * If this is a periodic timer, re-arm it
1487          * so it will fire again. We do this before
1488          * calling any deferred procedure calls because
1489          * it's possible the DPC might cancel the timer,
1490          * in which case it would be wrong for us to
1491          * re-arm it again afterwards.
1492          */
1493
1494         if (timer->k_period) {
1495                 timer->k_header.dh_inserted = TRUE;
1496                 callout_reset(timer->k_handle, 1 + timer->k_period * hz / 1000,
1497                               ntoskrnl_timercall, timer);
1498         } else {
1499                 callout_deactivate(timer->k_handle);
1500                 kfree(timer->k_handle, M_NDIS);
1501                 timer->k_handle = NULL;
1502         }
1503
1504         if (timer->k_dpc != NULL)
1505                 ntoskrnl_queue_dpc(timer->k_dpc, NULL, NULL);
1506
1507         ntoskrnl_wakeup(&timer->k_header);
1508 }
1509
1510 __stdcall void
1511 ntoskrnl_init_timer(ktimer *timer)
1512 {
1513         if (timer == NULL)
1514                 return;
1515
1516         ntoskrnl_init_timer_ex(timer,  EVENT_TYPE_NOTIFY);
1517 }
1518
1519 __stdcall void
1520 ntoskrnl_init_timer_ex(ktimer *timer, uint32_t type)
1521 {
1522         if (timer == NULL)
1523                 return;
1524
1525         INIT_LIST_HEAD((&timer->k_header.dh_waitlisthead));
1526         timer->k_header.dh_sigstate = FALSE;
1527         timer->k_header.dh_inserted = FALSE;
1528         timer->k_header.dh_type = type;
1529         timer->k_header.dh_size = OTYPE_TIMER;
1530         timer->k_handle = NULL;
1531
1532         return;
1533 }
1534
1535 /*
1536  * This is a wrapper for Windows deferred procedure calls that
1537  * have been placed on an NDIS thread work queue. We need it
1538  * since the DPC could be a _stdcall function. Also, as far as
1539  * I can tell, defered procedure calls must run at DISPATCH_LEVEL.
1540  */
1541 static void
1542 ntoskrnl_run_dpc(void *arg)
1543 {
1544         kdpc_func               dpcfunc;
1545         kdpc                    *dpc;
1546         uint8_t                 irql;
1547
1548         dpc = arg;
1549         dpcfunc = (kdpc_func)dpc->k_deferedfunc;
1550         irql = FASTCALL1(hal_raise_irql, DISPATCH_LEVEL);
1551         dpcfunc(dpc, dpc->k_deferredctx, dpc->k_sysarg1, dpc->k_sysarg2);
1552         FASTCALL1(hal_lower_irql, irql);
1553
1554         return;
1555 }
1556
1557 __stdcall void
1558 ntoskrnl_init_dpc(kdpc *dpc, void *dpcfunc, void *dpcctx)
1559 {
1560         if (dpc == NULL)
1561                 return;
1562
1563         dpc->k_deferedfunc = dpcfunc;
1564         dpc->k_deferredctx = dpcctx;
1565
1566         return;
1567 }
1568
1569 __stdcall uint8_t
1570 ntoskrnl_queue_dpc(kdpc *dpc, void *sysarg1, void *sysarg2)
1571 {
1572         dpc->k_sysarg1 = sysarg1;
1573         dpc->k_sysarg2 = sysarg2;
1574         if (ndis_sched(ntoskrnl_run_dpc, dpc, NDIS_SWI))
1575                 return(FALSE);
1576
1577         return(TRUE);
1578 }
1579
1580 __stdcall uint8_t
1581 ntoskrnl_dequeue_dpc(kdpc *dpc)
1582 {
1583         if (ndis_unsched(ntoskrnl_run_dpc, dpc, NDIS_SWI))
1584                 return(FALSE);
1585
1586         return(TRUE);
1587 }
1588
1589 __stdcall uint8_t
1590 ntoskrnl_set_timer_ex(ktimer *timer, int64_t duetime, uint32_t period,
1591                       kdpc *dpc)
1592 {
1593         struct timeval          tv;
1594         uint64_t                curtime;
1595         uint8_t                 pending;
1596         int                     ticks;
1597
1598         if (timer == NULL)
1599                 return(FALSE);
1600
1601         if (timer->k_header.dh_inserted == TRUE) {
1602                 if (timer->k_handle != NULL)
1603                         callout_stop(timer->k_handle);
1604                 timer->k_header.dh_inserted = FALSE;
1605                 pending = TRUE;
1606         } else
1607                 pending = FALSE;
1608
1609         timer->k_duetime = duetime;
1610         timer->k_period = period;
1611         timer->k_header.dh_sigstate = FALSE;
1612         timer->k_dpc = dpc;
1613
1614         if (duetime < 0) {
1615                 tv.tv_sec = - (duetime) / 10000000;
1616                 tv.tv_usec = (- (duetime) / 10) -
1617                     (tv.tv_sec * 1000000);
1618         } else {
1619                 ntoskrnl_time(&curtime);
1620                 if (duetime < curtime)
1621                         tv.tv_sec = tv.tv_usec = 0;
1622                 else {
1623                         tv.tv_sec = ((duetime) - curtime) / 10000000;
1624                         tv.tv_usec = ((duetime) - curtime) / 10 -
1625                             (tv.tv_sec * 1000000);
1626                 }
1627         }
1628
1629         ticks = 1 + tv.tv_sec * hz + tv.tv_usec * hz / 1000000;
1630         timer->k_header.dh_inserted = TRUE;
1631         if (timer->k_handle == NULL) {
1632                 timer->k_handle = kmalloc(sizeof(struct callout), M_NDIS,
1633                                          M_INTWAIT);
1634                 callout_init(timer->k_handle);
1635         }
1636         callout_reset(timer->k_handle, ticks, ntoskrnl_timercall, timer);
1637
1638         return(pending);
1639 }
1640
1641 __stdcall uint8_t
1642 ntoskrnl_set_timer(ktimer *timer, int64_t duetime, kdpc *dpc)
1643 {
1644         return (ntoskrnl_set_timer_ex(timer, duetime, 0, dpc));
1645 }
1646
1647 __stdcall uint8_t
1648 ntoskrnl_cancel_timer(ktimer *timer)
1649 {
1650         uint8_t                 pending;
1651
1652         if (timer == NULL)
1653                 return(FALSE);
1654
1655         if (timer->k_header.dh_inserted == TRUE) {
1656                 if (timer->k_handle != NULL) {
1657                         callout_stop(timer->k_handle);
1658                         kfree(timer->k_handle, M_NDIS);
1659                         timer->k_handle = NULL;
1660                 }
1661                 if (timer->k_dpc != NULL)
1662                         ntoskrnl_dequeue_dpc(timer->k_dpc);
1663                 pending = TRUE;
1664         } else
1665                 pending = FALSE;
1666
1667
1668         return(pending);
1669 }
1670
1671 __stdcall uint8_t
1672 ntoskrnl_read_timer(ktimer *timer)
1673 {
1674         return(timer->k_header.dh_sigstate);
1675 }
1676
1677 __stdcall static void
1678 dummy(void)
1679 {
1680         kprintf ("ntoskrnl dummy called...\n");
1681         return;
1682 }
1683
1684
1685 image_patch_table ntoskrnl_functbl[] = {
1686         { "RtlCompareMemory",           (FUNC)ntoskrnl_memcmp },
1687         { "RtlEqualUnicodeString",      (FUNC)ntoskrnl_unicode_equal },
1688         { "RtlCopyUnicodeString",       (FUNC)ntoskrnl_unicode_copy },
1689         { "RtlUnicodeStringToAnsiString", (FUNC)ntoskrnl_unicode_to_ansi },
1690         { "RtlAnsiStringToUnicodeString", (FUNC)ntoskrnl_ansi_to_unicode },
1691         { "RtlInitAnsiString",          (FUNC)ntoskrnl_init_ansi_string },
1692         { "RtlInitUnicodeString",       (FUNC)ntoskrnl_init_unicode_string },
1693         { "RtlFreeAnsiString",          (FUNC)ntoskrnl_free_ansi_string },
1694         { "RtlFreeUnicodeString",       (FUNC)ntoskrnl_free_unicode_string },
1695         { "RtlUnicodeStringToInteger",  (FUNC)ntoskrnl_unicode_to_int },
1696         { "sprintf",                    (FUNC)ksprintf },
1697         { "vsprintf",                   (FUNC)kvsprintf },
1698         { "_snprintf",                  (FUNC)ksnprintf },
1699         { "_vsnprintf",                 (FUNC)kvsnprintf },
1700         { "DbgPrint",                   (FUNC)ntoskrnl_dbgprint },
1701         { "DbgBreakPoint",              (FUNC)ntoskrnl_debugger },
1702         { "strncmp",                    (FUNC)strncmp },
1703         { "strcmp",                     (FUNC)strcmp },
1704         { "strncpy",                    (FUNC)strncpy },
1705         { "strcpy",                     (FUNC)strcpy },
1706         { "strlen",                     (FUNC)strlen },
1707         { "memcpy",                     (FUNC)memcpy },
1708         { "memmove",                    (FUNC)memcpy },
1709         { "memset",                     (FUNC)memset },
1710         { "IofCallDriver",              (FUNC)ntoskrnl_iofcalldriver },
1711         { "IofCompleteRequest",         (FUNC)ntoskrnl_iofcompletereq },
1712         { "IoBuildSynchronousFsdRequest", (FUNC)ntoskrnl_iobuildsynchfsdreq },
1713         { "KeWaitForSingleObject",      (FUNC)ntoskrnl_waitforobj },
1714         { "KeWaitForMultipleObjects",   (FUNC)ntoskrnl_waitforobjs },
1715         { "_allmul",                    (FUNC)_allmul },
1716         { "_alldiv",                    (FUNC)_alldiv },
1717         { "_allrem",                    (FUNC)_allrem },
1718         { "_allshr",                    (FUNC)_allshr },
1719         { "_allshl",                    (FUNC)_allshl },
1720         { "_aullmul",                   (FUNC)_aullmul },
1721         { "_aulldiv",                   (FUNC)_aulldiv },
1722         { "_aullrem",                   (FUNC)_aullrem },
1723         { "_aullshr",                   (FUNC)_aullshr },
1724         { "_aullshl",                   (FUNC)_aullshl },
1725         { "atoi",                       (FUNC)atoi },
1726         { "atol",                       (FUNC)atol },
1727         { "rand",                       (FUNC)rand },
1728         { "WRITE_REGISTER_USHORT",      (FUNC)ntoskrnl_writereg_ushort },
1729         { "READ_REGISTER_USHORT",       (FUNC)ntoskrnl_readreg_ushort },
1730         { "WRITE_REGISTER_ULONG",       (FUNC)ntoskrnl_writereg_ulong },
1731         { "READ_REGISTER_ULONG",        (FUNC)ntoskrnl_readreg_ulong },
1732         { "READ_REGISTER_UCHAR",        (FUNC)ntoskrnl_readreg_uchar },
1733         { "WRITE_REGISTER_UCHAR",       (FUNC)ntoskrnl_writereg_uchar },
1734         { "ExInitializePagedLookasideList", (FUNC)ntoskrnl_init_lookaside },
1735         { "ExDeletePagedLookasideList", (FUNC)ntoskrnl_delete_lookaside },
1736         { "ExInitializeNPagedLookasideList", (FUNC)ntoskrnl_init_nplookaside },
1737         { "ExDeleteNPagedLookasideList", (FUNC)ntoskrnl_delete_nplookaside },
1738         { "InterlockedPopEntrySList",   (FUNC)ntoskrnl_pop_slist },
1739         { "InterlockedPushEntrySList",  (FUNC)ntoskrnl_push_slist },
1740         { "ExInterlockedPopEntrySList", (FUNC)ntoskrnl_pop_slist_ex },
1741         { "ExInterlockedPushEntrySList",(FUNC)ntoskrnl_push_slist_ex },
1742         { "KefAcquireSpinLockAtDpcLevel", (FUNC)ntoskrnl_lock_dpc },
1743         { "KefReleaseSpinLockFromDpcLevel", (FUNC)ntoskrnl_unlock_dpc },
1744         { "InterlockedIncrement",       (FUNC)ntoskrnl_interlock_inc },
1745         { "InterlockedDecrement",       (FUNC)ntoskrnl_interlock_dec },
1746         { "ExInterlockedAddLargeStatistic",
1747                                         (FUNC)ntoskrnl_interlock_addstat },
1748         { "IoFreeMdl",                  (FUNC)ntoskrnl_freemdl },
1749         { "MmSizeOfMdl",                (FUNC)ntoskrnl_sizeofmdl },
1750         { "MmMapLockedPages",           (FUNC)ntoskrnl_mmaplockedpages },
1751         { "MmMapLockedPagesSpecifyCache",
1752                                         (FUNC)ntoskrnl_mmaplockedpages_cache },
1753         { "MmUnmapLockedPages",         (FUNC)ntoskrnl_munmaplockedpages },
1754         { "MmBuildMdlForNonPagedPool",  (FUNC)ntoskrnl_build_npaged_mdl },
1755         { "KeInitializeSpinLock",       (FUNC)ntoskrnl_init_lock },
1756         { "IoIsWdmVersionAvailable",    (FUNC)ntoskrnl_wdmver },
1757         { "IoGetDeviceProperty",        (FUNC)ntoskrnl_devprop },
1758         { "KeInitializeMutex",          (FUNC)ntoskrnl_init_mutex },
1759         { "KeReleaseMutex",             (FUNC)ntoskrnl_release_mutex },
1760         { "KeReadStateMutex",           (FUNC)ntoskrnl_read_mutex },
1761         { "KeInitializeEvent",          (FUNC)ntoskrnl_init_event },
1762         { "KeSetEvent",                 (FUNC)ntoskrnl_set_event },
1763         { "KeResetEvent",               (FUNC)ntoskrnl_reset_event },
1764         { "KeClearEvent",               (FUNC)ntoskrnl_clear_event },
1765         { "KeReadStateEvent",           (FUNC)ntoskrnl_read_event },
1766         { "KeInitializeTimer",          (FUNC)ntoskrnl_init_timer },
1767         { "KeInitializeTimerEx",        (FUNC)ntoskrnl_init_timer_ex },
1768         { "KeSetTimer",                 (FUNC)ntoskrnl_set_timer },
1769         { "KeSetTimerEx",               (FUNC)ntoskrnl_set_timer_ex },
1770         { "KeCancelTimer",              (FUNC)ntoskrnl_cancel_timer },
1771         { "KeReadStateTimer",           (FUNC)ntoskrnl_read_timer },
1772         { "KeInitializeDpc",            (FUNC)ntoskrnl_init_dpc },
1773         { "KeInsertQueueDpc",           (FUNC)ntoskrnl_queue_dpc },
1774         { "KeRemoveQueueDpc",           (FUNC)ntoskrnl_dequeue_dpc },
1775         { "ObReferenceObjectByHandle",  (FUNC)ntoskrnl_objref },
1776         { "ObfDereferenceObject",       (FUNC)ntoskrnl_objderef },
1777         { "ZwClose",                    (FUNC)ntoskrnl_zwclose },
1778         { "PsCreateSystemThread",       (FUNC)ntoskrnl_create_thread },
1779         { "PsTerminateSystemThread",    (FUNC)ntoskrnl_thread_exit },
1780
1781         /*
1782          * This last entry is a catch-all for any function we haven't
1783          * implemented yet. The PE import list patching routine will
1784          * use it for any function that doesn't have an explicit match
1785          * in this table.
1786          */
1787
1788         { NULL, (FUNC)dummy },
1789
1790         /* End of list. */
1791
1792         { NULL, NULL },
1793 };