Use tsc_frequency instead of tsc_freq.
[nvidia.git] / src / nvidia_os.c
CommitLineData
8b1b599b
SS
1/* _NVRM_COPYRIGHT_BEGIN_
2 *
3 * Copyright 2001-2002 by NVIDIA Corporation. All rights reserved. All
4 * information contained herein is proprietary and confidential to NVIDIA
5 * Corporation. Any use, reproduction, or disclosure without the written
6 * permission of NVIDIA Corporation is prohibited.
7 *
8 * _NVRM_COPYRIGHT_END_
9 */
10
11#include "nv-misc.h"
12#include "os-interface.h"
13#include "nv.h"
14#include "nv-freebsd.h"
15
16
bb9101fa
SS
17/* DragonFly compat */
18#include <machine/pmap_inval.h>
19
20void
21pmap_invalidate_range(struct pmap *pmap, vm_offset_t sva, vm_offset_t eva)
22{
23 struct pmap_inval_info info;
24 vm_offset_t va;
25
26 pmap_inval_init(&info);
27 for (va = sva; va < eva; va += PAGE_SIZE)
28 pmap_inval_add(&info, pmap, va);
29 pmap_inval_flush(&info);
30}
31
32
8b1b599b
SS
33/*
34 * The NVIDIA kernel module's malloc identifier, needed for both tracking
35 * and actual allocation/freeing purposes. M_NVIDIA is declared elsewhere
36 * to make it known to other parts of the kernel module (nv-freebsd.h).
37 */
38
39MALLOC_DEFINE(M_NVIDIA, "nvidia", "NVIDIA memory allocations");
40
41
42#define MAX_ERROR_STRING 256
43
44RM_STATUS NV_API_CALL os_alloc_contig_pages(
45 void **address,
46 U032 size
47)
48{
49 *address = contigmalloc(size, M_NVIDIA, 0, 0, ~0, PAGE_SIZE, 0);
50 return *address ? RM_OK : RM_ERROR;
51}
52
53void NV_API_CALL os_free_contig_pages(
54 void *address,
55 U032 size
56)
57{
58 contigfree(address, size, M_NVIDIA);
59}
60
61/*
62 * The core resource manager's favorite source of memory, this routine is
63 * called from different contexts, including ISRs. This means that it can
64 * not be allowed to sleep when memory is low.
65 */
66
67RM_STATUS NV_API_CALL os_alloc_mem(
68 void **address,
69 U032 size
70)
71{
72 /* XXX Fix me? (malloc flags) */
d5c62808 73 *address = malloc(size, M_NVIDIA, M_INTWAIT | M_ZERO | M_NULLOK);
8b1b599b
SS
74 return *address ? RM_OK : RM_ERROR;
75}
76
77void NV_API_CALL os_free_mem(void *address)
78{
79 free(address, M_NVIDIA);
80}
81
82#define NV_MSECS_PER_TICK (1000 / hz)
83#define NV_MSECS_TO_TICKS(ms) ((ms) * hz / 1000)
84#define NV_USECS_PER_TICK (1000000 / hz)
85#define NV_USECS_TO_TICKS(us) ((us) * hz / 1000000)
86
87RM_STATUS NV_API_CALL os_delay(U032 MilliSeconds)
88{
89 unsigned long MicroSeconds;
90 unsigned long ticks;
91 struct timeval tv_end, tv_aux;
92
93 getmicrotime(&tv_aux);
94
95 if (__NV_ITHREAD() && (MilliSeconds > NV_MAX_ISR_DELAY_MS))
96 return RM_ERROR;
97
98 if (__NV_ITHREAD()) {
99 DELAY(MilliSeconds * 1000);
100 return RM_OK;
101 }
102
103 MicroSeconds = MilliSeconds * 1000;
104 tv_end.tv_usec = MicroSeconds;
105 tv_end.tv_sec = 0;
106 /* tv_end = tv_aux + tv_end */
107 NV_TIMERADD(&tv_aux, &tv_end, &tv_end);
108
109 ticks = NV_USECS_TO_TICKS(MicroSeconds);
110
111 if (ticks > 0) {
112 do {
bb9101fa 113 tsleep((void *)os_delay, PCATCH, "delay", ticks);
8b1b599b
SS
114 getmicrotime(&tv_aux);
115 if (NV_TIMERCMP(&tv_aux, &tv_end, <)) {
116 /* tv_aux = tv_end - tv_aux */
117 NV_TIMERSUB(&tv_end, &tv_aux, &tv_aux);
118 MicroSeconds = tv_aux.tv_usec + (tv_aux.tv_sec * 1000000);
119 } else
120 MicroSeconds = 0;
121 } while ((ticks = NV_USECS_TO_TICKS(MicroSeconds)) > 0);
122 }
123
124 if (MicroSeconds > 0)
125 DELAY(MicroSeconds);
126
127 return RM_OK;
128}
129
130RM_STATUS NV_API_CALL os_delay_us(U032 MicroSeconds)
131{
132 if (__NV_ITHREAD() && (MicroSeconds > NV_MAX_ISR_DELAY_US))
133 return RM_ERROR;
134 DELAY(MicroSeconds);
135 return RM_OK;
136}
137
138U032 NV_API_CALL os_get_cpu_frequency(void)
139{
236fdf02 140 return ((tsc_frequency + 4999) / 1000000);
8b1b599b
SS
141}
142
143U032 NV_API_CALL os_get_current_process(void)
144{
145 return curproc->p_pid;
146}
147
148RM_STATUS NV_API_CALL os_kill_process(
149 U032 pid,
150 U032 sig
151)
152{
153 struct proc *p = pfind(pid);
154 if (!p) {
155 /*
156 * No process with the specified PID was found running. We notify
157 * the caller, but take no other action.
158 */
159 return RM_ERR_OPERATING_SYSTEM;
160 }
161
bb9101fa 162 ksignal(p, sig);
8b1b599b
SS
163
164 return RM_OK;
165}
166
167RM_STATUS NV_API_CALL os_get_current_time(
168 U032 *sec,
169 U032 *usec
170)
171{
172 struct timeval tv;
173
174 getmicrotime(&tv);
175
176 *sec = tv.tv_sec;
177 *usec = tv.tv_usec;
178
179 return RM_OK;
180}
181
182BOOL NV_API_CALL os_is_administrator(PHWINFO pDev)
183{
184 return suser(CURTHREAD) ? FALSE : TRUE;
185}
186
187U008 NV_API_CALL os_io_read_byte(
188 PHWINFO pDev,
189 U032 address
190)
191{
192 /* XXX Fix me? (bus_space access) */
193 return inb(address);
194}
195
196void NV_API_CALL os_io_write_byte(
197 PHWINFO pDev,
198 U032 address,
199 U008 value
200)
201{
202 /* XXX Fix me? (bus_space access) */
203 outb(address, value);
204}
205
206U016 NV_API_CALL os_io_read_word(
207 PHWINFO pDev,
208 U032 address
209)
210{
211 /* XXX Fix me? (bus_space access) */
212 return inw(address);
213}
214
215void NV_API_CALL os_io_write_word(
216 PHWINFO pDev,
217 U032 address,
218 U016 value
219)
220{
221 /* XXX Fix me? (bus_space access) */
222 return outw(address, value);
223}
224
225U032 NV_API_CALL os_io_read_dword(
226 PHWINFO pDev,
227 U032 address
228)
229{
230 /* XXX Fix me? (bus_space access) */
231 return inl(address);
232}
233
234void NV_API_CALL os_io_write_dword(
235 PHWINFO pDev,
236 U032 address,
237 U032 value
238)
239{
240 /* XXX Fix me? (bus_space access) */
241 outl(address, value);
242}
243
244/*
245 * XXX On FreeBSD/amd64, pmap_mapdev() returns an address range from the
246 * kernel's direct mapping of the first 4+GB; this direct mapping uses
247 * 2MB pages and may be cached. Ideally, we'd want to modify it for as long
248 * as we need uncached access to the I/O memory in question, but for now
249 * we'll have to create a private, uncached mapping to be safe.
250 *
251 * XXX This introduces aliasing concerns, but it's better than using the
252 * cached direct mapping.
253 */
254
255void* NV_API_CALL os_map_kernel_space(
256 NvU64 start,
257 NvU64 size,
258 U032 mode
259)
260{
261 void *vm;
262 U032 cache_bits = (mode != NV_MEMORY_CACHED) ? PG_N : 0;
263 vm_offset_t tva, va;
264
265#if defined(NVCPU_X86) && !defined(PAE)
266 if (start > 0xffffffff)
267 return NULL;
268#endif
269
270 if (start & PAGE_MASK)
271 return NULL;
272
273 size = NV_ALIGN_UP(size, PAGE_SIZE);
274
bb9101fa 275 va = kmem_alloc_nofault(&kernel_map, size);
8b1b599b
SS
276 vm = (void *)va;
277
278 if (vm != NULL) {
279 for (tva = va; tva < (va + size); tva += PAGE_SIZE) {
280 pt_entry_t *ptep = vtopte(tva);
281 pte_store(ptep, start | PG_RW | PG_V | PG_G | cache_bits);
282 start += PAGE_SIZE;
283 }
bb9101fa 284 pmap_invalidate_range(&kernel_pmap, va, tva);
8b1b599b
SS
285 }
286
287 return vm;
288}
289
290void NV_API_CALL os_unmap_kernel_space(
291 void *address,
292 NvU64 size
293)
294{
295 vm_offset_t tva, va;
296
297 size = NV_ALIGN_UP(size, PAGE_SIZE);
298
299 va = (vm_offset_t) address;
300 if (va != 0) {
301 for (tva = va; tva < (va + size); tva += PAGE_SIZE)
302 pte_clear(vtopte(tva));
bb9101fa
SS
303 pmap_invalidate_range(&kernel_pmap, va, tva);
304 kmem_free(&kernel_map, va, size);
8b1b599b
SS
305 }
306}
307
308void* NV_API_CALL os_map_kernel_space_high(
309 U032 pfn,
310 U032 size
311)
312{
313 U032 start;
314 if (!(pfn & ~0xfffff)) {
315 start = pfn << PAGE_SHIFT;
316 return os_map_kernel_space(start, size, NV_MEMORY_CACHED);
317 }
318 return NULL;
319}
320
321void NV_API_CALL os_unmap_kernel_space_high(
322 void *addr,
323 U032 pfn,
324 U032 size
325)
326{
327 os_unmap_kernel_space(addr, size);
328}
329
330RM_STATUS NV_API_CALL os_set_mem_range(
331 U032 start,
332 U032 size,
333 U032 mode
334)
335{
336 int arg;
337 struct mem_range_desc mrd;
338
339 mrd.mr_base = start;
340 mrd.mr_len = size;
341 mrd.mr_flags = MDF_WRITECOMBINE;
342
343 strcpy(mrd.mr_owner, "NVIDIA");
344 arg = MEMRANGE_SET_UPDATE;
345
346 if (mem_range_attr_set(&mrd, &arg))
347 return RM_ERROR;
348
349 return RM_OK;
350}
351
352RM_STATUS NV_API_CALL os_unset_mem_range(
353 U032 start,
354 U032 size
355)
356{
357 int arg;
358 struct mem_range_desc mrd;
359
360 mrd.mr_base = start;
361 mrd.mr_len = size;
362 arg = MEMRANGE_SET_REMOVE;
363
364 if (mem_range_attr_set(&mrd, &arg))
365 return RM_ERROR;
366
367 return RM_OK;
368}
369
370
371/*
372 * The current debug level is used to determine if certain debug messages
373 * are printed to the system console/log files or not. It defaults to the
374 * highest debug level, i.e. the lowest debug output.
375 */
376
377U032 cur_debuglevel = 0xffffffff;
378
379void NV_API_CALL os_dbg_init(void)
380{
381 U032 new_debuglevel;
382 nv_stack_t *sp;
383
384 NV_UMA_ZONE_ALLOC_STACK(sp);
385 if (sp == NULL)
386 return;
387
388 if (rm_read_registry_dword(sp, NULL, "NVreg", "ResmanDebugLevel",
389 &new_debuglevel) == RM_OK) {
390 if (new_debuglevel != 0xffffffff)
391 cur_debuglevel = new_debuglevel;
392 }
393
394 NV_UMA_ZONE_FREE_STACK(sp);
395}
396
397void NV_API_CALL os_dbg_set_level(U032 new_debuglevel)
398{
399 cur_debuglevel = new_debuglevel;
400}
401
402void NV_API_CALL NV_STACKWATCH_CALLBACK os_dbg_breakpoint(void)
403{
404#ifdef DEBUG
0fcfa2d0 405 Debugger("nv");
8b1b599b
SS
406#endif
407}
408
409void NV_API_CALL out_string(const char *message)
410{
411#if defined(DEBUG) || defined(QA_BUILD)
d5c62808 412 kprintf("%s", message);
8b1b599b
SS
413#endif
414}
415
416static char nv_error_string[MAX_ERROR_STRING];
417
418int NV_API_CALL nv_printf(
419 U032 debuglevel,
420 const char *format,
421 ...
422)
423{
424 char *message = nv_error_string;
bb9101fa 425 __va_list arglist;
8b1b599b
SS
426 int chars_written = 0;
427
428 if (debuglevel >= ((cur_debuglevel >> 4) & 3)) {
bb9101fa
SS
429 __va_start(arglist, format);
430 chars_written = kvsprintf(message, format, arglist);
431 __va_end(arglist);
432 kprintf("%s", message);
8b1b599b
SS
433 }
434
435 return chars_written;
436}
437
438int NV_API_CALL nv_snprintf(
439 char *buf,
440 unsigned int size,
441 const char *fmt,
442 ...
443)
444{
bb9101fa 445 __va_list arglist;
8b1b599b
SS
446 int chars_written;
447
bb9101fa
SS
448 __va_start(arglist, fmt);
449 chars_written = kvsnprintf(buf, size, fmt, arglist);
450 __va_end(arglist);
8b1b599b
SS
451
452 return chars_written;
453}
454
455void NV_API_CALL nv_os_log(
456 int loglevel,
457 const char *fmt,
458 void *ap
459)
460{
461 int l;
bb9101fa 462 ksprintf(nv_error_string, "NVRM: ");
8b1b599b 463 l = strlen(nv_error_string);
bb9101fa
SS
464 kvsnprintf(nv_error_string + l, MAX_ERROR_STRING - l, fmt, ap);
465 kprintf("%s", nv_error_string);
8b1b599b
SS
466}
467
468S032 NV_API_CALL os_mem_cmp(
469 const U008 *buf0,
470 const U008 *buf1,
471 U032 length
472)
473{
474 return memcmp(buf0, buf1, length);
475}
476
477U008* NV_API_CALL os_mem_copy(
478 U008 *dst,
479 const U008 *src,
480 U032 length
481)
482{
483#if defined(NVCPU_X86_64)
484 uint32_t i;
485 for (i = 0; i < length; i++) dst[i] = src[i];
486 return dst;
487#else
488 return memcpy(dst, src, length);
489#endif
490}
491
492RM_STATUS NV_API_CALL os_memcpy_from_user(
493 void *dst,
494 const void *src,
495 U032 length
496)
497{
bb9101fa 498 if (src < (void *) VM_MAX_USER_ADDRESS)
8b1b599b
SS
499 return copyin(src, dst, length) ? RM_ERR_INVALID_POINTER : RM_OK;
500
501 return os_mem_copy(dst, src, length) ? RM_ERR_INVALID_POINTER : RM_OK;
502}
503
504RM_STATUS NV_API_CALL os_memcpy_to_user(
505 void *dst,
506 const void *src,
507 U032 length
508)
509{
bb9101fa 510 if (dst < (void *) VM_MAX_USER_ADDRESS)
8b1b599b
SS
511 return copyout(src, dst, length) ? RM_ERR_INVALID_POINTER : RM_OK;
512
513 return os_mem_copy(dst, src, length) ? RM_ERR_INVALID_POINTER : RM_OK;
514}
515
516void* NV_API_CALL os_mem_set(
517 void *b,
518 U008 c,
519 U032 length
520)
521{
522 return memset(b, c, length);
523}
524
525S032 NV_API_CALL os_string_compare(
526 const U008 *s1,
527 const U008 *s2
528)
529{
530 return strcmp(s1, s2);
531}
532
533U008* NV_API_CALL os_string_copy(
534 U008 *dst,
535 const U008 *src
536)
537{
538 return strcpy(dst, src);
539}
540
541U032 NV_API_CALL os_string_length(const U008* s)
542{
543 return strlen(s);
544}
545
546RM_STATUS NV_API_CALL os_strncpy_from_user(
547 U008 *dst,
548 const U008 *src,
549 U032 n
550)
551{
552 return copyinstr(src, dst, n, NULL) ? RM_ERR_INVALID_POINTER : RM_OK;
553}
554
555U032 NV_API_CALL os_get_page_size(void)
556{
557 return PAGE_SIZE;
558}
559
560NvU64 NV_API_CALL os_get_page_mask(void)
561{
562 /*
563 * On FreeBSD, PAGE_MASK means (PAGE_SIZE - 1); on Linux it means the
564 * opposite, ~(PAGE_SIZE - 1); that is what this function is expected
565 * to return.
566 */
567 return ~PAGE_MASK;
568}
569
570NvU64 NV_API_CALL os_get_system_memory_size(void)
571{
572 return ((NvU64)physmem * PAGE_SIZE) / RM_PAGE_SIZE;
573}
574
575U032 NV_API_CALL os_get_cpu_count(void)
576{
bb9101fa 577 return ncpus;
8b1b599b
SS
578}
579
580RM_STATUS NV_API_CALL os_flush_cpu_cache(void)
581{
582 /*
583 * XXX This will do for now, but this may need to be extended
584 * to make IPI calls (flushing all caches).
585 */
586 __asm__ __volatile__("wbinvd": : :"memory");
587 return RM_OK;
588}
589
590void NV_API_CALL os_flush_cpu_write_combine_buffer(void)
591{
592 __asm__ __volatile__("sfence": : :"memory");
593}
594
595RM_STATUS NV_API_CALL os_raise_smp_barrier(void)
596{
597 return RM_OK;
598}
599
600RM_STATUS NV_API_CALL os_clear_smp_barrier(void)
601{
602 return RM_OK;
603}
604
605struct os_mutex {
606 nv_stack_t *sp;
bb9101fa 607 struct spinlock lock;
8b1b599b
SS
608 int refcnt;
609};
610
611RM_STATUS NV_API_CALL os_alloc_sema(void **semaphore)
612{
613 nv_stack_t *sp;
614 struct os_mutex *mtx;
615 RM_STATUS status;
616
617 NV_UMA_ZONE_ALLOC_STACK(sp);
618 if (sp == NULL)
619 return RM_ERR_NO_FREE_MEM;
620
621 status = os_alloc_mem((void **)&mtx, sizeof(struct os_mutex));
622 if (status != RM_OK) {
623 NV_UMA_ZONE_FREE_STACK(sp);
624 return status;
625 }
626
bb9101fa 627 spin_init(&mtx->lock);
8b1b599b
SS
628
629 mtx->sp = sp;
630 mtx->refcnt = 1;
631
632 *semaphore = (void *) mtx;
633
634 return RM_OK;
635}
636
637RM_STATUS NV_API_CALL os_free_sema(void *semaphore)
638{
639 struct os_mutex *mtx = semaphore;
640 nv_stack_t *sp;
641
642 sp = mtx->sp;
643 NV_UMA_ZONE_FREE_STACK(sp);
644
bb9101fa 645 spin_uninit(&mtx->lock);
8b1b599b
SS
646
647 os_free_mem(semaphore);
648
649 return RM_OK;
650}
651
652RM_STATUS NV_API_CALL os_acquire_sema(void *semaphore)
653{
654 struct os_mutex *mtx = semaphore;
655
bb9101fa 656 spin_lock_wr(&mtx->lock);
8b1b599b
SS
657 if (mtx->refcnt > 0)
658 rm_disable_interrupts(mtx->sp);
659 mtx->refcnt--;
660 if (mtx->refcnt < 0)
bb9101fa
SS
661 msleep(mtx, &mtx->lock, 0, "nvsemaq", 0);
662 spin_unlock_wr(&mtx->lock);
8b1b599b
SS
663
664 return RM_OK;
665}
666
667BOOL NV_API_CALL os_cond_acquire_sema(void *semaphore)
668{
669 struct os_mutex *mtx = semaphore;
670
bb9101fa 671 spin_lock_wr(&mtx->lock);
8b1b599b 672 if (mtx->refcnt < 1) {
bb9101fa 673 spin_unlock_wr(&mtx->lock);
8b1b599b
SS
674 return FALSE;
675 } else {
676 rm_disable_interrupts(mtx->sp);
677 mtx->refcnt--;
bb9101fa 678 spin_unlock_wr(&mtx->lock);
8b1b599b
SS
679 }
680
681 return TRUE;
682}
683
684RM_STATUS NV_API_CALL os_release_sema(void *semaphore)
685{
686 struct os_mutex *mtx = semaphore;
687
bb9101fa 688 spin_lock_wr(&mtx->lock);
8b1b599b 689 if (mtx->refcnt < 0)
bb9101fa 690 wakeup_one(mtx);
8b1b599b
SS
691 else
692 rm_enable_interrupts(mtx->sp);
693 mtx->refcnt++;
bb9101fa 694 spin_unlock_wr(&mtx->lock);
8b1b599b
SS
695
696 return RM_OK;
697}
698
699BOOL NV_API_CALL os_is_acquired_sema(void *semaphore)
700{
701 struct os_mutex *mtx = semaphore;
702 return (mtx->refcnt < 1);
703}
704
705BOOL NV_API_CALL os_pat_supported(void)
706{
707 /*
708 * FreeBSD has no native PAT support and there's no good
709 * way to implement it privately as we do on Linux.
710 */
711 return FALSE;
712}
713
714void* NV_API_CALL NV_STACKWATCH_CALLBACK os_get_stack_start(void *stack_pointer)
715{
bb9101fa 716 return (curthread->td_sp);
8b1b599b
SS
717}
718
719NvU64 NV_API_CALL os_get_current_pdpte(U032 address)
720{
721 return 0;
722}
723
724RM_STATUS NV_API_CALL os_set_mlock_capability()
725{
726 return (RM_ERROR);
727}
728
729S032 NV_API_CALL os_mlock_user_memory(
730 void *address,
731 U032 length
732)
733{
734 return -1;
735}
736
737S032 NV_API_CALL os_munlock_user_memory(
738 void *address,
739 U032 length
740)
741{
742 return -1;
743}
744
745RM_STATUS NV_API_CALL os_check_process_map_limit(
746 NvU64 proc_max_map_count
747)
748{
749 return (RM_ERROR);
750}
751
752void NV_API_CALL os_register_compatible_ioctl(
753 U032 cmd,
754 U032 size
755)
756{
757}
758
759void NV_API_CALL os_unregister_compatible_ioctl(
760 U032 cmd,
761 U032 size
762)
763{
764}
765
766RM_STATUS NV_API_CALL os_disable_console_access(void)
767{
768 return RM_OK;
769}
770
771RM_STATUS NV_API_CALL os_enable_console_access(void)
772{
773 return RM_OK;
774}
775