kernel: Make SMP support default (and non-optional).
[dragonfly.git] / sys / kern / kern_memio.c
CommitLineData
984263bc
MD
1/*-
2 * Copyright (c) 1988 University of Utah.
3 * Copyright (c) 1982, 1986, 1990 The Regents of the University of California.
4 * All rights reserved.
5 *
6 * This code is derived from software contributed to Berkeley by
7 * the Systems Programming Group of the University of Utah Computer
8 * Science Department, and code derived from software contributed to
9 * Berkeley by William Jolitz.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 * 3. All advertising materials mentioning features or use of this software
20 * must display the following acknowledgement:
21 * This product includes software developed by the University of
22 * California, Berkeley and its contributors.
23 * 4. Neither the name of the University nor the names of its contributors
24 * may be used to endorse or promote products derived from this software
25 * without specific prior written permission.
26 *
27 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
28 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
29 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
30 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
31 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
32 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
33 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
34 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
35 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
36 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
37 * SUCH DAMAGE.
38 *
39 * from: Utah $Hdr: mem.c 1.13 89/10/08$
40 * from: @(#)mem.c 7.2 (Berkeley) 5/9/91
41 * $FreeBSD: src/sys/i386/i386/mem.c,v 1.79.2.9 2003/01/04 22:58:01 njl Exp $
42 */
43
44/*
45 * Memory special file
46 */
47
48#include <sys/param.h>
49#include <sys/systm.h>
50#include <sys/buf.h>
51#include <sys/conf.h>
52#include <sys/fcntl.h>
53#include <sys/filio.h>
984263bc
MD
54#include <sys/kernel.h>
55#include <sys/malloc.h>
56#include <sys/memrange.h>
57#include <sys/proc.h>
895c1f85 58#include <sys/priv.h>
984263bc
MD
59#include <sys/random.h>
60#include <sys/signalvar.h>
61#include <sys/uio.h>
62#include <sys/vnode.h>
63
9f889dc4
MD
64#include <sys/signal2.h>
65#include <sys/mplock2.h>
66
984263bc
MD
67#include <vm/vm.h>
68#include <vm/pmap.h>
69#include <vm/vm_extern.h>
70
71
72static d_open_t mmopen;
73static d_close_t mmclose;
fef8985e
MD
74static d_read_t mmread;
75static d_write_t mmwrite;
984263bc
MD
76static d_ioctl_t mmioctl;
77static d_mmap_t memmmap;
6dcd6ba9 78static d_kqfilter_t mmkqfilter;
984263bc
MD
79
80#define CDEV_MAJOR 2
fef8985e 81static struct dev_ops mem_ops = {
88abd8b5 82 { "mem", 0, D_MPSAFE },
fef8985e
MD
83 .d_open = mmopen,
84 .d_close = mmclose,
85 .d_read = mmread,
86 .d_write = mmwrite,
87 .d_ioctl = mmioctl,
6dcd6ba9 88 .d_kqfilter = mmkqfilter,
fef8985e 89 .d_mmap = memmmap,
984263bc
MD
90};
91
7e071e7a 92static int rand_bolt;
984263bc 93static caddr_t zbuf;
b9a1f8e1 94static cdev_t zerodev = NULL;
984263bc
MD
95
96MALLOC_DEFINE(M_MEMDESC, "memdesc", "memory range descriptors");
b13267a5
MD
97static int mem_ioctl (cdev_t, u_long, caddr_t, int, struct ucred *);
98static int random_ioctl (cdev_t, u_long, caddr_t, int, struct ucred *);
984263bc
MD
99
100struct mem_range_softc mem_range_softc;
101
102
103static int
fef8985e 104mmopen(struct dev_open_args *ap)
984263bc 105{
b13267a5 106 cdev_t dev = ap->a_head.a_dev;
fef8985e 107 int error;
41c20dac 108
984263bc 109 switch (minor(dev)) {
fef8985e
MD
110 case 0:
111 case 1:
7e42c007
MD
112 if (ap->a_oflags & FWRITE) {
113 if (securelevel > 0 || kernel_mem_readonly)
114 return (EPERM);
115 }
fc26c9f1 116 error = 0;
fef8985e 117 break;
984263bc 118 case 14:
895c1f85 119 error = priv_check_cred(ap->a_cred, PRIV_ROOT, 0);
fef8985e 120 if (error != 0)
fc26c9f1 121 break;
7e42c007 122 if (securelevel > 0 || kernel_mem_readonly) {
fc26c9f1
MD
123 error = EPERM;
124 break;
125 }
126 error = cpu_set_iopl();
984263bc
MD
127 break;
128 default:
fc26c9f1 129 error = 0;
984263bc
MD
130 break;
131 }
fc26c9f1 132 return (error);
984263bc
MD
133}
134
135static int
fef8985e 136mmclose(struct dev_close_args *ap)
984263bc 137{
b13267a5 138 cdev_t dev = ap->a_head.a_dev;
fc26c9f1 139 int error;
984263bc
MD
140
141 switch (minor(dev)) {
984263bc 142 case 14:
fc26c9f1 143 error = cpu_clr_iopl();
984263bc
MD
144 break;
145 default:
fc26c9f1 146 error = 0;
984263bc
MD
147 break;
148 }
fc26c9f1 149 return (error);
984263bc
MD
150}
151
fef8985e 152
984263bc 153static int
b13267a5 154mmrw(cdev_t dev, struct uio *uio, int flags)
984263bc 155{
c9faf524 156 int o;
74fa35b9 157 u_int c;
984263bc 158 u_int poolsize;
74fa35b9 159 u_long v;
c9faf524 160 struct iovec *iov;
984263bc
MD
161 int error = 0;
162 caddr_t buf = NULL;
163
164 while (uio->uio_resid > 0 && error == 0) {
165 iov = uio->uio_iov;
166 if (iov->iov_len == 0) {
167 uio->uio_iov++;
168 uio->uio_iovcnt--;
169 if (uio->uio_iovcnt < 0)
170 panic("mmrw");
171 continue;
172 }
173 switch (minor(dev)) {
984263bc 174 case 0:
207bedb9
MD
175 /*
176 * minor device 0 is physical memory, /dev/mem
177 */
984263bc 178 v = uio->uio_offset;
74fa35b9 179 v &= ~(long)PAGE_MASK;
984263bc
MD
180 pmap_kenter((vm_offset_t)ptvmmap, v);
181 o = (int)uio->uio_offset & PAGE_MASK;
973c11b9 182 c = (u_int)(PAGE_SIZE - ((uintptr_t)iov->iov_base & PAGE_MASK));
984263bc
MD
183 c = min(c, (u_int)(PAGE_SIZE - o));
184 c = min(c, (u_int)iov->iov_len);
185 error = uiomove((caddr_t)&ptvmmap[o], (int)c, uio);
186 pmap_kremove((vm_offset_t)ptvmmap);
187 continue;
188
984263bc 189 case 1: {
207bedb9
MD
190 /*
191 * minor device 1 is kernel memory, /dev/kmem
192 */
c439ad8f 193 vm_offset_t saddr, eaddr;
460426e6
MD
194 int prot;
195
984263bc
MD
196 c = iov->iov_len;
197
198 /*
03debc4a
MD
199 * Make sure that all of the pages are currently
200 * resident so that we don't create any zero-fill
201 * pages.
984263bc 202 */
c439ad8f 203 saddr = trunc_page(uio->uio_offset);
984263bc 204 eaddr = round_page(uio->uio_offset + c);
460426e6 205 if (saddr > eaddr)
984263bc 206 return EFAULT;
460426e6
MD
207
208 /*
209 * Make sure the kernel addresses are mapped.
210 * platform_direct_mapped() can be used to bypass
211 * default mapping via the page table (virtual kernels
212 * contain a lot of out-of-band data).
213 */
214 prot = VM_PROT_READ;
215 if (uio->uio_rw != UIO_READ)
216 prot |= VM_PROT_WRITE;
217 error = kvm_access_check(saddr, eaddr, prot);
218 if (error)
219 return (error);
220 error = uiomove((caddr_t)(vm_offset_t)uio->uio_offset,
221 (int)c, uio);
984263bc
MD
222 continue;
223 }
984263bc 224 case 2:
207bedb9 225 /*
2ca4923a 226 * minor device 2 (/dev/null) is EOF/RATHOLE
207bedb9 227 */
984263bc
MD
228 if (uio->uio_rw == UIO_READ)
229 return (0);
230 c = iov->iov_len;
231 break;
984263bc 232 case 3:
207bedb9
MD
233 /*
234 * minor device 3 (/dev/random) is source of filth
c26b3d4d 235 * on read, seeder on write
207bedb9 236 */
984263bc 237 if (buf == NULL)
efda3bd0 238 buf = kmalloc(PAGE_SIZE, M_TEMP, M_WAITOK);
984263bc 239 c = min(iov->iov_len, PAGE_SIZE);
c26b3d4d
MD
240 if (uio->uio_rw == UIO_WRITE) {
241 error = uiomove(buf, (int)c, uio);
242 if (error == 0)
243 error = add_buffer_randomness(buf, c);
244 } else {
245 poolsize = read_random(buf, c);
246 if (poolsize == 0) {
247 if (buf)
efda3bd0 248 kfree(buf, M_TEMP);
c26b3d4d
MD
249 if ((flags & IO_NDELAY) != 0)
250 return (EWOULDBLOCK);
251 return (0);
252 }
253 c = min(c, poolsize);
254 error = uiomove(buf, (int)c, uio);
984263bc 255 }
984263bc 256 continue;
984263bc 257 case 4:
207bedb9
MD
258 /*
259 * minor device 4 (/dev/urandom) is source of muck
c26b3d4d 260 * on read, writes are disallowed.
207bedb9 261 */
c26b3d4d 262 c = min(iov->iov_len, PAGE_SIZE);
984263bc 263 if (uio->uio_rw == UIO_WRITE) {
c26b3d4d 264 error = EPERM;
984263bc
MD
265 break;
266 }
08f2f1bb 267 if (CURSIG(curthread->td_lwp) != 0) {
984263bc
MD
268 /*
269 * Use tsleep() to get the error code right.
270 * It should return immediately.
271 */
377d4740 272 error = tsleep(&rand_bolt, PCATCH, "urand", 1);
984263bc
MD
273 if (error != 0 && error != EWOULDBLOCK)
274 continue;
275 }
276 if (buf == NULL)
efda3bd0 277 buf = kmalloc(PAGE_SIZE, M_TEMP, M_WAITOK);
984263bc
MD
278 poolsize = read_random_unlimited(buf, c);
279 c = min(c, poolsize);
280 error = uiomove(buf, (int)c, uio);
281 continue;
984263bc 282 case 12:
207bedb9
MD
283 /*
284 * minor device 12 (/dev/zero) is source of nulls
c26b3d4d 285 * on read, write are disallowed.
207bedb9 286 */
984263bc
MD
287 if (uio->uio_rw == UIO_WRITE) {
288 c = iov->iov_len;
289 break;
290 }
291 if (zbuf == NULL) {
e7b4468c
SW
292 zbuf = (caddr_t)kmalloc(PAGE_SIZE, M_TEMP,
293 M_WAITOK | M_ZERO);
984263bc
MD
294 }
295 c = min(iov->iov_len, PAGE_SIZE);
296 error = uiomove(zbuf, (int)c, uio);
297 continue;
984263bc
MD
298 default:
299 return (ENODEV);
300 }
301 if (error)
302 break;
656849c6 303 iov->iov_base = (char *)iov->iov_base + c;
984263bc
MD
304 iov->iov_len -= c;
305 uio->uio_offset += c;
306 uio->uio_resid -= c;
307 }
308 if (buf)
efda3bd0 309 kfree(buf, M_TEMP);
984263bc
MD
310 return (error);
311}
312
fef8985e
MD
313static int
314mmread(struct dev_read_args *ap)
315{
316 return(mmrw(ap->a_head.a_dev, ap->a_uio, ap->a_ioflag));
317}
318
319static int
320mmwrite(struct dev_write_args *ap)
321{
322 return(mmrw(ap->a_head.a_dev, ap->a_uio, ap->a_ioflag));
323}
324
325
984263bc
MD
326
327
328
329/*******************************************************\
330* allow user processes to MMAP some memory sections *
331* instead of going through read/write *
332\*******************************************************/
fef8985e 333
984263bc 334static int
fef8985e 335memmmap(struct dev_mmap_args *ap)
984263bc 336{
b13267a5 337 cdev_t dev = ap->a_head.a_dev;
984263bc 338
fef8985e 339 switch (minor(dev)) {
984263bc 340 case 0:
fef8985e
MD
341 /*
342 * minor device 0 is physical memory
343 */
8d565796 344#if defined(__i386__)
fef8985e 345 ap->a_result = i386_btop(ap->a_offset);
b2b3ffcd
SS
346#elif defined(__x86_64__)
347 ap->a_result = x86_64_btop(ap->a_offset);
8d565796 348#endif
fef8985e 349 return 0;
984263bc 350 case 1:
fef8985e
MD
351 /*
352 * minor device 1 is kernel memory
353 */
8d565796 354#if defined(__i386__)
fef8985e 355 ap->a_result = i386_btop(vtophys(ap->a_offset));
b2b3ffcd
SS
356#elif defined(__x86_64__)
357 ap->a_result = x86_64_btop(vtophys(ap->a_offset));
8d565796 358#endif
fef8985e 359 return 0;
984263bc
MD
360
361 default:
fef8985e 362 return EINVAL;
984263bc
MD
363 }
364}
365
366static int
fef8985e 367mmioctl(struct dev_ioctl_args *ap)
984263bc 368{
b13267a5 369 cdev_t dev = ap->a_head.a_dev;
9f889dc4
MD
370 int error;
371
372 get_mplock();
984263bc
MD
373
374 switch (minor(dev)) {
375 case 0:
9f889dc4
MD
376 error = mem_ioctl(dev, ap->a_cmd, ap->a_data,
377 ap->a_fflag, ap->a_cred);
378 break;
984263bc
MD
379 case 3:
380 case 4:
9f889dc4
MD
381 error = random_ioctl(dev, ap->a_cmd, ap->a_data,
382 ap->a_fflag, ap->a_cred);
383 break;
384 default:
385 error = ENODEV;
386 break;
984263bc 387 }
9f889dc4
MD
388
389 rel_mplock();
390 return (error);
984263bc
MD
391}
392
393/*
394 * Operations for changing memory attributes.
395 *
396 * This is basically just an ioctl shim for mem_range_attr_get
397 * and mem_range_attr_set.
398 */
399static int
b13267a5 400mem_ioctl(cdev_t dev, u_long cmd, caddr_t data, int flags, struct ucred *cred)
984263bc
MD
401{
402 int nd, error = 0;
403 struct mem_range_op *mo = (struct mem_range_op *)data;
404 struct mem_range_desc *md;
405
406 /* is this for us? */
407 if ((cmd != MEMRANGE_GET) &&
408 (cmd != MEMRANGE_SET))
409 return (ENOTTY);
410
411 /* any chance we can handle this? */
412 if (mem_range_softc.mr_op == NULL)
413 return (EOPNOTSUPP);
414
415 /* do we have any descriptors? */
416 if (mem_range_softc.mr_ndesc == 0)
417 return (ENXIO);
418
419 switch (cmd) {
420 case MEMRANGE_GET:
421 nd = imin(mo->mo_arg[0], mem_range_softc.mr_ndesc);
422 if (nd > 0) {
423 md = (struct mem_range_desc *)
77652cad 424 kmalloc(nd * sizeof(struct mem_range_desc),
984263bc
MD
425 M_MEMDESC, M_WAITOK);
426 error = mem_range_attr_get(md, &nd);
427 if (!error)
428 error = copyout(md, mo->mo_desc,
429 nd * sizeof(struct mem_range_desc));
efda3bd0 430 kfree(md, M_MEMDESC);
984263bc
MD
431 } else {
432 nd = mem_range_softc.mr_ndesc;
433 }
434 mo->mo_arg[0] = nd;
435 break;
436
437 case MEMRANGE_SET:
77652cad 438 md = (struct mem_range_desc *)kmalloc(sizeof(struct mem_range_desc),
984263bc
MD
439 M_MEMDESC, M_WAITOK);
440 error = copyin(mo->mo_desc, md, sizeof(struct mem_range_desc));
441 /* clamp description string */
442 md->mr_owner[sizeof(md->mr_owner) - 1] = 0;
443 if (error == 0)
444 error = mem_range_attr_set(md, &mo->mo_arg[0]);
efda3bd0 445 kfree(md, M_MEMDESC);
984263bc
MD
446 break;
447 }
448 return (error);
449}
450
451/*
452 * Implementation-neutral, kernel-callable functions for manipulating
453 * memory range attributes.
454 */
455int
f123d5a1 456mem_range_attr_get(struct mem_range_desc *mrd, int *arg)
984263bc
MD
457{
458 /* can we handle this? */
459 if (mem_range_softc.mr_op == NULL)
460 return (EOPNOTSUPP);
461
462 if (*arg == 0) {
463 *arg = mem_range_softc.mr_ndesc;
464 } else {
465 bcopy(mem_range_softc.mr_desc, mrd, (*arg) * sizeof(struct mem_range_desc));
466 }
467 return (0);
468}
469
470int
f123d5a1 471mem_range_attr_set(struct mem_range_desc *mrd, int *arg)
984263bc
MD
472{
473 /* can we handle this? */
474 if (mem_range_softc.mr_op == NULL)
475 return (EOPNOTSUPP);
476
477 return (mem_range_softc.mr_op->set(&mem_range_softc, mrd, arg));
478}
479
984263bc
MD
480void
481mem_range_AP_init(void)
482{
483 if (mem_range_softc.mr_op && mem_range_softc.mr_op->initAP)
484 return (mem_range_softc.mr_op->initAP(&mem_range_softc));
485}
984263bc
MD
486
487static int
b13267a5 488random_ioctl(cdev_t dev, u_long cmd, caddr_t data, int flags, struct ucred *cred)
984263bc 489{
5f456c40
MD
490 int error;
491 int intr;
984263bc
MD
492
493 /*
984263bc
MD
494 * Even inspecting the state is privileged, since it gives a hint
495 * about how easily the randomness might be guessed.
496 */
5f456c40
MD
497 error = 0;
498
984263bc
MD
499 switch (cmd) {
500 /* Really handled in upper layer */
501 case FIOASYNC:
984263bc
MD
502 break;
503 case MEM_SETIRQ:
5f456c40 504 intr = *(int16_t *)data;
895c1f85 505 if ((error = priv_check_cred(cred, PRIV_ROOT, 0)) != 0)
984263bc 506 break;
5f456c40
MD
507 if (intr < 0 || intr >= MAX_INTS)
508 return (EINVAL);
7e071e7a 509 register_randintr(intr);
984263bc
MD
510 break;
511 case MEM_CLEARIRQ:
5f456c40 512 intr = *(int16_t *)data;
895c1f85 513 if ((error = priv_check_cred(cred, PRIV_ROOT, 0)) != 0)
984263bc 514 break;
5f456c40
MD
515 if (intr < 0 || intr >= MAX_INTS)
516 return (EINVAL);
7e071e7a 517 unregister_randintr(intr);
984263bc
MD
518 break;
519 case MEM_RETURNIRQ:
5f456c40
MD
520 error = ENOTSUP;
521 break;
522 case MEM_FINDIRQ:
523 intr = *(int16_t *)data;
895c1f85 524 if ((error = priv_check_cred(cred, PRIV_ROOT, 0)) != 0)
5f456c40
MD
525 break;
526 if (intr < 0 || intr >= MAX_INTS)
527 return (EINVAL);
528 intr = next_registered_randintr(intr);
529 if (intr == MAX_INTS)
530 return (ENOENT);
531 *(u_int16_t *)data = intr;
532 break;
533 default:
534 error = ENOTSUP;
984263bc
MD
535 break;
536 }
5f456c40 537 return (error);
984263bc
MD
538}
539
6dcd6ba9
SG
540static int
541mm_filter_read(struct knote *kn, long hint)
542{
543 return (1);
544}
545
2ca4923a
MD
546static int
547mm_filter_write(struct knote *kn, long hint)
548{
549 return (1);
550}
551
6dcd6ba9
SG
552static void
553dummy_filter_detach(struct knote *kn) {}
554
a081e067 555/* Implemented in kern_nrandom.c */
6dcd6ba9 556static struct filterops random_read_filtops =
a081e067 557 { FILTEROP_ISFD|FILTEROP_MPSAFE, NULL, dummy_filter_detach, random_filter_read };
6dcd6ba9
SG
558
559static struct filterops mm_read_filtops =
a081e067 560 { FILTEROP_ISFD|FILTEROP_MPSAFE, NULL, dummy_filter_detach, mm_filter_read };
6dcd6ba9 561
2ca4923a 562static struct filterops mm_write_filtops =
a081e067 563 { FILTEROP_ISFD|FILTEROP_MPSAFE, NULL, dummy_filter_detach, mm_filter_write };
2ca4923a 564
6dcd6ba9
SG
565int
566mmkqfilter(struct dev_kqfilter_args *ap)
567{
568 struct knote *kn = ap->a_kn;
569 cdev_t dev = ap->a_head.a_dev;
570
571 ap->a_result = 0;
572 switch (kn->kn_filter) {
573 case EVFILT_READ:
574 switch (minor(dev)) {
575 case 3:
576 kn->kn_fop = &random_read_filtops;
577 break;
578 default:
579 kn->kn_fop = &mm_read_filtops;
580 break;
581 }
582 break;
2ca4923a
MD
583 case EVFILT_WRITE:
584 kn->kn_fop = &mm_write_filtops;
585 break;
6dcd6ba9 586 default:
b287d649 587 ap->a_result = EOPNOTSUPP;
6dcd6ba9
SG
588 return (0);
589 }
590
591 return (0);
592}
593
984263bc 594int
f123d5a1 595iszerodev(cdev_t dev)
984263bc 596{
964d9977 597 return (zerodev == dev);
984263bc
MD
598}
599
600static void
601mem_drvinit(void *unused)
602{
603
604 /* Initialise memory range handling */
605 if (mem_range_softc.mr_op != NULL)
606 mem_range_softc.mr_op->init(&mem_range_softc);
607
fef8985e
MD
608 make_dev(&mem_ops, 0, UID_ROOT, GID_KMEM, 0640, "mem");
609 make_dev(&mem_ops, 1, UID_ROOT, GID_KMEM, 0640, "kmem");
610 make_dev(&mem_ops, 2, UID_ROOT, GID_WHEEL, 0666, "null");
611 make_dev(&mem_ops, 3, UID_ROOT, GID_WHEEL, 0644, "random");
612 make_dev(&mem_ops, 4, UID_ROOT, GID_WHEEL, 0644, "urandom");
b9a1f8e1 613 zerodev = make_dev(&mem_ops, 12, UID_ROOT, GID_WHEEL, 0666, "zero");
fef8985e 614 make_dev(&mem_ops, 14, UID_ROOT, GID_WHEEL, 0600, "io");
984263bc
MD
615}
616
617SYSINIT(memdev,SI_SUB_DRIVERS,SI_ORDER_MIDDLE+CDEV_MAJOR,mem_drvinit,NULL)
618