pipe - pre-MP work, change indexing to circular FIFO rindex/windex.
[dragonfly.git] / sys / kern / sys_pipe.c
CommitLineData
984263bc
MD
1/*
2 * Copyright (c) 1996 John S. Dyson
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice immediately at the beginning of the file, without modification,
10 * this list of conditions, and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * 3. Absolutely no warranty of function or purpose is made by the author
15 * John S. Dyson.
16 * 4. Modifications may be freely made to this file if the above conditions
17 * are met.
18 *
19 * $FreeBSD: src/sys/kern/sys_pipe.c,v 1.60.2.13 2002/08/05 15:05:15 des Exp $
c730be20 20 * $DragonFly: src/sys/kern/sys_pipe.c,v 1.50 2008/09/09 04:06:13 dillon Exp $
984263bc
MD
21 */
22
23/*
24 * This file contains a high-performance replacement for the socket-based
25 * pipes scheme originally used in FreeBSD/4.4Lite. It does not support
26 * all features of sockets, but does do everything that pipes normally
27 * do.
28 */
984263bc
MD
29#include <sys/param.h>
30#include <sys/systm.h>
fc7d5181 31#include <sys/kernel.h>
984263bc
MD
32#include <sys/proc.h>
33#include <sys/fcntl.h>
34#include <sys/file.h>
35#include <sys/filedesc.h>
36#include <sys/filio.h>
37#include <sys/ttycom.h>
38#include <sys/stat.h>
39#include <sys/poll.h>
40#include <sys/select.h>
41#include <sys/signalvar.h>
42#include <sys/sysproto.h>
43#include <sys/pipe.h>
44#include <sys/vnode.h>
45#include <sys/uio.h>
46#include <sys/event.h>
fc7d5181
MD
47#include <sys/globaldata.h>
48#include <sys/module.h>
49#include <sys/malloc.h>
50#include <sys/sysctl.h>
004d2de5 51#include <sys/socket.h>
984263bc
MD
52
53#include <vm/vm.h>
54#include <vm/vm_param.h>
55#include <sys/lock.h>
56#include <vm/vm_object.h>
57#include <vm/vm_kern.h>
58#include <vm/vm_extern.h>
59#include <vm/pmap.h>
60#include <vm/vm_map.h>
61#include <vm/vm_page.h>
62#include <vm/vm_zone.h>
63
dadab5e9
MD
64#include <sys/file2.h>
65
8100156a 66#include <machine/cpufunc.h>
984263bc
MD
67
68/*
69 * interfaces to the outside world
70 */
402ed7e1 71static int pipe_read (struct file *fp, struct uio *uio,
87de5057 72 struct ucred *cred, int flags);
402ed7e1 73static int pipe_write (struct file *fp, struct uio *uio,
87de5057
MD
74 struct ucred *cred, int flags);
75static int pipe_close (struct file *fp);
76static int pipe_shutdown (struct file *fp, int how);
77static int pipe_poll (struct file *fp, int events, struct ucred *cred);
402ed7e1 78static int pipe_kqfilter (struct file *fp, struct knote *kn);
87de5057
MD
79static int pipe_stat (struct file *fp, struct stat *sb, struct ucred *cred);
80static int pipe_ioctl (struct file *fp, u_long cmd, caddr_t data, struct ucred *cred);
984263bc
MD
81
82static struct fileops pipeops = {
b2d248cb
MD
83 .fo_read = pipe_read,
84 .fo_write = pipe_write,
85 .fo_ioctl = pipe_ioctl,
86 .fo_poll = pipe_poll,
87 .fo_kqfilter = pipe_kqfilter,
88 .fo_stat = pipe_stat,
89 .fo_close = pipe_close,
90 .fo_shutdown = pipe_shutdown
984263bc
MD
91};
92
93static void filt_pipedetach(struct knote *kn);
94static int filt_piperead(struct knote *kn, long hint);
95static int filt_pipewrite(struct knote *kn, long hint);
96
97static struct filterops pipe_rfiltops =
98 { 1, NULL, filt_pipedetach, filt_piperead };
99static struct filterops pipe_wfiltops =
100 { 1, NULL, filt_pipedetach, filt_pipewrite };
101
fc7d5181 102MALLOC_DEFINE(M_PIPE, "pipe", "pipe structures");
984263bc
MD
103
104/*
105 * Default pipe buffer size(s), this can be kind-of large now because pipe
106 * space is pageable. The pipe code will try to maintain locality of
107 * reference for performance reasons, so small amounts of outstanding I/O
108 * will not wipe the cache.
109 */
110#define MINPIPESIZE (PIPE_SIZE/3)
111#define MAXPIPESIZE (2*PIPE_SIZE/3)
112
984263bc
MD
113/*
114 * Limit the number of "big" pipes
115 */
08593aa1 116#define LIMITBIGPIPES 64
fc7d5181
MD
117#define PIPEQ_MAX_CACHE 16 /* per-cpu pipe structure cache */
118
119static int pipe_maxbig = LIMITBIGPIPES;
120static int pipe_maxcache = PIPEQ_MAX_CACHE;
39b0a1af 121static int pipe_bigcount;
fc7d5181 122static int pipe_nbig;
fc7d5181
MD
123static int pipe_bcache_alloc;
124static int pipe_bkmem_alloc;
fc7d5181
MD
125
126SYSCTL_NODE(_kern, OID_AUTO, pipe, CTLFLAG_RW, 0, "Pipe operation");
127SYSCTL_INT(_kern_pipe, OID_AUTO, nbig,
128 CTLFLAG_RD, &pipe_nbig, 0, "numer of big pipes allocated");
39b0a1af
MD
129SYSCTL_INT(_kern_pipe, OID_AUTO, bigcount,
130 CTLFLAG_RW, &pipe_bigcount, 0, "number of times pipe expanded");
fc7d5181
MD
131SYSCTL_INT(_kern_pipe, OID_AUTO, maxcache,
132 CTLFLAG_RW, &pipe_maxcache, 0, "max pipes cached per-cpu");
133SYSCTL_INT(_kern_pipe, OID_AUTO, maxbig,
134 CTLFLAG_RW, &pipe_maxbig, 0, "max number of big pipes");
135#if !defined(NO_PIPE_SYSCTL_STATS)
136SYSCTL_INT(_kern_pipe, OID_AUTO, bcache_alloc,
137 CTLFLAG_RW, &pipe_bcache_alloc, 0, "pipe buffer from pcpu cache");
fc7d5181
MD
138SYSCTL_INT(_kern_pipe, OID_AUTO, bkmem_alloc,
139 CTLFLAG_RW, &pipe_bkmem_alloc, 0, "pipe buffer from kmem");
fc7d5181 140#endif
984263bc 141
402ed7e1
RG
142static void pipeclose (struct pipe *cpipe);
143static void pipe_free_kmem (struct pipe *cpipe);
144static int pipe_create (struct pipe **cpipep);
145static __inline int pipelock (struct pipe *cpipe, int catch);
146static __inline void pipeunlock (struct pipe *cpipe);
147static __inline void pipeselwakeup (struct pipe *cpipe);
402ed7e1 148static int pipespace (struct pipe *cpipe, int size);
984263bc 149
984263bc
MD
150/*
151 * The pipe system call for the DTYPE_PIPE type of pipes
41c20dac
MD
152 *
153 * pipe_ARgs(int dummy)
984263bc
MD
154 */
155
156/* ARGSUSED */
157int
753fd850 158sys_pipe(struct pipe_args *uap)
984263bc 159{
dadab5e9
MD
160 struct thread *td = curthread;
161 struct proc *p = td->td_proc;
984263bc
MD
162 struct file *rf, *wf;
163 struct pipe *rpipe, *wpipe;
90b9818c 164 int fd1, fd2, error;
984263bc 165
dadab5e9 166 KKASSERT(p);
dadab5e9 167
984263bc
MD
168 rpipe = wpipe = NULL;
169 if (pipe_create(&rpipe) || pipe_create(&wpipe)) {
170 pipeclose(rpipe);
171 pipeclose(wpipe);
172 return (ENFILE);
173 }
174
90b9818c 175 error = falloc(p, &rf, &fd1);
984263bc
MD
176 if (error) {
177 pipeclose(rpipe);
178 pipeclose(wpipe);
179 return (error);
180 }
c7114eea 181 uap->sysmsg_fds[0] = fd1;
984263bc
MD
182
183 /*
184 * Warning: once we've gotten past allocation of the fd for the
185 * read-side, we can only drop the read side via fdrop() in order
186 * to avoid races against processes which manage to dup() the read
187 * side while we are blocked trying to allocate the write side.
188 */
984263bc 189 rf->f_type = DTYPE_PIPE;
fbb4eeab 190 rf->f_flag = FREAD | FWRITE;
984263bc 191 rf->f_ops = &pipeops;
fbb4eeab 192 rf->f_data = rpipe;
90b9818c 193 error = falloc(p, &wf, &fd2);
984263bc 194 if (error) {
259b8ea0 195 fsetfd(p, NULL, fd1);
9f87144f 196 fdrop(rf);
984263bc
MD
197 /* rpipe has been closed by fdrop(). */
198 pipeclose(wpipe);
199 return (error);
200 }
984263bc 201 wf->f_type = DTYPE_PIPE;
fbb4eeab 202 wf->f_flag = FREAD | FWRITE;
984263bc 203 wf->f_ops = &pipeops;
fbb4eeab 204 wf->f_data = wpipe;
c7114eea 205 uap->sysmsg_fds[1] = fd2;
984263bc
MD
206
207 rpipe->pipe_peer = wpipe;
208 wpipe->pipe_peer = rpipe;
259b8ea0
MD
209
210 fsetfd(p, rf, fd1);
211 fsetfd(p, wf, fd2);
9f87144f
MD
212 fdrop(rf);
213 fdrop(wf);
984263bc
MD
214
215 return (0);
216}
217
218/*
219 * Allocate kva for pipe circular buffer, the space is pageable
220 * This routine will 'realloc' the size of a pipe safely, if it fails
221 * it will retain the old buffer.
222 * If it fails it will return ENOMEM.
223 */
224static int
fc7d5181 225pipespace(struct pipe *cpipe, int size)
984263bc
MD
226{
227 struct vm_object *object;
228 caddr_t buffer;
229 int npages, error;
230
fc7d5181
MD
231 npages = round_page(size) / PAGE_SIZE;
232 object = cpipe->pipe_buffer.object;
984263bc
MD
233
234 /*
fc7d5181
MD
235 * [re]create the object if necessary and reserve space for it
236 * in the kernel_map. The object and memory are pageable. On
237 * success, free the old resources before assigning the new
238 * ones.
984263bc 239 */
fc7d5181
MD
240 if (object == NULL || object->size != npages) {
241 object = vm_object_allocate(OBJT_DEFAULT, npages);
e4846942 242 buffer = (caddr_t)vm_map_min(&kernel_map);
984263bc 243
e4846942 244 error = vm_map_find(&kernel_map, object, 0,
1b874851
MD
245 (vm_offset_t *)&buffer, size,
246 1,
247 VM_MAPTYPE_NORMAL,
248 VM_PROT_ALL, VM_PROT_ALL,
249 0);
984263bc 250
fc7d5181
MD
251 if (error != KERN_SUCCESS) {
252 vm_object_deallocate(object);
253 return (ENOMEM);
254 }
fc7d5181
MD
255 pipe_free_kmem(cpipe);
256 cpipe->pipe_buffer.object = object;
257 cpipe->pipe_buffer.buffer = buffer;
258 cpipe->pipe_buffer.size = size;
259 ++pipe_bkmem_alloc;
260 } else {
261 ++pipe_bcache_alloc;
fc7d5181 262 }
c600838f
MD
263 cpipe->pipe_buffer.rindex = 0;
264 cpipe->pipe_buffer.windex = 0;
984263bc
MD
265 return (0);
266}
267
268/*
fc7d5181
MD
269 * Initialize and allocate VM and memory for pipe, pulling the pipe from
270 * our per-cpu cache if possible. For now make sure it is sized for the
271 * smaller PIPE_SIZE default.
984263bc
MD
272 */
273static int
c972a82f 274pipe_create(struct pipe **cpipep)
984263bc 275{
fc7d5181 276 globaldata_t gd = mycpu;
984263bc
MD
277 struct pipe *cpipe;
278 int error;
279
fc7d5181
MD
280 if ((cpipe = gd->gd_pipeq) != NULL) {
281 gd->gd_pipeq = cpipe->pipe_peer;
282 --gd->gd_pipeqcount;
283 cpipe->pipe_peer = NULL;
284 } else {
efda3bd0 285 cpipe = kmalloc(sizeof(struct pipe), M_PIPE, M_WAITOK|M_ZERO);
fc7d5181
MD
286 }
287 *cpipep = cpipe;
288 if ((error = pipespace(cpipe, PIPE_SIZE)) != 0)
984263bc 289 return (error);
984263bc
MD
290 vfs_timestamp(&cpipe->pipe_ctime);
291 cpipe->pipe_atime = cpipe->pipe_ctime;
292 cpipe->pipe_mtime = cpipe->pipe_ctime;
984263bc
MD
293 return (0);
294}
295
296
297/*
298 * lock a pipe for I/O, blocking other access
299 */
300static __inline int
c972a82f 301pipelock(struct pipe *cpipe, int catch)
984263bc
MD
302{
303 int error;
304
305 while (cpipe->pipe_state & PIPE_LOCK) {
306 cpipe->pipe_state |= PIPE_LWANT;
377d4740 307 error = tsleep(cpipe, (catch ? PCATCH : 0), "pipelk", 0);
984263bc
MD
308 if (error != 0)
309 return (error);
310 }
311 cpipe->pipe_state |= PIPE_LOCK;
312 return (0);
313}
314
315/*
316 * unlock a pipe I/O lock
317 */
318static __inline void
c972a82f 319pipeunlock(struct pipe *cpipe)
984263bc
MD
320{
321
322 cpipe->pipe_state &= ~PIPE_LOCK;
323 if (cpipe->pipe_state & PIPE_LWANT) {
324 cpipe->pipe_state &= ~PIPE_LWANT;
325 wakeup(cpipe);
326 }
327}
328
329static __inline void
c972a82f 330pipeselwakeup(struct pipe *cpipe)
984263bc
MD
331{
332
333 if (cpipe->pipe_state & PIPE_SEL) {
334 cpipe->pipe_state &= ~PIPE_SEL;
335 selwakeup(&cpipe->pipe_sel);
336 }
337 if ((cpipe->pipe_state & PIPE_ASYNC) && cpipe->pipe_sigio)
338 pgsigio(cpipe->pipe_sigio, SIGIO, 0);
339 KNOTE(&cpipe->pipe_sel.si_note, 0);
340}
341
d9b2033e
MD
342/*
343 * MPALMOSTSAFE (acquires mplock)
344 */
984263bc 345static int
9ba76b73 346pipe_read(struct file *fp, struct uio *uio, struct ucred *cred, int fflags)
984263bc 347{
d9b2033e 348 struct pipe *rpipe;
984263bc
MD
349 int error;
350 int nread = 0;
9ba76b73 351 int nbio;
c600838f
MD
352 u_int size; /* total bytes available */
353 u_int rindex; /* contiguous bytes available */
984263bc 354
d9b2033e
MD
355 get_mplock();
356 rpipe = (struct pipe *) fp->f_data;
984263bc
MD
357 ++rpipe->pipe_busy;
358 error = pipelock(rpipe, 1);
359 if (error)
360 goto unlocked_error;
361
9ba76b73
MD
362 if (fflags & O_FBLOCKING)
363 nbio = 0;
364 else if (fflags & O_FNONBLOCKING)
365 nbio = 1;
366 else if (fp->f_flag & O_NONBLOCK)
367 nbio = 1;
368 else
369 nbio = 0;
370
984263bc 371 while (uio->uio_resid) {
c600838f
MD
372 size = rpipe->pipe_buffer.windex - rpipe->pipe_buffer.rindex;
373 if (size) {
374 rindex = rpipe->pipe_buffer.rindex &
375 (rpipe->pipe_buffer.size - 1);
376 if (size > rpipe->pipe_buffer.size - rindex)
377 size = rpipe->pipe_buffer.size - rindex;
378 if (size > (u_int)uio->uio_resid)
379 size = (u_int)uio->uio_resid;
380
381 error = uiomove(&rpipe->pipe_buffer.buffer[rindex],
984263bc
MD
382 size, uio);
383 if (error)
384 break;
c600838f 385 rpipe->pipe_buffer.rindex += size;
984263bc
MD
386
387 /*
388 * If there is no more to read in the pipe, reset
389 * its pointers to the beginning. This improves
390 * cache hit stats.
391 */
c600838f
MD
392 if (rpipe->pipe_buffer.rindex ==
393 rpipe->pipe_buffer.windex) {
394 rpipe->pipe_buffer.rindex = 0;
395 rpipe->pipe_buffer.windex = 0;
984263bc
MD
396 }
397 nread += size;
984263bc
MD
398 } else {
399 /*
400 * detect EOF condition
401 * read returns 0 on EOF, no need to set error
402 */
403 if (rpipe->pipe_state & PIPE_EOF)
404 break;
405
406 /*
407 * If the "write-side" has been blocked, wake it up now.
408 */
409 if (rpipe->pipe_state & PIPE_WANTW) {
410 rpipe->pipe_state &= ~PIPE_WANTW;
411 wakeup(rpipe);
412 }
413
414 /*
415 * Break if some data was read.
416 */
417 if (nread > 0)
418 break;
419
420 /*
8100156a
MD
421 * Unlock the pipe buffer for our remaining
422 * processing. We will either break out with an
423 * error or we will sleep and relock to loop.
984263bc
MD
424 */
425 pipeunlock(rpipe);
426
427 /*
428 * Handle non-blocking mode operation or
429 * wait for more data.
430 */
9ba76b73 431 if (nbio) {
984263bc
MD
432 error = EAGAIN;
433 } else {
434 rpipe->pipe_state |= PIPE_WANTR;
c730be20
MD
435 if ((error = tsleep(rpipe, PCATCH,
436 "piperd", 0)) == 0) {
984263bc 437 error = pipelock(rpipe, 1);
377d4740 438 }
984263bc
MD
439 }
440 if (error)
441 goto unlocked_error;
442 }
443 }
444 pipeunlock(rpipe);
445
446 if (error == 0)
447 vfs_timestamp(&rpipe->pipe_atime);
448unlocked_error:
449 --rpipe->pipe_busy;
450
451 /*
452 * PIPE_WANT processing only makes sense if pipe_busy is 0.
453 */
c600838f
MD
454 size = rpipe->pipe_buffer.windex - rpipe->pipe_buffer.rindex;
455
984263bc
MD
456 if ((rpipe->pipe_busy == 0) && (rpipe->pipe_state & PIPE_WANT)) {
457 rpipe->pipe_state &= ~(PIPE_WANT|PIPE_WANTW);
458 wakeup(rpipe);
c600838f 459 } else if (size < MINPIPESIZE) {
984263bc
MD
460 /*
461 * Handle write blocking hysteresis.
462 */
463 if (rpipe->pipe_state & PIPE_WANTW) {
464 rpipe->pipe_state &= ~PIPE_WANTW;
465 wakeup(rpipe);
466 }
467 }
468
c600838f 469 if ((rpipe->pipe_buffer.size - size) >= PIPE_BUF)
984263bc 470 pipeselwakeup(rpipe);
d9b2033e 471 rel_mplock();
984263bc
MD
472 return (error);
473}
474
d9b2033e
MD
475/*
476 * MPALMOSTSAFE - acquires mplock
477 */
984263bc 478static int
9ba76b73 479pipe_write(struct file *fp, struct uio *uio, struct ucred *cred, int fflags)
984263bc
MD
480{
481 int error = 0;
482 int orig_resid;
9ba76b73 483 int nbio;
984263bc 484 struct pipe *wpipe, *rpipe;
c600838f
MD
485 u_int windex;
486 u_int space;
984263bc 487
d9b2033e 488 get_mplock();
984263bc
MD
489 rpipe = (struct pipe *) fp->f_data;
490 wpipe = rpipe->pipe_peer;
491
492 /*
493 * detect loss of pipe read side, issue SIGPIPE if lost.
494 */
495 if ((wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) {
d9b2033e 496 rel_mplock();
984263bc
MD
497 return (EPIPE);
498 }
499 ++wpipe->pipe_busy;
500
9ba76b73
MD
501 if (fflags & O_FBLOCKING)
502 nbio = 0;
503 else if (fflags & O_FNONBLOCKING)
504 nbio = 1;
505 else if (fp->f_flag & O_NONBLOCK)
506 nbio = 1;
507 else
508 nbio = 0;
509
984263bc
MD
510 /*
511 * If it is advantageous to resize the pipe buffer, do
512 * so.
513 */
514 if ((uio->uio_resid > PIPE_SIZE) &&
39b0a1af 515 (pipe_nbig < pipe_maxbig) &&
39b0a1af 516 (wpipe->pipe_buffer.size <= PIPE_SIZE) &&
c600838f 517 (wpipe->pipe_buffer.rindex == wpipe->pipe_buffer.windex) &&
39b0a1af
MD
518 (error = pipelock(wpipe, 1)) == 0) {
519 /*
520 * Recheck after lock.
521 */
522 if ((pipe_nbig < pipe_maxbig) &&
39b0a1af 523 (wpipe->pipe_buffer.size <= PIPE_SIZE) &&
c600838f 524 (wpipe->pipe_buffer.rindex == wpipe->pipe_buffer.windex)) {
39b0a1af
MD
525 if (pipespace(wpipe, BIG_PIPE_SIZE) == 0) {
526 ++pipe_bigcount;
fc7d5181 527 pipe_nbig++;
39b0a1af 528 }
984263bc 529 }
39b0a1af 530 pipeunlock(wpipe);
984263bc
MD
531 }
532
533 /*
534 * If an early error occured unbusy and return, waking up any pending
535 * readers.
536 */
537 if (error) {
538 --wpipe->pipe_busy;
539 if ((wpipe->pipe_busy == 0) &&
540 (wpipe->pipe_state & PIPE_WANT)) {
541 wpipe->pipe_state &= ~(PIPE_WANT | PIPE_WANTR);
542 wakeup(wpipe);
543 }
d9b2033e 544 rel_mplock();
984263bc
MD
545 return(error);
546 }
547
548 KASSERT(wpipe->pipe_buffer.buffer != NULL, ("pipe buffer gone"));
549
550 orig_resid = uio->uio_resid;
551
552 while (uio->uio_resid) {
984263bc
MD
553 if (wpipe->pipe_state & PIPE_EOF) {
554 error = EPIPE;
555 break;
556 }
557
c600838f
MD
558 windex = wpipe->pipe_buffer.windex &
559 (wpipe->pipe_buffer.size - 1);
560 space = wpipe->pipe_buffer.size -
561 (wpipe->pipe_buffer.windex - wpipe->pipe_buffer.rindex);
984263bc
MD
562
563 /* Writes of size <= PIPE_BUF must be atomic. */
564 if ((space < uio->uio_resid) && (orig_resid <= PIPE_BUF))
565 space = 0;
566
c617bada
MD
567 /*
568 * Write to fill, read size handles write hysteresis. Also
569 * additional restrictions can cause select-based non-blocking
570 * writes to spin.
571 */
572 if (space > 0) {
984263bc 573 if ((error = pipelock(wpipe,1)) == 0) {
c600838f 574 u_int segsize;
984263bc 575
984263bc
MD
576 /*
577 * If a process blocked in uiomove, our
578 * value for space might be bad.
579 *
580 * XXX will we be ok if the reader has gone
581 * away here?
582 */
c600838f
MD
583 if (space > (wpipe->pipe_buffer.size -
584 (wpipe->pipe_buffer.windex -
585 wpipe->pipe_buffer.rindex))) {
984263bc 586 pipeunlock(wpipe);
08593aa1 587 continue;
984263bc 588 }
c600838f
MD
589 windex = wpipe->pipe_buffer.windex &
590 (wpipe->pipe_buffer.size - 1);
984263bc
MD
591
592 /*
593 * Transfer size is minimum of uio transfer
594 * and free space in pipe buffer.
595 */
c600838f
MD
596 if (space > (u_int)uio->uio_resid)
597 space = (u_int)uio->uio_resid;
598
984263bc
MD
599 /*
600 * First segment to transfer is minimum of
601 * transfer size and contiguous space in
602 * pipe buffer. If first segment to transfer
603 * is less than the transfer size, we've got
604 * a wraparound in the buffer.
605 */
c600838f
MD
606 segsize = wpipe->pipe_buffer.size - windex;
607 if (segsize > space)
608 segsize = space;
984263bc
MD
609
610 /* Transfer first segment */
611
c600838f
MD
612 error = uiomove(
613 &wpipe->pipe_buffer.buffer[windex],
614 segsize, uio);
984263bc 615
c600838f 616 if (error == 0 && segsize < space) {
984263bc
MD
617 /*
618 * Transfer remaining part now, to
619 * support atomic writes. Wraparound
620 * happened.
621 */
17cde63e
MD
622 error = uiomove(&wpipe->pipe_buffer.
623 buffer[0],
c600838f 624 space - segsize, uio);
984263bc 625 }
c600838f
MD
626 if (error == 0)
627 wpipe->pipe_buffer.windex += space;
984263bc
MD
628 pipeunlock(wpipe);
629 }
630 if (error)
631 break;
632
633 } else {
634 /*
c617bada
MD
635 * If the "read-side" has been blocked, wake it up now
636 * and yield to let it drain synchronously rather
637 * then block.
984263bc
MD
638 */
639 if (wpipe->pipe_state & PIPE_WANTR) {
640 wpipe->pipe_state &= ~PIPE_WANTR;
641 wakeup(wpipe);
642 }
643
644 /*
645 * don't block on non-blocking I/O
646 */
9ba76b73 647 if (nbio) {
984263bc
MD
648 error = EAGAIN;
649 break;
650 }
651
652 /*
653 * We have no more space and have something to offer,
654 * wake up select/poll.
655 */
656 pipeselwakeup(wpipe);
657
658 wpipe->pipe_state |= PIPE_WANTW;
c730be20 659 error = tsleep(wpipe, PCATCH, "pipewr", 0);
984263bc
MD
660 if (error != 0)
661 break;
662 /*
663 * If read side wants to go away, we just issue a signal
664 * to ourselves.
665 */
666 if (wpipe->pipe_state & PIPE_EOF) {
667 error = EPIPE;
668 break;
669 }
670 }
671 }
672
673 --wpipe->pipe_busy;
674
675 if ((wpipe->pipe_busy == 0) && (wpipe->pipe_state & PIPE_WANT)) {
676 wpipe->pipe_state &= ~(PIPE_WANT | PIPE_WANTR);
677 wakeup(wpipe);
c600838f 678 } else if (wpipe->pipe_buffer.windex != wpipe->pipe_buffer.rindex) {
984263bc
MD
679 /*
680 * If we have put any characters in the buffer, we wake up
681 * the reader.
682 */
683 if (wpipe->pipe_state & PIPE_WANTR) {
684 wpipe->pipe_state &= ~PIPE_WANTR;
685 wakeup(wpipe);
686 }
687 }
688
689 /*
690 * Don't return EPIPE if I/O was successful
691 */
c600838f 692 if ((wpipe->pipe_buffer.rindex == wpipe->pipe_buffer.windex) &&
984263bc
MD
693 (uio->uio_resid == 0) &&
694 (error == EPIPE)) {
695 error = 0;
696 }
697
698 if (error == 0)
699 vfs_timestamp(&wpipe->pipe_mtime);
700
701 /*
702 * We have something to offer,
703 * wake up select/poll.
704 */
c600838f 705 if (wpipe->pipe_buffer.rindex != wpipe->pipe_buffer.windex)
984263bc 706 pipeselwakeup(wpipe);
d9b2033e 707 rel_mplock();
984263bc
MD
708 return (error);
709}
710
711/*
d9b2033e
MD
712 * MPALMOSTSAFE - acquires mplock
713 *
984263bc
MD
714 * we implement a very minimal set of ioctls for compatibility with sockets.
715 */
716int
87de5057 717pipe_ioctl(struct file *fp, u_long cmd, caddr_t data, struct ucred *cred)
984263bc 718{
d9b2033e
MD
719 struct pipe *mpipe;
720 int error;
984263bc 721
d9b2033e
MD
722 get_mplock();
723 mpipe = (struct pipe *)fp->f_data;
984263bc 724
d9b2033e 725 switch (cmd) {
984263bc
MD
726 case FIOASYNC:
727 if (*(int *)data) {
728 mpipe->pipe_state |= PIPE_ASYNC;
729 } else {
730 mpipe->pipe_state &= ~PIPE_ASYNC;
731 }
d9b2033e
MD
732 error = 0;
733 break;
984263bc 734 case FIONREAD:
c600838f
MD
735 *(int *)data = mpipe->pipe_buffer.windex -
736 mpipe->pipe_buffer.rindex;
d9b2033e
MD
737 error = 0;
738 break;
984263bc 739 case FIOSETOWN:
d9b2033e
MD
740 error = fsetown(*(int *)data, &mpipe->pipe_sigio);
741 break;
984263bc
MD
742 case FIOGETOWN:
743 *(int *)data = fgetown(mpipe->pipe_sigio);
d9b2033e
MD
744 error = 0;
745 break;
984263bc 746 case TIOCSPGRP:
d9b2033e
MD
747 /* This is deprecated, FIOSETOWN should be used instead. */
748 error = fsetown(-(*(int *)data), &mpipe->pipe_sigio);
749 break;
984263bc 750
984263bc 751 case TIOCGPGRP:
d9b2033e 752 /* This is deprecated, FIOGETOWN should be used instead. */
984263bc 753 *(int *)data = -fgetown(mpipe->pipe_sigio);
d9b2033e
MD
754 error = 0;
755 break;
756 default:
757 error = ENOTTY;
758 break;
984263bc 759 }
d9b2033e
MD
760 rel_mplock();
761 return (error);
984263bc
MD
762}
763
d9b2033e
MD
764/*
765 * MPALMOSTSAFE - acquires mplock
766 */
984263bc 767int
87de5057 768pipe_poll(struct file *fp, int events, struct ucred *cred)
984263bc 769{
d9b2033e 770 struct pipe *rpipe;
984263bc
MD
771 struct pipe *wpipe;
772 int revents = 0;
c600838f 773 u_int space;
984263bc 774
d9b2033e
MD
775 get_mplock();
776 rpipe = (struct pipe *)fp->f_data;
984263bc 777 wpipe = rpipe->pipe_peer;
08593aa1 778 if (events & (POLLIN | POLLRDNORM)) {
c600838f 779 if ((rpipe->pipe_buffer.windex != rpipe->pipe_buffer.rindex) ||
08593aa1 780 (rpipe->pipe_state & PIPE_EOF)) {
984263bc 781 revents |= events & (POLLIN | POLLRDNORM);
08593aa1
MD
782 }
783 }
984263bc 784
08593aa1 785 if (events & (POLLOUT | POLLWRNORM)) {
c600838f 786 if (wpipe == NULL || (wpipe->pipe_state & PIPE_EOF)) {
984263bc 787 revents |= events & (POLLOUT | POLLWRNORM);
c600838f
MD
788 } else {
789 space = wpipe->pipe_buffer.windex -
790 wpipe->pipe_buffer.rindex;
791 space = wpipe->pipe_buffer.size - space;
792 if (space >= PIPE_BUF)
793 revents |= events & (POLLOUT | POLLWRNORM);
08593aa1
MD
794 }
795 }
984263bc
MD
796
797 if ((rpipe->pipe_state & PIPE_EOF) ||
798 (wpipe == NULL) ||
799 (wpipe->pipe_state & PIPE_EOF))
800 revents |= POLLHUP;
801
802 if (revents == 0) {
803 if (events & (POLLIN | POLLRDNORM)) {
87de5057 804 selrecord(curthread, &rpipe->pipe_sel);
984263bc
MD
805 rpipe->pipe_state |= PIPE_SEL;
806 }
807
808 if (events & (POLLOUT | POLLWRNORM)) {
87de5057 809 selrecord(curthread, &wpipe->pipe_sel);
984263bc
MD
810 wpipe->pipe_state |= PIPE_SEL;
811 }
812 }
d9b2033e 813 rel_mplock();
984263bc
MD
814 return (revents);
815}
816
d9b2033e
MD
817/*
818 * MPALMOSTSAFE - acquires mplock
819 */
984263bc 820static int
87de5057 821pipe_stat(struct file *fp, struct stat *ub, struct ucred *cred)
984263bc 822{
d9b2033e
MD
823 struct pipe *pipe;
824
825 get_mplock();
826 pipe = (struct pipe *)fp->f_data;
984263bc
MD
827
828 bzero((caddr_t)ub, sizeof(*ub));
829 ub->st_mode = S_IFIFO;
830 ub->st_blksize = pipe->pipe_buffer.size;
c600838f 831 ub->st_size = pipe->pipe_buffer.windex - pipe->pipe_buffer.rindex;
984263bc
MD
832 ub->st_blocks = (ub->st_size + ub->st_blksize - 1) / ub->st_blksize;
833 ub->st_atimespec = pipe->pipe_atime;
834 ub->st_mtimespec = pipe->pipe_mtime;
835 ub->st_ctimespec = pipe->pipe_ctime;
836 /*
837 * Left as 0: st_dev, st_ino, st_nlink, st_uid, st_gid, st_rdev,
838 * st_flags, st_gen.
839 * XXX (st_dev, st_ino) should be unique.
840 */
d9b2033e 841 rel_mplock();
984263bc
MD
842 return (0);
843}
844
d9b2033e
MD
845/*
846 * MPALMOSTSAFE - acquires mplock
847 */
984263bc 848static int
87de5057 849pipe_close(struct file *fp)
984263bc 850{
39b0a1af 851 struct pipe *cpipe;
984263bc 852
d9b2033e 853 get_mplock();
39b0a1af 854 cpipe = (struct pipe *)fp->f_data;
984263bc
MD
855 fp->f_ops = &badfileops;
856 fp->f_data = NULL;
857 funsetown(cpipe->pipe_sigio);
858 pipeclose(cpipe);
d9b2033e 859 rel_mplock();
984263bc
MD
860 return (0);
861}
862
004d2de5
MD
863/*
864 * Shutdown one or both directions of a full-duplex pipe.
d9b2033e
MD
865 *
866 * MPALMOSTSAFE - acquires mplock
004d2de5 867 */
004d2de5 868static int
87de5057 869pipe_shutdown(struct file *fp, int how)
004d2de5 870{
d9b2033e 871 struct pipe *rpipe;
004d2de5
MD
872 struct pipe *wpipe;
873 int error = EPIPE;
874
d9b2033e
MD
875 get_mplock();
876 rpipe = (struct pipe *)fp->f_data;
877
004d2de5
MD
878 switch(how) {
879 case SHUT_RDWR:
880 case SHUT_RD:
881 if (rpipe) {
882 rpipe->pipe_state |= PIPE_EOF;
883 pipeselwakeup(rpipe);
884 if (rpipe->pipe_busy)
885 wakeup(rpipe);
886 error = 0;
887 }
888 if (how == SHUT_RD)
889 break;
890 /* fall through */
891 case SHUT_WR:
892 if (rpipe && (wpipe = rpipe->pipe_peer) != NULL) {
893 wpipe->pipe_state |= PIPE_EOF;
894 pipeselwakeup(wpipe);
895 if (wpipe->pipe_busy)
896 wakeup(wpipe);
897 error = 0;
898 }
899 }
d9b2033e 900 rel_mplock();
004d2de5
MD
901 return (error);
902}
903
984263bc 904static void
dadab5e9 905pipe_free_kmem(struct pipe *cpipe)
984263bc 906{
984263bc
MD
907 if (cpipe->pipe_buffer.buffer != NULL) {
908 if (cpipe->pipe_buffer.size > PIPE_SIZE)
fc7d5181 909 --pipe_nbig;
e4846942 910 kmem_free(&kernel_map,
984263bc
MD
911 (vm_offset_t)cpipe->pipe_buffer.buffer,
912 cpipe->pipe_buffer.size);
913 cpipe->pipe_buffer.buffer = NULL;
fc7d5181 914 cpipe->pipe_buffer.object = NULL;
984263bc 915 }
984263bc
MD
916}
917
918/*
919 * shutdown the pipe
920 */
921static void
dadab5e9 922pipeclose(struct pipe *cpipe)
984263bc 923{
fc7d5181 924 globaldata_t gd;
984263bc
MD
925 struct pipe *ppipe;
926
fc7d5181
MD
927 if (cpipe == NULL)
928 return;
984263bc 929
fc7d5181 930 pipeselwakeup(cpipe);
984263bc 931
fc7d5181
MD
932 /*
933 * If the other side is blocked, wake it up saying that
934 * we want to close it down.
935 */
936 while (cpipe->pipe_busy) {
937 wakeup(cpipe);
938 cpipe->pipe_state |= PIPE_WANT | PIPE_EOF;
939 tsleep(cpipe, 0, "pipecl", 0);
940 }
984263bc 941
fc7d5181
MD
942 /*
943 * Disconnect from peer
944 */
945 if ((ppipe = cpipe->pipe_peer) != NULL) {
946 pipeselwakeup(ppipe);
947
948 ppipe->pipe_state |= PIPE_EOF;
949 wakeup(ppipe);
950 KNOTE(&ppipe->pipe_sel.si_note, 0);
951 ppipe->pipe_peer = NULL;
952 }
953
8100156a 954 if (cpipe->pipe_kva) {
910cc692 955 pmap_qremove(cpipe->pipe_kva, XIO_INTERNAL_PAGES);
e4846942 956 kmem_free(&kernel_map, cpipe->pipe_kva, XIO_INTERNAL_SIZE);
3641b7ca 957 cpipe->pipe_kva = 0;
8100156a
MD
958 }
959
fc7d5181
MD
960 /*
961 * free or cache resources
962 */
963 gd = mycpu;
964 if (gd->gd_pipeqcount >= pipe_maxcache ||
965 cpipe->pipe_buffer.size != PIPE_SIZE
966 ) {
984263bc 967 pipe_free_kmem(cpipe);
efda3bd0 968 kfree(cpipe, M_PIPE);
fc7d5181 969 } else {
fc7d5181
MD
970 cpipe->pipe_state = 0;
971 cpipe->pipe_busy = 0;
fc7d5181
MD
972 cpipe->pipe_peer = gd->gd_pipeq;
973 gd->gd_pipeq = cpipe;
974 ++gd->gd_pipeqcount;
984263bc
MD
975 }
976}
977
d9b2033e
MD
978/*
979 * MPALMOSTSAFE - acquires mplock
980 */
984263bc
MD
981static int
982pipe_kqfilter(struct file *fp, struct knote *kn)
983{
d9b2033e
MD
984 struct pipe *cpipe;
985
986 get_mplock();
987 cpipe = (struct pipe *)kn->kn_fp->f_data;
984263bc
MD
988
989 switch (kn->kn_filter) {
990 case EVFILT_READ:
991 kn->kn_fop = &pipe_rfiltops;
992 break;
993 case EVFILT_WRITE:
994 kn->kn_fop = &pipe_wfiltops;
995 cpipe = cpipe->pipe_peer;
d9b2033e 996 if (cpipe == NULL) {
984263bc 997 /* other end of pipe has been closed */
d9b2033e 998 rel_mplock();
41f57d15 999 return (EPIPE);
d9b2033e 1000 }
984263bc
MD
1001 break;
1002 default:
1003 return (1);
1004 }
1005 kn->kn_hook = (caddr_t)cpipe;
1006
1007 SLIST_INSERT_HEAD(&cpipe->pipe_sel.si_note, kn, kn_selnext);
d9b2033e 1008 rel_mplock();
984263bc
MD
1009 return (0);
1010}
1011
1012static void
1013filt_pipedetach(struct knote *kn)
1014{
1015 struct pipe *cpipe = (struct pipe *)kn->kn_hook;
1016
1017 SLIST_REMOVE(&cpipe->pipe_sel.si_note, kn, knote, kn_selnext);
1018}
1019
1020/*ARGSUSED*/
1021static int
1022filt_piperead(struct knote *kn, long hint)
1023{
1024 struct pipe *rpipe = (struct pipe *)kn->kn_fp->f_data;
1025 struct pipe *wpipe = rpipe->pipe_peer;
1026
c600838f 1027 kn->kn_data = rpipe->pipe_buffer.windex - rpipe->pipe_buffer.rindex;
984263bc
MD
1028
1029 if ((rpipe->pipe_state & PIPE_EOF) ||
1030 (wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) {
1031 kn->kn_flags |= EV_EOF;
1032 return (1);
1033 }
1034 return (kn->kn_data > 0);
1035}
1036
1037/*ARGSUSED*/
1038static int
1039filt_pipewrite(struct knote *kn, long hint)
1040{
1041 struct pipe *rpipe = (struct pipe *)kn->kn_fp->f_data;
1042 struct pipe *wpipe = rpipe->pipe_peer;
c600838f 1043 u_int32_t space;
984263bc
MD
1044
1045 if ((wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) {
1046 kn->kn_data = 0;
1047 kn->kn_flags |= EV_EOF;
1048 return (1);
1049 }
c600838f
MD
1050 space = wpipe->pipe_buffer.windex -
1051 wpipe->pipe_buffer.rindex;
1052 space = wpipe->pipe_buffer.size - space;
1053 kn->kn_data = space;
984263bc
MD
1054 return (kn->kn_data >= PIPE_BUF);
1055}