2f72866f63835a206be7f9e468a9a79081e9e3c4
[dragonfly.git] / sys / kern / sys_pipe.c
1 /*
2  * Copyright (c) 1996 John S. Dyson
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice immediately at the beginning of the file, without modification,
10  *    this list of conditions, and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  * 3. Absolutely no warranty of function or purpose is made by the author
15  *    John S. Dyson.
16  * 4. Modifications may be freely made to this file if the above conditions
17  *    are met.
18  *
19  * $FreeBSD: src/sys/kern/sys_pipe.c,v 1.60.2.13 2002/08/05 15:05:15 des Exp $
20  * $DragonFly: src/sys/kern/sys_pipe.c,v 1.50 2008/09/09 04:06:13 dillon Exp $
21  */
22
23 /*
24  * This file contains a high-performance replacement for the socket-based
25  * pipes scheme originally used in FreeBSD/4.4Lite.  It does not support
26  * all features of sockets, but does do everything that pipes normally
27  * do.
28  */
29 #include <sys/param.h>
30 #include <sys/systm.h>
31 #include <sys/kernel.h>
32 #include <sys/proc.h>
33 #include <sys/fcntl.h>
34 #include <sys/file.h>
35 #include <sys/filedesc.h>
36 #include <sys/filio.h>
37 #include <sys/ttycom.h>
38 #include <sys/stat.h>
39 #include <sys/poll.h>
40 #include <sys/select.h>
41 #include <sys/signalvar.h>
42 #include <sys/sysproto.h>
43 #include <sys/pipe.h>
44 #include <sys/vnode.h>
45 #include <sys/uio.h>
46 #include <sys/event.h>
47 #include <sys/globaldata.h>
48 #include <sys/module.h>
49 #include <sys/malloc.h>
50 #include <sys/sysctl.h>
51 #include <sys/socket.h>
52
53 #include <vm/vm.h>
54 #include <vm/vm_param.h>
55 #include <sys/lock.h>
56 #include <vm/vm_object.h>
57 #include <vm/vm_kern.h>
58 #include <vm/vm_extern.h>
59 #include <vm/pmap.h>
60 #include <vm/vm_map.h>
61 #include <vm/vm_page.h>
62 #include <vm/vm_zone.h>
63
64 #include <sys/file2.h>
65
66 #include <machine/cpufunc.h>
67
68 /*
69  * interfaces to the outside world
70  */
71 static int pipe_read (struct file *fp, struct uio *uio, 
72                 struct ucred *cred, int flags);
73 static int pipe_write (struct file *fp, struct uio *uio, 
74                 struct ucred *cred, int flags);
75 static int pipe_close (struct file *fp);
76 static int pipe_shutdown (struct file *fp, int how);
77 static int pipe_poll (struct file *fp, int events, struct ucred *cred);
78 static int pipe_kqfilter (struct file *fp, struct knote *kn);
79 static int pipe_stat (struct file *fp, struct stat *sb, struct ucred *cred);
80 static int pipe_ioctl (struct file *fp, u_long cmd, caddr_t data, struct ucred *cred);
81
82 static struct fileops pipeops = {
83         .fo_read = pipe_read, 
84         .fo_write = pipe_write,
85         .fo_ioctl = pipe_ioctl,
86         .fo_poll = pipe_poll,
87         .fo_kqfilter = pipe_kqfilter,
88         .fo_stat = pipe_stat,
89         .fo_close = pipe_close,
90         .fo_shutdown = pipe_shutdown
91 };
92
93 static void     filt_pipedetach(struct knote *kn);
94 static int      filt_piperead(struct knote *kn, long hint);
95 static int      filt_pipewrite(struct knote *kn, long hint);
96
97 static struct filterops pipe_rfiltops =
98         { 1, NULL, filt_pipedetach, filt_piperead };
99 static struct filterops pipe_wfiltops =
100         { 1, NULL, filt_pipedetach, filt_pipewrite };
101
102 MALLOC_DEFINE(M_PIPE, "pipe", "pipe structures");
103
104 /*
105  * Default pipe buffer size(s), this can be kind-of large now because pipe
106  * space is pageable.  The pipe code will try to maintain locality of
107  * reference for performance reasons, so small amounts of outstanding I/O
108  * will not wipe the cache.
109  */
110 #define MINPIPESIZE (PIPE_SIZE/3)
111 #define MAXPIPESIZE (2*PIPE_SIZE/3)
112
113 /*
114  * Limit the number of "big" pipes
115  */
116 #define LIMITBIGPIPES   64
117 #define PIPEQ_MAX_CACHE 16      /* per-cpu pipe structure cache */
118
119 static int pipe_maxbig = LIMITBIGPIPES;
120 static int pipe_maxcache = PIPEQ_MAX_CACHE;
121 static int pipe_bigcount;
122 static int pipe_nbig;
123 static int pipe_bcache_alloc;
124 static int pipe_bkmem_alloc;
125
126 SYSCTL_NODE(_kern, OID_AUTO, pipe, CTLFLAG_RW, 0, "Pipe operation");
127 SYSCTL_INT(_kern_pipe, OID_AUTO, nbig,
128         CTLFLAG_RD, &pipe_nbig, 0, "numer of big pipes allocated");
129 SYSCTL_INT(_kern_pipe, OID_AUTO, bigcount,
130         CTLFLAG_RW, &pipe_bigcount, 0, "number of times pipe expanded");
131 SYSCTL_INT(_kern_pipe, OID_AUTO, maxcache,
132         CTLFLAG_RW, &pipe_maxcache, 0, "max pipes cached per-cpu");
133 SYSCTL_INT(_kern_pipe, OID_AUTO, maxbig,
134         CTLFLAG_RW, &pipe_maxbig, 0, "max number of big pipes");
135 #if !defined(NO_PIPE_SYSCTL_STATS)
136 SYSCTL_INT(_kern_pipe, OID_AUTO, bcache_alloc,
137         CTLFLAG_RW, &pipe_bcache_alloc, 0, "pipe buffer from pcpu cache");
138 SYSCTL_INT(_kern_pipe, OID_AUTO, bkmem_alloc,
139         CTLFLAG_RW, &pipe_bkmem_alloc, 0, "pipe buffer from kmem");
140 #endif
141
142 static void pipeclose (struct pipe *cpipe);
143 static void pipe_free_kmem (struct pipe *cpipe);
144 static int pipe_create (struct pipe **cpipep);
145 static __inline int pipelock (struct pipe *cpipe, int catch);
146 static __inline void pipeunlock (struct pipe *cpipe);
147 static __inline void pipeselwakeup (struct pipe *cpipe);
148 static int pipespace (struct pipe *cpipe, int size);
149
150 /*
151  * The pipe system call for the DTYPE_PIPE type of pipes
152  *
153  * pipe_ARgs(int dummy)
154  */
155
156 /* ARGSUSED */
157 int
158 sys_pipe(struct pipe_args *uap)
159 {
160         struct thread *td = curthread;
161         struct proc *p = td->td_proc;
162         struct file *rf, *wf;
163         struct pipe *rpipe, *wpipe;
164         int fd1, fd2, error;
165
166         KKASSERT(p);
167
168         rpipe = wpipe = NULL;
169         if (pipe_create(&rpipe) || pipe_create(&wpipe)) {
170                 pipeclose(rpipe); 
171                 pipeclose(wpipe); 
172                 return (ENFILE);
173         }
174         
175         error = falloc(p, &rf, &fd1);
176         if (error) {
177                 pipeclose(rpipe);
178                 pipeclose(wpipe);
179                 return (error);
180         }
181         uap->sysmsg_fds[0] = fd1;
182
183         /*
184          * Warning: once we've gotten past allocation of the fd for the
185          * read-side, we can only drop the read side via fdrop() in order
186          * to avoid races against processes which manage to dup() the read
187          * side while we are blocked trying to allocate the write side.
188          */
189         rf->f_type = DTYPE_PIPE;
190         rf->f_flag = FREAD | FWRITE;
191         rf->f_ops = &pipeops;
192         rf->f_data = rpipe;
193         error = falloc(p, &wf, &fd2);
194         if (error) {
195                 fsetfd(p, NULL, fd1);
196                 fdrop(rf);
197                 /* rpipe has been closed by fdrop(). */
198                 pipeclose(wpipe);
199                 return (error);
200         }
201         wf->f_type = DTYPE_PIPE;
202         wf->f_flag = FREAD | FWRITE;
203         wf->f_ops = &pipeops;
204         wf->f_data = wpipe;
205         uap->sysmsg_fds[1] = fd2;
206
207         rpipe->pipe_peer = wpipe;
208         wpipe->pipe_peer = rpipe;
209
210         fsetfd(p, rf, fd1);
211         fsetfd(p, wf, fd2);
212         fdrop(rf);
213         fdrop(wf);
214
215         return (0);
216 }
217
218 /*
219  * Allocate kva for pipe circular buffer, the space is pageable
220  * This routine will 'realloc' the size of a pipe safely, if it fails
221  * it will retain the old buffer.
222  * If it fails it will return ENOMEM.
223  */
224 static int
225 pipespace(struct pipe *cpipe, int size)
226 {
227         struct vm_object *object;
228         caddr_t buffer;
229         int npages, error;
230
231         npages = round_page(size) / PAGE_SIZE;
232         object = cpipe->pipe_buffer.object;
233
234         /*
235          * [re]create the object if necessary and reserve space for it
236          * in the kernel_map.  The object and memory are pageable.  On
237          * success, free the old resources before assigning the new
238          * ones.
239          */
240         if (object == NULL || object->size != npages) {
241                 object = vm_object_allocate(OBJT_DEFAULT, npages);
242                 buffer = (caddr_t)vm_map_min(&kernel_map);
243
244                 error = vm_map_find(&kernel_map, object, 0,
245                                     (vm_offset_t *)&buffer, size,
246                                     1,
247                                     VM_MAPTYPE_NORMAL,
248                                     VM_PROT_ALL, VM_PROT_ALL,
249                                     0);
250
251                 if (error != KERN_SUCCESS) {
252                         vm_object_deallocate(object);
253                         return (ENOMEM);
254                 }
255                 pipe_free_kmem(cpipe);
256                 cpipe->pipe_buffer.object = object;
257                 cpipe->pipe_buffer.buffer = buffer;
258                 cpipe->pipe_buffer.size = size;
259                 ++pipe_bkmem_alloc;
260         } else {
261                 ++pipe_bcache_alloc;
262         }
263         cpipe->pipe_buffer.in = 0;
264         cpipe->pipe_buffer.out = 0;
265         cpipe->pipe_buffer.cnt = 0;
266         return (0);
267 }
268
269 /*
270  * Initialize and allocate VM and memory for pipe, pulling the pipe from
271  * our per-cpu cache if possible.  For now make sure it is sized for the
272  * smaller PIPE_SIZE default.
273  */
274 static int
275 pipe_create(struct pipe **cpipep)
276 {
277         globaldata_t gd = mycpu;
278         struct pipe *cpipe;
279         int error;
280
281         if ((cpipe = gd->gd_pipeq) != NULL) {
282                 gd->gd_pipeq = cpipe->pipe_peer;
283                 --gd->gd_pipeqcount;
284                 cpipe->pipe_peer = NULL;
285         } else {
286                 cpipe = kmalloc(sizeof(struct pipe), M_PIPE, M_WAITOK|M_ZERO);
287         }
288         *cpipep = cpipe;
289         if ((error = pipespace(cpipe, PIPE_SIZE)) != 0)
290                 return (error);
291         vfs_timestamp(&cpipe->pipe_ctime);
292         cpipe->pipe_atime = cpipe->pipe_ctime;
293         cpipe->pipe_mtime = cpipe->pipe_ctime;
294         return (0);
295 }
296
297
298 /*
299  * lock a pipe for I/O, blocking other access
300  */
301 static __inline int
302 pipelock(struct pipe *cpipe, int catch)
303 {
304         int error;
305
306         while (cpipe->pipe_state & PIPE_LOCK) {
307                 cpipe->pipe_state |= PIPE_LWANT;
308                 error = tsleep(cpipe, (catch ? PCATCH : 0), "pipelk", 0);
309                 if (error != 0) 
310                         return (error);
311         }
312         cpipe->pipe_state |= PIPE_LOCK;
313         return (0);
314 }
315
316 /*
317  * unlock a pipe I/O lock
318  */
319 static __inline void
320 pipeunlock(struct pipe *cpipe)
321 {
322
323         cpipe->pipe_state &= ~PIPE_LOCK;
324         if (cpipe->pipe_state & PIPE_LWANT) {
325                 cpipe->pipe_state &= ~PIPE_LWANT;
326                 wakeup(cpipe);
327         }
328 }
329
330 static __inline void
331 pipeselwakeup(struct pipe *cpipe)
332 {
333
334         if (cpipe->pipe_state & PIPE_SEL) {
335                 cpipe->pipe_state &= ~PIPE_SEL;
336                 selwakeup(&cpipe->pipe_sel);
337         }
338         if ((cpipe->pipe_state & PIPE_ASYNC) && cpipe->pipe_sigio)
339                 pgsigio(cpipe->pipe_sigio, SIGIO, 0);
340         KNOTE(&cpipe->pipe_sel.si_note, 0);
341 }
342
343 /*
344  * MPALMOSTSAFE (acquires mplock)
345  */
346 static int
347 pipe_read(struct file *fp, struct uio *uio, struct ucred *cred, int fflags)
348 {
349         struct pipe *rpipe;
350         int error;
351         int nread = 0;
352         int nbio;
353         u_int size;
354
355         get_mplock();
356         rpipe = (struct pipe *) fp->f_data;
357         ++rpipe->pipe_busy;
358         error = pipelock(rpipe, 1);
359         if (error)
360                 goto unlocked_error;
361
362         if (fflags & O_FBLOCKING)
363                 nbio = 0;
364         else if (fflags & O_FNONBLOCKING)
365                 nbio = 1;
366         else if (fp->f_flag & O_NONBLOCK)
367                 nbio = 1;
368         else
369                 nbio = 0;
370
371         while (uio->uio_resid) {
372                 if (rpipe->pipe_buffer.cnt > 0) {
373                         /*
374                          * normal pipe buffer receive
375                          */
376                         size = rpipe->pipe_buffer.size - rpipe->pipe_buffer.out;
377                         if (size > rpipe->pipe_buffer.cnt)
378                                 size = rpipe->pipe_buffer.cnt;
379                         if (size > (u_int) uio->uio_resid)
380                                 size = (u_int) uio->uio_resid;
381
382                         error = uiomove(&rpipe->pipe_buffer.buffer
383                                           [rpipe->pipe_buffer.out],
384                                         size, uio);
385                         if (error)
386                                 break;
387
388                         rpipe->pipe_buffer.out += size;
389                         if (rpipe->pipe_buffer.out >= rpipe->pipe_buffer.size)
390                                 rpipe->pipe_buffer.out = 0;
391
392                         rpipe->pipe_buffer.cnt -= size;
393
394                         /*
395                          * If there is no more to read in the pipe, reset
396                          * its pointers to the beginning.  This improves
397                          * cache hit stats.
398                          */
399                         if (rpipe->pipe_buffer.cnt == 0) {
400                                 rpipe->pipe_buffer.in = 0;
401                                 rpipe->pipe_buffer.out = 0;
402                         }
403                         nread += size;
404                 } else {
405                         /*
406                          * detect EOF condition
407                          * read returns 0 on EOF, no need to set error
408                          */
409                         if (rpipe->pipe_state & PIPE_EOF)
410                                 break;
411
412                         /*
413                          * If the "write-side" has been blocked, wake it up now.
414                          */
415                         if (rpipe->pipe_state & PIPE_WANTW) {
416                                 rpipe->pipe_state &= ~PIPE_WANTW;
417                                 wakeup(rpipe);
418                         }
419
420                         /*
421                          * Break if some data was read.
422                          */
423                         if (nread > 0)
424                                 break;
425
426                         /*
427                          * Unlock the pipe buffer for our remaining
428                          * processing.  We will either break out with an
429                          * error or we will sleep and relock to loop.
430                          */
431                         pipeunlock(rpipe);
432
433                         /*
434                          * Handle non-blocking mode operation or
435                          * wait for more data.
436                          */
437                         if (nbio) {
438                                 error = EAGAIN;
439                         } else {
440                                 rpipe->pipe_state |= PIPE_WANTR;
441                                 if ((error = tsleep(rpipe, PCATCH,
442                                                     "piperd", 0)) == 0) {
443                                         error = pipelock(rpipe, 1);
444                                 }
445                         }
446                         if (error)
447                                 goto unlocked_error;
448                 }
449         }
450         pipeunlock(rpipe);
451
452         if (error == 0)
453                 vfs_timestamp(&rpipe->pipe_atime);
454 unlocked_error:
455         --rpipe->pipe_busy;
456
457         /*
458          * PIPE_WANT processing only makes sense if pipe_busy is 0.
459          */
460         if ((rpipe->pipe_busy == 0) && (rpipe->pipe_state & PIPE_WANT)) {
461                 rpipe->pipe_state &= ~(PIPE_WANT|PIPE_WANTW);
462                 wakeup(rpipe);
463         } else if (rpipe->pipe_buffer.cnt < MINPIPESIZE) {
464                 /*
465                  * Handle write blocking hysteresis.
466                  */
467                 if (rpipe->pipe_state & PIPE_WANTW) {
468                         rpipe->pipe_state &= ~PIPE_WANTW;
469                         wakeup(rpipe);
470                 }
471         }
472
473         if ((rpipe->pipe_buffer.size - rpipe->pipe_buffer.cnt) >= PIPE_BUF)
474                 pipeselwakeup(rpipe);
475         rel_mplock();
476         return (error);
477 }
478
479 /*
480  * MPALMOSTSAFE - acquires mplock
481  */
482 static int
483 pipe_write(struct file *fp, struct uio *uio, struct ucred *cred, int fflags)
484 {
485         int error = 0;
486         int orig_resid;
487         int nbio;
488         struct pipe *wpipe, *rpipe;
489
490         get_mplock();
491         rpipe = (struct pipe *) fp->f_data;
492         wpipe = rpipe->pipe_peer;
493
494         /*
495          * detect loss of pipe read side, issue SIGPIPE if lost.
496          */
497         if ((wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) {
498                 rel_mplock();
499                 return (EPIPE);
500         }
501         ++wpipe->pipe_busy;
502
503         if (fflags & O_FBLOCKING)
504                 nbio = 0;
505         else if (fflags & O_FNONBLOCKING)
506                 nbio = 1;
507         else if (fp->f_flag & O_NONBLOCK)
508                 nbio = 1;
509         else
510                 nbio = 0;
511
512         /*
513          * If it is advantageous to resize the pipe buffer, do
514          * so.
515          */
516         if ((uio->uio_resid > PIPE_SIZE) &&
517             (pipe_nbig < pipe_maxbig) &&
518             (wpipe->pipe_buffer.size <= PIPE_SIZE) &&
519             (wpipe->pipe_buffer.cnt == 0) &&
520             (error = pipelock(wpipe, 1)) == 0) {
521                 /* 
522                  * Recheck after lock.
523                  */
524                 if ((pipe_nbig < pipe_maxbig) &&
525                     (wpipe->pipe_buffer.size <= PIPE_SIZE) &&
526                     (wpipe->pipe_buffer.cnt == 0)) {
527                         if (pipespace(wpipe, BIG_PIPE_SIZE) == 0) {
528                                 ++pipe_bigcount;
529                                 pipe_nbig++;
530                         }
531                 }
532                 pipeunlock(wpipe);
533         }
534
535         /*
536          * If an early error occured unbusy and return, waking up any pending
537          * readers.
538          */
539         if (error) {
540                 --wpipe->pipe_busy;
541                 if ((wpipe->pipe_busy == 0) && 
542                     (wpipe->pipe_state & PIPE_WANT)) {
543                         wpipe->pipe_state &= ~(PIPE_WANT | PIPE_WANTR);
544                         wakeup(wpipe);
545                 }
546                 rel_mplock();
547                 return(error);
548         }
549                 
550         KASSERT(wpipe->pipe_buffer.buffer != NULL, ("pipe buffer gone"));
551
552         orig_resid = uio->uio_resid;
553
554         while (uio->uio_resid) {
555                 int space;
556
557                 if (wpipe->pipe_state & PIPE_EOF) {
558                         error = EPIPE;
559                         break;
560                 }
561
562                 space = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt;
563
564                 /* Writes of size <= PIPE_BUF must be atomic. */
565                 if ((space < uio->uio_resid) && (orig_resid <= PIPE_BUF))
566                         space = 0;
567
568                 /* 
569                  * Write to fill, read size handles write hysteresis.  Also
570                  * additional restrictions can cause select-based non-blocking
571                  * writes to spin.
572                  */
573                 if (space > 0) {
574                         if ((error = pipelock(wpipe,1)) == 0) {
575                                 int size;       /* Transfer size */
576                                 int segsize;    /* first segment to transfer */
577
578                                 /* 
579                                  * If a process blocked in uiomove, our
580                                  * value for space might be bad.
581                                  *
582                                  * XXX will we be ok if the reader has gone
583                                  * away here?
584                                  */
585                                 if (space > wpipe->pipe_buffer.size - 
586                                     wpipe->pipe_buffer.cnt) {
587                                         pipeunlock(wpipe);
588                                         continue;
589                                 }
590
591                                 /*
592                                  * Transfer size is minimum of uio transfer
593                                  * and free space in pipe buffer.
594                                  */
595                                 if (space > uio->uio_resid)
596                                         size = uio->uio_resid;
597                                 else
598                                         size = space;
599                                 /*
600                                  * First segment to transfer is minimum of 
601                                  * transfer size and contiguous space in
602                                  * pipe buffer.  If first segment to transfer
603                                  * is less than the transfer size, we've got
604                                  * a wraparound in the buffer.
605                                  */
606                                 segsize = wpipe->pipe_buffer.size - 
607                                         wpipe->pipe_buffer.in;
608                                 if (segsize > size)
609                                         segsize = size;
610                                 
611                                 /* Transfer first segment */
612
613                                 error = uiomove(&wpipe->pipe_buffer.buffer
614                                                   [wpipe->pipe_buffer.in], 
615                                                 segsize, uio);
616                                 
617                                 if (error == 0 && segsize < size) {
618                                         /* 
619                                          * Transfer remaining part now, to
620                                          * support atomic writes.  Wraparound
621                                          * happened.
622                                          */
623                                         if (wpipe->pipe_buffer.in + segsize != 
624                                             wpipe->pipe_buffer.size)
625                                                 panic("Expected pipe buffer wraparound disappeared");
626                                                 
627                                         error = uiomove(&wpipe->pipe_buffer.
628                                                           buffer[0],
629                                                         size - segsize, uio);
630                                 }
631                                 if (error == 0) {
632                                         wpipe->pipe_buffer.in += size;
633                                         if (wpipe->pipe_buffer.in >=
634                                             wpipe->pipe_buffer.size) {
635                                                 if (wpipe->pipe_buffer.in != size - segsize + wpipe->pipe_buffer.size)
636                                                         panic("Expected wraparound bad");
637                                                 wpipe->pipe_buffer.in = size - segsize;
638                                         }
639                                 
640                                         wpipe->pipe_buffer.cnt += size;
641                                         if (wpipe->pipe_buffer.cnt > wpipe->pipe_buffer.size)
642                                                 panic("Pipe buffer overflow");
643                                 
644                                 }
645                                 pipeunlock(wpipe);
646                         }
647                         if (error)
648                                 break;
649
650                 } else {
651                         /*
652                          * If the "read-side" has been blocked, wake it up now
653                          * and yield to let it drain synchronously rather
654                          * then block.
655                          */
656                         if (wpipe->pipe_state & PIPE_WANTR) {
657                                 wpipe->pipe_state &= ~PIPE_WANTR;
658                                 wakeup(wpipe);
659                         }
660
661                         /*
662                          * don't block on non-blocking I/O
663                          */
664                         if (nbio) {
665                                 error = EAGAIN;
666                                 break;
667                         }
668
669                         /*
670                          * We have no more space and have something to offer,
671                          * wake up select/poll.
672                          */
673                         pipeselwakeup(wpipe);
674
675                         wpipe->pipe_state |= PIPE_WANTW;
676                         error = tsleep(wpipe, PCATCH, "pipewr", 0);
677                         if (error != 0)
678                                 break;
679                         /*
680                          * If read side wants to go away, we just issue a signal
681                          * to ourselves.
682                          */
683                         if (wpipe->pipe_state & PIPE_EOF) {
684                                 error = EPIPE;
685                                 break;
686                         }       
687                 }
688         }
689
690         --wpipe->pipe_busy;
691
692         if ((wpipe->pipe_busy == 0) && (wpipe->pipe_state & PIPE_WANT)) {
693                 wpipe->pipe_state &= ~(PIPE_WANT | PIPE_WANTR);
694                 wakeup(wpipe);
695         } else if (wpipe->pipe_buffer.cnt > 0) {
696                 /*
697                  * If we have put any characters in the buffer, we wake up
698                  * the reader.
699                  */
700                 if (wpipe->pipe_state & PIPE_WANTR) {
701                         wpipe->pipe_state &= ~PIPE_WANTR;
702                         wakeup(wpipe);
703                 }
704         }
705
706         /*
707          * Don't return EPIPE if I/O was successful
708          */
709         if ((wpipe->pipe_buffer.cnt == 0) &&
710             (uio->uio_resid == 0) &&
711             (error == EPIPE)) {
712                 error = 0;
713         }
714
715         if (error == 0)
716                 vfs_timestamp(&wpipe->pipe_mtime);
717
718         /*
719          * We have something to offer,
720          * wake up select/poll.
721          */
722         if (wpipe->pipe_buffer.cnt)
723                 pipeselwakeup(wpipe);
724         rel_mplock();
725         return (error);
726 }
727
728 /*
729  * MPALMOSTSAFE - acquires mplock
730  *
731  * we implement a very minimal set of ioctls for compatibility with sockets.
732  */
733 int
734 pipe_ioctl(struct file *fp, u_long cmd, caddr_t data, struct ucred *cred)
735 {
736         struct pipe *mpipe;
737         int error;
738
739         get_mplock();
740         mpipe = (struct pipe *)fp->f_data;
741
742         switch (cmd) {
743         case FIOASYNC:
744                 if (*(int *)data) {
745                         mpipe->pipe_state |= PIPE_ASYNC;
746                 } else {
747                         mpipe->pipe_state &= ~PIPE_ASYNC;
748                 }
749                 error = 0;
750                 break;
751         case FIONREAD:
752                 *(int *)data = mpipe->pipe_buffer.cnt;
753                 error = 0;
754                 break;
755         case FIOSETOWN:
756                 error = fsetown(*(int *)data, &mpipe->pipe_sigio);
757                 break;
758         case FIOGETOWN:
759                 *(int *)data = fgetown(mpipe->pipe_sigio);
760                 error = 0;
761                 break;
762         case TIOCSPGRP:
763                 /* This is deprecated, FIOSETOWN should be used instead. */
764                 error = fsetown(-(*(int *)data), &mpipe->pipe_sigio);
765                 break;
766
767         case TIOCGPGRP:
768                 /* This is deprecated, FIOGETOWN should be used instead. */
769                 *(int *)data = -fgetown(mpipe->pipe_sigio);
770                 error = 0;
771                 break;
772         default:
773                 error = ENOTTY;
774                 break;
775         }
776         rel_mplock();
777         return (error);
778 }
779
780 /*
781  * MPALMOSTSAFE - acquires mplock
782  */
783 int
784 pipe_poll(struct file *fp, int events, struct ucred *cred)
785 {
786         struct pipe *rpipe;
787         struct pipe *wpipe;
788         int revents = 0;
789
790         get_mplock();
791         rpipe = (struct pipe *)fp->f_data;
792         wpipe = rpipe->pipe_peer;
793         if (events & (POLLIN | POLLRDNORM)) {
794                 if ((rpipe->pipe_buffer.cnt > 0) ||
795                     (rpipe->pipe_state & PIPE_EOF)) {
796                         revents |= events & (POLLIN | POLLRDNORM);
797                 }
798         }
799
800         if (events & (POLLOUT | POLLWRNORM)) {
801                 if (wpipe == NULL || (wpipe->pipe_state & PIPE_EOF) ||
802                     ((wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >=
803                     PIPE_BUF)) {
804                         revents |= events & (POLLOUT | POLLWRNORM);
805                 }
806         }
807
808         if ((rpipe->pipe_state & PIPE_EOF) ||
809             (wpipe == NULL) ||
810             (wpipe->pipe_state & PIPE_EOF))
811                 revents |= POLLHUP;
812
813         if (revents == 0) {
814                 if (events & (POLLIN | POLLRDNORM)) {
815                         selrecord(curthread, &rpipe->pipe_sel);
816                         rpipe->pipe_state |= PIPE_SEL;
817                 }
818
819                 if (events & (POLLOUT | POLLWRNORM)) {
820                         selrecord(curthread, &wpipe->pipe_sel);
821                         wpipe->pipe_state |= PIPE_SEL;
822                 }
823         }
824         rel_mplock();
825         return (revents);
826 }
827
828 /*
829  * MPALMOSTSAFE - acquires mplock
830  */
831 static int
832 pipe_stat(struct file *fp, struct stat *ub, struct ucred *cred)
833 {
834         struct pipe *pipe;
835
836         get_mplock();
837         pipe = (struct pipe *)fp->f_data;
838
839         bzero((caddr_t)ub, sizeof(*ub));
840         ub->st_mode = S_IFIFO;
841         ub->st_blksize = pipe->pipe_buffer.size;
842         ub->st_size = pipe->pipe_buffer.cnt;
843         ub->st_blocks = (ub->st_size + ub->st_blksize - 1) / ub->st_blksize;
844         ub->st_atimespec = pipe->pipe_atime;
845         ub->st_mtimespec = pipe->pipe_mtime;
846         ub->st_ctimespec = pipe->pipe_ctime;
847         /*
848          * Left as 0: st_dev, st_ino, st_nlink, st_uid, st_gid, st_rdev,
849          * st_flags, st_gen.
850          * XXX (st_dev, st_ino) should be unique.
851          */
852         rel_mplock();
853         return (0);
854 }
855
856 /*
857  * MPALMOSTSAFE - acquires mplock
858  */
859 static int
860 pipe_close(struct file *fp)
861 {
862         struct pipe *cpipe;
863
864         get_mplock();
865         cpipe = (struct pipe *)fp->f_data;
866         fp->f_ops = &badfileops;
867         fp->f_data = NULL;
868         funsetown(cpipe->pipe_sigio);
869         pipeclose(cpipe);
870         rel_mplock();
871         return (0);
872 }
873
874 /*
875  * Shutdown one or both directions of a full-duplex pipe.
876  *
877  * MPALMOSTSAFE - acquires mplock
878  */
879 static int
880 pipe_shutdown(struct file *fp, int how)
881 {
882         struct pipe *rpipe;
883         struct pipe *wpipe;
884         int error = EPIPE;
885
886         get_mplock();
887         rpipe = (struct pipe *)fp->f_data;
888
889         switch(how) {
890         case SHUT_RDWR:
891         case SHUT_RD:
892                 if (rpipe) {
893                         rpipe->pipe_state |= PIPE_EOF;
894                         pipeselwakeup(rpipe);
895                         if (rpipe->pipe_busy)
896                                 wakeup(rpipe);
897                         error = 0;
898                 }
899                 if (how == SHUT_RD)
900                         break;
901                 /* fall through */
902         case SHUT_WR:
903                 if (rpipe && (wpipe = rpipe->pipe_peer) != NULL) {
904                         wpipe->pipe_state |= PIPE_EOF;
905                         pipeselwakeup(wpipe);
906                         if (wpipe->pipe_busy)
907                                 wakeup(wpipe);
908                         error = 0;
909                 }
910         }
911         rel_mplock();
912         return (error);
913 }
914
915 static void
916 pipe_free_kmem(struct pipe *cpipe)
917 {
918         if (cpipe->pipe_buffer.buffer != NULL) {
919                 if (cpipe->pipe_buffer.size > PIPE_SIZE)
920                         --pipe_nbig;
921                 kmem_free(&kernel_map,
922                         (vm_offset_t)cpipe->pipe_buffer.buffer,
923                         cpipe->pipe_buffer.size);
924                 cpipe->pipe_buffer.buffer = NULL;
925                 cpipe->pipe_buffer.object = NULL;
926         }
927 }
928
929 /*
930  * shutdown the pipe
931  */
932 static void
933 pipeclose(struct pipe *cpipe)
934 {
935         globaldata_t gd;
936         struct pipe *ppipe;
937
938         if (cpipe == NULL)
939                 return;
940
941         pipeselwakeup(cpipe);
942
943         /*
944          * If the other side is blocked, wake it up saying that
945          * we want to close it down.
946          */
947         while (cpipe->pipe_busy) {
948                 wakeup(cpipe);
949                 cpipe->pipe_state |= PIPE_WANT | PIPE_EOF;
950                 tsleep(cpipe, 0, "pipecl", 0);
951         }
952
953         /*
954          * Disconnect from peer
955          */
956         if ((ppipe = cpipe->pipe_peer) != NULL) {
957                 pipeselwakeup(ppipe);
958
959                 ppipe->pipe_state |= PIPE_EOF;
960                 wakeup(ppipe);
961                 KNOTE(&ppipe->pipe_sel.si_note, 0);
962                 ppipe->pipe_peer = NULL;
963         }
964
965         if (cpipe->pipe_kva) {
966                 pmap_qremove(cpipe->pipe_kva, XIO_INTERNAL_PAGES);
967                 kmem_free(&kernel_map, cpipe->pipe_kva, XIO_INTERNAL_SIZE);
968                 cpipe->pipe_kva = 0;
969         }
970
971         /*
972          * free or cache resources
973          */
974         gd = mycpu;
975         if (gd->gd_pipeqcount >= pipe_maxcache ||
976             cpipe->pipe_buffer.size != PIPE_SIZE
977         ) {
978                 pipe_free_kmem(cpipe);
979                 kfree(cpipe, M_PIPE);
980         } else {
981                 cpipe->pipe_state = 0;
982                 cpipe->pipe_busy = 0;
983                 cpipe->pipe_peer = gd->gd_pipeq;
984                 gd->gd_pipeq = cpipe;
985                 ++gd->gd_pipeqcount;
986         }
987 }
988
989 /*
990  * MPALMOSTSAFE - acquires mplock
991  */
992 static int
993 pipe_kqfilter(struct file *fp, struct knote *kn)
994 {
995         struct pipe *cpipe;
996
997         get_mplock();
998         cpipe = (struct pipe *)kn->kn_fp->f_data;
999
1000         switch (kn->kn_filter) {
1001         case EVFILT_READ:
1002                 kn->kn_fop = &pipe_rfiltops;
1003                 break;
1004         case EVFILT_WRITE:
1005                 kn->kn_fop = &pipe_wfiltops;
1006                 cpipe = cpipe->pipe_peer;
1007                 if (cpipe == NULL) {
1008                         /* other end of pipe has been closed */
1009                         rel_mplock();
1010                         return (EPIPE);
1011                 }
1012                 break;
1013         default:
1014                 return (1);
1015         }
1016         kn->kn_hook = (caddr_t)cpipe;
1017
1018         SLIST_INSERT_HEAD(&cpipe->pipe_sel.si_note, kn, kn_selnext);
1019         rel_mplock();
1020         return (0);
1021 }
1022
1023 static void
1024 filt_pipedetach(struct knote *kn)
1025 {
1026         struct pipe *cpipe = (struct pipe *)kn->kn_hook;
1027
1028         SLIST_REMOVE(&cpipe->pipe_sel.si_note, kn, knote, kn_selnext);
1029 }
1030
1031 /*ARGSUSED*/
1032 static int
1033 filt_piperead(struct knote *kn, long hint)
1034 {
1035         struct pipe *rpipe = (struct pipe *)kn->kn_fp->f_data;
1036         struct pipe *wpipe = rpipe->pipe_peer;
1037
1038         kn->kn_data = rpipe->pipe_buffer.cnt;
1039
1040         if ((rpipe->pipe_state & PIPE_EOF) ||
1041             (wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) {
1042                 kn->kn_flags |= EV_EOF; 
1043                 return (1);
1044         }
1045         return (kn->kn_data > 0);
1046 }
1047
1048 /*ARGSUSED*/
1049 static int
1050 filt_pipewrite(struct knote *kn, long hint)
1051 {
1052         struct pipe *rpipe = (struct pipe *)kn->kn_fp->f_data;
1053         struct pipe *wpipe = rpipe->pipe_peer;
1054
1055         if ((wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) {
1056                 kn->kn_data = 0;
1057                 kn->kn_flags |= EV_EOF; 
1058                 return (1);
1059         }
1060         kn->kn_data = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt;
1061         return (kn->kn_data >= PIPE_BUF);
1062 }