kernel -- file desc malloc zone overflow handling.

Allow null return from mallocs for file descriptor arrays. Fork and exec
will ENOMEM when we cannot allocate FD arrays.

This is only a partial solution to bug 2019.

diff --git a/sys/kern/kern_descrip.c b/sys/kern/kern_descrip.c
index f38aeaf..b0c43ba 100644 (file)
--- a/sys/kern/kern_descrip.c
+++ b/sys/kern/kern_descrip.c
@@ -1813,8 +1813,8 @@ fdshare(struct proc *p)
  *
  * MPSAFE
  */
-struct filedesc *
-fdcopy(struct proc *p)
+int
+fdcopy(struct proc *p, struct filedesc **fpp)
 {
        struct filedesc *fdp = p->p_fd;
        struct filedesc *newfdp;
@@ -1826,14 +1826,19 @@ fdcopy(struct proc *p)
         * Certain daemons might not have file descriptors. 
         */
        if (fdp == NULL)
-               return (NULL);
+               return (0);
 
        /*
         * Allocate the new filedesc and fd_files[] array.  This can race
         * with operations by other threads on the fdp so we have to be
         * careful.
         */
-       newfdp = kmalloc(sizeof(struct filedesc), M_FILEDESC, M_WAITOK | M_ZERO);
+       newfdp = kmalloc(sizeof(struct filedesc), 
+                        M_FILEDESC, M_WAITOK | M_ZERO | M_NULLOK);
+       if (newfdp == NULL) {
+               *fpp = NULL;
+               return (-1);
+       }
 again:
        spin_lock(&fdp->fd_spin);
        if (fdp->fd_lastfile < NDFILE) {
@@ -1925,7 +1930,8 @@ again:
                }
        }
        spin_unlock(&fdp->fd_spin);
-       return (newfdp);
+       *fpp = newfdp;
+       return (0);
 }
 
 /*

diff --git a/sys/kern/kern_exec.c b/sys/kern/kern_exec.c
index 6b5816f..e3d234e 100644 (file)
--- a/sys/kern/kern_exec.c
+++ b/sys/kern/kern_exec.c
@@ -338,7 +338,9 @@ interpret:
        if (p->p_fd->fd_refcnt > 1) {
                struct filedesc *tmp;
 
-               tmp = fdcopy(p);
+               error = fdcopy(p, &tmp);
+               if (error != 0)
+                       goto exec_fail;
                fdfree(p, tmp);
        }
 

diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c
index e926e66..086c2cd 100644 (file)
--- a/sys/kern/kern_fork.c
+++ b/sys/kern/kern_fork.c
@@ -283,7 +283,11 @@ fork1(struct lwp *lp1, int flags, struct proc **procp)
                if (flags & RFFDG) {
                        if (p1->p_fd->fd_refcnt > 1) {
                                struct filedesc *newfd;
-                               newfd = fdcopy(p1);
+                               error = fdcopy(p1, &newfd);
+                               if (error != 0) {
+                                       error = ENOMEM;
+                                       goto done;
+                               }
                                fdfree(p1, newfd);
                        }
                }
@@ -439,7 +443,11 @@ fork1(struct lwp *lp1, int flags, struct proc **procp)
                p2->p_fd = fdinit(p1);
                fdtol = NULL;
        } else if (flags & RFFDG) {
-               p2->p_fd = fdcopy(p1);
+               error = fdcopy(p1, &p2->p_fd);
+               if (error != 0) {
+                       error = ENOMEM;
+                       goto done;
+               }
                fdtol = NULL;
        } else {
                p2->p_fd = fdshare(p1);

diff --git a/sys/sys/filedesc.h b/sys/sys/filedesc.h
index 4e77991..2be21d5 100644 (file)
--- a/sys/sys/filedesc.h
+++ b/sys/sys/filedesc.h
@@ -164,7 +164,7 @@ void        fsetcred (struct file *fp, struct ucred *cr);
 void   fdinit_bootstrap(struct proc *p0, struct filedesc *fdp0, int cmask);
 struct filedesc *fdinit (struct proc *p);
 struct filedesc *fdshare (struct proc *p);
-struct filedesc *fdcopy (struct proc *p);
+int    fdcopy (struct proc *p, struct filedesc **fpp);
 void   fdfree (struct proc *p, struct filedesc *repl);
 int    fdrevoke(void *f_data, short f_type, struct ucred *cred);
 int    closef (struct file *fp, struct proc *p);