HAMMER VFS - Fix deadlock in rmdir
authorMatthew Dillon <dillon@apollo.backplane.com>
Sun, 25 Jan 2009 07:08:46 +0000 (23:08 -0800)
committerMatthew Dillon <dillon@apollo.backplane.com>
Sun, 25 Jan 2009 07:08:46 +0000 (23:08 -0800)
hammer_vop_rmdir() maintains a cursor at the directory entry being deleted,
and then calls hammer_ip_check_directory_empty() to check if the directory
is empty.  The check code creates a second, nested cursor.

Any locks on the first cursor must be released in order to allow the
function to block while scanning the second cursor to avoid possible
deadlocks.

sys/vfs/hammer/hammer_vnops.c

index 5b4e14a..84e36d1 100644 (file)
@@ -2816,13 +2816,25 @@ retry:
                 * If we are trying to remove a directory the directory must
                 * be empty.
                 *
-                * WARNING: hammer_ip_check_directory_empty() may have to
-                * terminate the cursor to avoid a deadlock.  It is ok to
-                * call hammer_done_cursor() twice.
+                * The check directory code can loop and deadlock/retry.  Our
+                * own cursor's node locks must be released to avoid a 3-way
+                * deadlock with the flusher if the check directory code
+                * blocks.
+                *
+                * If any changes whatsoever have been made to the cursor
+                * set EDEADLK and retry.
                 */
                if (error == 0 && ip->ino_data.obj_type ==
                                  HAMMER_OBJTYPE_DIRECTORY) {
+                       hammer_unlock_cursor(&cursor);
                        error = hammer_ip_check_directory_empty(trans, ip);
+                       hammer_lock_cursor(&cursor);
+                       if (cursor.flags & HAMMER_CURSOR_RETEST) {
+                               kprintf("HAMMER: Warning: avoided deadlock "
+                                       "on rmdir '%s'\n",
+                                       ncp->nc_name);
+                               error = EDEADLK;
+                       }
                }
 
                /*