diff --git a/Annex/Direct.hs b/Annex/Direct.hs
index 1db046e7b6..6d9f3e31db 100644
--- a/Annex/Direct.hs
+++ b/Annex/Direct.hs
@@ -206,9 +206,10 @@ toDirectGen k f = do
 {- Removes a direct mode file, while retaining its content in the annex. -}
 removeDirect :: Key -> FilePath -> Annex ()
 removeDirect k f = do
-	locs <- removeAssociatedFile k f
-	unlessM (inAnnex k) $
-		moveAnnex k f
+	otherlocs <- removeAssociatedFile k f
+	unless (null otherlocs) $
+		unlessM (inAnnex k) $
+			moveAnnex k f
 	liftIO $ do
 		nukeFile f
 		void $ tryIO $ removeDirectory $ parentDir f