Put non-object tmp files in .git/annex/misctmp, leaving .git/annex/tmp for only partially transferred objects.

This allows eg, putting .git/annex/tmp on a ram disk, if the disk IO
of temp object files is too annoying (and if you don't want to keep
partially transferred objects across reboots).

.git/annex/misctmp must be on the same filesystem as the git work tree,
since files are moved to there in a way that will not work cross-device,
as well as symlinked into there.

I first wanted to put the tmp objects in .git/annex/objects/tmp, but
that would pose transition problems on upgrade when partially transferred
objects existed.

git annex info does not currently show the size of .git/annex/misctemp,
since it should stay small. It would also be ok to make something clean it
out, periodically.

Annex/Content.hs

@@ -221,7 +221,7 @@ getViaTmpChecked check key action =
  -}
 prepGetViaTmpChecked :: Key -> Annex Bool -> Annex Bool
 prepGetViaTmpChecked key getkey = do
-	tmp <- fromRepo $ gitAnnexTmpLocation key
+	tmp <- fromRepo $ gitAnnexTmpObjectLocation key
 
 	e <- liftIO $ doesFileExist tmp
 	alreadythere <- if e
@@ -250,7 +250,7 @@ finishGetViaTmp check key action = do
 
 prepTmp :: Key -> Annex FilePath
 prepTmp key = do
-	tmp <- fromRepo $ gitAnnexTmpLocation key
+	tmp <- fromRepo $ gitAnnexTmpObjectLocation key
 	createAnnexDirectory (parentDir tmp)
 	return tmp
 

Annex/Init.hs

@@ -120,7 +120,7 @@ probeCrippledFileSystem = do
 #ifdef mingw32_HOST_OS
 	return True
 #else
-	tmp <- fromRepo gitAnnexTmpDir
+	tmp <- fromRepo gitAnnexTmpMiscDir
 	let f = tmp </> "gaprobe"
 	createAnnexDirectory tmp
 	liftIO $ writeFile f ""
@@ -157,7 +157,7 @@ probeFifoSupport = do
 #ifdef mingw32_HOST_OS
 	return False
 #else
-	tmp <- fromRepo gitAnnexTmpDir
+	tmp <- fromRepo gitAnnexTmpMiscDir
 	let f = tmp </> "gaprobe"
 	createAnnexDirectory tmp
 	liftIO $ do

Annex/Journal.hs

@@ -35,11 +35,11 @@ import Utility.WinLock
  -}
 setJournalFile :: JournalLocked -> FilePath -> String -> Annex ()
 setJournalFile _jl file content = do
+	tmp <- fromRepo gitAnnexTmpMiscDir
 	createAnnexDirectory =<< fromRepo gitAnnexJournalDir
-	createAnnexDirectory =<< fromRepo gitAnnexTmpDir
+	createAnnexDirectory tmp
 	-- journal file is written atomically
 	jfile <- fromRepo $ journalFile file
-	tmp <- fromRepo gitAnnexTmpDir
 	let tmpfile = tmp </> takeFileName jfile
 	liftIO $ do
 		writeBinaryFile tmpfile content

Annex/ReplaceFile.hs

@@ -24,7 +24,7 @@ import Annex.Exception
  -}
 replaceFile :: FilePath -> (FilePath -> Annex ()) -> Annex ()
 replaceFile file a = do
-	tmpdir <- fromRepo gitAnnexTmpDir
+	tmpdir <- fromRepo gitAnnexTmpMiscDir
 	void $ createAnnexDirectory tmpdir
 	bracketIO (setup tmpdir) nukeFile $ \tmpfile -> do
 		a tmpfile
@@ -36,4 +36,4 @@ replaceFile file a = do
 		return tmpfile
 	fallback tmpfile _ = do
 		createDirectoryIfMissing True $ parentDir file
-		rename tmpfile file
+		moveFile tmpfile file

Assistant/Threads/Committer.hs

@@ -448,7 +448,7 @@ safeToAdd havelsof delayadd pending inprocess = do
 			let segments = segmentXargs $ map keyFilename keysources
 			concat <$> forM segments (\fs -> Lsof.query $ "--" : fs)
 		, do
-			tmpdir <- fromRepo gitAnnexTmpDir
+			tmpdir <- fromRepo gitAnnexTmpMiscDir
 			liftIO $ Lsof.queryDir tmpdir
 		)
 

Assistant/Threads/TransferPoller.hs

@@ -35,7 +35,7 @@ transferPollerThread = namedThread "TransferPoller" $ do
 		{- Downloads are polled by checking the size of the
 		 - temp file being used for the transfer. -}
 		| transferDirection t == Download = do
-			let f = gitAnnexTmpLocation (transferKey t) g
+			let f = gitAnnexTmpObjectLocation (transferKey t) g
 			sz <- liftIO $ catchMaybeIO $
 				fromIntegral . fileSize <$> getFileStatus f
 			newsize t info sz

Assistant/XMPP/Git.hs

@@ -187,7 +187,7 @@ xmppPush cid gitpush = do
 		v <- liftIO $ getEnv "GIT_ANNEX_TMP_DIR"
 		case v of
 			Nothing -> do
-				tmp <- liftAnnex $ fromRepo gitAnnexTmpDir
+				tmp <- liftAnnex $ fromRepo gitAnnexTmpMiscDir
 				return $ tmp </> "xmppgit"
 			Just d -> return $ d </> "xmppgit"
 

Command/Add.hs

@@ -96,7 +96,7 @@ lockDown :: FilePath -> Annex (Maybe KeySource)
 lockDown file = ifM crippledFileSystem
 	( liftIO $ catchMaybeIO nohardlink
 	, do
-		tmp <- fromRepo gitAnnexTmpDir
+		tmp <- fromRepo gitAnnexTmpMiscDir
 		createAnnexDirectory tmp
 		eitherToMaybe <$> tryAnnexIO (go tmp)
 	)

Command/AddUrl.hs

@@ -114,7 +114,7 @@ addUrlFileQuvi relaxed quviurl videourl file = do
 			 - it later. -}
 			sizedkey <- addSizeUrlKey videourl key
 			prepGetViaTmpChecked sizedkey $ do
-				tmp <- fromRepo $ gitAnnexTmpLocation key
+				tmp <- fromRepo $ gitAnnexTmpObjectLocation key
 				showOutput
 				ok <- Transfer.download webUUID key (Just file) Transfer.forwardRetry $ const $ do
 					liftIO $ createDirectoryIfMissing True (parentDir tmp)
@@ -162,7 +162,7 @@ download url file = do
 	 - downloads, as the dummy key for a given url is stable. -}
 	dummykey <- addSizeUrlKey url =<< Backend.URL.fromUrl url Nothing
 	prepGetViaTmpChecked dummykey $ do
-		tmp <- fromRepo $ gitAnnexTmpLocation dummykey
+		tmp <- fromRepo $ gitAnnexTmpObjectLocation dummykey
 		showOutput
 		ifM (runtransfer dummykey tmp)
 			( do

Command/DropUnused.hs

@@ -27,7 +27,7 @@ seek ps = do
 	withUnusedMaps (start numcopies) ps
 
 start :: NumCopies -> UnusedMaps -> Int -> CommandStart
-start numcopies = startUnused "dropunused" (perform numcopies) (performOther gitAnnexBadLocation) (performOther gitAnnexTmpLocation)
+start numcopies = startUnused "dropunused" (perform numcopies) (performOther gitAnnexBadLocation) (performOther gitAnnexTmpObjectLocation)
 
 perform :: NumCopies -> Key -> CommandPerform
 perform numcopies key = maybe droplocal dropremote =<< Remote.byNameWithUUID =<< from

Command/Fsck.hs

@@ -147,7 +147,7 @@ performRemote key file backend numcopies remote =
 		]
 	withtmp a = do
 		pid <- liftIO getPID
-		t <- fromRepo gitAnnexTmpDir
+		t <- fromRepo gitAnnexTmpObjectDir
 		createAnnexDirectory t
 		let tmp = t </> "fsck" ++ show pid ++ "." ++ keyFile key
 		let cleanup = liftIO $ catchIO (removeFile tmp) (const noop)

Command/Info.hs

@@ -201,7 +201,7 @@ known_annex_size = stat "size of annexed files in working tree" $ json id $
 	showSizeKeys <$> cachedReferencedData
 
 tmp_size :: Stat
-tmp_size = staleSize "temporary directory size" gitAnnexTmpDir
+tmp_size = staleSize "temporary object directory size" gitAnnexTmpObjectDir
 
 bad_data_size :: Stat
 bad_data_size = staleSize "bad keys size" gitAnnexBadDir

Command/Unlock.hs

@@ -36,7 +36,7 @@ perform dest key = do
 	unlessM (checkDiskSpace Nothing key 0) $ error "cannot unlock"
 
 	src <- calcRepo $ gitAnnexLocation key
-	tmpdest <- fromRepo $ gitAnnexTmpLocation key
+	tmpdest <- fromRepo $ gitAnnexTmpObjectLocation key
 	liftIO $ createDirectoryIfMissing True (parentDir tmpdest)
 	showAction "copying"
 	ifM (liftIO $ copyFileExternal src tmpdest)

Command/Unused.hs

@@ -63,7 +63,7 @@ checkUnused :: CommandPerform
 checkUnused = chain 0
 	[ check "" unusedMsg $ findunused =<< Annex.getState Annex.fast
 	, check "bad" staleBadMsg $ staleKeysPrune gitAnnexBadDir False
-	, check "tmp" staleTmpMsg $ staleKeysPrune gitAnnexTmpDir True
+	, check "tmp" staleTmpMsg $ staleKeysPrune gitAnnexTmpObjectDir True
 	]
   where
 	findunused True = do

Locations.hs

@@ -23,8 +23,9 @@ module Locations (
 	annexLocation,
 	gitAnnexDir,
 	gitAnnexObjectDir,
-	gitAnnexTmpDir,
+	gitAnnexTmpMiscDir,
-	gitAnnexTmpLocation,
+	gitAnnexTmpObjectDir,
+	gitAnnexTmpObjectLocation,
 	gitAnnexBadDir,
 	gitAnnexBadLocation,
 	gitAnnexUnusedLog,
@@ -180,13 +181,17 @@ gitAnnexDir r = addTrailingPathSeparator $ Git.localGitDir r </> annexDir
 gitAnnexObjectDir :: Git.Repo -> FilePath
 gitAnnexObjectDir r = addTrailingPathSeparator $ Git.localGitDir r </> objectDir
 
-{- .git/annex/tmp/ is used for temp files -}
+{- .git/annex/misctmp/ is used for random temp files -}
-gitAnnexTmpDir :: Git.Repo -> FilePath
+gitAnnexTmpMiscDir :: Git.Repo -> FilePath
-gitAnnexTmpDir r = addTrailingPathSeparator $ gitAnnexDir r </> "tmp"
+gitAnnexTmpMiscDir r = addTrailingPathSeparator $ gitAnnexDir r </> "misctmp"
+
+{- .git/annex/tmp/ is used for temp files for key's contents -}
+gitAnnexTmpObjectDir :: Git.Repo -> FilePath
+gitAnnexTmpObjectDir r = addTrailingPathSeparator $ gitAnnexDir r </> "tmp"
 
 {- The temp file to use for a given key's content. -}
-gitAnnexTmpLocation :: Key -> Git.Repo -> FilePath
+gitAnnexTmpObjectLocation :: Key -> Git.Repo -> FilePath
-gitAnnexTmpLocation key r = gitAnnexTmpDir r </> keyFile key
+gitAnnexTmpObjectLocation key r = gitAnnexTmpObjectDir r </> keyFile key
 
 {- .git/annex/bad/ is used for bad files found during fsck -}
 gitAnnexBadDir :: Git.Repo -> FilePath

Logs/Transfer.hs

@@ -186,7 +186,7 @@ runTransfer t file shouldretry a = do
 		| transferDirection t == Upload =
 			liftIO $ readMVar metervar
 		| otherwise = do
-			f <- fromRepo $ gitAnnexTmpLocation (transferKey t)
+			f <- fromRepo $ gitAnnexTmpObjectLocation (transferKey t)
 			liftIO $ catchDefaultIO 0 $
 				fromIntegral . fileSize <$> getFileStatus f
 

Remote/Rsync.hs

@@ -247,7 +247,7 @@ sendParams = ifM crippledFileSystem
 withRsyncScratchDir :: (FilePath -> Annex a) -> Annex a
 withRsyncScratchDir a = do
 	p <- liftIO getPID
-	t <- fromRepo gitAnnexTmpDir
+	t <- fromRepo gitAnnexTmpObjectDir
 	createAnnexDirectory t
 	let tmp = t </> "rsynctmp" </> show p
 	nuke tmp

debian/changelog

@@ -28,6 +28,8 @@ git-annex (5.20140222) UNRELEASED; urgency=medium
   * Add build dep on regex-compat to fix build on mipsel, which lacks
     regex-tdfa.
   * Disable test suite on sparc, which is missing optparse-applicative.
+  * Put non-object tmp files in .git/annex/misctmp, leaving .git/annex/tmp
+    for only partially transferred objects.
 
  -- Joey Hess <joeyh@debian.org>  Fri, 21 Feb 2014 13:03:04 -0400