2012-07-03 14:58:40 +00:00
|
|
|
{- git-annex assistant transfer watching thread
|
|
|
|
-
|
|
|
|
- Copyright 2012 Joey Hess <joey@kitenet.net>
|
|
|
|
-
|
|
|
|
- Licensed under the GNU GPL version 3 or higher.
|
|
|
|
-}
|
|
|
|
|
|
|
|
module Assistant.Threads.TransferWatcher where
|
|
|
|
|
2012-07-20 23:29:59 +00:00
|
|
|
import Assistant.Common
|
2012-07-03 14:58:40 +00:00
|
|
|
import Assistant.DaemonStatus
|
when a Download finishes, queue Uploads
This ensures file propigate takes place in situations such as: Usb drive A
is connected to B. A's master branch is already in sync with B, but it is
being used to sneakernet some files around, so B downloads those. There is no
master branch change, so C does not request these files. B needs to upload
the files it just downloaded on to C, etc.
My first try at this, I saw loops happen. B uploaded to C, which then
tried to upload back to B (because it had not received the updated
git-annex branch from B yet). B already had the file, but it still created
a transfer info file from the incoming transfer, and its watcher saw
that be removed, and tried to upload back to C.
These loops should have been fixed by my previous commit. (They never
affected ssh remotes, only local ones, it seemed.) While C might still try
to upload to B, or to some other remote that already has the file, the
extra work dies out there.
2012-09-18 18:10:33 +00:00
|
|
|
import Assistant.TransferQueue
|
2012-10-18 19:37:57 +00:00
|
|
|
import Assistant.Drop
|
2012-09-24 17:16:08 +00:00
|
|
|
import Annex.Content
|
2012-07-03 14:58:40 +00:00
|
|
|
import Logs.Transfer
|
|
|
|
import Utility.DirWatcher
|
2013-03-11 02:24:13 +00:00
|
|
|
import Utility.DirWatcher.Types
|
2012-08-27 18:04:06 +00:00
|
|
|
import qualified Remote
|
2012-07-03 14:58:40 +00:00
|
|
|
|
2012-10-19 20:59:18 +00:00
|
|
|
import Control.Concurrent
|
|
|
|
|
2012-07-03 14:58:40 +00:00
|
|
|
{- This thread watches for changes to the gitAnnexTransferDir,
|
|
|
|
- and updates the DaemonStatus's map of ongoing transfers. -}
|
2012-10-29 17:09:58 +00:00
|
|
|
transferWatcherThread :: NamedThread
|
2013-01-26 06:09:33 +00:00
|
|
|
transferWatcherThread = namedThread "TransferWatcher" $ do
|
2012-10-29 17:09:58 +00:00
|
|
|
dir <- liftAnnex $ gitAnnexTransferDir <$> gitRepo
|
|
|
|
liftIO $ createDirectoryIfMissing True dir
|
|
|
|
let hook a = Just <$> asIO2 (runHandler a)
|
|
|
|
addhook <- hook onAdd
|
|
|
|
delhook <- hook onDel
|
|
|
|
modifyhook <- hook onModify
|
|
|
|
errhook <- hook onErr
|
2012-07-03 14:58:40 +00:00
|
|
|
let hooks = mkWatchHooks
|
2012-10-29 17:09:58 +00:00
|
|
|
{ addHook = addhook
|
|
|
|
, delHook = delhook
|
|
|
|
, modifyHook = modifyhook
|
|
|
|
, errHook = errhook
|
2012-07-03 14:58:40 +00:00
|
|
|
}
|
2012-10-29 17:09:58 +00:00
|
|
|
void $ liftIO $ watchDir dir (const False) hooks id
|
|
|
|
debug ["watching for transfers"]
|
2012-07-03 14:58:40 +00:00
|
|
|
|
2012-10-29 17:09:58 +00:00
|
|
|
type Handler = FilePath -> Assistant ()
|
2012-07-03 14:58:40 +00:00
|
|
|
|
|
|
|
{- Runs an action handler.
|
|
|
|
-
|
|
|
|
- Exceptions are ignored, otherwise a whole thread could be crashed.
|
|
|
|
-}
|
2012-10-29 17:09:58 +00:00
|
|
|
runHandler :: Handler -> FilePath -> Maybe FileStatus -> Assistant ()
|
|
|
|
runHandler handler file _filestatus =
|
2012-12-13 04:45:27 +00:00
|
|
|
either (liftIO . print) (const noop) =<< tryIO <~> handler file
|
2012-07-03 14:58:40 +00:00
|
|
|
|
|
|
|
{- Called when there's an error with inotify. -}
|
|
|
|
onErr :: Handler
|
2012-10-29 17:09:58 +00:00
|
|
|
onErr msg = error msg
|
2012-07-03 14:58:40 +00:00
|
|
|
|
2012-07-06 20:30:55 +00:00
|
|
|
{- Called when a new transfer information file is written. -}
|
2012-07-03 14:58:40 +00:00
|
|
|
onAdd :: Handler
|
2012-10-29 17:09:58 +00:00
|
|
|
onAdd file = case parseTransferFile file of
|
2012-07-03 14:58:40 +00:00
|
|
|
Nothing -> noop
|
2012-10-29 17:09:58 +00:00
|
|
|
Just t -> go t =<< liftAnnex (checkTransfer t)
|
|
|
|
where
|
|
|
|
go _ Nothing = noop -- transfer already finished
|
|
|
|
go t (Just info) = do
|
2013-03-01 19:23:59 +00:00
|
|
|
debug [ "transfer starting:", describeTransfer t info ]
|
2013-04-02 20:39:11 +00:00
|
|
|
r <- liftAnnex $ Remote.remoteFromUUID $ transferUUID t
|
2012-10-30 19:39:15 +00:00
|
|
|
updateTransferInfo t info { transferRemote = r }
|
2012-07-03 14:58:40 +00:00
|
|
|
|
2012-09-20 21:24:40 +00:00
|
|
|
{- Called when a transfer information file is updated.
|
|
|
|
-
|
|
|
|
- The only thing that should change in the transfer info is the
|
|
|
|
- bytesComplete, so that's the only thing updated in the DaemonStatus. -}
|
|
|
|
onModify :: Handler
|
2012-10-29 17:09:58 +00:00
|
|
|
onModify file = do
|
2012-09-20 21:24:40 +00:00
|
|
|
case parseTransferFile file of
|
|
|
|
Nothing -> noop
|
2012-10-29 17:09:58 +00:00
|
|
|
Just t -> go t =<< liftIO (readTransferInfoFile Nothing file)
|
2012-10-31 06:34:03 +00:00
|
|
|
where
|
|
|
|
go _ Nothing = noop
|
|
|
|
go t (Just newinfo) = alterTransferInfo t $
|
|
|
|
\i -> i { bytesComplete = bytesComplete newinfo }
|
2012-09-20 21:24:40 +00:00
|
|
|
|
|
|
|
{- This thread can only watch transfer sizes when the DirWatcher supports
|
|
|
|
- tracking modificatons to files. -}
|
|
|
|
watchesTransferSize :: Bool
|
|
|
|
watchesTransferSize = modifyTracked
|
|
|
|
|
2012-07-18 23:13:56 +00:00
|
|
|
{- Called when a transfer information file is removed. -}
|
2012-07-03 14:58:40 +00:00
|
|
|
onDel :: Handler
|
2012-10-29 17:09:58 +00:00
|
|
|
onDel file = case parseTransferFile file of
|
2012-07-03 14:58:40 +00:00
|
|
|
Nothing -> noop
|
2012-07-20 23:29:59 +00:00
|
|
|
Just t -> do
|
2012-10-29 17:09:58 +00:00
|
|
|
debug [ "transfer finishing:", show t]
|
2012-10-30 19:39:15 +00:00
|
|
|
minfo <- removeTransfer t
|
2012-10-19 20:59:18 +00:00
|
|
|
|
2012-10-29 17:09:58 +00:00
|
|
|
finished <- asIO2 finishedTransfer
|
|
|
|
void $ liftIO $ forkIO $ do
|
2012-10-19 20:59:18 +00:00
|
|
|
{- XXX race workaround delay. The location
|
2012-12-13 04:45:27 +00:00
|
|
|
- log needs to be updated before finishedTransfer
|
|
|
|
- runs. -}
|
2012-10-19 20:59:18 +00:00
|
|
|
threadDelay 10000000 -- 10 seconds
|
2012-10-29 17:09:58 +00:00
|
|
|
finished t minfo
|
when a Download finishes, queue Uploads
This ensures file propigate takes place in situations such as: Usb drive A
is connected to B. A's master branch is already in sync with B, but it is
being used to sneakernet some files around, so B downloads those. There is no
master branch change, so C does not request these files. B needs to upload
the files it just downloaded on to C, etc.
My first try at this, I saw loops happen. B uploaded to C, which then
tried to upload back to B (because it had not received the updated
git-annex branch from B yet). B already had the file, but it still created
a transfer info file from the incoming transfer, and its watcher saw
that be removed, and tried to upload back to C.
These loops should have been fixed by my previous commit. (They never
affected ssh remotes, only local ones, it seemed.) While C might still try
to upload to B, or to some other remote that already has the file, the
extra work dies out there.
2012-09-18 18:10:33 +00:00
|
|
|
|
2012-11-24 20:30:15 +00:00
|
|
|
{- Queue uploads of files downloaded to us, spreading them
|
2012-10-18 19:37:57 +00:00
|
|
|
- out to other reachable remotes.
|
|
|
|
-
|
2012-10-18 20:05:43 +00:00
|
|
|
- Downloading a file may have caused a remote to not want it;
|
2012-11-24 20:30:15 +00:00
|
|
|
- so check for drops from remotes.
|
2012-10-18 20:05:43 +00:00
|
|
|
-
|
|
|
|
- Uploading a file may cause the local repo, or some other remote to not
|
|
|
|
- want it; handle that too.
|
|
|
|
-}
|
2012-10-29 17:09:58 +00:00
|
|
|
finishedTransfer :: Transfer -> Maybe TransferInfo -> Assistant ()
|
|
|
|
finishedTransfer t (Just info)
|
|
|
|
| transferDirection t == Download =
|
|
|
|
whenM (liftAnnex $ inAnnex $ transferKey t) $ do
|
2013-03-01 19:58:44 +00:00
|
|
|
dodrops False
|
2013-03-01 19:23:59 +00:00
|
|
|
queueTransfersMatching (/= transferUUID t)
|
|
|
|
"newly received object"
|
|
|
|
Later (transferKey t) (associatedFile info) Upload
|
2013-03-01 19:58:44 +00:00
|
|
|
| otherwise = dodrops True
|
|
|
|
where
|
|
|
|
dodrops fromhere = handleDrops
|
|
|
|
("drop wanted after " ++ describeTransfer t info)
|
|
|
|
fromhere (transferKey t) (associatedFile info) Nothing
|
2012-10-29 17:09:58 +00:00
|
|
|
finishedTransfer _ _ = noop
|
|
|
|
|