diff --git a/Assistant.hs b/Assistant.hs index 413e5e90e1..42175dd82c 100644 --- a/Assistant.hs +++ b/Assistant.hs @@ -39,30 +39,32 @@ - and maintains the DaemonStatus currentTransfers map. - (This uses inotify on .git/annex/transfer/, so there are - additional inotify threads associated with it, too.) - - Thread 10: Transferrer + - Thread 10: TransferPoller + - Polls to determine how much of each ongoing transfer is complete. + - Thread 11: Transferrer - Waits for Transfers to be queued and does them. - - Thread 11: StatusLogger + - Thread 12: StatusLogger - Wakes up periodically and records the daemon's status to disk. - - Thread 12: SanityChecker + - Thread 13: SanityChecker - Wakes up periodically (rarely) and does sanity checks. - - Thread 13: MountWatcher + - Thread 14: MountWatcher - Either uses dbus to watch for drive mount events, or, when - there's no dbus, polls to find newly mounted filesystems. - Once a filesystem that contains a remote is mounted, updates - state about that remote, pulls from it, and queues a push to it, - as well as an update, and queues it onto the - ConnectedRemoteChan - - Thread 13: NetWatcher + - Thread 15: NetWatcher - Deals with network connection interruptions, which would cause - transfers to fail, and can be recovered from by waiting for a - network connection, and syncing with all network remotes. - Uses dbus to watch for network connections, or when dbus - cannot be used, assumes there's been one every 30 minutes. - - Thread 15: TransferScanner + - Thread 16: TransferScanner - Does potentially expensive checks to find data that needs to be - transferred from or to remotes, and queues Transfers. - Uses the ScanRemotes map. - - Thread 16: WebApp + - Thread 17: WebApp - Spawns more threads as necessary to handle clients. - Displays the DaemonStatus. - @@ -118,6 +120,7 @@ import Assistant.Threads.SanityChecker import Assistant.Threads.MountWatcher import Assistant.Threads.NetWatcher import Assistant.Threads.TransferScanner +import Assistant.Threads.TransferPoller #ifdef WITH_WEBAPP import Assistant.Threads.WebApp #else @@ -168,6 +171,7 @@ startAssistant assistant daemonize webappwaiter = do , assist $ pushRetryThread st dstatus pushmap , assist $ mergeThread st , assist $ transferWatcherThread st dstatus + , assist $ transferPollerThread st dstatus , assist $ transfererThread st dstatus transferqueue transferslots , assist $ daemonStatusThread st dstatus , assist $ sanityCheckerThread st dstatus transferqueue changechan diff --git a/Assistant/Threads/TransferPoller.hs b/Assistant/Threads/TransferPoller.hs new file mode 100644 index 0000000000..d720bcc453 --- /dev/null +++ b/Assistant/Threads/TransferPoller.hs @@ -0,0 +1,49 @@ +{- git-annex assistant transfer polling thread + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Threads.TransferPoller where + +import Assistant.Common +import Assistant.ThreadedMonad +import Assistant.DaemonStatus +import Logs.Transfer +import Utility.NotificationBroadcaster + +import Control.Concurrent +import qualified Data.Map as M + +thisThread :: ThreadName +thisThread = "TransferPoller" + +{- This thread polls the status of ongoing transfers, determining how much + - of each transfer is complete. -} +transferPollerThread :: ThreadState -> DaemonStatusHandle -> IO () +transferPollerThread st dstatus = do + g <- runThreadState st $ fromRepo id + tn <- newNotificationHandle =<< + transferNotifier <$> getDaemonStatus dstatus + forever $ do + threadDelay 500000 -- 0.5 seconds + ts <- currentTransfers <$> getDaemonStatus dstatus + if M.null ts + then waitNotification tn -- block until transfers running + else mapM_ (poll g) $ M.toList ts + where + poll g (t, info) + {- Downloads are polled by checking the size of the + - temp file being used for the transfer. -} + | transferDirection t == Download = do + let f = gitAnnexTmpLocation (transferKey t) g + sz <- catchMaybeIO $ + fromIntegral . fileSize + <$> getFileStatus f + when (bytesComplete info /= sz && isJust sz) $ do + putStrLn $ "download size " ++ show sz + updateTransferInfo dstatus t info + { bytesComplete = sz } + {- can't poll uploads -} + | otherwise = noop diff --git a/doc/design/assistant/progressbars.mdwn b/doc/design/assistant/progressbars.mdwn index ee73842743..33d736afdd 100644 --- a/doc/design/assistant/progressbars.mdwn +++ b/doc/design/assistant/progressbars.mdwn @@ -5,10 +5,24 @@ for it, in the terminal. Something better is needed for the [[webapp]]. There needs to be a way for the web app to know what the current progress is of all transfers. -To get this info for downloads, git-annex can watch the file as it arrives -and use its size. - -TODO: What about uploads? Will i have to parse rsync's progresss output? -Feed it via a named pipe? Ugh. Check into librsync. - This is one of those potentially hidden but time consuming problems. + +## downloads + +* Watch temp file as it's coming in and use its size. + This is the only option for some special remotes (ie, non-rsync). + Can either poll every .5 seconds or so to check file size, or + could use inotify. Implemented. +* Feed rsync output into a parser and parse out a progress value. Ugly, + failure prone, but potentially the least CPU-expensive option. +* Use librsync. Note: It's not wire-compatiable with the actual rsync + command. +* Set up a FIFO, have rsync read from or write to that, and the FIFO + feeder/reader then can update the transfer info. Generic enough to + work for most (all?) special remotes, but also the most expensive option, + involving another copy through memory of the whole file contents. + +## uploads + +Cannot use temp file, as we're not receiving it. Rsync progress parser, +librsync, and FIFO all work.