improve thread termination handling

The reason the DirWatcher had to wait for program termination was because
it used withINotify, so when it finished, its watcher threads were killed.
But since I have two DirWatcher threads now, that was not good, and could
perhaps explain the MVar problem I saw yesterday. In any case, fixed this
part of the code by making the DirWatcher return a handle that can be used
to stop it, and now the main Assistant thread is the only one calling
waitForTermination.

Assistant.hs

@@ -73,6 +73,7 @@ import Assistant.Threads.Merger
 import Assistant.Threads.SanityChecker
 import qualified Utility.Daemon
 import Utility.LogFile
+import Utility.ThreadScheduler
 
 import Control.Concurrent
 
@@ -99,8 +100,8 @@ startDaemon assistant foreground
 				_ <- forkIO $ mergeThread st
 				_ <- forkIO $ daemonStatusThread st dstatus
 				_ <- forkIO $ sanityCheckerThread st dstatus changechan
-				-- Does not return.
-				watchThread st dstatus changechan
+				_ <- forkIO $ watchThread st dstatus changechan
+				waitForTermination
 
 stopDaemon :: Annex ()
 stopDaemon = liftIO . Utility.Daemon.stopDaemon =<< fromRepo gitAnnexPidFile

Assistant/Threads/Merger.hs

@@ -35,7 +35,7 @@ mergeThread st = do
 		{ addHook = hook onAdd
 		, errHook = hook onErr
 		}
-	watchDir dir (const False) hooks id
+	void $ watchDir dir (const False) hooks id
 
 type Handler = Git.Repo -> FilePath -> Maybe FileStatus -> IO ()
 

Assistant/Threads/Watcher.hs

@@ -46,7 +46,7 @@ needLsof = error $ unlines
 	]
 
 watchThread :: ThreadState -> DaemonStatusHandle -> ChangeChan -> IO ()
-watchThread st dstatus changechan = watchDir "." ignored hooks startup
+watchThread st dstatus changechan = void $ watchDir "." ignored hooks startup
 	where
 		startup = statupScan st dstatus
 		hook a = Just $ runHandler st dstatus changechan a

Utility/DirWatcher.hs

@@ -17,7 +17,6 @@ import Utility.Types.DirWatcher
 #if WITH_INOTIFY
 import qualified Utility.INotify as INotify
 import qualified System.INotify as INotify
-import Utility.ThreadScheduler
 #endif
 #if WITH_KQUEUE
 import qualified Utility.Kqueue as Kqueue
@@ -72,19 +71,41 @@ closingTracked = undefined
 #endif
 #endif
 
+/* Starts a watcher thread. The runStartup action is passed a scanner action
+ * to run, that will return once the initial directory scan is complete.
+ * Once runStartup returns, the watcher thread continues running,
+ * and processing events. Returns a DirWatcherHandle that can be used
+ * to shutdown later.  */
 #if WITH_INOTIFY
-watchDir :: FilePath -> Pruner -> WatchHooks -> (IO () -> IO ()) -> IO ()
-watchDir dir prune hooks runstartup = INotify.withINotify $ \i -> do
+type DirWatcherHandle = INotify.INotify
+watchDir :: FilePath -> Pruner -> WatchHooks -> (IO () -> IO ()) -> IO DirWatcherHandle
+watchDir dir prune hooks runstartup = do
+	i <- INotify.initINotify
 	runstartup $ INotify.watchDir i dir prune hooks
-	waitForTermination -- Let the inotify thread run.
+	return i
 #else
 #if WITH_KQUEUE
-watchDir :: FilePath -> Pruner -> WatchHooks -> (IO Kqueue.Kqueue -> IO Kqueue.Kqueue) -> IO ()
+type DirWatcherHandle = ThreadID
+watchDir :: FilePath -> Pruner -> WatchHooks -> (IO Kqueue.Kqueue -> IO Kqueue.Kqueue) -> IO DirWatcherHandle
 watchDir dir ignored hooks runstartup = do
 	kq <- runstartup $ Kqueue.initKqueue dir ignored
-	Kqueue.runHooks kq hooks
+	forkIO $ Kqueue.runHooks kq hooks
 #else
-watchDir :: FilePath -> Pruner -> WatchHooks -> (IO () -> IO ()) -> IO ()
+type DirWatcherHandle = ()
+watchDir :: FilePath -> Pruner -> WatchHooks -> (IO () -> IO ()) -> IO DirWatcherHandle
 watchDir = undefined
 #endif
 #endif
+
+#if WITH_INOTIFY
+stopWatchDir :: DirWatcherHandle -> IO ()
+stopWatchDir = INotify.killINotify
+#else
+#if WITH_KQUEUE
+stopWatchDir :: DirWatcherHandle -> IO ()
+stopWatchDir = killThread
+#else
+stopWatchDir :: DirWatcherHandle -> IO ()
+stopWatchDir = undefined
+#endif
+#endif