OSX FSEvents support

Needs work to deal with directory renames better; otherwise seems to
basically work.

Makefile

@@ -26,14 +26,19 @@ FEATURES:=$(shell echo $(FEATURES) | sed -e 's/-DWITH_ASSISTANT//' -e 's/-DWITH_
 else
 # BSD system
 THREADFLAGS=-threaded
-OPTFLAGS?=-DWITH_KQUEUE
-clibs=Utility/libdiskfree.o Utility/libmounts.o Utility/libkqueue.o
 ifeq ($(OS),Darwin)
+# use fsevents for OSX
+OPTFLAGS?=-DWITH_FSEVENTS
+clibs=Utility/libdiskfree.o Utility/libmounts.o
 # Ensure OSX compiler builds for 32 bit when using 32 bit ghc
 GHCARCH:=$(shell ghc -e 'print System.Info.arch')
 ifeq ($(GHCARCH),i386)
 CFLAGS=-Wall -m32
 endif
+else
+# BSD system with kqueue
+OPTFLAGS?=-DWITH_KQUEUE
+clibs=Utility/libdiskfree.o Utility/libmounts.o Utility/libkqueue.o
 endif
 endif
 endif

Utility/DirWatcher.hs

@@ -1,7 +1,8 @@
 {- generic directory watching interface
  -
- - Uses either inotify or kqueue to watch a directory (and subdirectories)
+ - Uses inotify, or kqueue, or fsevents to watch a directory
- - for changes, and runs hooks for different sorts of events as they occur.
+ - (and subdirectories) for changes, and runs hooks for different
+ - sorts of events as they occur.
  -
  - Copyright 2012 Joey Hess <joey@kitenet.net>
  -
@@ -22,11 +23,15 @@ import qualified System.INotify as INotify
 import qualified Utility.Kqueue as Kqueue
 import Control.Concurrent
 #endif
+#if WITH_FSEVENTS
+import qualified Utility.FSEvents as FSEvents
+import qualified System.OSX.FSEvents as FSEvents
+#endif
 
 type Pruner = FilePath -> Bool
 
 canWatch :: Bool
-#if (WITH_INOTIFY || WITH_KQUEUE)
+#if (WITH_INOTIFY || WITH_KQUEUE || WITH_FSEVENTS)
 canWatch = True
 #else
 #if defined linux_HOST_OS
@@ -42,7 +47,7 @@ canWatch = False
  - OTOH, with kqueue, often only one event is received, indicating the most
  - recent state of the file. -}
 eventsCoalesce :: Bool
-#if WITH_INOTIFY
+#if (WITH_INOTIFY || WITH_FSEVENTS)
 eventsCoalesce = False
 #else
 #if WITH_KQUEUE
@@ -55,12 +60,15 @@ eventsCoalesce = undefined
 {- With inotify, file closing is tracked to some extent, so an add event
  - will always be received for a file once its writer closes it, and
  - (typically) not before. This may mean multiple add events for the same file.
+ - 
+ - fsevents behaves similarly, although different event types are used for
+ - creating and modification of the file.
  -
  - OTOH, with kqueue, add events will often be received while a file is
  - still being written to, and then no add event will be received once the
  - writer closes it. -}
 closingTracked :: Bool
-#if WITH_INOTIFY
+#if (WITH_INOTIFY || WITH_FSEVENTS)
 closingTracked = True
 #else
 #if WITH_KQUEUE
@@ -71,9 +79,11 @@ closingTracked = undefined
 #endif
 
 {- With inotify, modifications to existing files can be tracked.
- - Kqueue does not support this. -}
+ - Kqueue does not support this.
+ - Fsevents generates events when an existing file is reopened and rewritten,
+ - but not necessarily when it's opened once and modified repeatedly. -}
 modifyTracked :: Bool
-#if WITH_INOTIFY
+#if (WITH_INOTIFY || WITH_FSEVENTS)
 modifyTracked = True
 #else
 #if WITH_KQUEUE
@@ -83,9 +93,9 @@ modifyTracked = undefined
 #endif
 #endif
 
-{- Starts a watcher thread. The runStartup action is passed a scanner action
+{- Starts a watcher thread. The runstartup action is passed a scanner action
  - to run, that will return once the initial directory scan is complete.
- - Once runStartup returns, the watcher thread continues running,
+ - Once runstartup returns, the watcher thread continues running,
  - and processing events. Returns a DirWatcherHandle that can be used
  - to shutdown later. -}
 #if WITH_INOTIFY
@@ -103,11 +113,18 @@ watchDir dir prune hooks runstartup = do
 	kq <- runstartup $ Kqueue.initKqueue dir prune
 	forkIO $ Kqueue.runHooks kq hooks
 #else
+#if WITH_FSEVENTS
+type DirWatcherHandle = FSEvents.EventStream
+watchDir :: FilePath -> Pruner -> WatchHooks -> (IO FSEvents.EventStream -> IO FSEvents.EventStream) -> IO DirWatcherHandle
+watchDir dir prune hooks runstartup =
+	runstartup $ FSEvents.watchDir dir prune hooks
+#else
 type DirWatcherHandle = ()
 watchDir :: FilePath -> Pruner -> WatchHooks -> (IO () -> IO ()) -> IO DirWatcherHandle
 watchDir = undefined
 #endif
 #endif
+#endif
 
 #if WITH_INOTIFY
 stopWatchDir :: DirWatcherHandle -> IO ()
@@ -117,7 +134,12 @@ stopWatchDir = INotify.killINotify
 stopWatchDir :: DirWatcherHandle -> IO ()
 stopWatchDir = killThread
 #else
+#if WITH_FSEVENTS
+stopWatchDir :: DirWatcherHandle -> IO ()
+stopWatchDir = FSEvents.eventStreamDestroy
+#else
 stopWatchDir :: DirWatcherHandle -> IO ()
 stopWatchDir = undefined
 #endif
 #endif
+#endif

Utility/FSEvents.hs

@@ -0,0 +1,65 @@
+{- FSEvents interface
+ -
+ - Copyright 2012 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.FSEvents where
+
+import Common hiding (isDirectory)
+import Utility.Types.DirWatcher
+
+import System.OSX.FSEvents
+import qualified System.Posix.Files as Files
+import Data.Bits ((.&.))
+
+watchDir :: FilePath -> (FilePath -> Bool) -> WatchHooks -> IO EventStream
+watchDir dir ignored hooks = do
+	unlessM fileLevelEventsSupported $
+		error "Need at least OSX 10.7.0 for file-level FSEvents"
+	eventStreamCreate [dir] 1.0 True False True handle
+  where
+	handle evt
+		| not (hasflag eventFlagItemIsFile) = noop
+		| ignoredPath ignored (eventPath evt) = noop
+		| otherwise = do
+			{- More than one flag may be set, if events occurred
+			 - close together. 
+			 - 
+			 - Order is important..
+			 - If a file is added and then deleted, we'll see it's
+			 - not present, and addHook won't run.
+			 - OTOH, if a file is deleted and then re-added,
+			 - the delHook will run first, followed by the addHook.
+			 -}
+
+			{- Deletion events are received for both directories
+			 - and files, with no way to differentiate between
+			 - them. Deleting a directory always first yields
+			 - events deleting its contents though, so we
+			 - just always call delHook, and never delDirHook. -}
+			when (hasflag eventFlagItemRemoved) $
+				runhook delHook Nothing
+			{- TODO deal with moving whole directories -}
+			when (hasflag eventFlagItemCreated || hasflag eventFlagItemRenamed) $ do
+				ms <- getstatus $ eventPath evt
+				case ms of
+					Nothing -> noop
+					Just s
+						| Files.isSymbolicLink s -> 
+							runhook addSymlinkHook ms
+						| Files.isRegularFile s ->
+							runhook addHook ms
+						| otherwise -> noop
+			when (hasflag eventFlagItemModified) $ do
+				ms <- getstatus $ eventPath evt
+				runhook modifyHook ms
+	  where
+		getstatus = catchMaybeIO . getSymbolicLinkStatus
+		hasflag f = eventFlags evt .&. f /= 0
+		runhook h s = maybe noop (\a -> a (eventPath evt) s) (h hooks)
+
+{- Check each component of the path to see if it's ignored. -}
+ignoredPath :: (FilePath -> Bool) -> FilePath -> Bool
+ignoredPath ignored = any ignored . map dropTrailingPathSeparator . splitPath

debian/changelog

@@ -5,9 +5,12 @@ git-annex (3.20121212) UNRELEASED; urgency=low
     via symlinks. Note that direct mode is currently experimental. Many
     git and git-annex commands do not work, or can even cause data loss in
     direct mode.
-  * assistant: Support direct mode; however kqueue based systems (including
+  * assistant: Support direct mode.
-    OSX) do not yet support autocommitting after files are modified in
+  * OSX assistant: Now uses the FSEvents API to detect file changes.
-    direct mode.
+    This avoids issues with running out of file descriptors on large trees,
+    as well as allowing detection of modification of files in direct mode.
+    BSD systems still use kqueue, and cannot detect modifications of existing
+    files in direct mode.
   * kqueue: Fix bug that made broken symlinks not be noticed.
   * vicfg: Quote filename. Closes: #696193
   * Bugfix: Fixed bug parsing transfer info files, where the newline after
@@ -20,6 +23,9 @@ git-annex (3.20121212) UNRELEASED; urgency=low
   * SHA*E backends: Exclude non-alphanumeric characters from extensions.
   * migrate: Remove leading \ in SHA* checksums, and non-alphanumerics
     from extensions of SHA*E keys.
+ 
+  -- Joey Hess <joeyh@debian.org>  Thu, 13 Dec 2012 14:06:43 -0400
+ 
 
  -- Joey Hess <joeyh@debian.org>  Thu, 13 Dec 2012 14:06:43 -0400
 

git-annex.cabal

@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20121211
+Version: 3.20121212
 Cabal-Version: >= 1.8
 License: GPL
 Maintainer: Joey Hess <joey@kitenet.net>
@@ -84,9 +84,13 @@ Executable git-annex
     Build-Depends: hinotify
     CPP-Options: -DWITH_INOTIFY
   else
-    if (! os(windows) && ! os(solaris) && ! os(linux))
+    if os(darwin)
-      CPP-Options: -DWITH_KQUEUE
+      Build-Depends: hfsevents
-      C-Sources: Utility/libkqueue.c
+      CPP-Options: -DWITH_HFSEVENTS
+    else
+      if (! os(windows) && ! os(solaris) && ! os(linux))
+        CPP-Options: -DWITH_KQUEUE
+        C-Sources: Utility/libkqueue.c
 
   if os(linux) && flag(Dbus)
     Build-Depends: dbus (>= 0.10.3)