Merge branch 'master' into desymlink

Build/OSXMkLibs.hs

@@ -23,42 +23,53 @@ import Utility.Monad
 import Utility.SafeCommand
 import Utility.Path
 
+import qualified Data.Map as M
+import qualified Data.Set as S
+
+type LibMap = M.Map FilePath String
+
 {- Recursively find and install libs, until nothing new to install is found. -}
-mklibs :: FilePath -> [FilePath] -> IO [FilePath]
-mklibs appbase libdirs = do
-	new <- catMaybes <$> installLibs appbase
-	if null new
-		then return (libdirs++new)
-		else mklibs appbase (libdirs++new)
+mklibs :: FilePath -> [FilePath] -> LibMap -> IO ()
+mklibs appbase libdirs libmap = do
+	(new, libmap') <- installLibs appbase libmap
+	unless (null new) $
+		mklibs appbase (libdirs++new) libmap'
 
 {- Returns directories into which new libs were installed. -}
-installLibs :: FilePath -> IO [Maybe FilePath]
-installLibs appbase = do
-	needlibs <- otool appbase
-	forM needlibs $ \lib -> do
-		let dest = appbase </> takeFileName lib
+installLibs :: FilePath -> LibMap -> IO ([FilePath], LibMap)
+installLibs appbase libmap = do
+	(needlibs, libmap') <- otool appbase libmap
+	libs <- forM needlibs $ \lib -> do
+		let shortlib = fromMaybe (error "internal") (M.lookup lib libmap')
+		let fulllib = dropWhile (== '/') lib
+		let dest = appbase </> fulllib
+		let symdest = appbase </> shortlib
 		ifM (doesFileExist dest)
 			( return Nothing
 			, do
-				createDirectoryIfMissing True appbase
-				putStrLn $ "installing " ++ lib
+				createDirectoryIfMissing True (parentDir dest)
+				putStrLn $ "installing " ++ lib ++ " as " ++ shortlib
 				_ <- boolSystem "cp" [File lib, File dest]
 				_ <- boolSystem "chmod" [Param "644", File dest]
+				_ <- boolSystem "ln" [Param "-s", File fulllib, File symdest]
 				return $ Just appbase
 			)
+	return (catMaybes libs, libmap')
 
 {- Returns libraries to install. -}
-otool :: FilePath -> IO [FilePath]
-otool appbase = do
+otool :: FilePath -> LibMap -> IO ([FilePath], LibMap)
+otool appbase libmap = do
 	files <- filterM doesFileExist =<< dirContentsRecursive appbase
-	l <- forM files $ \file -> do
-		libs <- filter unprocessed . parseOtool
-			<$> readProcess "otool" ["-L", file]
-		forM_ libs $ \lib -> install_name_tool file lib
-		return libs
-	return $ nub $ concat l
+	process [] files libmap
   where
 	unprocessed s = not ("@executable_path" `isInfixOf` s)
+	process c [] m = return (nub $ concat c, m)
+	process c (file:rest) m = do
+		_ <- boolSystem "chmod" [Param "755", File file]
+		libs <- filter unprocessed . parseOtool
+			<$> readProcess "otool" ["-L", file]
+		m' <- install_name_tool file libs m
+		process (libs:c) rest m'
 
 parseOtool :: String -> [FilePath]
 parseOtool = catMaybes . map parse . lines
@@ -67,27 +78,47 @@ parseOtool = catMaybes . map parse . lines
 		| "\t" `isPrefixOf` l = headMaybe $ words l
 		| otherwise = Nothing
 
-{- Adjusts binaries to use libraries in paths relative to the executable.
- -
- - Assumes all executables are installed into the same directory, and
- - the libraries will be installed in subdirectories that match their
- - absolute paths. -}
-install_name_tool :: FilePath -> FilePath -> IO ()
-install_name_tool binary lib = do
-	ok <- boolSystem "install_name_tool"
+{- Adjusts binaries to use libraries bundled with it, rather than the
+ - system libraries. -}
+install_name_tool :: FilePath -> [FilePath] -> LibMap -> IO LibMap
+install_name_tool _ [] libmap = return libmap
+install_name_tool binary libs libmap = do
+	let (libnames, libmap') = getLibNames libs libmap
+	let params = concatMap change $ zip libs libnames
+	ok <- boolSystem "install_name_tool" $ params ++ [File binary]
+	unless ok $
+		error $ "install_name_tool failed for " ++ binary
+	return libmap'
+  where
+	change (lib, libname) =
 		[ Param "-change"
 		, File lib
-		, Param $ "@executable_path" ++ (dropFileName lib)
-		, File binary
+		, Param $ "@executable_path/" ++ libname
 		]
-	unless ok $
-		hPutStrLn stderr $ "install_name_tool failed for " ++ binary
+
+getLibNames :: [FilePath] -> LibMap -> ([FilePath], LibMap)
+getLibNames libs libmap = go [] libs libmap
+  where
+	go c [] m = (reverse c, m)
+	go c (l:rest) m =
+		let (f, m') = getLibName l m
+		in go (f:c) rest m'
+
+{- Uses really short names for the library files it installs, because
+ - binaries have arbitrarily short RPATH field limits. -}
+getLibName :: FilePath -> LibMap -> (FilePath, LibMap)
+getLibName lib libmap = case M.lookup lib libmap of
+	Just n -> (n, libmap)
+	Nothing -> (nextfreename, M.insert lib nextfreename libmap)
+  where
+	names = map (\c -> [c]) ['A' .. 'Z'] ++
+		[[n, l] | n <- ['0' .. '9'], l <- ['A' .. 'Z']]
+	used = S.fromList $ M.elems libmap
+	nextfreename = fromMaybe (error "ran out of short library names!") $ 
+		headMaybe $ dropWhile (`S.member` used) names
 
 main :: IO ()
 main = getArgs >>= go
   where
 	go [] = error "specify OSXAPP_BASE"
-	go (appbase:_) = do
-		libdirs <- mklibs appbase []
-		writeFile (appbase </> "libdirs") $
-			unlines $ nub libdirs
+	go (appbase:_) = mklibs appbase [] M.empty

Makefile

@@ -7,7 +7,7 @@ BASEFLAGS=-Wall -outputdir $(GIT_ANNEX_TMP_BUILD_DIR) -IUtility
 #
 # If you're using an old version of yesod, enable -DWITH_OLD_YESOD
 # Or with an old version of the uri library, enable -DWITH_OLD_URI
-FEATURES?=$(GIT_ANNEX_LOCAL_FEATURES) -DWITH_ASSISTANT -DWITH_S3 -DWITH_WEBDAV -DWITH_WEBAPP -DWITH_PAIRING -DWITH_XMPP -DWITH_DNS
+FEATURES?=$(GIT_ANNEX_LOCAL_FEATURES) -DWITH_ASSISTANT -DWITH_S3 -DWITH_WEBDAV -DWITH_WEBAPP -DWITH_PAIRING -DWITH_XMPP -DWITH_HOST
 
 bins=git-annex
 mans=git-annex.1 git-annex-shell.1
@@ -210,7 +210,7 @@ osxapp:
 	hdiutil create -size 640m -format UDRW -srcfolder $(GIT_ANNEX_TMP_BUILD_DIR)/build-dmg \
 		-volname git-annex -o tmp/git-annex.dmg
 	rm -f tmp/git-annex.dmg.bz2
-	bzip2 tmp/git-annex.dmg
+	bzip2 --fast tmp/git-annex.dmg
 
 # used by ./ghci
 getflags:

debian/changelog

@@ -29,6 +29,7 @@ git-annex (3.20121128) UNRELEASED; urgency=low
   * webapp: Allow user to specify the port when setting up a ssh or rsync
     remote.
   * assistant: Fix syncing to just created ssh remotes.
+  * Enable WebDAV support in Debian package. Closes: #695532
 
  -- Joey Hess <joeyh@debian.org>  Wed, 28 Nov 2012 13:31:07 -0400
 

debian/rules

@@ -2,9 +2,9 @@
 
 ARCH = $(shell dpkg-architecture -qDEB_BUILD_ARCH)
 ifeq (install ok installed,$(shell dpkg-query -W -f '$${Status}' libghc-yesod-dev 2>/dev/null))
-export FEATURES=-DWITH_ASSISTANT -DWITH_S3 -DWITH_OLD_URI -DWITH_OLD_YESOD -DWITH_WEBAPP -DWITH_PAIRING -DWITH_XMPP
+export FEATURES=-DWITH_ASSISTANT -DWITH_S3 -DWITH_WEBDAV -DWITH_HOST -DWITH_OLD_URI -DWITH_OLD_YESOD -DWITH_WEBAPP -DWITH_PAIRING -DWITH_XMPP
 else
-export FEATURES=-DWITH_ASSISTANT -DWITH_S3 -DWITH_OLD_URI -DWITH_PAIRING -DWITH_XMPP
+export FEATURES=-DWITH_ASSISTANT -DWITH_S3 -DWITH_WEBDAV -DWITH_HOST -DWITH_OLD_URI -DWITH_PAIRING -DWITH_XMPP
 endif
 
 %:

doc/bugs/Proxy_support.mdwn

@@ -0,0 +1,18 @@
+What steps will reproduce the problem?
+
+Adding a e.g box.com repository from behind a http proxy via webapp.
+
+What is the expected output? What do you see instead?
+
+Connection should be made. But there is an error message:
+
+"Internal Server Error
+connect: does not exist (Connection refused): user error"
+
+What version of git-annex are you using? On what operating system?
+
+3.20121127 on Archlinux
+
+Please provide any additional information below.
+
+I don't use networkmanager if proxy information is obtained from it. There should be a fallback to environment variables.

doc/bugs/yesod-form_missing.mdwn

@@ -19,3 +19,5 @@ git checkout commit 90b62db1defdd223294935ec0bbaac883cd20c04 on OS X Lion
 Please provide any additional information below.
 
 adding yesod-form to the build depends in git-annex.cabal does indeed fix the problem!
+
+> [[done]] --[[Joey]] 

doc/design/assistant/blog/day_148__direct_mode.mdwn

@@ -0,0 +1,42 @@
+Got object sending working in direct mode. However, I don't yet have a
+reliable way to deal with files being modified while they're being
+transferred. I have code that detects it on the sending side, but the
+receiver is still free to move the wrong content into its annex, and record
+that it has the content. So that's not acceptable, and I'll need to work
+on it some more. However, at this point I can use a direct mode repository
+as a remote and transfer files from and to it.
+
+----
+
+Automated updating of the cached mtime, etc data. Next I need to automate
+generation of the key to filename mapping files. I'm thinking that I'll make
+`git annex sync` do it. Then, once I get committing and 
+merging working in direct mode repositories (which is likely to be a
+good week's worth of work), the workflow for using these repositories
+will be something like this:
+
+	git config annex.direct true
+	git annex sync # pulls any changes, merges, updates maps and caches
+	git annex get
+	# modify files
+	git annex sync # commits and pushes changes
+
+And once I get direct mode repositories working to this degree at the
+command line, I can get on with adding support to the assistant.
+
+----
+
+Also did some more work today on the OSX app. Am in the middle of getting
+it to modify the binaries in the app to change the paths to the libraries they
+depend on. This will avoid the hacky environment variable it is currently
+using, and make runshell a much more usable environment. It's the right way
+to do it. (I can't believe I just said RPATH was the right way to do
+anything.)
+
+In the middle of this, I discovered
+<http://hackage.haskell.org/package/cabal-macosx>, which does the same
+type of thing.
+
+Anyway, I have to do some crazy hacks to work around short library name
+fields in executables that I don't want to have to be specially rebuilt in
+order to build the webapp. Like git.

doc/design/assistant/desymlink.mdwn

@@ -80,3 +80,9 @@ is converted to a real file when it becomes present.
   the user uses directly. Would need some git-annex commands
   to merge changes into the repo, update caches, and commit changes.
   This could all be done by "git annex sync".
+
+## TODO
+
+* Deal with files changing as they're being transferred from a direct mode
+  repository to another git repository. The remote repo currently will 
+  accept the bad data and update the location log to say it has the key.

doc/forum/External_drive_syncs_git-annex_branch_but_not_master_branch/comment_1_9a909e3d89061adacbd8ed370520250c._comment

@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://edheil.wordpress.com/"
+ ip="173.162.44.162"
+ subject="comment 1"
+ date="2012-12-08T21:25:49Z"
+ content="""
+apologies if I misunderstand your question, but I believe that git-annex puts the metadata in the repository's sync/master branch, which you can merge into the repository's local master either manually or by running \"git annex sync\" in the repository.
+
+"""]]

doc/forum/External_drive_syncs_git-annex_branch_but_not_master_branch/comment_2_0dd489b264374b7b1065b89e1ff7561b._comment

@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="spwhitton"
+ ip="163.1.166.255"
+ subject="comment 2"
+ date="2012-12-08T21:57:37Z"
+ content="""
+It does, which is what one would have expected I guess, but I didn't look.  Thanks so much.
+"""]]

doc/forum/gadu_-_git-annex_disk_usage/comment_3_38296fef5a2dc5794c2dc09df676b8c1._comment

@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="Steve"
+ ip="92.104.175.136"
+ subject="gadu 0.03 is up"
+ date="2012-12-09T13:05:10Z"
+ content="""
+* 1K blocksize is now the default
+* \".\" is now the default path
+* implemented -B/--block-size option
+* --help is no longer -h, it only has a long option like du
+* implemented -h/--human-readable option
+
+du will take up to yottabytes for the --block-size option.  I had been fudging the sizes with a size_t thinking 16 exabytes was plenty big enough for now, but since I was implementing --block-size I went ahead and converted everything to use the GNU MP.  So libgmp is now a dependency.
+
+--human-readable probably doesn't have exactly the same output, but I think it is good enough.  I tried to make the options work mostly the same as du from core-utils.  Let me know if you find other discrepancies.
+
+I'll see about making the git tree available soon, but it may have to wait until next weekend.  I may also look into a forum for the website, or a mailing list.
+"""]]

doc/install/OSX.mdwn

@@ -16,7 +16,8 @@ sudo brew update
 sudo brew install haskell-platform git ossp-uuid md5sha1sum coreutils pcre libgsasl gnutls libidn libgsasl pkg-config libxml2
 sudo brew link libxml2
 cabal update
-cabal install git-annex --bindir=$HOME/bin
+PATH=$HOME/bin:$PATH
+cabal install c2hs git-annex --bindir=$HOME/bin
 </pre>
 
 ## using MacPorts
@@ -31,14 +32,15 @@ sudo port install git-core ossp-uuid md5sha1sum coreutils pcre
 sudo ln -s /opt/local/include/pcre.h  /usr/include/pcre.h # This is hack that allows pcre-light to find pcre
 
 sudo cabal update
-cabal install git-annex --bindir=$HOME/bin
+PATH=$HOME/bin:$PATH
+cabal install c2hs git-annex --bindir=$HOME/bin
 </pre>
 
 ## PATH setup
 
 Do not forget to add to your PATH variable your ~/bin folder. In your .bashrc, for example:
 <pre>
-PATH=$HOME/bin:/usr/bin/local:$PATH
+PATH=$HOME/bin:$PATH
 </pre>
 
 See also:

doc/install/cabal.mdwn

@@ -1,7 +1,8 @@
 As a haskell package, git-annex can be installed using cabal. For example:
 
 	cabal update
-        cabal install c2hs git-annex --bindir=$HOME/bin
+	PATH=$HOME/bin:$PATH
+	cabal install c2hs git-annex --bindir=$HOME/bin
 
 The above downloads the latest release and installs it into a ~/bin/
 directory, which you can put in your PATH.
@@ -10,6 +11,7 @@ But maybe you want something newer (or older). Then [[download]] the version
 you want, and use cabal as follows inside its source tree:
 
 	cabal update
+	PATH=$HOME/bin:$PATH
 	cabal install c2hs --bindir=$HOME/bin
 	cabal install --only-dependencies
 	cabal configure

git-annex.cabal

@@ -95,7 +95,7 @@ Executable git-annex
   if flag(Webapp) && flag(Assistant)
     Build-Depends: yesod, yesod-static, case-insensitive,
      http-types, transformers, wai, wai-logger, warp, blaze-builder,
-     crypto-api, hamlet, clientsession, aeson,
+     crypto-api, hamlet, clientsession, aeson, yesod-form,
      template-haskell, yesod-default (>= 1.1.0), data-default
     CPP-Options: -DWITH_WEBAPP
 

standalone/osx/git-annex.app/Contents/MacOS/git-annex

@@ -17,7 +17,7 @@ cd "$orig"
 
 # If this is a standalone app, set a variable that git-annex can use to
 # install itself.
-if [ -e "$base/bin/git-annex" ]; then
+if [ -e "$base/git-annex" ]; then
 	GIT_ANNEX_APP_BASE="$base"
 	export GIT_ANNEX_APP_BASE
 fi

standalone/osx/git-annex.app/Contents/MacOS/git-annex-webapp

@@ -17,7 +17,7 @@ cd "$orig"
 
 # If this is a standalone app, set a variable that git-annex can use to
 # install itself.
-if [ -e "$base/bin/git-annex" ]; then
+if [ -e "$base/git-annex" ]; then
 	GIT_ANNEX_APP_BASE="$base"
 	export GIT_ANNEX_APP_BASE
 fi

standalone/osx/git-annex.app/Contents/MacOS/runshell

@@ -1,6 +1,6 @@
 #!/bin/sh
-# Runs a shell command (or interactive shell) using the binaries and
-# libraries bundled with this app.
+# Runs a shell command (or interactive shell) using the binaries 
+# bundled with this app.
 
 set -e
 
@@ -11,12 +11,12 @@ if [ ! -d "$base" ]; then
 	exit 1
 fi
 
-if [ ! -e "$base/bin/git-annex" ]; then
-	echo "** base directory $base does not contain bin/git-annex" >&2
+if [ ! -e "$base/git-annex" ]; then
+	echo "** base directory $base does not contain git-annex" >&2
 	exit 1
 fi
-if [ ! -e "$base/bin/git" ]; then
-	echo "** base directory $base does not contain bin/git" >&2
+if [ ! -e "$base/git" ]; then
+	echo "** base directory $base does not contain git" >&2
 	exit 1
 fi
 
@@ -46,20 +46,6 @@ export ORIG_PATH
 PATH=$base/bin:$PATH
 export PATH
 
-# This makes the linker first look in the library directories bundled
-# in this app. Only if it fails to find a library there will a system
-# library be used.
-# 
-# This seems to work better than DYLD_LIBRARY_PATH, which can force
-# the wrong version of a library to be used, if the app bundles two
-# different versions of a single library. And it seems to work better
-# than DYLD_FALLBACK_LIBRARY_PATH, which fails to override old system
-# versions of libraries when a program in the app needs a newer version.
-#ORIG_DYLD_ROOT_PATH="$DYLD_ROOT_PATH"
-#export ORIG_DYLD_ROOT_PATH
-#DYLD_ROOT_PATH=$base
-#export DYLD_ROOT_PATH
-
 ORIG_GIT_EXEC_PATH="$GIT_EXEC_PATH"
 export ORIG_GIT_EXEC_PATH
 GIT_EXEC_PATH=$base