Merge branch 'master' into adjustedbranch

Annex/FileMatcher.hs

@@ -33,6 +33,7 @@ import Git.CheckAttr (unspecifiedAttr)
 
 #ifdef WITH_MAGICMIME
 import Magic
+import Utility.Env
 #endif
 
 import Data.Either
@@ -131,7 +132,12 @@ mkLargeFilesParser = do
 #ifdef WITH_MAGICMIME
 	magicmime <- liftIO $ catchMaybeIO $ do
 		m <- magicOpen [MagicMimeType]
-		liftIO $ magicLoadDefault m
+		liftIO $ do
+			md <- getEnv "GIT_ANNEX_DIR"
+			case md of
+				Nothing -> magicLoadDefault m
+				Just d -> magicLoad m
+					(d </> "magic" </> "magic.mgc")
 		return m
 #endif
 	let parse = parseToken $ commonTokens

Annex/MetaData.hs

@@ -1,6 +1,6 @@
 {- git-annex metadata
  -
- - Copyright 2014 Joey Hess <id@joeyh.name>
+ - Copyright 2014-2016 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -8,6 +8,8 @@
 module Annex.MetaData (
 	genMetaData,
 	dateMetaData,
+	parseModMeta,
+	parseMetaDataMatcher,
 	module X
 ) where
 
@@ -17,6 +19,7 @@ import Types.MetaData as X
 import Annex.MetaData.StandardFields as X
 import Logs.MetaData
 import Annex.CatFile
+import Utility.Glob
 
 import qualified Data.Set as S
 import qualified Data.Map as M
@@ -53,3 +56,37 @@ dateMetaData mtime old = MetaData $ M.fromList $ filter isnew
   where
 	isnew (f, _) = S.null (currentMetaDataValues f old)
 	(y, m, _d) = toGregorian $ utctDay mtime
+
+{- Parses field=value, field+=value, field-=value, field?=value -}
+parseModMeta :: String -> Either String ModMeta
+parseModMeta p = case lastMaybe f of
+	Just '+' -> AddMeta <$> mkMetaField f' <*> v
+	Just '-' -> DelMeta <$> mkMetaField f' <*> (Just <$> v)
+	Just '?' -> MaybeSetMeta <$> mkMetaField f' <*> v
+	_ -> SetMeta <$> mkMetaField f <*> v
+  where
+	(f, sv) = separate (== '=') p
+	f' = beginning f
+	v = pure (toMetaValue sv)
+
+{- Parses field=value, field<value, field<=value, field>value, field>=value -}
+parseMetaDataMatcher :: String -> Either String (MetaField, MetaValue -> Bool)
+parseMetaDataMatcher p = (,)
+	<$> mkMetaField f
+	<*> pure matcher
+  where
+	(f, op_v) = break (`elem` "=<>") p
+	matcher = case op_v of
+		('=':v) -> checkglob v
+		('<':'=':v) -> checkcmp (<=) v
+		('<':v) -> checkcmp (<) v
+		('>':'=':v) -> checkcmp (>=) v
+		('>':v) -> checkcmp (>) v
+		_ -> checkglob ""
+	checkglob v =
+		let cglob = compileGlob v CaseInsensative
+		in matchGlob cglob . fromMetaValue
+	checkcmp cmp v v' = case (doubleval v, doubleval (fromMetaValue v')) of
+		(Just d, Just d') -> d' `cmp` d
+		_ -> False
+	doubleval v = readish v :: Maybe Double

Backend/Hash.hs

@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-{-# LANGUAGE BangPatterns, CPP #-}
+{-# LANGUAGE CPP #-}
 
 module Backend.Hash (
 	backends,
@@ -171,8 +171,9 @@ hashFile hash file filesize = go hash
 	go (SkeinHash hashsize) = use (skeinHasher hashsize)
 	
 	use hasher = liftIO $ do
-		!h <- hasher <$> L.readFile file
-		return h
+		h <- hasher <$> L.readFile file
+		-- Force full evaluation so file is read and closed.
+		return (length h `seq` h)
 	
 	usehasher hashsize = case shaHasher hashsize filesize of
 		Left sha -> use sha

Command/MetaData.hs

@@ -46,10 +46,14 @@ optParser desc = MetaDataOptions
 			( long "tag" <> short 't' <> metavar "TAG"
 			<> help "set a tag"
 			))
-		<|> (AddMeta tagMetaField . mkMetaValue (CurrentlySet False) <$> strOption
+		<|> (DelMeta tagMetaField . Just . toMetaValue <$> strOption
 			( long "untag" <> short 'u' <> metavar "TAG"
 			<> help "remove a tag"
 			))
+		<|> option (eitherReader (\f -> DelMeta <$> mkMetaField f <*> pure Nothing))
+			( long "remove"  <> short 'r' <> metavar "FIELD"
+			<> help "remove all values of a field"
+			)
 
 seek :: MetaDataOptions -> CommandSeek
 seek o = do

Git/Env.hs

@@ -12,6 +12,9 @@ module Git.Env where
 import Git
 import Git.Types
 import Utility.Env
+#ifdef __ANDROID__
+import Common
+#endif
 
 {- Adjusts the gitEnv of a Repo. Copies the system environment if the repo
  - does not have any gitEnv yet. -}

Git/LsTree.hs

@@ -73,7 +73,7 @@ parseLsTree l = TreeItem
 	{ mode = smode
 	, typeobj = t
 	, sha = Ref s
-	, file = asTopFilePath $ Git.Filename.decode f
+	, file = sfile
 	}
   where
 	-- l = <mode> SP <type> SP <sha> TAB <file>

Limit.hs

@@ -23,6 +23,7 @@ import Types.TrustLevel
 import Types.Group
 import Types.FileMatcher
 import Types.MetaData
+import Annex.MetaData
 import Logs.MetaData
 import Logs.Group
 import Logs.Unused
@@ -278,14 +279,12 @@ addMetaData :: String -> Annex ()
 addMetaData = addLimit . limitMetaData
 
 limitMetaData :: MkLimit Annex
-limitMetaData s = case parseMetaData s of
+limitMetaData s = case parseMetaDataMatcher s of
 	Left e -> Left e
-	Right (f, v) ->
-		let cglob = compileGlob (fromMetaValue v) CaseInsensative
-		in Right $ const $ checkKey (check f cglob)
+	Right (f, matching) -> Right $ const $ checkKey (check f matching)
   where
-	check f cglob k = not . S.null 
-		. S.filter (matchGlob cglob . fromMetaValue) 
+	check f matching k = not . S.null 
+		. S.filter matching
 		. metaDataValues f <$> getCurrentMetaData k
 
 addTimeLimit :: String -> Annex ()

Makefile

@@ -147,6 +147,8 @@ linuxstandalone-nobuild: Build/Standalone Build/LinuxMkLibs
 	install -d "$(LINUXSTANDALONE_DEST)/git-core"
 	(cd "$(shell git --exec-path)" && tar c .) | (cd "$(LINUXSTANDALONE_DEST)"/git-core && tar x)
 	install -d "$(LINUXSTANDALONE_DEST)/templates"
+	install -d "$(LINUXSTANDALONE_DEST)/magic"
+	cp /usr/share/file/magic.mgc "$(LINUXSTANDALONE_DEST)/magic"
 	
 	./Build/LinuxMkLibs "$(LINUXSTANDALONE_DEST)"
 	
@@ -199,6 +201,12 @@ osxapp: Build/Standalone Build/OSXMkLibs
 
 	(cd "$(shell git --exec-path)" && tar c .) | (cd "$(OSXAPP_BASE)" && tar x)
 	install -d "$(OSXAPP_BASE)/templates"
+	install -d "$(OSXAPP_BASE)/magic"
+	if [ -e "$(OSX_MAGIC_FILE)" ]; then \
+		cp "$(OSX_MAGIC_FILE)" "$(OSXAPP_BASE)/magic/magic.mgc"; \
+	else \
+		echo "** OSX_MAGIC_FILE not set; not including it" >&2; \
+	fi
 
 	# OSX looks in man dir nearby the bin
 	$(MAKE) install-mans DESTDIR="$(OSXAPP_BASE)/.." SHAREDIR="" PREFIX=""

Types/MetaData.hs

@@ -36,8 +36,6 @@ module Types.MetaData (
 	metaDataValues,
 	ModMeta(..),
 	modMeta,
-	parseModMeta,
-	parseMetaData,
 	prop_metadata_sane,
 	prop_metadata_serialize
 ) where
@@ -221,9 +219,13 @@ metaDataValues f (MetaData m) = fromMaybe S.empty (M.lookup f m)
 {- Ways that existing metadata can be modified -}
 data ModMeta
 	= AddMeta MetaField MetaValue
-	| DelMeta MetaField MetaValue
-	| SetMeta MetaField MetaValue -- removes any existing values
-	| MaybeSetMeta MetaField MetaValue -- when field has no existing value
+	| DelMeta MetaField (Maybe MetaValue)
+	-- ^ delete value of a field. With Just, only that specific value
+	-- is deleted; with Nothing, all current values are deleted.
+	| SetMeta MetaField MetaValue
+	-- ^ removes any existing values
+	| MaybeSetMeta MetaField MetaValue
+	-- ^ set when field has no existing value
 	deriving (Show)
 
 {- Applies a ModMeta, generating the new MetaData.
@@ -231,7 +233,10 @@ data ModMeta
  - values set in the input metadata. It only contains changed values. -}
 modMeta :: MetaData -> ModMeta -> MetaData
 modMeta _ (AddMeta f v) = updateMetaData f v emptyMetaData
-modMeta _ (DelMeta f oldv) = updateMetaData f (unsetMetaValue oldv) emptyMetaData
+modMeta _ (DelMeta f (Just oldv)) =
+	updateMetaData f (unsetMetaValue oldv) emptyMetaData
+modMeta m (DelMeta f Nothing) = MetaData $ M.singleton f $
+	S.fromList $ map unsetMetaValue $ S.toList $ currentMetaDataValues f m
 modMeta m (SetMeta f v) = updateMetaData f v $
 	foldr (updateMetaData f) emptyMetaData $
 		map unsetMetaValue $ S.toList $ currentMetaDataValues f m
@@ -239,26 +244,6 @@ modMeta m (MaybeSetMeta f v)
 	| S.null (currentMetaDataValues f m) = updateMetaData f v emptyMetaData
 	| otherwise = emptyMetaData
 
-{- Parses field=value, field+=value, field-=value, field?=value -}
-parseModMeta :: String -> Either String ModMeta
-parseModMeta p = case lastMaybe f of
-	Just '+' -> AddMeta <$> mkMetaField f' <*> v
-	Just '-' -> DelMeta <$> mkMetaField f' <*> v
-	Just '?' -> MaybeSetMeta <$> mkMetaField f' <*> v
-	_ -> SetMeta <$> mkMetaField f <*> v
-  where
-	(f, sv) = separate (== '=') p
-	f' = beginning f
-	v = pure (toMetaValue sv)
-
-{- Parses field=value -}
-parseMetaData :: String -> Either String (MetaField, MetaValue)
-parseMetaData p = (,)
-	<$> mkMetaField f
-	<*> pure (toMetaValue v)
-  where
-	(f, v) = separate (== '=') p
-
 {- Avoid putting too many fields in the map; extremely large maps make
  - the seriaization test slow due to the sheer amount of data.
  - It's unlikely that more than 100 fields of metadata will be used. -}

debian/changelog

@@ -1,4 +1,10 @@
-git-annex (6.20160218) UNRELEASED; urgency=medium
+git-annex (6.20160230) UNRELEASED; urgency=medium
+
+  * metadata: Added -r to remove all current values of a field.
+
+ -- Joey Hess <id@joeyh.name>  Mon, 29 Feb 2016 13:00:30 -0400
+
+git-annex (6.20160229) unstable; urgency=medium
 
   * Update perlmagick build dependency. Closes: #789225
   * Fix memory leak in last release, which affected commands like
@@ -11,8 +17,19 @@ git-annex (6.20160218) UNRELEASED; urgency=medium
     so any system gpg will be preferred over it.
   * Avoid crashing when built with MagicMime support, but when the magic
     database cannot be loaded.
+  * Include magic database in the linux and OSX standalone builds.
+  * Fix memory leak when hashing files, which triggered during fsck
+    when an external hash program was not used.
+    (This leak was introduced in version 6.20160114.)
+  * Support --metadata field<number, --metadata field>number etc
+    to match ranges of numeric values.
+  * Similarly, support preferred content expressions like
+    metadata=field<number and metadata=field>number
+  * The pre-commit-annex hook script that automatically extracts
+    metadata has been updated to also use exiftool.
+    Thanks, Klaus Ethgen.
 
- -- Joey Hess <id@joeyh.name>  Thu, 18 Feb 2016 13:09:21 -0400
+ -- Joey Hess <id@joeyh.name>  Mon, 29 Feb 2016 12:41:49 -0400
 
 git-annex (6.20160217) unstable; urgency=medium
 

debian/copyright

@@ -44,6 +44,11 @@ Copyright: 2001 Ian Lynagh
            2010-2015 Joey Hess <id@joeyh.name>
 License: GPL-3+
 
+Files: doc/tips/automatically_adding_metadata/pre-commit-annex 
+Copyright: 2014 Joey Hess <id@joeyh.name>
+           2016 Klaus Ethgen <Klaus@Ethgen.ch>
+License: GPL-3+
+
 Files: Utility/libmounts.c
 Copyright: 1980, 1989, 1993, 1994 The Regents of the University of California
            2001 David Rufino <daverufino@btinternet.com>

doc/bugs/checksum_loads_whole_file_into_memory.mdwn

@@ -0,0 +1,24 @@
+Using eg, fsck with the MD5 backend loads whole files into memory.
+
+May only happen for very large files (40 gb) or in other specific
+circumstances, including ghc version used for buildd etc.
+
+Observed with 6.20160217-g95bbdb8, linux standalone amd64.
+
+Not observed with 5.20151218-g5008846.
+
+Commit 7482853ddddc21f2696dcfbc82d737f03032134a may be relevant,
+but I don't understand how yet. A small test program using the same
+code doesn't exhibit the problem, even when built in the identical build
+environment as the 6.20160217-g95bbdb8 that has the problem.
+
+> Update: Reverted 7482853ddddc21f2696dcfbc82d737f03032134a and indeed the
+> problem got fixed. But, reverting that commit breaks the test suite on
+> windows and has a FD leak, so is not desirable. This needs more
+> investigation. --[[Joey]]
+
+>> I see it now, the checksum is a String and it was only forced to WHNF,
+>> so the hashing didn't fully complete and the file got buffered.
+>> Probably only occurred when fscking, and not when adding a file,
+>> due to differing use patterns of the checksum.
+>> [[fixed|done]] --[[Joey]] 

doc/bugs/duplicate_progress_reports_in_parallel___39__get__39__.mdwn

@@ -0,0 +1,22 @@
+[[!format sh """
+$> git annex version                    
+git-annex version: 6.20160213+gitg9597a21-1~ndall+1
+...
+$> git annex get -J 5 .
+get docs/freesurfer.groupanalysis.ppt (from origin...) (checksum...) ok
+get docs/freesurfer.future_directions.2007.ppt (from origin...) (checksum...) ok
+get docs/freesurfer.groupanalysis.short.ppt (from origin...) (checksum...) ok
+get distribution/trctrain/trctraindata.tar.gz (from origin...) 
+47%          80.2MB/s 3s
+47%          80.2MB/s 3s
+get docs/freesurfer.inferring_architectonics.ppt (from origin...) 
+33%           8.1MB/s 4s
+33%           8.1MB/s 4s
+get docs/freesurfer.intro.2011.ppt (from origin...) 
+64%           7.3MB/s 1s
+64%           7.3MB/s 1s
+get docs/freesurfer.intro.mmclass.ppt (from origin...)
+# End of transcript or log.
+"""]]
+
+[[!meta author=yoh]]

doc/bugs/using_regular_magic_file__warning_pollutes_stderr.mdwn

@@ -0,0 +1,25 @@
+### Please describe the problem.
+
+Although probably not an annex issue but thought to let you know since you might see
+a quick resolution on your end
+
+Here is the log from datalad:
+
+[[!format sh """
+2016-02-25 22:53:32,886 [DEBUG] Running: ['git', '-c', 'receive.autogc=false', '-c', 'annex.alwayscommit=false', 'annex', 'add', '--debug', '--json', '-c', 'annex.largefiles=exclude=CHANGES* and exclude=README* and exclude=*.[mc] and exclude=dataset*.json and (exclude=*.txt or include=*/*.txt)  and (exclude=*.json or include=*/*.json) and (exclude=*.tsv or include=*/*.tsv)', 'README.txt'] (cmd.py:351)
+2016-02-25 22:53:32,979 [ERROR] stderr| /etc/magic, 4: Warning: using regular magic file `/usr/share/misc/magic' (cmd.py:351)
+
+
+or outside (file is already under annex)
+
+$> git annex --debug add -c 'annex.largefiles=exclude=CHANGES* and exclude=README*' README.txt
+/etc/magic, 4: Warning: using regular magic file `/usr/share/misc/magic'
+
+"""]]
+
+annex is up to date: 6.20160225+gitg229db26-1~ndall+1
+
+edit1: that is happening on jessie with file 1:5.22+15-2+deb8u1 if that is relevant
+[[!meta author=yoh]]
+
+> [[fixed|done]] --[[Joey]]

doc/bugs/using_regular_magic_file__warning_pollutes_stderr/comment_1_407787292dd6e6d1aff7193634f8a7bf._comment

@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-02-26T15:52:43Z"
+ content="""
+Reproduced in a clean chroot using the linux standalone tarball.
+
+I think it does this when it can't find any magic file.
+
+Fixing by including the magic database in the bundle. This will also let me
+re-enable magicmime on OSX.
+"""]]

doc/design/adjusted_branches/comment_1_5bdbf54ae75f77c69c8f557f4dbac13d._comment

@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
+ subject="would it be possible to get a branch only with annexed content?"
+ date="2016-02-25T03:04:26Z"
+ content="""
+I am yet to grasp all the glorious plans here, but wondered to ask:  it seems like it then should be possible to establish a branch only with annexed archive? may be matching some preferred content expression?
+Usecase:  a relatively large git/annex repository mixing in code, large data files, and some pre-built binaries.  If I could seamlessly and reproducibly (with the progress of their master) tease those apart (for separate Debian packaging ;)), that would be really handy
+
+[[!meta author=yoh]]
+"""]]

doc/devblog/day_367__adjusted_branches_proof_of_concept.mdwn

@@ -0,0 +1,18 @@
+Now I have a proof of concept [[design/adjusted_branches]]
+implementation, that creates a branch where all locked files
+are adjusted to be unlocked. It works!
+
+Building the adjusted branch is pretty fast; around 2 thousand files
+per second. And, I have a trick in my back pocket that could double that
+speed. It's important this be quite fast, because it'll be done often.
+
+Checking out the adjusted branch can be bit slow though, since git runs
+`git annex smudge` once per unlocked file. So that might need to be
+optimised somehow. On the other hand, this should be done only rarely.
+
+I like that it generates reproducible git commits so the same adjustments
+of the same branch will always have the same sha, no matter when and where
+it's done. Implementing that involved parsing git commit objects.
+
+Next step will be merging pulled changes into the adjusted branch, while
+maintaining the desired adjustments.

doc/forum/Help_with_assistant_and_Adobe_Lightroom.mdwn

@@ -0,0 +1,3 @@
+I am trying to use the git-annex assistant to automatically sync photos from my Lightroom catalog on Mac OS X to a remote archive server. I hit a snag due to a race condition combined with some peculiar behavior of Adobe Lightroom: it dereferences symlinks and uses the name of the target when importing (see https://forums.adobe.com/thread/568863?tstart=0) I observe the following problematic workflow: 1. Files are imported using the original filename. 2. The assistant eventually moves the content to .git/annex/objects and creates symlinks to replace the originals. 3. The photos no longer appear in the Library, however Lightroom will not allow me to re-import as it thinks the files have already been imported and are duplicates.
+
+I know a few other folks are using git-annex and Lightroom together, but I cannot tell if they are using the assistant or manually managing the annex. This implementation is for my wife who is not technical enough to use git on the command line so that's out of the question. It seems like direct mode is my best option for now, however, I'd like to avoid it since it's being deprecated. Is there some other option that I'm missing? Is anyone else out there successfully using the assistant and Lightroom together? I'd love some advice.

doc/forum/How_to_improve_performance_with_v6_repos__63__.mdwn

@@ -0,0 +1,3 @@
+I've been using git-annex to manage a repo of ~4000 files (an MP3 collection). I used to use direct mode, but switched to v6 and am using unlocked files instead. However, even small commits have been taking 10-20 minutes. Looking at `ps` output shows `git-annex smudge` processes running against all of the files in the repo, one by one.
+
+Is there any way I can speed things up?

doc/forum/How_to_improve_performance_with_v6_repos__63__/comment_1_56e0fead15a88ea12262e6091972c6d6._comment

@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="page"
+ subject="Some additional notes"
+ date="2016-02-28T11:44:39Z"
+ content="""
+Just to add some info here. I just started using git-annex and v6 sounded like the way to go. I tried to add my photo collection (around 50k files) and all operations take a huge amount of time. I did 'git add photos/' (to add them in unlocked mode), which took hours. Then I run 'git annex sync', which took >10h again to do the commit. After that i run 'git annex sync' from another remote, and it's syncing items at ~2/s (smudge processes as the op mentions, not blocked either by cpu nor io). I'm not sure if I'm doing something wrong, but this is completely unusable for me. I'm all ears if I can try something, otherwise I'm gonna give v5 a try, I guess.
+"""]]

doc/forum/Multiple_remotes_with_the_same_path.mdwn

@@ -0,0 +1,67 @@
+This is a followup to <https://git-annex.branchable.com/forum/basic_usage_questions/> – I'm actually testing the "two repos at same URL" situation now, with git-annex 6.20160211 and 6.20160221.
+
+So I have a git-annex repo on two hosts at `rain:~/Attic/Software`, `frost:~/Attic/Software`, and a third clone at `/mnt/portable_HD/Attic/Software`. This means that the repo on the portable HD has two remotes with identical paths, but corresponding to different repositories:
+
+    rain    /home/grawity/Attic/Software
+    frost   /home/grawity/Attic/Software
+
+[I was told earlier](/forum/basic_usage_questions) that this configuration would work fine and git-annex would not get confused. However, that doesn't seem to be the case (unless I misunderstood what it considers to be the "right thing"?) I see that whenever I run a command like `git annex info` on the portable\_HD repo, it overrides `remote.{rain,frost}.annex-uuid` with whatever UUID it sees right now – resulting in `git annex info` output such as:
+
+    ┌ frost /run/media/grawity/vol4_grimoire/Attic/Software master
+    ┘ git config -l | grep annex-uuid
+    remote.origin.annex-uuid=0ebc2083-f95e-4637-bd3e-09db8471daf3
+    remote.rain.annex-uuid=3e342a37-6c35-40e5-99a1-1f140e6c363d    <--
+    remote.frost.annex-uuid=524b8690-9b3e-48d4-b1a8-c0edb35c1ccf   <--
+    remote.fs1.annex-uuid=ed83c81c-1a95-4acb-b76c-b6724fe88873
+    
+    ┌ frost /run/media/grawity/vol4_grimoire/Attic/Software master
+    ┘ git annex info --fast --verbose --debug
+    repository mode: indirect
+    trusted repositories: [2016-02-29 08:09:29.284136] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"]
+    [2016-02-29 08:09:29.286872] process done ExitSuccess
+    [2016-02-29 08:09:29.286954] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"]
+    [2016-02-29 08:09:29.289167] process done ExitSuccess
+    [2016-02-29 08:09:29.289316] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..fc873fc472a8c5d0db3c91a3868d9adc072f7076","-n1","--pretty=%H"]
+    [2016-02-29 08:09:29.291836] process done ExitSuccess
+    [2016-02-29 08:09:29.291941] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..23c27db2f2a02515ba39e7d0bb8653fa786b6aea","-n1","--pretty=%H"]
+    [2016-02-29 08:09:29.294762] process done ExitSuccess
+    [2016-02-29 08:09:29.294869] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..690a90da7fc28be67d5053c0a6c3050cca614eaa","-n1","--pretty=%H"]
+    [2016-02-29 08:09:29.301018] process done ExitSuccess
+    [2016-02-29 08:09:29.30112] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..94375fb892d1c9f5b70bdd5917c85c5c18a13168","-n1","--pretty=%H"]
+    [2016-02-29 08:09:29.3031] process done ExitSuccess
+    [2016-02-29 08:09:29.303182] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..4a7f17e644e86776b510606beb6f3cb3819d8256","-n1","--pretty=%H"]
+    [2016-02-29 08:09:29.304916] process done ExitSuccess
+    [2016-02-29 08:09:29.305455] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"]
+    [2016-02-29 08:09:29.308946] read: git ["config","--null","--list"]
+    [2016-02-29 08:09:29.312347] process done ExitSuccess
+    [2016-02-29 08:09:29.312939] read: git ["config","--null","--list"]
+    [2016-02-29 08:09:29.3179] process done ExitSuccess
+    [2016-02-29 08:09:29.318494] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","config","remote.rain.annex-uuid","524b8690-9b3e-48d4-b1a8-c0edb35c1ccf"]
+    [2016-02-29 08:09:29.320648] process done ExitSuccess
+    [2016-02-29 08:09:29.320735] read: git ["config","--null","--list"]
+    [2016-02-29 08:09:29.327697] process done ExitSuccess
+    0
+    semitrusted repositories: 9
+    	00000000-0000-0000-0000-000000000001 -- web
+     	00000000-0000-0000-0000-000000000002 -- bittorrent
+     	0ebc2083-f95e-4637-bd3e-09db8471daf3 -- origin
+     	3e14eb23-1b16-4b52-8798-56efb550ab00 -- [vol4_grimoire]:/Attic/Software [here]
+     	3e342a37-6c35-40e5-99a1-1f140e6c363d -- grawity@rain:~/Attic/Software              <--
+     	524b8690-9b3e-48d4-b1a8-c0edb35c1ccf -- grawity@frost:~/Downloads/Software [rain]  <--
+     	b9c0c485-07e5-4166-b5ac-ba971faab98a -- [vol3_tombstone]:/Attic/Software
+     	c3c6dd39-ebc7-475a-9992-f99ca01a7f3a -- grawity@wolke:~/Attic/Software
+     	ed83c81c-1a95-4acb-b76c-b6724fe88873 -- fs1:/Attic/Software [fs1]
+    untrusted repositories: 0
+    transfers in progress: none
+    available local disk space: 920.96 gigabytes (+1 megabyte reserved)
+    
+    ┌ frost /run/media/grawity/vol4_grimoire/Attic/Software master
+    ┘ git config -l | grep annex-uuid
+    remote.origin.annex-uuid=0ebc2083-f95e-4637-bd3e-09db8471daf3
+    remote.rain.annex-uuid=524b8690-9b3e-48d4-b1a8-c0edb35c1ccf    <--
+    remote.frost.annex-uuid=524b8690-9b3e-48d4-b1a8-c0edb35c1ccf   <--
+    remote.fs1.annex-uuid=ed83c81c-1a95-4acb-b76c-b6724fe88873
+
+Notice how the `[rain]` remote tag now shows up next to the wrong remote. Among other things, **this means I cannot use `git annex find --in frost --not --in rain`** and similar commands (unless I specify all the UUIDs by hand). And even though I trust git-annex to not lose any data, this behavior is a bit confusing.
+
+Is it possible that git-annex could detect such situations and avoid updating an UUID if it's _already attached_ to another remote?

doc/forum/Multiple_remotes_with_the_same_path/comment_1_78f2d8e3906c11ba5753478d08973bd2._comment

@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-02-29T15:47:39Z"
+ content="""
+git-annex is not getting confused; it's looking at the current content of
+the remote and using that uuid. Failing to do so would constitute
+confusion.
+
+If you want the remote to only work when the drive is plugged into a single
+host, you need to make the remote path only reach a repository on that
+host. One way would be a symlink, eg `/home/grawity/Attic/rain-Software ->
+Software` on rain.
+
+Or, just use the repository description, eg
+`--in grawity@rain:~/Attic/Software`. Note that the description can be
+changed to something easier to type.
+"""]]

doc/git-annex-matching-options.mdwn

@@ -115,6 +115,15 @@ file contents are present at either of two repositories.
   matches the glob. The values of metadata fields are matched case
   insensitively.
 
+* `--metadata field<number` / `--metadata field>number`
+* `--metadata field<=number` / `--metadata field>=number`
+
+  Matches only files that have a metadata field attached with a value that
+  is a number and is less than or greater than the specified number.
+
+  (Note that you will need to quote the second parameter to avoid
+  the shell doing redirection.)
+
 * `--want-get`
 
   Matches files that the preferred content settings for the repository

doc/git-annex-metadata.mdwn

@@ -19,34 +19,39 @@ When run without any -s or -t parameters, displays the current metadata.
 
 # OPTIONS
 
-* `-g field`
+* `-g field` / `--get field`
 
   Get the value(s) of a single field.
 
   The values will be output one per line, with no other output, so
   this is suitable for use in a script.
 
-* `-s field=value`
+* `-s field=value` / `--set field=value`
 
   Set a field's value, removing any old values.
 
-* `-s field+=value`
+* `-s field+=value` / `--set field+=value`
 
   Add an additional value, preserving any old values.
 
-* `-s field-=value`
-
-  Remove a value.
-
-* `-s field?=value`
+* `-s field?=value` / `--set field?=value`
 
   Set a value, but only if the field does not already have a value set.
-  
-* `-t tag`
+
+* `-s field-=value` / `--set field-=value`
+
+  Remove a value from a field, leaving any other values that the field has
+  set.
+
+* `-r field` / `--remove field`
+
+  Remove all current values of the field.
+
+* `-t tag` / `--tag tag`
 
   Set a tag. Note that a tag is just a value of the "tag" field.
 
-* `-u tag`
+* `-u tag` / `--unset tag`
 
   Unset a tag.
 

doc/git-annex-preferred-content.mdwn

@@ -119,6 +119,15 @@ elsewhere to allow removing it).
 
   To match author metadata, use `metadata=author=*Smith`
 
+* `metadata=field<number` / `metadata=field>number` 
+* `metadata=field<=number` / `metadata=field>=number`
+
+  Matches only files that have a metadata field attached with a value that
+  is a number and is less than or greater than the specified number.
+
+  To match PDFs with between 100 and 200 pages (assuming something has set
+  that metadata), use `metadata=pagecount>=100 and metadata=pagecount<=200`
+
 * `present`
 
   Makes content be wanted if it's present, but not otherwise.

doc/news/version_6.20160229.mdwn

@@ -0,0 +1,24 @@
+git-annex 6.20160229 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Update perlmagick build dependency. Closes: #[789225](http://bugs.debian.org/789225)
+   * Fix memory leak in last release, which affected commands like
+     git-annex status when a large non-annexed file is present in the work
+     tree.
+   * fsck: When the only copy of a file is in a dead repository, mention
+     the repository.
+   * info: Mention when run in a dead repository.
+   * Linux and OSX standalone builds put the bundled gpg last in PATH,
+     so any system gpg will be preferred over it.
+   * Avoid crashing when built with MagicMime support, but when the magic
+     database cannot be loaded.
+   * Include magic database in the linux and OSX standalone builds.
+   * Fix memory leak when hashing files, which triggered during fsck
+     when an external hash program was not used.
+     (This leak was introduced in version 6.20160114.)
+   * Support --metadata field<number, --metadata field>number etc
+     to match ranges of numeric values.
+   * Similarly, support preferred content expressions like
+     metadata=field<number and metadata=field>number
+   * The pre-commit-annex hook script that automatically extracts
+     metadata has been updated to also use exiftool.
+     Thanks, Klaus Ethgen."""]]

doc/preferred_content.mdwn

@@ -67,3 +67,4 @@ they were added in.
 * "metadata=" 5.20140221
 * "lackingcopies=", "approxlackingcopies=", "unused=" 5.20140127
 * "inpreferreddir=" 4.20130501
+* "metadata=field<number" etc 6.20160227

doc/tips/automatically_adding_metadata.mdwn

@@ -2,23 +2,48 @@ git-annex's [[metadata]] works best when files have a lot of useful
 metadata attached to them.
 
 To make git-annex automatically set the year and month when adding files,
-run `git config annex.genmetadata true`.
+run: `git config annex.genmetadata true`
+
+## git commit hook
 
 A git commit hook can be set up to extract lots of metadata from files
-like photos, mp3s, etc.
+like photos, mp3s, etc. Whenever annexed files are committed, their
+metadata will be extracted and stored.
 
-1. Install the `extract` utility, from <http://www.gnu.org/software/libextractor/>  
-   `apt-get install extract`
-2. Download [[pre-commit-annex]] and install it in your git-annex repository
-   as `.git/hooks/pre-commit-annex`.  
-   Remember to make the script executable!
-3. Run: `git config metadata.extract "artist album title camera_make video_dimensions"`
+Download [[pre-commit-annex]] and install it in your git-annex repository
+as `.git/hooks/pre-commit-annex`  
+Remember to make the script executable! `chmod +x .git/hooks/pre-commit-annex`
 
-Now any fields you list in metadata.extract to will be extracted and
-stored when files are committed.
+### using extract
+
+The git commit hook can use extract to get metadata.
+
+Install it from <http://www.gnu.org/software/libextractor/>  
+`apt-get install extract`
+
+Configure which metadata fields to ask extract for: `git config metadata.extract "artist album title camera_make video_dimensions"`
 
 To get a list of all possible fields, run: `extract -L | sed 's/ /_/g'`
 
+### using exiftool
+
+The git commit hook can also use exiftool to get metadata.
+
+Install it from <http://owl.phy.queensu.ca/~phil/exiftool/>  
+`apt-get install libimage-exiftool-perl`
+
+Configure which metadata fields to ask exiftool for: `git config metadata.exiftool "Model ImageSize FocusRange GPSAltitude GPSCoordinates"`
+
+To get a list of all possible fields, run: `exiftool -list`
+
+### using both extract and exiftool
+
+If you want some metadata that extract knows about, and other metadata
+that exiftool knows about, just install them both, and set both
+`metadata.extract` and `metadata.exiftool`.
+
+### overwriting existing metadata
+
 By default, if a git-annex already has a metadata field for a file,
 its value will not be overwritten with metadata taken from files.
 To allow overwriting, run: `git config metadata.overwrite true`

doc/tips/automatically_adding_metadata/pre-commit-annex

@@ -1,4 +1,20 @@
-#!/bin/sh
+#! /bin/sh
+#
+# Copyright (C) 2014 Joey Hess <id@joeyh.name>
+# Copyright (C) 2016 Klaus Ethgen <Klaus@Ethgen.ch>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This script can be used to add git-annex metadata to files when they're
 # committed. It is typically installed as .git/hooks/pre-commit-annex
@@ -6,67 +22,97 @@
 # You can also run this script by hand, passing it the names of files
 # already checked into git-annex, and it will extract/refresh the git-annex
 # metadata from the files.
-#
-# Copyright 2014 Joey Hess <id@joeyh.name>
-# License: GPL-3+
 
-extract="$(git config metadata.extract || true)" 
-want="$(perl -e 'print (join("|", map {s/_/ /g; "^$_ - "} (split " ", shift())))' "$extract")"
-
-if [ -z "$want" ]; then
-	exit 0
+tool="$(git config metadata.tool || :)"
+if [ -z "$tool" ]; then
+	tool=extract
 fi
-
-case "$(git config --bool metadata.overwrite || true)" in
-	true)
-		overwrite=1
+case "$tool" in
+	exiftool)
+		tool_exec="exiftool -unknown -zip -veryShort -ignoreMinorErrors -use MWG -dateFormat '%Y-%m-%dT%H:%M:%S'"
 	;;
 	*)
-		overwrite=""
+		tool_exec="$tool"
 	;;
 esac
 
-addmeta () {
+extract_fields="$(git config metadata.extract || :)"
+if [ -n "$extract_fields" ]; then
+	tools=extract
+	extract_want="^($(echo "$extract_fields" | sed -e 's/ /|/g' -e 's/_/ /g'))"
+fi
+exiftool_fields="$(git config metadata.exiftool || :)"
+if [ -n "$exiftool_fields" ]; then
+	tools="exiftool $tools"
+	exiftool_want="^($(echo "$exiftool_fields" | sed -e 's/ /|/g' -e 's/_/ /g'))"
+fi
+if [ -z "$tools" ]; then
+	exit 0
+fi
+
+case "$(git config --bool metadata.overwrite || :)" in
+	true)
+		equal="="
+	;;
+	*)
+		equal="?="
+	;;
+esac
+
+if git rev-parse --verify HEAD >/dev/null 2>&1; then
+	against="HEAD"
+else
+	# Initial commit: diff against an empty tree object
+	against="4b825dc642cb6eb9a060e54bf8d69288fbee4904"
+fi
+
+addmeta() {
 	file="$1"
 	field="$2"
 	value="$3"
-	afield="$(echo "$field" | tr ' ' _)"
-	if [ "$overwrite" ]; then
-		p="$afield=$value"
-
-	else
-		p="$afield?=$value"
-	fi
-	git -c annex.alwayscommit=false annex metadata "$file" -s "$p" --quiet
+	afield="$(echo "$field" | tr ' ' '_')"
+	git -c annex.alwayscommit=false annex metadata \
+		--set "$afield$equal$value" --quiet -- "$file"
 }
 
-if git rev-parse --verify HEAD >/dev/null 2>&1; then
-	against=HEAD
-else
-	# Initial commit: diff against an empty tree object
-	against=4b825dc642cb6eb9a060e54bf8d69288fbee4904
-fi
-
-IFS="
-"
-
-process () {
+process() {
 	if [ -e "$f" ]; then
 		echo "adding metadata for $f"
-		for l in $(extract "$f" | egrep "$want"); do
-			field="${l%% - *}"
-			value="${l#* - }"
-			addmeta "$f" "$field" "$value"
+		for tool in $tools; do
+			case "$tool" in
+				exiftool)
+					tool_exec="exiftool -unknown -zip -veryShort -ignoreMinorErrors -use MWG -dateFormat '%Y-%m-%dT%H:%M:%S'"
+				;;
+				*)
+					tool_exec="$tool"
+				;;
+			esac
+			LC_ALL=C $tool_exec "./$f" | eval egrep --text -i \""\$${tool}_want"\" | while read line; do
+				case "$tool" in
+					extract)
+						field="${line%% - *}"
+						value="${line#* - }"
+					;;
+					exiftool)
+						field="${line%%: *}"
+						value="${line#*: }"
+					;;
+				esac
+				
+				if [ -n "$value" ]; then
+					addmeta "$f" "$field" "$value"
+				fi
+			done
 		done
 	fi
 }
 
 if [ -n "$*" ]; then
-	for f in $@; do
+	for f in "$@"; do
 		process "$f"
 	done
 else
-	for f in $(git diff-index --name-only --cached $against); do
+	for f in "$(git diff-index --name-only --cached $against)"; do
 		process "$f"
 	done
 fi

doc/tips/centralized_git_repository_tutorial/on_your_own_server/comment_1_ae5af47faf95f008f1b07dbed5181286._comment

@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="tom_clune"
+ subject="git annex using the "wrong" ssh socket"
+ date="2016-02-24T19:23:13Z"
+ content="""
+To avoid frequent typing of pin + RSA passcode + password,  we typically establish an ssh control master just once.   This works fine with regular git commands, but the git-annex command apparently try to create a different socket.  Even that would be ok, except that apparently it is a new socket each time we enter a command.
+
+With sufficient \"-vvvv\" we see things like:  
+
+    ...
+    debug1: Executing proxy command: exec ssh -l fred proxy.xxx.yyy direct host
+    ...
+
+(Note I have eliminated references to the actual machine names and userid's.)
+
+If the command had instead been:
+
+     exec ssh -l fred proxy.xxx.yyy direct /home/fred/.ssh/master_host:22
+
+everything would have worked fine.  In fact, we are now using:
+
+     git config remote.origin.annex-ssh-options '-S /home/fred/.ssh/master_host:22'
+
+and this eliminates the issue.    But it would be nice if git annex could somehow automatically use
+the pre-existing connection.     Is there a better way to achieve this?
+
+
+"""]]

doc/todo/standalone__47__android__47__buildchroot-inchroot_should_install_libmagic-dev/comment_3_501a89c819d0e1531a8bc6548c837a77._comment

@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="divergentdave@5c17d06f6d67c6f157b76a4cc95ca764b7d2f899"
+ nickname="divergentdave"
+ subject="comment 3"
+ date="2016-02-26T03:56:41Z"
+ content="""
+It looks like the flag in that change was incorrect. I had to change it to `--flags=\"-magicmime -concurrentoutput\"`. (no hyphen between current and output) After that, I was able to reinstall the libraries.
+
+I'd be happy to lend a hand, expect more patches!
+"""]]

doc/walkthrough/unused_data/comment_4_2110ed6316b6a0df4ef3e1c8bc97ab99._comment

@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="tom_clune"
+ subject="dropping files after changing branches/tags"
+ date="2016-02-24T22:19:00Z"
+ content="""
+I have a use case in which a colleague wishes to have a working copy of my data repository to use with the current version of my model.    When a new version of the model is available they would likewise update their git-annex clone of my data.    The colleague wants to drop any files that have been made obsolete by this change, but I do not see an efficient way to make this determination.   They could of course drop everything and then do git annex get .   but that could be very expensive if only a small subset of the files have actually changed.
+
+I'm probably just missing something basic, as this seems to be a reasonably frequent use case.
+
+
+"""]]

git-annex.cabal

@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 6.20160217
+Version: 6.20160229
 Cabal-Version: >= 1.8
 License: GPL-3
 Maintainer: Joey Hess <id@joeyh.name>

standalone/osx/git-annex.app/Contents/MacOS/runshell

@@ -77,7 +77,11 @@ export ORIG_GIT_TEMPLATE_DIR
 GIT_TEMPLATE_DIR="$bundle/templates"
 export GIT_TEMPLATE_DIR
 
-# Indicate which variables were exported above.
+GIT_ANNEX_DIR="$bundle"
+export GIT_ANNEX_DIR
+
+# Indicate which variables were exported above and should be cleaned
+# when running non-bundled programs.
 GIT_ANNEX_STANDLONE_ENV="PATH GIT_EXEC_PATH GIT_TEMPLATE_DIR"
 export GIT_ANNEX_STANDLONE_ENV