Merge branch 'master' into tasty-tests

This commit is contained in:
Joey Hess 2013-11-15 15:42:58 -04:00
commit 27179ad6fc
26 changed files with 265 additions and 54 deletions

View file

@ -29,7 +29,6 @@ module Annex.Content (
preseedTmp,
freezeContent,
thawContent,
cleanObjectLoc,
dirKeys,
) where
@ -255,11 +254,9 @@ moveAnnex key src = withObjectLoc key storeobject storedirect
where
storeobject dest = ifM (liftIO $ doesFileExist dest)
( alreadyhave
, do
createContentDir dest
, modifyContent dest $ do
liftIO $ moveFile src dest
freezeContent dest
freezeContentDir dest
)
storeindirect = storeobject =<< calcRepo (gitAnnexLocation key)
@ -273,7 +270,6 @@ moveAnnex key src = withObjectLoc key storeobject storedirect
storedirect = storedirect' storeindirect
storedirect' fallback [] = fallback
storedirect' fallback (f:fs) = do
thawContentDir =<< calcRepo (gitAnnexLocation key)
thawContent src
v <- isAnnexLink f
if Just key == v
@ -349,11 +345,11 @@ withObjectLoc key indirect direct = ifM isDirect
where
goindirect = indirect =<< calcRepo (gitAnnexLocation key)
cleanObjectLoc :: Key -> Annex ()
cleanObjectLoc key = do
cleanObjectLoc :: Key -> Annex () -> Annex ()
cleanObjectLoc key cleaner = do
file <- calcRepo $ gitAnnexLocation key
unlessM crippledFileSystem $
void $ liftIO $ catchMaybeIO $ allowWrite $ parentDir file
void $ tryAnnexIO $ thawContentDir file
cleaner
liftIO $ removeparents file (3 :: Int)
where
removeparents _ 0 = noop
@ -369,13 +365,10 @@ cleanObjectLoc key = do
removeAnnex :: Key -> Annex ()
removeAnnex key = withObjectLoc key remove removedirect
where
remove file = do
thawContentDir file
remove file = cleanObjectLoc key $ do
liftIO $ nukeFile file
removeInodeCache key
cleanObjectLoc key
removedirect fs = do
thawContentDir =<< calcRepo (gitAnnexLocation key)
cache <- recordedInodeCache key
removeInodeCache key
mapM_ (resetfile cache) fs
@ -389,12 +382,10 @@ removeAnnex key = withObjectLoc key remove removedirect
{- Moves a key's file out of .git/annex/objects/ -}
fromAnnex :: Key -> FilePath -> Annex ()
fromAnnex key dest = do
fromAnnex key dest = cleanObjectLoc key $ do
file <- calcRepo $ gitAnnexLocation key
thawContentDir file
thawContent file
liftIO $ moveFile file dest
cleanObjectLoc key
{- Moves a key out of .git/annex/objects/ into .git/annex/bad, and
- returns the file it was moved to. -}
@ -404,9 +395,8 @@ moveBad key = do
bad <- fromRepo gitAnnexBadDir
let dest = bad </> takeFileName src
createAnnexDirectory (parentDir dest)
thawContentDir src
liftIO $ moveFile src dest
cleanObjectLoc key
cleanObjectLoc key $
liftIO $ moveFile src dest
logStatus key InfoMissing
return dest

View file

@ -10,6 +10,7 @@ module Annex.Content.Direct (
associatedFilesRelative,
removeAssociatedFile,
removeAssociatedFileUnchecked,
removeAssociatedFiles,
addAssociatedFile,
goodContent,
recordedInodeCache,
@ -64,8 +65,8 @@ changeAssociatedFiles key transform = do
files <- associatedFilesRelative key
let files' = transform files
when (files /= files') $ do
createContentDir mapping
liftIO $ viaTmp write mapping $ unlines files'
modifyContent mapping $
liftIO $ viaTmp write mapping $ unlines files'
top <- fromRepo Git.repoPath
return $ map (top </>) files'
where
@ -75,6 +76,13 @@ changeAssociatedFiles key transform = do
hPutStr h content
hClose h
{- Removes the list of associated files. -}
removeAssociatedFiles :: Key -> Annex ()
removeAssociatedFiles key = do
mapping <- calcRepo $ gitAnnexMapping key
modifyContent mapping $
liftIO $ nukeFile mapping
{- Removes an associated file. Returns new associatedFiles value.
- Checks if this was the last copy of the object, and updates location
- log. -}
@ -142,16 +150,16 @@ addInodeCache key cache = do
{- Writes inode cache for a key. -}
writeInodeCache :: Key -> [InodeCache] -> Annex ()
writeInodeCache key caches = withInodeCacheFile key $ \f -> do
createContentDir f
liftIO $ writeFile f $
unlines $ map showInodeCache caches
writeInodeCache key caches = withInodeCacheFile key $ \f ->
modifyContent f $
liftIO $ writeFile f $
unlines $ map showInodeCache caches
{- Removes an inode cache. -}
removeInodeCache :: Key -> Annex ()
removeInodeCache key = withInodeCacheFile key $ \f -> do
createContentDir f -- also thaws directory
liftIO $ nukeFile f
removeInodeCache key = withInodeCacheFile key $ \f ->
modifyContent f $
liftIO $ nukeFile f
withInodeCacheFile :: Key -> (FilePath -> Annex a) -> Annex a
withInodeCacheFile key a = a =<< calcRepo (gitAnnexInodeCache key)

View file

@ -138,28 +138,33 @@ mergeDirect d branch g = do
- and the commit sha passed in, along with the old sha of the tree
- before the merge. Uses git diff-tree to find files that changed between
- the two shas, and applies those changes to the work tree.
-
- There are really only two types of changes: An old item can be deleted,
- or a new item added. Two passes are made, first deleting and then
- adding. This is to handle cases where eg, a file is deleted and a
- directory is added. The diff-tree output may list these in the opposite
- order, but we cannot really add the directory until the file with the
- same name is remvoed.
-}
mergeDirectCleanup :: FilePath -> Git.Ref -> Git.Ref -> Annex ()
mergeDirectCleanup d oldsha newsha = do
(items, cleanup) <- inRepo $ DiffTree.diffTreeRecursive oldsha newsha
makeabs <- flip fromTopFilePath <$> gitRepo
forM_ items (updated makeabs)
let fsitems = zip (map (makeabs . DiffTree.file) items) items
forM_ fsitems $
go DiffTree.srcsha DiffTree.srcmode moveout moveout_raw
forM_ fsitems $
go DiffTree.dstsha DiffTree.dstmode movein movein_raw
void $ liftIO cleanup
liftIO $ removeDirectoryRecursive d
where
updated makeabs item = do
let f = makeabs (DiffTree.file item)
void $ tryAnnex $
go f DiffTree.srcsha DiffTree.srcmode moveout moveout_raw
void $ tryAnnex $
go f DiffTree.dstsha DiffTree.dstmode movein movein_raw
where
go f getsha getmode a araw
| getsha item == nullSha = noop
| otherwise = maybe (araw f) (\k -> void $ a k f)
=<< catKey (getsha item) (getmode item)
go getsha getmode a araw (f, item)
| getsha item == nullSha = noop
| otherwise = void $
tryAnnex . maybe (araw f) (\k -> void $ a k f)
=<< catKey (getsha item) (getmode item)
moveout = removeDirect
moveout k f = removeDirect k f
{- Files deleted by the merge are removed from the work tree.
- Empty work tree directories are removed, per git behavior. -}
@ -205,11 +210,11 @@ toDirectGen k f = do
where
fromindirect loc = do
{- Move content from annex to direct file. -}
thawContentDir loc
updateInodeCache k loc
void $ addAssociatedFile k f
thawContent loc
replaceFile f $ liftIO . moveFile loc
modifyContent loc $ do
thawContent loc
replaceFile f $ liftIO . moveFile loc
fromdirect loc = do
replaceFile f $
liftIO . void . copyFileExternal loc

View file

@ -13,12 +13,14 @@ module Annex.Perms (
createContentDir,
freezeContentDir,
thawContentDir,
modifyContent,
) where
import Common.Annex
import Utility.FileMode
import Git.SharedRepository
import qualified Annex
import Annex.Exception
import Config
import System.Posix.Types
@ -103,3 +105,13 @@ createContentDir dest = do
liftIO $ allowWrite dir
where
dir = parentDir dest
{- Creates the content directory for a file if it doesn't already exist,
- or thaws it if it does, then runs an action to modify the file, and
- finally, freezes the content directory. -}
modifyContent :: FilePath -> Annex a -> Annex a
modifyContent f a = do
createContentDir f -- also thaws it
v <- tryAnnex a
freezeContentDir f
either throwAnnex return v

View file

@ -218,9 +218,10 @@ verifyLocationLog key desc = do
{- Since we're checking that a key's file is present, throw
- in a permission fixup here too. -}
when (present && not direct) $ do
file <- calcRepo $ gitAnnexLocation key
file <- calcRepo $ gitAnnexLocation key
when (present && not direct) $
freezeContent file
whenM (liftIO $ doesDirectoryExist $ parentDir file) $
freezeContentDir file
{- In direct mode, modified files will show up as not present,

View file

@ -20,9 +20,9 @@ import Config
import qualified Annex
import Annex.Direct
import Annex.Content
import Annex.Content.Direct
import Annex.CatFile
import Annex.Version
import Annex.Perms
import Annex.Exception
import Init
import qualified Command.Add
@ -77,7 +77,8 @@ perform = do
Just s
| isSymbolicLink s -> void $ flip whenAnnexed f $
\_ (k, _) -> do
cleandirect k
removeInodeCache k
removeAssociatedFiles k
return Nothing
| otherwise ->
maybe noop (fromdirect f)
@ -87,8 +88,8 @@ perform = do
fromdirect f k = do
showStart "indirect" f
thawContentDir =<< calcRepo (gitAnnexLocation k)
cleandirect k -- clean before content directory gets frozen
removeInodeCache k
removeAssociatedFiles k
whenM (liftIO $ not . isSymbolicLink <$> getSymbolicLinkStatus f) $ do
v <-tryAnnexIO (moveAnnex k f)
case v of
@ -103,10 +104,6 @@ perform = do
warnlocked e = do
warning $ show e
warning "leaving this file as-is; correct this problem and run git annex add on it"
cleandirect k = do
liftIO . nukeFile =<< calcRepo (gitAnnexInodeCache k)
liftIO . nukeFile =<< calcRepo (gitAnnexMapping k)
cleanup :: CommandCleanup
cleanup = do

9
debian/changelog vendored
View file

@ -27,6 +27,15 @@ git-annex (5.20131102) UNRELEASED; urgency=low
* repair: Handle case where index file is corrupt, but all objects are ok.
* assistant: Notice on startup when the index file is corrupt, and
auto-repair.
* Fix direct mode merge bug when a direct mode file was deleted and replaced
with a directory. An ordering problem caused the directory to not get
created in this case.
Thanks to Tim for the test case.
* Direct mode .git/annex/objects directories are no longer left writable,
because that allowed writing to symlinks of files that are not present,
which followed the link and put bad content in an object location.
Thanks to Tim for the test case.
* fsck: Fix up .git/annex/object directory permissions.
-- Joey Hess <joeyh@debian.org> Wed, 06 Nov 2013 16:14:14 -0400

View file

@ -0,0 +1,8 @@
[[!comment format=mdwn
username="https://www.google.com/accounts/o8/id?id=AItOawmkXtBdMgE1d9nCz2iBc4f85xh4izZ_auU"
nickname="Ulrich"
subject="Using a portable drive as another transfer device? cool."
date="2013-11-14T19:05:56Z"
content="""
Thanks - I was hoping that it is that easy. I'll try that as soon as I have a working version of the latest git-annex (trying to build with brew for Mac OS X 10.9, but without success so far).
"""]]

View file

@ -40,3 +40,5 @@ remote types: git gcrypt S3 bup directory rsync web webdav glacier hook
Linux ceilingcat 3.11.6-1-ARCH #1 SMP PREEMPT Fri Oct 18 23:22:36 CEST 2013 x86_64 GNU/Linux
"""]]
> [[fixed|done]] --[[Joey]]

View file

@ -0,0 +1,12 @@
[[!comment format=mdwn
username="http://joeyh.name/"
ip="209.250.56.246"
subject="comment 2"
date="2013-11-15T17:39:37Z"
content="""
My initial guess was wrong.. This is not actually a bug in conflicted merge resolution at all.
The bug is that in direct mode, it diffs the old and new tree when doing a normal merge, so see what files in the work tree need to be changed. This was written to go through the diff and replay the deletes and adds. In this case, since f/f and f are different items, they can appear in either order in the diff But the code only worked when f was first deleted, and f/f was then added. And it turns out that in this case, the diff had the two items the other way around.
So, I think it needs to do 2 passes, first deleting and then adding.
"""]]

View file

@ -43,3 +43,6 @@ remote types: git gcrypt S3 bup directory rsync web webdav glacier hook
Linux ceilingcat 3.11.6-1-ARCH #1 SMP PREEMPT Fri Oct 18 23:22:36 CEST 2013 x86_64 GNU/Linux
"""]]
> [[fixed|done]]; direct mode now freezes the content directory as indirect
> mode already did. fsck will fix up the permissions too. --[[Joey]]

View file

@ -0,0 +1,15 @@
[[!comment format=mdwn
username="http://joeyh.name/"
ip="209.250.56.246"
subject="comment 3"
date="2013-11-15T17:48:42Z"
content="""
Hmm. In your script, when you run `git annex sync` in a and then `echo aaaa > f`, f already exists at that point as a symlink. This actually ends up following the link and writing to .git/annex/objects. (fsck will detect that junk has been written there)
That's a bug on its own; seems like direct mode is neglecting to lock down the .git/annex/objects directories to prevent writing to them like this.
----
However, this means that your script does not demonstrate the bug you originally reported.
You remove b/f and sync, and since a/f has not been changed, the deleting is correctly synced to a, removing a/f.
"""]]

View file

@ -0,0 +1,14 @@
[[!comment format=mdwn
username="http://joeyh.name/"
ip="209.250.56.246"
subject="comment 4"
date="2013-11-15T17:52:38Z"
content="""
Looking back at the original bug description:
\"repo A a symlink to a file whose contents aren't yet available, are overwritten, while at repo B the file is deleted.\"
I think the \"overwritten\" is key. I suspect you were always doing echo > f where f was a symlink, and this does not actually overwrite the symlink, it just puts data (that fsck will reject) into the annex.
So, proceeding as if the real bug here is the ability to write through symlink in direct mode, unless told otherwise..
"""]]

View file

@ -216,3 +216,5 @@ Binary Images:
# End of transcript or log.
"""]]
> [[fixed|done]] --[[Joey]]

View file

@ -0,0 +1,10 @@
[[!comment format=mdwn
username="http://joeyh.name/"
ip="209.250.56.246"
subject="comment 22"
date="2013-11-14T18:55:54Z"
content="""
Native 10.9 autobuilder is now running. <https://downloads.kitenet.net/git-annex/autobuild/x86_64-apple-mavericks/git-annex.dmg.bz2>
Lacks XMPP support until I get libxml2 linked on the autobuilder. Only tested on the build machine so far.
"""]]

View file

@ -0,0 +1,8 @@
[[!comment format=mdwn
username="https://www.google.com/accounts/o8/id?id=AItOawnmF_9CAtfqdZkC4e-_dCX-rK5bqh4RWkw"
nickname="Carl"
subject="comment 23"
date="2013-11-14T19:22:16Z"
content="""
Seems to work on my Mavericks Macbook Air. Thank you!
"""]]

View file

@ -0,0 +1,52 @@
### Please describe the problem.
When I use the web app and try to create a remote on a remote server (via ssh connection) the assistant shows a gpg error.
### What steps will reproduce the problem?
1. Start the the web app using git-annex web app
2. create a local repository
3. create a new repository on a Remote server (Set up a repository on a remote server using ssh).
4. provide correct server address, user, port, etc.
Then gpg fails.
### What version of git-annex are you using? On what operating system?
git-annex version: 4.20131106
build flags: Assistant Webapp Pairing Testsuite S3 WebDAV FsEvents XMPP DNS Feeds Quvi TDFA CryptoHash
key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL
remote types: git gcrypt S3 bup directory rsync web webdav glacier hook
local repository version: unknown
default repository version: 3
supported repository versions: 3 4
upgrade supported from repository versions: 0 1 2
On Mac OS X 10.9 Mavericks, build 13A603.
### Please provide any additional information below.
[[!format sh """
# If you can, paste a complete transcript of the problem occurring here.
# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
[2013-11-14 21:03:47 CET] main: starting assistant version 4.20131106
[2013-11-14 21:03:47 CET] Cronner: You should enable consistency checking to protect your data.
(Recording state in git...)
(scanning...) [2013-11-14 21:03:47 CET] Watcher: Performing startup scan
(started...) [2013-11-14 21:04:47 CET] Cronner: Consistency check in progress
[2013-11-14 21:05:21 CET] Committer: Adding sunflower.html test.html cindy.css d3.js d3.min.js Accessors.js Essentials.js List.js Namespace.js and 6 other files
(Recording state in git...)
add /Users/ulli/Documents/annex/test.html (checksum...) ok
### several similar adds removed for privacy reasons.
[2013-11-14 21:05:22 CET] Committer: Committing changes to git
ok
(Recording state in git...)
(Recording state in git...)
14/Nov/2013:21:21:05 +0100 [Error#yesod-core] user error (gpg ["--quiet","--trust-model","always","--with-colons","--list-secret-keys","--fixed-list-mode"] exited 127) @(yesod-core-1.2.5:Yesod.Core.Class.Yesod ./Yesod/Core/Class/Yesod.hs:485:5)
# End of transcript or log.
"""]]

View file

@ -0,0 +1,8 @@
[[!comment format=mdwn
username="https://www.google.com/accounts/o8/id?id=AItOawmkXtBdMgE1d9nCz2iBc4f85xh4izZ_auU"
nickname="Ulrich"
subject="Easy to fix."
date="2013-11-14T20:36:17Z"
content="""
Well, this only happens when gpg is not available. Everything works fine after a quick \"brew install gpg\".
"""]]

View file

@ -0,0 +1,14 @@
[[!comment format=mdwn
username="http://joeyh.name/"
ip="209.250.56.246"
subject="comment 2"
date="2013-11-14T22:10:32Z"
content="""
gpg is included in the bundle though:
<pre>
oberon:tmp joeyh$ /Volumes/git-annex/git-annex.app/Contents/MacOS/runshell
bash-3.2$ which gpg
/Volumes/git-annex/git-annex.app/Contents/MacOS/bundle/gpg
</pre>
"""]]

View file

@ -7,3 +7,5 @@ As told in http://git-annex.branchable.com/bugs/OSX_app_issues/#comment-2a69d531
Manually editing /Applications/git-annex.app/Contents/MacOS/runshell as told in http://git-annex.branchable.com/bugs/OSX_app_issues/#comment-5579c2150ad4d2ccc207a253fe57612a fixes the issue.
Furthermore, this build is quite outdated...
> [[fixed|done]] --[[Joey]]

View file

@ -0,0 +1,14 @@
The user survey is producing some interesting and useful results!
Added two more polls: [using with](http://git-annex-survey.branchable.com/polls/2013/using_with/) and [blocking problems](http://git-annex-survey.branchable.com/polls/2013/blocking_problems/)
(There were some load issues so if you were unable to vote yesterday, try
again..)
Worked on getting the autobuilder for OS X Mavericks set up. Eventually
succeeded, after patching a few packages to work around a cpp that thinks
it should parse haskell files as if they're C code.
Also, Jimmy has resuscitated the OS X Lion autobuilder.
A not too bad bug in automatic merge conflict resolution has been reported,
so I will need to dig into that tomorrow. Didn't feel up to it today, so
instead have been spending the remaining time finishing up a branch that
switches the test suite to use the tasty test framework.

View file

@ -0,0 +1,11 @@
[[!comment format=mdwn
username="https://www.google.com/accounts/o8/id?id=AItOawnZEanlyzay_QlEAL0CWpyZcRTyN7vay8U"
nickname="Carlo"
subject="comment 6"
date="2013-11-15T13:43:06Z"
content="""
Since git web--browse defaults to firefox on my system (Ubuntu/XMonad), I did this on Ubuntu to get the system default browser.
git config --global web.browser xdg-open
"""]]

View file

@ -0,0 +1 @@
Is there any kind of "lastpresent" in the preferred-content expression? If set, git-annex would see if "git log --follow $path -n 1" (or some configurable -n) was present.

View file

@ -0,0 +1,11 @@
[[!comment format=mdwn
username="https://www.google.com/accounts/o8/id?id=AItOawmkXtBdMgE1d9nCz2iBc4f85xh4izZ_auU"
nickname="Ulrich"
subject="So what does &quot;git annex drop&quot; do in direct mode?"
date="2013-11-15T15:07:57Z"
content="""
I just tried a git annex drop on a file in a direct mode repository, and that just did not change anything at all, as far as I can see. Actually I don't know what to expect, but doesn't that mean that I should have to switch to indirect mode in order to be able to drop large files to save space?
Funny is that git annex drop did not complain at all but just reported \"ok\" after quite a while. And when I tried to drop a file that was not available in any other repository, it failed (which is expected behavior). So what is it what was \"ok\"?
"""]]

View file

@ -0,0 +1 @@
Especially on Mac OSX (and Windows, and maybe Android), it would be great to be able to check in the webapp if an upgrade is available. A deeper integration with these OS would be even better: for example on Mac OSX, an icon on the status bar list available upgrades for some programs, including LibreOffice and others which are not installed by default.

View file

@ -0,0 +1 @@
In the same idea than https://git-annex.branchable.com/todo/Check_if_an_upgrade_is_available_in_the_webapp/, it would be great to be able to download and install git-annex upgrades directly from the webapp.