Merge branch 'master' into desymlink

Conflicts:
	Annex/CatFile.hs
	Annex/Content.hs
	Git/LsFiles.hs
	Git/LsTree.hs
This commit is contained in:
Joey Hess 2012-12-13 00:29:06 -04:00
commit b080a58b76
108 changed files with 2031 additions and 1615 deletions

View file

@ -175,8 +175,7 @@ get' staleok file = fromjournal =<< getJournalFile file
| otherwise = do
update
frombranch
frombranch = withIndex $
L.unpack <$> catFile fullname file
frombranch = withIndex $ L.unpack <$> catFile fullname file
{- Applies a function to modifiy the content of a file.
-

View file

@ -62,11 +62,12 @@ otool appbase libmap = do
files <- filterM doesFileExist =<< dirContentsRecursive appbase
process [] files libmap
where
unprocessed s = not ("@executable_path" `isInfixOf` s)
want s = not ("@executable_path" `isInfixOf` s)
&& not (".framework" `isInfixOf` s)
process c [] m = return (nub $ concat c, m)
process c (file:rest) m = do
_ <- boolSystem "chmod" [Param "755", File file]
libs <- filter unprocessed . parseOtool
libs <- filter want . parseOtool
<$> readProcess "otool" ["-L", file]
m' <- install_name_tool file libs m
process (libs:c) rest m'

View file

@ -101,10 +101,12 @@ localToUrl reference r
| repoIsUrl r = r
| otherwise = r { location = Url $ fromJust $ parseURI absurl }
where
absurl =
Url.scheme reference ++ "//" ++
Url.authority reference ++
repoPath r
absurl = concat
[ Url.scheme reference
, "//"
, Url.authority reference
, repoPath r
]
{- Calculates a list of a repo's configured remotes, by parsing its config. -}
fromRemotes :: Repo -> IO [Repo]
@ -127,8 +129,7 @@ remoteNamed n constructor = do
remoteNamedFromKey :: String -> IO Repo -> IO Repo
remoteNamedFromKey k = remoteNamed basename
where
basename = join "." $ reverse $ drop 1 $
reverse $ drop 1 $ split "." k
basename = join "." $ reverse $ drop 1 $ reverse $ drop 1 $ split "." k
{- Constructs a new Repo for one of a Repo's remotes using a given
- location (ie, an url). -}
@ -153,8 +154,7 @@ fromRemoteLocation s repo = gen $ calcloc s
endswith suffix k &&
startswith v l
filterconfig f = filter f $
concatMap splitconfigs $
M.toList $ fullconfig repo
concatMap splitconfigs $ M.toList $ fullconfig repo
splitconfigs (k, vs) = map (\v -> (k, v)) vs
(prefix, suffix) = ("url." , ".insteadof")
urlstyle v = isURI v || ":" `isInfixOf` v && "//" `isInfixOf` v

View file

@ -69,6 +69,12 @@ stagedDetails l repo = do
where
(metadata, file) = separate (== '\t') s
{- Returns a list of files that have unstaged changes. -}
changedUnstaged :: [FilePath] -> Repo -> IO ([FilePath], IO Bool)
changedUnstaged l = pipeNullSplit params
where
params = Params "diff --name-only -z --" : map File l
{- Returns a list of the files in the specified locations that are staged
- for commit, and whose type has changed. -}
typeChangedStaged :: [FilePath] -> Repo -> IO ([FilePath], IO Bool)

View file

@ -52,8 +52,8 @@ matching :: Ref -> Repo -> IO [(Ref, Branch)]
matching ref repo = map gen . lines <$>
pipeReadStrict [Param "show-ref", Param $ show ref] repo
where
gen l = let (r, b) = separate (== ' ') l in
(Ref r, Ref b)
gen l = let (r, b) = separate (== ' ') l
in (Ref r, Ref b)
{- List of (refs, branches) matching a given ref spec.
- Duplicate refs are filtered out. -}

View file

@ -7,7 +7,7 @@ BASEFLAGS=-Wall -outputdir $(GIT_ANNEX_TMP_BUILD_DIR) -IUtility
#
# If you're using an old version of yesod, enable -DWITH_OLD_YESOD
# Or with an old version of the uri library, enable -DWITH_OLD_URI
FEATURES?=$(GIT_ANNEX_LOCAL_FEATURES) -DWITH_ASSISTANT -DWITH_S3 -DWITH_WEBDAV -DWITH_WEBAPP -DWITH_PAIRING -DWITH_XMPP -DWITH_HOST
FEATURES?=$(GIT_ANNEX_LOCAL_FEATURES) -DWITH_ASSISTANT -DWITH_S3 -DWITH_WEBDAV -DWITH_WEBAPP -DWITH_PAIRING -DWITH_XMPP -DWITH_DNS
bins=git-annex
mans=git-annex.1 git-annex-shell.1
@ -160,8 +160,15 @@ linuxstandalone:
ln -sf git-annex "$(LINUXSTANDALONE_DEST)/bin/git-annex-shell"
zcat standalone/licences.gz > $(LINUXSTANDALONE_DEST)/LICENSE
set -e; \
for bin in $(THIRDPARTY_BINS); do \
cp "$$(which "$$bin")" "$(LINUXSTANDALONE_DEST)/bin/" || echo "failed to install $$bin"; \
p="$$(PATH=$$PATH:/usr/sbin:/sbin:/usr/local/sbin which "$$bin")"; \
if [ -z "$$p" ]; then \
echo "** missing $$bin" >&2; \
exit 1; \
else \
cp "$$p" "$(LINUXSTANDALONE_DEST)/bin/"; \
fi; \
done
install -d "$(LINUXSTANDALONE_DEST)/git-core"
@ -200,7 +207,13 @@ osxapp:
cp $(OSXAPP_BASE)/LICENSE $(GIT_ANNEX_TMP_BUILD_DIR)/build-dmg/LICENSE.txt
for bin in $(THIRDPARTY_BINS); do \
cp "$$(which "$$bin")" "$(OSXAPP_BASE)" || echo "failed to install $$bin"; \
p="$$(PATH=$$PATH:/usr/sbin:/sbin:/usr/local/sbin which "$$bin")"; \
if [ -z "$$p" ]; then \
echo "** missing $$bin" >&2; \
exit 1; \
else \
cp "$$p" "$(OSXAPP_BASE)"; \
fi; \
done
(cd "$(shell git --exec-path)" && tar c .) | (cd "$(OSXAPP_BASE)" && tar x)

View file

@ -36,6 +36,6 @@ descStandardGroup FullArchiveGroup = "full archive: archives all files not archi
preferredContent :: StandardGroup -> String
preferredContent ClientGroup = "exclude=*/archive/* and exclude=archive/*"
preferredContent TransferGroup = "not (inallgroup=client and copies=client:2) and " ++ preferredContent ClientGroup
preferredContent BackupGroup = "" -- all content is preferred
preferredContent BackupGroup = "include=*"
preferredContent SmallArchiveGroup = "(include=*/archive/* or include=archive/*) and " ++ preferredContent FullArchiveGroup
preferredContent FullArchiveGroup = "not (copies=archive:1 or copies=smallarchive:1)"

View file

@ -73,12 +73,12 @@ moveFile src dest = tryIO (rename src dest) >>= onrename
where
rethrow = throw e
mv tmp _ = do
ok <- boolSystem "mv" [Param "-f",
Param src, Param tmp]
ok <- boolSystem "mv" [Param "-f", Param src, Param tmp]
unless ok $ do
-- delete any partial
_ <- tryIO $ removeFile tmp
rethrow
isdir f = do
r <- tryIO $ getFileStatus f
case r of

View file

@ -118,9 +118,7 @@ decode_c s = unescape ("", s)
handle (x:n1:n2:n3:rest)
| isescape x && alloctal = (fromoctal, rest)
where
alloctal = isOctDigit n1 &&
isOctDigit n2 &&
isOctDigit n3
alloctal = isOctDigit n1 && isOctDigit n2 && isOctDigit n3
fromoctal = [chr $ readoctal [n1, n2, n3]]
readoctal o = Prelude.read $ "0o" ++ o :: Int
-- \C is used for a few special characters

View file

@ -117,8 +117,7 @@ userDesktopDir :: IO FilePath
userDesktopDir = maybe fallback return =<< (parse <$> xdg_user_dir)
where
parse = maybe Nothing (headMaybe . lines)
xdg_user_dir = catchMaybeIO $
readProcess "xdg-user-dir" ["DESKTOP"]
xdg_user_dir = catchMaybeIO $ readProcess "xdg-user-dir" ["DESKTOP"]
fallback = xdgEnvHome "DESKTOP_DIR" "Desktop"
xdgEnvHome :: String -> String -> IO String

View file

@ -249,11 +249,9 @@ runHooks kq hooks = do
withstatus change $ dispatchadd dirmap
dispatchadd dirmap change s
| Files.isSymbolicLink s =
callhook addSymlinkHook (Just s) change
| Files.isSymbolicLink s = callhook addSymlinkHook (Just s) change
| Files.isDirectory s = recursiveadd dirmap change
| Files.isRegularFile s =
callhook addHook (Just s) change
| Files.isRegularFile s = callhook addHook (Just s) change
| otherwise = noop
recursiveadd dirmap change = do

View file

@ -62,8 +62,7 @@ parse s = bundle $ go [] $ lines s
| otherwise = parsefail
go _ _ = parsefail
parseprocess l =
case splitnull l of
parseprocess l = case splitnull l of
[pid, 'c':cmdline, ""] ->
case readish pid of
(Just n) -> ProcessInfo n cmdline
@ -71,8 +70,7 @@ parse s = bundle $ go [] $ lines s
_ -> parsefail
parsefiles c [] = (c, [])
parsefiles c (l:ls) =
case splitnull l of
parsefiles c (l:ls) = case splitnull l of
['a':mode, 'n':file, ""] ->
parsefiles ((file, parsemode mode):c) ls
(('p':_):_) -> (c, l:ls)

View file

@ -18,5 +18,4 @@ import Control.Applicative
getHostname :: IO (Maybe String)
getHostname = catchMaybeIO uname_node
where
uname_node = takeWhile (/= '\n') <$>
readProcess "uname" ["-n"]
uname_node = takeWhile (/= '\n') <$> readProcess "uname" ["-n"]

View file

@ -71,9 +71,9 @@ touchBoth file atime mtime follow =
r <- c_utimensat at_fdcwd f ptr flags
when (r /= 0) $ throwErrno "touchBoth"
where
flags = if follow
then 0
else at_symlink_nofollow
flags
| follow = 0
| otherwise = at_symlink_nofollow
#else
#if 0
@ -109,9 +109,9 @@ touchBoth file atime mtime follow =
when (r /= 0) $
throwErrno "touchBoth"
where
syscall = if follow
then c_lutimes
else c_utimes
syscall
| follow = c_lutimes
| otherwise = c_utimes
#else
#warning "utimensat and lutimes not available; building without symlink timestamp preservation support"

4
debian/changelog vendored
View file

@ -1,4 +1,4 @@
git-annex (3.20121128) UNRELEASED; urgency=low
git-annex (3.20121211) unstable; urgency=low
* webapp: Defaults to sharing box.com account info with friends, allowing
one-click enabling of the repository.
@ -31,7 +31,7 @@ git-annex (3.20121128) UNRELEASED; urgency=low
* assistant: Fix syncing to just created ssh remotes.
* Enable WebDAV support in Debian package. Closes: #695532
-- Joey Hess <joeyh@debian.org> Wed, 28 Nov 2012 13:31:07 -0400
-- Joey Hess <joeyh@debian.org> Tue, 11 Dec 2012 11:25:03 -0400
git-annex (3.20121127) unstable; urgency=low

2
debian/rules vendored
View file

@ -2,7 +2,7 @@
ARCH = $(shell dpkg-architecture -qDEB_BUILD_ARCH)
ifeq (install ok installed,$(shell dpkg-query -W -f '$${Status}' libghc-yesod-dev 2>/dev/null))
export FEATURES=-DWITH_ASSISTANT -DWITH_S3 -DWITH_WEBDAV -DWITH_HOST -DWITH_OLD_URI -DWITH_OLD_YESOD -DWITH_WEBAPP -DWITH_PAIRING -DWITH_XMPP
export FEATURES=-DWITH_ASSISTANT -DWITH_S3 -DWITH_WEBDAV -DWITH_HOST -DWITH_OLD_URI -DWITH_PAIRING -DWITH_XMPP -DWITH_WEBAPP -DWITH_OLD_YESOD
else
export FEATURES=-DWITH_ASSISTANT -DWITH_S3 -DWITH_WEBDAV -DWITH_HOST -DWITH_OLD_URI -DWITH_PAIRING -DWITH_XMPP
endif

View file

@ -1,3 +1,38 @@
## version 3.20121211
This release of the git-annex assistant (which is still in beta)
consists of mostly bugfixes, user interface improvements, and improvements
to existing features.
In general, anything you can configure with the assistant's web app
will work. Some examples of use cases supported by this release include:
* Using Box.com's 5 gigabytes of free storage space as a cloud transfer
point between between repositories that cannot directly contact
one-another. (Many other cloud providers are also supported, from Rsync.net
to Amazon S3, to your own ssh server.)
* Archiving or backing up files to Amazon Glacier. See [[archival_walkthrough]].
* [[Sharing repositories with friends|share_with_a_friend_walkthrough]]
contacted through a Jabber server (such as Google Talk).
* [[Pairing|pairing_walkthrough]] two computers that are on the same local
network (or VPN) and automatically keeping the files in the annex in
sync as changes are made to them.
* Cloning your repository to removable drives, USB keys, etc. The assistant
will notice when the drive is mounted and keep it in sync.
Such a drive can be stored as an offline backup, or transported between
computers to keep them in sync.
The following are known limitations of this release of the git-annex
assistant:
* The Max OSX standalone app may not work on all versions of Max OSX.
Please test!
* On Mac OSX and BSD operating systems, the assistant uses kqueue to watch
files. Kqueue has to open every directory it watches, so too many
directories will run it out of the max number of open files (typically
1024), and fail. See [[bugs/Issue_on_OSX_with_some_system_limits]]
for a workaround.
## version 3.20121126
This adds several features to the git-annex assistant, which is still in beta.

View file

@ -1,4 +1,6 @@
This is a collection of problem reports for the standalone OSX app.
If you have a problem using it, post it here. --[[Joey]]
(Some things that should be fixed now have been moved to [[old]].)
[[!tag /design/assistant/OSX]]

View file

@ -0,0 +1,41 @@
[[!comment format=mdwn
username="https://www.google.com/accounts/o8/id?id=AItOawkfHTPsiAcHEEN7Xl7WxiZmYq-vX7azxFY"
nickname="Vincent"
subject="OS/X build 2012-12-12"
date="2012-12-13T00:59:51Z"
content="""
I installed this today from the .dmg.bz2, md5sum 1bb50b3ee5eda3cd7f4b4a70cdae1855 on OS/X 10.8.2
uname -a
Darwin foo 12.2.0 Darwin Kernel Version 12.2.0: Sat Aug 25 00:48:52 PDT 2012; root:xnu-2050.18.24~1/RELEASE_X86_64 x86_64
I installed the app to the Applications folder.
I had chrome and firefox running, recent versions.
Double-click and it opens a new chrome window. This came up behind the existing (iconified) window. A nit, but just so you know.
The configuration part of the app is shown, so far so good.
I type in the path I want it to use (~/work/annex) and press the create button.
It hangs forever trying to access localhost:55163
$ ps aux|grep git
me 85291 100.0 0.0 2460884 4160 ?? R 11:42am 12:03.72 git init --quiet /Users/me/work/annex/
me 85233 0.0 0.3 2687204 44064 ?? S 11:42am 0:00.44 git-annex webapp -psn_0_50204638
me 85226 0.0 0.0 2433432 868 ?? S 11:42am 0:00.00 /bin/sh /Applications/git-annex.app/Contents/MacOS/git-annex-webapp -psn_0_50204638
me 85515 0.0 0.0 2432768 620 s000 S+ 11:54am 0:00.00 grep git
$ netstat -an |grep 55163
tcp4 0 0 127.0.0.1.55163 127.0.0.1.55207 CLOSE_WAIT
tcp4 0 0 127.0.0.1.55163 127.0.0.1.55206 CLOSE_WAIT
tcp4 0 0 127.0.0.1.55163 127.0.0.1.55205 CLOSE_WAIT
tcp4 0 0 127.0.0.1.55163 127.0.0.1.55201 ESTABLISHED
tcp4 0 0 127.0.0.1.55201 127.0.0.1.55163 ESTABLISHED
tcp4 0 0 127.0.0.1.55163 127.0.0.1.55199 CLOSE_WAIT
tcp4 0 0 127.0.0.1.55163 127.0.0.1.55197 CLOSE_WAIT
tcp4 0 0 127.0.0.1.55163 *.* LISTEN
I was plugged into wired ethernet, no other interfaces up, no VPN.
I have macports but no haskell packages, which ghc returns nothing.
"""]]

View file

@ -0,0 +1,23 @@
[[!comment format=mdwn
username="https://www.google.com/accounts/o8/id?id=AItOawkfHTPsiAcHEEN7Xl7WxiZmYq-vX7azxFY"
nickname="Vincent"
subject="comment 15"
date="2012-12-13T01:04:44Z"
content="""
following up to #14.
dtruss -p <git --init process>
shows the same symptom as reported earlier
SYSCALL(args) = return
workq_kernreturn(0x1, 0x10F31E000, 0x0) = -1 Err#22
workq_kernreturn(0x1, 0x10F31E000, 0x0) = -1 Err#22
workq_kernreturn(0x1, 0x10F31E000, 0x0) = -1 Err#22
workq_kernreturn(0x1, 0x10F31E000, 0x0) = -1 Err#22
workq_kernreturn(0x1, 0x10F31E000, 0x0) = -1 Err#22
workq_kernreturn(0x1, 0x10F31E000, 0x0) = -1 Err#22
...
workq_kernreturn(0x1, 0x10F31E000, 0x0) = -1 Err#22
dtrace: 339527 drops on CPU 0
"""]]

View file

@ -0,0 +1 @@
These issues should be fixed now.

View file

@ -0,0 +1,5 @@
[Due to some stupid issue on my and AT&T's part] one of my remote repositories is currently unreachable. I would like to tell the webapp/assistant to not attempt to sync with it, or, at least, modify this error message to be more specific (by telling me which repository is unreachable).
In a red bubble it says: "Synced with rose 60justin"
That verbage is the same if they all succeed. The only difference is the red instead of green. Would be nice to know exactly which machine to kick (if I didn't already know, eg I was syncing only with repositories not under my control).

View file

@ -0,0 +1,33 @@
What steps will reproduce the problem?
~$ mkdir testannex
~$ cd testannex/
testannex$ git init
Initialized empty Git repository in /Users/ed/testannex/.git/
testannex$ git annex init "test annex"
init test annex ok
(Recording state in git...)
testannex$ echo "file1" > file1
testannex$ git annex add file1
add file1 (checksum...) ok
(Recording state in git...)
testannex$ mkdir directory
testannex$ mv file1 directory/
testannex$ cat directory/file1
cat: directory/file1: No such file or directory
testannex$ git annex fix directory/file1
git-annex: directory/file1 not found
What is the expected output? What do you see instead?
git annex fix should fix the symlink. It looks like maybe it's *following* the symlink?
What version of git-annex are you using? On what operating system?
checkout: 20d195f compiled on OS X 10.7 using cabal.
Please provide any additional information below.
git annex assistant is not noticing file renames either.

View file

@ -0,0 +1,20 @@
I updated haskell platform, and now
<pre>
[jtang@x00 git-annex (master)]$ make test
Assistant/Threads/NetWatcher.hs:26:2:
warning: #warning Building without dbus support; will poll for network connection changes [-Wcpp]
Assistant/Threads/MountWatcher.hs:33:2:
warning: #warning Building without dbus support; will use mtab polling [-Wcpp]
test.hs:11:8:
Could not find module `Test.HUnit.Tools'
Perhaps you meant Test.HUnit.Text (from HUnit-1.2.5.1)
Use -v to see a list of the files searched for.
** failed to build the test suite
make: *** [test] Error 1
</pre>
Looks like a missing dep somewhere with testpack or quickcheck... I haven't had time to figure it out yet, its not git-annex specific but I thought I might log it as a reminder for myself just in case if the osxapp is more borked than usual, I probably need to flush my .cabal directory of installed userland dependancies.

View file

@ -0,0 +1,8 @@
[[!comment format=mdwn
username="http://joeyh.name/"
ip="4.153.8.117"
subject="comment 1"
date="2012-12-10T19:18:59Z"
content="""
Test.HUnit.Tools is part of testpack: <http://hackage.haskell.org/package/testpack>
"""]]

View file

@ -0,0 +1,45 @@
[[!comment format=mdwn
username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
nickname="Jimmy"
subject="comment 2"
date="2012-12-11T08:29:07Z"
content="""
yea its a problem with testpack rather than git-annex's test suite,
<pre>
[jtang@laplace git-annex (master)]$ cabal install testpack
Resolving dependencies...
Configuring testpack-2.1.2...
Building testpack-2.1.2...
Preprocessing library testpack-2.1.2...
[1 of 3] Compiling Test.QuickCheck.Instances ( src/Test/QuickCheck/Instances.hs, dist/build/Test/QuickCheck/Instances.o )
[2 of 3] Compiling Test.QuickCheck.Tools ( src/Test/QuickCheck/Tools.hs, dist/build/Test/QuickCheck/Tools.o )
src/Test/QuickCheck/Tools.hs:33:9:
Warning: Fields of `MkResult' not initialised: abort
In the expression:
MkResult
{ok = Just (expected == actual), expect = True,
interrupted = False,
reason = \"Result: expected \"
++ show expected ++ \", got \" ++ show actual,
stamp = [], callbacks = []}
In an equation for `@=?':
expected @=? actual
= MkResult
{ok = Just (expected == actual), expect = True,
interrupted = False,
reason = \"Result: expected \"
++ show expected ++ \", got \" ++ show actual,
stamp = [], callbacks = []}
[3 of 3] Compiling Test.HUnit.Tools ( src/Test/HUnit/Tools.hs, dist/build/Test/HUnit/Tools.o )
src/Test/HUnit/Tools.hs:131:57:
`maxDiscard' is not a (visible) constructor field name
src/Test/HUnit/Tools.hs:177:40: Not in scope: `maxDiscard'
cabal: Error: some packages failed to install:
testpack-2.1.2 failed during the building phase. The exception was:
ExitFailure 1
</pre>
"""]]

View file

@ -13,9 +13,10 @@ and use cases to add. Feel free to chip in with comments! --[[Joey]]
We are, approximately, here:
* Month 6 "9k bonus round": [[!traillink Android]] or [[!traillink desymlink]]
* Month 7: user-driven features and polishing
* Month 8: whatever I don't get to in month 6
* Month 6 "9k bonus round": [[!traillink desymlink]]
* Month 7: user-driven features and polishing;
[presentation at LCA2013](https://lca2013.linux.org.au/schedule/30059/view_talk)
* Month 8: [[!traillink Android]]
* Months 9-11: more user-driven features and polishing (see remaining TODO items in all pages above)
* Month 12: "Windows purgatory" [[Windows]]

View file

@ -0,0 +1,16 @@
[[!comment format=mdwn
username="http://lj.rossia.org/users/imz/"
ip="79.165.59.119"
subject="&quot;removing&quot; vs drop"
date="2012-12-12T13:20:42Z"
content="""
I don't understand the difference behind:
> Removing objects also works (and puts back a broken symlink)
and
> \"drop\" won't work because they rely on the symlink to map back to the key.
If a file is removed (its content, which is replaced by a symlink), then it's not present there, so effectively it should be counted as \"dropped\" at this place. So, removing a file without counting it as dropped is something inconsistent, isn't it? Do I misunderstand something?
"""]]

View file

@ -0,0 +1,8 @@
[[!comment format=mdwn
username="http://joeyh.name/"
ip="4.153.8.117"
subject="comment 2"
date="2012-12-12T23:45:42Z"
content="""
`git annex drop` is a user-level operation built on top of lower-level object removal functions that are also used by other things.
"""]]

View file

@ -0,0 +1,15 @@
Made `git annex sync` update the file mappings in direct mode.
To do this efficiently, it uses `git diff-tree` to find files that are
changed by the sync, and only updates those mappings. I'm rather happy
with this, as a first step to fully supporting sync in direct mode.
Finished the overhaul of the OSX app's library handling. It seems to work
well, and will fix a whole class of ways the OSX app could break.
Fixed a bug in the preferred content settings for backup repositories,
introduced by some changes I made to preferred content handling 4 days ago.
Fixed the Debian package to build with WebDAV support, which I forgot to
turn on before.
Planning a release tomorrow.

View file

@ -0,0 +1,53 @@
Yesterday I cut another release. However, getting an OSX build took until
12:12 pm today because of a confusion about the location of lsof on OSX. The
OSX build is now available, and I'm looking forward to hearing if it's working!
----
Today I've been working on making `git annex sync` commit in direct mode.
For this I needed to find all new, modified, and deleted files, and I also
need the git SHA from the index for all non-new files. There's not really
an ideal git command to use to query this. For now I'm using
`git ls-files --others --stage`, which works but lists more files than I
really need to look at. It might be worth using one of the Haskell libraries
that can directly read git's index.. but for now I'll stick with `ls-files`.
It has to check all direct mode files whose content is present, which means
one stat per file (on top of the stat that git already does), as well as one
retrieval of the key per file (using the single `git cat-file` process that
git-annex talks to).
This is about as efficient as I can make it, except that unmodified
annexed files whose content is not present are listed due to --stage,
and so it has to stat those too, and currently also feeds them into `git add`.
The assistant will be able to avoid all this work, except once at startup.
Anyway, direct mode committing is working!
For now, `git annex sync` in direct mode also adds new files. This because
`git annex add` doesn't work yet in direct mode.
It's possible for a direct mode file to be changed during a commit,
which would be a problem since committing involves things like calculating
the key and caching the mtime/etc, that would be screwed up. I took
care to handle that case; it checks the mtime/etc cache before and after
generating a key for the file, and if it detects the file has changed,
avoids committing anything. It could retry, but if the file is a VM disk
image or something else that's constantly modified, commit retrying forever
would not be good.
----
For `git annex sync` to be usable in direct mode, it still needs
to handle merging. It looks like I may be able to just enhance the automatic
conflict resolution code to know about typechanged direct mode files.
The other missing piece before this can really be used is that currently
the key to file mapping is only maintained for files added locally, or
that come in via `git annex sync`. Something needs to set up that mapping
for files present when the repo is initally cloned. Maybe the thing
to do is to have a `git annex directmode` command that enables/disables
direct mode and can setup the the mapping, as well as any necessary unlocks
and setting the trust level to untrusted.

View file

@ -89,8 +89,8 @@ is converted to a real file when it becomes present.
* `git annex sync` updates the key to files mappings for files changed,
but needs much other work to handle direct mode:
* Generate git commit, without running `git commit`, because it will
want to stage the full files.
* Update location logs for any files deleted by a commit.
want to stage the full files. **done**
* Update location logs for any files deleted by a commit. **done**
* Generate a git merge, without running `git merge` (or possibly running
it in a scratch repo?), because it will stumble over the direct files.
* Drop contents of files deleted by a merge (including updating the

View file

@ -0,0 +1,10 @@
[[!comment format=mdwn
username="https://www.google.com/accounts/o8/id?id=AItOawniCRkhl_W87gOK5eElfsef3FoUsUFpAr4"
nickname="Alexandre"
subject="Simplifying this kind of setup"
date="2012-12-10T14:33:08Z"
content="""
Maybe it is possible to avoid the XMPP account setup and transferring via XMPP, maybe getting notifications through the SSH connection is possible.
I'm thinking about a \"git-annex-shell server\" unix socket to which clients would connect using the SSH connection and get update notifications from other clients.
"""]]

View file

@ -0,0 +1,8 @@
[[!comment format=mdwn
username="https://www.google.com/accounts/o8/id?id=AItOawkk3K0AUduAybbBO_LRRGKOe2zcGeezbzI"
nickname="Nathan"
subject="comment 5"
date="2012-12-11T04:15:49Z"
content="""
Thanks, Joey; I was using the standalone build, and it seems to be behaving better now.
"""]]

View file

@ -0,0 +1,8 @@
[[!comment format=mdwn
username="http://sunny256.sunbase.org/"
nickname="sunny256"
subject="comment 4"
date="2012-12-09T20:13:47Z"
content="""
Thanks a lot, Steve. Awesome, got everything on my wishlist. :) A very useful utility, and works perfectly. Will be using this a lot. git-annex-utils is a good name for this, I'm sure if you place it on GitHub or somewhere else you'll get lots of contributions and this could grow to be a project containing many useful utilities for git-annex.
"""]]

View file

@ -0,0 +1,18 @@
[[!comment format=mdwn
username="Steve"
ip="92.104.175.136"
subject="comment 5"
date="2012-12-10T04:07:53Z"
content="""
I pay attention to feedback ;)
I'm not done with it yet, I want to add in some options to limit what gets counted.
For example: If you have two annexed files that contain the same content using the same backend, they will be stored only once in the .git/annex/objects directory but be counted twice by gadu.
I want to fix that, but I'll leave an option to keep that behavior if you want. I also want to add options to count or not count files that exist in a certain repo. It will be very easy to add options to only count files that you have or don't have locally as well.
Making it pay attention to environment variables that git and git-annex do would also be a good idea. (like GIT_DIR, etc...)
I'm open to good ideas that anybody has, unfortunately I can only work on it on the weekends for now.
"""]]

View file

@ -0,0 +1,8 @@
[[!comment format=mdwn
username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
nickname="Jimmy"
subject="comment 4"
date="2012-12-10T17:00:43Z"
content="""
For those that care, I've updated my autobuilder to the latest version of haskell-platform 2012.4.0.0 and it appears to be building correctly.
"""]]

View file

@ -1,4 +0,0 @@
git-annex 3.20121017 released with [[!toggle text="these changes"]]
[[!toggleable text="""
* Fix zombie cleanup reversion introduced in 3.20121009.
* Additional fix to support git submodules."""]]

View file

@ -0,0 +1,32 @@
git-annex 3.20121211 released with [[!toggle text="these changes"]]
[[!toggleable text="""
* webapp: Defaults to sharing box.com account info with friends, allowing
one-click enabling of the repository.
* Fix broken .config/git-annex/program installed by standalone tarball.
* assistant: Retrival from glacier now handled.
* Include ssh in standalone tarball and OSX app.
* watch: Avoid leaving hard links to files behind in .git/annex/tmp
if a file is deleted or moved while it's being quarantined in preparation
to being added to the annex.
* Allow `git annex drop --from web`; of course this does not remove
any file from the web, but it does make git-annex remove all urls
associated with a file.
* webapp: S3 and Glacier forms now have a select list of all
currently-supported AWS regions.
* webdav: Avoid trying to set props, avoiding incompatability with
livedrive.com. Needs DAV version 0.3.
* webapp: Prettify error display.
* webapp: Fix bad interaction between required fields and modals.
* webapp: Added help buttons and links next to fields that require
explanations.
* webapp: Encryption can be disabled when setting up remotes.
* assistant: Avoid trying to drop content from remotes that don't have it.
* assistant: Allow periods in ssh key comments.
* get/copy --auto: Transfer data even if it would exceed numcopies,
when preferred content settings want it.
* drop --auto: Fix dropping content when there are no preferred content
settings.
* webapp: Allow user to specify the port when setting up a ssh or rsync
remote.
* assistant: Fix syncing to just created ssh remotes.
* Enable WebDAV support in Debian package. Closes: #[695532](http://bugs.debian.org/695532)"""]]

View file

@ -116,6 +116,8 @@ will be added later.
All content is preferred.
`include=*`
### small archive
Only prefers content that's located in an "archive" directory, and

View file

@ -0,0 +1,8 @@
[[!comment format=mdwn
username="http://joeyh.name/"
ip="4.153.8.117"
subject="comment 4"
date="2012-12-10T19:46:01Z"
content="""
It was a bug in the backup group's preferred content pagespec, introduced by the changes I made to fix the previous problem. Now fixed.
"""]]

View file

@ -0,0 +1,9 @@
[[!comment format=mdwn
username="http://edheil.wordpress.com/"
ip="173.162.44.162"
subject="comment 5"
date="2012-12-11T16:03:04Z"
content="""
thanks!
"""]]

View file

@ -23,9 +23,9 @@ git-annex is designed for scalability. The key points are:
improves this will improve. Scaling to hundreds of thousands of files
is not a problem, scaling beyond that and git will start to get slow.
To some degree, git-annex works around innefficiencies in git; for
example it batches input sent to certian git commands that are slow
when run in an emormous repository.
To some degree, git-annex works around inefficiencies in git; for
example it batches input sent to certain git commands that are slow
when run in an enormous repository.
* It can use as much, or as little bandwidth as is available. In
particular, any interrupted file transfer can be resumed by git-annex.

View file

@ -1,5 +1,5 @@
Name: git-annex
Version: 3.20121128
Version: 3.20121211
Cabal-Version: >= 1.8
License: GPL
Maintainer: Joey Hess <joey@kitenet.net>

View file

@ -43,7 +43,7 @@ fi
# system binaries.
ORIG_PATH="$PATH"
export ORIG_PATH
PATH=$base/bin:$PATH
PATH=$base:$PATH
export PATH
ORIG_GIT_EXEC_PATH="$GIT_EXEC_PATH"