Merge branch 'master' into desymlink

Conflicts:
	debian/changelog
	doc/design/assistant/desymlink.mdwn
This commit is contained in:
Joey Hess 2012-12-17 13:29:10 -04:00
commit 40fa6936e4
52 changed files with 675 additions and 42 deletions

View file

@ -24,6 +24,7 @@ import Assistant.Drop
import Logs.Transfer
import Utility.DirWatcher
import Utility.Types.DirWatcher
import Utility.Lsof
import qualified Annex
import qualified Annex.Queue
import qualified Git.Command
@ -39,7 +40,8 @@ import qualified Data.ByteString.Lazy as L
checkCanWatch :: Annex ()
checkCanWatch
| canWatch =
| canWatch = do
liftIO setupLsof
unlessM (liftIO (inPath "lsof") <||> Annex.getState Annex.force)
needLsof
| otherwise = error "watch mode is not available on this system"

View file

@ -26,7 +26,7 @@ tests =
, TestCase "wget" $ testCmd "wget" "wget --version >/dev/null"
, TestCase "bup" $ testCmd "bup" "bup --version >/dev/null"
, TestCase "gpg" $ testCmd "gpg" "gpg --version >/dev/null"
, TestCase "lsof" $ testCmd "lsof" "lsof -v >/dev/null 2>&1"
, TestCase "lsof" $ findCmdPath "lsof" "lsof"
, TestCase "ssh connection caching" getSshConnectionCaching
] ++ shaTestCases
[ (1, "da39a3ee5e6b4b0d3255bfef95601890afd80709")
@ -50,7 +50,7 @@ shaTestCases l = map make l
zip (shacmds n) (repeat check)
where
key = "sha" ++ show n
check = "</dev/null | grep -q '" ++ knowngood ++ "'"
check = "</dev/null 2>/dev/null | grep -q '" ++ knowngood ++ "'"
shacmds n = concatMap (\x -> [x, 'g':x, osxpath </> x]) $
map (\x -> "sha" ++ show n ++ x) ["sum", ""]
{- Max OSX sometimes puts GNU tools outside PATH, so look in

View file

@ -64,6 +64,7 @@ otool appbase libmap = do
where
want s = not ("@executable_path" `isInfixOf` s)
&& not (".framework" `isInfixOf` s)
&& not ("libSystem.B" `isInfixOf` s)
process c [] m = return (nub $ concat c, m)
process c (file:rest) m = do
_ <- boolSystem "chmod" [Param "755", File file]

78
Build/Standalone.hs Normal file
View file

@ -0,0 +1,78 @@
{- Makes standalone bundle.
-
- Copyright 2012 Joey Hess <joey@kitenet.net>
-
- Licensed under the GNU GPL version 3 or higher.
-}
{-# LANGUAGE CPP #-}
module Build.Standalone where
import Control.Applicative
import Control.Monad.IfElse
import System.Environment
import Data.Maybe
import System.FilePath
import System.Directory
import System.IO
import Control.Monad
import Data.List
import Build.SysConfig as SysConfig
import Utility.PartialPrelude
import Utility.Directory
import Utility.Process
import Utility.Monad
import Utility.SafeCommand
import Utility.Path
{- Programs that git-annex uses, to include in the bundle.
-
- These may be just the command name, or the full path to it. -}
thirdpartyProgs :: [FilePath]
thirdpartyProgs = catMaybes
[ Just "git"
, Just "cp"
, Just "xargs"
, Just "gpg"
, Just "rsync"
, Just "ssh"
, Just "sh"
, headMaybe $ words SysConfig.uuid -- may include parameters
, ifset SysConfig.curl "curl"
, ifset SysConfig.wget "wget"
, ifset SysConfig.bup "bup"
, SysConfig.lsof
, SysConfig.sha1
, SysConfig.sha256
, SysConfig.sha512
, SysConfig.sha224
, SysConfig.sha384
]
where
ifset True s = Just s
ifset False _ = Nothing
progDir :: FilePath -> FilePath
#ifdef darwin_HOST_OS
progDir topdir = topdir
#else
progDir topdir = topdir </> "bin"
#endif
installProg :: FilePath -> FilePath -> IO ()
installProg dir prog = searchPath prog >>= go
where
go Nothing = error $ "cannot find " ++ prog ++ " in PATH"
go (Just f) = unlessM (boolSystem "install" [File f, File dir]) $
error $ "install failed for " ++ prog
main = getArgs >>= go
where
go [] = error "specify topdir"
go (topdir:_) = do
let dir = progDir topdir
createDirectoryIfMissing True dir
forM_ thirdpartyProgs $ installProg dir

View file

@ -2,9 +2,14 @@
module Build.TestConfig where
import Utility.Path
import Utility.Monad
import System.IO
import System.Cmd
import System.Exit
import System.FilePath
import System.Directory
type ConfigKey = String
data ConfigValue =
@ -98,6 +103,23 @@ searchCmd success failure cmdsparams = search cmdsparams
then success c
else search cs
{- Finds a command, either in PATH or perhaps in a sbin directory not in
- PATH. If it's in PATH the config is set to just the command name,
- but if it's found outside PATH, the config is set to the full path to
- the command. -}
findCmdPath :: ConfigKey -> String -> Test
findCmdPath k command = do
ifM (inPath command)
( return $ Config k $ MaybeStringConfig $ Just command
, do
r <- getM find ["/usr/sbin", "/sbin", "/usr/local/sbin"]
return $ Config k $ MaybeStringConfig r
)
where
find d =
let f = d </> command
in ifM (doesFileExist f) ( return (Just f), return Nothing )
quiet :: String -> String
quiet s = s ++ " >/dev/null 2>&1"

View file

@ -143,9 +143,6 @@ sdist: clean $(mans)
hackage: sdist
@cabal upload dist/*.tar.gz
THIRDPARTY_BINS=git curl lsof xargs rsync uuid wget gpg \
sha1sum sha224sum sha256sum sha384sum sha512sum cp ssh sh
LINUXSTANDALONE_DEST=$(GIT_ANNEX_TMP_BUILD_DIR)/git-annex.linux
linuxstandalone:
$(MAKE) git-annex
@ -160,19 +157,11 @@ linuxstandalone:
ln -sf git-annex "$(LINUXSTANDALONE_DEST)/bin/git-annex-shell"
zcat standalone/licences.gz > $(LINUXSTANDALONE_DEST)/LICENSE
set -e; \
for bin in $(THIRDPARTY_BINS); do \
p="$$(PATH=$$PATH:/usr/sbin:/sbin:/usr/local/sbin which "$$bin")"; \
if [ -z "$$p" ]; then \
echo "** missing $$bin" >&2; \
exit 1; \
else \
cp "$$p" "$(LINUXSTANDALONE_DEST)/bin/"; \
fi; \
done
runghc Build/Standalone.hs "$(LINUXSTANDALONE_DEST)"
install -d "$(LINUXSTANDALONE_DEST)/git-core"
(cd "$(shell git --exec-path)" && tar c .) | (cd "$(LINUXSTANDALONE_DEST)"/git-core && tar x)
install -d "$(LINUXSTANDALONE_DEST)/templates"
touch "$(LINUXSTANDALONE_DEST)/libdirs.tmp"
for lib in $$(ldd "$(LINUXSTANDALONE_DEST)"/bin/* $$(find "$(LINUXSTANDALONE_DEST)"/git-core/ -type f) | grep -v -f standalone/linux/glibc-libs | grep -v "not a dynamic executable" | egrep '^ ' | sed 's/^\t//' | sed 's/.*=> //' | cut -d ' ' -f 1 | sort | uniq); do \
@ -206,17 +195,10 @@ osxapp:
gzcat standalone/licences.gz > $(OSXAPP_BASE)/LICENSE
cp $(OSXAPP_BASE)/LICENSE $(GIT_ANNEX_TMP_BUILD_DIR)/build-dmg/LICENSE.txt
for bin in $(THIRDPARTY_BINS); do \
p="$$(PATH=$$PATH:/usr/sbin:/sbin:/usr/local/sbin which "$$bin")"; \
if [ -z "$$p" ]; then \
echo "** missing $$bin" >&2; \
exit 1; \
else \
cp "$$p" "$(OSXAPP_BASE)"; \
fi; \
done
runghc Build/Standalone.hs $(OSXAPP_BASE)
(cd "$(shell git --exec-path)" && tar c .) | (cd "$(OSXAPP_BASE)" && tar x)
install -d "$(OSXAPP_BASE)/templates"
runghc Build/OSXMkLibs.hs $(OSXAPP_BASE)
rm -f tmp/git-annex.dmg

View file

@ -80,7 +80,7 @@ getDirInfo dir = do
return $ DirInfo dir contents
where
getDirEnt f = catchMaybeIO $ do
s <- getFileStatus (dir </> f)
s <- getSymbolicLinkStatus (dir </> f)
return $ DirEnt f (fileID s) (isDirectory s)
{- Difference between the dirCaches of two DirInfos. -}

View file

@ -10,8 +10,10 @@
module Utility.Lsof where
import Common
import Build.SysConfig as SysConfig
import System.Posix.Types
import System.Posix.Env
data LsofOpenMode = OpenReadWrite | OpenReadOnly | OpenWriteOnly | OpenUnknown
deriving (Show, Eq)
@ -21,6 +23,17 @@ type CmdLine = String
data ProcessInfo = ProcessInfo ProcessID CmdLine
deriving (Show)
{- lsof is not in PATH on all systems, so SysConfig may have the absolute
- path where the program was found. Make sure at runtime that lsof is
- available, and if it's not in PATH, adjust PATH to contain it. -}
setupLsof :: IO ()
setupLsof = do
let cmd = fromMaybe "lsof" SysConfig.lsof
when (isAbsolute cmd) $ do
path <- getSearchPath
let path' = takeDirectory cmd : path
setEnv "PATH" (join [searchPathSeparator] path') True
{- Checks each of the files in a directory to find open files.
- Note that this will find hard links to files elsewhere that are open. -}
queryDir :: FilePath -> IO [(FilePath, LsofOpenMode, ProcessInfo)]

View file

@ -132,11 +132,25 @@ relHome path = do
then "~/" ++ relPathDirToFile home path
else path
{- Checks if a command is available in PATH. -}
{- Checks if a command is available in PATH.
-
- The command may be fully-qualified, in which case, this succeeds as
- long as it exists. -}
inPath :: String -> IO Bool
inPath command = getSearchPath >>= anyM indir
inPath command = isJust <$> searchPath command
{- Finds a command in PATH and returns the full path to it.
-
- The command may be fully qualified already, in which case it will
- be returned if it exists.
-}
searchPath :: String -> IO (Maybe FilePath)
searchPath command
| isAbsolute command = check command
| otherwise = getSearchPath >>= getM indir
where
indir d = doesFileExist $ d </> command
indir d = check $ d </> command
check f = ifM (doesFileExist f) ( return (Just f), return Nothing )
{- Checks if a filename is a unix dotfile. All files inside dotdirs
- count as dotfiles. -}

1
debian/changelog vendored
View file

@ -5,6 +5,7 @@ git-annex (3.20121212) UNRELEASED; urgency=low
via symlinks. Note that direct mode is currently experimental. Many
git and git-annex commands do not work, or can even cause data loss in
direct mode.
* kqueue: Fix bug that made broken symlinks not be noticed.
-- Joey Hess <joeyh@debian.org> Thu, 13 Dec 2012 14:06:43 -0400

View file

@ -0,0 +1,8 @@
[[!comment format=mdwn
username="http://joeyh.name/"
ip="4.153.8.117"
subject="comment 16"
date="2012-12-13T22:40:59Z"
content="""
Today's daily build of the OSX app has a further change that *might* help. I removed the system library and frameworks from the bundle.
"""]]

View file

@ -0,0 +1,59 @@
[[!comment format=mdwn
username="https://www.google.com/accounts/o8/id?id=AItOawkfHTPsiAcHEEN7Xl7WxiZmYq-vX7azxFY"
nickname="Vincent"
subject="OS/X daily 2012-12-13"
date="2012-12-14T09:22:42Z"
content="""
Thanks for the update - I tried again, similar results.
same platform. installed image md5sum 1bb50b3ee5eda3cd7f4b4a70cdae1855
Procedure was the same.
- download, bunzip2, mount, drag app to Applications.
- chrome had one tab open, iconified
- double-click application icon
Chrome opens up and shows the config window. Type in the same path
after checking that the final element of the path did not already exist.
ps aux|grep git
me 89194 99.0 0.0 2460884 4160 ?? R 8:03pm 0:12.58 git init --quiet /Users/me/me/annex/
me 89245 0.2 0.0 2423356 220 s001 R+ 8:07pm 0:00.00 grep git
me 89182 0.0 0.3 2668772 44208 ?? S 8:03pm 0:00.30 git-annex webapp -psn_0_55022710
me 89177 0.0 0.0 2433432 868 ?? S 8:03pm 0:00.00 /bin/sh /Applications/git-annex.app/Contents/MacOS/git-annex-webapp -psn_0_55022710
I ran dtruss on the two processes of interest, including when I sent them kill -9 in case that showed anything of interest.
Mail me if you need that but the gist is git init was doing
workq_kernreturn(0x1, 0x1019CC000, 0x0) = -1 Err#22
workq_kernreturn(0x1, 0x1019CC000, 0x0) = -1 Err#22
workq_kernreturn(0x1, 0x1019CC000, 0x0) = -1 Err#22
...
workq_kernreturn(0x1, 0x1019CC000, 0x0) = -1 Err#22
dtrace: 2006687 drops on CPU 0
workq_kernreturn(0x1, 0x1019CC000, 0x0) = -1 Err#22
and the git-annex webapp was doing stuff like
psynch_cvwait(0x7FD7F1418888, 0x2BE010002BF00, 0x600) = -1 Err#260
sigreturn(0x7FFF543DD810, 0x1E, 0x2) = 0 Err#-2
__pthread_canceled(0x0, 0x2BE010002BF00, 0x7FFF543DD8C8) = -1 Err#22
psynch_cvwait(0x7FD7F1418888, 0x2C8010002C900, 0x600) = -1 Err#260
sigreturn(0x7FFF543DD810, 0x1E, 0x2) = 0 Err#-2
__pthread_canceled(0x0, 0x2C8010002C900, 0x7FFF543DD8C8) = -1 Err#22
psynch_cvwait(0x7FD7F1418888, 0x2D0010002D100, 0x600) = -1 Err#260
sigreturn(0x7FFF543DD810, 0x1E, 0x2) = 0 Err#-2
__pthread_canceled(0x0, 0x2D0010002D100, 0x7FFF543DD8C8) = -1 Err#22
psynch_cvwait(0x7FD7F1418888, 0x2D7010002D800, 0x600) = -1 Err#260
sigreturn(0x7FFF543DD810, 0x1E, 0x10DCEDC00) = 0 Err#-2
__pthread_canceled(0x0, 0x2D7010002D800, 0x7FFF543DD8C8) = -1 Err#22
psynch_cvwait(0x7FD7F14189D8, 0x230100002400, 0x2300) = 0 0
read(0x6, \"\377\0\", 0x1000) = 1 0
setitimer(0x0, 0x116903E50, 0x0) = 0 0
write(0x7, \"\377\0\", 0x1) = 1 0
sigreturn(0x7FFF543DD810, 0x1E, 0x0) = 0 Err#-2
__pthread_canceled(0x0, 0x2BC010002BD00, 0x7FFF543DD8C8) = -1 Err#22
which may be unrelated browser event loop stuff I guess.
"""]]

View file

@ -0,0 +1,12 @@
[[!comment format=mdwn
username="http://joeyh.name/"
ip="4.153.8.117"
subject="comment 18"
date="2012-12-14T20:55:12Z"
content="""
I've built the app on 10.8.2, let's hope that'll finally put this problem to rest.
Temporarily available here: <http://downloads.kitenet.net/tmp/git-annex.dmg.bz2>
(This build currently lacks XMPP support.)
"""]]

View file

@ -0,0 +1,11 @@
[[!comment format=mdwn
username="https://www.google.com/accounts/o8/id?id=AItOawkfHTPsiAcHEEN7Xl7WxiZmYq-vX7azxFY"
nickname="Vincent"
subject="comment 19"
date="2012-12-16T05:27:06Z"
content="""
downloaded the 10.8.2 build, md5 9fc31ec6dcf0088d3723d1b25110f7f7.
git --init works instantaneously.
"""]]

View file

@ -0,0 +1,10 @@
[[!comment format=mdwn
username="http://joeyh.name/"
ip="4.153.8.117"
subject="comment 20"
date="2012-12-17T16:23:05Z"
content="""
@Vincent OMG, it works!?!
\o/
"""]]

View file

@ -22,3 +22,5 @@ I'd expect to see git-annex run. "git-annex" doesn't run and what I see (in the
**Please provide any additional information below.**
[[!tag /design/assistant/OSX]]
> This was fixed a while ago. [[done]] --[[Joey]]

View file

@ -0,0 +1,16 @@
with rsync, it is sometimes the case that one needs to specify ssh options -- typical examples from the rsync man page are `rsync -e 'ssh -p 2234'`. as git-annex does the shell splitting of the arguments in `annex-rsync-options` (see [[special remotes/rsync]]) itself by looking for whitespace, these options can't be passed directly. (`annex-rsync-options = -e 'ssh -p 2234'` gets split to `["rsync", "-e", "'ssh", "-p", "2234'"]` instead of `["rsync", "-e", "ssh -p 2234"]`).
git-annex should respect shell splitting rules when looking at annex-rsync-options. (i suppose there is a haskell library or module for that; in python, we have the `shlex` module for that).
## workaround
put this in .git/ssh and mark it as executable:
#!/bin/sh
exec ssh -p 2234 $@
put this in your git annex config in the particular remote's section:
annex-rsync-options = -e /local/path/to/your/repo/.git/ssh
(typical bug report information: observed with git-annex 3.20121127 on debian)

View file

@ -0,0 +1,23 @@
[[!comment format=mdwn
username="http://joeyh.name/"
ip="4.153.8.117"
subject="comment 1"
date="2012-12-13T17:03:08Z"
content="""
Due to the way git-annex runs rsync, which involves a specific -e parameter it constructs that, you cannot pass -e in annex-rsync-options anyway; or if you do you'll bypass use of git-annex-shell, which is not desirable. I have not checked which, but would not recommend use of it.
There is no need for ugly workarounds. Just use ~/.ssh/config to configure the hostname to use the nonstandard port it needs. For example:
<pre>
Host example.com
Port 2234
</pre>
Or, to make a separate example.com-2234 host that can be used to use the nonstandard port:
<pre>
Host example.com-2234
Hostname example.com
Port 2234
</pre>
"""]]

View file

@ -0,0 +1,68 @@
What steps will reproduce the problem?
In one terminal, I created a new annex and started the assistant watching it.
In another, I added a file file1; assistant noticed it and added it to the annex.
I moved file1 to a directory directory1; the link broke and assistant did not fix it.
I created a file called file2 inside directory 2; assistant noticed it and added it to the annex.
I moved file2 back up to the annex root directory; the link broke and assistant did not fix it.
I created a file file3 in the annex root directory; assistant noticed it and added it to the annex.
Here is the content of the first terminal, where I created the annex and ran assistant:
~$ mkdir testannex
~$ cd testannex/
testannex$ git init .
Initialized empty Git repository in /Users/ed/testannex/.git/
testannex$ git annex init "test annex"
init test annex ok
(Recording state in git...)
testannex$ git annex assistant --foreground
assistant . (scanning...) (started...) add file1 (checksum...) ok
(Recording state in git...)
(Recording state in git...)
add directory1/file2 (checksum...) ok
(Recording state in git...)
(Recording state in git...)
add file3 (checksum...)
here is the content of the second terminal, where I created and moved files:
~$ cd testannex
testannex$ echo "file1 content" > file1
testannex$ mkdir directory1
testannex$ ls -l file1
lrwxr-xr-x 1 ed staff 180 Dec 13 15:40 file1 -> .git/annex/objects/FX/51/SHA256E-s14--edac79763e630b1b77aefb6c284bcb0362dea71c0548be0e793ffa8fd5907b80/SHA256E-s14--edac79763e630b1b77aefb6c284bcb0362dea71c0548be0e793ffa8fd5907b80
testannex$ mv file1 directory1/
testannex$ cd directory1/
directory1$ cat file1
cat: file1: No such file or directory
directory1$ echo "file2 content" > file2
directory1$ cat file2
file2 content
directory1$ cat file1
cat: file1: No such file or directory
directory1$ mv file2 ../
directory1$ cd ..
testannex$ echo "file3 content" > file3
testannex$
What is the expected output? What do you see instead?
The links do not break when moved to another directory.
What version of git-annex are you using? On what operating system?
One compiled using cabal from checkout 739c937
Please provide any additional information below.
> [[fixed|done]]; this turned out to be an kqueue specific bug,
> the kqueue code statted new files, but that files for a broken symlink.
> Using lstat instead fixed this. --[[Joey]]

View file

@ -0,0 +1,9 @@
[[!comment format=mdwn
username="http://edheil.wordpress.com/"
ip="173.162.44.162"
subject="comment 1"
date="2012-12-13T20:50:18Z"
content="""
(Operating system is OS X Lion)
"""]]

View file

@ -0,0 +1,9 @@
[[!comment format=mdwn
username="http://edheil.wordpress.com/"
ip="173.162.44.162"
subject="comment 2"
date="2012-12-15T05:47:25Z"
content="""
thanks, fix worked for me! Now the links are fixed virtually instantaneously when the file is moved.
"""]]

View file

@ -31,3 +31,6 @@ Please provide any additional information below.
git annex assistant is not noticing file renames either.
> git-annex commands (other than `git annex add`) only operate on files
> checked into git, which `directory/file1` is not, since you did not use
> `git mv`. Once you `git add` the file, it'll work. [[done]] --[[Joey]]

View file

@ -0,0 +1,24 @@
[[!comment format=mdwn
username="http://edheil.wordpress.com/"
ip="173.162.44.162"
subject="comment 1"
date="2012-12-13T20:14:37Z"
content="""
Thanks for the response! It's good to know how it works right now.
The reason I was expecting it to work on the files even if I moved them without git rename, is this:
Imaginary use case:
I am using git annex assistant, and not using the command line at all. Maybe I don't know anything about git. I am on a machine in the \"client\" group, so it drops content in \"archive\" subdirectories after storing it safely in a repository in the \"backup\" group.
To add content to git annex, I drag and drop it into the git annex directory. Assistant notices, and it gets added to the annex (it is now a symlink). Yay!
To archive content, I drag and drop the file (its symlink actually) from the git annex directory to an annex/archive directory, assistant notices, and it gets moved off to backup directories, and this symlink becomes dead. This doesn't work because git annex assistant doesn't notice renames.
To retrieve content from an archive, I drag and drop the dead link in the archive directory to a parent directory, and git annex notices and grabs the content from a backup somewhere. This doesn't work, because git annex assistant doesn't notice renames.
These kinds of operations are necessary if I'm going to archive and unarchive files, or otherwise move and manage them, while the assistant is running, without using git from the command line.
"""]]

View file

@ -0,0 +1,8 @@
[[!comment format=mdwn
username="http://joeyh.name/"
ip="4.153.8.117"
subject="comment 2"
date="2012-12-13T20:24:32Z"
content="""
The assistant does notice renames, and also automatically fixes links.
"""]]

View file

@ -0,0 +1,10 @@
[[!comment format=mdwn
username="http://edheil.wordpress.com/"
ip="173.162.44.162"
subject="comment 3"
date="2012-12-13T20:39:57Z"
content="""
The assistant doesn't seem to be noticing renames or fixing files anymore, for me; that was what got me started on this bug report, though it sounds like I took it in an irrelevant direction by focusing on the command line and \"git annex fix.\"
I'll double check, and submit a new bug report if I can confirm that assistant isn't doing what it should.
"""]]

View file

@ -20,3 +20,6 @@ What version of git-annex are you using? On what operating system?
git annex Version: 3.20121017 on Mac OS X 10.7.5
[[!tag /design/assistant/OSX]]
> Libraries are now handled better in the OSX app and this should be able
> to happen anymore. [[done]] --[[Joey]]

View file

@ -2,4 +2,10 @@ The git-annex assistant is being
[crowd funded on Kickstarter](http://www.kickstarter.com/projects/joeyh/git-annex-assistant-like-dropbox-but-with-your-own/).
I'll be blogging about my progress here on a semi-daily basis.
[[!sidebar content="""
[[!calendar type="month" pages="page(design/assistant/blog/*)"]]
[[!calendar type="month" month="-1" pages="page(design/assistant/blog/*)"]]
[[!calendar type="month" month="-2" pages="page(design/assistant/blog/*)"]]
"""]]
[[!inline pages="page(design/assistant/blog/*)" show=0]]

View file

@ -1,6 +1,6 @@
More bugfixes today. The assistant now seems to have enough users that
they're turning up interesting bugs, which is good. But does keep me too
busy to add many more bugs^Wcode.
busy to add many more bugs\^Wcode.
The fun one today made it bloat to eat all memory when logging out of a
Linux desktop. I tracked that back to a bug in the Haskell DBUS library

View file

@ -0,0 +1,59 @@
Built `git annex direct` and `git annex indirect` to toggle back and forth
between direct mode. Made `git annex status` show if the repository is in
direct mode. Now *only* merging is needed for direct mode to be basically
usable.
I can do a little demo now. Pay attention to the "@" ls shows at the end
of symlinks.
joey@gnu:~/tmp/bench/rdirect>ls
myfile@ otherfile@
joey@gnu:~/tmp/bench/rdirect>git annex find
otherfile
# So, two files, only one present in this repo.
joey@gnu:~/tmp/bench/rdirect>git annex direct
commit
# On branch master
# Your branch is ahead of 'origin/master' by 7 commits.
#
nothing to commit (working directory clean)
ok
direct myfile ok
direct otherfile ok
direct ok
joey@gnu:~/tmp/bench/rdirect>ls
myfile@ otherfile
# myfile is still a broken symlink because we don't have its content
joey@gnu:~/tmp/bench/rdirect>git annex get myfile
get myfile (from origin...) ok
(Recording state in git...)
joey@gnu:~/tmp/bench/rdirect>ls
myfile otherfile
joey@gnu:~/tmp/bench/rdirect>echo "look mom, no symlinks" >> myfile
joey@gnu:~/tmp/bench/rdirect>git annex sync
add myfile (checksum...) ok
commit
(Recording state in git...)
[master 0e8de9b] git-annex automatic sync
...
ok
joey@gnu:~/tmp/bench/rdirect>git annex indirect
commit ok
indirect myfile ok
indirect otherfile ok
indirect ok
joey@gnu:~/tmp/bench/rdirect>ls
myfile@ otherfile@
I'd like `git annex direct` to set the repository to untrusted, but
I didn't do it. Partly because having `git annex indirect` set it back to
semitrusted seems possibly wrong -- the user might not trust a repo even in
indirect mode. Or might fully trust it. The docs will encourage users to
set direct mode repos to untrusted -- in direct mode you're operating
without large swathes of git-annex's carefully constructed safety net.
(When the assistant later uses direct mode, it'll untrust the repository
automatically.)

View file

@ -0,0 +1,18 @@
Fixed a bug in the kqueue code that made the assistant not notice when a
file was renamed into a subdirectory. This turned out to be because the
symlink got broken, and it was using `stat` on the file. Switching to
`lstat` fixed that.
Improved installation of programs into standalone bundles. Now it uses
the programs detected by configure, rather than a separate hardcoded list.
Also improved handling of lsof, which is not always in PATH.
Made a OSX 10.8.2 build of the app, which is nearly my last gasp attempt
at finding a way around this crazy `git init` spinning problem with Jimmy's
daily builds are used with newer OSX versions. Try it here:
<http://downloads.kitenet.net/tmp/git-annex.dmg.bz2>
----
Mailed out the Kickstarter T-shirt rewards today, to people in the US.
Have to fill out a bunch of forms before I can mail the non-US ones.

View file

@ -0,0 +1,9 @@
[[!comment format=mdwn
username="https://www.google.com/accounts/o8/id?id=AItOawmYiJgOvC4IDYkr2KIjMlfVD9r_1Sij_jY"
nickname="Douglas"
subject="OSX troubles"
date="2012-12-17T16:07:41Z"
content="""
Have you considered making a homebrew installable version of git-annex? It may make things easier for you when it comes to build dependencies.
"""]]

View file

@ -0,0 +1,8 @@
[[!comment format=mdwn
username="http://joeyh.name/"
ip="4.153.8.117"
subject="comment 2"
date="2012-12-17T16:16:47Z"
content="""
I would *love* *love* *love* for there to be a homebrew for git-annex. But I know nothing about homebrew so am not in a very good position to do it.
"""]]

View file

@ -62,6 +62,7 @@ is converted to a real file when it becomes present.
can map to multiple files. And that when a file is deleted or moved, the
mapping needs to be updated.
* May need a reverse mapping, from files in the tree to keys? TBD
(Currently, getting by looking up symlinks using `git cat-file`)
(Needed to make things like `git annex drop` that want to map from the
file back to the key work.)
* The existing watch code detects when a file gets closed, and in this

View file

@ -11,6 +11,10 @@ who share a repository, that is stored in the [[cloud]].
See <http://git-annex.branchable.com/design/assistant/blog/day_114__xmpp/#comment-aaba579f92cb452caf26ac53071a6788>
* Assistant.Sync.manualPull doesn't handle XMPP remotes yet.
This is needed to handle getting back in sync after reconnection.
* Support use of a single XMPP account with several separate git-annex repos.
This probably works for the simple push notification use of XMPP. But
XMPP pairing and the pushes over XMPP assume that anyone you're paired with
is intending to sync to your repository.
## design goals

View file

@ -0,0 +1 @@
I use git-annex mostly to archive my photos. They're generally large raw files and I am encrypting them going to the S3 special remote. Are there any plans to compress objects before encrypting them? It would save on S3 costs, only add some overhead on get and copy, and I don't think there would be any problems looking up data later.

View file

@ -0,0 +1,8 @@
[[!comment format=mdwn
username="http://joeyh.name/"
ip="4.153.8.117"
subject="comment 1"
date="2012-12-14T18:53:04Z"
content="""
gpg compresses encrypted content by default, so all encrypted special remotes get compression for free
"""]]

View file

@ -0,0 +1,5 @@
I worked out how to retroactively annex a large file that had been checked into a git repo some time ago. I thought this might be useful for others, so I am posting it here.
> This is a great tip, so I've moved it to
> [[tips|tips/How_to_retroactively_annex_a_file_already_in_a_git_repo]].
> --[[Joey]]

View file

@ -0,0 +1,24 @@
New user? Can't figure out the basics? Add it here - what you wanted, what you tried.
#### I wanted to keep track of some files I had all organized in a directory outside of my ~/annex:
$ cd ~/annex
$ git annex add /path/to/some/photos
fatal: '/path/to/some/photos' is outside repository
But git-annex doesn't work that way. I had to do this instead
$ rsync -a /path/to/some/photos
$ git annex add photos
(Recording state in git...)
$ git annex status
... lots of helpful info...
#### I just have the OS/X app, can I do commandline stuff?
yes
$ /Applications/git-annex.app/Contents/MacOS/git-annex add photos/
(Recording state in git...)
but perhaps there is a better way.

View file

@ -0,0 +1,10 @@
[[!comment format=mdwn
username="http://joeyh.name/"
ip="4.153.8.117"
subject="comment 1"
date="2012-12-17T16:21:36Z"
content="""
For importing your photos, you can use `git annex import /path/to/some/photos` .. this will move them from the given location into the current directory, and add them to the annex. It preserves subdirectory structure too.
For using the OSX app at the command line, the only thing that is guaranteed to work properly is to run `/Applications/git-annex.app/Contents/MacOS/runshell`, which will modify your PATH and other environment appropriately.
"""]]

View file

@ -0,0 +1,3 @@
Is there a way to store a file in an annex repo without the assistant trying to commit it? My particular issue is that git annex watch tries to add my .thunderbird folder, and I don't want it in annex. Is that a supported use case?
Also, for my better understanding, does watch keep track of what files it saw last time and only look for new things on startup? Or does it try to commit anything that's not already in git (whether symlink or not)?

View file

@ -0,0 +1,27 @@
[[!comment format=mdwn
username="http://edheil.wordpress.com/"
ip="173.162.44.162"
subject="comment 1"
date="2012-12-16T00:10:39Z"
content="""
wow, are you git annexing your whole home directory? that sounds extreme and problematic, since everything in git annex becomes read-only!!
This how to make git annex ignore something though: make git ignore it in the first place.
Stop the git assistant if it's running.
create a file called \".gitignore\" in the root of your annex directory; add a line to that file for everything you want git (and therefore git annex) to ignore, e.g. \".thunderbird\"
e.g.:
echo '.thunderbird' > '.gitignore'
check that file into git:
git add .gitignore
git commit -m 'check in .gitignore'
Now you should be able to start up git-annex again and have it ignore .thunderbird, or any other path you put in .gitignore
"""]]

View file

@ -0,0 +1,8 @@
[[!comment format=mdwn
username="andy"
ip="108.92.50.62"
subject="Thanks!"
date="2012-12-16T00:31:09Z"
content="""
That does what I needed. I'd forgotten about that. :)
"""]]

View file

@ -0,0 +1,10 @@
[[!comment format=mdwn
username="http://joeyh.name/"
ip="4.153.8.117"
subject="comment 3"
date="2012-12-17T16:32:31Z"
content="""
AFAIK, the assistant does *not* currently honor .gitgnore completely. There are parts of it that do, and places where running git commands to check .gitignore would be very expensive, and so I've TODOed it for later, as I'll need to write my own .gitignore parser.
So, use caution..
"""]]

View file

@ -2,7 +2,7 @@
[[!table format=dsv header=yes data="""
detailed instructions | quick install
[[OSX]] | [download git-annex.app](http://downloads.kitenet.net/git-annex/OSX/git-annex.dmg.bz2) **beta; [[known_problems|/bugs/OSX_app_issues]]**
[[OSX]] | [download git-annex.app](http://downloads.kitenet.net/git-annex/OSX/current/)
[[Linux|linux_standalone]] | [download prebuilt linux tarball](http://downloads.kitenet.net/git-annex/linux/)
[[Debian]] | `apt-get install git-annex`
[[Ubuntu]] | `apt-get install git-annex`

View file

@ -1,14 +1,21 @@
## git-annex.app
For easy installation, [Jimmy Tang](http://www.sgenomics.org/~jtang/)
builds a standalone git-annex.app of the git-annex assistant.
For easy installation, use the
[beta release of git-annex.app](http://downloads.kitenet.net/git-annex/OSX/current/).
* [beta release of git-annex.app](http://downloads.kitenet.net/git-annex/OSX/git-annex.dmg.bz2)
* [daily build of git-annex.app](http://www.sgenomics.org/~jtang/gitbuilder-git-annex-x00-x86_64-apple-darwin10.8.0-binary/ref/master/git-annex.dmg.bz2) ([build logs](http://www.sgenomics.org/~jtang/gitbuilder-git-annex-x00-x86_64-apple-darwin10.8.0-binary/))
Be sure to select the build matching your version of OSX.
This is still a work in progress. See [[/bugs/OSX_app_issues]] for problem
reports.
## autobuilds
[Jimmy Tang](http://www.sgenomics.org/~jtang/) autobuilds
the app. These autobuilds only work on OSX Lion, not Mountain Lion.
* [autobuild of git-annex.app](http://www.sgenomics.org/~jtang/gitbuilder-git-annex-x00-x86_64-apple-darwin10.8.0-binary/ref/master/git-annex.dmg.bz2) ([build logs](http://www.sgenomics.org/~jtang/gitbuilder-git-annex-x00-x86_64-apple-darwin10.8.0-binary/))
* [past builds](http://www.sgenomics.org/~jtang/gitbuilder-git-annex-x00-x86_64-apple-darwin10.8.0-binary/sha1/) -- directories are named from the commitid's
This is known to not work on all OSX systems. [[/bugs/OSX_app_issues]] is collecting reports of problems with it in one place.
## using Brew
<pre>

View file

@ -0,0 +1,19 @@
I worked out how to retroactively annex a large file that had been checked into a git repo some time ago. I thought this might be useful for others, so I am posting it here.
Suppose you have a git repo where somebody had checked in a large file you would like to have annexed, but there are a bunch of commits after it and you don't want to loose history, but you also don't want everybody to have to retrieve the large file when they clone the repo. This will re-write history as if the file had been annexed when it was originally added.
This command works for me, it relies on the current behavior of git which is to use a directory named .git-rewrite/t/ at the top of the git tree for the extracted tree. This will not be fast and it will rewrite history, so be sure that everybody who has a copy of your repo is OK with accepting the new history. If the behavior of git changes, you can specify the directory to use with the -d option. Currently, the t/ directory is created inside the directory you specify, so "-d ./.git-rewrite/" should be roughly equivalent to the default.
Enough with the explanation, on to the command:
<pre>
git filter-branch --tree-filter 'for FILE in file1 file2 file3;do if [ -f "$FILE" ] && [ ! -L "$FILE" ];then git rm --cached "$FILE";git annex add "$FILE";ln -sf `readlink "$FILE"|sed -e "s:^../../::"` "$FILE";fi;done' --tag-name-filter cat -- --all
</pre>
replace file1 file2 file3... with whatever paths you want retroactively annexed. If you wanted bigfile1.bin in the top dir and subdir1/bigfile2.bin to be retroactively annexed try:
<pre>
git filter-branch --tree-filter 'for FILE in bigfile1.bin subdir1/bigfile2.bin;do if [ -f "$FILE" ] && [ ! -L "$FILE" ];then git rm --cached "$FILE";git annex add "$FILE";ln -sf `readlink "$FILE"|sed -e "s:^../../::"` "$FILE";fi;done' --tag-name-filter cat -- --all
</pre>
**If your repo has tags** then you should take a look at the git-filter-branch man page about the --tag-name-filter option and decide what you want to do. By default this will re-write the tags "nearly properly".
You'll probably also want to look at the git-filter-branch man page's section titled "CHECKLIST FOR SHRINKING A REPOSITORY" if you want to free up the space in the existing repo that you just changed history on.

View file

@ -0,0 +1,8 @@
[[!comment format=mdwn
username="http://edheil.wordpress.com/"
ip="173.162.44.162"
subject="comment 1"
date="2012-12-16T00:11:38Z"
content="""
Man, I wish you'd written this a couple weeks ago. :) I was never able to figure that incantation out and ended up unannexing and re-annexing the whole thing to get rid of the file I inadvertently checked into git instead of the annex.
"""]]

View file

@ -84,7 +84,7 @@ Executable git-annex
Build-Depends: hinotify
CPP-Options: -DWITH_INOTIFY
else
if (! os(windows) && ! os(solaris))
if (! os(windows) && ! os(solaris) && ! os(linux))
CPP-Options: -DWITH_KQUEUE
C-Sources: Utility/libkqueue.c
@ -119,7 +119,7 @@ Test-Suite test
old-locale, time, pcre-light, extensible-exceptions, dataenc, SHA,
process, json, HTTP, base (>= 4.5 && < 4.7), monad-control,
transformers-base, lifted-base, IfElse, text, QuickCheck >= 2.1,
bloomfilter, edit-distance, process
bloomfilter, edit-distance, process, SafeSemaphore
Other-Modules: Utility.Touch
Include-Dirs: Utility
C-Sources: Utility/libdiskfree.c

View file

@ -58,8 +58,13 @@ export ORIG_GIT_EXEC_PATH
GIT_EXEC_PATH=$base/git-core
export GIT_EXEC_PATH
ORIG_GIT_TEMPLATE_DIR="$GIT_TEMPLATE_DIR"
export ORIG_GIT_TEMPLATE_DIR
GIT_TEMPLATE_DIR="$base/templates"
export GIT_TEMPLATE_DIR
# Indicate which variables were exported above.
GIT_ANNEX_STANDLONE_ENV="PATH LD_LIBRARY_PATH GIT_EXEC_PATH"
GIT_ANNEX_STANDLONE_ENV="PATH LD_LIBRARY_PATH GIT_EXEC_PATH GIT_TEMPLATE_DIR"
export GIT_ANNEX_STANDLONE_ENV
if [ "$1" ]; then

View file

@ -51,8 +51,13 @@ export ORIG_GIT_EXEC_PATH
GIT_EXEC_PATH=$base
export GIT_EXEC_PATH
ORIG_GIT_TEMPLATE_DIR="$GIT_TEMPLATE_DIR"
export ORIG_GIT_TEMPLATE_DIR
GIT_TEMPLATE_DIR="$base/templates"
export GIT_TEMPLATE_DIR
# Indicate which variables were exported above.
GIT_ANNEX_STANDLONE_ENV="PATH GIT_EXEC_PATH"
GIT_ANNEX_STANDLONE_ENV="PATH GIT_EXEC_PATH GIT_TEMPLATE_DIR"
export GIT_ANNEX_STANDLONE_ENV
if [ "$1" ]; then
@ -60,5 +65,5 @@ if [ "$1" ]; then
shift 1
exec "$cmd" "$@"
else
sh
$SHELL
fi