Merge branch 'master' into incrementalfsck
This commit is contained in:
Joey Hess
commit
7e723d2f56
41 changed files with 620 additions and 28 deletions
|
|
@ -124,14 +124,16 @@ perform relaxed url file = ifAnnexed file addurl geturl
|
|||
next $ return True
|
||||
| otherwise = do
|
||||
headers <- getHttpHeaders
|
||||
ifM (Url.withUserAgent $ Url.check url headers $ keySize key)
|
||||
( do
|
||||
(exists, samesize) <- Url.withUserAgent $ Url.check url headers $ keySize key
|
||||
if exists && samesize
|
||||
then do
|
||||
setUrlPresent key url
|
||||
next $ return True
|
||||
, do
|
||||
warning $ "failed to verify url exists: " ++ url
|
||||
else do
|
||||
warning $ if exists
|
||||
then "url does not have expected file size (use --relaxed to bypass this check) " ++ url
|
||||
else "failed to verify url exists: " ++ url
|
||||
stop
|
||||
)
|
||||
|
||||
addUrlFile :: Bool -> URLString -> FilePath -> Annex Bool
|
||||
addUrlFile relaxed url file = do
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ data StatInfo = StatInfo
|
|||
type StatState = StateT StatInfo Annex
|
||||
|
||||
def :: [Command]
|
||||
def = [command "status" paramPaths seek
|
||||
def = [noCommit $ command "status" paramPaths seek
|
||||
SectionQuery "shows status information about the annex"]
|
||||
|
||||
seek :: [CommandSeek]
|
||||
|
|
|
|||
|
|
@ -241,7 +241,7 @@ inAnnex r key
|
|||
where
|
||||
checkhttp headers = do
|
||||
showChecking r
|
||||
ifM (anyM (\u -> Url.withUserAgent $ Url.check u headers (keySize key)) (keyUrls r key))
|
||||
ifM (anyM (\u -> Url.withUserAgent $ Url.checkBoth u headers (keySize key)) (keyUrls r key))
|
||||
( return $ Right True
|
||||
, return $ Left "not found"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -118,7 +118,7 @@ checkKey' key us = firsthit us (Right False) $ \u -> do
|
|||
#endif
|
||||
DefaultDownloader -> do
|
||||
headers <- getHttpHeaders
|
||||
Right <$> Url.withUserAgent (Url.check u' headers $ keySize key)
|
||||
Right <$> Url.withUserAgent (Url.checkBoth u' headers $ keySize key)
|
||||
where
|
||||
firsthit [] miss _ = return miss
|
||||
firsthit (u:rest) _ a = do
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ module Utility.Url (
|
|||
URLString,
|
||||
UserAgent,
|
||||
check,
|
||||
checkBoth,
|
||||
exists,
|
||||
download,
|
||||
downloadQuiet
|
||||
|
|
@ -32,12 +33,18 @@ type UserAgent = String
|
|||
|
||||
{- Checks that an url exists and could be successfully downloaded,
|
||||
- also checking that its size, if available, matches a specified size. -}
|
||||
check :: URLString -> Headers -> Maybe Integer -> Maybe UserAgent -> IO Bool
|
||||
checkBoth :: URLString -> Headers -> Maybe Integer -> Maybe UserAgent -> IO Bool
|
||||
checkBoth url headers expected_size ua = do
|
||||
v <- check url headers expected_size ua
|
||||
return (fst v && snd v)
|
||||
check :: URLString -> Headers -> Maybe Integer -> Maybe UserAgent -> IO (Bool, Bool)
|
||||
check url headers expected_size = handle <$$> exists url headers
|
||||
where
|
||||
handle (False, _) = False
|
||||
handle (True, Nothing) = True
|
||||
handle (True, s) = expected_size == s
|
||||
handle (False, _) = (False, False)
|
||||
handle (True, Nothing) = (True, True)
|
||||
handle (True, s) = case expected_size of
|
||||
Just _ -> (True, expected_size == s)
|
||||
Nothing -> (True, True)
|
||||
|
||||
{- Checks that an url exists and could be successfully downloaded,
|
||||
- also returning its size if available.
|
||||
|
|
|
|||
3
debian/changelog
vendored
3
debian/changelog
vendored
|
|
@ -12,6 +12,9 @@ git-annex (4.20131003) UNRELEASED; urgency=low
|
|||
appends stuff to it to make a temporary filename. Closes: #725512
|
||||
* status: Fix space leak in local mode, introduced in version 4.20130920.
|
||||
* import: Skip .git directories.
|
||||
* Remove bogus runshell loop check.
|
||||
* addurl: Improve message when adding url with wrong size to existing file.
|
||||
* Fixed handling of URL keys that have no recorded size.
|
||||
|
||||
-- Joey Hess <joeyh@debian.org> Thu, 03 Oct 2013 15:41:24 -0400
|
||||
|
||||
|
|
|
|||
|
|
@ -34,6 +34,36 @@ git-annex folders. Just like that you can share files.
|
|||
|
||||
----
|
||||
|
||||
## Requirements
|
||||
|
||||
For local pairing to work, you must have sshd (ssh server daemon) installed and working on all machines involved. That means you must allow at least local connections to sshd. On most Linux distributions, sshd is packaged in either openssh (openSUSE) or openssh-server (Debian).
|
||||
|
||||
It is highly recommended that you disable root login, disable password login to sshd and just enable key based authentication instead. No one will be able to login without your key.
|
||||
|
||||
To disable root, after installing sshd, edit the sshd config (usually /etc/ssh/sshd_config file) and disable root login by adding:
|
||||
|
||||
PermitRootLogin no
|
||||
|
||||
Restart sshd. See man sshd_config for details.
|
||||
|
||||
To disable password login and enable key based authentication, edit the sshd config (just like above) by uncommenting and changing the following options:
|
||||
|
||||
ChallengeResponseAuthentication no
|
||||
PasswordAuthentication no
|
||||
UsePAM no
|
||||
|
||||
PubkeyAuthentication yes
|
||||
|
||||
Restart sshd. See man sshd_config for details.
|
||||
|
||||
You can also restrict login to your local network only (not allow internet users from trying to log into your computer). Edit the hosts.deny file (usually /etc/hosts.deny) by adding the following:
|
||||
|
||||
sshd : ALL EXCEPT LOCAL
|
||||
|
||||
Do note that restricting login to your local network may or may not block git-annex. Also note that this will not work on Mac OSX because Apple decided to disable this feature and replace it with a crippled version made by Apple.
|
||||
|
||||
## Tips
|
||||
|
||||
Something to keep in mind, especially if pairing doesn't seem to be
|
||||
working, is that the two computers need to be on the same network for this
|
||||
pairing process to work. Sometimes a building will have more than one
|
||||
|
|
|
|||
|
|
@ -0,0 +1,8 @@
|
|||
[[!comment format=mdwn
|
||||
username="https://www.google.com/accounts/o8/id?id=AItOawmVV_nBwlsyCv53BXoJt8YpCX_wZPfzpyo"
|
||||
nickname="Peter"
|
||||
subject="Progress"
|
||||
date="2013-10-10T01:17:08Z"
|
||||
content="""
|
||||
Is there any type of script / tool / patch which does the --fast but with a copy instead of only a hard link? Can someone point me towards how I'm supposed to do this? I'm a technical user, however I don't really fancy having try to go learn the source code of git-annex to fix this really bad flaw :-/
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,104 @@
|
|||
[[!comment format=mdwn
|
||||
username="https://www.google.com/accounts/o8/id?id=AItOawmVV_nBwlsyCv53BXoJt8YpCX_wZPfzpyo"
|
||||
nickname="Peter"
|
||||
subject="Productive Annoyance"
|
||||
date="2013-10-10T04:30:47Z"
|
||||
content="""
|
||||
Ok, so I'm annoyed by this enough (and desperate enough to want to get my data back) that I wrote up a few scripts to help with this. I make no claims regarding how well these will work, but they seem to work with some minimal testing on a Fedora 17 machine.
|
||||
|
||||
READ THROUGH THESE SCRIPTS BEFORE RUNNING THEM TO MAKE SURE YOU ARE OK WITH WHAT THEY ARE DOING!!!
|
||||
|
||||
First, a script to create a bad git-annex: one with missing files (with a few corner case names) after a git unannex. Specify the directory you'd like to make the annex at the top of the file. ALL CONTENTS OF THIS DIRECTORY WILL BE REMOVED!!!
|
||||
|
||||
#!/bin/bash
|
||||
|
||||
#This is the folder you'd like to create and unannex
|
||||
FOLDERTOUNANNEX='/tmp/badAnnex'
|
||||
|
||||
pushd .
|
||||
|
||||
if [ ! -d \"$FOLDERTOUNANNEX\" ] ; then
|
||||
mkdir \"$FOLDERTOUNANNEX\"
|
||||
fi
|
||||
|
||||
cd \"$FOLDERTOUNANNEX\"
|
||||
|
||||
rm -rf *
|
||||
|
||||
mkdir subdir
|
||||
echo \"hi\" > 1one.txt
|
||||
echo \"hi\" > 2two.txt
|
||||
echo \"hi\" > \"3thr re ee.boo\"
|
||||
echo \"hi\" > \"4f o u r.boo\"
|
||||
echo \"hi\" > 5
|
||||
echo \"hi\" > \"6\"
|
||||
echo \"hi\" > \"subdir/7\"
|
||||
echo \"hi\" > \"subdir/8.cat\"
|
||||
echo \"hi\" > \"subdir/9.cat\"
|
||||
|
||||
echo \"* annex.backend=SHA512E\" > .gitattributes
|
||||
|
||||
chmod g-r 5 6
|
||||
chmod o-r 6
|
||||
|
||||
ls -la
|
||||
|
||||
git init
|
||||
git annex init \"stupid\"
|
||||
git annex add *
|
||||
ls -la
|
||||
git annex unannex *
|
||||
ls -la
|
||||
|
||||
popd
|
||||
|
||||
|
||||
Then, a script to recover the files left missing by the above script. Note this might be very slow as it has to generate SHA512 hashes for all the files in your annex. Again, change the paths at the top of this file to work in your environment:
|
||||
|
||||
#!/bin/bash
|
||||
|
||||
#Set this to some place outside your annex, where we can store our hashes while we search for them
|
||||
#It will be fastest if this is on a different physical disk than the annexed folder
|
||||
#You can manually delete the file afterwards
|
||||
HASHFILE='/backup3/tmp.sha'
|
||||
#This is the folder you'd like to unannex
|
||||
FOLDERTOUNANNEX='/tmp/badAnnex'
|
||||
|
||||
|
||||
|
||||
|
||||
HASHLEN=128
|
||||
|
||||
pushd .
|
||||
cd \"$FOLDERTOUNANNEX\"
|
||||
|
||||
find \"$FOLDERTOUNANNEX\" ! -path '*.git*' -exec sha512sum \{\} \; > \"$HASHFILE\"
|
||||
|
||||
find -L \"$FOLDERTOUNANNEX\" -type l | while read BROKENFILE; do
|
||||
POINTSTO=`file \"$BROKENFILE\" | sed -r 's/^.*broken symbolic link to .(.*).$/\1/g'`
|
||||
|
||||
HASH=`echo \"$POINTSTO\" | sed -r \"s/^.*--([^-\/.]{$HASHLEN}).*$/\1/g\"`
|
||||
|
||||
EXT=`echo \"$POINTSTO\" | sed -r \"s/^.*--[^-\/.]{$HASHLEN}(.[^.]+)?$/\1/g\"`
|
||||
|
||||
echo \"-\"
|
||||
echo \"FILE:$BROKENFILE\"
|
||||
echo \"POINTSTO:$POINTSTO\"
|
||||
echo \"HASH:$HASH\"
|
||||
echo \"EXT:$EXT\"
|
||||
|
||||
SOURCEFILE=`grep $HASH $HASHFILE | grep -m 1 \"$EXT\" | sed -r \"s/^.{$HASHLEN} (.*)$/\1/g\"`
|
||||
|
||||
echo \"SOURCEFILE:$SOURCEFILE\"
|
||||
if [ -f \"$SOURCEFILE\" ];
|
||||
then
|
||||
cp --backup --suffix=\"~GIT_ANNEX_IS_DANGEROUS~\" -a \"$SOURCEFILE\" \"$BROKENFILE\"
|
||||
else
|
||||
echo \"ERROR: Cant find sourcefile\"
|
||||
fi
|
||||
done;
|
||||
|
||||
popd
|
||||
|
||||
I have not yet run this repair script on my rather large broken annex. I cannot seem to figure out how to restore file ownership and permissions which seem to have been lost when the second file is just linked to the matching previously annexed file (note: this is visible after \"fixing\" the bad annex created by the first script above in that after \"fixing\" file \"6\" is readable by other, whereas originally he was NOT readable by other. The permissions of 6 have been copied from 5.) Any thoughts or improvements on this are appreciated.
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
### Please describe the problem.
|
||||
|
||||
As described in [[tips/Internet_Archive_via_S3]], there is a problem using S3 with buckets that have capital letters. The bug lies either in the hS3 library or in archive.org itself.
|
||||
|
||||
### What steps will reproduce the problem?
|
||||
|
||||
Try to add an [[special_remotes/S3]] remote with capital letters in the bucket name.
|
||||
|
||||
### What version of git-annex are you using? On what operating system?
|
||||
|
||||
[[!format txt """
|
||||
git-annex version: 4.20130921-g434dc22
|
||||
build flags: Assistant Webapp Pairing Testsuite S3 WebDAV Inotify DBus XMPP DNS Feeds Quvi
|
||||
local repository version: 3
|
||||
default repository version: 3
|
||||
supported repository versions: 3 4
|
||||
upgrade supported from repository versions: 0 1 2
|
||||
"""]]
|
||||
|
||||
### Please provide any additional information below.
|
||||
|
||||
[[!format sh """
|
||||
# If you can, paste a complete transcript of the problem occurring here.
|
||||
# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
|
||||
anarcat@angela:video$ git annex initremote archive-moglenrepublica type=S3 host=s3.us.archive.org bucket=Republica2012-EbenMoglen-FreedomOfThoughtRequiresFreeMedia
|
||||
initremote archive-moglenrepublica (Internet Archive mode) git-annex: The request signature we calculated does not match the signature you provided. Check your AWS Secret Access Key and signing method. For more information, see REST Authentication and SOAP Authentication for details.
|
||||
# End of transcript or log.
|
||||
"""]]
|
||||
|
||||
Just thought it would be better to have a separate thread for this bug. :)
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
### What steps will reproduce the problem?
|
||||
[[!format sh """
|
||||
C:\Users\Bruno>mkdir annex
|
||||
|
||||
C:\Users\Bruno>cd annex
|
||||
|
||||
C:\Users\Bruno\annex>git init
|
||||
Initialized empty Git repository in C:/Users/Bruno/annex/.git/
|
||||
|
||||
C:\Users\Bruno\annex>git annex init
|
||||
init
|
||||
Detected a crippled filesystem.
|
||||
|
||||
Enabling direct mode.
|
||||
|
||||
Detected a filesystem without fifo support.
|
||||
|
||||
Disabling ssh connection caching.
|
||||
ok
|
||||
(Recording state in git...)
|
||||
|
||||
C:\Users\Bruno\annex>echo test > test
|
||||
|
||||
C:\Users\Bruno\annex>git annex add .
|
||||
add test (checksum...) ok
|
||||
(Recording state in git...)
|
||||
|
||||
C:\Users\Bruno\annex>git commit -a -m added
|
||||
[master (root-commit) 2eea610] added
|
||||
1 file changed, 1 insertion(+)
|
||||
create mode 120000 test
|
||||
|
||||
C:\Users\Bruno\annex>git annex sync
|
||||
(Recording state in git...)
|
||||
fatal: unable to access '../../../../C:\Users\Bruno\annex\.git/config': Invalid argument
|
||||
|
||||
git-annex: user error (xargs ["-0","git","--git-dir=C:\\Users\\Bruno\\annex\\.git","--work-tree=C:\\Users\\Bruno\\annex","add","-f"] exited 123)
|
||||
failed
|
||||
git-annex: sync: 1 failed
|
||||
"""]]
|
||||
|
||||
### What version of git-annex are you using? On what operating system?
|
||||
Windows 8 (64 bits)
|
||||
|
||||
git version 1.8.4.msysgit.0
|
||||
|
||||
[[!format sh """
|
||||
git-annex version: 4.20131008-ge115441
|
||||
build flags: Pairing Testsuite S3 WebDAV DNS Feeds Quvi TDFA CryptoHash
|
||||
key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL
|
||||
remote types: git gcrypt S3 bup directory rsync web webdav glacier hook
|
||||
local repository version: 4
|
||||
default repository version: 3
|
||||
supported repository versions: 3 4
|
||||
upgrade supported from repository versions: 2
|
||||
"""]]
|
||||
|
||||
### Please provide any additional information below.
|
||||
C:\Users\Bruno\annex\.git\config exists
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
[[!comment format=mdwn
|
||||
username="https://www.google.com/accounts/o8/id?id=AItOawknwkXgi8SnK4QT32ANl3GMKvFLyQGeHqo"
|
||||
nickname="Florian"
|
||||
subject="comment 7"
|
||||
date="2013-10-08T16:18:07Z"
|
||||
content="""
|
||||
Any comment from on this from one of the maintainers would be great. Apparently this was not fixed in the latest (2013-10-02 18:46) build.
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
### Please describe the problem.
|
||||
|
||||
When cloning/syncing a repository (and probably doing some nono's in the process),
|
||||
git annex will happily delete files.
|
||||
This cost me several files, which just by coincidence were not totally important, still its *very* unsettling.
|
||||
|
||||
### What steps will reproduce the problem?
|
||||
|
||||
I did not try to exactly reproduce it yet (sorry, no time right now), but here is vaguley what I did (sorry its been a process spread over
|
||||
several hours and i was doing lots of other things in parallel so I'm fuzzy about details):
|
||||
|
||||
* have a repository in direct mode on your local harddrive, say ~/myannex
|
||||
* git clone ~/myannex to /usbhd/myannex, git annex init.
|
||||
The usbhd is a FAT, git annex recognizes it as "crippled filessytem".
|
||||
* Git annex get all from ~/myannex. So far, so good.
|
||||
* create several files on ~/myannex, git annex add them
|
||||
* do a git annex add on them, abort it (realizing SHA256E takes forever, so changing to WORM), repeat add
|
||||
(not sure wheter i did a git annex sync here)
|
||||
* create several files on /usbhd/myannex, git annex add them
|
||||
(not sure wheter i did a git annex sync here again)
|
||||
|
||||
All repos are in direct mode.
|
||||
|
||||
From here on i don't remember the exact order of events, one definite -- probably important -- nono i did,
|
||||
was:
|
||||
I executed a git annex sync/get on the usbhd in a sub-directory (i.e. not in the git base dir), say /usbhd/myannex/foo/bar/,
|
||||
so it went on creating /usbhd/myannex/foo/bar/foo, which of course was not intended.
|
||||
However /usbhd/myannex/foo/bar/foo contained FAT-crippled-symlinks to the new files in ~/myannex (good).
|
||||
|
||||
In order to avoid a potential messy situation i just renamed /usbhd/myannex/foo to /usbhd/myannex/foo_bak
|
||||
(which I just realize while writing this, saved me the hash values of my files, yay :))
|
||||
|
||||
However when I tried to repeat the procedure, it seems that the new files would not appear on the usbhd (for a reason i totally don't get, maybe I synced back before realizing my sub-foo mistake).
|
||||
When I synced usbhd -> ~/myannex again, git-annex happily deleted my new files there which obviously quite upset me.
|
||||
|
||||
Again, sorry for this horribly chaotic description,
|
||||
I'll try and deliver a more reproducible description, but for the next 2 weeks at least I'm too busy for that.
|
||||
|
||||
|
||||
|
||||
### What version of git-annex are you using? On what operating system?
|
||||
|
||||
Archlinux,
|
||||
3.9.2-1-ARCH #1 SMP PREEMPT Sat May 11 20:31:08 CEST 2013 x86_64 GNU/Linux
|
||||
|
||||
aur/git-annex-bin 4.20131002-1
|
||||
|
||||
### Please provide any additional information below.
|
||||
|
||||
[[!format sh """
|
||||
# If you can, paste a complete transcript of the problem occurring here.
|
||||
# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
|
||||
|
||||
|
||||
# End of transcript or log.
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
[[!comment format=mdwn
|
||||
username="http://droggl.myopenid.com/"
|
||||
ip="92.76.150.86"
|
||||
subject="comment 1"
|
||||
date="2013-10-08T07:46:35Z"
|
||||
content="""
|
||||
Just in case you wonder: the most important (in the sense of not backupped elswhere) files where some scripts which i did hash with SHA256E and not with WORM.
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
[[!comment format=mdwn
|
||||
username="http://cstork.org/"
|
||||
nickname="Chris Stork"
|
||||
subject="git annex get/sync don't delete files"
|
||||
date="2013-10-10T11:43:29Z"
|
||||
content="""
|
||||
AFAIU it's impossible for the 'get' subcommand to delete anything because it only copies files to .git/annex/objects.
|
||||
|
||||
'sync' should also never delete files because it only adds files to the annex (i.e. checksums them and sets up the links and directories in .git/annex), commits things to git and pulls/pushes the git-annex specific branches to/from other repos.
|
||||
|
||||
So I think some other 'nono' must have messed up your files.
|
||||
|
||||
If you ever used (maybe unintentionally) used indirect mode there seem to be a rather high chance that some of your otherwise lost content is still in .git/annex/objects. Did you check if there's are any files in .git/annex/objects ?
|
||||
"""]]
|
||||
24
doc/bugs/webapp:_difficult_to_abort_adding_a_repository.mdwn
Normal file
24
doc/bugs/webapp:_difficult_to_abort_adding_a_repository.mdwn
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
### Please describe the problem.
|
||||
I could not find a way to abort the addition of a new remote repository.
|
||||
|
||||
### What steps will reproduce the problem?
|
||||
- start adding a remote repository (unencrypted, with git-annex installed);
|
||||
- forget to create the folder on the remote host;
|
||||
- navigate away from the repository page;
|
||||
- the dashboard says the repository is partially set-up, and the only thing one can do is look at the log (which says the folder is missing).
|
||||
|
||||
I was able to solve it by creating another repository with the exact same data.
|
||||
|
||||
### What version of git-annex are you using? On what operating system?
|
||||
|
||||
Version: 4.20131002-gf25991c on OS X 10.8.5
|
||||
|
||||
### Please provide any additional information below.
|
||||
|
||||
[[!format sh """
|
||||
# If you can, paste a complete transcript of the problem occurring here.
|
||||
# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
|
||||
|
||||
|
||||
# End of transcript or log.
|
||||
"""]]
|
||||
|
|
@ -24,17 +24,17 @@ check that nothing else is using it, fix the problem, and redo the commit.
|
|||
|
||||
## incremental fsck
|
||||
|
||||
Add webapp UI to enable incremental fsck, and choose when to start and how
|
||||
long to run each day, and what the goal is (ie, to fsck all files once
|
||||
per month).
|
||||
TODO: Add webapp UI to enable incremental fsck
|
||||
|
||||
Of course, incremental fsck will run as an niced (and ioniced) background
|
||||
job. There will need to be a button in the webapp to stop it, in case it's
|
||||
annoying.
|
||||
annoying. **done**
|
||||
|
||||
When fsck finds a damanged file, queue a download of the file from a
|
||||
remote. If no accessible remote has the file, prompt the user to eg, connect
|
||||
a drive containing it.
|
||||
remote. **done**
|
||||
|
||||
TODO: If no accessible remote has the file, prompt the user to eg, connect
|
||||
a drive containing it
|
||||
|
||||
## git-annex-shell remote fsck
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ Now in the
|
|||
(starting September 2013).
|
||||
|
||||
* Month 1 [[!traillink assistant/encrypted_git_remotes]]
|
||||
* Month 2 [[!traillink assistant/disaster_recovery]]
|
||||
* **Month 2 [[!traillink assistant/disaster_recovery]]**
|
||||
* Month 3 user-driven features and polishing
|
||||
* Month 4 improve special remote interface & git-annex enhancement contest
|
||||
* Month 5 [[!traillink assistant/xmpp_security]]
|
||||
|
|
|
|||
10
doc/devblog/day_29__scheduling.mdwn
Normal file
10
doc/devblog/day_29__scheduling.mdwn
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
Spent most of the day building some generic types for scheduling recurring
|
||||
events. Not sure if rolling my own was a good idea, but that's what I did.
|
||||
|
||||
In the incrementalfsck branch, I have hooked this up in `git-annex vicfg`,
|
||||
which now accepts and parses scheduled events like
|
||||
"fsck self every day at any time for 60 minutes" and
|
||||
"fsck self on day 1 of weeks divisible by 2 at 3:45 for 120 minutes", and
|
||||
stores them in the git-annex branch. The exact syntax is of course subject
|
||||
to change, but also doesn't matter a whole lot since the webapp will have
|
||||
a better interface.
|
||||
17
doc/devblog/day_30__cronner.mdwn
Normal file
17
doc/devblog/day_30__cronner.mdwn
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
Lots of progress from yesterday's modest start of building data types for
|
||||
scheduling. Last night I wrote the hairy calendar code to calculate when
|
||||
next to run a scheduled event. (This is actually quite superior to `cron`,
|
||||
which checks every second to see if it should run each event!) Today I
|
||||
built a "Cronner" thread that handles spawning threads to handle each
|
||||
scheduled event. It even notices when changes have been made to the its
|
||||
schedule and stops/starts event threads appropriately.
|
||||
|
||||
Everything is hooked up, building, and there's a good chance it works
|
||||
without too many bugs, but while I've tested all the pure code (mostly
|
||||
automatically with quickcheck properties), I have not run the Cronner
|
||||
thread at all. And there is some tricky stuff in there, like noticing
|
||||
that the machine was asleep past when it expected to wake up, and deciding
|
||||
if it should still run a scheduled event, or should wait until next time.
|
||||
So tomorrow we'll see..
|
||||
|
||||
Today's work was sponsored by Ethan Aubin.
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
[[!comment format=mdwn
|
||||
username="http://nullroute.eu.org/~grawity/"
|
||||
nickname="Mantas"
|
||||
subject="comment 1"
|
||||
date="2013-10-09T06:04:15Z"
|
||||
content="""
|
||||
The SysV Unix *cron* learned this same thing [in 1979][1].
|
||||
|
||||
[1]: http://en.wikipedia.org/wiki/Cron#Multi-user_capability
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
[[!comment format=mdwn
|
||||
username="http://joeyh.name/"
|
||||
ip="4.154.4.22"
|
||||
subject="comment 2"
|
||||
date="2013-10-11T17:07:24Z"
|
||||
content="""
|
||||
However, if you strace a traditional cron, you will be sad at how it's implemented. So much statting of files, and checking of entries every second..
|
||||
|
||||
0 polling or redundant calulcation in my code!
|
||||
"""]]
|
||||
17
doc/devblog/day_31__blah.mdwn
Normal file
17
doc/devblog/day_31__blah.mdwn
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
Some neat stuff is coming up, but today was a pretty blah day for me.
|
||||
I did get the Cronner tested and working (only had a few little bugs). But
|
||||
I got stuck for quite a while making the Cronner stop `git-annex fsck`
|
||||
processes it was running when their jobs get removed. I had some code to do
|
||||
this that worked when run standalone, but not when run from git-annex.
|
||||
|
||||
After considerable head-scratching, I found out this was due to
|
||||
`forkProcess` masking aync exceptions, which seems to be probably
|
||||
[a bug](http://ghc.haskell.org/trac/ghc/ticket/8433). Luckily was able to
|
||||
work around it. Async exceptions continue to strike me as the worst part of
|
||||
the worst part of Haskell (the worst part being exceptions in general).
|
||||
|
||||
Was more productive after that.. Got the assistant to automatically queue
|
||||
re-downloads of any files that fsck throws out due to having bad contents,
|
||||
and made the webapp display an alert while fscking is running, which will
|
||||
go to the page to configure fsck schedules. Now all I need to do is
|
||||
build the UI of that page.
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
Hello,
|
||||
|
||||
I went through several of the questions on the forum but could not find the answer to this. I have the following scenario, which seems very similar to the nomad case.
|
||||
|
||||
I have a server at home with many big media files. I want to keep a subset of these files on my laptop, to watch when I travel. When I'm done watching the files, I want to either delete them (i.e, they should no longer be on any machine) or archive them (i.e., they should remain on the server, but no longer be on the laptop). I think I know how to do this from the command line, but I still have a few questions.
|
||||
|
||||
- How do I drop a file from _every_ repository? If I understood correctly, `git annex drop foo` will only drop foo locally.
|
||||
- Is it possible to follow this use case using the assistant, or should I use the command line instead (which would be very fine with me)? In particular, how do I choose the file I want to put on the laptop? (It seems `git annex get` is the command line way, is there an assistant equivalent?)
|
||||
- Can I use both the assistant and the command line at the same time? Is there a description of the command-line version of what the assistant is doing behind the scenes?
|
||||
- Can the server repository be both a "transfer" repository (to hold the files I haven't watch) and an "archive" repository (to keep the files I've watched and decided to keep), or do I need two repositories (on the server) for that?
|
||||
|
||||
Thanks a lot for any suggestion.
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
[[!comment format=mdwn
|
||||
username="ringprince"
|
||||
ip="134.76.140.110"
|
||||
subject="comment 1"
|
||||
date="2013-10-10T07:08:21Z"
|
||||
content="""
|
||||
I hope it is. I'd love to use it.
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
[[!comment format=mdwn
|
||||
username="https://id.koumbit.net/anarcat"
|
||||
ip="72.0.72.144"
|
||||
subject="comment 2"
|
||||
date="2013-10-10T09:22:00Z"
|
||||
content="""
|
||||
it is, please publish.
|
||||
"""]]
|
||||
22
doc/forum/git_annex_sync_dies___40__sometimes__41__.mdwn
Normal file
22
doc/forum/git_annex_sync_dies___40__sometimes__41__.mdwn
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
I've set up git annex on my laptop and on a remote server using the gitolite v3 git-annex branch.
|
||||
When I 'git annex sync' from my laptop (mac OSX) to the server, and 'git annex copy . --to server' all works fine.
|
||||
|
||||
Later, I tried to clone the repository on another machine (linux 2.6.32). again all is well. I 'git annex init' in the new clone and then try to 'git annex sync' -- now I run into problems.
|
||||
|
||||
Specifically, the first request for the configlist (when I do 'git annex --debug sync' all is fine until the configlist request) somehow breaks and my ssh session shows:
|
||||
|
||||
muxserver_listen bind(): Input/output error
|
||||
|
||||
This seems to confuse the client, who now believes that git annex is not installed on the server (it is).
|
||||
|
||||
If I issue the same command as git annex via ssh, all seems to work ok with the caveats below:
|
||||
|
||||
I have a feeling it's related to the -o 'ControlPersist=yes' argument that git-annex appears to be giving ssh. If I include this option when I run via ssh, ssh dies:
|
||||
|
||||
>ssh git@server -o "ControlMaster=auto" -o "ControlPersist=yes" "git-annex-shell 'configlist' '/~/PRJ'"
|
||||
command-line: line 0: Bad configuration option: ControlPersist
|
||||
|
||||
|
||||
(note that it appears to be ok to leave "ControlMaster=auto" in).
|
||||
|
||||
any thoughts?
|
||||
|
|
@ -31,4 +31,4 @@ of Bath CS department.
|
|||
Eskild Hustvedt has contributed a
|
||||
[gitannex-install](https://github.com/zerodogg/scriptbucket/blob/master/gitannex-install)
|
||||
script to manage keeping up to date with new releases using the standalone
|
||||
built.
|
||||
build.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,11 @@
|
|||
[[!comment format=mdwn
|
||||
username="https://openid.stackexchange.com/user/2eeca722-8d1c-41fe-8fda-13af7eaff6ae"
|
||||
nickname="Diggory"
|
||||
subject="Outdated in 13.04"
|
||||
date="2013-10-08T16:36:10Z"
|
||||
content="""
|
||||
13.04 Raring also has fairly outdated packages: 3.20121112ubuntu4
|
||||
Is there a PPA or some such for this? I suppose since Saucy will be out soon this is less critical, but would still be useful.
|
||||
|
||||
These don't have direct mode, which (I think; I haven't tested it yet) is the feature I've been waiting for.
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
[[!comment format=mdwn
|
||||
username="https://www.google.com/accounts/o8/id?id=AItOawlkA6XinbeOdnEDxEGQUWyjqPGh0kdMXr4"
|
||||
nickname="Blake"
|
||||
subject="Updating git-annex via cabal"
|
||||
date="2013-10-09T21:21:22Z"
|
||||
content="""
|
||||
What is the best way to update git-annex and the dependencies via cabal? Just the `$ cabal install *` commands again?
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
[[!comment format=mdwn
|
||||
username="http://joeyh.name/"
|
||||
ip="4.154.4.22"
|
||||
subject="comment 21"
|
||||
date="2013-10-11T17:09:33Z"
|
||||
content="""
|
||||
If you run the \"cabal update\" command followed by the \"cabal install\" command it'll upgrade git-annex.
|
||||
"""]]
|
||||
27
doc/news/git_annex_fall_of_code.mdwn
Normal file
27
doc/news/git_annex_fall_of_code.mdwn
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
Thanks to my recent successful git-annex
|
||||
[crowdfunding campaign](http://campaign.joeyh.name/), $1000 has been set
|
||||
aside to award others for their contributions to git-annex by
|
||||
the end of 2013. This is not a lot of money, but I hope that the
|
||||
reward and recognition will encourage some more people to work on git-annex.
|
||||
|
||||
You don't need to know Haskell! You could contribute some interesting and
|
||||
useful special remote hooks, or write better documentation.
|
||||
|
||||
On the other hand, if you have learned Haskell, this might be your first
|
||||
chance to make some money with it! Some ideas for things to work on that
|
||||
would be particularly award-worthy:
|
||||
|
||||
* Port git-annex to IOS, using [ghc-ios](https://github.com/ghc-ios).
|
||||
I don't plan to ever do this myself, but you could make it happen.
|
||||
* Get the git-annex assistant working in Windows, building on the existing
|
||||
port to Windows. Otherwise, this is on my roadmap for February.
|
||||
* Solve existing [[bugs]] or [[todo]] items. There are more than I can
|
||||
keep up with.
|
||||
|
||||
The amount of the awards will vary depending on the size of the
|
||||
contribution. Since this is getting started later than is ideal,
|
||||
I reserve the right to extend it past the end of the year if I don't get
|
||||
enough participants.
|
||||
|
||||
If you'd like to participate in this program, just email me at
|
||||
<id@joeyh.name>. --[[Joey]]
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
[[!comment format=mdwn
|
||||
username="http://joeyh.name/"
|
||||
ip="4.154.4.22"
|
||||
subject="comment 10"
|
||||
date="2013-10-11T17:12:00Z"
|
||||
content="""
|
||||
Georg, `drop --auto` will only drop files that are not preferred content. I'd need to know what preferred content expression you're using to say more.
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
[[!comment format=mdwn
|
||||
username="https://www.google.com/accounts/o8/id?id=AItOawkC0W3ZQERUaTkHoks6k68Tsp1tz510nGo"
|
||||
nickname="Georg"
|
||||
subject="drop only files located in archive"
|
||||
date="2013-10-08T17:53:08Z"
|
||||
content="""
|
||||
Is there a way to drop only the files that are located in an \"archive\" directory?
|
||||
I want to drop all files when calling
|
||||
|
||||
git annex drop --auto
|
||||
|
||||
if I move them to the archive. But I want to keep the files that are outside of the archive, even if they are already present in other repos. As far as I have seen and tested, as soon as I have the files in an other repo all files get dropped, also the ones outside the archive directory. Or do I have to increase \"numcopies\" in order to circumvent the \"(not copies=semitrusted+:1)\" case?
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
[[!comment format=mdwn
|
||||
username="http://elmimmo.myopenid.com/"
|
||||
nickname="chocolate.camera"
|
||||
subject="How to sync content with git-annex, not assistant"
|
||||
date="2013-10-11T09:58:12Z"
|
||||
content="""
|
||||
Sure assistant can sync git-annex data across remotes. But how do I tell a repo to sync git-annex data, but not so manually as to having to know what exactly needs to be copied from/to where?
|
||||
"""]]
|
||||
|
|
@ -31,7 +31,7 @@ specify `x-archive-meta*` headers to add metadata as explained in their
|
|||
[documentation](http://www.archive.org/help/abouts3.txt).
|
||||
|
||||
[[!template id=note text="""
|
||||
/!\ There seems to be a bug in either hS3 or the archive that breaks
|
||||
/!\ There seems to be a [[bug|bugs/S3 buckets with capital letters breaks authentication]] in either hS3 or the archive that breaks
|
||||
authentication when the bucket name contains spaces or upper-case letters..
|
||||
use all lowercase and no spaces when making the bucket with `initremote`.
|
||||
"""]]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,34 @@
|
|||
[[!comment format=mdwn
|
||||
username="https://id.koumbit.net/anarcat"
|
||||
ip="72.0.72.144"
|
||||
subject="how to use with simply addurl?"
|
||||
date="2013-10-09T22:27:27Z"
|
||||
content="""
|
||||
It doesn't seem like git annex addurl by itself supports the archive.org urls...
|
||||
|
||||
[[!format txt \"\"\"
|
||||
anarcat@marcos:presentations$ git annex addurl --file=re_publica_2012___Eben_Moglen___Freedom_of_Thought_Requires_Free_Media.webm http://archive.org/download/Republica2012-EbenMoglen-FreedomOfThoughtRequiresFreeMedia/re_publica_2012___Eben_Moglen___Freedom_of_Thought_Requires_Free_Media.webm
|
||||
addurl re_publica_2012___Eben_Moglen___Freedom_of_Thought_Requires_Free_Media.webm
|
||||
failed to verify url exists: http://archive.org/download/Republica2012-EbenMoglen-FreedomOfThoughtRequiresFreeMedia/re_publica_2012___Eben_Moglen___Freedom_of_Thought_Requires_Free_Media.webm
|
||||
failed
|
||||
git-annex: addurl: 1 failed
|
||||
\"\"\"]]
|
||||
|
||||
I also tried the \"details\" url (<http://archive.org/details/Republica2012-EbenMoglen-FreedomOfThoughtRequiresFreeMedia>) - but that just downloads the webpage, not the video either...
|
||||
|
||||
Even the ultimate video URL doesn't work:
|
||||
|
||||
[[!format txt \"\"\"
|
||||
anarcat@marcos:presentations$ git annex addurl --debug --file=re_publica_2012___Eben_Moglen___Freedom_of_Thought_Requires_Free_Media.webm http://ia601009.us.archive.org/9/items/Republica2012-EbenMoglen-FreedomOfThoughtRequiresFreeMedia/re_publica_2012___Eben_Moglen___Freedom_of_Thought_Requires_Free_Media.webm
|
||||
[2013-10-09 18:26:30 EDT] call: quvi [\"-v\",\"mute\",\"--support\",\"http://ia601009.us.archive.org/9/items/Republica2012-EbenMoglen-FreedomOfThoughtRequiresFreeMedia/re_publica_2012___Eben_Moglen___Freedom_of_Thought_Requires_Free_Media.webm\"]
|
||||
addurl re_publica_2012___Eben_Moglen___Freedom_of_Thought_Requires_Free_Media.webm [2013-10-09 18:26:30 EDT] read: curl [\"-s\",\"--head\",\"-L\",\"http://ia601009.us.archive.org/9/items/Republica2012-EbenMoglen-FreedomOfThoughtRequiresFreeMedia/re_publica_2012___Eben_Moglen___Freedom_of_Thought_Requires_Free_Media.webm\",\"-w\",\"%{http_code}\"]
|
||||
|
||||
failed to verify url exists: http://ia601009.us.archive.org/9/items/Republica2012-EbenMoglen-FreedomOfThoughtRequiresFreeMedia/re_publica_2012___Eben_Moglen___Freedom_of_Thought_Requires_Free_Media.webm
|
||||
failed
|
||||
git-annex: addurl: 1 failed
|
||||
\"\"\"]]
|
||||
|
||||
... even though that URL actually gives out a proper 200 OK response code.
|
||||
|
||||
Any ideas? --[[anarcat]]
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
[[!comment format=mdwn
|
||||
username="http://joeyh.name/"
|
||||
ip="4.154.4.22"
|
||||
subject="comment 2"
|
||||
date="2013-10-11T17:08:27Z"
|
||||
content="""
|
||||
This was a misleading error message. The url you are trying to add to the file does not match the size recorded for the file already in the annex. (Or possibly the file's key has no recorded size). If you really want to add the url to the file despite it being a different encoding, you can use --relaxed, although fsck may not like the result if you ever end up downloading that url..
|
||||
|
||||
(Please file bug reports for problems in the future, rather than posting comments on only vaguely related pages which as we can see here can turn out to be entirely offtopic.)
|
||||
"""]]
|
||||
|
|
@ -8,10 +8,14 @@ Here are some ways you can use this awesome stuff..
|
|||
|
||||
[[!toc ]]
|
||||
|
||||
This page will show how to set it up at the command line, but the git-annex
|
||||
[[assistant]] can also be used to help you set up encrypted git
|
||||
repositories.
|
||||
|
||||
## prerequisites
|
||||
|
||||
* Install
|
||||
[git-remote-gcrypt](https://github.com/blake2-ppc/git-remote-gcrypt/)
|
||||
[git-remote-gcrypt](https://github.com/joeyh/git-remote-gcrypt/)
|
||||
* Install git-annex version 4.20130909 or newer.
|
||||
|
||||
## encrypted backup drive
|
||||
|
|
|
|||
|
|
@ -19,9 +19,6 @@ if [ ! -e "$base/bin/git" ]; then
|
|||
echo "** base directory $base does not contain bin/git" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ -n "$GIT_ANNEX_STANDLONE_ENV" ]; then
|
||||
echo "** runshell loop detected!" >&2
|
||||
fi
|
||||
|
||||
# Get absolute path to base, to avoid breakage when things change directories.
|
||||
orig="$(pwd)"
|
||||
|
|
|
|||
|
|
@ -21,9 +21,6 @@ if [ ! -e "$bundle/git" ]; then
|
|||
echo "** bundle directory $bundle does not contain git" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ -n "$GIT_ANNEX_STANDLONE_ENV" ]; then
|
||||
echo "** runshell loop detected!" >&2
|
||||
fi
|
||||
|
||||
# Get absolute path to base, to avoid breakage when things change directories.
|
||||
orig="$(pwd)"
|
||||
|
|
|
|||
Loading…
Reference in a new issue