mirrors/git-annex
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Merge branch 'master' into smudge

Browse source
This commit is contained in:
Joey Hess 2015-12-07 12:29:34 -04:00
commit 62a2fba1cd
Failed to extract signature
23 changed files with 283 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

5
Annex/ReplaceFile.hs
View file

@ -27,9 +27,10 @@ replaceFile :: FilePath -> (FilePath -> Annex ()) -> Annex ()
replaceFile file action = do replaceFile file action = do
misctmpdir <- fromRepo gitAnnexTmpMiscDir misctmpdir <- fromRepo gitAnnexTmpMiscDir
void $ createAnnexDirectory misctmpdir void $ createAnnexDirectory misctmpdir
let basetmp = takeFileName file filemax <- liftIO $ fileNameLengthLimit misctmpdir
let basetmp = take (filemax `div` 2) (takeFileName file)
withTmpDirIn misctmpdir basetmp $ \tmpdir -> do withTmpDirIn misctmpdir basetmp $ \tmpdir -> do
let tmpfile = tmpdir <> basetmp let tmpfile = tmpdir </> basetmp
action tmpfile action tmpfile
liftIO $ replaceFileFrom tmpfile file liftIO $ replaceFileFrom tmpfile file

2
Assistant/Threads/XMPPClient.hs
View file

@ -7,7 +7,7 @@
module Assistant.Threads.XMPPClient where module Assistant.Threads.XMPPClient where
import Assistant.Common import Assistant.Common hiding (ProtocolError)
import Assistant.XMPP import Assistant.XMPP
import Assistant.XMPP.Client import Assistant.XMPP.Client
import Assistant.NetMessager import Assistant.NetMessager

2
Assistant/WebApp/Common.hs
View file

@ -13,5 +13,5 @@ import Assistant.WebApp.Page as X
import Assistant.WebApp.Form as X import Assistant.WebApp.Form as X
import Assistant.WebApp.Types as X import Assistant.WebApp.Types as X
import Assistant.WebApp.RepoId as X import Assistant.WebApp.RepoId as X
import Utility.Yesod as X hiding (textField, passwordField, insertBy, replace, joinPath, deleteBy, delete, insert, Key, Option) import Utility.Yesod as X hiding (textField, passwordField, insertBy, replace, joinPath, deleteBy, delete, insert, Key, Option, PermissionDenied)
import Data.Text as X (Text) import Data.Text as X (Text)

19
Backend/Hash.hs
View file

@ -108,17 +108,16 @@ selectExtension f
{- A key's checksum is checked during fsck. -} {- A key's checksum is checked during fsck. -}
checkKeyChecksum :: Hash -> Key -> FilePath -> Annex Bool checkKeyChecksum :: Hash -> Key -> FilePath -> Annex Bool
checkKeyChecksum hash key file = go `catchHardwareFault` hwfault checkKeyChecksum hash key file = catchIOErrorType HardwareFault hwfault $ do
fast <- Annex.getState Annex.fast
mstat <- liftIO $ catchMaybeIO $ getFileStatus file
case (mstat, fast) of
(Just stat, False) -> do
filesize <- liftIO $ getFileSize' file stat
showAction "checksum"
check <$> hashFile hash file filesize
_ -> return True
where where
go = do
fast <- Annex.getState Annex.fast
mstat <- liftIO $ catchMaybeIO $ getFileStatus file
case (mstat, fast) of
(Just stat, False) -> do
filesize <- liftIO $ getFileSize' file stat
showAction "checksum"
check <$> hashFile hash file filesize
_ -> return True
expected = keyHash key expected = keyHash key
check s check s
| s == expected = True | s == expected = True

15
Utility/Exception.hs
View file

@ -20,7 +20,8 @@ module Utility.Exception (
catchNonAsync, catchNonAsync,
tryNonAsync, tryNonAsync,
tryWhenExists, tryWhenExists,
catchHardwareFault, catchIOErrorType,
IOErrorType(..)
) where ) where
import Control.Monad.Catch as X hiding (Handler) import Control.Monad.Catch as X hiding (Handler)
@ -88,11 +89,11 @@ tryWhenExists a = do
v <- tryJust (guard . isDoesNotExistError) a v <- tryJust (guard . isDoesNotExistError) a
return (eitherToMaybe v) return (eitherToMaybe v)
{- Catches only exceptions caused by hardware faults. {- Catches only IO exceptions of a particular type.
- Ie, disk IO error. -} - Ie, use HardwareFault to catch disk IO errors. -}
catchHardwareFault :: MonadCatch m => m a -> (IOException -> m a) -> m a catchIOErrorType :: MonadCatch m => IOErrorType -> (IOException -> m a) -> m a -> m a
catchHardwareFault a onhardwareerr = catchIO a onlyhw catchIOErrorType errtype onmatchingerr a = catchIO a onlymatching
where where
onlyhw e onlymatching e
| ioeGetErrorType e == HardwareFault = onhardwareerr e | ioeGetErrorType e == errtype = onmatchingerr e
| otherwise = throwM e | otherwise = throwM e

1
Utility/Path.hs
View file

@ -288,7 +288,6 @@ fileNameLengthLimit dir = do
if l <= 0 if l <= 0
then return 255 then return 255
else return $ minimum [l, 255] else return $ minimum [l, 255]
where
#endif #endif
{- Given a string that we'd like to use as the basis for FilePath, but that {- Given a string that we'd like to use as the basis for FilePath, but that

5
Utility/Tmp.hs
View file

@ -88,8 +88,9 @@ withTmpDirIn tmpdir template = bracketIO create remove
makenewdir (tmpdir </> template) (0 :: Int) makenewdir (tmpdir </> template) (0 :: Int)
makenewdir t n = do makenewdir t n = do
let dir = t ++ "." ++ show n let dir = t ++ "." ++ show n
either (const $ makenewdir t $ n + 1) (const $ return dir) catchIOErrorType AlreadyExists (const $ makenewdir t $ n + 1) $ do
=<< tryIO (createDirectory dir) createDirectory dir
return dir
{- It's not safe to use a FilePath of an existing file as the template {- It's not safe to use a FilePath of an existing file as the template
- for openTempFile, because if the FilePath is really long, the tmpfile - for openTempFile, because if the FilePath is really long, the tmpfile

7
debian/changelog vendored
View file

@ -1,5 +1,3 @@
git-annex (6.20151117) UNRELEASED; urgency=medium
* annex.version increased to 6, but version 5 is also still supported. * annex.version increased to 6, but version 5 is also still supported.
* The upgrade to version 6 is not done fully automatically, because * The upgrade to version 6 is not done fully automatically, because
upgrading a direct mode repository to version 6 will prevent old upgrading a direct mode repository to version 6 will prevent old
@ -9,6 +7,9 @@ git-annex (6.20151117) UNRELEASED; urgency=medium
* init: Configure .git/info/attributes to use git-annex as a smudge * init: Configure .git/info/attributes to use git-annex as a smudge
filter. Note that this changes the default behavior of git add in a filter. Note that this changes the default behavior of git add in a
newly initialized repository; it will add files to the annex. newly initialized repository; it will add files to the annex.
git-annex (5.20151117) UNRELEASED; urgency=medium
* Build with -j1 again to get reproducible build. * Build with -j1 again to get reproducible build.
* Display progress meter in -J mode when copying from a local git repo, * Display progress meter in -J mode when copying from a local git repo,
to a local git repo, and from a remote git repo. to a local git repo, and from a remote git repo.
@ -29,6 +30,8 @@ git-annex (6.20151117) UNRELEASED; urgency=medium
This was a reversion caused by the relative path changes in 5.20150113. This was a reversion caused by the relative path changes in 5.20150113.
* dropunused: Make more robust when trying to drop an object that has * dropunused: Make more robust when trying to drop an object that has
already been dropped. already been dropped.
* Fix reversion in handling of long filenames, particularly when using
addurl/importfeed, which was introduced in the previous release.
-- Joey Hess <id@joeyh.name> Mon, 16 Nov 2015 16:49:34 -0400 -- Joey Hess <id@joeyh.name> Mon, 16 Nov 2015 16:49:34 -0400

2
doc/bugs/20151116_tests_fail_on_OS_X.mdwn
View file

@ -36,3 +36,5 @@ FAIL (0.29s)
(This could be due to a bug in git-annex, or an incompatability (This could be due to a bug in git-annex, or an incompatability
with utilities, such as git, installed on this system.) with utilities, such as git, installed on this system.)
``` ```
> [[fixed|done]] --[[Joey]]

12
doc/bugs/20151116_tests_fail_on_OS_X/comment_2_59f7803b9a9ee939a8a50605fe5c4682._comment Normal file
View file

@ -0,0 +1,12 @@
[[!comment format=mdwn
username="joey"
subject="""comment 2"""
date="2015-12-06T20:02:49Z"
content="""
Ok, this involves where the test suite is run.
The addurl test adds `file://`pwd`/somefile`, and that will create a file
with a name like `_home_you_sub_dir_path_here_somefile`. Which can easily
exceed filename length limits of 255 bytes.
There was indeed a reversion in addurl's handling of that.
"""]]

28
doc/bugs/Location_change_of_remote_DNS_ignored.mdwn Normal file
View file

@ -0,0 +1,28 @@
### Please describe the problem.
git-annex ignores changing of remote location in .git/config
### What steps will reproduce the problem?
1. Change Hostname of previously working remote so that the existing remote will no longer work and produce "ssh: connect to host <DNS hostname> port 22: Network is unreachable" errors.
2. Stop all running git-annex processes.
3. Edit DNS name in corresponding remote .git/config.
4. Restart git-annex.
5. Turns out: git-annex still uses the old remote's DNS name.
### What version of git-annex are you using? On what operating system?
newest one available.
5.20151116-gbe86081
### Please provide any additional information below.
See following entry in log, occurring plenty often:
[[!format sh """
ssh: connect to host some.unreachable.dns.net port 22: Network is unreachable
rsync: connection unexpec
rsync failed -- run git annex again to resume file transfer
tedly closed (0 bytes received so far) [Receiver]
rsync error: unexplained error (code 255) at io.c(226) [Receiver=3.1.1]
"""]]
### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
Trust me I love git-annex. Had dreams of something like git-annex for almost 10 years. I instantly got stuck on git-annex when I have read the first few sentences about it. Since then, things in my life have changed in a quite serious manner.
THANKS for your help!

8
doc/bugs/Location_change_of_remote_DNS_ignored/comment_1_449bfb3a594f99cf5e71a6e18d6d3dbc._comment Normal file
View file

@ -0,0 +1,8 @@
[[!comment format=mdwn
username="torpidus"
subject="comment 1"
date="2015-12-06T20:29:25Z"
content="""
**self-edit**:
Changed location of .git/config once again as described but instead of re-starting assistant, I ran \"git-annex sync\" which ended with no errors. After that, assistant started correctly and also had the corrected remote location. From my point of view, this thread has been resolved - even though I wonder what changed the value back to the old one in the first place.
"""]]

11
doc/bugs/Watcher_crashed:_.git__92__annex__92__objects__92__...sha256....pdf__92__:_openTempFile:_does_not_exist/comment_3_1ef917c10f944e6c874aa40affae7602._comment Normal file
View file

@ -0,0 +1,11 @@
[[!comment format=mdwn
username="OlivierBerger"
subject="Difficult to tell"
date="2015-12-04T10:21:26Z"
content="""
Sorry, Joey, but I don't have the details of the version which crahed on us, and we updated afterwards : no time to try and reproduce fully : really busy producing videos, and anxious to find a solution on backing up videos reliably (which by the way I solved like this : https://www-public.tem-tsp.eu/~berger_o/weblog/2015/11/26/handling-video-files-produced-for-a-mooc-on-windows-with-git-and-git-annex/).
I don't see however how we could have touched the contents of the .git/ : we were only doing Windows explorer actions : dragging files here and there, renaming dirs, saving stuff in Camtasia, but all inside subdirs of the git clone.
Sorry for the poor quality of the report, but we were in a real hurry and had to find solutions without distracting us from the main topic : video editing :-/
"""]]

7
doc/bugs/Windows_:_git-annex:_failed_to_read_sha_from_git_write-tree/comment_4_2e5791aa8f568f86d0185a553227afa8._comment Normal file
View file

@ -0,0 +1,7 @@
[[!comment format=mdwn
username="OlivierBerger"
subject="Again, difficult to tell"
date="2015-12-04T12:24:06Z"
content="""
Well, I really don't understand either what may have caused the crashes... but the path issue might be a hint. I think our repo was cloned in a path something like : \"C:\Users\Majdi\git\git-annex.tem-tsp.eu\testscpo\" which already takes around 48 chars.
"""]]

36
doc/bugs/git-annex_cannot_connect_to_freenet_cloud___40__webdav__41__.mdwn Normal file
View file

@ -0,0 +1,36 @@
### Please describe the problem.
git-annex's WebDAV support does not like (aka it does not work) the WebDAV server of the freenet cloud.
### What steps will reproduce the problem?
My first attempt was:
WEBDAV_USERNAME='XXX' WEBDAV_PASSWORD='XXX' git annex initremote webdav type=webdav url='https://webmail.freenet.de/webdav' encryption=none
initremote webdav (testing WebDAV server...)
git-annex: WebDAV test failed: StatusCodeException (Status {statusCode = 401, statusMessage = "Unauthorized"}) [("Date","Fri, 04 Dec 2015 12:20:57 GMT"),("Server","Apache/2.2.16 (Debian)"),("WWW-Authenticate","Basic realm=\"MD-Cloud\""),
("Vary","Accept-Encoding"),("Content-Encoding","gzip"),("Content-Length","20"),("Connection","close"),("Content-Type","text/html; charset=iso-8859-15"),("X-Response-Body-Start",""),("X-Request-URL","MKCOL https://webmail.freenet.de:443/webdav/tmp")]
(CJ {expose = []}): user error failed
git-annex: initremote: 1 failed
Ok this fails (what is the error?). However, it does create a folder "tmp" in the "cloud". A second attempt yields another error:
WEBDAV_USERNAME='XXX' WEBDAV_PASSWORD='XXX' git annex initremote webdav type=webdav url='https://webmail.freenet.de/webdav' encryption=none
initremote webdav (testing WebDAV server...)
git-annex: WebDAV test failed: StatusCodeException (Status {statusCode = 501, statusMessage = "Not Implemented"}) [("Date","Fri, 04 Dec 2015 12:21:22 GMT"),("Server","Apache/2.2.16 (Debian)"),("Content-Length","349"),("Connection","close"),
("Content-Type","application/xml; charset=utf-8"),("X-Response-Body-Start","<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<d:error xmlns:d=\"DAV:\" xmlns:s=\"http://sabredav.org/ns\">\n <s:exception>Sabre\\DAV\\Exception\\NotImplemented</s:exception>\n
<s:message>This server is not compatible with OS/X finder. Consider using a different WebDAV client or webserver.</s:message>\n <s:sabredav-version>1.8.6</s:sabredav-version>\n</d:error>\n"),("X-Request-URL","PUT https://webmail.freenet.de:443/webdav
/tmp/git-annex-test")] (CJ {expose = []}): user error failed
git-annex: initremote: 1 failed
which is I guess the same. The WebDAV server does support writing and locking files. I tried writing using davfs2 and locking-unlocking using cadaver. I guess in the end that it's the server's fault, but it would be great to know what exactly fails at this point :).
### What version of git-annex are you using? On what operating system?
git-annex version: 5.20151116-gbe86081
Gentoo Linux
### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
First time trying WebDAV support.

50
doc/bugs/importfeed_fails_on_createSymbolicLink___40__but_ln_-s_works__41__.mdwn Normal file
View file

@ -0,0 +1,50 @@
### Please describe the problem.
`git-annex importfeed` fails to import some feed with the error `createSymbolicLink: invalid argument (File name too long)` in a case in which `ln -s` works. The item of the feed is not added, and the importing aborts at this entry.
This is a regression: some entries that trigger the problem where imported just fine three months ago.
This looks suspiciously like [this bug](../20151116_tests_fail_on_OS_X).
### What steps will reproduce the problem?
In a freshly `git init`-ed and `git-annex init`-ed repo:
[[!format sh """
$ git-annex importfeed --relaxed http://www.rtve.es/api/programas/1938/audios.rss
(checking known urls...) importfeed http://www.rtve.es/api/programas/1938/audios.rss
/tmp/feed1907 100%[================================================================>] 91,96K 31,4KB/s en 2,9s
addurl Documentos_RNE/Documentos_RNE___La_guerra__un_recorrido_por_la_historia_de_la_peor_lacra_de_la_Humanidad___05_1215.mp3 ok
addurl Documentos_RNE/Documentos_RNE___El_proceso_de_Burgos._El_juicio_contra_ETA_que_arrinconó_al_Franquismo___28_11_15.mp3 ok
addurl Documentos_RNE/Documentos_RNE___Carmen_Martín_Gaite__la_escritura__como_afición_y_refugio___21_11_15.mp3 ok
addurl Documentos_RNE/Documentos_RNE___El_crimen_del_Expreso_de_Andalucía___14_11_15.mp3 ok
addurl Documentos_RNE/Documentos_RNE___La_Belle_Époque._La_Europa_que_bailaba_al_borde_del_abismo___07_11_15.mp3 ok
addurl Documentos_RNE/Documentos_RNE___Cruz_Roja_Española._Siglo_y_medio_de_labor_humanitaria__de_los_heridos_de_guerra_a_los_excluidos_de_la_crisis___31_10_15.mp3
git-annex: ../.git/annex/objects/F7/x3/URL--http&c%%mvod.lvlt.rtve.es%resour-cb7157debded21ea4ab1c330aaa54b42/URL--http&c%%mvod.lvlt.rtve.es%resour-cb7157debded21ea4ab1c330aaa54b42: createSymbolicLink: invalid argument (File name too long)
failed
(recording state in git...)
git-annex: importfeed: 1 failed
"""]]
Just to make sure, I checked that the link can be created:
[[!format sh """
$ ln -s '../.git/annex/objects/F7/x3/URL--http&c%%mvod.lvlt.rtve.es%resour-cb7157debded21ea4ab1c330aaa54b42/URL--http&c%%mvod.lvlt.rtve.es%resour-cb7157debded21ea4ab1c330aaa54b42' Documentos_RNE/Documentos_RNE___Cruz_Roja_Española._Siglo_y_medio_de_labor_humanitaria__de_los_heridos_de_guerra_a_los_excluidos_de_la_crisis___31_10_15.mp3
"""]]
I also checked in ghci that System.Posix.Files.createSymbolicLink succeeds in creating the same link.
(But I didn’t compile git-annex myself, so I’m not sure that the version of the lib used in the debian package is the same as the one I checked; and I’m not sure whether git-annex uses exactly that function or another with the same name, by the way).
### What version of git-annex are you using? On what operating system?
Debian amd64 unstable package 5.20151116-1.
(I also tried version i386, with the same result).
### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
That’s just a little hiccup in, up to now, various months of merry use! ;-)
> [[fixed|done]] --[[Joey]]

24
doc/bugs/importfeed_fails_on_createSymbolicLink___40__but_ln_-s_works__41__/comment_1_c831256a081b181a121910aca3151e45._comment Normal file
View file

@ -0,0 +1,24 @@
[[!comment format=mdwn
username="joey"
subject="""comment 1"""
date="2015-12-06T19:50:59Z"
content="""
I can reproduce this.
symlink("../.git/annex/objects/fK/9M/URL--http&c%%mvod.lvlt.rtve.es%resour-39e58395b01e2385d46bade56127cfc4/URL--http&c%%mvod.lvlt.rtve.es%resour-39e58395b01e2385d46bade56127cfc4", ".git/annex/misctmp/Documentos_RNE___Coss\303\255o_y_la_Casona_de_Tudanca__refugio_de_la_cultura_espa\303\261ola_entre_las_monta\303\261as_de_Cantabria___28_08_15.mp3.0Documentos_RNE___Coss\303\255o_y_la_Casona_de_Tudanca__refugio_de_la_cultura_espa\303\261ola_entre_las_monta\303\261as_de_Cantabria___28_08_15.mp3") = -1 ENAMETOOLONG (File name too long)
The problem is not the length of the link target, which is only 170
characters and will work on any OS. The basename of the symlink
being created is pretty long, 294 characters, and that's the
cause of the failure.
joey@darkstar:~>ln -s /dev/null $(perl -e 'print ("a" x 294)')
ln: failed to create symbolic link ‘aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa’ -> ‘/dev/null’: File name too long
The limit is 255 characters in a single path component (`pathconf _PC_NAME_MAX`).
So, the issue is caused by the rss feed having a long title for this
item.
And this used to work; it's a recent reversion. Fixing.
"""]]

13
doc/coding_style.mdwn
View file

@ -98,3 +98,16 @@ If you feel that this coding style leads to excessive amounts of horizontal
or vertical whitespace around your code, making it hard to fit enough of it or vertical whitespace around your code, making it hard to fit enough of it
on the screen, consider finding a better abstraction, so the code that on the screen, consider finding a better abstraction, so the code that
does fit on the screen is easily understandable. ;) does fit on the screen is easily understandable. ;)
-----
Note for emacs users: You can put the following snippet into a file called
`.dir-locals.el` at root of git-annex's source tree to use tabs for indentation:
((nil . ((indent-tabs-mode . t)
(tab-width . 8)
(fill-column . 80)))
;; Warn about spaces used for indentation:
(haskell-mode . ((eval . (highlight-regexp "^ *")))))
Also consider [haskell-tab-indent-mode](https://spwhitton.name/tech/code/haskell-tab-indent/). The standard indentation modes that come with haskell-mode do not work well with tabs for indentation. This mode works well for hacking on git-annex.

24
doc/devblog/day_341__starting_smudge.mdwn Normal file
View file

@ -0,0 +1,24 @@
I've gotten git-annex working as a smudge/clean filter today in the
`smudge` branch. It works ok in a local git repository. `git add` lets
git-annex decide if it wants to annex a file's content, and checking out
branches and other git commands involving those files works pretty well.
It can sometimes be slow; git's smudge interface necessarily needs to
copy the content of files around, particularly when checking out files,
and so it's never going to be as fast as the good old git-annex symlink
approach. Most of the slow parts are things that can't be done in direct
mode repos though, like switching branches, so that isn't a regression.
No git-annex commands to manage the annexed content work yet. That
will need a key to worktree file mapping to be maintained, and implementing
that mapping and ensuring its always consistent is probably going to be
the harder part of this.
Also there's the question of how to handle upgrades from direct mode
repositories. This will be an upgrade from annex.version 5 to 6, and you
won't want to do it until all computers that have clones of a repository
have upgraded to git-annex 6.x, since older versions won't be able to work
with the upgraded repository. So, the repository upgrade will need to be
run manually initially, and it seems I'll need to keep supporting direct
mode for v5 repos in a transition period, which will probably be measured
in years.

8
doc/forum/Find_out_if_file_is_in_annex__63__.mdwn Normal file
View file

@ -0,0 +1,8 @@
Hi,
I am currently trying to move nearly all of my files into git-annex, and I was wondering if there is a (plumbing-level?) way to easily figure out if a file I find on a random hard disk is already contained in my annex repository?
I.e., is there an easy way to compute the key for a file (which is not in an annex) and look up if the file is already contained in a given annex repository? I've looked a bit at the plumbing-level commands but none seem to exactly fit the bill :-/
Cheers,
Alex

9
doc/forum/Future_proofing___47___disaster_recovery_with_an_encrypted_special_remote.mdwn Normal file
View file

@ -0,0 +1,9 @@
I am interested in using `git annex` to manage encrypted backups to Amazon S3/Glacier. So `git annex` will be used with the main file directory in direct mode and an encrypted S3 or Glacier remote set up in archive mode and then `git annex add .` and `git annex sync` will be run periodically. The intent is for this set up to be a backup for catastrophic failure, so I want to make sure I take care of future-proofing and disaster recovery properly. So my basic question is what would I need to have backed up and what would I have to do if the computer with the main repository died. I try to break that out into more specific questions below.
0. Do the S3/Glacier remotes just store the contents of `.git/annex/objects` in encrypted form and nothing else? So if I was left with nothing but the AWS bucket and couldn't get `git annex` to work for whatever reason, I could recover my files by hand if I had the encryption key (though I wouldn't know the file names or directory structure)?
1. For `shared` encryption, I see the cipher text in `remote.log` in the `git-annex` branch. Assuming I didn't have access to `git annex`, what would I need to do to convert that cipher text into a form that I could use with `gpg` to decrypt files?
2. Same question but for `hybrid` encryption rather than `shared`. I assume the answer is similar but I need to decrypt the cipher first with my gpg key? How do I do that?
3. Assuming I did have access to `git annex`, what would I need to create a new repo on a new computer with access to all of the files in the S3/Glacier bucket? I think I would need my Amazon credentials, my gpg key if using hybrid or public key encryption, and the `.git` folder as it was the last time files were pushed to the S3/Glacier remote (which would have the necessary decryption information for shared encryption). Is that right? I guess mainly I am checking that the remote does not store any metadata about the repo, so for `git annex` to be able to pull files back out I would need a backup of the `.git` directory and that back up would need to be up to date (can't just copy remote.log and have `git annex` work out the rest from the remote's contents). So for a full backup, my script would need to `tar` the `.git` directory, encrypt it, and push it to S3/Glacier separately after `git annex` does a sync. Then I could recover everything as long as I had a secure backup of my Amazon credentials and my encryption key(s).

13
doc/forum/assistant_and_archive_folder.mdwn Normal file
View file

@ -0,0 +1,13 @@
I have a setup with a windows desktop and a linux laptop with both an annex repository, interconnected via a backup repository in the middle (connected via ssh).
Both computers are setup as clients and all 3 machines are running the assistant.
Today i tried creating an archive folder and a putting some files in it. To my understanding those files should have been dropped in the 2 machines but kept on the server.. instead I have them copied on each machine. Listing where the files are on both computers shows that both know that themselves and the server have a copy, but they don't know anything about each other.
AFAIK it looks like the bug reported in https://git-annex.branchable.com/forum/question_about_assistant_and___47__archive__47__/
laptop: git-annex version: 5.20151116+gitg5416a1a-1~ndall+1
server: git-annex version: 5.20151109-g08bb3b1
windows: the latest build on the website

8
doc/special_remotes/comment_29_8393a14f084d022986c8245ee01f4198._comment Normal file
View file

@ -0,0 +1,8 @@
[[!comment format=mdwn
username="encryptio@2557a3f5b4ef0cddf8f011d7dbb3e76d46ed9c79"
nickname="encryptio"
subject="git-annex-remote-b2"
date="2015-12-04T03:19:29Z"
content="""
I wrote an external remote for [Backblaze's B2](https://www.backblaze.com/b2/cloud-storage.html) object storage service: [git-annex-remote-b2](https://github.com/encryptio/git-annex-remote-b2). I can't be arsed to write a full page on it, but the README covers everything an experienced git-annex user needs to know to use it.
"""]]