Merge branch 'master' into s3-aws

This commit is contained in:
Joey Hess 2014-10-28 12:39:22 -04:00
commit e535ff8fa4
27 changed files with 205 additions and 67 deletions

View file

@ -18,6 +18,7 @@ import Assistant.DaemonStatus
import qualified Remote
import Remote.List
import qualified Git.Remote
import qualified Git.Remote.Remove
import Logs.Trust
import qualified Annex
@ -34,7 +35,7 @@ disableRemote uuid = do
remote <- fromMaybe (error "unknown remote")
<$> liftAnnex (Remote.remoteFromUUID uuid)
liftAnnex $ do
inRepo $ Git.Remote.remove (Remote.name remote)
inRepo $ Git.Remote.Remove.remove (Remote.name remote)
void $ remoteListRefresh
updateSyncRemotes
return remote

View file

@ -25,7 +25,7 @@ import Assistant.WebApp.RepoList
import Assistant.WebApp.Configurators
import Assistant.XMPP
#endif
import qualified Git.Remote
import qualified Git.Remote.Remove
import Remote.List
import Creds
@ -125,7 +125,7 @@ getBuddyListR nid = do
waitNotifier getBuddyListBroadcaster nid
p <- widgetToPageContent buddyListDisplay
giveUrlRenderer $ [hamlet|^{pageBody p}|]
withUrlRenderer $ [hamlet|^{pageBody p}|]
buddyListDisplay :: Widget
buddyListDisplay = do
@ -211,7 +211,7 @@ getDisconnectXMPPR = do
rs <- filter Remote.isXMPPRemote . syncRemotes
<$> liftAssistant getDaemonStatus
liftAnnex $ do
mapM_ (inRepo . Git.Remote.remove . Remote.name) rs
mapM_ (inRepo . Git.Remote.Remove.remove . Remote.name) rs
void remoteListRefresh
removeCreds xmppCredsFile
liftAssistant $ do

View file

@ -66,7 +66,7 @@ getTransfersR nid = do
waitNotifier getTransferBroadcaster nid
p <- widgetToPageContent transfersDisplay
giveUrlRenderer $ [hamlet|^{pageBody p}|]
withUrlRenderer $ [hamlet|^{pageBody p}|]
{- The main dashboard. -}
dashboard :: Bool -> Widget

View file

@ -14,6 +14,7 @@ import Assistant.Gpg
import Utility.Gpg
import qualified Git.Command
import qualified Git.Remote
import qualified Git.Remote.Remove
import qualified Git.Construct
import qualified Annex.Branch
import qualified Git.GCrypt
@ -76,7 +77,7 @@ getGCryptRemoteName u repoloc = do
(M.lookup "name" <=< M.lookup u) <$> readRemoteLog
, return Nothing
)
void $ inRepo $ Git.Remote.remove tmpremote
void $ inRepo $ Git.Remote.Remove.remove tmpremote
maybe missing return mname
where
missing = error $ "Cannot find configuration for the gcrypt remote at " ++ repoloc

View file

@ -66,7 +66,7 @@ customPage' with_longpolling navbaritem content = do
when with_longpolling $
addScript $ StaticR js_longpolling_js
$(widgetFile "page")
giveUrlRenderer $(Hamlet.hamletFile $ hamletTemplate "bootstrap")
withUrlRenderer $(Hamlet.hamletFile $ hamletTemplate "bootstrap")
Just msg -> error msg
where
navdetails i = (navBarName i, navBarRoute i, Just i == navbaritem)

View file

@ -91,7 +91,7 @@ getRepoListR :: NotificationId -> RepoSelector -> Handler Html
getRepoListR nid reposelector = do
waitNotifier getRepoListBroadcaster nid
p <- widgetToPageContent $ repoListDisplay reposelector
giveUrlRenderer $ [hamlet|^{pageBody p}|]
withUrlRenderer $ [hamlet|^{pageBody p}|]
mainRepoSelector :: RepoSelector
mainRepoSelector = RepoSelector

View file

@ -73,7 +73,7 @@ getSideBarR nid = do
liftIO $ threadDelay 100000
page <- widgetToPageContent sideBarDisplay
giveUrlRenderer $ [hamlet|^{pageBody page}|]
withUrlRenderer $ [hamlet|^{pageBody page}|]
{- Called by the client to close an alert. -}
getCloseAlert :: AlertId -> Handler ()

View file

@ -78,7 +78,7 @@ instance Yesod WebApp where
addScript $ StaticR js_jquery_full_js
addScript $ StaticR js_bootstrap_js
$(widgetFile "error")
giveUrlRenderer $(hamletFile $ hamletTemplate "bootstrap")
withUrlRenderer $(hamletFile $ hamletTemplate "bootstrap")
instance RenderMessage WebApp FormMessage where
renderMessage _ _ = defaultFormMessage

View file

@ -12,8 +12,6 @@ module Git.Remote where
import Common
import Git
import Git.Types
import qualified Git.Command
import qualified Git.BuildVersion
import Data.Char
import qualified Data.Map as M
@ -44,17 +42,6 @@ makeLegalName s = case filter legal $ replace "/" "_" s of
legal '.' = True
legal c = isAlphaNum c
remove :: RemoteName -> Repo -> IO ()
remove remotename = Git.Command.run
[ Param "remote"
-- name of this subcommand changed
, Param $
if Git.BuildVersion.older "1.8.0"
then "rm"
else "remove"
, Param remotename
]
data RemoteLocation = RemoteUrl String | RemotePath FilePath
remoteLocationIsUrl :: RemoteLocation -> Bool

27
Git/Remote/Remove.hs Normal file
View file

@ -0,0 +1,27 @@
{- git remote stuff
-
- Copyright 2012 Joey Hess <joey@kitenet.net>
-
- Licensed under the GNU GPL version 3 or higher.
-}
{-# LANGUAGE CPP #-}
module Git.Remote.Remove where
import Common
import Git
import Git.Types
import qualified Git.Command
import qualified Git.BuildVersion
remove :: RemoteName -> Repo -> IO ()
remove remotename = Git.Command.run
[ Param "remote"
-- name of this subcommand changed
, Param $
if Git.BuildVersion.older "1.8.0"
then "rm"
else "remove"
, Param remotename
]

View file

@ -1,9 +1,9 @@
{- Yesod stuff, that's typically found in the scaffolded site.
-
- Also a bit of a compatability layer to make it easier to support yesod
- 1.1 and 1.2 in the same code base.
- 1.1-1.4 in the same code base.
-
- Copyright 2012, 2013 Joey Hess <joey@kitenet.net>
- Copyright 2012-2014 Joey Hess <joey@kitenet.net>
-
- Licensed under the GNU GPL version 3 or higher.
-}
@ -17,8 +17,10 @@ module Utility.Yesod
, widgetFile
, hamletTemplate
#endif
#if ! MIN_VERSION_yesod(1,4,0)
, withUrlRenderer
#endif
#if ! MIN_VERSION_yesod(1,2,0)
, giveUrlRenderer
, Html
#endif
) where
@ -41,6 +43,11 @@ import Data.Default (def)
import Text.Hamlet hiding (Html)
#endif
#endif
#if ! MIN_VERSION_yesod(1,4,0)
#if MIN_VERSION_yesod(1,2,0)
import Data.Text (Text)
#endif
#endif
#ifndef __NO_TH__
widgetFile :: String -> Q Exp
@ -69,8 +76,13 @@ liftH = lift
{- Misc new names for stuff. -}
#if ! MIN_VERSION_yesod(1,2,0)
giveUrlRenderer :: forall master sub. HtmlUrl (Route master) -> GHandler sub master RepHtml
giveUrlRenderer = hamletToRepHtml
withUrlRenderer :: forall master sub. HtmlUrl (Route master) -> GHandler sub master RepHtml
withUrlRenderer = hamletToRepHtml
type Html = RepHtml
#else
#if ! MIN_VERSION_yesod_core(1,2,20)
withUrlRenderer :: MonadHandler m => ((Route (HandlerSite m) -> [(Text, Text)] -> Text) -> output) -> m output
withUrlRenderer = giveUrlRenderer
#endif
#endif

4
debian/changelog vendored
View file

@ -1,4 +1,4 @@
git-annex (5.20141014) UNRELEASED; urgency=medium
git-annex (5.20141024) unstable; urgency=medium
* vicfg: Deleting configurations now resets to the default, where
before it has no effect.
@ -20,7 +20,7 @@ git-annex (5.20141014) UNRELEASED; urgency=medium
file. Before, the old version of the creds could be left there, and
would continue to be used.
-- Joey Hess <joeyh@debian.org> Tue, 14 Oct 2014 14:09:24 -0400
-- Joey Hess <joeyh@debian.org> Fri, 24 Oct 2014 13:03:29 -0400
git-annex (5.20141013) unstable; urgency=medium

View file

@ -0,0 +1,14 @@
[[!comment format=mdwn
username="https://www.google.com/accounts/o8/id?id=AItOawmUJBh1lYmvfCCiGr3yrdx-QhuLCSRnU5c"
nickname="Justin"
subject="comment 2"
date="2014-10-24T04:49:20Z"
content="""
Oh jeez, I screwed that up wrt HEAD and GET. Sorry. The cost per HEAD on Google is 1/10 the price of GET, so we're talking $.13 to HEAD my 130k-file annex, which is totally reasonable.
One can GET a bucket, which is what I was looking at. This returns up to 1000 elements of its contents (and there's a way to iterate over larger buckets). Of course this would only be useful if the majority of files in the bucket were of interest to git-annex, and it sounds like more trouble than it's worth at the prices I'm seeing.
There might be a throughput improvement to be had by keeping the connection alive, although in my brief investigation, I think there may be a larger gain to be had by pipelining the various steps. Based on the fact that git-annex oomed when trying to upload a large file from my rpi, it seems like maybe the whole file is encrypted in memory before it's uploaded? And certainly the HEAD(s) appear not to be done in parallel with the upload.
Sorry again for that HEAD/GET fail.
"""]]

View file

@ -0,0 +1,12 @@
[[!comment format=mdwn
username="http://joeyh.name/"
ip="209.250.56.96"
subject="comment 3"
date="2014-10-24T16:02:23Z"
content="""
The OOM is [[S3_memory_leaks]]; fixed in the s3-aws branch.
Yeah, GET of a bucket is doable. Another problem with it though is, if the bucket has a lot of contents, such as many files, or large files split into many chunks, that all has to be buffered in memory or processed as a stream. It would make sense in operations where git-annex knows it wants to check every key in a bucket. `git annex unused --from $s3remote` is the case that springs to mind where it could be quite useful to do that. Integrating it with `get`, not so much.
I'd be inclined to demote this to a wishlist todo item to try to use bucket GET for `unused`. And/or rethink whether it makes sense for `copy --to` to run in --fast mode by default. I've been back and forth on that question before, but just from a runtime perspective, not from a 13 cents perspective. ;)
"""]]

View file

@ -0,0 +1,11 @@
[[!comment format=mdwn
username="http://joeyh.name/"
ip="209.250.56.96"
subject="comment 11"
date="2014-10-23T21:05:15Z"
content="""
When it resumes, it will start at 0% but jump forward to the resume point pretty quickly, after verifying which chunks have already been sent.
If any full chunk gets transferred, I'd expect it to resume. This may not be very obvious it's happening for smaller files.
I have been running `git annex testremote` against S3 special remotes today, and have not managed to reproduce this problem (using either the old S3 or the new AWS libraries). It could be anything, including a problem with your network or the network between you and the S3 endpoint. Have you tried using a different S3 region?
"""]]

View file

@ -0,0 +1,14 @@
[[!comment format=mdwn
username="http://joeyh.name/"
ip="209.250.56.96"
subject="comment 1"
date="2014-10-23T18:52:48Z"
content="""
The S3 library that git-annex is using does not support the authentication method that this region uses.
It is supported by the aws library that git-annex uses in the `s3-aws` branch in git, and I already added the region there this morning.
I can't merge `s3-aws` yet; the neccessary version of the aws library is not yet available in eg, Debian. And even upgrading aws from cabal seems to result in dependency hell, due to its needing a newer version of scientific. This should all sort itself out in time.
If you need this region, you'll need to try to build git-annex's s3-aws branch, for now.
"""]]

View file

@ -0,0 +1,8 @@
[[!comment format=mdwn
username="http://joeyh.name/"
ip="209.250.56.96"
subject="comment 2"
date="2014-10-23T19:51:46Z"
content="""
Looks like the cabal dependency hell is managable; if done in system without anything installed, cabal manages to install the new aws, and everything else, except for the dbus library. Still not ready to be merged though.
"""]]

View file

@ -0,0 +1,13 @@
New AWS region in Germany announced today. git-annex doesn't support it
yet, unless you're using the `s3-aws` branch.
I cleaned up that branch, got it building again, and re-tested it with
`testremote`, and then fixed a problem the test suite found that was
caused by some changes in the haskell aws library.
Unfortunately, s3-aws is [not ready to be merged](http://git-annex.branchable.com/bugs/new_AWS_region___40__eu-central-1__41__)
because of some cabal dependency problems involving `dbus` and `random`. I did
go ahead and update Debian's haskell-aws package to cherry-pick
from a newer version the change needed for Inernet Archive
support, which allows building the s3-aws branch on Debian.
Getting closer..

View file

@ -0,0 +1,16 @@
I'm having trouble sharing an rsync remote between two users (as a "transfer" repository). `git-annex` thinks `rsync` fails because it can't set the permissions on the root directory on the destination. I've added `--perms --chmod=Dug+rwxs,Fug+rw,o-rwx` to `annex-rsync-options` so all users in the specified group on the destination can create and delete files. However, I see in the logs errors like:
sending incremental file list
./
rsync: failed to set permissions on "/home/shared-xfer/.": Operation not permitted (1)
2fe/985/GPGHMACSHA1--b36ce0a18718e7ea7cf9827c2748608df7152dfc/GPGHMACSHA1--b36ce0a18718e7ea7cf9827c2748608df7152dfc
32,768 0% 0.00kB/s 0:00:00
3,359,405 100% 4.78MB/s 0:00:00 (xfr#1, to-chk=0/5)
rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1165) [sender=3.1.1]
[2014-10-18 15:16:24 EDT] chat: gpg ["--batch","--no-tty","--use-agent","--quiet","--trust-model","always","--batch","--passphrase-fd","22","--symmetric","--force-mdc","--no-textmode"]
[2014-10-18 15:16:24 EDT] read: rsync ["-e","'ssh' '-S' '.git/annex/ssh/blah@blah' '-o' 'ControlMaster=auto' '-o' 'ControlPersist=yes' '-l' 'blah' '-T'","--perms","--chmod=Dug+rwxs,Fug+rw,o-rwx","--progress","--recursive","--partial","--partial-dir=.rsync-partial","/blah/.git/annex/tmp/rsynctmp/24863/","blah@blah:/home/shared-xfer/"]
rsync failed -- run git annex again to resume file transfer
The transfer actually completes fine, except for setting permissions of the root directory on the destination (which I don't really care about, because I have manually set them to be correct). Is it possible for `git-annex` to copy the contents of the directory to rsync remotes? For the example above, instead of `/blah/.git/annex/tmp/rsynctmp/24863/`, `/blah/.git/annex/tmp/rsynctmp/24863/2fe` would work. I can't come up with an alternate solution for this server, except using the same remote user (which I'd prefer not to do).

View file

@ -0,0 +1,13 @@
I accidentally deleted a file from a git-annex repository with a plain "rm". How can I restore it from other repositories that have it?
I tried using
git annex copy --from $REMOTE $REMOVED_FILE
but git-annex complaints about
$REMOVED_FILE not found
I suppose that I could switch to indirect mode and do checkout and older version, but I'd prefer not to. My repository contains thousands of quite big files; it would take a lot of time and the probability of something going wrong during the conversion is quite high (see other bugs reported during import).
I'm using git-annex 5.20141024-g613f396.

View file

@ -2,10 +2,10 @@
[[!table format=dsv header=yes data="""
detailed instructions | quick install
[[OSX]] | [download git-annex.app](http://downloads.kitenet.net/git-annex/OSX/current/)
[[OSX]] | [download git-annex.app](https://downloads.kitenet.net/git-annex/OSX/current/)
&nbsp;&nbsp;[[OSX/Homebrew]] | `brew install git-annex`
[[Android]] | [download git-annex.apk](http://downloads.kitenet.net/git-annex/android/current/) **beta**
[[Linux|linux_standalone]] | [download prebuilt linux tarball](http://downloads.kitenet.net/git-annex/linux/current/)
[[Android]] | [download git-annex.apk](https://downloads.kitenet.net/git-annex/android/current/) **beta**
[[Linux|linux_standalone]] | [download prebuilt linux tarball](https://downloads.kitenet.net/git-annex/linux/current/)
&nbsp;&nbsp;[[Debian]] | `apt-get install git-annex`
&nbsp;&nbsp;[[Ubuntu]] | `apt-get install git-annex`
&nbsp;&nbsp;[[Fedora]] | `yum install git-annex`
@ -16,10 +16,10 @@ detailed instructions | quick install
&nbsp;&nbsp;[[ScientificLinux5]] |
&nbsp;&nbsp;[[openSUSE]] |
&nbsp;&nbsp;[[Docker]] |
[[Windows]] | [download installer](http://downloads.kitenet.net/git-annex/windows/current/) **beta**
[[Windows]] | [download installer](https://downloads.kitenet.net/git-annex/windows/current/) **beta**
"""]]
All the downloads above use https for security. For added security, see
All the download links above use https for security. For added security, see
[[verifying_downloads]].
## Building it yourself

View file

@ -9,8 +9,8 @@ file accompanying your package. Just append .sig to the url.
For example, on Linux:
$ wget http://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-amd64.tar.gz
$ wget http://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-amd64.tar.gz.sig
$ wget https://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-amd64.tar.gz
$ wget https://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-amd64.tar.gz.sig
You can then download the public key, and check that the package is signed
with it.
@ -26,6 +26,8 @@ But, how do you know that the gpg-pubkey.asc you downloaded
is the right key? The answer is the GPG web of trust.
* Joey Hess generates these git-annex packages,
and has a GPG key, [C910D9222512E3C Joey Hess <id@joeyh.name>](http://pgp.cs.uu.nl/stats/2512E3C7.html), which has
and has a GPG key, [C910D9222512E3C7 Joey Hess <id@joeyh.name>](http://pgp.cs.uu.nl/stats/2512E3C7.html), which has
been verified and signed by many people.
* Joey's GPG key has signed the git-annex distribution signing key.
Don't take this page's word about this, check it yourself!

View file

@ -1,27 +0,0 @@
git-annex 5.20140915 released with [[!toggle text="these changes"]]
[[!toggleable text="""
* New annex.hardlink setting. Closes: #[758593](http://bugs.debian.org/758593)
* init: Automatically detect when a repository was cloned with --shared,
and set annex.hardlink=true, as well as marking the repository as
untrusted.
* Fix parsing of ipv6 address in git remote address when it was not
formatted as an url.
* The annex-rsync-transport configuration is now also used when checking
if a key is present on a rsync remote, and when dropping a key from
the remote.
* Promote file not found warning message to an error.
* Fix transfer lock file FD leak that could occur when two separate
git-annex processes were both working to perform the same set of
transfers.
* sync: Ensure that pending changes to git-annex branch are committed
before push when in direct mode. (Fixing a very minor reversion.)
* WORM backend: Switched to include the relative path to the file inside
the repository, rather than just the file's base name. Note that if you're
relying on such things to keep files separate with WORM, you should really
be using a better backend.
* Rather than crashing when there's a problem with the requested bloomfilter
capacity/accuracy, fall back to a reasonable default bloom filter size.
* Fix build with optparse-applicative 0.10. Closes: #[761484](http://bugs.debian.org/761484)
* webapp: Fixed visual glitch in xmpp pairing that was reported live by a
user who tracked me down in front of a coffee cart in Portland.
(New bug reporting method of choice?)"""]]

View file

@ -0,0 +1,21 @@
git-annex 5.20141024 released with [[!toggle text="these changes"]]
[[!toggleable text="""
* vicfg: Deleting configurations now resets to the default, where
before it has no effect.
* Remove hurd stuff from cabal file, since hackage currently rejects
it, and the test suite fails on hurd.
* initremote: Don't allow creating a special remote that has the same
name as an existing git remote.
* Windows: Use haskell setenv library to clean up several ugly workarounds
for inability to manipulate the environment on windows. This includes
making git-annex not re-exec itself on start on windows, and making the
test suite on Windows run tests without forking.
* glacier: Fix pipe setup when calling glacier-cli to retrieve an object.
* info: When run on a single annexed file, displays some info about the
file, including its key and size.
* info: When passed the name or uuid of a remote, displays info about that
remote. Remotes that support encryption, chunking, or embedded
creds will include that in their info.
* enableremote: When the remote has creds, update the local creds cache
file. Before, the old version of the creds could be left there, and
would continue to be used."""]]

View file

@ -0,0 +1,13 @@
[[!comment format=mdwn
username="https://www.google.com/accounts/o8/id?id=AItOawn3p4i4lk_zMilvjnJ9sS6g2nerpgz0Fjc"
nickname="Matthias"
subject="Keep historical data, but delete data never referenced"
date="2014-10-26T12:06:25Z"
content="""
Is there an easy solution for the following? There are two kinds of \"unused\" I would like to treat differently:
1. Kind \"really unused\": Was added once to the annex, but symlink was never committed
2. Kind \"only history\": A commit contains a symlink to the data, but no active branch
I want to preserve \"only history\", and only drop \"really unused\". What is an elegant way to do this? Thanks for your suggestions.
"""]]

View file

@ -1,5 +1,5 @@
Name: git-annex
Version: 5.20141013
Version: 5.20141024
Cabal-Version: >= 1.8
License: GPL-3
Maintainer: Joey Hess <joey@kitenet.net>

View file

@ -9,7 +9,7 @@
For full details, see #
<a href="http://git-annex.branchable.com/">the git-annex website</a>.
<hr>
git-annex is © 2010-2013 Joey Hess. It is free software, licensed #
git-annex is © 2010-2014 Joey Hess. It is free software, licensed #
under the terms of the GNU General Public License, version 3 or above. #
This webapp is licensed under the terms of the GNU Affero General #
Public License, version 3 or above. #