From f8fceecd6343d25a5b872eca1aba27a7fe8017e0 Mon Sep 17 00:00:00 2001 From: "http://joeyh.name/" Date: Thu, 23 Oct 2014 18:52:49 +0000 Subject: [PATCH 01/20] Added a comment --- ...ent_1_23d65c7e05f6e7312bbdbd039bf8d308._comment | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/bugs/new_AWS_region___40__eu-central-1__41__/comment_1_23d65c7e05f6e7312bbdbd039bf8d308._comment diff --git a/doc/bugs/new_AWS_region___40__eu-central-1__41__/comment_1_23d65c7e05f6e7312bbdbd039bf8d308._comment b/doc/bugs/new_AWS_region___40__eu-central-1__41__/comment_1_23d65c7e05f6e7312bbdbd039bf8d308._comment new file mode 100644 index 0000000000..e56e1beaff --- /dev/null +++ b/doc/bugs/new_AWS_region___40__eu-central-1__41__/comment_1_23d65c7e05f6e7312bbdbd039bf8d308._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.96" + subject="comment 1" + date="2014-10-23T18:52:48Z" + content=""" +The S3 library that git-annex is using does not support the authentication method that this region uses. + +It is supported by the aws library that git-annex uses in the `s3-aws` branch in git, and I already added the region there this morning. + +I can't merge `s3-aws` yet; the neccessary version of the aws library is not yet available in eg, Debian. And even upgrading aws from cabal seems to result in dependency hell, due to its needing a newer version of scientific. This should all sort itself out in time. + +If you need this region, you'll need to try to build git-annex's s3-aws branch, for now. +"""]] From 9ac288d94f54f8e8c55c07ee2ed439a4f077f07e Mon Sep 17 00:00:00 2001 From: "http://joeyh.name/" Date: Thu, 23 Oct 2014 19:51:46 +0000 Subject: [PATCH 02/20] Added a comment --- .../comment_2_6ceb3251b7067591d34cd146d19ccea9._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/new_AWS_region___40__eu-central-1__41__/comment_2_6ceb3251b7067591d34cd146d19ccea9._comment diff --git a/doc/bugs/new_AWS_region___40__eu-central-1__41__/comment_2_6ceb3251b7067591d34cd146d19ccea9._comment b/doc/bugs/new_AWS_region___40__eu-central-1__41__/comment_2_6ceb3251b7067591d34cd146d19ccea9._comment new file mode 100644 index 0000000000..fab525b668 --- /dev/null +++ b/doc/bugs/new_AWS_region___40__eu-central-1__41__/comment_2_6ceb3251b7067591d34cd146d19ccea9._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.96" + subject="comment 2" + date="2014-10-23T19:51:46Z" + content=""" +Looks like the cabal dependency hell is managable; if done in system without anything installed, cabal manages to install the new aws, and everything else, except for the dbus library. Still not ready to be merged though. +"""]] From 2f1e6fcb738e1f29281c5285236d5fd5b74a3c92 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 23 Oct 2014 17:00:49 -0400 Subject: [PATCH 03/20] devblog --- doc/devblog/day__228_new_AWS.mdwn | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/devblog/day__228_new_AWS.mdwn diff --git a/doc/devblog/day__228_new_AWS.mdwn b/doc/devblog/day__228_new_AWS.mdwn new file mode 100644 index 0000000000..3003a79852 --- /dev/null +++ b/doc/devblog/day__228_new_AWS.mdwn @@ -0,0 +1,13 @@ +New AWS region in Germany announced today. git-annex doesn't support it +yet, unless you're using the `s3-aws` branch. + +I cleaned up that branch, got it building again, and re-tested it with +`testremote`, and then fixed a problem the test suite found that was +caused by some changes in the haskell aws library. + +Unfortunately, s3-aws is [not ready to be merged](http://git-annex.branchable.com/bugs/new_AWS_region___40__eu-central-1__41__) +because of some cabal dependency problems involving `dbus` and `random`. I did +go ahead and update Debian's haskell-aws package to cherry-pick +from a newer version the change needed for Inernet Archive +support, which allows building the s3-aws branch on Debian. +Getting closer.. From 805abd3c8dc80b6af9a897fb96a213f5d25b1c81 Mon Sep 17 00:00:00 2001 From: "http://joeyh.name/" Date: Thu, 23 Oct 2014 21:05:15 +0000 Subject: [PATCH 04/20] Added a comment --- ...mment_11_65cb9b1dd69cd657f2cfde036eb60417._comment | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/bugs/Upload_to_S3_fails_/comment_11_65cb9b1dd69cd657f2cfde036eb60417._comment diff --git a/doc/bugs/Upload_to_S3_fails_/comment_11_65cb9b1dd69cd657f2cfde036eb60417._comment b/doc/bugs/Upload_to_S3_fails_/comment_11_65cb9b1dd69cd657f2cfde036eb60417._comment new file mode 100644 index 0000000000..ec59d6a207 --- /dev/null +++ b/doc/bugs/Upload_to_S3_fails_/comment_11_65cb9b1dd69cd657f2cfde036eb60417._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.96" + subject="comment 11" + date="2014-10-23T21:05:15Z" + content=""" +When it resumes, it will start at 0% but jump forward to the resume point pretty quickly, after verifying which chunks have already been sent. +If any full chunk gets transferred, I'd expect it to resume. This may not be very obvious it's happening for smaller files. + +I have been running `git annex testremote` against S3 special remotes today, and have not managed to reproduce this problem (using either the old S3 or the new AWS libraries). It could be anything, including a problem with your network or the network between you and the S3 endpoint. Have you tried using a different S3 region? +"""]] From 31a1f061c64189a8e80105b0b6d8bd94752d76cd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 23 Oct 2014 20:25:31 -0400 Subject: [PATCH 05/20] typo --- Assistant/WebApp/Configurators/AWS.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Assistant/WebApp/Configurators/AWS.hs b/Assistant/WebApp/Configurators/AWS.hs index 7c91d70b0f..701c2c8704 100644 --- a/Assistant/WebApp/Configurators/AWS.hs +++ b/Assistant/WebApp/Configurators/AWS.hs @@ -162,7 +162,7 @@ getEnableS3R :: UUID -> Handler Html #ifdef WITH_S3 getEnableS3R uuid = do m <- liftAnnex readRemoteLog - if maybe False S3.confgiIA (M.lookup uuid m) + if maybe False S3.configIA (M.lookup uuid m) then redirect $ EnableIAR uuid else postEnableS3R uuid #else From ce9076370edc4c73984b35e6f2fb0ea885d333b9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 23 Oct 2014 20:26:46 -0400 Subject: [PATCH 06/20] deal with yesod's second gratuitous rename of the same function in a year --- Assistant/WebApp/Configurators/XMPP.hs | 2 +- Assistant/WebApp/DashBoard.hs | 2 +- Assistant/WebApp/Page.hs | 2 +- Assistant/WebApp/RepoList.hs | 2 +- Assistant/WebApp/SideBar.hs | 2 +- Assistant/WebApp/Types.hs | 2 +- Utility/Yesod.hs | 22 +++++++++++++++++----- 7 files changed, 23 insertions(+), 11 deletions(-) diff --git a/Assistant/WebApp/Configurators/XMPP.hs b/Assistant/WebApp/Configurators/XMPP.hs index 5d03c65998..cabae9dbd3 100644 --- a/Assistant/WebApp/Configurators/XMPP.hs +++ b/Assistant/WebApp/Configurators/XMPP.hs @@ -125,7 +125,7 @@ getBuddyListR nid = do waitNotifier getBuddyListBroadcaster nid p <- widgetToPageContent buddyListDisplay - giveUrlRenderer $ [hamlet|^{pageBody p}|] + withUrlRenderer $ [hamlet|^{pageBody p}|] buddyListDisplay :: Widget buddyListDisplay = do diff --git a/Assistant/WebApp/DashBoard.hs b/Assistant/WebApp/DashBoard.hs index bf2366161a..acf628078b 100644 --- a/Assistant/WebApp/DashBoard.hs +++ b/Assistant/WebApp/DashBoard.hs @@ -66,7 +66,7 @@ getTransfersR nid = do waitNotifier getTransferBroadcaster nid p <- widgetToPageContent transfersDisplay - giveUrlRenderer $ [hamlet|^{pageBody p}|] + withUrlRenderer $ [hamlet|^{pageBody p}|] {- The main dashboard. -} dashboard :: Bool -> Widget diff --git a/Assistant/WebApp/Page.hs b/Assistant/WebApp/Page.hs index 4d2e58b11c..378b2f228a 100644 --- a/Assistant/WebApp/Page.hs +++ b/Assistant/WebApp/Page.hs @@ -66,7 +66,7 @@ customPage' with_longpolling navbaritem content = do when with_longpolling $ addScript $ StaticR js_longpolling_js $(widgetFile "page") - giveUrlRenderer $(Hamlet.hamletFile $ hamletTemplate "bootstrap") + withUrlRenderer $(Hamlet.hamletFile $ hamletTemplate "bootstrap") Just msg -> error msg where navdetails i = (navBarName i, navBarRoute i, Just i == navbaritem) diff --git a/Assistant/WebApp/RepoList.hs b/Assistant/WebApp/RepoList.hs index f3b6e8bcd1..3a0ee49a01 100644 --- a/Assistant/WebApp/RepoList.hs +++ b/Assistant/WebApp/RepoList.hs @@ -91,7 +91,7 @@ getRepoListR :: NotificationId -> RepoSelector -> Handler Html getRepoListR nid reposelector = do waitNotifier getRepoListBroadcaster nid p <- widgetToPageContent $ repoListDisplay reposelector - giveUrlRenderer $ [hamlet|^{pageBody p}|] + withUrlRenderer $ [hamlet|^{pageBody p}|] mainRepoSelector :: RepoSelector mainRepoSelector = RepoSelector diff --git a/Assistant/WebApp/SideBar.hs b/Assistant/WebApp/SideBar.hs index d7a0bb78fa..2e1a62eda6 100644 --- a/Assistant/WebApp/SideBar.hs +++ b/Assistant/WebApp/SideBar.hs @@ -73,7 +73,7 @@ getSideBarR nid = do liftIO $ threadDelay 100000 page <- widgetToPageContent sideBarDisplay - giveUrlRenderer $ [hamlet|^{pageBody page}|] + withUrlRenderer $ [hamlet|^{pageBody page}|] {- Called by the client to close an alert. -} getCloseAlert :: AlertId -> Handler () diff --git a/Assistant/WebApp/Types.hs b/Assistant/WebApp/Types.hs index 1dbc146b0f..711af91c40 100644 --- a/Assistant/WebApp/Types.hs +++ b/Assistant/WebApp/Types.hs @@ -78,7 +78,7 @@ instance Yesod WebApp where addScript $ StaticR js_jquery_full_js addScript $ StaticR js_bootstrap_js $(widgetFile "error") - giveUrlRenderer $(hamletFile $ hamletTemplate "bootstrap") + withUrlRenderer $(hamletFile $ hamletTemplate "bootstrap") instance RenderMessage WebApp FormMessage where renderMessage _ _ = defaultFormMessage diff --git a/Utility/Yesod.hs b/Utility/Yesod.hs index afe10a1112..348af7f2f8 100644 --- a/Utility/Yesod.hs +++ b/Utility/Yesod.hs @@ -1,9 +1,9 @@ {- Yesod stuff, that's typically found in the scaffolded site. - - Also a bit of a compatability layer to make it easier to support yesod - - 1.1 and 1.2 in the same code base. + - 1.1-1.4 in the same code base. - - - Copyright 2012, 2013 Joey Hess + - Copyright 2012-2014 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} @@ -17,8 +17,10 @@ module Utility.Yesod , widgetFile , hamletTemplate #endif +#if ! MIN_VERSION_yesod(1,4,0) + , withUrlRenderer +#endif #if ! MIN_VERSION_yesod(1,2,0) - , giveUrlRenderer , Html #endif ) where @@ -41,6 +43,11 @@ import Data.Default (def) import Text.Hamlet hiding (Html) #endif #endif +#if ! MIN_VERSION_yesod(1,4,0) +#if MIN_VERSION_yesod(1,2,0) +import Data.Text (Text) +#endif +#endif #ifndef __NO_TH__ widgetFile :: String -> Q Exp @@ -69,8 +76,13 @@ liftH = lift {- Misc new names for stuff. -} #if ! MIN_VERSION_yesod(1,2,0) -giveUrlRenderer :: forall master sub. HtmlUrl (Route master) -> GHandler sub master RepHtml -giveUrlRenderer = hamletToRepHtml +withUrlRenderer :: forall master sub. HtmlUrl (Route master) -> GHandler sub master RepHtml +withUrlRenderer = hamletToRepHtml type Html = RepHtml +#else +#if ! MIN_VERSION_yesod(1,4,0) +withUrlRenderer :: MonadHandler m => ((Route (HandlerSite m) -> [(Text, Text)] -> Text) -> output) -> m output +withUrlRenderer = giveUrlRenderer +#endif #endif From 23bd3564d12e588dc32e96afbca5560cb4a70d9a Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawmUJBh1lYmvfCCiGr3yrdx-QhuLCSRnU5c" Date: Fri, 24 Oct 2014 04:49:20 +0000 Subject: [PATCH 07/20] Added a comment --- ...ent_2_8fdeb3352ac00a76625ec9c007e5b76a._comment | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/bugs/Issue_fewer_S3_GET_requests/comment_2_8fdeb3352ac00a76625ec9c007e5b76a._comment diff --git a/doc/bugs/Issue_fewer_S3_GET_requests/comment_2_8fdeb3352ac00a76625ec9c007e5b76a._comment b/doc/bugs/Issue_fewer_S3_GET_requests/comment_2_8fdeb3352ac00a76625ec9c007e5b76a._comment new file mode 100644 index 0000000000..b3b2594450 --- /dev/null +++ b/doc/bugs/Issue_fewer_S3_GET_requests/comment_2_8fdeb3352ac00a76625ec9c007e5b76a._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmUJBh1lYmvfCCiGr3yrdx-QhuLCSRnU5c" + nickname="Justin" + subject="comment 2" + date="2014-10-24T04:49:20Z" + content=""" +Oh jeez, I screwed that up wrt HEAD and GET. Sorry. The cost per HEAD on Google is 1/10 the price of GET, so we're talking $.13 to HEAD my 130k-file annex, which is totally reasonable. + +One can GET a bucket, which is what I was looking at. This returns up to 1000 elements of its contents (and there's a way to iterate over larger buckets). Of course this would only be useful if the majority of files in the bucket were of interest to git-annex, and it sounds like more trouble than it's worth at the prices I'm seeing. + +There might be a throughput improvement to be had by keeping the connection alive, although in my brief investigation, I think there may be a larger gain to be had by pipelining the various steps. Based on the fact that git-annex oomed when trying to upload a large file from my rpi, it seems like maybe the whole file is encrypted in memory before it's uploaded? And certainly the HEAD(s) appear not to be done in parallel with the upload. + +Sorry again for that HEAD/GET fail. +"""]] From c7684de78eaa8bb8516f21716084e1425ca72843 Mon Sep 17 00:00:00 2001 From: "http://joeyh.name/" Date: Fri, 24 Oct 2014 16:02:24 +0000 Subject: [PATCH 08/20] Added a comment --- ...mment_3_6ccbb1cff7bc6b4640220d98f7ce21c3._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/bugs/Issue_fewer_S3_GET_requests/comment_3_6ccbb1cff7bc6b4640220d98f7ce21c3._comment diff --git a/doc/bugs/Issue_fewer_S3_GET_requests/comment_3_6ccbb1cff7bc6b4640220d98f7ce21c3._comment b/doc/bugs/Issue_fewer_S3_GET_requests/comment_3_6ccbb1cff7bc6b4640220d98f7ce21c3._comment new file mode 100644 index 0000000000..353281db65 --- /dev/null +++ b/doc/bugs/Issue_fewer_S3_GET_requests/comment_3_6ccbb1cff7bc6b4640220d98f7ce21c3._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="209.250.56.96" + subject="comment 3" + date="2014-10-24T16:02:23Z" + content=""" +The OOM is [[S3_memory_leaks]]; fixed in the s3-aws branch. + +Yeah, GET of a bucket is doable. Another problem with it though is, if the bucket has a lot of contents, such as many files, or large files split into many chunks, that all has to be buffered in memory or processed as a stream. It would make sense in operations where git-annex knows it wants to check every key in a bucket. `git annex unused --from $s3remote` is the case that springs to mind where it could be quite useful to do that. Integrating it with `get`, not so much. + +I'd be inclined to demote this to a wishlist todo item to try to use bucket GET for `unused`. And/or rethink whether it makes sense for `copy --to` to run in --fast mode by default. I've been back and forth on that question before, but just from a runtime perspective, not from a 13 cents perspective. ;) +"""]] From 613f396efd7a6e5d992feba8d4029c72fb3d8c11 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 24 Oct 2014 12:11:27 -0400 Subject: [PATCH 09/20] more accurate version bounds --- Utility/Yesod.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Utility/Yesod.hs b/Utility/Yesod.hs index 348af7f2f8..ecd4eaf942 100644 --- a/Utility/Yesod.hs +++ b/Utility/Yesod.hs @@ -81,7 +81,7 @@ withUrlRenderer = hamletToRepHtml type Html = RepHtml #else -#if ! MIN_VERSION_yesod(1,4,0) +#if ! MIN_VERSION_yesod_core(1,2,20) withUrlRenderer :: MonadHandler m => ((Route (HandlerSite m) -> [(Text, Text)] -> Text) -> output) -> m output withUrlRenderer = giveUrlRenderer #endif From 5bdcb2342467b315f438a8f53373f66dac6dd0e9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 24 Oct 2014 13:03:50 -0400 Subject: [PATCH 10/20] prep release --- debian/changelog | 4 ++-- git-annex.cabal | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index c658ce1862..8af44b3bb3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (5.20141014) UNRELEASED; urgency=medium +git-annex (5.20141024) unstable; urgency=medium * vicfg: Deleting configurations now resets to the default, where before it has no effect. @@ -20,7 +20,7 @@ git-annex (5.20141014) UNRELEASED; urgency=medium file. Before, the old version of the creds could be left there, and would continue to be used. - -- Joey Hess Tue, 14 Oct 2014 14:09:24 -0400 + -- Joey Hess Fri, 24 Oct 2014 13:03:29 -0400 git-annex (5.20141013) unstable; urgency=medium diff --git a/git-annex.cabal b/git-annex.cabal index 270e1b74aa..9ff85fd55b 100644 --- a/git-annex.cabal +++ b/git-annex.cabal @@ -1,5 +1,5 @@ Name: git-annex -Version: 5.20141013 +Version: 5.20141024 Cabal-Version: >= 1.8 License: GPL-3 Maintainer: Joey Hess From d0f0bd71dc11562a7819eb781a4900dbd3d85ac2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 24 Oct 2014 13:35:08 -0400 Subject: [PATCH 11/20] add news item for git-annex 5.20141024 --- doc/news/version_5.20140915.mdwn | 27 --------------------------- doc/news/version_5.20141024.mdwn | 21 +++++++++++++++++++++ 2 files changed, 21 insertions(+), 27 deletions(-) delete mode 100644 doc/news/version_5.20140915.mdwn create mode 100644 doc/news/version_5.20141024.mdwn diff --git a/doc/news/version_5.20140915.mdwn b/doc/news/version_5.20140915.mdwn deleted file mode 100644 index 7aa6477c9f..0000000000 --- a/doc/news/version_5.20140915.mdwn +++ /dev/null @@ -1,27 +0,0 @@ -git-annex 5.20140915 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * New annex.hardlink setting. Closes: #[758593](http://bugs.debian.org/758593) - * init: Automatically detect when a repository was cloned with --shared, - and set annex.hardlink=true, as well as marking the repository as - untrusted. - * Fix parsing of ipv6 address in git remote address when it was not - formatted as an url. - * The annex-rsync-transport configuration is now also used when checking - if a key is present on a rsync remote, and when dropping a key from - the remote. - * Promote file not found warning message to an error. - * Fix transfer lock file FD leak that could occur when two separate - git-annex processes were both working to perform the same set of - transfers. - * sync: Ensure that pending changes to git-annex branch are committed - before push when in direct mode. (Fixing a very minor reversion.) - * WORM backend: Switched to include the relative path to the file inside - the repository, rather than just the file's base name. Note that if you're - relying on such things to keep files separate with WORM, you should really - be using a better backend. - * Rather than crashing when there's a problem with the requested bloomfilter - capacity/accuracy, fall back to a reasonable default bloom filter size. - * Fix build with optparse-applicative 0.10. Closes: #[761484](http://bugs.debian.org/761484) - * webapp: Fixed visual glitch in xmpp pairing that was reported live by a - user who tracked me down in front of a coffee cart in Portland. - (New bug reporting method of choice?)"""]] \ No newline at end of file diff --git a/doc/news/version_5.20141024.mdwn b/doc/news/version_5.20141024.mdwn new file mode 100644 index 0000000000..b50a81b855 --- /dev/null +++ b/doc/news/version_5.20141024.mdwn @@ -0,0 +1,21 @@ +git-annex 5.20141024 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * vicfg: Deleting configurations now resets to the default, where + before it has no effect. + * Remove hurd stuff from cabal file, since hackage currently rejects + it, and the test suite fails on hurd. + * initremote: Don't allow creating a special remote that has the same + name as an existing git remote. + * Windows: Use haskell setenv library to clean up several ugly workarounds + for inability to manipulate the environment on windows. This includes + making git-annex not re-exec itself on start on windows, and making the + test suite on Windows run tests without forking. + * glacier: Fix pipe setup when calling glacier-cli to retrieve an object. + * info: When run on a single annexed file, displays some info about the + file, including its key and size. + * info: When passed the name or uuid of a remote, displays info about that + remote. Remotes that support encryption, chunking, or embedded + creds will include that in their info. + * enableremote: When the remote has creds, update the local creds cache + file. Before, the old version of the creds could be left there, and + would continue to be used."""]] \ No newline at end of file From 45b2a87520634738358ee57bafb56319eace5c93 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 24 Oct 2014 13:57:36 -0400 Subject: [PATCH 12/20] actually use https for download urls The urls on sub-pages were all https, but not on the main page. --- doc/install.mdwn | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/install.mdwn b/doc/install.mdwn index adaaa3eb6e..171177fbf0 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -2,10 +2,10 @@ [[!table format=dsv header=yes data=""" detailed instructions | quick install -[[OSX]] | [download git-annex.app](http://downloads.kitenet.net/git-annex/OSX/current/) +[[OSX]] | [download git-annex.app](https://downloads.kitenet.net/git-annex/OSX/current/)   [[OSX/Homebrew]] | `brew install git-annex` -[[Android]] | [download git-annex.apk](http://downloads.kitenet.net/git-annex/android/current/) **beta** -[[Linux|linux_standalone]] | [download prebuilt linux tarball](http://downloads.kitenet.net/git-annex/linux/current/) +[[Android]] | [download git-annex.apk](https://downloads.kitenet.net/git-annex/android/current/) **beta** +[[Linux|linux_standalone]] | [download prebuilt linux tarball](https://downloads.kitenet.net/git-annex/linux/current/)   [[Debian]] | `apt-get install git-annex`   [[Ubuntu]] | `apt-get install git-annex`   [[Fedora]] | `yum install git-annex` @@ -16,7 +16,7 @@ detailed instructions | quick install   [[ScientificLinux5]] |   [[openSUSE]] |   [[Docker]] | -[[Windows]] | [download installer](http://downloads.kitenet.net/git-annex/windows/current/) **beta** +[[Windows]] | [download installer](https://downloads.kitenet.net/git-annex/windows/current/) **beta** """]] All the downloads above use https for security. For added security, see From 508355a5cfb080b6dc31189aba85405422100fe9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 24 Oct 2014 13:58:32 -0400 Subject: [PATCH 13/20] use https here too --- doc/install/verifying_downloads.mdwn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/install/verifying_downloads.mdwn b/doc/install/verifying_downloads.mdwn index c3413d4315..ca588ceecb 100644 --- a/doc/install/verifying_downloads.mdwn +++ b/doc/install/verifying_downloads.mdwn @@ -9,8 +9,8 @@ file accompanying your package. Just append .sig to the url. For example, on Linux: - $ wget http://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-amd64.tar.gz - $ wget http://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-amd64.tar.gz.sig + $ wget https://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-amd64.tar.gz + $ wget https://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-amd64.tar.gz.sig You can then download the public key, and check that the package is signed with it. From 027e30a5df850223249bdc000f03e66819d9d248 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 24 Oct 2014 14:02:53 -0400 Subject: [PATCH 14/20] I had left off the last byte of my key id --- doc/install/verifying_downloads.mdwn | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/install/verifying_downloads.mdwn b/doc/install/verifying_downloads.mdwn index ca588ceecb..b3664488f9 100644 --- a/doc/install/verifying_downloads.mdwn +++ b/doc/install/verifying_downloads.mdwn @@ -26,6 +26,8 @@ But, how do you know that the gpg-pubkey.asc you downloaded is the right key? The answer is the GPG web of trust. * Joey Hess generates these git-annex packages, - and has a GPG key, [C910D9222512E3C Joey Hess ](http://pgp.cs.uu.nl/stats/2512E3C7.html), which has + and has a GPG key, [C910D9222512E3C7 Joey Hess ](http://pgp.cs.uu.nl/stats/2512E3C7.html), which has been verified and signed by many people. * Joey's GPG key has signed the git-annex distribution signing key. + +Don't take this page's word about this, check it yourself! From 2e1c53cf58a27df89c10d87c86964cffeb1ac351 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 24 Oct 2014 14:05:29 -0400 Subject: [PATCH 15/20] be more precise --- doc/install.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/install.mdwn b/doc/install.mdwn index 171177fbf0..478044593b 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -19,7 +19,7 @@ detailed instructions | quick install [[Windows]] | [download installer](https://downloads.kitenet.net/git-annex/windows/current/) **beta** """]] -All the downloads above use https for security. For added security, see +All the download links above use https for security. For added security, see [[verifying_downloads]]. ## Building it yourself From 9ca3e14481c3c130776e4bfdbfd0f464f475cf9f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 24 Oct 2014 17:05:27 -0400 Subject: [PATCH 16/20] update copyright year --- templates/documentation/about.hamlet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/documentation/about.hamlet b/templates/documentation/about.hamlet index cfea1bfc8e..26e512415c 100644 --- a/templates/documentation/about.hamlet +++ b/templates/documentation/about.hamlet @@ -9,7 +9,7 @@ For full details, see # the git-annex website.
- git-annex is © 2010-2013 Joey Hess. It is free software, licensed # + git-annex is © 2010-2014 Joey Hess. It is free software, licensed # under the terms of the GNU General Public License, version 3 or above. # This webapp is licensed under the terms of the GNU Affero General # Public License, version 3 or above. # From da224d83f8016d1e967023131dff963b97e3b619 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawlVB7wf1TFFrNPvM62fg78X-simVV-R1Cg" Date: Sat, 25 Oct 2014 02:36:44 +0000 Subject: [PATCH 17/20] Initial post --- ...irectory_to_rsync_remotes_possible__63__.mdwn | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 doc/forum/Copy_contents_of_directory_to_rsync_remotes_possible__63__.mdwn diff --git a/doc/forum/Copy_contents_of_directory_to_rsync_remotes_possible__63__.mdwn b/doc/forum/Copy_contents_of_directory_to_rsync_remotes_possible__63__.mdwn new file mode 100644 index 0000000000..abc2f8f263 --- /dev/null +++ b/doc/forum/Copy_contents_of_directory_to_rsync_remotes_possible__63__.mdwn @@ -0,0 +1,16 @@ +I'm having trouble sharing an rsync remote between two users (as a "transfer" repository). `git-annex` thinks `rsync` fails because it can't set the permissions on the root directory on the destination. I've added `--perms --chmod=Dug+rwxs,Fug+rw,o-rwx` to `annex-rsync-options` so all users in the specified group on the destination can create and delete files. However, I see in the logs errors like: + + sending incremental file list + ./ + rsync: failed to set permissions on "/home/shared-xfer/.": Operation not permitted (1) + 2fe/985/GPGHMACSHA1--b36ce0a18718e7ea7cf9827c2748608df7152dfc/GPGHMACSHA1--b36ce0a18718e7ea7cf9827c2748608df7152dfc + + 32,768 0% 0.00kB/s 0:00:00 + 3,359,405 100% 4.78MB/s 0:00:00 (xfr#1, to-chk=0/5) + rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1165) [sender=3.1.1] + [2014-10-18 15:16:24 EDT] chat: gpg ["--batch","--no-tty","--use-agent","--quiet","--trust-model","always","--batch","--passphrase-fd","22","--symmetric","--force-mdc","--no-textmode"] + [2014-10-18 15:16:24 EDT] read: rsync ["-e","'ssh' '-S' '.git/annex/ssh/blah@blah' '-o' 'ControlMaster=auto' '-o' 'ControlPersist=yes' '-l' 'blah' '-T'","--perms","--chmod=Dug+rwxs,Fug+rw,o-rwx","--progress","--recursive","--partial","--partial-dir=.rsync-partial","/blah/.git/annex/tmp/rsynctmp/24863/","blah@blah:/home/shared-xfer/"] + + rsync failed -- run git annex again to resume file transfer + +The transfer actually completes fine, except for setting permissions of the root directory on the destination (which I don't really care about, because I have manually set them to be correct). Is it possible for `git-annex` to copy the contents of the directory to rsync remotes? For the example above, instead of `/blah/.git/annex/tmp/rsynctmp/24863/`, `/blah/.git/annex/tmp/rsynctmp/24863/2fe` would work. I can't come up with an alternate solution for this server, except using the same remote user (which I'd prefer not to do). From 04d6a431bbc5b9de6f205ab85d7e6cf2a509f0d2 Mon Sep 17 00:00:00 2001 From: "http://svario.it/gioele" Date: Sat, 25 Oct 2014 20:34:55 +0000 Subject: [PATCH 18/20] new forum post "Recovering deleted file in direct mode" --- .../Recovering_deleted_file_in_direct_mode.mdwn | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/forum/Recovering_deleted_file_in_direct_mode.mdwn diff --git a/doc/forum/Recovering_deleted_file_in_direct_mode.mdwn b/doc/forum/Recovering_deleted_file_in_direct_mode.mdwn new file mode 100644 index 0000000000..9dc1bdaf1f --- /dev/null +++ b/doc/forum/Recovering_deleted_file_in_direct_mode.mdwn @@ -0,0 +1,13 @@ +I accidentally deleted a file from a git-annex repository with a plain "rm". How can I restore it from other repositories that have it? + +I tried using + + git annex copy --from $REMOTE $REMOVED_FILE + +but git-annex complaints about + + $REMOVED_FILE not found + +I suppose that I could switch to indirect mode and do checkout and older version, but I'd prefer not to. My repository contains thousands of quite big files; it would take a lot of time and the probability of something going wrong during the conversion is quite high (see other bugs reported during import). + +I'm using git-annex 5.20141024-g613f396. From be0523f87100c8250732330833c9ac04ebe55db2 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawn3p4i4lk_zMilvjnJ9sS6g2nerpgz0Fjc" Date: Sun, 26 Oct 2014 12:06:25 +0000 Subject: [PATCH 19/20] Added a comment: Keep historical data, but delete data never referenced --- ...ment_2_e2ccd9a209c7a104004e998135d4b675._comment | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/walkthrough/unused_data/comment_2_e2ccd9a209c7a104004e998135d4b675._comment diff --git a/doc/walkthrough/unused_data/comment_2_e2ccd9a209c7a104004e998135d4b675._comment b/doc/walkthrough/unused_data/comment_2_e2ccd9a209c7a104004e998135d4b675._comment new file mode 100644 index 0000000000..91075ce24b --- /dev/null +++ b/doc/walkthrough/unused_data/comment_2_e2ccd9a209c7a104004e998135d4b675._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawn3p4i4lk_zMilvjnJ9sS6g2nerpgz0Fjc" + nickname="Matthias" + subject="Keep historical data, but delete data never referenced" + date="2014-10-26T12:06:25Z" + content=""" +Is there an easy solution for the following? There are two kinds of \"unused\" I would like to treat differently: + +1. Kind \"really unused\": Was added once to the annex, but symlink was never committed +2. Kind \"only history\": A commit contains a symlink to the data, but no active branch + +I want to preserve \"only history\", and only drop \"really unused\". What is an elegant way to do this? Thanks for your suggestions. +"""]] From 20a497b181f73e29f4e5935126a542dfe05b259c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 27 Oct 2014 11:24:21 -0400 Subject: [PATCH 20/20] move remote removal into separate module This allows using Git.Remote w/o needing to have Git.BuildVersion, which requires configure. It will simplify github-backup when these libraries are used there. --- Assistant/DeleteRemote.hs | 3 ++- Assistant/WebApp/Configurators/XMPP.hs | 4 ++-- Assistant/WebApp/Gpg.hs | 3 ++- Git/Remote.hs | 13 ------------- Git/Remote/Remove.hs | 27 ++++++++++++++++++++++++++ 5 files changed, 33 insertions(+), 17 deletions(-) create mode 100644 Git/Remote/Remove.hs diff --git a/Assistant/DeleteRemote.hs b/Assistant/DeleteRemote.hs index a900753a79..968b214c10 100644 --- a/Assistant/DeleteRemote.hs +++ b/Assistant/DeleteRemote.hs @@ -18,6 +18,7 @@ import Assistant.DaemonStatus import qualified Remote import Remote.List import qualified Git.Remote +import qualified Git.Remote.Remove import Logs.Trust import qualified Annex @@ -34,7 +35,7 @@ disableRemote uuid = do remote <- fromMaybe (error "unknown remote") <$> liftAnnex (Remote.remoteFromUUID uuid) liftAnnex $ do - inRepo $ Git.Remote.remove (Remote.name remote) + inRepo $ Git.Remote.Remove.remove (Remote.name remote) void $ remoteListRefresh updateSyncRemotes return remote diff --git a/Assistant/WebApp/Configurators/XMPP.hs b/Assistant/WebApp/Configurators/XMPP.hs index cabae9dbd3..ab2d183758 100644 --- a/Assistant/WebApp/Configurators/XMPP.hs +++ b/Assistant/WebApp/Configurators/XMPP.hs @@ -25,7 +25,7 @@ import Assistant.WebApp.RepoList import Assistant.WebApp.Configurators import Assistant.XMPP #endif -import qualified Git.Remote +import qualified Git.Remote.Remove import Remote.List import Creds @@ -211,7 +211,7 @@ getDisconnectXMPPR = do rs <- filter Remote.isXMPPRemote . syncRemotes <$> liftAssistant getDaemonStatus liftAnnex $ do - mapM_ (inRepo . Git.Remote.remove . Remote.name) rs + mapM_ (inRepo . Git.Remote.Remove.remove . Remote.name) rs void remoteListRefresh removeCreds xmppCredsFile liftAssistant $ do diff --git a/Assistant/WebApp/Gpg.hs b/Assistant/WebApp/Gpg.hs index e7946d1bb7..fa20db2f4b 100644 --- a/Assistant/WebApp/Gpg.hs +++ b/Assistant/WebApp/Gpg.hs @@ -14,6 +14,7 @@ import Assistant.Gpg import Utility.Gpg import qualified Git.Command import qualified Git.Remote +import qualified Git.Remote.Remove import qualified Git.Construct import qualified Annex.Branch import qualified Git.GCrypt @@ -76,7 +77,7 @@ getGCryptRemoteName u repoloc = do (M.lookup "name" <=< M.lookup u) <$> readRemoteLog , return Nothing ) - void $ inRepo $ Git.Remote.remove tmpremote + void $ inRepo $ Git.Remote.Remove.remove tmpremote maybe missing return mname where missing = error $ "Cannot find configuration for the gcrypt remote at " ++ repoloc diff --git a/Git/Remote.hs b/Git/Remote.hs index 7e8e5f8171..156e30891e 100644 --- a/Git/Remote.hs +++ b/Git/Remote.hs @@ -12,8 +12,6 @@ module Git.Remote where import Common import Git import Git.Types -import qualified Git.Command -import qualified Git.BuildVersion import Data.Char import qualified Data.Map as M @@ -44,17 +42,6 @@ makeLegalName s = case filter legal $ replace "/" "_" s of legal '.' = True legal c = isAlphaNum c -remove :: RemoteName -> Repo -> IO () -remove remotename = Git.Command.run - [ Param "remote" - -- name of this subcommand changed - , Param $ - if Git.BuildVersion.older "1.8.0" - then "rm" - else "remove" - , Param remotename - ] - data RemoteLocation = RemoteUrl String | RemotePath FilePath remoteLocationIsUrl :: RemoteLocation -> Bool diff --git a/Git/Remote/Remove.hs b/Git/Remote/Remove.hs new file mode 100644 index 0000000000..949019b220 --- /dev/null +++ b/Git/Remote/Remove.hs @@ -0,0 +1,27 @@ +{- git remote stuff + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Git.Remote.Remove where + +import Common +import Git +import Git.Types +import qualified Git.Command +import qualified Git.BuildVersion + +remove :: RemoteName -> Repo -> IO () +remove remotename = Git.Command.run + [ Param "remote" + -- name of this subcommand changed + , Param $ + if Git.BuildVersion.older "1.8.0" + then "rm" + else "remove" + , Param remotename + ]