Merge branch 'master' into gitlab
This commit is contained in:
Joey Hess
commit
f763f4552e
23 changed files with 572 additions and 226 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -4,7 +4,6 @@ Setup
|
||||||
*.o
|
*.o
|
||||||
tmp
|
tmp
|
||||||
test
|
test
|
||||||
build-stamp
|
|
||||||
Build/SysConfig.hs
|
Build/SysConfig.hs
|
||||||
Build/InstallDesktopFile
|
Build/InstallDesktopFile
|
||||||
Build/EvilSplicer
|
Build/EvilSplicer
|
||||||
|
|
|
||||||
|
|
@ -46,6 +46,8 @@ data AddUrlOptions = AddUrlOptions
|
||||||
{ addUrls :: CmdParams
|
{ addUrls :: CmdParams
|
||||||
, fileOption :: Maybe FilePath
|
, fileOption :: Maybe FilePath
|
||||||
, pathdepthOption :: Maybe Int
|
, pathdepthOption :: Maybe Int
|
||||||
|
, prefixOption :: Maybe String
|
||||||
|
, suffixOption :: Maybe String
|
||||||
, relaxedOption :: Bool
|
, relaxedOption :: Bool
|
||||||
, rawOption :: Bool
|
, rawOption :: Bool
|
||||||
}
|
}
|
||||||
|
|
@ -59,7 +61,15 @@ optParser desc = AddUrlOptions
|
||||||
))
|
))
|
||||||
<*> optional (option auto
|
<*> optional (option auto
|
||||||
( long "pathdepth" <> metavar paramNumber
|
( long "pathdepth" <> metavar paramNumber
|
||||||
<> help "path components to use in filename"
|
<> help "number of url path components to use in filename"
|
||||||
|
))
|
||||||
|
<*> optional (strOption
|
||||||
|
( long "prefix" <> metavar paramValue
|
||||||
|
<> help "add a prefix to the filename"
|
||||||
|
))
|
||||||
|
<*> optional (strOption
|
||||||
|
( long "suffix" <> metavar paramValue
|
||||||
|
<> help "add a suffix to the filename"
|
||||||
))
|
))
|
||||||
<*> parseRelaxedOption
|
<*> parseRelaxedOption
|
||||||
<*> parseRawOption
|
<*> parseRawOption
|
||||||
|
|
@ -80,13 +90,13 @@ seek :: AddUrlOptions -> CommandSeek
|
||||||
seek o = forM_ (addUrls o) $ \u -> do
|
seek o = forM_ (addUrls o) $ \u -> do
|
||||||
r <- Remote.claimingUrl u
|
r <- Remote.claimingUrl u
|
||||||
if Remote.uuid r == webUUID || rawOption o
|
if Remote.uuid r == webUUID || rawOption o
|
||||||
then void $ commandAction $ startWeb (relaxedOption o) (fileOption o) (pathdepthOption o) u
|
then void $ commandAction $ startWeb o u
|
||||||
else checkUrl r u (fileOption o) (relaxedOption o) (pathdepthOption o)
|
else checkUrl r o u
|
||||||
|
|
||||||
checkUrl :: Remote -> URLString -> Maybe FilePath -> Bool -> Maybe Int -> Annex ()
|
checkUrl :: Remote -> AddUrlOptions -> URLString -> Annex ()
|
||||||
checkUrl r u optfile relaxed pathdepth = do
|
checkUrl r o u = do
|
||||||
pathmax <- liftIO $ fileNameLengthLimit "."
|
pathmax <- liftIO $ fileNameLengthLimit "."
|
||||||
let deffile = fromMaybe (urlString2file u pathdepth pathmax) optfile
|
let deffile = fromMaybe (urlString2file u (pathdepthOption o) pathmax) (fileOption o)
|
||||||
go deffile =<< maybe
|
go deffile =<< maybe
|
||||||
(error $ "unable to checkUrl of " ++ Remote.name r)
|
(error $ "unable to checkUrl of " ++ Remote.name r)
|
||||||
(tryNonAsync . flip id u)
|
(tryNonAsync . flip id u)
|
||||||
|
|
@ -98,14 +108,15 @@ checkUrl r u optfile relaxed pathdepth = do
|
||||||
warning (show e)
|
warning (show e)
|
||||||
next $ next $ return False
|
next $ next $ return False
|
||||||
go deffile (Right (UrlContents sz mf)) = do
|
go deffile (Right (UrlContents sz mf)) = do
|
||||||
let f = fromMaybe (maybe deffile fromSafeFilePath mf) optfile
|
let f = adjustFile o (fromMaybe (maybe deffile fromSafeFilePath mf) (fileOption o))
|
||||||
void $ commandAction $
|
void $ commandAction $
|
||||||
startRemote r relaxed f u sz
|
startRemote r (relaxedOption o) f u sz
|
||||||
go deffile (Right (UrlMulti l))
|
go deffile (Right (UrlMulti l))
|
||||||
| isNothing optfile =
|
| isNothing (fileOption o) =
|
||||||
forM_ l $ \(u', sz, f) ->
|
forM_ l $ \(u', sz, f) -> do
|
||||||
|
let f' = adjustFile o (deffile </> fromSafeFilePath f)
|
||||||
void $ commandAction $
|
void $ commandAction $
|
||||||
startRemote r relaxed (deffile </> fromSafeFilePath f) u' sz
|
startRemote r (relaxedOption o) f' u' sz
|
||||||
| otherwise = error $ unwords
|
| otherwise = error $ unwords
|
||||||
[ "That url contains multiple files according to the"
|
[ "That url contains multiple files according to the"
|
||||||
, Remote.name r
|
, Remote.name r
|
||||||
|
|
@ -151,8 +162,8 @@ downloadRemoteFile r relaxed uri file sz = do
|
||||||
where
|
where
|
||||||
loguri = setDownloader uri OtherDownloader
|
loguri = setDownloader uri OtherDownloader
|
||||||
|
|
||||||
startWeb :: Bool -> Maybe FilePath -> Maybe Int -> String -> CommandStart
|
startWeb :: AddUrlOptions -> String -> CommandStart
|
||||||
startWeb relaxed optfile pathdepth s = go $ fromMaybe bad $ parseURI urlstring
|
startWeb o s = go $ fromMaybe bad $ parseURI urlstring
|
||||||
where
|
where
|
||||||
(urlstring, downloader) = getDownloader s
|
(urlstring, downloader) = getDownloader s
|
||||||
bad = fromMaybe (error $ "bad url " ++ urlstring) $
|
bad = fromMaybe (error $ "bad url " ++ urlstring) $
|
||||||
|
|
@ -170,22 +181,22 @@ startWeb relaxed optfile pathdepth s = go $ fromMaybe bad $ parseURI urlstring
|
||||||
#endif
|
#endif
|
||||||
regulardownload url = do
|
regulardownload url = do
|
||||||
pathmax <- liftIO $ fileNameLengthLimit "."
|
pathmax <- liftIO $ fileNameLengthLimit "."
|
||||||
urlinfo <- if relaxed
|
urlinfo <- if relaxedOption o
|
||||||
then pure $ Url.UrlInfo True Nothing Nothing
|
then pure $ Url.UrlInfo True Nothing Nothing
|
||||||
else Url.withUrlOptions (Url.getUrlInfo urlstring)
|
else Url.withUrlOptions (Url.getUrlInfo urlstring)
|
||||||
file <- case optfile of
|
file <- adjustFile o <$> case fileOption o of
|
||||||
Just f -> pure f
|
Just f -> pure f
|
||||||
Nothing -> case Url.urlSuggestedFile urlinfo of
|
Nothing -> case Url.urlSuggestedFile urlinfo of
|
||||||
Nothing -> pure $ url2file url pathdepth pathmax
|
Nothing -> pure $ url2file url (pathdepthOption o) pathmax
|
||||||
Just sf -> do
|
Just sf -> do
|
||||||
let f = truncateFilePath pathmax $
|
let f = truncateFilePath pathmax $
|
||||||
sanitizeFilePath sf
|
sanitizeFilePath sf
|
||||||
ifM (liftIO $ doesFileExist f <||> doesDirectoryExist f)
|
ifM (liftIO $ doesFileExist f <||> doesDirectoryExist f)
|
||||||
( pure $ url2file url pathdepth pathmax
|
( pure $ url2file url (pathdepthOption o) pathmax
|
||||||
, pure f
|
, pure f
|
||||||
)
|
)
|
||||||
showStart "addurl" file
|
showStart "addurl" file
|
||||||
next $ performWeb relaxed urlstring file urlinfo
|
next $ performWeb (relaxedOption o) urlstring file urlinfo
|
||||||
#ifdef WITH_QUVI
|
#ifdef WITH_QUVI
|
||||||
badquvi = error $ "quvi does not know how to download url " ++ urlstring
|
badquvi = error $ "quvi does not know how to download url " ++ urlstring
|
||||||
usequvi = do
|
usequvi = do
|
||||||
|
|
@ -193,11 +204,11 @@ startWeb relaxed optfile pathdepth s = go $ fromMaybe bad $ parseURI urlstring
|
||||||
<$> withQuviOptions Quvi.forceQuery [Quvi.quiet, Quvi.httponly] urlstring
|
<$> withQuviOptions Quvi.forceQuery [Quvi.quiet, Quvi.httponly] urlstring
|
||||||
let link = fromMaybe badquvi $ headMaybe $ Quvi.pageLinks page
|
let link = fromMaybe badquvi $ headMaybe $ Quvi.pageLinks page
|
||||||
pathmax <- liftIO $ fileNameLengthLimit "."
|
pathmax <- liftIO $ fileNameLengthLimit "."
|
||||||
let file = flip fromMaybe optfile $
|
let file = adjustFile o $ flip fromMaybe (fileOption o) $
|
||||||
truncateFilePath pathmax $ sanitizeFilePath $
|
truncateFilePath pathmax $ sanitizeFilePath $
|
||||||
Quvi.pageTitle page ++ "." ++ fromMaybe "m" (Quvi.linkSuffix link)
|
Quvi.pageTitle page ++ "." ++ fromMaybe "m" (Quvi.linkSuffix link)
|
||||||
showStart "addurl" file
|
showStart "addurl" file
|
||||||
next $ performQuvi relaxed urlstring (Quvi.linkUrl link) file
|
next $ performQuvi (relaxedOption o) urlstring (Quvi.linkUrl link) file
|
||||||
#else
|
#else
|
||||||
usequvi = error "not built with quvi support"
|
usequvi = error "not built with quvi support"
|
||||||
#endif
|
#endif
|
||||||
|
|
@ -367,3 +378,9 @@ urlString2file :: URLString -> Maybe Int -> Int -> FilePath
|
||||||
urlString2file s pathdepth pathmax = case Url.parseURIRelaxed s of
|
urlString2file s pathdepth pathmax = case Url.parseURIRelaxed s of
|
||||||
Nothing -> error $ "bad uri " ++ s
|
Nothing -> error $ "bad uri " ++ s
|
||||||
Just u -> url2file u pathdepth pathmax
|
Just u -> url2file u pathdepth pathmax
|
||||||
|
|
||||||
|
adjustFile :: AddUrlOptions -> FilePath -> FilePath
|
||||||
|
adjustFile o = addprefix . addsuffix
|
||||||
|
where
|
||||||
|
addprefix f = maybe f (++ f) (prefixOption o)
|
||||||
|
addsuffix f = maybe f (f ++) (suffixOption o)
|
||||||
|
|
|
||||||
8
Makefile
8
Makefile
|
|
@ -12,9 +12,7 @@ ifdef VIM
|
||||||
all=fast
|
all=fast
|
||||||
endif
|
endif
|
||||||
|
|
||||||
build: build-stamp
|
build: $(all)
|
||||||
build-stamp: $(all)
|
|
||||||
touch $@
|
|
||||||
|
|
||||||
Build/SysConfig.hs: configure.hs Build/TestConfig.hs Build/Configure.hs
|
Build/SysConfig.hs: configure.hs Build/TestConfig.hs Build/Configure.hs
|
||||||
if [ "$(CABAL)" = ./Setup ]; then ghc --make Setup; fi
|
if [ "$(CABAL)" = ./Setup ]; then ghc --make Setup; fi
|
||||||
|
|
@ -87,7 +85,7 @@ docs: mans
|
||||||
|
|
||||||
clean:
|
clean:
|
||||||
rm -rf tmp dist git-annex $(mans) configure *.tix .hpc \
|
rm -rf tmp dist git-annex $(mans) configure *.tix .hpc \
|
||||||
doc/.ikiwiki html dist tags Build/SysConfig.hs build-stamp \
|
doc/.ikiwiki html dist tags Build/SysConfig.hs \
|
||||||
Setup Build/InstallDesktopFile Build/EvilSplicer \
|
Setup Build/InstallDesktopFile Build/EvilSplicer \
|
||||||
Build/Standalone Build/OSXMkLibs Build/LinuxMkLibs \
|
Build/Standalone Build/OSXMkLibs Build/LinuxMkLibs \
|
||||||
Build/DistributionUpdate Build/BuildVersion \
|
Build/DistributionUpdate Build/BuildVersion \
|
||||||
|
|
@ -264,4 +262,4 @@ distributionupdate:
|
||||||
ghc -Wall --make Build/DistributionUpdate -XPackageImports -optP-include -optPdist/build/autogen/cabal_macros.h
|
ghc -Wall --make Build/DistributionUpdate -XPackageImports -optP-include -optPdist/build/autogen/cabal_macros.h
|
||||||
./Build/DistributionUpdate
|
./Build/DistributionUpdate
|
||||||
|
|
||||||
.PHONY: git-annex git-union-merge git-recover-repository tags build-stamp
|
.PHONY: git-annex git-union-merge tags
|
||||||
|
|
|
||||||
13
debian/changelog
vendored
13
debian/changelog
vendored
|
|
@ -7,16 +7,18 @@ git-annex (5.20150714) UNRELEASED; urgency=medium
|
||||||
edge cases of option parsing. (For example, the metadata command no
|
edge cases of option parsing. (For example, the metadata command no
|
||||||
longer accepts the combination of --get and --set, which never actually
|
longer accepts the combination of --get and --set, which never actually
|
||||||
worked.)
|
worked.)
|
||||||
* Bash completion code is built-in to git-annex, and can be enabled by
|
* Bash completion file is now included in the git-annex source tree,
|
||||||
running: source <(git-annex --bash-completion-script git-annex)
|
and installed into Debian package (and any other packages built using make
|
||||||
|
install). This bash completion is generated by the option parser, so it
|
||||||
|
covers all commands, all options, and will never go out of date!
|
||||||
* As well as tab completing "git-annex" commands, "git annex" will also tab
|
* As well as tab completing "git-annex" commands, "git annex" will also tab
|
||||||
complete. However, git's bash completion script needs a patch,
|
complete. However, git's bash completion script needs a patch,
|
||||||
which I've submitted, for this to work prefectly.
|
which I've submitted, for this to work prefectly.
|
||||||
* Debian package (and any other packages built using make install)
|
|
||||||
now includes bash completion.
|
|
||||||
* version --raw now works when run outside a git repository.
|
* version --raw now works when run outside a git repository.
|
||||||
* assistant --startdelay now works when run outside a git repository.
|
* assistant --startdelay now works when run outside a git repository.
|
||||||
* dead now accepts multiple --key options.
|
* dead now accepts multiple --key options.
|
||||||
|
* addurl now accepts --prefix and --suffix options to adjust the
|
||||||
|
filenames used.
|
||||||
* sync --content: Fix bug that caused files to be uploaded to eg,
|
* sync --content: Fix bug that caused files to be uploaded to eg,
|
||||||
more archive remotes than wanted copies, only to later be dropped
|
more archive remotes than wanted copies, only to later be dropped
|
||||||
to satisfy the preferred content settings.
|
to satisfy the preferred content settings.
|
||||||
|
|
@ -25,6 +27,9 @@ git-annex (5.20150714) UNRELEASED; urgency=medium
|
||||||
permalinks in rss feeds, it now also looks at guids.
|
permalinks in rss feeds, it now also looks at guids.
|
||||||
* importfeed: Look at not only permalinks, but now also guids
|
* importfeed: Look at not only permalinks, but now also guids
|
||||||
to identify previously downloaded files.
|
to identify previously downloaded files.
|
||||||
|
* Adjust debian build deps: The webapp can now build on arm64, s390x
|
||||||
|
and hurd-i386. WebDAV support is also available on those architectures.
|
||||||
|
* Debian package now maintained by Richard Hartmann.
|
||||||
|
|
||||||
-- Joey Hess <id@joeyh.name> Fri, 10 Jul 2015 16:36:42 -0400
|
-- Joey Hess <id@joeyh.name> Fri, 10 Jul 2015 16:36:42 -0400
|
||||||
|
|
||||||
|
|
|
||||||
28
debian/control
vendored
28
debian/control
vendored
|
|
@ -15,7 +15,6 @@ Build-Depends:
|
||||||
libghc-aws-dev (>= 0.9.2-2~),
|
libghc-aws-dev (>= 0.9.2-2~),
|
||||||
libghc-conduit-dev,
|
libghc-conduit-dev,
|
||||||
libghc-resourcet-dev,
|
libghc-resourcet-dev,
|
||||||
libghc-dav-dev (>= 1.0) [amd64 armel armhf i386 kfreebsd-amd64 kfreebsd-i386 powerpc ppc64el hurd-i386],
|
|
||||||
libghc-quickcheck2-dev,
|
libghc-quickcheck2-dev,
|
||||||
libghc-monad-control-dev (>= 0.3),
|
libghc-monad-control-dev (>= 0.3),
|
||||||
libghc-exceptions-dev (>= 0.6),
|
libghc-exceptions-dev (>= 0.6),
|
||||||
|
|
@ -32,17 +31,18 @@ Build-Depends:
|
||||||
libghc-stm-dev (>= 2.3),
|
libghc-stm-dev (>= 2.3),
|
||||||
libghc-dbus-dev (>= 0.10.7) [linux-any],
|
libghc-dbus-dev (>= 0.10.7) [linux-any],
|
||||||
libghc-fdo-notify-dev (>= 0.3) [linux-any],
|
libghc-fdo-notify-dev (>= 0.3) [linux-any],
|
||||||
libghc-yesod-dev (>= 1.2.6.1) [i386 amd64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el],
|
libghc-yesod-dev (>= 1.2.6.1) [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386],
|
||||||
libghc-yesod-core-dev (>= 1.2.19) [i386 amd64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el],
|
libghc-yesod-core-dev (>= 1.2.19) [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386],
|
||||||
libghc-yesod-form-dev (>= 1.3.15) [i386 amd64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el],
|
libghc-yesod-form-dev (>= 1.3.15) [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386],
|
||||||
libghc-yesod-static-dev (>= 1.2.4) [i386 amd64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el],
|
libghc-yesod-static-dev (>= 1.2.4) [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386],
|
||||||
libghc-yesod-default-dev (>= 1.2.0) [i386 amd64 armel armhf kfreebsd-amd64 powerpc ppc64el],
|
libghc-yesod-default-dev (>= 1.2.0) [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386],
|
||||||
libghc-shakespeare-dev (>= 2.0.0) [i386 amd64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el],
|
libghc-shakespeare-dev (>= 2.0.0) [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386],
|
||||||
libghc-clientsession-dev [i386 amd64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el],
|
libghc-clientsession-dev [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386],
|
||||||
libghc-warp-dev (>= 3.0.0.5) [i386 amd64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el],
|
libghc-warp-dev (>= 3.0.0.5) [i386 amd64 armel armhf armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386],
|
||||||
libghc-warp-tls-dev [i386 amd64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el],
|
libghc-warp-tls-dev [i386 amd64 armel armhf armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386],
|
||||||
libghc-wai-dev [i386 amd64 kfreebsd-i386 armel armhf kfreebsd-amd64 powerpc ppc64el],
|
libghc-wai-dev [i386 amd64 armel armhf armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386],
|
||||||
libghc-wai-extra-dev [i386 amd64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el],
|
libghc-wai-extra-dev [i386 amd64 armel armhf armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386],
|
||||||
|
libghc-dav-dev (>= 1.0) [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 powerpc ppc64el s390x hurd-i386],
|
||||||
libghc-securemem-dev,
|
libghc-securemem-dev,
|
||||||
libghc-byteable-dev,
|
libghc-byteable-dev,
|
||||||
libghc-dns-dev,
|
libghc-dns-dev,
|
||||||
|
|
@ -70,7 +70,7 @@ Build-Depends:
|
||||||
libghc-tasty-quickcheck-dev,
|
libghc-tasty-quickcheck-dev,
|
||||||
libghc-tasty-rerun-dev,
|
libghc-tasty-rerun-dev,
|
||||||
libghc-optparse-applicative-dev (>= 0.10),
|
libghc-optparse-applicative-dev (>= 0.10),
|
||||||
lsof [!kfreebsd-i386 !kfreebsd-amd64 !hurd-any],
|
lsof [linux-any],
|
||||||
ikiwiki,
|
ikiwiki,
|
||||||
perlmagick,
|
perlmagick,
|
||||||
git (>= 1:1.8.1),
|
git (>= 1:1.8.1),
|
||||||
|
|
@ -79,7 +79,7 @@ Build-Depends:
|
||||||
curl,
|
curl,
|
||||||
openssh-client,
|
openssh-client,
|
||||||
git-remote-gcrypt (>= 0.20130908-6),
|
git-remote-gcrypt (>= 0.20130908-6),
|
||||||
Maintainer: Gergely Nagy <algernon@madhouse-project.org>
|
Maintainer: Richard Hartmann <richih@debian.org>
|
||||||
Standards-Version: 3.9.6
|
Standards-Version: 3.9.6
|
||||||
Vcs-Git: git://git.kitenet.net/git-annex
|
Vcs-Git: git://git.kitenet.net/git-annex
|
||||||
Homepage: http://git-annex.branchable.com/
|
Homepage: http://git-annex.branchable.com/
|
||||||
|
|
|
||||||
46
doc/bugs/OOM_while_configuring_git-annex.mdwn
Normal file
46
doc/bugs/OOM_while_configuring_git-annex.mdwn
Normal file
|
|
@ -0,0 +1,46 @@
|
||||||
|
### Please describe the problem.
|
||||||
|
When running configure, the system runs out of memory:
|
||||||
|
|
||||||
|
[[https://kojipkgs.fedoraproject.org//work/tasks/1587/10431587/build.log]]
|
||||||
|
|
||||||
|
built with versions from:
|
||||||
|
|
||||||
|
[[https://kojipkgs.fedoraproject.org//work/tasks/1587/10431587/root.log]]
|
||||||
|
|
||||||
|
### What steps will reproduce the problem?
|
||||||
|
Rebuilding on koji. Also occurs in mock. Haven't tried a naked build (yet).
|
||||||
|
|
||||||
|
### What version of git-annex are you using? On what operating system?
|
||||||
|
Currently, 5.20141203. Newest snapshot requires deps not yet packaged in Fedora.
|
||||||
|
|
||||||
|
### Please provide any additional information below.
|
||||||
|
|
||||||
|
[[!format sh """
|
||||||
|
+ ./Setup configure --prefix=/usr --libdir=/usr/lib --docdir=/usr/share/doc/git-annex '--libsubdir=$compiler/$pkgid' '--datasubdir=$pkgid' --ghc --enable-executable-dynamic '--ghc-options= -optc-O2 -optc-g -optc-pipe -optc-Wall -optc-Werror=format-security -optc-Wp,-D_FORTIFY_SOURCE=2 -optc-fexceptions -optc-fstack-protector-strong -optc--param=ssp-buffer-size=4 -optc-grecord-gcc-switches -optc-specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -optc-m32 -optc-march=i686 -optc-mtune=atom -optc-fasynchronous-unwind-tables -optl-Wl,-z,relro'
|
||||||
|
checking version...fatal: Not a git repository (or any of the parent directories): .git
|
||||||
|
5.20141203
|
||||||
|
checking UPGRADE_LOCATION... not available
|
||||||
|
checking git... yes
|
||||||
|
checking git version... 2.4.6
|
||||||
|
checking cp -a... yes
|
||||||
|
checking cp -p... yes
|
||||||
|
checking cp --preserve=timestamps... yes
|
||||||
|
checking cp --reflink=auto... yes
|
||||||
|
checking xargs -0... yes
|
||||||
|
checking rsync... yes
|
||||||
|
checking curl... yes
|
||||||
|
checking wget... no
|
||||||
|
checking bup... no
|
||||||
|
checking nice... yes
|
||||||
|
checking ionice... yes
|
||||||
|
checking nocache... no
|
||||||
|
checking gpg... gpg2
|
||||||
|
checking lsof... not available
|
||||||
|
checking git-remote-gcrypt... not available
|
||||||
|
checking ssh connection caching... yes
|
||||||
|
checking sha1... sha1sum
|
||||||
|
checking sha256... sha256sum
|
||||||
|
checking sha512... sha512sum
|
||||||
|
checking sha224... sha224sum
|
||||||
|
checking sha384...Setup: out of memory (requested 1048576 bytes)
|
||||||
|
"""]]
|
||||||
|
|
@ -0,0 +1,10 @@
|
||||||
|
[[!comment format=mdwn
|
||||||
|
username="joey"
|
||||||
|
subject="""comment 1"""
|
||||||
|
date="2015-07-22T03:36:11Z"
|
||||||
|
content="""
|
||||||
|
This is due to cabal's dependency resolver not finding a solution and
|
||||||
|
recursing too much. Passing -v3 will show what the dependency problem is.
|
||||||
|
|
||||||
|
(Using stackage is a good way to avoid these kinds of dependency problems.)
|
||||||
|
"""]]
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
[[!comment format=mdwn
|
||||||
|
username="threadshuffle"
|
||||||
|
subject="same problem"
|
||||||
|
date="2015-07-21T19:01:52Z"
|
||||||
|
content="""
|
||||||
|
Hi, i have the same problem. I can't compile from source but ive looked at the process and it seems that GIT_SSH/GIT_ANNEX_SSHOPTION are set ok. Ive also looked through the source even though my haskell is quite limited and eventually managed to get something working by bypassing the
|
||||||
|
<pre><code>g' <- addGitEnv g sshOptionsEnv val
|
||||||
|
addGitEnv g' \"GIT_SSH\" command</code></pre>
|
||||||
|
lines but i can't remember right now how i did it, though i remember it was by setting something similar to GIT_SSH (probably just it to 'c:\program...\git\bin\git.exe' or ssh.exe with a combination of GIT_ANNEX_SSHOPTION='').
|
||||||
|
It didn't show the 'git annex unknown command' stuff but it did mess up the repo (said it synced the annex/direct/master synced/* branches but the changed didn't appear on the remote/origin).
|
||||||
|
If there's anything i can try without actually compiling, im all up for it.
|
||||||
|
"""]]
|
||||||
|
|
@ -0,0 +1,49 @@
|
||||||
|
### Please describe the problem.
|
||||||
|
|
||||||
|
It seems that the `synced/git-annex` branch (which I am a little confused about in the first place), doesn't get synced automatically by the assistant.
|
||||||
|
|
||||||
|
I was expecting the assistant to regularly do the equivalent of `git annex sync`. However, after a client pushed changes to the `git-annex` branch, I had to manually do a `git annex sync` for the changes of the `synced/git-annex` branch to be merged down into the local `git-annex`.
|
||||||
|
|
||||||
|
### What steps will reproduce the problem?
|
||||||
|
|
||||||
|
I have 3 machines in this setup (to simplify: there are more, but those are sufficient). Let's call them foo, bar and quux. foo and quuex are connected to bar through password-less SSH connexions.
|
||||||
|
|
||||||
|
foo commits a file to git-annex. the assistant syncs that to bar in the `synced/git-annex` branch.
|
||||||
|
|
||||||
|
quux syncs with bar, and seems to ignore the `synced/git-annex` branch.
|
||||||
|
|
||||||
|
git-annex sync on quux syncs the `synced/git-annex` branch into the local `git-annex`, working around the issue.
|
||||||
|
|
||||||
|
### What version of git-annex are you using? On what operating system?
|
||||||
|
|
||||||
|
foo is 5.20150610+gitg608172f-1~ndall+1 on Debian 7.8 (wheezy).
|
||||||
|
|
||||||
|
bar and quux are 5.20150409+git126-ga29f683-1~ndall+1 and 5.20150610+gitg608172f-1~ndall+1 (respectively) on Ubuntu 12.04 (precise) .
|
||||||
|
|
||||||
|
### Please provide any additional information below.
|
||||||
|
|
||||||
|
I guess a more general question is how and how often do those branches get merged by the assistant... it's still unclear to me how this works.
|
||||||
|
|
||||||
|
[[!format sh """
|
||||||
|
$ sudo -u www-data -H git annex sync
|
||||||
|
(merging origin/synced/git-annex into git-annex...)
|
||||||
|
(recording state in git...)
|
||||||
|
commit ok
|
||||||
|
pull origin
|
||||||
|
Auto packing the repository in background for optimum performance.
|
||||||
|
See "git help gc" for manual housekeeping.
|
||||||
|
|
||||||
|
Already up-to-date.
|
||||||
|
ok
|
||||||
|
push origin
|
||||||
|
Counting objects: 373637, done.
|
||||||
|
# End of transcript or log.
|
||||||
|
"""]]
|
||||||
|
|
||||||
|
This was started at 20:23 UTC. Note that the sync had run previously under the assistant:
|
||||||
|
|
||||||
|
<pre>
|
||||||
|
[2015-07-22 20:09:06 UTC] RemoteControl: Syncing with origin
|
||||||
|
</pre>
|
||||||
|
|
||||||
|
Available, as usual, for further debugging. :) -- [[anarcat]]
|
||||||
30
doc/devblog/day_302-305__gitlab.mdwn
Normal file
30
doc/devblog/day_302-305__gitlab.mdwn
Normal file
|
|
@ -0,0 +1,30 @@
|
||||||
|
I've been working on adding GitLab support to the webapp for the past 3
|
||||||
|
days.
|
||||||
|
|
||||||
|
That's not the only thing I've been working on; I've continued to work on
|
||||||
|
the older parts of the backlog, which is now shrunk to 91 messages, and
|
||||||
|
made some minor improvements and bugfixes.
|
||||||
|
|
||||||
|
But, GitLab support in the webapp has certianly taken longer than I'd have
|
||||||
|
expected. Only had to write 82 lines of GitLab specific code so far, but it
|
||||||
|
went slowly. The user will need to cut and paste repository url and
|
||||||
|
ssh public key back and forth between the webapp and GitLab for now. And
|
||||||
|
the way GitLab repositories use git-annex makes it a bit tricky to set up;
|
||||||
|
in one case the webapp has to do a forced push dry run to check if the
|
||||||
|
repository on GitLab can be accessed by ssh.
|
||||||
|
|
||||||
|
I found a way to adapt the existing code for setting up a ssh server to
|
||||||
|
also support GitLab, so beyond the repo url prompt and ssh key setup,
|
||||||
|
everything will be reused. I have something that works now, but there are
|
||||||
|
lots of cases to test (encrypted repositories, enabling existing
|
||||||
|
repositories, etc), so will need to work on it a bit more before merging
|
||||||
|
this feature.
|
||||||
|
|
||||||
|
Also took some time to split the [centralized git repository tutorial](http://git-annex.branchable.com/tips/centralized_git_repository_tutorial/)
|
||||||
|
into three parts, one for each of GitHub, GitLab, and self-administered servers.
|
||||||
|
|
||||||
|
----
|
||||||
|
|
||||||
|
The git-annex package in Debian unstable hasn't been updated for 8 months.
|
||||||
|
This is about to change; Richard Hartmann has stepped up and is preparing
|
||||||
|
an upload of a recent version. Yay!
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
[[!comment format=mdwn
|
||||||
|
username="fusionx86@2cab672ef75a8502e153ceb90177dde38985dba9"
|
||||||
|
nickname="fusionx86"
|
||||||
|
subject="comment 5"
|
||||||
|
date="2015-07-20T20:41:00Z"
|
||||||
|
content="""
|
||||||
|
Also, I created git-annex-shell.exe as a copy of git-annex.exe which you recommended here: https://git-annex.branchable.com/bugs/no_git-annex_shell_on_Windows/.
|
||||||
|
|
||||||
|
I also made sure UAC and Windows Firewall were disabled and they already were.
|
||||||
|
|
||||||
|
What environment variables should exist to support annex? I see references to GIT_ANNEX_SSHOPTION and GIT_SSH, but those aren't currently configured on my Windows server. The git, git-annex and git-annex-shell files are in the global path and accessible from anywhere.
|
||||||
|
"""]]
|
||||||
|
|
@ -48,13 +48,22 @@ be used to get better filenames.
|
||||||
|
|
||||||
* `--pathdepth=N`
|
* `--pathdepth=N`
|
||||||
|
|
||||||
This causes a shorter filename to be used. For example,
|
Rather than basing the filename on the whole url, this causes a path to
|
||||||
`--pathdepth=1` will use "dir/subdir/bigfile",
|
be constructed, starting at the specified depth within the path of the
|
||||||
while `--pathdepth=3` will use "bigfile".
|
url.
|
||||||
|
|
||||||
|
For example, adding the url http://www.example.com/dir/subdir/bigfile
|
||||||
|
with `--pathdepth=1` will use "dir/subdir/bigfile",
|
||||||
|
while `--pathdepth=3` will use "bigfile".
|
||||||
|
|
||||||
It can also be negative; `--pathdepth=-2` will use the last
|
It can also be negative; `--pathdepth=-2` will use the last
|
||||||
two parts of the url.
|
two parts of the url.
|
||||||
|
|
||||||
|
* `--prefix=foo` `--suffix=bar`
|
||||||
|
|
||||||
|
Use to adjust the filenames that are created by addurl. For example,
|
||||||
|
`--suffix=.mp3` can be used to add an extension to the file.
|
||||||
|
|
||||||
# SEE ALSO
|
# SEE ALSO
|
||||||
|
|
||||||
[[git-annex]](1)
|
[[git-annex]](1)
|
||||||
|
|
|
||||||
|
|
@ -34,7 +34,7 @@ To make the import process add metadata to the imported files from the feed,
|
||||||
|
|
||||||
Other available variables for templates: feedauthor, itemauthor, itemsummary, itemdescription, itemrights, itemid, itempubdate, title, author
|
Other available variables for templates: feedauthor, itemauthor, itemsummary, itemdescription, itemrights, itemid, itempubdate, title, author
|
||||||
|
|
||||||
* `--relaxed`, `--fast`, `--raw`, `--template`
|
* `--relaxed`, `--fast`, `--raw`
|
||||||
|
|
||||||
These options behave the same as when using [[git-annex-addurl]](1).
|
These options behave the same as when using [[git-annex-addurl]](1).
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -31,32 +31,7 @@ don't want things to be system wide)
|
||||||
|
|
||||||
$ export PATH=/usr/hs/bin:$PATH
|
$ export PATH=/usr/hs/bin:$PATH
|
||||||
|
|
||||||
Once the packages are installed and are in your execution path, using
|
Once the GHC packages are installed and are in your execution path, using
|
||||||
cabal to configure and build git-annex just makes life easier, it
|
cabal to build git-annex just makes life easier, it
|
||||||
should install all the needed dependancies.
|
should install all the needed dependancies. See "minimal build with cabal
|
||||||
|
and stackage" in [[fromsource]] for instructions.
|
||||||
$ cabal update
|
|
||||||
$ git clone git://git.kitenet.net/git-annex
|
|
||||||
$ cd git-annex
|
|
||||||
$ make git-annex.1
|
|
||||||
$ cabal configure
|
|
||||||
$ cabal build
|
|
||||||
$ cabal install
|
|
||||||
|
|
||||||
Or if you want to install it globallly for everyone (otherwise it will
|
|
||||||
get installed into $HOME/.cabal/bin)
|
|
||||||
|
|
||||||
$ cabal install --global
|
|
||||||
|
|
||||||
The above will take a while to compile and install the needed
|
|
||||||
dependancies. I would suggest any user who does should run the tests
|
|
||||||
that comes with git-annex to make sure everything is functioning as
|
|
||||||
expected.
|
|
||||||
|
|
||||||
I haven't had a chance or need to install git-annex on a SL6 based
|
|
||||||
system yet, but I would assume something similar to the above steps
|
|
||||||
would be required to do so.
|
|
||||||
|
|
||||||
The above is almost a cut and paste of <http://jcftang.github.com/2012/06/15/installing-git-annex-on-sl5/>, the above could probably be refined, it was what worked for me on SL5. Please feel free to re-edit and chop out or add useless bits of text in the above!
|
|
||||||
|
|
||||||
Note: from the minor testing, it appears the compiled binaries from SL5 will work on SL6.
|
|
||||||
|
|
|
||||||
|
|
@ -48,8 +48,8 @@ which is a more stable and consistent version of the Hackage repository.
|
||||||
|
|
||||||
Inside the source tree, run:
|
Inside the source tree, run:
|
||||||
|
|
||||||
cabal configure -f"-assistant -webapp -webdav -pairing -xmpp -dns"
|
|
||||||
cabal install -j -f"-assistant -webapp -webdav -pairing -xmpp -dns" --only-dependencies
|
cabal install -j -f"-assistant -webapp -webdav -pairing -xmpp -dns" --only-dependencies
|
||||||
|
cabal configure -f"-assistant -webapp -webdav -pairing -xmpp -dns"
|
||||||
cabal build -j
|
cabal build -j
|
||||||
PATH=$HOME/bin:$PATH
|
PATH=$HOME/bin:$PATH
|
||||||
cabal install --bindir=$HOME/bin
|
cabal install --bindir=$HOME/bin
|
||||||
|
|
@ -65,8 +65,8 @@ Using [Stackage](http://www.stackage.org/) is again a good idea here!
|
||||||
|
|
||||||
Once the C libraries are installed, run inside the source tree:
|
Once the C libraries are installed, run inside the source tree:
|
||||||
|
|
||||||
cabal configure
|
|
||||||
cabal install -j --only-dependencies
|
cabal install -j --only-dependencies
|
||||||
|
cabal configure
|
||||||
cabal build -j
|
cabal build -j
|
||||||
PATH=$HOME/bin:$PATH
|
PATH=$HOME/bin:$PATH
|
||||||
cabal install --bindir=$HOME/bin
|
cabal install --bindir=$HOME/bin
|
||||||
|
|
|
||||||
|
|
@ -1,142 +1,18 @@
|
||||||
The [[walkthrough]] builds up a decentralized git repository setup, but
|
The [[walkthrough]] builds up a decentralized git repository setup, but
|
||||||
git-annex can also be used with a centralized bare repository, just like
|
git-annex can also be used with a centralized git repository.
|
||||||
git can. This tutorial shows how to set up a centralized repository hosted on
|
|
||||||
GitHub on GitLab or your own git server.
|
|
||||||
|
|
||||||
## set up the repository, and make a checkout
|
We have separate tutorials depending on where the centralized git
|
||||||
|
repository is hosted.
|
||||||
|
|
||||||
I've created a repository for technical talk videos, which you can
|
* [[centralized_git_repository_tutorial/On_GitHub]] --
|
||||||
[fork on Github](https://github.com/joeyh/techtalks).
|
However, GitHub does not currently let git-annex
|
||||||
Or make your own repository on GitHub (or GitLab elsewhere) now.
|
store the contents of large files there. So, things get a little more
|
||||||
|
complicated when using it.
|
||||||
|
|
||||||
On your laptop, [[install]] git-annex, and clone the repository:
|
* [[centralized_git_repository_tutorial/On_GitLab]] --
|
||||||
|
This service is similar to GitHub, but supports
|
||||||
|
git-annex.
|
||||||
|
|
||||||
# git clone git@github.com:joeyh/techtalks.git
|
* [[centralized_git_repository_tutorial/On_your_own_server]] --
|
||||||
# cd techtalks
|
use any unix system with ssh and git and git-annex installed.
|
||||||
|
A VPS, a home server, etc.
|
||||||
Tell git-annex to use the repository, and describe where this clone is
|
|
||||||
located:
|
|
||||||
|
|
||||||
# git annex init 'my laptop'
|
|
||||||
init my laptop ok
|
|
||||||
|
|
||||||
Let's tell git-annex that GitHub doesn't support running git-annex-shell there.
|
|
||||||
|
|
||||||
# git config remote.origin.annex-ignore true
|
|
||||||
|
|
||||||
This means you can't store annexed file *contents* on GitHub; it would
|
|
||||||
really be better to host the bare repository on your own server, which
|
|
||||||
would not have this limitation. (If you want to do that, check out
|
|
||||||
[[using_gitolite_with_git-annex]].) Or, you could use GitLab, which
|
|
||||||
*does* [support git-annex on their servers](https://about.gitlab.com/2015/02/17/gitlab-annex-solves-the-problem-of-versioning-large-binaries-with-git/).
|
|
||||||
|
|
||||||
## add files to the repository
|
|
||||||
|
|
||||||
Add some files, obtained however.
|
|
||||||
|
|
||||||
# youtube-dl -t 'http://www.youtube.com/watch?v=b9FagOVqxmI'
|
|
||||||
# git annex add *.mp4
|
|
||||||
add Haskell_Amuse_Bouche-b9FagOVqxmI.mp4 (checksum) ok
|
|
||||||
(Recording state in git...)
|
|
||||||
# git commit -m "added a video. I have not watched it yet but it sounds interesting"
|
|
||||||
|
|
||||||
This file is available directly from the web; so git-annex can download it:
|
|
||||||
|
|
||||||
# git annex addurl http://kitenet.net/~joey/screencasts/git-annex_coding_in_haskell.ogg
|
|
||||||
addurl kitenet.net_~joey_screencasts_git-annex_coding_in_haskell.ogg
|
|
||||||
(downloading http://kitenet.net/~joey/screencasts/git-annex_coding_in_haskell.ogg ...)
|
|
||||||
(checksum...) ok
|
|
||||||
(Recording state in git...)
|
|
||||||
# git commit -a -m 'added a screencast I made'
|
|
||||||
|
|
||||||
Feel free to rename the files, etc, using normal git commands:
|
|
||||||
|
|
||||||
# git mv Haskell_Amuse_Bouche-b9FagOVqxmI.mp4 Haskell_Amuse_Bouche.mp4
|
|
||||||
# git mv kitenet.net_~joey_screencasts_git-annex_coding_in_haskell.ogg git-annex_coding_in_haskell.ogg
|
|
||||||
# git commit -m 'better filenames'
|
|
||||||
|
|
||||||
Now push your changes back to the central repository. As well as pushing
|
|
||||||
the master branch, remember to push the git-annex branch, which is used to
|
|
||||||
track the file contents.
|
|
||||||
|
|
||||||
# git push origin master git-annex
|
|
||||||
To git@github.com:joeyh/techtalks.git
|
|
||||||
* [new branch] master -> master
|
|
||||||
* [new branch] git-annex -> git-annex
|
|
||||||
|
|
||||||
That push went fast, because it didn't upload large videos to GitHub.
|
|
||||||
To check this, you can ask git-annex where the contents of the videos are:
|
|
||||||
|
|
||||||
# git annex whereis
|
|
||||||
whereis Haskell_Amuse_Bouche.mp4 (1 copy)
|
|
||||||
767e8558-0955-11e1-be83-cbbeaab7fff8 -- here
|
|
||||||
ok
|
|
||||||
whereis git-annex_coding_in_haskell.ogg (2 copies)
|
|
||||||
00000000-0000-0000-0000-000000000001 -- web
|
|
||||||
767e8558-0955-11e1-be83-cbbeaab7fff8 -- here
|
|
||||||
ok
|
|
||||||
|
|
||||||
## make more checkouts
|
|
||||||
|
|
||||||
So far you have a central repository, and a checkout on a laptop.
|
|
||||||
Let's make another checkout that's used as a backup. You can put it anywhere
|
|
||||||
you like, just make it be somewhere your laptop can access. A few options:
|
|
||||||
|
|
||||||
* Put it on a USB drive that you can plug into the laptop.
|
|
||||||
* Put it on a desktop.
|
|
||||||
* Put it on some server in the local network.
|
|
||||||
* Put it on a remote VPS.
|
|
||||||
|
|
||||||
I'll use the VPS option, but these instructions should work for
|
|
||||||
any of the above.
|
|
||||||
|
|
||||||
# ssh server
|
|
||||||
server# sudo apt-get install git-annex
|
|
||||||
|
|
||||||
Clone the central repository as before. (If the clone fails, you need
|
|
||||||
to add your server's ssh public key to github -- see
|
|
||||||
[this page](http://help.github.com/ssh-issues/).)
|
|
||||||
|
|
||||||
server# git clone git@github.com:joeyh/techtalks.git
|
|
||||||
server# cd techtalks
|
|
||||||
server# git config remote.origin.annex-ignore true
|
|
||||||
server# git annex init 'backup'
|
|
||||||
init backup (merging origin/git-annex into git-annex...) ok
|
|
||||||
|
|
||||||
Notice that the server does not have the contents of any of the files yet.
|
|
||||||
If you run `ls`, you'll see broken symlinks. We want to populate this
|
|
||||||
backup with the file contents, by copying them from your laptop.
|
|
||||||
|
|
||||||
Back on your laptop, you need to configure a git remote for the backup.
|
|
||||||
Adjust the ssh url as needed to point to wherever the backup is. (If it
|
|
||||||
was on a local USB drive, you'd use the path to the repository instead.)
|
|
||||||
|
|
||||||
# git remote add backup ssh://server/~/techtalks
|
|
||||||
|
|
||||||
Now git-annex on your laptop knows how to reach the backup repository,
|
|
||||||
and can do things like copy files to it:
|
|
||||||
|
|
||||||
# git annex copy --to backup git-annex_coding_in_haskell.ogg
|
|
||||||
copy git-annex_coding_in_haskell.ogg (checking backup...)
|
|
||||||
12877824 2% 255.11kB/s 00:00
|
|
||||||
ok
|
|
||||||
|
|
||||||
You can also `git annex move` files to it, to free up space on your laptop.
|
|
||||||
And then you can `git annex get` files back to your laptop later on, as
|
|
||||||
desired.
|
|
||||||
|
|
||||||
After you use git-annex to move files around, remember to push,
|
|
||||||
which will broadcast its updated location information.
|
|
||||||
|
|
||||||
# git push origin master git-annex
|
|
||||||
|
|
||||||
## take it farther
|
|
||||||
|
|
||||||
Of course you can create as many checkouts as you desire. If you have a
|
|
||||||
desktop machine too, you can make a checkout there, and use `git remote
|
|
||||||
add` to also let your desktop access the backup repository.
|
|
||||||
|
|
||||||
You can add remotes for each direct connection between machines you find you
|
|
||||||
need -- so make the laptop have the desktop as a remote, and the desktop
|
|
||||||
have the laptop as a remote, and then on either machine git-annex can
|
|
||||||
access files stored on the other.
|
|
||||||
|
|
|
||||||
129
doc/tips/centralized_git_repository_tutorial/on_GitHub.mdwn
Normal file
129
doc/tips/centralized_git_repository_tutorial/on_GitHub.mdwn
Normal file
|
|
@ -0,0 +1,129 @@
|
||||||
|
This tutorial shows how to set up a centralized repository hosted on
|
||||||
|
GitHub.
|
||||||
|
|
||||||
|
GitHub does not currently let git-annex store the contents of large files
|
||||||
|
there. This doesn't prevent using git-annex with GitHub, it just means you
|
||||||
|
have to set up some other centralized location for the large files.
|
||||||
|
|
||||||
|
## set up the repository, and make a checkout
|
||||||
|
|
||||||
|
I've created a repository for technical talk videos, which you can
|
||||||
|
[fork on Github](https://github.com/joeyh/techtalks).
|
||||||
|
Or make your own repository on GitHub now.
|
||||||
|
|
||||||
|
On your laptop, [[install]] git-annex, and clone the repository:
|
||||||
|
|
||||||
|
# git clone git@github.com:joeyh/techtalks.git
|
||||||
|
# cd techtalks
|
||||||
|
|
||||||
|
Tell git-annex to use the repository, and describe where this clone is
|
||||||
|
located:
|
||||||
|
|
||||||
|
# git annex init 'my laptop'
|
||||||
|
init my laptop ok
|
||||||
|
|
||||||
|
## add files to the repository
|
||||||
|
|
||||||
|
Add some files, obtained however.
|
||||||
|
|
||||||
|
# git annex add *.mp4
|
||||||
|
add Haskell_Amuse_Bouche-b9OVqxmI.mp4 (checksum) ok
|
||||||
|
(Recording state in git...)
|
||||||
|
# git commit -m "added a video. I have not watched it yet but it sounds interesting"
|
||||||
|
|
||||||
|
This file is available on the web; so git-annex can download it:
|
||||||
|
|
||||||
|
# git annex addurl http://kitenet.net/~joey/screencasts/git-annex_coding_in_haskell.ogg
|
||||||
|
addurl kitenet.net_~joey_screencasts_git-annex_coding_in_haskell.ogg
|
||||||
|
(downloading http://kitenet.net/~joey/screencasts/git-annex_coding_in_haskell.ogg ...)
|
||||||
|
(checksum...) ok
|
||||||
|
(Recording state in git...)
|
||||||
|
# git commit -a -m 'added a screencast I made'
|
||||||
|
|
||||||
|
Feel free to rename the files, etc, using normal git commands:
|
||||||
|
|
||||||
|
# git mv Haskell_Amuse_Bouche-b9OVqxmI.mp4 Haskell_Amuse_Bouche.mp4
|
||||||
|
# git mv kitenet.net_~joey_screencasts_git-annex_coding_in_haskell.ogg git-annex_coding_in_haskell.ogg
|
||||||
|
# git commit -m 'better filenames'
|
||||||
|
|
||||||
|
Now push your changes back to the central repository on GitHub. As well as
|
||||||
|
pushing the master branch, remember to push the git-annex branch, which is
|
||||||
|
used to track the file contents. You can do this push manually as shown
|
||||||
|
below, or you can just run `git annex sync` to do the same thing.
|
||||||
|
|
||||||
|
# git push origin master git-annex
|
||||||
|
To git@github.com:joeyh/techtalks.git
|
||||||
|
* [new branch] master -> master
|
||||||
|
* [new branch] git-annex -> git-annex
|
||||||
|
|
||||||
|
That push went fast, because it didn't upload large videos to GitHub.
|
||||||
|
To check this, you can ask git-annex where the contents of the videos are:
|
||||||
|
|
||||||
|
# git annex whereis
|
||||||
|
whereis Haskell_Amuse_Bouche.mp4 (1 copy)
|
||||||
|
767e8558-0955-11e1-be83-cbbeaab7fff8 -- here
|
||||||
|
ok
|
||||||
|
whereis git-annex_coding_in_haskell.ogg (2 copies)
|
||||||
|
00000000-0000-0000-0000-000000000001 -- web
|
||||||
|
767e8558-0955-11e1-be83-cbbeaab7fff8 -- here
|
||||||
|
ok
|
||||||
|
|
||||||
|
## make more checkouts
|
||||||
|
|
||||||
|
So far you have a central repository, and a checkout on a laptop.
|
||||||
|
You, or anyone you allow to can clone the central repository, and
|
||||||
|
use git-annex with it.
|
||||||
|
|
||||||
|
But, since GitHub doesn't currently support storing large files there
|
||||||
|
with git-annex, other checkouts of your repository won't be able to
|
||||||
|
access the files you added to the repository on your laptop.
|
||||||
|
|
||||||
|
# git clone git@github.com:myrepo/techtalks.git
|
||||||
|
# git annex get Haskell_Amuse_Bouche-b9OVqxmI.mp4
|
||||||
|
get Haskell_Amuse_Bouche-b9OVqxmI.mp4
|
||||||
|
|
||||||
|
Try making some of these repositories available:
|
||||||
|
767e8558-0955-11e1-be83-cbbeaab7fff8 -- my laptop
|
||||||
|
failed
|
||||||
|
|
||||||
|
## add a special remote
|
||||||
|
|
||||||
|
So, to complete your setup, you need to set up a repository where git-annex
|
||||||
|
can store the contents of large files. This is often done by setting up
|
||||||
|
a [[special_remote|special_remotes]]. One free option is explained in
|
||||||
|
[[using_box.com_as_a_special_remote]]. Another useful approach is
|
||||||
|
explained in [[public_Amazon_S3_remote]].
|
||||||
|
|
||||||
|
Once you have the special remote set up on your laptop, you can
|
||||||
|
send files to it:
|
||||||
|
|
||||||
|
# git annex copy --to myspecialremote Haskell_Amuse_Bouche-b9OVqxmI.mp4
|
||||||
|
copy Haskell_Amuse_Bouche-b9OVqxmI.mp4 (to myspecialremote...)
|
||||||
|
100% 255.11kB/s
|
||||||
|
ok
|
||||||
|
|
||||||
|
You can also `git annex move` files to it, to free up space on your laptop.
|
||||||
|
And then you can `git annex get` files back to your laptop later on, as
|
||||||
|
desired.
|
||||||
|
|
||||||
|
After you use git-annex to move files around, remember to sync,
|
||||||
|
which will broadcast its updated location information.
|
||||||
|
|
||||||
|
# git annex sync
|
||||||
|
|
||||||
|
After setting up the special remote and storing some files on it,
|
||||||
|
you can download them on other clones. You'll first need to enable the same
|
||||||
|
special remote on the clones.
|
||||||
|
|
||||||
|
# git annex sync
|
||||||
|
# git annex enableremote myspecialremote
|
||||||
|
# git annex get git-annex_coding_in_haskell.ogg
|
||||||
|
100% 255.11kB/s
|
||||||
|
ok
|
||||||
|
|
||||||
|
## take it farther
|
||||||
|
|
||||||
|
You can add remotes for each direct connection between machines you find you
|
||||||
|
need -- so make the laptop have the desktop as a remote, and the desktop
|
||||||
|
have the laptop as a remote, and then on either machine git-annex can
|
||||||
|
access files stored on the other.
|
||||||
76
doc/tips/centralized_git_repository_tutorial/on_GitLab.mdwn
Normal file
76
doc/tips/centralized_git_repository_tutorial/on_GitLab.mdwn
Normal file
|
|
@ -0,0 +1,76 @@
|
||||||
|
This tutorial shows how to set up a centralized repository hosted on
|
||||||
|
GitLab.
|
||||||
|
|
||||||
|
Since GitLab has [added support for git-annex on their servers](https://about.gitlab.com/2015/02/17/gitlab-annex-solves-the-problem-of-versioning-large-binaries-with-git/),
|
||||||
|
you can store your large files on GitLab, quite easily.
|
||||||
|
|
||||||
|
Note that as I'm writing this, GitLab is providing this service for free,
|
||||||
|
and I don't know how much data they're willing to host for free.
|
||||||
|
|
||||||
|
## create the repository
|
||||||
|
|
||||||
|
Go to <https://gitlab.com/> and sign up for an account, and create the
|
||||||
|
repository there. Take note of the SSH clone url for the repository, which
|
||||||
|
will be something like `git@gitlab.com:yourlogin/annex.git`.
|
||||||
|
|
||||||
|
We want to clone this locally, on your laptop. (If the clone fails, you
|
||||||
|
need to generate a ssh key and add it to GitLab.)
|
||||||
|
|
||||||
|
# git clone git@gitlab.com:yourlogin/annex.git
|
||||||
|
# cd annex
|
||||||
|
|
||||||
|
Tell git-annex to use the repository, and describe where this clone is
|
||||||
|
located:
|
||||||
|
|
||||||
|
# git annex init 'my laptop'
|
||||||
|
init my laptop ok
|
||||||
|
|
||||||
|
Add some files, obtained however.
|
||||||
|
|
||||||
|
# git annex add *.mp4
|
||||||
|
add Haskell_Amuse_Bouche-b9OVqxmI.mp4 (checksum) ok
|
||||||
|
(Recording state in git...)
|
||||||
|
# git commit -m "added a video. I have not watched it yet but it sounds interesting"
|
||||||
|
|
||||||
|
Feel free to rename the files, etc, using normal git commands:
|
||||||
|
|
||||||
|
# git mv Haskell_Amuse_Bouche-b9OVqxmI.mp4 Haskell_Amuse_Bouche.mp4
|
||||||
|
# git commit -m 'better filenames'
|
||||||
|
|
||||||
|
## push to GitLab
|
||||||
|
|
||||||
|
Now make a first push to the GitLab repository.
|
||||||
|
As well as pushing the master branch, remember to push the git-annex
|
||||||
|
branch, which is used to track the file contents.
|
||||||
|
|
||||||
|
# git push origin master git-annex
|
||||||
|
To git@gitlab.com:yourlogin/annex.git
|
||||||
|
* [new branch] master -> master
|
||||||
|
* [new branch] git-annex -> git-annex
|
||||||
|
|
||||||
|
That push went fast, because it didn't upload the large file contents yet.
|
||||||
|
|
||||||
|
So, to finish up, tell git-annex to sync all the data in the repository
|
||||||
|
to GitLab:
|
||||||
|
|
||||||
|
# git annex sync --content
|
||||||
|
...
|
||||||
|
|
||||||
|
## make more checkouts
|
||||||
|
|
||||||
|
So far you have a central repository on GitLab, and a checkout on a laptop.
|
||||||
|
Let's make another checkout elsewhere. Clone the central repository as before.
|
||||||
|
(If the clone fails, you need to generate a ssh key and add it to GitLab.)
|
||||||
|
|
||||||
|
elsewhere# git clone git@gitlab.com:yourlogin/annex.git
|
||||||
|
elsewhere# cd annex
|
||||||
|
|
||||||
|
Notice that your clone does not have the contents of any of the files yet.
|
||||||
|
If you run `ls`, you'll see broken symlinks. It's easy to download them from
|
||||||
|
GitLab either by running `git annex sync --content`, or by asking
|
||||||
|
git-annex to download individual files:
|
||||||
|
|
||||||
|
# git annex get Haskell_Amuse_Bouche.mp4
|
||||||
|
get Haskell_Amuse_Bouche.mp4 (from origin...)
|
||||||
|
12877824 2% 255.11kB/s 00:00
|
||||||
|
ok
|
||||||
|
|
@ -0,0 +1,88 @@
|
||||||
|
This tutorial shows how to set up a centralized git repository
|
||||||
|
hosted on your own git server, which can be any unix system with
|
||||||
|
ssh and git and git-annex installed. A VPS, a home server, etc.
|
||||||
|
|
||||||
|
This sets up a very simple git server. More complex setups are possible.
|
||||||
|
See for example [[using_gitolite_with_git-annex]].
|
||||||
|
|
||||||
|
## set up the server
|
||||||
|
|
||||||
|
On the server, you'll want to [[install]] git, and git-annex, if you haven't
|
||||||
|
already.
|
||||||
|
|
||||||
|
server# sudo apt-get install git git-annex
|
||||||
|
|
||||||
|
Decide where to put the repository on the server, and create a bare git repo
|
||||||
|
there. In your home directory is a simple choice:
|
||||||
|
|
||||||
|
server# cd
|
||||||
|
server# git init annex.git --bare --shared
|
||||||
|
|
||||||
|
That's the server setup done!
|
||||||
|
|
||||||
|
## make a checkout
|
||||||
|
|
||||||
|
Now on your laptop, clone the git repository from the server:
|
||||||
|
|
||||||
|
laptop# git clone ssh://example.com/~/annex.git
|
||||||
|
Cloning into 'annex'...
|
||||||
|
warning: You appear to have cloned an empty repository.
|
||||||
|
Checking connectivity... done.
|
||||||
|
|
||||||
|
|
||||||
|
Tell git-annex to use the repository, and describe where this clone is
|
||||||
|
located:
|
||||||
|
|
||||||
|
|
||||||
|
laptop# cd annex
|
||||||
|
laptop# git annex init 'my laptop'
|
||||||
|
init my laptop ok
|
||||||
|
|
||||||
|
## add files to the repository
|
||||||
|
|
||||||
|
Add some files, obtained however.
|
||||||
|
|
||||||
|
# git annex add *.mp4
|
||||||
|
add Haskell_Amuse_Bouche-b9OVqxmI.mp4 (checksum) ok
|
||||||
|
(Recording state in git...)
|
||||||
|
# git commit -m "added a video. I have not watched it yet but it sounds interesting"
|
||||||
|
|
||||||
|
Feel free to rename the files, etc, using normal git commands:
|
||||||
|
|
||||||
|
# git mv Haskell_Amuse_Bouche-b9OVqxmI.mp4 Haskell_Amuse_Bouche.mp4
|
||||||
|
# git commit -m 'better filenames'
|
||||||
|
|
||||||
|
Now push your changes back to the central repository on your server. As
|
||||||
|
well as pushing the master branch, remember to push the git-annex branch,
|
||||||
|
which is used to track the file contents.
|
||||||
|
|
||||||
|
# git push origin master git-annex
|
||||||
|
To git@github.com:joeyh/techtalks.git
|
||||||
|
* [new branch] master -> master
|
||||||
|
* [new branch] git-annex -> git-annex
|
||||||
|
|
||||||
|
That push went fast, because it didn't upload large videos to the server.
|
||||||
|
|
||||||
|
So, to finish up, tell git-annex to sync all the data in the repository
|
||||||
|
to your server:
|
||||||
|
|
||||||
|
# git annex sync --content
|
||||||
|
...
|
||||||
|
|
||||||
|
## make more checkouts
|
||||||
|
|
||||||
|
So far you have a central repository on your server, and a checkout on a laptop.
|
||||||
|
Let's make another checkout elsewhere. Clone the central repository as before.
|
||||||
|
|
||||||
|
elsewhere# git clone ssh://example.com/~/annex.git
|
||||||
|
elsewhere# cd annex
|
||||||
|
|
||||||
|
Notice that your clone does not have the contents of any of the files yet.
|
||||||
|
If you run `ls`, you'll see broken symlinks. It's easy to download them from
|
||||||
|
your server either by running `git annex sync --content`, or by asking
|
||||||
|
git-annex to download individual files:
|
||||||
|
|
||||||
|
# git annex get Haskell_Amuse_Bouche.mp4
|
||||||
|
get Haskell_Amuse_Bouche.mp4 (from origin...)
|
||||||
|
12877824 2% 255.11kB/s 00:00
|
||||||
|
ok
|
||||||
|
|
@ -1 +1,4 @@
|
||||||
Is it possible to add a '--dir' option to addurl (or some other mechanic) to make git annex create the symlinks in the specified directory?
|
Is it possible to add a '--dir' option to addurl (or some other mechanic) to make git annex create the symlinks in the specified directory?
|
||||||
|
|
||||||
|
> --prefix makes sense, and might as well also add --suffix. [[done]]
|
||||||
|
> --[[Joey]]
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,13 @@
|
||||||
|
[[!comment format=mdwn
|
||||||
|
username="CandyAngel"
|
||||||
|
subject="comment 2"
|
||||||
|
date="2015-07-21T12:11:45Z"
|
||||||
|
content="""
|
||||||
|
I was hoping something like tar's *-C*, or unzip's *-d* options would be easy to implement, given that git-annex will already create required directories when using --pathdepth. As far as I can tell, you can't mix --file and --pathdepth.. I'd have to reimplemnt --pathdepth and the directory creation in my script, which while not a major problem in Perl (aside from reinventing a potentially inferior wheel), may be if someone is using shell scripting.
|
||||||
|
|
||||||
|
Maybe this request can be more generalised to *--prefix*?
|
||||||
|
|
||||||
|
Then people could use --prefix to either just prefix the filename or set a directory for it to be put into. I had a look at Command/AddUrl.hs to see how feasible that would be but I'm struggling to follow it (not knowing Haskell yet).
|
||||||
|
|
||||||
|
In the meantime(?), I have made my script remember the original directory it was started in and it cd's about as needed.
|
||||||
|
"""]]
|
||||||
|
|
@ -0,0 +1,7 @@
|
||||||
|
[[!comment format=mdwn
|
||||||
|
username="CandyAngel"
|
||||||
|
subject="comment 3"
|
||||||
|
date="2015-07-22T07:41:53Z"
|
||||||
|
content="""
|
||||||
|
Yay! Awesome, thank you :)
|
||||||
|
"""]]
|
||||||
|
|
@ -105,10 +105,6 @@ Flag network-uri
|
||||||
Description: Get Network.URI from the network-uri package
|
Description: Get Network.URI from the network-uri package
|
||||||
Default: True
|
Default: True
|
||||||
|
|
||||||
Flag new-time
|
|
||||||
Description: Build with new version of time and without old-locale
|
|
||||||
Default: True
|
|
||||||
|
|
||||||
Executable git-annex
|
Executable git-annex
|
||||||
Main-Is: git-annex.hs
|
Main-Is: git-annex.hs
|
||||||
Build-Depends:
|
Build-Depends:
|
||||||
|
|
@ -127,7 +123,8 @@ Executable git-annex
|
||||||
monad-control, transformers,
|
monad-control, transformers,
|
||||||
bloomfilter, edit-distance,
|
bloomfilter, edit-distance,
|
||||||
resourcet, http-conduit, http-types,
|
resourcet, http-conduit, http-types,
|
||||||
esqueleto, persistent-sqlite, persistent, persistent-template
|
esqueleto, persistent-sqlite, persistent, persistent-template,
|
||||||
|
time, old-locale
|
||||||
CC-Options: -Wall
|
CC-Options: -Wall
|
||||||
GHC-Options: -Wall -fno-warn-tabs
|
GHC-Options: -Wall -fno-warn-tabs
|
||||||
Extensions: PackageImports
|
Extensions: PackageImports
|
||||||
|
|
@ -144,11 +141,6 @@ Executable git-annex
|
||||||
else
|
else
|
||||||
Build-Depends: network (< 2.6), network (>= 2.0)
|
Build-Depends: network (< 2.6), network (>= 2.0)
|
||||||
|
|
||||||
if flag(new-time)
|
|
||||||
Build-Depends: time (>= 1.5)
|
|
||||||
else
|
|
||||||
Build-Depends: time, old-locale
|
|
||||||
|
|
||||||
if flag(Production)
|
if flag(Production)
|
||||||
GHC-Options: -O2
|
GHC-Options: -O2
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue