S3: Support signature=anonymous to access a S3 bucket anonymously

This can be used, for example, with importtree=yes to import from a public
bucket.

This needs a patch that has not yet landed in the aws library, and will
need to be adjusted to support compiling with old versions of the library,
so is not yet suitable for merging.
See https://github.com/aristidb/aws/pull/281

The stack.yaml changes are provided to show how to build against the aws
fork and will need to be reverted as well.

Sponsored-by: Dartmouth College's DANDI project
This commit is contained in:
Joey Hess 2022-10-10 16:52:18 -04:00
parent 90f9671e00
commit ca91c3ba91
No known key found for this signature in database
GPG key ID: DB12DB0FF05F8F38
4 changed files with 35 additions and 16 deletions

View file

@ -6,6 +6,9 @@ git-annex (10.20221004) UNRELEASED; urgency=medium
do not operate on a repository that has an empty name. do not operate on a repository that has an empty name.
* move: Fix openFile crash with -J * move: Fix openFile crash with -J
(Fixes a reversion in 8.20201103) (Fixes a reversion in 8.20201103)
* S3: Support signature=anonymous to access a S3 bucket anonymously.
This can be used, for example, with importtree=yes to import from
a public bucket.
-- Joey Hess <id@joeyh.name> Mon, 03 Oct 2022 13:36:42 -0400 -- Joey Hess <id@joeyh.name> Mon, 03 Oct 2022 13:36:42 -0400

View file

@ -1,6 +1,6 @@
{- S3 remotes {- S3 remotes
- -
- Copyright 2011-2020 Joey Hess <id@joeyh.name> - Copyright 2011-2022 Joey Hess <id@joeyh.name>
- -
- Licensed under the GNU AGPL version 3 or higher. - Licensed under the GNU AGPL version 3 or higher.
-} -}
@ -156,7 +156,9 @@ requeststyleField = Accepted "requeststyle"
signatureField :: RemoteConfigField signatureField :: RemoteConfigField
signatureField = Accepted "signature" signatureField = Accepted "signature"
newtype SignatureVersion = SignatureVersion Int data SignatureVersion
= SignatureVersion Int
| Anonymous
signatureVersionParser :: RemoteConfigField -> FieldDesc -> RemoteConfigFieldParser signatureVersionParser :: RemoteConfigField -> FieldDesc -> RemoteConfigFieldParser
signatureVersionParser f fd = signatureVersionParser f fd =
@ -165,10 +167,17 @@ signatureVersionParser f fd =
where where
go "v2" = Just (SignatureVersion 2) go "v2" = Just (SignatureVersion 2)
go "v4" = Just (SignatureVersion 4) go "v4" = Just (SignatureVersion 4)
go "anonymous" = Just Anonymous
go _ = Nothing go _ = Nothing
defver = SignatureVersion 2 defver = SignatureVersion 2
isAnonymous :: ParsedRemoteConfig -> Bool
isAnonymous c =
case getRemoteConfigValue signatureField c of
Just Anonymous -> True
_ -> False
portField :: RemoteConfigField portField :: RemoteConfigField
portField = Accepted "port" portField = Accepted "port"
@ -272,7 +281,9 @@ s3Setup' ss u mcreds c gc
(c', encsetup) <- encryptionSetup (c `M.union` defaults) gc (c', encsetup) <- encryptionSetup (c `M.union` defaults) gc
pc <- either giveup return . parseRemoteConfig c' pc <- either giveup return . parseRemoteConfig c'
=<< configParser remote c' =<< configParser remote c'
c'' <- setRemoteCredPair ss encsetup pc gc (AWS.creds u) mcreds c'' <- if isAnonymous pc
then pure c'
else setRemoteCredPair ss encsetup pc gc (AWS.creds u) mcreds
pc' <- either giveup return . parseRemoteConfig c'' pc' <- either giveup return . parseRemoteConfig c''
=<< configParser remote c'' =<< configParser remote c''
info <- extractS3Info pc' info <- extractS3Info pc'
@ -286,7 +297,9 @@ s3Setup' ss u mcreds c gc
showNote "Internet Archive mode" showNote "Internet Archive mode"
pc <- either giveup return . parseRemoteConfig c pc <- either giveup return . parseRemoteConfig c
=<< configParser remote c =<< configParser remote c
c' <- setRemoteCredPair ss noEncryptionUsed pc gc (AWS.creds u) mcreds c' <- if isAnonymous pc
then pure c
else setRemoteCredPair ss noEncryptionUsed pc gc (AWS.creds u) mcreds
-- Ensure user enters a valid bucket name, since -- Ensure user enters a valid bucket name, since
-- this determines the name of the archive.org item. -- this determines the name of the archive.org item.
let validbucket = replace " " "-" $ map toLower $ let validbucket = replace " " "-" $ map toLower $
@ -841,17 +854,20 @@ type S3HandleVar = TVar (Either (Annex (Maybe S3Handle)) (Maybe S3Handle))
{- Prepares a S3Handle for later use. Does not connect to S3 or do anything {- Prepares a S3Handle for later use. Does not connect to S3 or do anything
- else expensive. -} - else expensive. -}
mkS3HandleVar :: ParsedRemoteConfig -> RemoteGitConfig -> UUID -> Annex S3HandleVar mkS3HandleVar :: ParsedRemoteConfig -> RemoteGitConfig -> UUID -> Annex S3HandleVar
mkS3HandleVar c gc u = liftIO $ newTVarIO $ Left $ do mkS3HandleVar c gc u = liftIO $ newTVarIO $ Left $
mcreds <- getRemoteCredPair c gc (AWS.creds u) if isAnonymous c
case mcreds of then go =<< liftIO AWS.anonymousCredentials
Just creds -> do else do
awscreds <- liftIO $ genCredentials creds mcreds <- getRemoteCredPair c gc (AWS.creds u)
let awscfg = AWS.Configuration AWS.Timestamp awscreds debugMapper Nothing case mcreds of
ou <- getUrlOptions Just creds -> go =<< liftIO (genCredentials creds)
return $ Just $ S3Handle (httpManager ou) awscfg s3cfg Nothing -> return Nothing
Nothing -> return Nothing
where where
s3cfg = s3Configuration c s3cfg = s3Configuration c
go awscreds = do
let awscfg = AWS.Configuration AWS.Timestamp awscreds debugMapper Nothing
ou <- getUrlOptions
return $ Just $ S3Handle (httpManager ou) awscfg s3cfg
withS3Handle :: S3HandleVar -> (Maybe S3Handle -> Annex a) -> Annex a withS3Handle :: S3HandleVar -> (Maybe S3Handle -> Annex a) -> Annex a
withS3Handle hv a = liftIO (readTVarIO hv) >>= \case withS3Handle hv a = liftIO (readTVarIO hv) >>= \case

View file

@ -82,6 +82,7 @@ the S3 remote.
* `signature` - This controls the S3 signature version to use. * `signature` - This controls the S3 signature version to use.
"v2" is currently the default, "v4" is needed to use some S3 services. "v2" is currently the default, "v4" is needed to use some S3 services.
If you get some kind of authentication error, try "v4". If you get some kind of authentication error, try "v4".
To access a S3 bucket anonymously, use "anonymous".
* `bucket` - S3 requires that buckets have a globally unique name, * `bucket` - S3 requires that buckets have a globally unique name,
so by default, a bucket name is chosen based on the remote name so by default, a bucket name is chosen based on the remote name

View file

@ -11,10 +11,10 @@ flags:
gitlfs: true gitlfs: true
packages: packages:
- '.' - '.'
resolver: lts-18.13 resolver: lts-19.16
extra-deps: extra-deps:
- IfElse-0.85 - IfElse-0.85
- aws-0.22 - /home/joey/tmp/aws
- bloomfilter-2.0.1.0 - bloomfilter-2.0.1.0
- git-lfs-1.2.0 - git-lfs-1.2.0
- http-client-restricted-0.0.4 - http-client-restricted-0.0.4
@ -24,4 +24,3 @@ extra-deps:
- base16-bytestring-0.1.1.7 - base16-bytestring-0.1.1.7
- base64-bytestring-1.0.0.3 - base64-bytestring-1.0.0.3
- bencode-0.6.1.1 - bencode-0.6.1.1
- http-client-0.7.9