From 04ec726d3b54bf475484d7bbd777bbd4e29c960e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 6 Feb 2023 14:07:33 -0400 Subject: [PATCH] S3 region= S3: Support a region= configuration useful for some non-Amazon S3 implementations. This feature needs git-annex to be built with aws-0.24. datacenter= sets both the AWS hostname and region in one setting, which is easy when using AWS, but not useful for other hosts. So kept datacenter as-is, but added this additional config. Sponsored-By: Brett Eisenberg on Patreon --- CHANGELOG | 7 +++++ Remote/S3.hs | 27 ++++++++++++++++--- ...S3_remote_errors_with_garage_endpoint.mdwn | 3 +++ ..._3d2033edcd49e62045339e341996fd28._comment | 10 +++++++ doc/special_remotes/S3.mdwn | 13 ++++++--- 5 files changed, 53 insertions(+), 7 deletions(-) create mode 100644 doc/bugs/S3_remote_errors_with_garage_endpoint/comment_9_3d2033edcd49e62045339e341996fd28._comment diff --git a/CHANGELOG b/CHANGELOG index ee9aa3961d..6cbcaf5535 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,10 @@ +git-annex (10.20230127) UNRELEASED; urgency=medium + + * S3: Support a region= configuration useful for some non-Amazon S3 + implementations. This feature needs git-annex to be built with aws-0.24. + + -- Joey Hess Mon, 06 Feb 2023 13:39:18 -0400 + git-annex (10.20230126) upstream; urgency=medium * Change --metadata comparisons < > <= and >= to fall back to diff --git a/Remote/S3.hs b/Remote/S3.hs index f5014202eb..3ea9ed813d 100644 --- a/Remote/S3.hs +++ b/Remote/S3.hs @@ -1,6 +1,6 @@ {- S3 remotes - - - Copyright 2011-2022 Joey Hess + - Copyright 2011-2023 Joey Hess - - Licensed under the GNU AGPL version 3 or higher. -} @@ -83,6 +83,8 @@ remote = specialRemoteType $ RemoteType (FieldDesc "S3 server hostname (default is Amazon S3)") , optionalStringParser datacenterField (FieldDesc "S3 datacenter to use (US, EU, us-west-1, ..)") + , optionalStringParser regionField + (FieldDesc "S3 region to use") , optionalStringParser partsizeField (FieldDesc "part size for multipart upload (eg 1GiB)") , optionalStringParser storageclassField @@ -129,6 +131,9 @@ hostField = Accepted "host" datacenterField :: RemoteConfigField datacenterField = Accepted "datacenter" +regionField :: RemoteConfigField +regionField = Accepted "region" + partsizeField :: RemoteConfigField partsizeField = Accepted "partsize" @@ -914,6 +919,7 @@ s3Configuration c = cfg } where h = fromJust $ getRemoteConfigValue hostField c + r = encodeBS <$> getRemoteConfigValue regionField c datacenter = fromJust $ getRemoteConfigValue datacenterField c -- When the default S3 host is configured, connect directly to -- the S3 endpoint for the configured datacenter. @@ -948,8 +954,14 @@ s3Configuration c = cfg | otherwise -> AWS.HTTP cfg = case getRemoteConfigValue signatureField c of Just (SignatureVersion 4) -> - S3.s3v4 proto endpoint False S3.SignWithEffort - _ -> S3.s3 proto endpoint False + (S3.s3v4 proto endpoint False S3.SignWithEffort) +#if MIN_VERSION_aws(0,24,0) + { S3.s3Region = r } +#endif + _ -> (S3.s3 proto endpoint False) +#if MIN_VERSION_aws(0,24,0) + { S3.s3Region = r } +#endif data S3Info = S3Info { bucket :: S3.Bucket @@ -964,6 +976,7 @@ data S3Info = S3Info , public :: Bool , publicurl :: Maybe URLString , host :: Maybe String + , region :: Maybe String } extractS3Info :: ParsedRemoteConfig -> Annex S3Info @@ -987,6 +1000,7 @@ extractS3Info c = do getRemoteConfigValue publicField c , publicurl = getRemoteConfigValue publicurlField c , host = getRemoteConfigValue hostField c + , region = getRemoteConfigValue regionField c } putObject :: S3Info -> T.Text -> RequestBody -> S3.PutObject @@ -1126,7 +1140,12 @@ debugMapper level t = forward "Remote.S3" (T.unpack t) s3Info :: ParsedRemoteConfig -> S3Info -> [(String, String)] s3Info c info = catMaybes [ Just ("bucket", fromMaybe "unknown" (getBucketName c)) - , Just ("endpoint", w82s (BS.unpack (S3.s3Endpoint s3c))) + , Just ("endpoint", decodeBS (S3.s3Endpoint s3c)) +#if MIN_VERSION_aws(0,24,0) + , case S3.s3Region s3c of + Nothing -> Nothing + Just r -> Just ("region", decodeBS r) +#endif , Just ("port", show (S3.s3Port s3c)) , Just ("protocol", map toLower (show (S3.s3Protocol s3c))) , Just ("storage class", showstorageclass (getStorageClass c)) diff --git a/doc/bugs/S3_remote_errors_with_garage_endpoint.mdwn b/doc/bugs/S3_remote_errors_with_garage_endpoint.mdwn index 1bb1f330eb..a9a7c5a8f0 100644 --- a/doc/bugs/S3_remote_errors_with_garage_endpoint.mdwn +++ b/doc/bugs/S3_remote_errors_with_garage_endpoint.mdwn @@ -28,3 +28,6 @@ Fedora Silverblue 37 / git-annex-10.20221212-1.fc37.x86_64 Yes, many years ago - now trying to get it up and running with my self-hosted S3 endpoint. [1]: https://garagehq.deuxfleurs.fr/ + +> [[fixed|done]] although it needs git-annex to be built against +> a not yet released version of aws. --[[Joey]] diff --git a/doc/bugs/S3_remote_errors_with_garage_endpoint/comment_9_3d2033edcd49e62045339e341996fd28._comment b/doc/bugs/S3_remote_errors_with_garage_endpoint/comment_9_3d2033edcd49e62045339e341996fd28._comment new file mode 100644 index 0000000000..046002c5c9 --- /dev/null +++ b/doc/bugs/S3_remote_errors_with_garage_endpoint/comment_9_3d2033edcd49e62045339e341996fd28._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 9""" + date="2023-02-06T17:33:25Z" + content=""" +Implemented this in aws: https://github.com/aristidb/aws/pull/284 +Which should be released as version 0.24. + +git-annex will support region= when built with that version of aws. +"""]] diff --git a/doc/special_remotes/S3.mdwn b/doc/special_remotes/S3.mdwn index 5bda70f98a..6329bb093b 100644 --- a/doc/special_remotes/S3.mdwn +++ b/doc/special_remotes/S3.mdwn @@ -34,9 +34,12 @@ the S3 remote. Think carefully about who can access your repository before using embedcreds without gpg encryption. -* `datacenter` - Defaults to "US". Other values include "EU" (which is EU/Ireland), - "us-west-1", "us-west-2", "ap-southeast-1", "ap-southeast-2", and - "sa-east-1". +* `datacenter` - Specifies which Amazon datacenter + to use for the bucket. Defaults to "US". Other values include "EU" + (which is EU/Ireland), "us-west-1", "us-west-2", "ap-southeast-1", + "ap-southeast-2", and "sa-east-1". See Amazon's documentation for a + complete list. Configuring this is equivilant to configuring both + `host` and `region`. * `storageclass` - Default is "STANDARD". Consult S3 provider documentation for pricing details and available @@ -63,6 +66,10 @@ the S3 remote. * `host` - Specify in order to use a different, S3 compatable service. +* `region` - Specify the region to use. Only makes sense to use when + you also set `host`. + (Requires a git-annex built with aws-0.24.) + * `protocol` - Either "http" (the default) or "https". Setting protocol=https implies port=443.