diff --git a/CHANGELOG b/CHANGELOG index 5931689e59..c80d47f2b0 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -11,6 +11,8 @@ git-annex (10.20241032) UNRELEASED; urgency=medium (Needs aws-0.24.3) * S3: Send git-annex or other configured User-Agent. (Needs aws-0.24.3) + * S3: Fix infinite loop and memory blowup when importing from an + unversioned S3 bucket that is large enough to need pagination. -- Joey Hess Mon, 11 Nov 2024 12:26:00 -0400 diff --git a/Remote/S3.hs b/Remote/S3.hs index b7d13f11f5..299f7d7644 100644 --- a/Remote/S3.hs +++ b/Remote/S3.hs @@ -589,14 +589,23 @@ listImportableContentsS3 hv r info c = continuelistunversioned h [] rsp continuelistunversioned h l rsp - | S3.gbrIsTruncated rsp = do - rsp' <- sendS3Handle h $ - (S3.getBucket (bucket info)) - { S3.gbMarker = S3.gbrNextMarker rsp - , S3.gbPrefix = fileprefix - } - continuelistunversioned h (rsp:l) rsp' - | otherwise = return $ + | S3.gbrIsTruncated rsp = + let marker = + S3.gbrNextMarker rsp + <|> + (S3.objectKey <$> lastMaybe (S3.gbrContents rsp)) + in case marker of + Just _ -> do + rsp' <- sendS3Handle h $ + (S3.getBucket (bucket info)) + { S3.gbMarker = marker + , S3.gbPrefix = fileprefix + } + continuelistunversioned h (rsp:l) rsp' + Nothing -> nomore + | otherwise = nomore + where + nomore = return $ mkImportableContentsUnversioned info (reverse (rsp:l)) continuelistversioned h l rsp