From 4b87669ae229c89eadb4ff88eba927e105c003c4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Nov 2024 16:09:27 -0400 Subject: [PATCH] S3 use last Key when there is no Marker element Fix infinite loop and memory blowup when importing from an unversioned S3 bucket that is large enough to need pagination. I don't think there actually ever will be a Marker element, a delimiter is not set. Probably this code path was never tested with pagination! Also the aws library's lack of any docs made it easy to mess up. Versioned buckets seem to not have the same problem. The API docs for ListObjectVersions say that NextKeyMarker will always be provided when paginating. --- CHANGELOG | 2 ++ Remote/S3.hs | 25 +++++++++++++++++-------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 5931689e59..c80d47f2b0 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -11,6 +11,8 @@ git-annex (10.20241032) UNRELEASED; urgency=medium (Needs aws-0.24.3) * S3: Send git-annex or other configured User-Agent. (Needs aws-0.24.3) + * S3: Fix infinite loop and memory blowup when importing from an + unversioned S3 bucket that is large enough to need pagination. -- Joey Hess Mon, 11 Nov 2024 12:26:00 -0400 diff --git a/Remote/S3.hs b/Remote/S3.hs index b7d13f11f5..299f7d7644 100644 --- a/Remote/S3.hs +++ b/Remote/S3.hs @@ -589,14 +589,23 @@ listImportableContentsS3 hv r info c = continuelistunversioned h [] rsp continuelistunversioned h l rsp - | S3.gbrIsTruncated rsp = do - rsp' <- sendS3Handle h $ - (S3.getBucket (bucket info)) - { S3.gbMarker = S3.gbrNextMarker rsp - , S3.gbPrefix = fileprefix - } - continuelistunversioned h (rsp:l) rsp' - | otherwise = return $ + | S3.gbrIsTruncated rsp = + let marker = + S3.gbrNextMarker rsp + <|> + (S3.objectKey <$> lastMaybe (S3.gbrContents rsp)) + in case marker of + Just _ -> do + rsp' <- sendS3Handle h $ + (S3.getBucket (bucket info)) + { S3.gbMarker = marker + , S3.gbPrefix = fileprefix + } + continuelistunversioned h (rsp:l) rsp' + Nothing -> nomore + | otherwise = nomore + where + nomore = return $ mkImportableContentsUnversioned info (reverse (rsp:l)) continuelistversioned h l rsp