work around minimum part size problem

When uploading the last part of a file, which was 640229 bytes, S3 rejected
that part: "Your proposed upload is smaller than the minimum allowed size"

I don't know what the minimum is, but the fix is just to include the last
part into the previous part. Since this can result in a part that's
double-sized, use half-sized parts normally.
This commit is contained in:
Joey Hess 2014-11-04 16:06:13 -04:00
parent ad2125e24a
commit a42022d8ff
2 changed files with 19 additions and 8 deletions

View file

@ -181,9 +181,16 @@ store r h = fileStorer $ \k f p -> do
}
uploadid <- S3.imurUploadId <$> sendS3Handle h startreq
-- The actual part size will be a even multiple of the
-- 32k chunk size that hGetUntilMetered uses.
let partsz' = (partsz `div` toInteger defaultChunkSize) * toInteger defaultChunkSize
{- The actual part size will be a even multiple of the
- 32k chunk size that hGetUntilMetered uses.
-
- Also, half-size parts are used. This is so that
- the final part of a file can be rolled into the
- last full-size part, which avoids a problem when the
- final part could otherwise be too small for S3 to accept
- it.
-}
let partsz' = (partsz `div` toInteger defaultChunkSize `div` 2) * toInteger defaultChunkSize
-- Send parts of the file, taking care to stream each part
-- w/o buffering in memory, since the parts can be large.
@ -195,7 +202,7 @@ store r h = fileStorer $ \k f p -> do
else do
-- Calculate size of part that will
-- be read.
let sz = if fsz - pos < partsz'
let sz = if fsz - pos < partsz' * 2
then fsz - pos
else partsz'
let p' = offsetMeterUpdate p (toBytesProcessed pos)