From f008843fc5b15a0a21fcdadcf573de8699a3436c Mon Sep 17 00:00:00 2001
From: Dan Stillman <dstillman@zotero.org>
Date: Thu, 24 Apr 2014 17:51:47 -0400
Subject: [PATCH] Automatically retry intermittent Amazon S3 upload timeouts

"Your socket connection to the server was not read from or written to
within the timeout period."

I can't reproduce these reliably, so this is fairly untested, but it
seems to work. It backs off exponentially when the error occurs, and
halves the delay on successful requests.

Eventually all 50x file sync errors should also be retried
automatically, but that can't really happen in 4.0.
---
 chrome/content/zotero/xpcom/storage/zfs.js | 32 ++++++++++++++++++++--
 1 file changed, 29 insertions(+), 3 deletions(-)

diff --git a/chrome/content/zotero/xpcom/storage/zfs.js b/chrome/content/zotero/xpcom/storage/zfs.js
index de4bb40f04..9d2bcc6464 100644
--- a/chrome/content/zotero/xpcom/storage/zfs.js
+++ b/chrome/content/zotero/xpcom/storage/zfs.js
@@ -31,6 +31,8 @@ Zotero.Sync.Storage.ZFS = (function () {
 		"Zotero-API-Version" : ZOTERO_CONFIG.API_VERSION
 	};
 	var _cachedCredentials = false;
+	var _s3Backoff = 1;
+	var _maxS3Backoff = 60;
 	
 	/**
 	 * Get file metadata on storage server
@@ -463,9 +465,29 @@ Zotero.Sync.Storage.ZFS = (function () {
 				onStop: function (httpRequest, status, response, data) {
 					data.request.setChannel(false);
 					
-					deferred.resolve(
-						onUploadComplete(httpRequest, status, response, data)
-					);
+					// For timeouts from S3, which seem to happen intermittently,
+					// wait a little and try again
+					let timeoutMessage = "Your socket connection to the server was not read from or "
+						+ "written to within the timeout period.";
+					if (status == 400 && response.indexOf(timeoutMessage) != -1) {
+						let libraryKey = Zotero.Items.getLibraryKeyHash(item);
+						let msg = "S3 returned 400 in Zotero.Sync.Storage.ZFS.onUploadComplete()"
+							+ " (" + libraryKey + ") -- retrying"
+						Components.utils.reportError(msg);
+						Zotero.debug(msg, 1);
+						Zotero.debug(response, 1);
+						if (_s3Backoff < _maxS3Backoff) {
+							_s3Backoff *= 2;
+						}
+						Zotero.debug("Delaying " + libraryKey + " upload for " + _s3Backoff + " seconds");
+						Q.delay(_s3Backoff * 1000)
+						.then(function () {
+							deferred.resolve(postFile(request, item, url, uploadKey, params));
+						});
+						return;
+					}
+					
+					deferred.resolve(onUploadComplete(httpRequest, status, response, data));
 				},
 				onCancel: function (httpRequest, status, data) {
 					onUploadCancel(httpRequest, status, data)
@@ -504,6 +526,10 @@ Zotero.Sync.Storage.ZFS = (function () {
 			
 			switch (status) {
 				case 201:
+					// Decrease backoff delay on successful upload
+					if (_s3Backoff > 1) {
+						_s3Backoff /= 2;
+					}
 					break;
 				
 				case 500: