Replace Zotero.Utilities.Internal.saveURI() with Zotero.HTTP.download()
Adds a new function, Zotero.HTTP.download(), that uses Zotero.HTTP.request(). This fixes downloads via authenticated proxies in Zotero 7 and gives us other request() functionality (e.g., 5xx retrying) for free. The downside is that this is probably less efficient, potentially loading large downloads in memory. We should create a replacement for request() based on fetch() that supports getting the body as a ReadableStream. Fixes #5062
This commit is contained in:
parent
ccc1800f21
commit
1f401f0897
9 changed files with 289 additions and 577 deletions
|
@ -921,20 +921,7 @@ Zotero.Attachments = new function () {
|
|||
}
|
||||
}
|
||||
else {
|
||||
Zotero.debug("Saving file with saveURI()");
|
||||
const nsIWBP = Components.interfaces.nsIWebBrowserPersist;
|
||||
var wbp = Components.classes["@mozilla.org/embedding/browser/nsWebBrowserPersist;1"]
|
||||
.createInstance(nsIWBP);
|
||||
wbp.persistFlags = nsIWBP.PERSIST_FLAGS_FROM_CACHE;
|
||||
var ioService = Components.classes["@mozilla.org/network/io-service;1"]
|
||||
.getService(Components.interfaces.nsIIOService);
|
||||
var nsIURL = ioService.newURI(url, null, null);
|
||||
var deferred = Zotero.Promise.defer();
|
||||
wbp.progressListener = new Zotero.WebProgressFinishListener(function () {
|
||||
deferred.resolve();
|
||||
});
|
||||
Zotero.Utilities.Internal.saveURI(wbp, nsIURL, tmpFile);
|
||||
yield deferred.promise;
|
||||
yield Zotero.HTTP.download(url, tmpFile);
|
||||
}
|
||||
|
||||
var attachmentItem;
|
||||
|
@ -1100,20 +1087,18 @@ Zotero.Attachments = new function () {
|
|||
|
||||
let enforcingFileType = false;
|
||||
try {
|
||||
await new Zotero.Promise(function (resolve) {
|
||||
var wbp = Components.classes["@mozilla.org/embedding/browser/nsWebBrowserPersist;1"]
|
||||
.createInstance(Components.interfaces.nsIWebBrowserPersist);
|
||||
if (options.cookieSandbox) {
|
||||
options.cookieSandbox.attachToInterfaceRequestor(wbp);
|
||||
let headers = {};
|
||||
if (options.referrer) {
|
||||
headers.Referer = options.referrer;
|
||||
}
|
||||
await Zotero.HTTP.download(
|
||||
url,
|
||||
path,
|
||||
{
|
||||
headers,
|
||||
cookieSandbox: options.cookieSandbox
|
||||
}
|
||||
|
||||
wbp.progressListener = new Zotero.WebProgressFinishListener(() => resolve());
|
||||
var headers = {};
|
||||
if (options.referrer) {
|
||||
headers.Referer = options.referrer;
|
||||
}
|
||||
Zotero.Utilities.Internal.saveURI(wbp, url, path, headers);
|
||||
});
|
||||
);
|
||||
|
||||
if (options.enforceFileType) {
|
||||
enforcingFileType = true;
|
||||
|
|
|
@ -448,16 +448,15 @@ Zotero.File = new function(){
|
|||
|
||||
this.download = async function (uri, path) {
|
||||
var uriStr = uri.spec || uri;
|
||||
|
||||
const isHTTP = uriStr.startsWith('http');
|
||||
if (uriStr.startsWith('http')) {
|
||||
Zotero.warn("Zotero.File.download() is deprecated for HTTP(S) URLs -- use Zotero.HTTP.download()");
|
||||
return Zotero.HTTP.download(uri, path);
|
||||
}
|
||||
|
||||
Zotero.debug(`Saving ${uriStr} to ${path.pathQueryRef || path}`);
|
||||
|
||||
if (isHTTP && Zotero.HTTP.browserIsOffline()) {
|
||||
let msg = `Download failed: ${Zotero.appName} is currently offline`;
|
||||
Zotero.debug(msg, 2);
|
||||
throw new Error(msg);
|
||||
}
|
||||
|
||||
var deferred = Zotero.Promise.defer();
|
||||
const inputChannel = NetUtil.newChannel({
|
||||
uri,
|
||||
|
|
|
@ -3,6 +3,7 @@
|
|||
* @namespace
|
||||
*/
|
||||
Zotero.HTTP = new function() {
|
||||
this.disableErrorRetry = false;
|
||||
var _errorDelayIntervals = [2500, 5000, 10000, 20000, 40000, 60000, 120000, 240000, 300000];
|
||||
var _errorDelayMax = 60 * 60 * 1000; // 1 hour
|
||||
|
||||
|
@ -169,7 +170,7 @@ Zotero.HTTP = new function() {
|
|||
continue;
|
||||
}
|
||||
// Don't retry if errorDelayMax is 0
|
||||
if (options.errorDelayMax === 0) {
|
||||
if (options.errorDelayMax === 0 || Zotero.HTTP.disableErrorRetry) {
|
||||
throw e;
|
||||
}
|
||||
// Automatically retry other 5xx errors by default
|
||||
|
@ -269,7 +270,10 @@ Zotero.HTTP = new function() {
|
|||
|
||||
var deferred = Zotero.Promise.defer();
|
||||
|
||||
if (!this.mock || url.startsWith('resource://') || url.startsWith('chrome://')) {
|
||||
if (!this.mock
|
||||
|| options.noMock
|
||||
|| url.startsWith('resource://')
|
||||
|| url.startsWith('chrome://')) {
|
||||
var xmlhttp = new XMLHttpRequest();
|
||||
}
|
||||
else {
|
||||
|
@ -324,11 +328,13 @@ Zotero.HTTP = new function() {
|
|||
channel.loadFlags |= Components.interfaces.nsIRequest.LOAD_BYPASS_CACHE;
|
||||
}
|
||||
|
||||
// Don't follow redirects
|
||||
let notificationCallbacks = options.notificationCallbacks || {};
|
||||
if (options.followRedirects === false) {
|
||||
channel.notificationCallbacks = {
|
||||
QueryInterface: ChromeUtils.generateQI([Ci.nsIInterfaceRequestor, Ci.nsIChannelEventSync]),
|
||||
getInterface: ChromeUtils.generateQI([Ci.nsIChannelEventSink]),
|
||||
if (notificationCallbacks.asyncOnChannelRedirect) {
|
||||
throw new Error("Can't set asyncOnChannelRedirect and followRedirects = false");
|
||||
}
|
||||
notificationCallbacks = {
|
||||
...notificationCallbacks,
|
||||
asyncOnChannelRedirect: function (oldChannel, newChannel, flags, callback) {
|
||||
redirectStatus = (flags & Ci.nsIChannelEventSink.REDIRECT_PERMANENT) ? 301 : 302;
|
||||
redirectLocation = newChannel.URI.spec;
|
||||
|
@ -337,6 +343,9 @@ Zotero.HTTP = new function() {
|
|||
}
|
||||
};
|
||||
}
|
||||
if (notificationCallbacks) {
|
||||
channel.notificationCallbacks = wrapNotificationCallbacks(notificationCallbacks);
|
||||
}
|
||||
}
|
||||
|
||||
// Set responseType
|
||||
|
@ -545,6 +554,61 @@ Zotero.HTTP = new function() {
|
|||
return deferred.promise;
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* Create an nsIInterfaceRequestor object based on the callbacks provided
|
||||
*/
|
||||
function wrapNotificationCallbacks(callbacks) {
|
||||
return {
|
||||
QueryInterface: ChromeUtils.generateQI(["nsIInterfaceRequestor"]),
|
||||
|
||||
getInterface(aIID) {
|
||||
// Handle nsIProgressEventSink (for onProgress, onStatus)
|
||||
if (aIID.equals(Ci.nsIProgressEventSink) && (callbacks.onProgress || callbacks.onStatus)) {
|
||||
return {
|
||||
onProgress: callbacks.onProgress || function () {},
|
||||
onStatus: callbacks.onStatus || function () {},
|
||||
};
|
||||
}
|
||||
|
||||
// Handle nsIChannelEventSink (for asyncOnChannelRedirect)
|
||||
if (aIID.equals(Ci.nsIChannelEventSink) && callbacks.asyncOnChannelRedirect) {
|
||||
return {
|
||||
asyncOnChannelRedirect: callbacks.asyncOnChannelRedirect,
|
||||
};
|
||||
}
|
||||
|
||||
throw Components.Exception("No interface available", Cr.NS_ERROR_NO_INTERFACE);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Download a file
|
||||
*
|
||||
* @param {nsIURI|String} url - URL to request
|
||||
* @param {String} path - Path to save file to
|
||||
* @param {Object} [options] - See `Zotero.HTTP.request()`
|
||||
*/
|
||||
this.download = async function(uri, path, options = {}) {
|
||||
// TODO: Convert request() to fetch() and use ReadableStream
|
||||
var req = await this.request(
|
||||
'GET',
|
||||
uri,
|
||||
{
|
||||
...options,
|
||||
responseType: 'blob',
|
||||
// Downloads can have channel notification callbacks, etc., so always do them for real
|
||||
noMock: true
|
||||
}
|
||||
);
|
||||
var bytes = await IOUtils.write(path, await req.response.bytes());
|
||||
Zotero.debug(`Saved file to ${path} (${bytes} byte${bytes != 1 ? 's' : ''})`);
|
||||
return req;
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* Send an HTTP GET request via XMLHTTPRequest
|
||||
*
|
||||
|
@ -1087,6 +1151,9 @@ Zotero.HTTP = new function() {
|
|||
|
||||
|
||||
this.getDisplayURI = function (uri, noCredentials) {
|
||||
if (typeof uri == 'string') {
|
||||
uri = NetUtil.newURI(uri);
|
||||
}
|
||||
if (!uri.password) return uri;
|
||||
uri = uri.mutate();
|
||||
if (noCredentials) {
|
||||
|
|
|
@ -1,276 +0,0 @@
|
|||
/*
|
||||
***** BEGIN LICENSE BLOCK *****
|
||||
|
||||
Copyright © 2009 Center for History and New Media
|
||||
George Mason University, Fairfax, Virginia, USA
|
||||
http://zotero.org
|
||||
|
||||
This file is part of Zotero.
|
||||
|
||||
Zotero is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
Zotero is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with Zotero. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
***** END LICENSE BLOCK *****
|
||||
*/
|
||||
|
||||
|
||||
/**
|
||||
* Stream listener that can handle both download and upload requests
|
||||
*
|
||||
* Possible properties of data object:
|
||||
* - onStart: f(request)
|
||||
* - onProgress: f(request, progress, progressMax)
|
||||
* - onStop: f(request, status, response)
|
||||
* - onCancel: f(request, status)
|
||||
* - streams: array of streams to close on completion
|
||||
*/
|
||||
Zotero.Sync.Storage.StreamListener = function (data) {
|
||||
this._data = data;
|
||||
}
|
||||
|
||||
Zotero.Sync.Storage.StreamListener.prototype = {
|
||||
_channel: null,
|
||||
|
||||
// nsIProgressEventSink
|
||||
onProgress: function (request, progress, progressMax) {
|
||||
Zotero.debug("onProgress with " + progress + "/" + progressMax);
|
||||
this._onProgress(request, progress, progressMax);
|
||||
},
|
||||
|
||||
onStatus: function (request, status, statusArg) {
|
||||
Zotero.debug('onStatus with ' + status);
|
||||
},
|
||||
|
||||
// nsIRequestObserver
|
||||
// Note: For uploads, this isn't called until data is done uploading
|
||||
onStartRequest: function (request) {
|
||||
Zotero.debug('onStartRequest');
|
||||
this._response = "";
|
||||
|
||||
this._onStart(request);
|
||||
},
|
||||
|
||||
onStopRequest: function (request, status) {
|
||||
Zotero.debug('onStopRequest with ' + status);
|
||||
|
||||
// Some errors from https://developer.mozilla.org/en-US/docs/Table_Of_Errors
|
||||
var msg = "";
|
||||
switch (status) {
|
||||
// Normal
|
||||
case 0:
|
||||
break;
|
||||
|
||||
// NS_BINDING_ABORTED
|
||||
case 0x804b0002:
|
||||
msg = "Request cancelled";
|
||||
break;
|
||||
|
||||
// NS_ERROR_NET_INTERRUPT
|
||||
case 0x804B0047:
|
||||
msg = "Request interrupted";
|
||||
break;
|
||||
|
||||
// NS_ERROR_NET_TIMEOUT
|
||||
case 0x804B000E:
|
||||
msg = "Request timed out";
|
||||
break;
|
||||
|
||||
default:
|
||||
msg = "Request failed";
|
||||
break;
|
||||
}
|
||||
|
||||
if (msg) {
|
||||
msg += " in Zotero.Sync.Storage.StreamListener.onStopRequest() (" + status + ")";
|
||||
Components.utils.reportError(msg);
|
||||
Zotero.debug(msg, 1);
|
||||
}
|
||||
|
||||
this._onStop(request, status);
|
||||
},
|
||||
|
||||
// nsIWebProgressListener
|
||||
onProgressChange: function (wp, request, curSelfProgress,
|
||||
maxSelfProgress, curTotalProgress, maxTotalProgress) {
|
||||
//Zotero.debug("onProgressChange with " + curTotalProgress + "/" + maxTotalProgress);
|
||||
|
||||
// onProgress gets called too, so this isn't necessary
|
||||
//this._onProgress(request, curTotalProgress, maxTotalProgress);
|
||||
},
|
||||
|
||||
onStateChange: function (wp, request, stateFlags, status) {
|
||||
Zotero.debug("onStateChange with " + stateFlags);
|
||||
|
||||
if (stateFlags & Components.interfaces.nsIWebProgressListener.STATE_IS_REQUEST) {
|
||||
if (stateFlags & Components.interfaces.nsIWebProgressListener.STATE_START) {
|
||||
this._onStart(request);
|
||||
}
|
||||
else if (stateFlags & Components.interfaces.nsIWebProgressListener.STATE_STOP) {
|
||||
this._onStop(request, status);
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
onStatusChange: function (progress, request, status, message) {
|
||||
Zotero.debug("onStatusChange with '" + message + "'");
|
||||
},
|
||||
onLocationChange: function () {
|
||||
Zotero.debug('onLocationChange');
|
||||
},
|
||||
onSecurityChange: function () {
|
||||
Zotero.debug('onSecurityChange');
|
||||
},
|
||||
|
||||
// nsIStreamListener
|
||||
onDataAvailable: function (request, stream, sourceOffset, length) {
|
||||
Zotero.debug('onDataAvailable');
|
||||
var scriptableInputStream =
|
||||
Components.classes["@mozilla.org/scriptableinputstream;1"]
|
||||
.createInstance(Components.interfaces.nsIScriptableInputStream);
|
||||
scriptableInputStream.init(stream);
|
||||
|
||||
var data = scriptableInputStream.read(length);
|
||||
Zotero.debug(data);
|
||||
this._response += data;
|
||||
},
|
||||
|
||||
// nsIChannelEventSink
|
||||
//
|
||||
// If this._data.onChannelRedirect exists, it should return a promise resolving to true to
|
||||
// follow the redirect or false to cancel it
|
||||
onChannelRedirect: Zotero.Promise.coroutine(function* (oldChannel, newChannel, flags) {
|
||||
Zotero.debug('onChannelRedirect');
|
||||
|
||||
if (this._data && this._data.onChannelRedirect) {
|
||||
let result = yield this._data.onChannelRedirect(oldChannel, newChannel, flags);
|
||||
if (!result) {
|
||||
oldChannel.cancel(Components.results.NS_BINDING_ABORTED);
|
||||
newChannel.cancel(Components.results.NS_BINDING_ABORTED);
|
||||
Zotero.debug("Cancelling redirect");
|
||||
// TODO: Prevent onStateChange error
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// if redirecting, store the new channel
|
||||
this._channel = newChannel;
|
||||
}),
|
||||
|
||||
asyncOnChannelRedirect: function (oldChan, newChan, flags, redirectCallback) {
|
||||
Zotero.debug('asyncOnRedirect');
|
||||
|
||||
this.onChannelRedirect(oldChan, newChan, flags)
|
||||
.then(function (result) {
|
||||
redirectCallback.onRedirectVerifyCallback(
|
||||
result ? Components.results.NS_SUCCEEDED : Components.results.NS_FAILED
|
||||
);
|
||||
})
|
||||
.catch(function (e) {
|
||||
Zotero.logError(e);
|
||||
redirectCallback.onRedirectVerifyCallback(Components.results.NS_FAILED);
|
||||
});
|
||||
},
|
||||
|
||||
// nsIHttpEventSink
|
||||
onRedirect: function (oldChannel, newChannel) {
|
||||
Zotero.debug('onRedirect');
|
||||
|
||||
var newURL = Zotero.HTTP.getDisplayURI(newChannel.URI).spec;
|
||||
Zotero.debug("Redirecting to " + newURL);
|
||||
},
|
||||
|
||||
|
||||
//
|
||||
// Private methods
|
||||
//
|
||||
_onStart: function (request) {
|
||||
Zotero.debug('Starting request');
|
||||
if (this._data && this._data.onStart) {
|
||||
this._data.onStart(request);
|
||||
}
|
||||
},
|
||||
|
||||
_onProgress: function (request, progress, progressMax) {
|
||||
if (this._data && this._data.onProgress) {
|
||||
this._data.onProgress(request, progress, progressMax);
|
||||
}
|
||||
},
|
||||
|
||||
_onStop: function (request, status) {
|
||||
var cancelled = status == 0x804b0002; // NS_BINDING_ABORTED
|
||||
|
||||
if (!cancelled && status == 0 && request instanceof Components.interfaces.nsIHttpChannel) {
|
||||
request.QueryInterface(Components.interfaces.nsIHttpChannel);
|
||||
try {
|
||||
status = request.responseStatus;
|
||||
}
|
||||
catch (e) {
|
||||
Zotero.debug("Request responseStatus not available", 1);
|
||||
status = 0;
|
||||
}
|
||||
Zotero.debug('Request ended with status code ' + status);
|
||||
request.QueryInterface(Components.interfaces.nsIRequest);
|
||||
}
|
||||
else {
|
||||
Zotero.debug('Request ended with status ' + status);
|
||||
status = 0;
|
||||
}
|
||||
|
||||
if (this._data.streams) {
|
||||
for (let stream of this._data.streams) {
|
||||
stream.close();
|
||||
}
|
||||
}
|
||||
|
||||
if (cancelled) {
|
||||
if (this._data.onCancel) {
|
||||
this._data.onCancel(request, status);
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (this._data.onStop) {
|
||||
this._data.onStop(request, status, this._response);
|
||||
}
|
||||
}
|
||||
|
||||
this._channel = null;
|
||||
},
|
||||
|
||||
// nsIInterfaceRequestor
|
||||
getInterface: function (iid) {
|
||||
try {
|
||||
return this.QueryInterface(iid);
|
||||
}
|
||||
catch (e) {
|
||||
throw Components.results.NS_NOINTERFACE;
|
||||
}
|
||||
},
|
||||
|
||||
QueryInterface: function(iid) {
|
||||
if (iid.equals(Components.interfaces.nsISupports) ||
|
||||
iid.equals(Components.interfaces.nsIInterfaceRequestor) ||
|
||||
iid.equals(Components.interfaces.nsIChannelEventSink) ||
|
||||
iid.equals(Components.interfaces.nsIProgressEventSink) ||
|
||||
iid.equals(Components.interfaces.nsIHttpEventSink) ||
|
||||
iid.equals(Components.interfaces.nsIStreamListener) ||
|
||||
iid.equals(Components.interfaces.nsIWebProgressListener)) {
|
||||
return this;
|
||||
}
|
||||
throw Components.results.NS_NOINTERFACE;
|
||||
},
|
||||
|
||||
_safeSpec: function (uri) {
|
||||
return uri.scheme + '://' + uri.username + ':********@'
|
||||
+ uri.hostPort + uri.pathQueryRef
|
||||
},
|
||||
};
|
|
@ -278,8 +278,8 @@ Zotero.Sync.Storage.Mode.WebDAV.prototype = {
|
|||
* @param {Zotero.Sync.Storage.Request} request
|
||||
* @return {Promise<Zotero.Sync.Storage.Result>}
|
||||
*/
|
||||
downloadFile: Zotero.Promise.coroutine(function* (request) {
|
||||
yield this._init();
|
||||
downloadFile: async function (request) {
|
||||
await this._init();
|
||||
|
||||
var item = Zotero.Sync.Storage.Utilities.getItemFromRequest(request);
|
||||
if (!item) {
|
||||
|
@ -294,7 +294,7 @@ Zotero.Sync.Storage.Mode.WebDAV.prototype = {
|
|||
}
|
||||
|
||||
// Retrieve modification time from server
|
||||
var metadata = yield this._getStorageFileMetadata(item, request);
|
||||
var metadata = await this._getStorageFileMetadata(item, request);
|
||||
|
||||
if (!request.isRunning()) {
|
||||
Zotero.debug("Download request '" + request.name
|
||||
|
@ -307,7 +307,7 @@ Zotero.Sync.Storage.Mode.WebDAV.prototype = {
|
|||
return new Zotero.Sync.Storage.Result;
|
||||
}
|
||||
|
||||
var fileModTime = yield item.attachmentModificationTime;
|
||||
var fileModTime = await item.attachmentModificationTime;
|
||||
if (metadata.mtime == fileModTime) {
|
||||
Zotero.debug("File mod time matches remote file -- skipping download of "
|
||||
+ item.libraryKey);
|
||||
|
@ -315,10 +315,10 @@ Zotero.Sync.Storage.Mode.WebDAV.prototype = {
|
|||
var updateItem = item.attachmentSyncState != 1
|
||||
item.attachmentSyncedModificationTime = metadata.mtime;
|
||||
item.attachmentSyncState = "in_sync";
|
||||
yield item.saveTx({ skipAll: true });
|
||||
await item.saveTx({ skipAll: true });
|
||||
// DEBUG: Necessary?
|
||||
if (updateItem) {
|
||||
yield item.updateSynced(false);
|
||||
await item.updateSynced(false);
|
||||
}
|
||||
|
||||
return new Zotero.Sync.Storage.Result({
|
||||
|
@ -329,9 +329,8 @@ Zotero.Sync.Storage.Mode.WebDAV.prototype = {
|
|||
var uri = this._getItemURI(item);
|
||||
|
||||
var destPath = OS.Path.join(Zotero.getTempDirectory().path, item.key + '.tmp');
|
||||
yield Zotero.File.removeIfExists(destPath);
|
||||
await Zotero.File.removeIfExists(destPath);
|
||||
|
||||
var deferred = Zotero.Promise.defer();
|
||||
var requestData = {
|
||||
item,
|
||||
mtime: metadata.mtime,
|
||||
|
@ -339,88 +338,71 @@ Zotero.Sync.Storage.Mode.WebDAV.prototype = {
|
|||
compressed: true
|
||||
};
|
||||
|
||||
var listener = new Zotero.Sync.Storage.StreamListener(
|
||||
{
|
||||
onStart: function (req) {
|
||||
if (request.isFinished()) {
|
||||
Zotero.debug("Download request " + request.name
|
||||
+ " stopped before download started -- closing channel");
|
||||
req.cancel(0x804b0002); // NS_BINDING_ABORTED
|
||||
deferred.resolve(new Zotero.Sync.Storage.Result);
|
||||
return new Promise(async (resolve, reject) => {
|
||||
try {
|
||||
let req = await Zotero.HTTP.download(
|
||||
uri,
|
||||
destPath,
|
||||
{
|
||||
successCodes: [200, 404],
|
||||
noCache: true,
|
||||
notificationCallbacks: {
|
||||
onProgress: function (a, b, c) {
|
||||
request.onProgress(a, b, c)
|
||||
},
|
||||
},
|
||||
errorDelayIntervals: this.ERROR_DELAY_INTERVALS,
|
||||
errorDelayMax: this.ERROR_DELAY_MAX,
|
||||
}
|
||||
},
|
||||
onProgress: function (a, b, c) {
|
||||
request.onProgress(a, b, c)
|
||||
},
|
||||
onStop: Zotero.Promise.coroutine(function* (req, status, res) {
|
||||
request.setChannel(false);
|
||||
|
||||
if (status == 404) {
|
||||
let msg = "Remote ZIP file not found for item " + item.libraryKey;
|
||||
Zotero.debug(msg, 2);
|
||||
Components.utils.reportError(msg);
|
||||
|
||||
// Delete the orphaned prop file
|
||||
try {
|
||||
yield this._deleteStorageFiles([item.key + ".prop"]);
|
||||
}
|
||||
catch (e) {
|
||||
Zotero.logError(e);
|
||||
}
|
||||
|
||||
deferred.resolve(new Zotero.Sync.Storage.Result);
|
||||
return;
|
||||
}
|
||||
else if (status != 200) {
|
||||
try {
|
||||
this._throwFriendlyError("GET", dispURL, status);
|
||||
}
|
||||
catch (e) {
|
||||
deferred.reject(e);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Don't try to process if the request has been cancelled
|
||||
if (request.isFinished()) {
|
||||
Zotero.debug("Download request " + request.name
|
||||
+ " is no longer running after file download");
|
||||
deferred.resolve(new Zotero.Sync.Storage.Result);
|
||||
return;
|
||||
}
|
||||
|
||||
Zotero.debug("Finished download of " + destPath);
|
||||
);
|
||||
|
||||
if (req.status == 404) {
|
||||
let msg = "Remote ZIP file not found for item " + item.libraryKey;
|
||||
Zotero.debug(msg, 2);
|
||||
Cu.reportError(msg);
|
||||
|
||||
// Delete the orphaned prop file
|
||||
try {
|
||||
deferred.resolve(
|
||||
Zotero.Sync.Storage.Local.processDownload(requestData)
|
||||
);
|
||||
await this._deleteStorageFiles([item.key + ".prop"]);
|
||||
}
|
||||
catch (e) {
|
||||
deferred.reject(e);
|
||||
}
|
||||
}.bind(this)),
|
||||
onCancel: function (req, status) {
|
||||
Zotero.debug("Request cancelled");
|
||||
if (deferred.promise.isPending()) {
|
||||
deferred.resolve(new Zotero.Sync.Storage.Result);
|
||||
Zotero.logError(e);
|
||||
}
|
||||
|
||||
resolve(new Zotero.Sync.Storage.Result);
|
||||
return;
|
||||
}
|
||||
|
||||
// Don't try to process if the request has been cancelled
|
||||
if (request.isFinished()) {
|
||||
Zotero.debug("Download request " + request.name
|
||||
+ " is no longer running after file download");
|
||||
resolve(new Zotero.Sync.Storage.Result);
|
||||
return;
|
||||
}
|
||||
|
||||
Zotero.debug("Finished download of " + destPath);
|
||||
|
||||
resolve(
|
||||
Zotero.Sync.Storage.Local.processDownload(requestData)
|
||||
);
|
||||
}
|
||||
);
|
||||
|
||||
// Don't display password in console
|
||||
var dispURL = Zotero.HTTP.getDisplayURI(uri).spec;
|
||||
Zotero.debug('Saving ' + dispURL);
|
||||
const nsIWBP = Components.interfaces.nsIWebBrowserPersist;
|
||||
var wbp = Components.classes["@mozilla.org/embedding/browser/nsWebBrowserPersist;1"]
|
||||
.createInstance(nsIWBP);
|
||||
wbp.persistFlags = nsIWBP.PERSIST_FLAGS_BYPASS_CACHE;
|
||||
wbp.progressListener = listener;
|
||||
Zotero.Utilities.Internal.saveURI(wbp, uri, destPath);
|
||||
|
||||
return deferred.promise;
|
||||
}),
|
||||
catch (e) {
|
||||
if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
|
||||
try {
|
||||
let dispURL = Zotero.HTTP.getDisplayURI(uri).spec;
|
||||
this._throwFriendlyError("GET", dispURL, e.xmlhttp.status);
|
||||
}
|
||||
catch (e) {
|
||||
reject(e);
|
||||
}
|
||||
return;
|
||||
}
|
||||
Zotero.logError(e);
|
||||
reject(new Error(Zotero.Sync.Storage.defaultError));
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
|
||||
uploadFile: Zotero.Promise.coroutine(function* (request) {
|
||||
|
|
|
@ -49,7 +49,7 @@ Zotero.Sync.Storage.Mode.ZFS.prototype = {
|
|||
* @param {Zotero.Sync.Storage.Request} request
|
||||
* @return {Promise<Zotero.Sync.Storage.Result>}
|
||||
*/
|
||||
downloadFile: Zotero.Promise.coroutine(function* (request) {
|
||||
downloadFile: async function (request) {
|
||||
var item = Zotero.Sync.Storage.Utilities.getItemFromRequest(request);
|
||||
if (!item) {
|
||||
throw new Error("Item '" + request.name + "' not found");
|
||||
|
@ -63,195 +63,147 @@ Zotero.Sync.Storage.Mode.ZFS.prototype = {
|
|||
|
||||
var destPath = OS.Path.join(Zotero.getTempDirectory().path, item.key + '.tmp');
|
||||
|
||||
// saveURI() below appears not to create empty files for Content-Length: 0,
|
||||
// so we create one here just in case, which also lets us check file access
|
||||
// Create an empty file to check file access
|
||||
try {
|
||||
yield IOUtils.write(destPath, new Uint8Array());
|
||||
await IOUtils.write(destPath, new Uint8Array());
|
||||
}
|
||||
catch (e) {
|
||||
Zotero.File.checkFileAccessError(e, destPath, 'create');
|
||||
}
|
||||
|
||||
var deferred = Zotero.Promise.defer();
|
||||
var requestData = {item};
|
||||
|
||||
var listener = new Zotero.Sync.Storage.StreamListener(
|
||||
{
|
||||
onStart: function (req) {
|
||||
if (request.isFinished()) {
|
||||
Zotero.debug("Download request " + request.name
|
||||
+ " stopped before download started -- closing channel");
|
||||
req.cancel(Components.results.NS_BINDING_ABORTED);
|
||||
deferred.resolve(new Zotero.Sync.Storage.Result);
|
||||
}
|
||||
},
|
||||
onChannelRedirect: Zotero.Promise.coroutine(function* (oldChannel, newChannel, flags) {
|
||||
// These will be used in processDownload() if the download succeeds
|
||||
oldChannel.QueryInterface(Components.interfaces.nsIHttpChannel);
|
||||
|
||||
Zotero.debug("CHANNEL HERE FOR " + item.libraryKey + " WITH " + oldChannel.status);
|
||||
Zotero.debug(oldChannel.URI.spec);
|
||||
Zotero.debug(newChannel.URI.spec);
|
||||
|
||||
var header;
|
||||
try {
|
||||
header = "Zotero-File-Modification-Time";
|
||||
requestData.mtime = parseInt(oldChannel.getResponseHeader(header));
|
||||
header = "Zotero-File-MD5";
|
||||
requestData.md5 = oldChannel.getResponseHeader(header);
|
||||
header = "Zotero-File-Compressed";
|
||||
requestData.compressed = oldChannel.getResponseHeader(header) == 'Yes';
|
||||
}
|
||||
catch (e) {
|
||||
deferred.reject(new Error(`${header} header not set in file request for ${item.libraryKey}`));
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!(yield OS.File.exists(path))) {
|
||||
return true;
|
||||
}
|
||||
|
||||
var updateHash = false;
|
||||
var fileModTime = yield item.attachmentModificationTime;
|
||||
if (requestData.mtime == fileModTime) {
|
||||
Zotero.debug("File mod time matches remote file -- skipping download of "
|
||||
+ item.libraryKey);
|
||||
}
|
||||
// If not compressed, check hash, in case only timestamp changed
|
||||
else if (!requestData.compressed && (yield item.attachmentHash) == requestData.md5) {
|
||||
Zotero.debug("File hash matches remote file -- skipping download of "
|
||||
+ item.libraryKey);
|
||||
updateHash = true;
|
||||
}
|
||||
else {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Update local metadata and stop request, skipping file download
|
||||
yield OS.File.setDates(path, null, new Date(requestData.mtime));
|
||||
item.attachmentSyncedModificationTime = requestData.mtime;
|
||||
if (updateHash) {
|
||||
item.attachmentSyncedHash = requestData.md5;
|
||||
}
|
||||
item.attachmentSyncState = "in_sync";
|
||||
yield item.saveTx({ skipAll: true });
|
||||
|
||||
deferred.resolve(new Zotero.Sync.Storage.Result({
|
||||
localChanges: true
|
||||
}));
|
||||
|
||||
return false;
|
||||
}),
|
||||
onProgress: function (req, progress, progressMax) {
|
||||
request.onProgress(progress, progressMax);
|
||||
},
|
||||
onStop: function (req, status, res) {
|
||||
request.setChannel(false);
|
||||
|
||||
if (status != 200) {
|
||||
if (status == 404) {
|
||||
Zotero.debug("Remote file not found for item " + item.libraryKey);
|
||||
// Don't refresh item pane rows when nothing happened
|
||||
request.skipProgressBarUpdate = true;
|
||||
deferred.resolve(new Zotero.Sync.Storage.Result);
|
||||
return;
|
||||
}
|
||||
|
||||
// Check for SSL certificate error
|
||||
if (status == 0) {
|
||||
try {
|
||||
Zotero.HTTP.checkSecurity(req);
|
||||
}
|
||||
catch (e) {
|
||||
deferred.reject(e);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// If S3 connection is interrupted, delay and retry, or bail if too many
|
||||
// consecutive failures
|
||||
if (status == 0 || status == 500 || status == 503) {
|
||||
if (++this._s3ConsecutiveFailures < this._maxS3ConsecutiveFailures) {
|
||||
let libraryKey = item.libraryKey;
|
||||
let msg = "S3 returned 0 for " + libraryKey + " -- retrying download"
|
||||
Components.utils.reportError(msg);
|
||||
Zotero.debug(msg, 1);
|
||||
if (this._s3Backoff < this._maxS3Backoff) {
|
||||
this._s3Backoff *= 2;
|
||||
}
|
||||
Zotero.debug("Delaying " + libraryKey + " download for "
|
||||
+ this._s3Backoff + " seconds", 2);
|
||||
Zotero.Promise.delay(this._s3Backoff * 1000)
|
||||
.then(function () {
|
||||
deferred.resolve(this.downloadFile(request));
|
||||
}.bind(this));
|
||||
return;
|
||||
}
|
||||
|
||||
Zotero.debug(this._s3ConsecutiveFailures
|
||||
+ " consecutive S3 failures -- aborting", 1);
|
||||
this._s3ConsecutiveFailures = 0;
|
||||
}
|
||||
|
||||
var msg = "Unexpected status code " + status + " for GET " + uri;
|
||||
Zotero.debug(msg, 1);
|
||||
Components.utils.reportError(msg);
|
||||
// Output saved content, in case an error was captured
|
||||
try {
|
||||
let sample = Zotero.File.getContents(destPath, null, 4096);
|
||||
if (sample) {
|
||||
Zotero.debug(sample, 1);
|
||||
}
|
||||
}
|
||||
catch (e) {
|
||||
Zotero.debug(e, 1);
|
||||
}
|
||||
deferred.reject(new Error(Zotero.Sync.Storage.defaultError));
|
||||
return;
|
||||
}
|
||||
|
||||
// Don't try to process if the request has been cancelled
|
||||
if (request.isFinished()) {
|
||||
Zotero.debug("Download request " + request.name
|
||||
+ " is no longer running after file download", 2);
|
||||
deferred.resolve(new Zotero.Sync.Storage.Result);
|
||||
return;
|
||||
}
|
||||
|
||||
Zotero.debug("Finished download of " + destPath);
|
||||
|
||||
try {
|
||||
deferred.resolve(
|
||||
Zotero.Sync.Storage.Local.processDownload(requestData)
|
||||
);
|
||||
}
|
||||
catch (e) {
|
||||
deferred.reject(e);
|
||||
}
|
||||
}.bind(this),
|
||||
onCancel: function (req, status) {
|
||||
Zotero.debug("Request cancelled");
|
||||
if (deferred.promise.isPending()) {
|
||||
deferred.resolve(new Zotero.Sync.Storage.Result);
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
var params = this._getRequestParams(item.libraryID, `items/${item.key}/file`);
|
||||
var uri = this.apiClient.buildRequestURI(params);
|
||||
var headers = this.apiClient.getHeaders();
|
||||
|
||||
Zotero.debug('Saving ' + uri);
|
||||
const nsIWBP = Components.interfaces.nsIWebBrowserPersist;
|
||||
var wbp = Components.classes["@mozilla.org/embedding/browser/nsWebBrowserPersist;1"]
|
||||
.createInstance(nsIWBP);
|
||||
wbp.persistFlags = nsIWBP.PERSIST_FLAGS_BYPASS_CACHE;
|
||||
wbp.progressListener = listener;
|
||||
Zotero.Utilities.Internal.saveURI(wbp, uri, destPath, headers);
|
||||
|
||||
return deferred.promise;
|
||||
}),
|
||||
return new Promise(async (resolve, reject) => {
|
||||
try {
|
||||
let req = await Zotero.HTTP.download(
|
||||
uri,
|
||||
destPath,
|
||||
{
|
||||
successCodes: [200, 404],
|
||||
headers: this.apiClient.getHeaders(),
|
||||
noCache: true,
|
||||
notificationCallbacks: {
|
||||
asyncOnChannelRedirect: async function (oldChannel, newChannel, flags, callback) {
|
||||
// These will be used in processDownload() if the download succeeds
|
||||
oldChannel.QueryInterface(Components.interfaces.nsIHttpChannel);
|
||||
|
||||
Zotero.debug(`Handling ${oldChannel.responseStatus} redirect for ${item.libraryKey}`);
|
||||
Zotero.debug(oldChannel.URI.spec);
|
||||
Zotero.debug(newChannel.URI.spec);
|
||||
|
||||
var header;
|
||||
try {
|
||||
header = "Zotero-File-Modification-Time";
|
||||
requestData.mtime = parseInt(oldChannel.getResponseHeader(header));
|
||||
header = "Zotero-File-MD5";
|
||||
requestData.md5 = oldChannel.getResponseHeader(header);
|
||||
header = "Zotero-File-Compressed";
|
||||
requestData.compressed = oldChannel.getResponseHeader(header) == 'Yes';
|
||||
}
|
||||
catch (_e) {
|
||||
reject(new Error(`${header} header not set in file request for ${item.libraryKey}`));
|
||||
callback.onRedirectVerifyCallback(Cr.NS_ERROR_ABORT);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!(await IOUtils.exists(path))) {
|
||||
callback.onRedirectVerifyCallback(Cr.NS_OK);
|
||||
return;
|
||||
}
|
||||
|
||||
var updateHash = false;
|
||||
var fileModTime = await item.attachmentModificationTime;
|
||||
if (requestData.mtime == fileModTime) {
|
||||
Zotero.debug("File mod time matches remote file -- skipping download of "
|
||||
+ item.libraryKey);
|
||||
}
|
||||
// If not compressed, check hash, in case only timestamp changed
|
||||
else if (!requestData.compressed && (await item.attachmentHash) == requestData.md5) {
|
||||
Zotero.debug("File hash matches remote file -- skipping download of "
|
||||
+ item.libraryKey);
|
||||
updateHash = true;
|
||||
}
|
||||
else {
|
||||
callback.onRedirectVerifyCallback(Cr.NS_OK);
|
||||
return;
|
||||
}
|
||||
|
||||
// Update local metadata and stop request, skipping file download
|
||||
await OS.File.setDates(path, null, new Date(requestData.mtime));
|
||||
item.attachmentSyncedModificationTime = requestData.mtime;
|
||||
if (updateHash) {
|
||||
item.attachmentSyncedHash = requestData.md5;
|
||||
}
|
||||
item.attachmentSyncState = "in_sync";
|
||||
await item.saveTx({ skipAll: true });
|
||||
|
||||
resolve(new Zotero.Sync.Storage.Result({
|
||||
localChanges: true
|
||||
}));
|
||||
|
||||
callback.onRedirectVerifyCallback(Cr.NS_ERROR_ABORT);
|
||||
},
|
||||
|
||||
onProgress: function (req, progress, progressMax) {
|
||||
request.onProgress(progress, progressMax);
|
||||
},
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
if (req.status == 404) {
|
||||
Zotero.debug("Remote file not found for item " + item.libraryKey);
|
||||
// Don't refresh item pane rows when nothing happened
|
||||
request.skipProgressBarUpdate = true;
|
||||
resolve(new Zotero.Sync.Storage.Result);
|
||||
return;
|
||||
}
|
||||
|
||||
// Don't try to process if the request has been cancelled
|
||||
if (request.isFinished()) {
|
||||
Zotero.debug(`Download request ${request.name} is no longer running after file download`, 2);
|
||||
resolve(new Zotero.Sync.Storage.Result);
|
||||
return;
|
||||
}
|
||||
|
||||
Zotero.debug("Finished download of " + destPath);
|
||||
|
||||
resolve(await Zotero.Sync.Storage.Local.processDownload(requestData));
|
||||
}
|
||||
catch (e) {
|
||||
if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
|
||||
// If S3 connection is interrupted, delay and retry, or bail if too many
|
||||
// consecutive failures
|
||||
if (e.xmlhttp.status == 0) {
|
||||
if (++this._s3ConsecutiveFailures < this._maxS3ConsecutiveFailures) {
|
||||
let libraryKey = item.libraryKey;
|
||||
let msg = "S3 returned 0 for " + libraryKey + " -- retrying download";
|
||||
Zotero.logError(msg);
|
||||
if (this._s3Backoff < this._maxS3Backoff) {
|
||||
this._s3Backoff *= 2;
|
||||
}
|
||||
Zotero.debug("Delaying " + libraryKey + " download for "
|
||||
+ this._s3Backoff + " seconds", 2);
|
||||
Zotero.Promise.delay(this._s3Backoff * 1000)
|
||||
.then(function () {
|
||||
resolve(this.downloadFile(request));
|
||||
}.bind(this));
|
||||
return;
|
||||
}
|
||||
|
||||
Zotero.debug(this._s3ConsecutiveFailures
|
||||
+ " consecutive S3 failures -- aborting", 1);
|
||||
this._s3ConsecutiveFailures = 0;
|
||||
}
|
||||
}
|
||||
Zotero.logError(e);
|
||||
reject(new Error(Zotero.Sync.Storage.defaultError));
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
|
||||
uploadFile: Zotero.Promise.coroutine(function* (request) {
|
||||
|
|
|
@ -459,6 +459,8 @@ Zotero.Utilities.Internal = {
|
|||
* @param {Zotero.CookieSandbox} [cookieSandbox]
|
||||
*/
|
||||
saveURI: function (wbp, uri, target, headers, cookieSandbox) {
|
||||
Zotero.warn("Zotero.Utilities.Internal.saveURI() is deprecated -- use Zotero.HTTP.download()");
|
||||
|
||||
// Handle gzip encoding
|
||||
wbp.persistFlags |= wbp.PERSIST_FLAGS_AUTODETECT_APPLY_CONVERSION;
|
||||
// If not explicitly using cache, skip it
|
||||
|
|
|
@ -144,7 +144,6 @@ const xpcomFilesLocal = [
|
|||
'storage/storageRequest',
|
||||
'storage/storageResult',
|
||||
'storage/storageUtilities',
|
||||
'storage/streamListener',
|
||||
'storage/zfs',
|
||||
'storage/webdav',
|
||||
'syncedSettings',
|
||||
|
|
|
@ -103,6 +103,7 @@ describe("Zotero.Sync.Storage.Mode.ZFS", function () {
|
|||
httpd.stop(() => defer.resolve());
|
||||
yield defer.promise;
|
||||
win.close();
|
||||
Zotero.HTTP.disableErrorRetry = false;
|
||||
})
|
||||
|
||||
after(function* () {
|
||||
|
@ -211,6 +212,7 @@ describe("Zotero.Sync.Storage.Mode.ZFS", function () {
|
|||
item.attachmentSyncState = "to_download";
|
||||
yield item.saveTx();
|
||||
|
||||
Zotero.HTTP.disableErrorRetry = true;
|
||||
httpd.registerPathHandler(
|
||||
`/users/1/items/${item.key}/file`,
|
||||
{
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue