mirrors/zotero
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions 1

Replace Zotero.Utilities.Internal.saveURI() with Zotero.HTTP.download()

Browse source 
Adds a new function, Zotero.HTTP.download(), that uses
Zotero.HTTP.request(). This fixes downloads via authenticated proxies in
Zotero 7 and gives us other request() functionality (e.g., 5xx retrying)
for free.

The downside is that this is probably less efficient, potentially
loading large downloads in memory. We should create a replacement for
request() based on fetch() that supports getting the body as a
ReadableStream.

Fixes #5062
This commit is contained in:
Dan Stillman 2025-03-05 04:35:48 -05:00
parent ccc1800f21
commit 1f401f0897
9 changed files with 289 additions and 577 deletions
Download patch file Download diff file Expand all files Collapse all files

39
chrome/content/zotero/xpcom/attachments.js
View file

@ -921,20 +921,7 @@ Zotero.Attachments = new function () {
}
}
else {
Zotero.debug("Saving file with saveURI()");
const nsIWBP = Components.interfaces.nsIWebBrowserPersist;
var wbp = Components.classes["@mozilla.org/embedding/browser/nsWebBrowserPersist;1"]
.createInstance(nsIWBP);
wbp.persistFlags = nsIWBP.PERSIST_FLAGS_FROM_CACHE;
var ioService = Components.classes["@mozilla.org/network/io-service;1"]
.getService(Components.interfaces.nsIIOService);
var nsIURL = ioService.newURI(url, null, null);
var deferred = Zotero.Promise.defer();
wbp.progressListener = new Zotero.WebProgressFinishListener(function () {
deferred.resolve();
});
Zotero.Utilities.Internal.saveURI(wbp, nsIURL, tmpFile);
yield deferred.promise;
yield Zotero.HTTP.download(url, tmpFile);
}
var attachmentItem;
@ -1100,20 +1087,18 @@ Zotero.Attachments = new function () {
let enforcingFileType = false;
try {
await new Zotero.Promise(function (resolve) {
var wbp = Components.classes["@mozilla.org/embedding/browser/nsWebBrowserPersist;1"]
.createInstance(Components.interfaces.nsIWebBrowserPersist);
if (options.cookieSandbox) {
options.cookieSandbox.attachToInterfaceRequestor(wbp);
let headers = {};
if (options.referrer) {
headers.Referer = options.referrer;
}
await Zotero.HTTP.download(
url,
path,
{
headers,
cookieSandbox: options.cookieSandbox
}
wbp.progressListener = new Zotero.WebProgressFinishListener(() => resolve());
var headers = {};
if (options.referrer) {
headers.Referer = options.referrer;
}
Zotero.Utilities.Internal.saveURI(wbp, url, path, headers);
});
);
if (options.enforceFileType) {
enforcingFileType = true;

11
chrome/content/zotero/xpcom/file.js
View file

@ -448,16 +448,15 @@ Zotero.File = new function(){
this.download = async function (uri, path) {
var uriStr = uri.spec || uri;
const isHTTP = uriStr.startsWith('http');
if (uriStr.startsWith('http')) {
Zotero.warn("Zotero.File.download() is deprecated for HTTP(S) URLs -- use Zotero.HTTP.download()");
return Zotero.HTTP.download(uri, path);
}
Zotero.debug(`Saving ${uriStr} to ${path.pathQueryRef || path}`);
if (isHTTP && Zotero.HTTP.browserIsOffline()) {
let msg = `Download failed: ${Zotero.appName} is currently offline`;
Zotero.debug(msg, 2);
throw new Error(msg);
}
var deferred = Zotero.Promise.defer();
const inputChannel = NetUtil.newChannel({
uri,

79
chrome/content/zotero/xpcom/http.js
View file

@ -3,6 +3,7 @@
* @namespace
*/
Zotero.HTTP = new function() {
this.disableErrorRetry = false;
var _errorDelayIntervals = [2500, 5000, 10000, 20000, 40000, 60000, 120000, 240000, 300000];
var _errorDelayMax = 60 * 60 * 1000; // 1 hour
@ -169,7 +170,7 @@ Zotero.HTTP = new function() {
continue;
}
// Don't retry if errorDelayMax is 0
if (options.errorDelayMax === 0) {
if (options.errorDelayMax === 0 || Zotero.HTTP.disableErrorRetry) {
throw e;
}
// Automatically retry other 5xx errors by default
@ -269,7 +270,10 @@ Zotero.HTTP = new function() {
var deferred = Zotero.Promise.defer();
if (!this.mock || url.startsWith('resource://') || url.startsWith('chrome://')) {
if (!this.mock
|| options.noMock
|| url.startsWith('resource://')
|| url.startsWith('chrome://')) {
var xmlhttp = new XMLHttpRequest();
}
else {
@ -324,11 +328,13 @@ Zotero.HTTP = new function() {
channel.loadFlags |= Components.interfaces.nsIRequest.LOAD_BYPASS_CACHE;
}
// Don't follow redirects
let notificationCallbacks = options.notificationCallbacks || {};
if (options.followRedirects === false) {
channel.notificationCallbacks = {
QueryInterface: ChromeUtils.generateQI([Ci.nsIInterfaceRequestor, Ci.nsIChannelEventSync]),
getInterface: ChromeUtils.generateQI([Ci.nsIChannelEventSink]),
if (notificationCallbacks.asyncOnChannelRedirect) {
throw new Error("Can't set asyncOnChannelRedirect and followRedirects = false");
}
notificationCallbacks = {
...notificationCallbacks,
asyncOnChannelRedirect: function (oldChannel, newChannel, flags, callback) {
redirectStatus = (flags & Ci.nsIChannelEventSink.REDIRECT_PERMANENT) ? 301 : 302;
redirectLocation = newChannel.URI.spec;
@ -337,6 +343,9 @@ Zotero.HTTP = new function() {
}
};
}
if (notificationCallbacks) {
channel.notificationCallbacks = wrapNotificationCallbacks(notificationCallbacks);
}
}
// Set responseType
@ -545,6 +554,61 @@ Zotero.HTTP = new function() {
return deferred.promise;
};
/**
* Create an nsIInterfaceRequestor object based on the callbacks provided
*/
function wrapNotificationCallbacks(callbacks) {
return {
QueryInterface: ChromeUtils.generateQI(["nsIInterfaceRequestor"]),
getInterface(aIID) {
// Handle nsIProgressEventSink (for onProgress, onStatus)
if (aIID.equals(Ci.nsIProgressEventSink) && (callbacks.onProgress || callbacks.onStatus)) {
return {
onProgress: callbacks.onProgress || function () {},
onStatus: callbacks.onStatus || function () {},
};
}
// Handle nsIChannelEventSink (for asyncOnChannelRedirect)
if (aIID.equals(Ci.nsIChannelEventSink) && callbacks.asyncOnChannelRedirect) {
return {
asyncOnChannelRedirect: callbacks.asyncOnChannelRedirect,
};
}
throw Components.Exception("No interface available", Cr.NS_ERROR_NO_INTERFACE);
}
};
}
/**
* Download a file
*
* @param {nsIURI|String} url - URL to request
* @param {String} path - Path to save file to
* @param {Object} [options] - See `Zotero.HTTP.request()`
*/
this.download = async function(uri, path, options = {}) {
// TODO: Convert request() to fetch() and use ReadableStream
var req = await this.request(
'GET',
uri,
{
...options,
responseType: 'blob',
// Downloads can have channel notification callbacks, etc., so always do them for real
noMock: true
}
);
var bytes = await IOUtils.write(path, await req.response.bytes());
Zotero.debug(`Saved file to ${path} (${bytes} byte${bytes != 1 ? 's' : ''})`);
return req;
};
/**
* Send an HTTP GET request via XMLHTTPRequest
*
@ -1087,6 +1151,9 @@ Zotero.HTTP = new function() {
this.getDisplayURI = function (uri, noCredentials) {
if (typeof uri == 'string') {
uri = NetUtil.newURI(uri);
}
if (!uri.password) return uri;
uri = uri.mutate();
if (noCredentials) {

276
chrome/content/zotero/xpcom/storage/streamListener.js
View file

@ -1,276 +0,0 @@
/*
***** BEGIN LICENSE BLOCK *****
Copyright © 2009 Center for History and New Media
George Mason University, Fairfax, Virginia, USA
http://zotero.org
This file is part of Zotero.
Zotero is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Zotero is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with Zotero. If not, see <http://www.gnu.org/licenses/>.
***** END LICENSE BLOCK *****
*/
/**
* Stream listener that can handle both download and upload requests
*
* Possible properties of data object:
* - onStart: f(request)
* - onProgress: f(request, progress, progressMax)
* - onStop: f(request, status, response)
* - onCancel: f(request, status)
* - streams: array of streams to close on completion
*/
Zotero.Sync.Storage.StreamListener = function (data) {
this._data = data;
}
Zotero.Sync.Storage.StreamListener.prototype = {
_channel: null,
// nsIProgressEventSink
onProgress: function (request, progress, progressMax) {
Zotero.debug("onProgress with " + progress + "/" + progressMax);
this._onProgress(request, progress, progressMax);
},
onStatus: function (request, status, statusArg) {
Zotero.debug('onStatus with ' + status);
},
// nsIRequestObserver
// Note: For uploads, this isn't called until data is done uploading
onStartRequest: function (request) {
Zotero.debug('onStartRequest');
this._response = "";
this._onStart(request);
},
onStopRequest: function (request, status) {
Zotero.debug('onStopRequest with ' + status);
// Some errors from https://developer.mozilla.org/en-US/docs/Table_Of_Errors
var msg = "";
switch (status) {
// Normal
case 0:
break;
// NS_BINDING_ABORTED
case 0x804b0002:
msg = "Request cancelled";
break;
// NS_ERROR_NET_INTERRUPT
case 0x804B0047:
msg = "Request interrupted";
break;
// NS_ERROR_NET_TIMEOUT
case 0x804B000E:
msg = "Request timed out";
break;
default:
msg = "Request failed";
break;
}
if (msg) {
msg += " in Zotero.Sync.Storage.StreamListener.onStopRequest() (" + status + ")";
Components.utils.reportError(msg);
Zotero.debug(msg, 1);
}
this._onStop(request, status);
},
// nsIWebProgressListener
onProgressChange: function (wp, request, curSelfProgress,
maxSelfProgress, curTotalProgress, maxTotalProgress) {
//Zotero.debug("onProgressChange with " + curTotalProgress + "/" + maxTotalProgress);
// onProgress gets called too, so this isn't necessary
//this._onProgress(request, curTotalProgress, maxTotalProgress);
},
onStateChange: function (wp, request, stateFlags, status) {
Zotero.debug("onStateChange with " + stateFlags);
if (stateFlags & Components.interfaces.nsIWebProgressListener.STATE_IS_REQUEST) {
if (stateFlags & Components.interfaces.nsIWebProgressListener.STATE_START) {
this._onStart(request);
}
else if (stateFlags & Components.interfaces.nsIWebProgressListener.STATE_STOP) {
this._onStop(request, status);
}
}
},
onStatusChange: function (progress, request, status, message) {
Zotero.debug("onStatusChange with '" + message + "'");
},
onLocationChange: function () {
Zotero.debug('onLocationChange');
},
onSecurityChange: function () {
Zotero.debug('onSecurityChange');
},
// nsIStreamListener
onDataAvailable: function (request, stream, sourceOffset, length) {
Zotero.debug('onDataAvailable');
var scriptableInputStream =
Components.classes["@mozilla.org/scriptableinputstream;1"]
.createInstance(Components.interfaces.nsIScriptableInputStream);
scriptableInputStream.init(stream);
var data = scriptableInputStream.read(length);
Zotero.debug(data);
this._response += data;
},
// nsIChannelEventSink
//
// If this._data.onChannelRedirect exists, it should return a promise resolving to true to
// follow the redirect or false to cancel it
onChannelRedirect: Zotero.Promise.coroutine(function* (oldChannel, newChannel, flags) {
Zotero.debug('onChannelRedirect');
if (this._data && this._data.onChannelRedirect) {
let result = yield this._data.onChannelRedirect(oldChannel, newChannel, flags);
if (!result) {
oldChannel.cancel(Components.results.NS_BINDING_ABORTED);
newChannel.cancel(Components.results.NS_BINDING_ABORTED);
Zotero.debug("Cancelling redirect");
// TODO: Prevent onStateChange error
return false;
}
}
// if redirecting, store the new channel
this._channel = newChannel;
}),
asyncOnChannelRedirect: function (oldChan, newChan, flags, redirectCallback) {
Zotero.debug('asyncOnRedirect');
this.onChannelRedirect(oldChan, newChan, flags)
.then(function (result) {
redirectCallback.onRedirectVerifyCallback(
result ? Components.results.NS_SUCCEEDED : Components.results.NS_FAILED
);
})
.catch(function (e) {
Zotero.logError(e);
redirectCallback.onRedirectVerifyCallback(Components.results.NS_FAILED);
});
},
// nsIHttpEventSink
onRedirect: function (oldChannel, newChannel) {
Zotero.debug('onRedirect');
var newURL = Zotero.HTTP.getDisplayURI(newChannel.URI).spec;
Zotero.debug("Redirecting to " + newURL);
},
//
// Private methods
//
_onStart: function (request) {
Zotero.debug('Starting request');
if (this._data && this._data.onStart) {
this._data.onStart(request);
}
},
_onProgress: function (request, progress, progressMax) {
if (this._data && this._data.onProgress) {
this._data.onProgress(request, progress, progressMax);
}
},
_onStop: function (request, status) {
var cancelled = status == 0x804b0002; // NS_BINDING_ABORTED
if (!cancelled && status == 0 && request instanceof Components.interfaces.nsIHttpChannel) {
request.QueryInterface(Components.interfaces.nsIHttpChannel);
try {
status = request.responseStatus;
}
catch (e) {
Zotero.debug("Request responseStatus not available", 1);
status = 0;
}
Zotero.debug('Request ended with status code ' + status);
request.QueryInterface(Components.interfaces.nsIRequest);
}
else {
Zotero.debug('Request ended with status ' + status);
status = 0;
}
if (this._data.streams) {
for (let stream of this._data.streams) {
stream.close();
}
}
if (cancelled) {
if (this._data.onCancel) {
this._data.onCancel(request, status);
}
}
else {
if (this._data.onStop) {
this._data.onStop(request, status, this._response);
}
}
this._channel = null;
},
// nsIInterfaceRequestor
getInterface: function (iid) {
try {
return this.QueryInterface(iid);
}
catch (e) {
throw Components.results.NS_NOINTERFACE;
}
},
QueryInterface: function(iid) {
if (iid.equals(Components.interfaces.nsISupports) ||
iid.equals(Components.interfaces.nsIInterfaceRequestor) ||
iid.equals(Components.interfaces.nsIChannelEventSink) ||
iid.equals(Components.interfaces.nsIProgressEventSink) ||
iid.equals(Components.interfaces.nsIHttpEventSink) ||
iid.equals(Components.interfaces.nsIStreamListener) ||
iid.equals(Components.interfaces.nsIWebProgressListener)) {
return this;
}
throw Components.results.NS_NOINTERFACE;
},
_safeSpec: function (uri) {
return uri.scheme + '://' + uri.username + ':********@'
+ uri.hostPort + uri.pathQueryRef
},
};

146
chrome/content/zotero/xpcom/storage/webdav.js
View file

@ -278,8 +278,8 @@ Zotero.Sync.Storage.Mode.WebDAV.prototype = {
* @param {Zotero.Sync.Storage.Request} request
* @return {Promise<Zotero.Sync.Storage.Result>}
*/
downloadFile: Zotero.Promise.coroutine(function* (request) {
yield this._init();
downloadFile: async function (request) {
await this._init();
var item = Zotero.Sync.Storage.Utilities.getItemFromRequest(request);
if (!item) {
@ -294,7 +294,7 @@ Zotero.Sync.Storage.Mode.WebDAV.prototype = {
}
// Retrieve modification time from server
var metadata = yield this._getStorageFileMetadata(item, request);
var metadata = await this._getStorageFileMetadata(item, request);
if (!request.isRunning()) {
Zotero.debug("Download request '" + request.name
@ -307,7 +307,7 @@ Zotero.Sync.Storage.Mode.WebDAV.prototype = {
return new Zotero.Sync.Storage.Result;
}
var fileModTime = yield item.attachmentModificationTime;
var fileModTime = await item.attachmentModificationTime;
if (metadata.mtime == fileModTime) {
Zotero.debug("File mod time matches remote file -- skipping download of "
+ item.libraryKey);
@ -315,10 +315,10 @@ Zotero.Sync.Storage.Mode.WebDAV.prototype = {
var updateItem = item.attachmentSyncState != 1
item.attachmentSyncedModificationTime = metadata.mtime;
item.attachmentSyncState = "in_sync";
yield item.saveTx({ skipAll: true });
await item.saveTx({ skipAll: true });
// DEBUG: Necessary?
if (updateItem) {
yield item.updateSynced(false);
await item.updateSynced(false);
}
return new Zotero.Sync.Storage.Result({
@ -329,9 +329,8 @@ Zotero.Sync.Storage.Mode.WebDAV.prototype = {
var uri = this._getItemURI(item);
var destPath = OS.Path.join(Zotero.getTempDirectory().path, item.key + '.tmp');
yield Zotero.File.removeIfExists(destPath);
await Zotero.File.removeIfExists(destPath);
var deferred = Zotero.Promise.defer();
var requestData = {
item,
mtime: metadata.mtime,
@ -339,88 +338,71 @@ Zotero.Sync.Storage.Mode.WebDAV.prototype = {
compressed: true
};
var listener = new Zotero.Sync.Storage.StreamListener(
{
onStart: function (req) {
if (request.isFinished()) {
Zotero.debug("Download request " + request.name
+ " stopped before download started -- closing channel");
req.cancel(0x804b0002); // NS_BINDING_ABORTED
deferred.resolve(new Zotero.Sync.Storage.Result);
return new Promise(async (resolve, reject) => {
try {
let req = await Zotero.HTTP.download(
uri,
destPath,
{
successCodes: [200, 404],
noCache: true,
notificationCallbacks: {
onProgress: function (a, b, c) {
request.onProgress(a, b, c)
},
},
errorDelayIntervals: this.ERROR_DELAY_INTERVALS,
errorDelayMax: this.ERROR_DELAY_MAX,
}
},
onProgress: function (a, b, c) {
request.onProgress(a, b, c)
},
onStop: Zotero.Promise.coroutine(function* (req, status, res) {
request.setChannel(false);
if (status == 404) {
let msg = "Remote ZIP file not found for item " + item.libraryKey;
Zotero.debug(msg, 2);
Components.utils.reportError(msg);
// Delete the orphaned prop file
try {
yield this._deleteStorageFiles([item.key + ".prop"]);
}
catch (e) {
Zotero.logError(e);
}
deferred.resolve(new Zotero.Sync.Storage.Result);
return;
}
else if (status != 200) {
try {
this._throwFriendlyError("GET", dispURL, status);
}
catch (e) {
deferred.reject(e);
}
return;
}
// Don't try to process if the request has been cancelled
if (request.isFinished()) {
Zotero.debug("Download request " + request.name
+ " is no longer running after file download");
deferred.resolve(new Zotero.Sync.Storage.Result);
return;
}
Zotero.debug("Finished download of " + destPath);
);
if (req.status == 404) {
let msg = "Remote ZIP file not found for item " + item.libraryKey;
Zotero.debug(msg, 2);
Cu.reportError(msg);
// Delete the orphaned prop file
try {
deferred.resolve(
Zotero.Sync.Storage.Local.processDownload(requestData)
);
await this._deleteStorageFiles([item.key + ".prop"]);
}
catch (e) {
deferred.reject(e);
}
}.bind(this)),
onCancel: function (req, status) {
Zotero.debug("Request cancelled");
if (deferred.promise.isPending()) {
deferred.resolve(new Zotero.Sync.Storage.Result);
Zotero.logError(e);
}
resolve(new Zotero.Sync.Storage.Result);
return;
}
// Don't try to process if the request has been cancelled
if (request.isFinished()) {
Zotero.debug("Download request " + request.name
+ " is no longer running after file download");
resolve(new Zotero.Sync.Storage.Result);
return;
}
Zotero.debug("Finished download of " + destPath);
resolve(
Zotero.Sync.Storage.Local.processDownload(requestData)
);
}
);
// Don't display password in console
var dispURL = Zotero.HTTP.getDisplayURI(uri).spec;
Zotero.debug('Saving ' + dispURL);
const nsIWBP = Components.interfaces.nsIWebBrowserPersist;
var wbp = Components.classes["@mozilla.org/embedding/browser/nsWebBrowserPersist;1"]
.createInstance(nsIWBP);
wbp.persistFlags = nsIWBP.PERSIST_FLAGS_BYPASS_CACHE;
wbp.progressListener = listener;
Zotero.Utilities.Internal.saveURI(wbp, uri, destPath);
return deferred.promise;
}),
catch (e) {
if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
try {
let dispURL = Zotero.HTTP.getDisplayURI(uri).spec;
this._throwFriendlyError("GET", dispURL, e.xmlhttp.status);
}
catch (e) {
reject(e);
}
return;
}
Zotero.logError(e);
reject(new Error(Zotero.Sync.Storage.defaultError));
}
});
},
uploadFile: Zotero.Promise.coroutine(function* (request) {

310
chrome/content/zotero/xpcom/storage/zfs.js
View file

@ -49,7 +49,7 @@ Zotero.Sync.Storage.Mode.ZFS.prototype = {
* @param {Zotero.Sync.Storage.Request} request
* @return {Promise<Zotero.Sync.Storage.Result>}
*/
downloadFile: Zotero.Promise.coroutine(function* (request) {
downloadFile: async function (request) {
var item = Zotero.Sync.Storage.Utilities.getItemFromRequest(request);
if (!item) {
throw new Error("Item '" + request.name + "' not found");
@ -63,195 +63,147 @@ Zotero.Sync.Storage.Mode.ZFS.prototype = {
var destPath = OS.Path.join(Zotero.getTempDirectory().path, item.key + '.tmp');
// saveURI() below appears not to create empty files for Content-Length: 0,
// so we create one here just in case, which also lets us check file access
// Create an empty file to check file access
try {
yield IOUtils.write(destPath, new Uint8Array());
await IOUtils.write(destPath, new Uint8Array());
}
catch (e) {
Zotero.File.checkFileAccessError(e, destPath, 'create');
}
var deferred = Zotero.Promise.defer();
var requestData = {item};
var listener = new Zotero.Sync.Storage.StreamListener(
{
onStart: function (req) {
if (request.isFinished()) {
Zotero.debug("Download request " + request.name
+ " stopped before download started -- closing channel");
req.cancel(Components.results.NS_BINDING_ABORTED);
deferred.resolve(new Zotero.Sync.Storage.Result);
}
},
onChannelRedirect: Zotero.Promise.coroutine(function* (oldChannel, newChannel, flags) {
// These will be used in processDownload() if the download succeeds
oldChannel.QueryInterface(Components.interfaces.nsIHttpChannel);
Zotero.debug("CHANNEL HERE FOR " + item.libraryKey + " WITH " + oldChannel.status);
Zotero.debug(oldChannel.URI.spec);
Zotero.debug(newChannel.URI.spec);
var header;
try {
header = "Zotero-File-Modification-Time";
requestData.mtime = parseInt(oldChannel.getResponseHeader(header));
header = "Zotero-File-MD5";
requestData.md5 = oldChannel.getResponseHeader(header);
header = "Zotero-File-Compressed";
requestData.compressed = oldChannel.getResponseHeader(header) == 'Yes';
}
catch (e) {
deferred.reject(new Error(`${header} header not set in file request for ${item.libraryKey}`));
return false;
}
if (!(yield OS.File.exists(path))) {
return true;
}
var updateHash = false;
var fileModTime = yield item.attachmentModificationTime;
if (requestData.mtime == fileModTime) {
Zotero.debug("File mod time matches remote file -- skipping download of "
+ item.libraryKey);
}
// If not compressed, check hash, in case only timestamp changed
else if (!requestData.compressed && (yield item.attachmentHash) == requestData.md5) {
Zotero.debug("File hash matches remote file -- skipping download of "
+ item.libraryKey);
updateHash = true;
}
else {
return true;
}
// Update local metadata and stop request, skipping file download
yield OS.File.setDates(path, null, new Date(requestData.mtime));
item.attachmentSyncedModificationTime = requestData.mtime;
if (updateHash) {
item.attachmentSyncedHash = requestData.md5;
}
item.attachmentSyncState = "in_sync";
yield item.saveTx({ skipAll: true });
deferred.resolve(new Zotero.Sync.Storage.Result({
localChanges: true
}));
return false;
}),
onProgress: function (req, progress, progressMax) {
request.onProgress(progress, progressMax);
},
onStop: function (req, status, res) {
request.setChannel(false);
if (status != 200) {
if (status == 404) {
Zotero.debug("Remote file not found for item " + item.libraryKey);
// Don't refresh item pane rows when nothing happened
request.skipProgressBarUpdate = true;
deferred.resolve(new Zotero.Sync.Storage.Result);
return;
}
// Check for SSL certificate error
if (status == 0) {
try {
Zotero.HTTP.checkSecurity(req);
}
catch (e) {
deferred.reject(e);
return;
}
}
// If S3 connection is interrupted, delay and retry, or bail if too many
// consecutive failures
if (status == 0 || status == 500 || status == 503) {
if (++this._s3ConsecutiveFailures < this._maxS3ConsecutiveFailures) {
let libraryKey = item.libraryKey;
let msg = "S3 returned 0 for " + libraryKey + " -- retrying download"
Components.utils.reportError(msg);
Zotero.debug(msg, 1);
if (this._s3Backoff < this._maxS3Backoff) {
this._s3Backoff *= 2;
}
Zotero.debug("Delaying " + libraryKey + " download for "
+ this._s3Backoff + " seconds", 2);
Zotero.Promise.delay(this._s3Backoff * 1000)
.then(function () {
deferred.resolve(this.downloadFile(request));
}.bind(this));
return;
}
Zotero.debug(this._s3ConsecutiveFailures
+ " consecutive S3 failures -- aborting", 1);
this._s3ConsecutiveFailures = 0;
}
var msg = "Unexpected status code " + status + " for GET " + uri;
Zotero.debug(msg, 1);
Components.utils.reportError(msg);
// Output saved content, in case an error was captured
try {
let sample = Zotero.File.getContents(destPath, null, 4096);
if (sample) {
Zotero.debug(sample, 1);
}
}
catch (e) {
Zotero.debug(e, 1);
}
deferred.reject(new Error(Zotero.Sync.Storage.defaultError));
return;
}
// Don't try to process if the request has been cancelled
if (request.isFinished()) {
Zotero.debug("Download request " + request.name
+ " is no longer running after file download", 2);
deferred.resolve(new Zotero.Sync.Storage.Result);
return;
}
Zotero.debug("Finished download of " + destPath);
try {
deferred.resolve(
Zotero.Sync.Storage.Local.processDownload(requestData)
);
}
catch (e) {
deferred.reject(e);
}
}.bind(this),
onCancel: function (req, status) {
Zotero.debug("Request cancelled");
if (deferred.promise.isPending()) {
deferred.resolve(new Zotero.Sync.Storage.Result);
}
}
}
);
var params = this._getRequestParams(item.libraryID, `items/${item.key}/file`);
var uri = this.apiClient.buildRequestURI(params);
var headers = this.apiClient.getHeaders();
Zotero.debug('Saving ' + uri);
const nsIWBP = Components.interfaces.nsIWebBrowserPersist;
var wbp = Components.classes["@mozilla.org/embedding/browser/nsWebBrowserPersist;1"]
.createInstance(nsIWBP);
wbp.persistFlags = nsIWBP.PERSIST_FLAGS_BYPASS_CACHE;
wbp.progressListener = listener;
Zotero.Utilities.Internal.saveURI(wbp, uri, destPath, headers);
return deferred.promise;
}),
return new Promise(async (resolve, reject) => {
try {
let req = await Zotero.HTTP.download(
uri,
destPath,
{
successCodes: [200, 404],
headers: this.apiClient.getHeaders(),
noCache: true,
notificationCallbacks: {
asyncOnChannelRedirect: async function (oldChannel, newChannel, flags, callback) {
// These will be used in processDownload() if the download succeeds
oldChannel.QueryInterface(Components.interfaces.nsIHttpChannel);
Zotero.debug(`Handling ${oldChannel.responseStatus} redirect for ${item.libraryKey}`);
Zotero.debug(oldChannel.URI.spec);
Zotero.debug(newChannel.URI.spec);
var header;
try {
header = "Zotero-File-Modification-Time";
requestData.mtime = parseInt(oldChannel.getResponseHeader(header));
header = "Zotero-File-MD5";
requestData.md5 = oldChannel.getResponseHeader(header);
header = "Zotero-File-Compressed";
requestData.compressed = oldChannel.getResponseHeader(header) == 'Yes';
}
catch (_e) {
reject(new Error(`${header} header not set in file request for ${item.libraryKey}`));
callback.onRedirectVerifyCallback(Cr.NS_ERROR_ABORT);
return;
}
if (!(await IOUtils.exists(path))) {
callback.onRedirectVerifyCallback(Cr.NS_OK);
return;
}
var updateHash = false;
var fileModTime = await item.attachmentModificationTime;
if (requestData.mtime == fileModTime) {
Zotero.debug("File mod time matches remote file -- skipping download of "
+ item.libraryKey);
}
// If not compressed, check hash, in case only timestamp changed
else if (!requestData.compressed && (await item.attachmentHash) == requestData.md5) {
Zotero.debug("File hash matches remote file -- skipping download of "
+ item.libraryKey);
updateHash = true;
}
else {
callback.onRedirectVerifyCallback(Cr.NS_OK);
return;
}
// Update local metadata and stop request, skipping file download
await OS.File.setDates(path, null, new Date(requestData.mtime));
item.attachmentSyncedModificationTime = requestData.mtime;
if (updateHash) {
item.attachmentSyncedHash = requestData.md5;
}
item.attachmentSyncState = "in_sync";
await item.saveTx({ skipAll: true });
resolve(new Zotero.Sync.Storage.Result({
localChanges: true
}));
callback.onRedirectVerifyCallback(Cr.NS_ERROR_ABORT);
},
onProgress: function (req, progress, progressMax) {
request.onProgress(progress, progressMax);
},
},
}
);
if (req.status == 404) {
Zotero.debug("Remote file not found for item " + item.libraryKey);
// Don't refresh item pane rows when nothing happened
request.skipProgressBarUpdate = true;
resolve(new Zotero.Sync.Storage.Result);
return;
}
// Don't try to process if the request has been cancelled
if (request.isFinished()) {
Zotero.debug(`Download request ${request.name} is no longer running after file download`, 2);
resolve(new Zotero.Sync.Storage.Result);
return;
}
Zotero.debug("Finished download of " + destPath);
resolve(await Zotero.Sync.Storage.Local.processDownload(requestData));
}
catch (e) {
if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
// If S3 connection is interrupted, delay and retry, or bail if too many
// consecutive failures
if (e.xmlhttp.status == 0) {
if (++this._s3ConsecutiveFailures < this._maxS3ConsecutiveFailures) {
let libraryKey = item.libraryKey;
let msg = "S3 returned 0 for " + libraryKey + " -- retrying download";
Zotero.logError(msg);
if (this._s3Backoff < this._maxS3Backoff) {
this._s3Backoff *= 2;
}
Zotero.debug("Delaying " + libraryKey + " download for "
+ this._s3Backoff + " seconds", 2);
Zotero.Promise.delay(this._s3Backoff * 1000)
.then(function () {
resolve(this.downloadFile(request));
}.bind(this));
return;
}
Zotero.debug(this._s3ConsecutiveFailures
+ " consecutive S3 failures -- aborting", 1);
this._s3ConsecutiveFailures = 0;
}
}
Zotero.logError(e);
reject(new Error(Zotero.Sync.Storage.defaultError));
}
});
},
uploadFile: Zotero.Promise.coroutine(function* (request) {

2
chrome/content/zotero/xpcom/utilities_internal.js
View file

@ -459,6 +459,8 @@ Zotero.Utilities.Internal = {
* @param {Zotero.CookieSandbox} [cookieSandbox]
*/
saveURI: function (wbp, uri, target, headers, cookieSandbox) {
Zotero.warn("Zotero.Utilities.Internal.saveURI() is deprecated -- use Zotero.HTTP.download()");
// Handle gzip encoding
wbp.persistFlags |= wbp.PERSIST_FLAGS_AUTODETECT_APPLY_CONVERSION;
// If not explicitly using cache, skip it

1
chrome/content/zotero/zotero.mjs
View file

@ -144,7 +144,6 @@ const xpcomFilesLocal = [
'storage/storageRequest',
'storage/storageResult',
'storage/storageUtilities',
'storage/streamListener',
'storage/zfs',
'storage/webdav',
'syncedSettings',

2
test/tests/zfsTest.js
View file

@ -103,6 +103,7 @@ describe("Zotero.Sync.Storage.Mode.ZFS", function () {
httpd.stop(() => defer.resolve());
yield defer.promise;
win.close();
Zotero.HTTP.disableErrorRetry = false;
})
after(function* () {
@ -211,6 +212,7 @@ describe("Zotero.Sync.Storage.Mode.ZFS", function () {
item.attachmentSyncState = "to_download";
yield item.saveTx();
Zotero.HTTP.disableErrorRetry = true;
httpd.registerPathHandler(
`/users/1/items/${item.key}/file`,
{