WebDAV file sync overhaul for 5.0

Also:

- Remove last-sync-time mechanism for both WebDAV and ZFS, since it can
  be determined by storage properties (mtime/md5) in data sync
- Add option to include synced storage properties in item toJSON()
  instead of local file properties
- Set "Fake-Server-Match" header in setHTTPResponse() test support
  function, which can be used for request count assertions -- see
  resetRequestCount() and assertRequestCount() in webdavTest.js
- Allow string (e.g., 'to_download') instead of constant in
  Zotero.Sync.Data.Local.setSyncState()
- Misc storage tweaks
This commit is contained in:
Dan Stillman 2015-12-23 04:52:09 -05:00
parent 6844deba60
commit c5a9987f37
32 changed files with 3182 additions and 2789 deletions

View file

@ -28,11 +28,17 @@ Components.utils.import("resource://gre/modules/Services.jsm");
Zotero_Preferences.Sync = {
init: Zotero.Promise.coroutine(function* () {
this.updateStorageSettings(null, null, true);
this.updateStorageSettingsUI();
var username = Zotero.Users.getCurrentUsername() || "";
var apiKey = Zotero.Sync.Data.Local.getAPIKey();
this.displayFields(apiKey ? username : "");
var pass = Zotero.Sync.Runner.getStorageController('webdav').password;
if (pass) {
document.getElementById('storage-password').value = pass;
}
if (apiKey) {
try {
var keyInfo = yield Zotero.Sync.Runner.checkAccess(
@ -57,13 +63,6 @@ Zotero_Preferences.Sync = {
}
}
}
// TEMP: Disabled
//var pass = Zotero.Sync.Storage.WebDAV.password;
//if (pass) {
// document.getElementById('storage-password').value = pass;
//}
}),
displayFields: function (username) {
@ -249,17 +248,13 @@ Zotero_Preferences.Sync = {
return true;
}),
updateStorageSettings: function (enabled, protocol, skipWarnings) {
if (enabled === null) {
enabled = document.getElementById('pref-storage-enabled').value;
}
updateStorageSettingsUI: Zotero.Promise.coroutine(function* () {
this.unverifyStorageServer();
var oldProtocol = document.getElementById('pref-storage-protocol').value;
if (protocol === null) {
protocol = oldProtocol;
}
var protocol = document.getElementById('pref-storage-protocol').value;
var enabled = document.getElementById('pref-storage-enabled').value;
var storageSettings = document.getElementById('storage-settings');
var protocolMenu = document.getElementById('storage-protocol');
@ -275,66 +270,13 @@ Zotero_Preferences.Sync = {
sep.hidden = true;
}
var menulists = storageSettings.getElementsByTagName('menulist');
for each(var menulist in menulists) {
if (menulist.className == 'storage-personal') {
menulist.disabled = !enabled;
}
var menulists = document.querySelectorAll('#storage-settings menulist.storage-personal');
for (let menulist of menulists) {
menulist.disabled = !enabled;
}
if (!skipWarnings) {
// WARN if going between
}
if (oldProtocol == 'zotero' && protocol == 'webdav') {
var sql = "SELECT COUNT(*) FROM version WHERE schema LIKE 'storage_zfs%'";
if (Zotero.DB.valueQuery(sql)) {
var ps = Components.classes["@mozilla.org/embedcomp/prompt-service;1"]
.getService(Components.interfaces.nsIPromptService);
var buttonFlags = (ps.BUTTON_POS_0) * (ps.BUTTON_TITLE_IS_STRING)
+ (ps.BUTTON_POS_1) * (ps.BUTTON_TITLE_IS_STRING)
+ ps.BUTTON_DELAY_ENABLE;
var account = Zotero.Sync.Server.username;
var index = ps.confirmEx(
null,
Zotero.getString('zotero.preferences.sync.purgeStorage.title'),
Zotero.getString('zotero.preferences.sync.purgeStorage.desc'),
buttonFlags,
Zotero.getString('zotero.preferences.sync.purgeStorage.confirmButton'),
Zotero.getString('zotero.preferences.sync.purgeStorage.cancelButton'), null, null, {}
);
if (index == 0) {
var sql = "INSERT OR IGNORE INTO settings VALUES (?,?,?)";
Zotero.DB.query(sql, ['storage', 'zfsPurge', 'user']);
Zotero.Sync.Storage.ZFS.purgeDeletedStorageFiles()
.then(function () {
ps.alert(
null,
Zotero.getString("general.success"),
"Attachment files from your personal library have been removed from the Zotero servers."
);
})
.catch(function (e) {
Zotero.debug(e, 1);
Components.utils.reportError(e);
ps.alert(
null,
Zotero.getString("general.error"),
"An error occurred. Please try again later."
);
});
}
}
}
var self = this;
setTimeout(function () {
self.updateStorageTerms();
}, 1)
},
this.updateStorageTerms();
}),
updateStorageSettingsGroups: function (enabled) {
@ -364,14 +306,90 @@ Zotero_Preferences.Sync = {
},
unverifyStorageServer: function () {
Zotero.Prefs.set('sync.storage.verified', false);
Zotero.Sync.Storage.WebDAV.clearCachedCredentials();
Zotero.Sync.Storage.resetAllSyncStates(null, true, false);
},
onStorageSettingsKeyPress: Zotero.Promise.coroutine(function* (event) {
if (event.keyCode == 13) {
yield this.onStorageSettingsChange();
yield this.verifyStorageServer();
}
}),
verifyStorageServer: function () {
onStorageSettingsChange: Zotero.Promise.coroutine(function* () {
// Clean URL
var urlPref = document.getElementById('pref-storage-url');
urlPref.value = urlPref.value.replace(/(^https?:\/\/|\/zotero\/?$|\/$)/g, '');
var oldProtocol = document.getElementById('pref-storage-protocol').value;
var oldEnabled = document.getElementById('pref-storage-enabled').value;
yield Zotero.Promise.delay(1);
var newProtocol = document.getElementById('pref-storage-protocol').value;
var newEnabled = document.getElementById('pref-storage-enabled').value;
if (oldProtocol != newProtocol) {
yield Zotero.Sync.Storage.Local.resetModeSyncStates(oldProtocol);
}
if (oldProtocol == 'webdav') {
this.unverifyStorageServer();
Zotero.Sync.Runner.resetStorageController(oldProtocol);
var username = document.getElementById('storage-username').value;
var password = document.getElementById('storage-password').value;
if (username) {
Zotero.Sync.Runner.getStorageController('webdav').password = password;
}
}
if (oldProtocol == 'zotero' && newProtocol == 'webdav') {
var sql = "SELECT COUNT(*) FROM settings "
+ "WHERE schema='storage' AND key='zfsPurge' AND value='user'";
if (!Zotero.DB.valueQueryAsync(sql)) {
var ps = Components.classes["@mozilla.org/embedcomp/prompt-service;1"]
.getService(Components.interfaces.nsIPromptService);
var buttonFlags = (ps.BUTTON_POS_0) * (ps.BUTTON_TITLE_IS_STRING)
+ (ps.BUTTON_POS_1) * (ps.BUTTON_TITLE_IS_STRING)
+ ps.BUTTON_DELAY_ENABLE;
var account = Zotero.Sync.Server.username;
var index = ps.confirmEx(
null,
Zotero.getString('zotero.preferences.sync.purgeStorage.title'),
Zotero.getString('zotero.preferences.sync.purgeStorage.desc'),
buttonFlags,
Zotero.getString('zotero.preferences.sync.purgeStorage.confirmButton'),
Zotero.getString('zotero.preferences.sync.purgeStorage.cancelButton'), null, null, {}
);
if (index == 0) {
var sql = "INSERT OR IGNORE INTO settings VALUES (?,?,?)";
yield Zotero.DB.queryAsync(sql, ['storage', 'zfsPurge', 'user']);
try {
yield Zotero.Sync.Storage.ZFS.purgeDeletedStorageFiles();
ps.alert(
null,
Zotero.getString("general.success"),
"Attachment files from your personal library have been removed from the Zotero servers."
);
}
catch (e) {
Zotero.logError(e);
ps.alert(
null,
Zotero.getString("general.error"),
"An error occurred. Please try again later."
);
}
}
}
}
this.updateStorageSettingsUI();
}),
verifyStorageServer: Zotero.Promise.coroutine(function* () {
Zotero.debug("Verifying storage");
var verifyButton = document.getElementById("storage-verify");
@ -385,78 +403,56 @@ Zotero_Preferences.Sync = {
abortButton.hidden = false;
progressMeter.hidden = false;
var success = false;
var request = null;
var onDone = false;
Zotero.Sync.Storage.WebDAV.checkServer()
// Get the XMLHttpRequest for possible cancelling
.progress(function (obj) {
request = obj.xmlhttp;
})
.finally(function () {
var controller = Zotero.Sync.Runner.getStorageController('webdav');
try {
yield controller.checkServer({
// Get the XMLHttpRequest for possible cancelling
onRequest: r => request = r
})
success = true;
}
catch (e) {
if (e instanceof controller.VerificationError) {
switch (e.error) {
case "NO_URL":
urlField.focus();
break;
case "NO_USERNAME":
usernameField.focus();
break;
case "NO_PASSWORD":
case "AUTH_FAILED":
passwordField.focus();
break;
}
}
success = yield controller.handleVerificationError(e);
}
finally {
verifyButton.hidden = false;
abortButton.hidden = true;
progressMeter.hidden = true;
})
.spread(function (uri, status) {
switch (status) {
case Zotero.Sync.Storage.ERROR_NO_URL:
onDone = function () {
urlField.focus();
};
break;
case Zotero.Sync.Storage.ERROR_NO_USERNAME:
onDone = function () {
usernameField.focus();
};
break;
case Zotero.Sync.Storage.ERROR_NO_PASSWORD:
case Zotero.Sync.Storage.ERROR_AUTH_FAILED:
onDone = function () {
passwordField.focus();
};
break;
}
}
if (success) {
Zotero.debug("WebDAV verification succeeded");
return Zotero.Sync.Storage.WebDAV.checkServerCallback(uri, status, window);
})
.then(function (success) {
if (success) {
Zotero.debug("WebDAV verification succeeded");
var promptService = Components.classes["@mozilla.org/embedcomp/prompt-service;1"]
.getService(Components.interfaces.nsIPromptService);
promptService.alert(
window,
Zotero.getString('sync.storage.serverConfigurationVerified'),
Zotero.getString('sync.storage.fileSyncSetUp')
);
Zotero.Prefs.set("sync.storage.verified", true);
}
else {
Zotero.debug("WebDAV verification failed");
if (onDone) {
setTimeout(function () {
onDone();
}, 1);
}
}
})
.catch(function (e) {
Zotero.debug("WebDAV verification failed");
Zotero.debug(e, 1);
Components.utils.reportError(e);
Zotero.Utilities.Internal.errorPrompt(Zotero.getString('general.error'), e);
if (onDone) {
setTimeout(function () {
onDone();
}, 1);
}
})
.done();
Zotero.alert(
window,
Zotero.getString('sync.storage.serverConfigurationVerified'),
Zotero.getString('sync.storage.fileSyncSetUp')
);
}
else {
Zotero.logError("WebDAV verification failed");
}
abortButton.onclick = function () {
if (request) {
@ -468,6 +464,12 @@ Zotero_Preferences.Sync = {
progressMeter.hidden = true;
}
}
}),
unverifyStorageServer: function () {
Zotero.debug("Unverifying storage");
Zotero.Prefs.set('sync.storage.verified', false);
},

View file

@ -34,8 +34,7 @@
<preference id="pref-sync-username" name="extensions.zotero.sync.server.username" type="unichar" instantApply="true"/>
<preference id="pref-sync-fulltext-enabled" name="extensions.zotero.sync.fulltext.enabled" type="bool"/>
<preference id="pref-storage-enabled" name="extensions.zotero.sync.storage.enabled" type="bool"/>
<preference id="pref-storage-protocol" name="extensions.zotero.sync.storage.protocol" type="string"
onchange="Zotero_Preferences.Sync.unverifyStorageServer()"/>
<preference id="pref-storage-protocol" name="extensions.zotero.sync.storage.protocol" type="string"/>
<preference id="pref-storage-scheme" name="extensions.zotero.sync.storage.scheme" type="string" instantApply="true"/>
<preference id="pref-storage-url" name="extensions.zotero.sync.storage.url" type="string"/>
<preference id="pref-storage-username" name="extensions.zotero.sync.storage.username" type="string"/>
@ -152,14 +151,14 @@
<hbox>
<checkbox label="&zotero.preferences.sync.fileSyncing.myLibrary;"
preference="pref-storage-enabled"
oncommand="Zotero_Preferences.Sync.updateStorageSettings(this.checked, null)"/>
oncommand="Zotero_Preferences.Sync.onStorageSettingsChange()"/>
<menulist id="storage-protocol" class="storage-personal"
style="margin-left: .5em"
preference="pref-storage-protocol"
oncommand="Zotero_Preferences.Sync.updateStorageSettings(null, this.value)">
oncommand="Zotero_Preferences.Sync.onStorageSettingsChange()">
<menupopup>
<menuitem label="Zotero" value="zotero"/>
<menuitem label="WebDAV" value="webdav" disabled="true"/><!-- TEMP -->
<menuitem label="WebDAV" value="webdav"/>
</menupopup>
</menulist>
</hbox>
@ -190,13 +189,8 @@
<label value="://"/>
<textbox id="storage-url" flex="1"
preference="pref-storage-url"
onkeypress="if (Zotero.isMac &amp;&amp; event.keyCode == 13) {
this.blur();
setTimeout(Zotero_Preferences.Sync.verifyStorageServer, 1);
}"
onchange="Zotero_Preferences.Sync.unverifyStorageServer();
this.value = this.value.replace(/(^https?:\/\/|\/zotero\/?$|\/$)/g, '');
Zotero.Prefs.set('sync.storage.url', this.value)"/>
onkeypress="Zotero_Preferences.Sync.onStorageSettingsKeyPress(event)"
onchange="Zotero_Preferences.Sync.onStorageSettingsChange()"/>
<label value="/zotero/"/>
</hbox>
</row>
@ -205,27 +199,16 @@
<hbox>
<textbox id="storage-username"
preference="pref-storage-username"
onkeypress="if (Zotero.isMac &amp;&amp; event.keyCode == 13) {
this.blur();
setTimeout(Zotero_Preferences.Sync.verifyStorageServer, 1); }"
onchange="Zotero_Preferences.Sync.unverifyStorageServer();
Zotero.Prefs.set('sync.storage.username', this.value);
var pass = document.getElementById('storage-password');
if (pass.value) {
Zotero.Sync.Storage.WebDAV.password = pass.value;
}"/>
onkeypress="Zotero_Preferences.Sync.onStorageSettingsKeyPress(event)"
onchange="Zotero_Preferences.Sync.onStorageSettingsChange()"/>
</hbox>
</row>
<row>
<label value="&zotero.preferences.sync.password;"/>
<hbox>
<textbox id="storage-password" flex="0" type="password"
onkeypress="if (Zotero.isMac &amp;&amp; event.keyCode == 13) {
this.blur();
setTimeout(Zotero_Preferences.Sync.verifyStorageServer, 1);
}"
onchange="Zotero_Preferences.Sync.unverifyStorageServer();
Zotero.Sync.Storage.WebDAV.password = this.value;"/>
onkeypress="Zotero_Preferences.Sync.onStorageSettingsKeyPress(event)"
onchange="Zotero_Preferences.Sync.onStorageSettingsChange()"/>
</hbox>
</row>
<row>

View file

@ -2294,9 +2294,7 @@ Zotero.Item.prototype.renameAttachmentFile = Zotero.Promise.coroutine(function*
yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncedHash(this.id, null, false);
yield Zotero.Sync.Storage.Local.setSyncState(
this.id, Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD
);
yield Zotero.Sync.Storage.Local.setSyncState(this.id, "to_upload");
}.bind(this));
return true;
@ -2692,11 +2690,12 @@ Zotero.defineProperty(Zotero.Item.prototype, 'attachmentSyncState', {
}
switch (val) {
case Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD:
case Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD:
case Zotero.Sync.Storage.SYNC_STATE_IN_SYNC:
case Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD:
case Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD:
case Zotero.Sync.Storage.Local.SYNC_STATE_TO_UPLOAD:
case Zotero.Sync.Storage.Local.SYNC_STATE_TO_DOWNLOAD:
case Zotero.Sync.Storage.Local.SYNC_STATE_IN_SYNC:
case Zotero.Sync.Storage.Local.SYNC_STATE_FORCE_UPLOAD:
case Zotero.Sync.Storage.Local.SYNC_STATE_FORCE_DOWNLOAD:
case Zotero.Sync.Storage.Local.SYNC_STATE_IN_CONFLICT:
break;
default:
@ -3670,6 +3669,9 @@ Zotero.Item.prototype._eraseData = Zotero.Promise.coroutine(function* (env) {
Components.utils.reportError(e);
}
}
// Zotero.Sync.EventListeners.ChangeListener needs to know if this was a storage file
env.notifierData[this.id].storageDeleteLog = this.isImportedAttachment();
}
// Regular item
else {
@ -3905,8 +3907,14 @@ Zotero.Item.prototype.toJSON = Zotero.Promise.coroutine(function* (options = {})
}
if (this.isFileAttachment()) {
obj.mtime = (yield this.attachmentModificationTime) || null;
obj.md5 = (yield this.attachmentHash) || null;
if (options.syncedStorageProperties) {
obj.mtime = yield Zotero.Sync.Storage.Local.getSyncedModificationTime(this.id);
obj.md5 = yield Zotero.Sync.Storage.Local.getSyncedHash(this.id);
}
else {
obj.mtime = (yield this.attachmentModificationTime) || null;
obj.md5 = (yield this.attachmentHash) || null;
}
}
}

View file

@ -33,6 +33,7 @@ Zotero.Library = function(params = {}) {
this._hasCollections = null;
this._hasSearches = null;
this._storageDownloadNeeded = false;
Zotero.Utilities.assignProps(
this,
@ -42,8 +43,8 @@ Zotero.Library = function(params = {}) {
'editable',
'filesEditable',
'libraryVersion',
'storageVersion',
'lastSync',
'lastStorageSync'
]
);
@ -64,7 +65,7 @@ Zotero.Library = function(params = {}) {
// DB columns
Zotero.defineProperty(Zotero.Library, '_dbColumns', {
value: Object.freeze([
'type', 'editable', 'filesEditable', 'version', 'lastSync', 'lastStorageSync'
'type', 'editable', 'filesEditable', 'version', 'storageVersion', 'lastSync'
])
});
@ -172,7 +173,7 @@ Zotero.defineProperty(Zotero.Library.prototype, 'hasTrash', {
// Create other accessors
(function() {
let accessors = ['editable', 'filesEditable', 'lastStorageSync'];
let accessors = ['editable', 'filesEditable', 'storageVersion'];
for (let i=0; i<accessors.length; i++) {
let prop = Zotero.Library._colToProp(accessors[i]);
Zotero.defineProperty(Zotero.Library.prototype, accessors[i], {
@ -182,6 +183,11 @@ Zotero.defineProperty(Zotero.Library.prototype, 'hasTrash', {
}
})()
Zotero.defineProperty(Zotero.Library.prototype, 'storageDownloadNeeded', {
get: function () { return this._storageDownloadNeeded; },
set: function (val) { this._storageDownloadNeeded = !!val; },
})
Zotero.Library.prototype._isValidProp = function(prop) {
let prefix = '_library';
if (prop.indexOf(prefix) !== 0 || prop.length == prefix.length) {
@ -235,8 +241,10 @@ Zotero.Library.prototype._set = function(prop, val) {
val = !!val;
break;
case '_libraryVersion':
let newVal = Number.parseInt(val, 10);
if (newVal != val) throw new Error(prop + ' must be an integer');
var newVal = Number.parseInt(val, 10);
if (newVal != val) {
throw new Error(`${prop} must be an integer (${typeof val} '${val}' given)`);
}
val = newVal
// Allow -1 to indicate that a full sync is needed
@ -247,6 +255,17 @@ Zotero.Library.prototype._set = function(prop, val) {
break;
case '_libraryStorageVersion':
var newVal = parseInt(val);
if (newVal != val) {
throw new Error(`${prop} must be an integer (${typeof val} '${val}' given)`);
}
val = newVal;
// Ensure that it is never decreasing
if (val < this._libraryStorageVersion) throw new Error(prop + ' cannot decrease');
break;
case '_libraryLastSync':
if (!val) {
val = false;
@ -257,18 +276,6 @@ Zotero.Library.prototype._set = function(prop, val) {
val = new Date(Math.floor(val.getTime()/1000) * 1000);
}
break;
case '_libraryLastStorageSync':
if (parseInt(val) != val) {
Zotero.debug(val);
throw new Error("timestamp must be an integer");
}
if (val > 9999999999) {
Zotero.debug(val);
throw new Error("timestamp must be in seconds");
}
val = parseInt(val);
break;
}
if (this[prop] == val) return; // Unchanged
@ -293,8 +300,8 @@ Zotero.Library.prototype._loadDataFromRow = function(row) {
this._libraryEditable = !!row._libraryEditable;
this._libraryFilesEditable = !!row._libraryFilesEditable;
this._libraryVersion = row._libraryVersion;
this._libraryStorageVersion = row._libraryStorageVersion;
this._libraryLastSync = row._libraryLastSync !== 0 ? new Date(row._libraryLastSync * 1000) : false;
this._libraryLastStorageSync = row._libraryLastStorageSync || false;
this._hasCollections = !!row.hasCollections;
this._hasSearches = !!row.hasSearches;
@ -394,7 +401,7 @@ Zotero.Library.prototype._initSave = Zotero.Promise.method(function(env) {
Zotero.Libraries._ensureExists(this._libraryID);
if (!Object.keys(this._changed).length) {
Zotero.debug("No data changed in " + this._objectType + " " + this.id + ". Not saving.", 4);
Zotero.debug(`No data changed in ${this._objectType} ${this.id} -- not saving`, 4);
return false;
}
}

View file

@ -666,11 +666,11 @@ Zotero.File = new function(){
var zw = Components.classes["@mozilla.org/zipwriter;1"]
.createInstance(Components.interfaces.nsIZipWriter);
zw.open(this.pathToFile(zipPath), 0x04 | 0x08 | 0x20); // open rw, create, truncate
var entries = yield _zipDirectory(dirPath, dirPath, zw);
var entries = yield _addZipEntries(dirPath, dirPath, zw);
if (entries.length == 0) {
Zotero.debug('No files to add -- removing ZIP file');
zw.close();
zipPath.remove(null);
yield OS.File.remove(zipPath);
return false;
}
@ -716,7 +716,7 @@ Zotero.File = new function(){
});
var _zipDirectory = Zotero.Promise.coroutine(function* (rootPath, path, zipWriter) {
var _addZipEntries = Zotero.Promise.coroutine(function* (rootPath, path, zipWriter) {
var entries = [];
let iterator;
try {
@ -727,7 +727,7 @@ Zotero.File = new function(){
return;
}
if (entry.isDir) {
entries.concat(yield _zipDirectory(rootPath, path, zipWriter));
entries.concat(yield _addZipEntries(rootPath, path, zipWriter));
return;
}
if (entry.name.startsWith('.')) {

View file

@ -635,74 +635,6 @@ Zotero.HTTP = new function() {
this.WebDAV = {};
/**
* Send a WebDAV PROP* request via XMLHTTPRequest
*
* Returns false if browser is offline
*
* @param {String} method PROPFIND or PROPPATCH
* @param {nsIURI} uri
* @param {String} body XML string
* @param {Function} callback
* @param {Object} requestHeaders e.g. { Depth: 0 }
*/
this.WebDAV.doProp = function (method, uri, body, callback, requestHeaders) {
switch (method) {
case 'PROPFIND':
case 'PROPPATCH':
break;
default:
throw ("Invalid method '" + method
+ "' in Zotero.HTTP.doProp");
}
if (requestHeaders && requestHeaders.depth != undefined) {
var depth = requestHeaders.depth;
}
// Don't display password in console
var disp = Zotero.HTTP.getDisplayURI(uri);
var bodyStart = body.substr(0, 1024);
Zotero.debug("HTTP " + method + " "
+ (depth != undefined ? "(depth " + depth + ") " : "")
+ (body.length > 1024 ?
bodyStart + "... (" + body.length + " chars)" : bodyStart)
+ " to " + disp.spec);
if (Zotero.HTTP.browserIsOffline()) {
Zotero.debug("Browser is offline", 2);
return false;
}
var xmlhttp = Components.classes["@mozilla.org/xmlextras/xmlhttprequest;1"]
.createInstance();
// Prevent certificate/authentication dialogs from popping up
xmlhttp.mozBackgroundRequest = true;
xmlhttp.open(method, uri.spec, true);
if (requestHeaders) {
for (var header in requestHeaders) {
xmlhttp.setRequestHeader(header, requestHeaders[header]);
}
}
xmlhttp.setRequestHeader("Content-Type", 'text/xml; charset="utf-8"');
var useMethodjit = Components.utils.methodjit;
/** @ignore */
xmlhttp.onreadystatechange = function() {
// XXX Remove when we drop support for Fx <24
if(useMethodjit !== undefined) Components.utils.methodjit = useMethodjit;
_stateChange(xmlhttp, callback);
};
xmlhttp.send(body);
return xmlhttp;
}
/**
* Send a WebDAV MKCOL request via XMLHTTPRequest
*
@ -736,49 +668,6 @@ Zotero.HTTP = new function() {
}
/**
* Send a WebDAV PUT request via XMLHTTPRequest
*
* @param {nsIURI} url
* @param {String} body String body to PUT
* @param {Function} onDone
* @return {XMLHTTPRequest}
*/
this.WebDAV.doPut = function (uri, body, callback) {
// Don't display password in console
var disp = Zotero.HTTP.getDisplayURI(uri);
var bodyStart = "'" + body.substr(0, 1024) + "'";
Zotero.debug("HTTP PUT "
+ (body.length > 1024 ?
bodyStart + "... (" + body.length + " chars)" : bodyStart)
+ " to " + disp.spec);
if (Zotero.HTTP.browserIsOffline()) {
return false;
}
var xmlhttp = Components.classes["@mozilla.org/xmlextras/xmlhttprequest;1"]
.createInstance();
// Prevent certificate/authentication dialogs from popping up
xmlhttp.mozBackgroundRequest = true;
xmlhttp.open("PUT", uri.spec, true);
// Some servers (e.g., Jungle Disk DAV) return a 200 response code
// with Content-Length: 0, which triggers a "no element found" error
// in Firefox, so we override to text
xmlhttp.overrideMimeType("text/plain");
var useMethodjit = Components.utils.methodjit;
/** @ignore */
xmlhttp.onreadystatechange = function() {
// XXX Remove when we drop support for Fx <24
if(useMethodjit !== undefined) Components.utils.methodjit = useMethodjit;
_stateChange(xmlhttp, callback);
};
xmlhttp.send(body);
return xmlhttp;
}
/**
* Send a WebDAV PUT request via XMLHTTPRequest
*

View file

@ -33,7 +33,7 @@ Zotero.Schema = new function(){
var _dbVersions = [];
var _schemaVersions = [];
var _maxCompatibility = 1;
var _maxCompatibility = 2;
var _repositoryTimer;
var _remoteUpdateInProgress = false, _localUpdateInProgress = false;
@ -2280,6 +2280,18 @@ Zotero.Schema = new function(){
yield Zotero.DB.queryAsync("CREATE TABLE feeds (\n libraryID INTEGER PRIMARY KEY,\n name TEXT NOT NULL,\n url TEXT NOT NULL UNIQUE,\n lastUpdate TIMESTAMP,\n lastCheck TIMESTAMP,\n lastCheckError TEXT,\n cleanupAfter INT,\n refreshInterval INT,\n FOREIGN KEY (libraryID) REFERENCES libraries(libraryID) ON DELETE CASCADE\n)");
yield Zotero.DB.queryAsync("CREATE TABLE feedItems (\n itemID INTEGER PRIMARY KEY,\n guid TEXT NOT NULL UNIQUE,\n readTime TIMESTAMP,\n FOREIGN KEY (itemID) REFERENCES items(itemID) ON DELETE CASCADE\n)");
}
if (i == 81) {
yield _updateDBVersion('compatibility', 2);
yield Zotero.DB.queryAsync("ALTER TABLE libraries RENAME TO librariesOld");
yield Zotero.DB.queryAsync("CREATE TABLE libraries (\n libraryID INTEGER PRIMARY KEY,\n type TEXT NOT NULL,\n editable INT NOT NULL,\n filesEditable INT NOT NULL,\n version INT NOT NULL DEFAULT 0,\n storageVersion INT NOT NULL DEFAULT 0,\n lastSync INT NOT NULL DEFAULT 0\n)");
yield Zotero.DB.queryAsync("INSERT INTO libraries SELECT libraryID, type, editable, filesEditable, version, 0, lastSync FROM librariesOld");
yield Zotero.DB.queryAsync("DROP TABLE librariesOld");
yield Zotero.DB.queryAsync("PRAGMA foreign_keys = ON");
yield Zotero.DB.queryAsync("DELETE FROM version WHERE schema LIKE ?", "storage_%");
}
}
yield _updateDBVersion('userdata', toVersion);

View file

@ -25,34 +25,6 @@
Zotero.Sync.Storage = new function () {
//
// Constants
//
this.SYNC_STATE_TO_UPLOAD = 0;
this.SYNC_STATE_TO_DOWNLOAD = 1;
this.SYNC_STATE_IN_SYNC = 2;
this.SYNC_STATE_FORCE_UPLOAD = 3;
this.SYNC_STATE_FORCE_DOWNLOAD = 4;
this.SYNC_STATE_IN_CONFLICT = 5;
this.SUCCESS = 1;
this.ERROR_NO_URL = -1;
this.ERROR_NO_USERNAME = -2;
this.ERROR_NO_PASSWORD = -3;
this.ERROR_OFFLINE = -4;
this.ERROR_UNREACHABLE = -5;
this.ERROR_SERVER_ERROR = -6;
this.ERROR_NOT_DAV = -7;
this.ERROR_BAD_REQUEST = -8;
this.ERROR_AUTH_FAILED = -9;
this.ERROR_FORBIDDEN = -10;
this.ERROR_PARENT_DIR_NOT_FOUND = -11;
this.ERROR_ZOTERO_DIR_NOT_FOUND = -12;
this.ERROR_ZOTERO_DIR_NOT_WRITABLE = -13;
this.ERROR_NOT_ALLOWED = -14;
this.ERROR_UNKNOWN = -15;
this.ERROR_FILE_MISSING_AFTER_UPLOAD = -16;
this.ERROR_NONEXISTENT_FILE_NOT_MISSING = -17;
// TEMP
this.__defineGetter__("defaultError", function () Zotero.getString('sync.storage.error.default', Zotero.appName));
@ -111,40 +83,6 @@ Zotero.Sync.Storage = new function () {
}
this.checkServerPromise = function (mode) {
return mode.checkServer()
.spread(function (uri, status) {
var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"]
.getService(Components.interfaces.nsIWindowMediator);
var lastWin = wm.getMostRecentWindow("navigator:browser");
var success = mode.checkServerCallback(uri, status, lastWin, true);
if (!success) {
Zotero.debug(mode.name + " verification failed");
var e = new Zotero.Error(
Zotero.getString('sync.storage.error.verificationFailed', mode.name),
0,
{
dialogButtonText: Zotero.getString('sync.openSyncPreferences'),
dialogButtonCallback: function () {
var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"]
.getService(Components.interfaces.nsIWindowMediator);
var lastWin = wm.getMostRecentWindow("navigator:browser");
lastWin.ZoteroPane.openPreferences('zotero-prefpane-sync');
}
}
);
throw e;
}
})
.then(function () {
Zotero.debug(mode.name + " file sync is successfully set up");
Zotero.Prefs.set("sync.storage.verified", true);
});
}
this.getItemDownloadImageNumber = function (item) {
var numImages = 64;
@ -191,57 +129,6 @@ Zotero.Sync.Storage = new function () {
}
this.resetAllSyncStates = function (syncState, includeUserFiles, includeGroupFiles) {
if (!includeUserFiles && !includeGroupFiles) {
includeUserFiles = true;
includeGroupFiles = true;
}
if (!syncState) {
syncState = this.SYNC_STATE_TO_UPLOAD;
}
switch (syncState) {
case this.SYNC_STATE_TO_UPLOAD:
case this.SYNC_STATE_TO_DOWNLOAD:
case this.SYNC_STATE_IN_SYNC:
break;
default:
throw ("Invalid sync state '" + syncState + "' in "
+ "Zotero.Sync.Storage.resetAllSyncStates()");
}
//var sql = "UPDATE itemAttachments SET syncState=?, storageModTime=NULL, storageHash=NULL";
var sql = "UPDATE itemAttachments SET syncState=?";
var params = [syncState];
if (includeUserFiles && !includeGroupFiles) {
sql += " WHERE itemID IN (SELECT itemID FROM items WHERE libraryID = ?)";
params.push(Zotero.Libraries.userLibraryID);
}
else if (!includeUserFiles && includeGroupFiles) {
sql += " WHERE itemID IN (SELECT itemID FROM items WHERE libraryID != ?)";
params.push(Zotero.Libraries.userLibraryID);
}
Zotero.DB.query(sql, [syncState]);
var sql = "DELETE FROM version WHERE schema LIKE 'storage_%'";
Zotero.DB.query(sql);
}
function error(e) {
if (_syncInProgress) {
Zotero.Sync.Storage.QueueManager.cancel(true);

View file

@ -32,31 +32,29 @@ if (!Zotero.Sync.Storage) {
* An Engine manages file sync processes for a given library
*
* @param {Object} options
* @param {Zotero.Sync.APIClient} options.apiClient
* @param {Integer} options.libraryID
* @param {Object} options.controller - Storage controller instance (ZFS_Controller/WebDAV_Controller)
* @param {Function} [onError] - Function to run on error
* @param {Boolean} [stopOnError]
*/
Zotero.Sync.Storage.Engine = function (options) {
if (options.apiClient == undefined) {
throw new Error("options.apiClient not set");
}
if (options.libraryID == undefined) {
throw new Error("options.libraryID not set");
}
if (options.controller == undefined) {
throw new Error("options.controller not set");
}
this.apiClient = options.apiClient;
this.background = options.background;
this.firstInSession = options.firstInSession;
this.lastFullFileCheck = options.lastFullFileCheck;
this.libraryID = options.libraryID;
this.library = Zotero.Libraries.get(options.libraryID);
this.controller = options.controller;
this.local = Zotero.Sync.Storage.Local;
this.utils = Zotero.Sync.Storage.Utilities;
this.mode = this.local.getModeForLibrary(this.libraryID);
var modeClass = this.utils.getClassForMode(this.mode);
this.controller = new modeClass(options);
this.setStatus = options.setStatus || function () {};
this.onError = options.onError || function (e) {};
this.stopOnError = options.stopOnError || false;
@ -87,12 +85,37 @@ Zotero.Sync.Storage.Engine.prototype.start = Zotero.Promise.coroutine(function*
Zotero.debug("Starting file sync for " + this.library.name);
if (!this.controller.verified) {
Zotero.debug(`${this.mode} file sync is not active`);
Zotero.debug(`${this.controller.name} file sync is not active -- verifying`);
throw new Error("Storage mode verification not implemented");
// TODO: Check server
try {
yield this.controller.checkServer();
}
catch (e) {
let wm = Components.classes["@mozilla.org/appshell/window-mediator;1"]
.getService(Components.interfaces.nsIWindowMediator);
let lastWin = wm.getMostRecentWindow("navigator:browser");
let success = yield this.controller.handleVerificationError(e, lastWin, true);
if (!success) {
Zotero.debug(this.controller.name + " verification failed", 2);
throw new Zotero.Error(
Zotero.getString('sync.storage.error.verificationFailed', this.controller.name),
0,
{
dialogButtonText: Zotero.getString('sync.openSyncPreferences'),
dialogButtonCallback: function () {
let wm = Components.classes["@mozilla.org/appshell/window-mediator;1"]
.getService(Components.interfaces.nsIWindowMediator);
let lastWin = wm.getMostRecentWindow("navigator:browser");
lastWin.ZoteroPane.openPreferences('zotero-prefpane-sync');
}
}
);
}
}
}
if (this.controller.cacheCredentials) {
yield this.controller.cacheCredentials();
}
@ -101,7 +124,10 @@ Zotero.Sync.Storage.Engine.prototype.start = Zotero.Promise.coroutine(function*
var lastSyncTime = null;
var downloadAll = this.local.downloadOnSync(libraryID);
if (downloadAll) {
lastSyncTime = yield this.controller.getLastSyncTime(libraryID);
if (!this.library.storageDownloadNeeded) {
this.library.storageVersion = this.library.libraryVersion;
yield this.library.saveTx();
}
}
// Check for updated files to upload
@ -131,18 +157,11 @@ Zotero.Sync.Storage.Engine.prototype.start = Zotero.Promise.coroutine(function*
var downloadForced = yield this.local.checkForForcedDownloads(libraryID);
// If we don't have any forced downloads, we can skip downloads if the last sync time hasn't
// changed or doesn't exist on the server (meaning there are no files)
// If we don't have any forced downloads, we can skip downloads if no storage metadata has
// changed (meaning nothing else has uploaded files since the last successful file sync)
if (downloadAll && !downloadForced) {
if (lastSyncTime) {
if (this.library.lastStorageSync == lastSyncTime) {
Zotero.debug("Last " + this.mode.toUpperCase() + " sync id hasn't changed for "
+ this.library.name + " -- skipping file downloads");
downloadAll = false;
}
}
else {
Zotero.debug(`No last ${this.mode} sync time for ${this.library.name}`
if (this.library.storageVersion == this.library.libraryVersion) {
Zotero.debug("No remote storage changes for " + this.library.name
+ " -- skipping file downloads");
downloadAll = false;
}
@ -189,6 +208,7 @@ Zotero.Sync.Storage.Engine.prototype.start = Zotero.Promise.coroutine(function*
}
// Process the results
var downloadSuccessful = false;
var changes = new Zotero.Sync.Storage.Result;
for (let type of ['download', 'upload']) {
let results = yield promises[type];
@ -202,32 +222,33 @@ Zotero.Sync.Storage.Engine.prototype.start = Zotero.Promise.coroutine(function*
}
}
}
Zotero.debug(`File ${type} sync finished for ${this.library.name}`);
changes.updateFromResults(results.filter(p => p.isFulfilled()).map(p => p.value()));
}
// If files were uploaded, update the remote last-sync time
if (changes.remoteChanges) {
lastSyncTime = yield this.controller.setLastSyncTime(libraryID);
if (!lastSyncTime) {
throw new Error("Last sync time not set after sync");
if (type == 'download' && results.every(p => !p.isRejected())) {
downloadSuccessful = true;
}
}
// If there's a remote last-sync time from either the check before downloads or when it
// was changed after uploads, store that locally so we know we can skip download checks
// next time
if (lastSyncTime) {
this.library.lastStorageSync = lastSyncTime;
if (downloadSuccessful) {
this.library.storageDownloadNeeded = false;
this.library.storageVersion = this.library.libraryVersion;
yield this.library.saveTx();
}
// If WebDAV sync, purge deleted and orphaned files
if (this.mode == 'webdav') {
// For ZFS, this purges all files on server based on flag set when switching from ZFS
// to WebDAV in prefs. For WebDAV, this purges locally deleted files on server.
try {
yield this.controller.purgeDeletedStorageFiles(libraryID);
}
catch (e) {
Zotero.logError(e);
}
// If WebDAV sync, purge orphaned files
if (this.controller.mode == 'webdav') {
try {
yield this.controller.purgeDeletedStorageFiles(libraryID);
yield this.controller.purgeOrphanedStorageFiles(libraryID);
}
catch (e) {
@ -253,16 +274,16 @@ Zotero.Sync.Storage.Engine.prototype.stop = function () {
Zotero.Sync.Storage.Engine.prototype.queueItem = Zotero.Promise.coroutine(function* (item) {
switch (yield this.local.getSyncState(item.id)) {
case Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD:
case Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD:
case Zotero.Sync.Storage.Local.SYNC_STATE_TO_DOWNLOAD:
case Zotero.Sync.Storage.Local.SYNC_STATE_FORCE_DOWNLOAD:
var type = 'download';
var onStart = Zotero.Promise.method(function (request) {
return this.controller.downloadFile(request);
}.bind(this));
break;
case Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD:
case Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD:
case Zotero.Sync.Storage.Local.SYNC_STATE_TO_UPLOAD:
case Zotero.Sync.Storage.Local.SYNC_STATE_FORCE_UPLOAD:
var type = 'upload';
var onStart = Zotero.Promise.method(function (request) {
return this.controller.uploadFile(request);

View file

@ -1,4 +1,14 @@
Zotero.Sync.Storage.Local = {
//
// Constants
//
SYNC_STATE_TO_UPLOAD: 0,
SYNC_STATE_TO_DOWNLOAD: 1,
SYNC_STATE_IN_SYNC: 2,
SYNC_STATE_FORCE_UPLOAD: 3,
SYNC_STATE_FORCE_DOWNLOAD: 4,
SYNC_STATE_IN_CONFLICT: 5,
lastFullFileCheck: {},
uploadCheckFiles: [],
@ -101,7 +111,7 @@ Zotero.Sync.Storage.Local = {
libraryID,
Zotero.Attachments.LINK_MODE_IMPORTED_FILE,
Zotero.Attachments.LINK_MODE_IMPORTED_URL,
Zotero.Sync.Storage.SYNC_STATE_IN_SYNC,
this.SYNC_STATE_IN_SYNC,
minTime
];
var itemIDs = yield Zotero.DB.columnQueryAsync(sql, params);
@ -174,8 +184,8 @@ Zotero.Sync.Storage.Local = {
let params = [
Zotero.Attachments.LINK_MODE_IMPORTED_FILE,
Zotero.Attachments.LINK_MODE_IMPORTED_URL,
Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD,
Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
this.SYNC_STATE_TO_UPLOAD,
this.SYNC_STATE_IN_SYNC
];
if (libraryID !== false) {
sql += " AND libraryID=?";
@ -251,7 +261,7 @@ Zotero.Sync.Storage.Local = {
var path = item.getFilePath();
if (!path) {
Zotero.debug("Marking pathless attachment " + lk + " as in-sync");
return Zotero.Sync.Storage.SYNC_STATE_IN_SYNC;
return this.SYNC_STATE_IN_SYNC;
}
var fileName = OS.Path.basename(path);
var file;
@ -272,7 +282,7 @@ Zotero.Sync.Storage.Local = {
// If file is already marked for upload, skip check. Even if the file was changed
// both locally and remotely, conflicts are checked at upload time, so we don't need
// to worry about it here.
if ((yield this.getSyncState(item.id)) == Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD) {
if ((yield this.getSyncState(item.id)) == this.SYNC_STATE_TO_UPLOAD) {
Zotero.debug("File is already marked for upload");
return false;
}
@ -296,7 +306,7 @@ Zotero.Sync.Storage.Local = {
Zotero.debug(`Marking attachment ${lk} for download (stored mtime: ${mtime})`);
// DEBUG: Always set here, or allow further steps?
return Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD;
return this.SYNC_STATE_FORCE_DOWNLOAD;
}
var same = !this.checkFileModTime(item, fmtime, mtime);
@ -330,7 +340,7 @@ Zotero.Sync.Storage.Local = {
// Mark file for upload
Zotero.debug("Marking attachment " + lk + " as changed "
+ "(" + mtime + " != " + fmtime + ")");
return Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD;
return this.SYNC_STATE_TO_UPLOAD;
}
catch (e) {
if (e instanceof OS.File.Error &&
@ -342,7 +352,7 @@ Zotero.Sync.Storage.Local = {
// Handle long filenames on OS X/Linux
|| (e.unixErrno && e.unixErrno == 63))) {
Zotero.debug("Marking attachment " + lk + " as missing");
return Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD;
return this.SYNC_STATE_TO_DOWNLOAD;
}
if (e instanceof OS.File.Error) {
@ -372,7 +382,7 @@ Zotero.Sync.Storage.Local = {
* @return {Boolean} - True if file modification time differs from remote mod time,
* false otherwise
*/
checkFileModTime(item, fmtime, mtime) {
checkFileModTime: function (item, fmtime, mtime) {
var libraryKey = item.libraryKey;
if (fmtime == mtime) {
@ -406,7 +416,7 @@ Zotero.Sync.Storage.Local = {
var sql = "SELECT COUNT(*) FROM items JOIN itemAttachments USING (itemID) "
+ "WHERE libraryID=? AND syncState=?";
return !!(yield Zotero.DB.valueQueryAsync(
sql, [libraryID, Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD]
sql, [libraryID, this.SYNC_STATE_FORCE_DOWNLOAD]
));
}),
@ -420,10 +430,10 @@ Zotero.Sync.Storage.Local = {
getFilesToDownload: function (libraryID, forcedOnly) {
var sql = "SELECT itemID FROM itemAttachments JOIN items USING (itemID) "
+ "WHERE libraryID=? AND syncState IN (?";
var params = [libraryID, Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD];
var params = [libraryID, this.SYNC_STATE_FORCE_DOWNLOAD];
if (!forcedOnly) {
sql += ",?";
params.push(Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD);
params.push(this.SYNC_STATE_TO_DOWNLOAD);
}
sql += ") "
// Skip attachments with empty path, which can't be saved, and files with .zotero*
@ -444,8 +454,8 @@ Zotero.Sync.Storage.Local = {
+ "WHERE libraryID=? AND syncState IN (?,?) AND linkMode IN (?,?)";
var params = [
libraryID,
Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD,
Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD,
this.SYNC_STATE_TO_UPLOAD,
this.SYNC_STATE_FORCE_UPLOAD,
Zotero.Attachments.LINK_MODE_IMPORTED_FILE,
Zotero.Attachments.LINK_MODE_IMPORTED_URL
];
@ -473,17 +483,22 @@ Zotero.Sync.Storage.Local = {
/**
* @param {Integer} itemID
* @param {Integer} syncState Constant from Zotero.Sync.Storage
* @param {Integer} itemID
* @param {Integer|String} syncState - Zotero.Sync.Storage.Local.SYNC_STATE_* or last part
* as string (e.g., "TO_UPLOAD")
*/
setSyncState: Zotero.Promise.method(function (itemID, syncState) {
if (typeof syncState == 'string') {
syncState = this["SYNC_STATE_" + syncState.toUpperCase()];
}
switch (syncState) {
case Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD:
case Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD:
case Zotero.Sync.Storage.SYNC_STATE_IN_SYNC:
case Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD:
case Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD:
case Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT:
case this.SYNC_STATE_TO_UPLOAD:
case this.SYNC_STATE_TO_DOWNLOAD:
case this.SYNC_STATE_IN_SYNC:
case this.SYNC_STATE_FORCE_UPLOAD:
case this.SYNC_STATE_FORCE_DOWNLOAD:
case this.SYNC_STATE_IN_CONFLICT:
break;
default:
@ -495,6 +510,14 @@ Zotero.Sync.Storage.Local = {
}),
resetModeSyncStates: Zotero.Promise.coroutine(function* (mode) {
var sql = "UPDATE itemAttachments SET syncState=? "
+ "WHERE itemID IN (SELECT itemID FROM items WHERE libraryID=?)";
var params = [this.SYNC_STATE_TO_UPLOAD, Zotero.Libraries.userLibraryID];
yield Zotero.DB.queryAsync(sql, params);
}),
/**
* @param {Integer} itemID
* @return {Integer|NULL} Mod time as timestamp in ms,
@ -513,7 +536,7 @@ Zotero.Sync.Storage.Local = {
/**
* @param {Integer} itemID
* @param {Integer} mtime - File modification time as timestamp in ms
* @param {Boolean} [updateItem=FALSE] - Update clientDateModified field of attachment item
* @param {Boolean} [updateItem=FALSE] - Mark attachment item as unsynced
*/
setSyncedModificationTime: Zotero.Promise.coroutine(function* (itemID, mtime, updateItem) {
if (mtime < 0) {
@ -528,9 +551,8 @@ Zotero.Sync.Storage.Local = {
yield Zotero.DB.queryAsync(sql, [mtime, itemID]);
if (updateItem) {
// Update item date modified so the new mod time will be synced
let sql = "UPDATE items SET clientDateModified=? WHERE itemID=?";
yield Zotero.DB.queryAsync(sql, [Zotero.DB.transactionDateTime, itemID]);
let item = yield Zotero.Items.getAsync(itemID);
yield item.updateSynced(false);
}
}),
@ -553,8 +575,7 @@ Zotero.Sync.Storage.Local = {
/**
* @param {Integer} itemID
* @param {String} hash File hash
* @param {Boolean} [updateItem=FALSE] Update dateModified field of
* attachment item
* @param {Boolean} [updateItem=FALSE] - Mark attachment item as unsynced
*/
setSyncedHash: Zotero.Promise.coroutine(function* (itemID, hash, updateItem) {
if (hash !== null && hash.length != 32) {
@ -567,9 +588,8 @@ Zotero.Sync.Storage.Local = {
yield Zotero.DB.queryAsync(sql, [hash, itemID]);
if (updateItem) {
// Update item date modified so the new mod time will be synced
var sql = "UPDATE items SET clientDateModified=? WHERE itemID=?";
yield Zotero.DB.queryAsync(sql, [Zotero.DB.transactionDateTime, itemID]);
let item = yield Zotero.Items.getAsync(itemID);
yield item.updateSynced(false);
}
}),
@ -658,7 +678,7 @@ Zotero.Sync.Storage.Local = {
yield Zotero.DB.executeTransaction(function* () {
yield this.setSyncedHash(item.id, md5);
yield this.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
yield this.setSyncState(item.id, this.SYNC_STATE_IN_SYNC);
yield this.setSyncedModificationTime(item.id, mtime);
}.bind(this));
@ -998,7 +1018,7 @@ Zotero.Sync.Storage.Local = {
sql,
[
{ int: libraryID },
Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
this.SYNC_STATE_IN_CONFLICT
]
);
var keyVersionPairs = rows.map(function (row) {
@ -1073,16 +1093,16 @@ Zotero.Sync.Storage.Local = {
let mtime = io.dataOut[i].dateModified;
// Local
if (mtime == conflict.left.dateModified) {
syncState = Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD;
syncState = this.SYNC_STATE_FORCE_UPLOAD;
}
// Remote
else {
syncState = Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD;
syncState = this.SYNC_STATE_FORCE_DOWNLOAD;
}
let itemID = Zotero.Items.getIDFromLibraryAndKey(libraryID, conflict.left.key);
yield Zotero.Sync.Storage.Local.setSyncState(itemID, syncState);
}
});
}.bind(this));
return true;
})
}

View file

@ -2,10 +2,10 @@ Zotero.Sync.Storage.Utilities = {
getClassForMode: function (mode) {
switch (mode) {
case 'zfs':
return Zotero.Sync.Storage.ZFS_Module;
return Zotero.Sync.Storage.Mode.ZFS;
case 'webdav':
return Zotero.Sync.Storage.WebDAV_Module;
return Zotero.Sync.Storage.Mode.WebDAV;
default:
throw new Error("Invalid storage mode '" + mode + "'");

File diff suppressed because it is too large Load diff

View file

@ -23,8 +23,11 @@
***** END LICENSE BLOCK *****
*/
if (!Zotero.Sync.Storage.Mode) {
Zotero.Sync.Storage.Mode = {};
}
Zotero.Sync.Storage.ZFS_Module = function (options) {
Zotero.Sync.Storage.Mode.ZFS = function (options) {
this.options = options;
this.apiClient = options.apiClient;
@ -33,76 +36,17 @@ Zotero.Sync.Storage.ZFS_Module = function (options) {
this._maxS3Backoff = 60;
this._maxS3ConsecutiveFailures = 5;
};
Zotero.Sync.Storage.ZFS_Module.prototype = {
Zotero.Sync.Storage.Mode.ZFS.prototype = {
mode: "zfs",
name: "ZFS",
verified: true,
/**
* @return {Promise} A promise for the last sync time
*/
getLastSyncTime: Zotero.Promise.coroutine(function* (libraryID) {
var params = this._getRequestParams(libraryID, "laststoragesync");
var uri = this.apiClient.buildRequestURI(params);
try {
let req = yield this.apiClient.makeRequest(
"GET", uri, { successCodes: [200, 404], debug: true }
);
// Not yet synced
if (req.status == 404) {
Zotero.debug("No last sync time for library " + libraryID);
return null;
}
let ts = req.responseText;
let date = new Date(ts * 1000);
Zotero.debug("Last successful ZFS sync for library " + libraryID + " was " + date);
return ts;
}
catch (e) {
Zotero.logError(e);
throw e;
}
}),
setLastSyncTime: Zotero.Promise.coroutine(function* (libraryID) {
var params = this._getRequestParams(libraryID, "laststoragesync");
var uri = this.apiClient.buildRequestURI(params);
try {
var req = yield this.apiClient.makeRequest(
"POST", uri, { successCodes: [200, 404], debug: true }
);
}
catch (e) {
var msg = "Unexpected status code " + e.xmlhttp.status + " setting last file sync time";
Zotero.logError(e);
throw new Error(Zotero.Sync.Storage.defaultError);
}
// Not yet synced
//
// TODO: Don't call this at all if no files uploaded
if (req.status == 404) {
return;
}
var time = req.responseText;
if (parseInt(time) != time) {
Zotero.logError(`Unexpected response ${time} setting last file sync time`);
throw new Error(Zotero.Sync.Storage.defaultError);
}
return parseInt(time);
}),
/**
* Begin download process for individual file
*
* @param {Zotero.Sync.Storage.Request} request
* @return {Promise<Boolean>} - True if file download, false if not
* @return {Promise<Zotero.Sync.Storage.Result>}
*/
downloadFile: Zotero.Promise.coroutine(function* (request) {
var item = Zotero.Sync.Storage.Utilities.getItemFromRequest(request);
@ -140,7 +84,7 @@ Zotero.Sync.Storage.ZFS_Module.prototype = {
Zotero.debug("Download request " + request.name
+ " stopped before download started -- closing channel");
req.cancel(Components.results.NS_BINDING_ABORTED);
deferred.resolve(false);
deferred.resolve(new Zotero.Sync.Storage.Result);
}
},
onChannelRedirect: Zotero.Promise.coroutine(function* (oldChannel, newChannel, flags) {
@ -193,9 +137,7 @@ Zotero.Sync.Storage.ZFS_Module.prototype = {
yield Zotero.Sync.Storage.Local.setSyncedModificationTime(
item.id, requestData.mtime
);
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
);
yield Zotero.Sync.Storage.Local.setSyncState(item.id, "in_sync");
});
return false;
}),
@ -259,7 +201,7 @@ Zotero.Sync.Storage.ZFS_Module.prototype = {
if (request.isFinished()) {
Zotero.debug("Download request " + request.name
+ " is no longer running after file download", 2);
deferred.resolve(false);
deferred.resolve(new Zotero.Sync.Storage.Result);
return;
}
@ -271,14 +213,13 @@ Zotero.Sync.Storage.ZFS_Module.prototype = {
);
}
catch (e) {
Zotero.debug("REJECTING");
deferred.reject(e);
}
}.bind(this),
onCancel: function (req, status) {
Zotero.debug("Request cancelled");
if (deferred.promise.isPending()) {
deferred.resolve(false);
deferred.resolve(new Zotero.Sync.Storage.Result);
}
}
}
@ -290,8 +231,7 @@ Zotero.Sync.Storage.ZFS_Module.prototype = {
Zotero.debug('Saving ' + uri);
const nsIWBP = Components.interfaces.nsIWebBrowserPersist;
var wbp = Components
.classes["@mozilla.org/embedding/browser/nsWebBrowserPersist;1"]
var wbp = Components.classes["@mozilla.org/embedding/browser/nsWebBrowserPersist;1"]
.createInstance(nsIWBP);
wbp.persistFlags = nsIWBP.PERSIST_FLAGS_BYPASS_CACHE;
wbp.progressListener = listener;
@ -308,7 +248,6 @@ Zotero.Sync.Storage.ZFS_Module.prototype = {
if (!created) {
return new Zotero.Sync.Storage.Result;
}
return this._processUploadFile(request);
}
return this._processUploadFile(request);
}),
@ -327,23 +266,7 @@ Zotero.Sync.Storage.ZFS_Module.prototype = {
Zotero.debug("Unlinking synced files on ZFS");
var uri = this.userURI;
uri.spec += "removestoragefiles?";
// Unused
for each(var value in values) {
switch (value) {
case 'user':
uri.spec += "user=1&";
break;
case 'group':
uri.spec += "group=1&";
break;
default:
throw new Error("Invalid zfsPurge value '" + value + "'");
}
}
uri.spec = uri.spec.substr(0, uri.spec.length - 1);
uri.spec += "removestoragefiles";
yield Zotero.HTTP.request("POST", uri, "");
@ -438,10 +361,9 @@ Zotero.Sync.Storage.ZFS_Module.prototype = {
}
// Build POST body
var mtime = yield item.attachmentModificationTime;
var params = {
mtime: yield item.attachmentModificationTime,
md5: yield item.attachmentHash,
mtime,
filename,
filesize: (yield OS.File.stat(uploadPath)).size
};
@ -521,7 +443,7 @@ Zotero.Sync.Storage.ZFS_Module.prototype = {
// TEMP
//
// Passed through to _updateItemFileInfo()
json.mtime = mtime;
json.mtime = params.mtime;
json.md5 = params.md5;
if (storedHash) {
json.storedHash = storedHash;
@ -619,16 +541,12 @@ Zotero.Sync.Storage.ZFS_Module.prototype = {
item.id, fileModTime
);
yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, fileHash);
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
);
yield Zotero.Sync.Storage.Local.setSyncState(item.id, "in_sync");
});
return new Zotero.Sync.Storage.Result;
}
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
);
yield Zotero.Sync.Storage.Local.setSyncState(item.id, "in_conflict");
return new Zotero.Sync.Storage.Result({
fileSyncRequired: true
});
@ -847,9 +765,7 @@ Zotero.Sync.Storage.ZFS_Module.prototype = {
_updateItemFileInfo: Zotero.Promise.coroutine(function* (item, params) {
// Mark as in-sync
yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
);
yield Zotero.Sync.Storage.Local.setSyncState(item.id, "in_sync");
// Store file mod time and hash
yield Zotero.Sync.Storage.Local.setSyncedModificationTime(item.id, params.mtime);
@ -1016,7 +932,7 @@ Zotero.Sync.Storage.ZFS_Module.prototype = {
// Check for conflict
if ((yield Zotero.Sync.Storage.Local.getSyncState(item.id))
!= Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD) {
!= Zotero.Sync.Storage.Local.SYNC_STATE_FORCE_UPLOAD) {
if (info) {
// Local file time
var fmtime = yield item.attachmentModificationTime;
@ -1036,7 +952,7 @@ Zotero.Sync.Storage.ZFS_Module.prototype = {
yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.setSyncedModificationTime(item.id, fmtime);
yield Zotero.Sync.Storage.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
item.id, Zotero.Sync.Storage.Local.SYNC_STATE_IN_SYNC
);
});
return {

View file

@ -1657,7 +1657,7 @@ Zotero.Sync.Server.Data = new function() {
// Mark new attachments for download
if (isNewObject) {
obj.attachmentSyncState =
Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD;
Zotero.Sync.Storage.Local.SYNC_STATE_TO_DOWNLOAD;
}
// Set existing attachments for mtime update check
else {

View file

@ -709,9 +709,12 @@ Zotero.Sync.Data.Engine.prototype._uploadObjects = Zotero.Promise.coroutine(func
toCache.push(current);
}
else {
let j = yield obj.toJSON();
j.version = json.libraryVersion;
toCache.push(j);
// This won't reflect the actual version of the item on the server, but
// it will guarantee that the item won't be redownloaded unnecessarily
// in the case of a full sync, because the version will be higher than
// whatever version is on the server.
batch[index].version = json.libraryVersion
toCache.push(batch[index]);
}
numSuccessful++;
@ -891,6 +894,7 @@ Zotero.Sync.Data.Engine.prototype._getJSONForObject = function (objectType, id)
includeKey: true,
includeVersion: true, // DEBUG: remove?
includeDate: true,
syncedStorageProperties: true,
patchBase: cacheObj ? cacheObj.data : false
});
});

View file

@ -43,58 +43,55 @@ Zotero.Sync.EventListeners.ChangeListener = new function () {
var storageForLibrary = {};
return Zotero.DB.executeTransaction(function* () {
for (let i = 0; i < ids.length; i++) {
let id = ids[i];
if (extraData[id] && extraData[id].skipDeleteLog) {
continue;
}
var libraryID, key;
if (type == 'setting') {
[libraryID, key] = ids[i].split("/");
}
else {
let d = extraData[ids[i]];
libraryID = d.libraryID;
key = d.key;
}
if (!key) {
throw new Error("Key not provided in notifier object");
}
yield Zotero.DB.queryAsync(
syncSQL,
[
syncObjectTypeID,
libraryID,
key
]
);
if (type == 'item') {
if (storageForLibrary[libraryID] === undefined) {
storageForLibrary[libraryID] =
Zotero.Sync.Storage.Local.getModeForLibrary(libraryID) == 'webdav';
}
if (storageForLibrary[libraryID] && oldItem.itemType == 'attachment' &&
[
Zotero.Attachments.LINK_MODE_IMPORTED_FILE,
Zotero.Attachments.LINK_MODE_IMPORTED_URL
].indexOf(oldItem.linkMode) != -1) {
return Zotero.Utilities.Internal.forEachChunkAsync(
ids,
100,
function (chunk) {
return Zotero.DB.executeTransaction(function* () {
for (let id of chunk) {
if (extraData[id] && extraData[id].skipDeleteLog) {
continue;
}
if (type == 'setting') {
var [libraryID, key] = id.split("/");
}
else {
var { libraryID, key } = extraData[id];
}
if (!key) {
throw new Error("Key not provided in notifier object");
}
yield Zotero.DB.queryAsync(
storageSQL,
syncSQL,
[
syncObjectTypeID,
libraryID,
key
]
);
if (type == 'item') {
if (storageForLibrary[libraryID] === undefined) {
storageForLibrary[libraryID] =
Zotero.Sync.Storage.Local.getModeForLibrary(libraryID) == 'webdav';
}
if (storageForLibrary[libraryID] && extraData[id].storageDeleteLog) {
yield Zotero.DB.queryAsync(
storageSQL,
[
libraryID,
key
]
);
}
}
}
}
});
}
});
);
});
}

View file

@ -478,6 +478,7 @@ Zotero.Sync.Data.Local = {
let isNewObject = false;
let skipCache = false;
let storageDetailsChanged = false;
let obj = yield objectsClass.getByLibraryAndKeyAsync(
libraryID, objectKey, { noCache: true }
);
@ -495,12 +496,22 @@ Zotero.Sync.Data.Local = {
let jsonDataLocal = yield obj.toJSON();
// For items, check if mtime or file hash changed in metadata,
// which would indicate that a remote storage sync took place and
// a download is needed
if (objectType == 'item' && obj.isImportedAttachment()) {
if (jsonDataLocal.mtime != jsonData.mtime
|| jsonDataLocal.md5 != jsonData.md5) {
storageDetailsChanged = true;
}
}
let result = this._reconcileChanges(
objectType,
cachedJSON.data,
jsonDataLocal,
jsonData,
['dateAdded', 'dateModified']
['mtime', 'md5', 'dateAdded', 'dateModified']
);
// If no changes, update local version number and mark as synced
@ -510,6 +521,15 @@ Zotero.Sync.Data.Local = {
obj.version = json.version;
obj.synced = true;
yield obj.save();
if (objectType == 'item') {
yield this._onItemProcessed(
obj,
jsonData,
isNewObject,
storageDetailsChanged
);
}
continue;
}
@ -599,15 +619,12 @@ Zotero.Sync.Data.Local = {
obj, jsonData, options, { skipCache }
);
if (saved) {
// Delete older versions of the item in the cache
yield this.deleteCacheObjectVersions(
objectType, libraryID, jsonData.key, null, jsonData.version - 1
);
// Mark updated attachments for download
if (objectType == 'item' && obj.isImportedAttachment()) {
yield this._checkAttachmentForDownload(
obj, jsonData.mtime, isNewObject
if (objectType == 'item') {
yield this._onItemProcessed(
obj,
jsonData,
isNewObject,
storageDetailsChanged
);
}
}
@ -694,6 +711,27 @@ Zotero.Sync.Data.Local = {
}),
_onItemProcessed: Zotero.Promise.coroutine(function* (item, jsonData, isNewObject, storageDetailsChanged) {
// Delete older versions of the item in the cache
yield this.deleteCacheObjectVersions(
'item', item.libraryID, jsonData.key, null, jsonData.version - 1
);
// Mark updated attachments for download
if (item.isImportedAttachment()) {
// If storage changes were made (attachment mtime or hash), mark
// library as requiring download
if (isNewObject || storageDetailsChanged) {
Zotero.Libraries.get(item.libraryID).storageDownloadNeeded = true;
}
yield this._checkAttachmentForDownload(
item, jsonData.mtime, isNewObject
);
}
}),
_checkAttachmentForDownload: Zotero.Promise.coroutine(function* (item, mtime, isNewObject) {
var markToDownload = false;
if (!isNewObject) {
@ -724,9 +762,7 @@ Zotero.Sync.Data.Local = {
markToDownload = true;
}
if (markToDownload) {
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
);
yield Zotero.Sync.Storage.Local.setSyncState(item.id, "to_download");
}
}),

View file

@ -62,6 +62,7 @@ Zotero.Sync.Runner_Module = function (options = {}) {
var _syncEngines = [];
var _storageEngines = [];
var _storageControllers = {};
var _lastSyncStatus;
var _currentSyncStatusLabel;
@ -476,6 +477,9 @@ Zotero.Sync.Runner_Module = function (options = {}) {
Object.assign(opts, options);
opts.libraryID = libraryID;
let mode = Zotero.Sync.Storage.Local.getModeForLibrary(libraryID);
opts.controller = this.getStorageController(mode, opts);
let tries = 3;
while (true) {
if (tries == 0) {
@ -549,6 +553,25 @@ Zotero.Sync.Runner_Module = function (options = {}) {
}.bind(this));
/**
* Get a storage controller for a given mode ('zfs', 'webdav'),
* caching it if necessary
*/
this.getStorageController = function (mode, options) {
if (_storageControllers[mode]) {
return _storageControllers[mode];
}
var modeClass = Zotero.Sync.Storage.Utilities.getClassForMode(mode);
return _storageControllers[mode] = new modeClass(options);
},
// TODO: Call on API key change
this.resetStorageController = function (mode) {
delete _storageControllers[mode];
},
/**
* Download a single file on demand (not within a sync process)
*/

View file

@ -235,15 +235,6 @@ var ZoteroPane = new function()
catch (e) {}
}
// Hide sync debugging menu by default
if (Zotero.Prefs.get('sync.debugMenu')) {
var sep = document.getElementById('zotero-tb-actions-sync-separator');
sep.hidden = false;
sep.nextSibling.hidden = false;
sep.nextSibling.nextSibling.hidden = false;
sep.nextSibling.nextSibling.nextSibling.hidden = false;
}
if (Zotero.openPane) {
Zotero.openPane = false;
setTimeout(function () {

View file

@ -114,10 +114,6 @@
label="Search for Shared Libraries" oncommand="Zotero.Zeroconf.findInstances()"/>
<menuseparator id="zotero-tb-actions-plugins-separator"/>
<menuitem id="zotero-tb-actions-timeline" label="&zotero.toolbar.timeline.label;" command="cmd_zotero_createTimeline"/>
<menuseparator hidden="true" id="zotero-tb-actions-sync-separator"/>
<menuitem hidden="true" label="WebDAV Sync Debugging" disabled="true"/>
<menuitem hidden="true" label=" Purge Deleted Storage Files" oncommand="Zotero.Sync.Storage.purgeDeletedStorageFiles('webdav', function(results) { Zotero.debug(results); })"/>
<menuitem hidden="true" label=" Purge Orphaned Storage Files" oncommand="Zotero.Sync.Storage.purgeOrphanedStorageFiles('webdav', function(results) { Zotero.debug(results); })"/>
<menuseparator id="zotero-tb-actions-separator"/>
<menuitem id="zotero-tb-actions-prefs" label="&zotero.toolbar.preferences.label;"
oncommand="ZoteroPane_Local.openPreferences()"/>

View file

@ -920,6 +920,9 @@ sync.storage.error.webdav.fileMissingAfterUpload = A potential problem was foun
sync.storage.error.webdav.nonexistentFileNotMissing = Your WebDAV server is claiming that a nonexistent file exists. Contact your WebDAV server administrator for assistance.
sync.storage.error.webdav.serverConfig.title = WebDAV Server Configuration Error
sync.storage.error.webdav.serverConfig = Your WebDAV server returned an internal error.
sync.storage.error.webdav.requestError = Your WebDAV server returned an HTTP %1$S error for a %2$S request.
sync.storage.error.webdav.checkSettingsOrContactAdmin = If you receive this message repeatedly, check your WebDAV server settings or contact your WebDAV server administrator.
sync.storage.error.webdav.url = URL: %S
sync.storage.error.zfs.restart = A file sync error occurred. Please restart %S and/or your computer and try syncing again.\n\nIf the error persists, there may be a problem with either your computer or your network: security software, proxy server, VPN, etc. Try disabling any security/firewall software you're using or, if this is a laptop, try from a different network.
sync.storage.error.zfs.tooManyQueuedUploads = You have too many queued uploads. Please try again in %S minutes.

View file

@ -1,4 +1,4 @@
-- 80
-- 81
-- Copyright (c) 2009 Center for History and New Media
-- George Mason University, Fairfax, Virginia, USA
@ -261,8 +261,8 @@ CREATE TABLE libraries (
editable INT NOT NULL,
filesEditable INT NOT NULL,
version INT NOT NULL DEFAULT 0,
lastSync INT NOT NULL DEFAULT 0,
lastStorageSync INT NOT NULL DEFAULT 0
storageVersion INT NOT NULL DEFAULT 0,
lastSync INT NOT NULL DEFAULT 0
);
CREATE TABLE users (

View file

@ -724,8 +724,14 @@ function setHTTPResponse(server, baseURL, response, responses) {
responseArray[1]["Content-Type"] = "text/plain";
responseArray[2] = response.text || "";
}
if (!response.headers) {
response.headers = {};
}
response.headers["Fake-Server-Match"] = 1;
for (let i in response.headers) {
responseArray[1][i] = response.headers[i];
}
server.respondWith(response.method, baseURL + response.url, responseArray);
}

View file

@ -616,7 +616,7 @@ describe("Zotero.Item", function () {
// DEBUG: Is this necessary?
assert.equal(
(yield Zotero.Sync.Storage.Local.getSyncState(item.id)),
Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD
Zotero.Sync.Storage.Local.SYNC_STATE_TO_UPLOAD
);
assert.isNull(yield Zotero.Sync.Storage.Local.getSyncedHash(item.id));
})
@ -874,6 +874,49 @@ describe("Zotero.Item", function () {
assert.strictEqual(json.deleted, 1);
})
it("should output attachment fields from file", function* () {
var file = getTestDataDirectory();
file.append('test.png');
var item = yield Zotero.Attachments.importFromFile({ file });
yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncedModificationTime(
item.id, new Date().getTime()
);
yield Zotero.Sync.Storage.Local.setSyncedHash(
item.id, 'b32e33f529942d73bea4ed112310f804'
);
});
var json = yield item.toJSON();
assert.equal(json.linkMode, 'imported_file');
assert.equal(json.filename, 'test.png');
assert.isUndefined(json.path);
assert.equal(json.mtime, (yield item.attachmentModificationTime));
assert.equal(json.md5, (yield item.attachmentHash));
})
it("should output synced storage values with .syncedStorageProperties", function* () {
var item = new Zotero.Item('attachment');
item.attachmentLinkMode = 'imported_file';
item.fileName = 'test.txt';
yield item.saveTx();
var mtime = new Date().getTime();
var md5 = 'b32e33f529942d73bea4ed112310f804';
yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncedModificationTime(item.id, mtime);
yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, md5);
});
var json = yield item.toJSON({
syncedStorageProperties: true
});
assert.equal(json.mtime, mtime);
assert.equal(json.md5, md5);
})
it("should output unset storage properties as null", function* () {
var item = new Zotero.Item('attachment');
item.attachmentLinkMode = 'imported_file';
@ -881,7 +924,6 @@ describe("Zotero.Item", function () {
var id = yield item.saveTx();
var json = yield item.toJSON();
Zotero.debug(json);
assert.isNull(json.mtime);
assert.isNull(json.md5);
})
@ -960,21 +1002,6 @@ describe("Zotero.Item", function () {
assert.strictEqual(json.deleted, 1);
})
})
// TODO: Expand to all fields
it("should handle attachment fields", function* () {
var file = getTestDataDirectory();
file.append('test.png');
var item = yield Zotero.Attachments.importFromFile({
file: file
});
var json = yield item.toJSON();
assert.equal(json.linkMode, 'imported_file');
assert.equal(json.filename, 'test.png');
assert.isUndefined(json.path);
assert.equal(json.md5, '93da8f1e5774c599f0942dcecf64b11c');
assert.typeOf(json.mtime, 'number');
})
})
describe("#fromJSON()", function () {

View file

@ -63,26 +63,6 @@ describe("Zotero.Library", function() {
});
});
describe("#lastStorageSync", function () {
it("should set and get a time in seconds", function* () {
var library = yield createGroup();
var time = Math.round(new Date().getTime() / 1000);
library.lastStorageSync = time;
yield library.saveTx();
var dbTime = yield Zotero.DB.valueQueryAsync(
"SELECT lastStorageSync FROM libraries WHERE libraryID=?", library.libraryID
);
assert.equal(dbTime, time);
assert.equal(library.lastStorageSync, time);
});
it("should throw if setting time in milliseconds", function* () {
var library = Zotero.Libraries.userLibrary;
assert.throws(() => library.lastStorageSync = new Date().getTime(), "timestamp must be in seconds");
})
})
describe("#editable", function() {
it("should return editable status", function() {
let library = Zotero.Libraries.get(Zotero.Libraries.userLibraryID);

View file

@ -1,819 +0,0 @@
"use strict";
describe("Zotero.Sync.Storage.Engine", function () {
Components.utils.import("resource://zotero-unit/httpd.js");
var win;
var apiKey = Zotero.Utilities.randomString(24);
var port = 16213;
var baseURL = `http://localhost:${port}/`;
var server;
var responses = {};
var setup = Zotero.Promise.coroutine(function* (options = {}) {
server = sinon.fakeServer.create();
server.autoRespond = true;
Components.utils.import("resource://zotero/concurrentCaller.js");
var caller = new ConcurrentCaller(1);
caller.setLogger(msg => Zotero.debug(msg));
caller.stopOnError = true;
Components.utils.import("resource://zotero/config.js");
var client = new Zotero.Sync.APIClient({
baseURL,
apiVersion: options.apiVersion || ZOTERO_CONFIG.API_VERSION,
apiKey,
caller,
background: options.background || true
});
var engine = new Zotero.Sync.Storage.Engine({
apiClient: client,
libraryID: options.libraryID || Zotero.Libraries.userLibraryID,
stopOnError: true
});
return { engine, client, caller };
});
function setResponse(response) {
setHTTPResponse(server, baseURL, response, responses);
}
function parseQueryString(str) {
var queryStringParams = str.split('&');
var params = {};
for (let param of queryStringParams) {
let [ key, val ] = param.split('=');
params[key] = decodeURIComponent(val);
}
return params;
}
function assertAPIKey(request) {
assert.equal(request.requestHeaders["Zotero-API-Key"], apiKey);
}
//
// Tests
//
before(function* () {
})
beforeEach(function* () {
yield resetDB({
thisArg: this,
skipBundledFiles: true
});
win = yield loadZoteroPane();
Zotero.HTTP.mock = sinon.FakeXMLHttpRequest;
this.httpd = new HttpServer();
this.httpd.start(port);
yield Zotero.Users.setCurrentUserID(1);
yield Zotero.Users.setCurrentUsername("testuser");
// Set download-on-sync by default
Zotero.Sync.Storage.Local.downloadOnSync(
Zotero.Libraries.userLibraryID, true
);
})
afterEach(function* () {
var defer = new Zotero.Promise.defer();
this.httpd.stop(() => defer.resolve());
yield defer.promise;
win.close();
})
after(function* () {
this.timeout(60000);
//yield resetDB();
win.close();
})
describe("ZFS", function () {
describe("Syncing", function () {
it("should skip downloads if no last storage sync time", function* () {
var { engine, client, caller } = yield setup();
setResponse({
method: "GET",
url: "users/1/laststoragesync",
status: 404
});
var result = yield engine.start();
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
// Check last sync time
assert.isFalse(Zotero.Libraries.userLibrary.lastStorageSync);
})
it("should skip downloads if unchanged last storage sync time", function* () {
var { engine, client, caller } = yield setup();
var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
var library = Zotero.Libraries.userLibrary;
library.lastStorageSync = newStorageSyncTime;
yield library.saveTx();
setResponse({
method: "GET",
url: "users/1/laststoragesync",
status: 200,
text: "" + newStorageSyncTime
});
var result = yield engine.start();
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
// Check last sync time
assert.equal(library.lastStorageSync, newStorageSyncTime);
})
it("should ignore a remotely missing file", function* () {
var { engine, client, caller } = yield setup();
var item = new Zotero.Item("attachment");
item.attachmentLinkMode = 'imported_file';
item.attachmentPath = 'storage:test.txt';
yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
);
var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
setResponse({
method: "GET",
url: "users/1/laststoragesync",
status: 200,
text: "" + newStorageSyncTime
});
this.httpd.registerPathHandler(
`/users/1/items/${item.key}/file`,
{
handle: function (request, response) {
response.setStatusLine(null, 404, null);
}
}
);
var result = yield engine.start();
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
// Check last sync time
assert.equal(Zotero.Libraries.userLibrary.lastStorageSync, newStorageSyncTime);
})
it("should handle a remotely failing file", function* () {
var { engine, client, caller } = yield setup();
var item = new Zotero.Item("attachment");
item.attachmentLinkMode = 'imported_file';
item.attachmentPath = 'storage:test.txt';
yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
);
var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
setResponse({
method: "GET",
url: "users/1/laststoragesync",
status: 200,
text: "" + newStorageSyncTime
});
this.httpd.registerPathHandler(
`/users/1/items/${item.key}/file`,
{
handle: function (request, response) {
response.setStatusLine(null, 500, null);
}
}
);
// TODO: In stopOnError mode, this the promise is rejected.
// This should probably test with stopOnError mode turned off instead.
var e = yield getPromiseError(engine.start());
assert.equal(e.message, Zotero.Sync.Storage.defaultError);
})
it("should download a missing file", function* () {
var { engine, client, caller } = yield setup();
var item = new Zotero.Item("attachment");
item.attachmentLinkMode = 'imported_file';
item.attachmentPath = 'storage:test.txt';
// TODO: Test binary data
var text = Zotero.Utilities.randomString();
yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
);
var mtime = "1441252524905";
var md5 = Zotero.Utilities.Internal.md5(text)
var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
setResponse({
method: "GET",
url: "users/1/laststoragesync",
status: 200,
text: "" + newStorageSyncTime
});
var s3Path = `pretend-s3/${item.key}`;
this.httpd.registerPathHandler(
`/users/1/items/${item.key}/file`,
{
handle: function (request, response) {
if (!request.hasHeader('Zotero-API-Key')) {
response.setStatusLine(null, 403, "Forbidden");
return;
}
var key = request.getHeader('Zotero-API-Key');
if (key != apiKey) {
response.setStatusLine(null, 403, "Invalid key");
return;
}
response.setStatusLine(null, 302, "Found");
response.setHeader("Zotero-File-Modification-Time", mtime, false);
response.setHeader("Zotero-File-MD5", md5, false);
response.setHeader("Zotero-File-Compressed", "No", false);
response.setHeader("Location", baseURL + s3Path, false);
}
}
);
this.httpd.registerPathHandler(
"/" + s3Path,
{
handle: function (request, response) {
response.setStatusLine(null, 200, "OK");
response.write(text);
}
}
);
var result = yield engine.start();
assert.isTrue(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
// Check last sync time
assert.equal(Zotero.Libraries.userLibrary.lastStorageSync, newStorageSyncTime);
var contents = yield Zotero.File.getContentsAsync(yield item.getFilePathAsync());
assert.equal(contents, text);
})
it("should upload new files", function* () {
var { engine, client, caller } = yield setup();
// Single file
var file1 = getTestDataDirectory();
file1.append('test.png');
var item1 = yield Zotero.Attachments.importFromFile({ file: file1 });
var mtime1 = yield item1.attachmentModificationTime;
var hash1 = yield item1.attachmentHash;
var path1 = item1.getFilePath();
var filename1 = 'test.png';
var size1 = (yield OS.File.stat(path1)).size;
var contentType1 = 'image/png';
var prefix1 = Zotero.Utilities.randomString();
var suffix1 = Zotero.Utilities.randomString();
var uploadKey1 = Zotero.Utilities.randomString(32, 'abcdef0123456789');
// HTML file with auxiliary image
var file2 = OS.Path.join(getTestDataDirectory().path, 'snapshot', 'index.html');
var parentItem = yield createDataObject('item');
var item2 = yield Zotero.Attachments.importSnapshotFromFile({
file: file2,
url: 'http://example.com/',
parentItemID: parentItem.id,
title: 'Test',
contentType: 'text/html',
charset: 'utf-8'
});
var mtime2 = yield item2.attachmentModificationTime;
var hash2 = yield item2.attachmentHash;
var path2 = item2.getFilePath();
var filename2 = 'index.html';
var size2 = (yield OS.File.stat(path2)).size;
var contentType2 = 'text/html';
var charset2 = 'utf-8';
var prefix2 = Zotero.Utilities.randomString();
var suffix2 = Zotero.Utilities.randomString();
var uploadKey2 = Zotero.Utilities.randomString(32, 'abcdef0123456789');
var deferreds = [];
setResponse({
method: "GET",
url: "users/1/laststoragesync",
status: 404
});
// https://github.com/cjohansen/Sinon.JS/issues/607
let fixSinonBug = ";charset=utf-8";
server.respond(function (req) {
// Get upload authorization for single file
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item1.key}/file`
&& req.requestBody.indexOf('upload=') == -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
let parts = req.requestBody.split('&');
let params = {};
for (let part of parts) {
let [key, val] = part.split('=');
params[key] = decodeURIComponent(val);
}
assert.equal(params.md5, hash1);
assert.equal(params.mtime, mtime1);
assert.equal(params.filename, filename1);
assert.equal(params.filesize, size1);
assert.equal(params.contentType, contentType1);
req.respond(
200,
{
"Content-Type": "application/json"
},
JSON.stringify({
url: baseURL + "pretend-s3/1",
contentType: contentType1,
prefix: prefix1,
suffix: suffix1,
uploadKey: uploadKey1
})
);
}
// Get upload authorization for multi-file zip
else if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item2.key}/file`
&& req.requestBody.indexOf('upload=') == -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
// Verify ZIP hash
let tmpZipPath = OS.Path.join(
Zotero.getTempDirectory().path,
item2.key + '.zip'
);
deferreds.push({
promise: Zotero.Utilities.Internal.md5Async(tmpZipPath)
.then(function (md5) {
assert.equal(params.zipMD5, md5);
})
});
let parts = req.requestBody.split('&');
let params = {};
for (let part of parts) {
let [key, val] = part.split('=');
params[key] = decodeURIComponent(val);
}
Zotero.debug(params);
assert.equal(params.md5, hash2);
assert.notEqual(params.zipMD5, hash2);
assert.equal(params.mtime, mtime2);
assert.equal(params.filename, filename2);
assert.equal(params.zipFilename, item2.key + ".zip");
assert.isTrue(parseInt(params.filesize) == params.filesize);
assert.equal(params.contentType, contentType2);
assert.equal(params.charset, charset2);
req.respond(
200,
{
"Content-Type": "application/json"
},
JSON.stringify({
url: baseURL + "pretend-s3/2",
contentType: 'application/zip',
prefix: prefix2,
suffix: suffix2,
uploadKey: uploadKey2
})
);
}
// Upload single file to S3
else if (req.method == "POST" && req.url == baseURL + "pretend-s3/1") {
assert.equal(req.requestHeaders["Content-Type"], contentType1 + fixSinonBug);
assert.equal(req.requestBody.size, (new Blob([prefix1, File(file1), suffix1]).size));
req.respond(201, {}, "");
}
// Upload multi-file ZIP to S3
else if (req.method == "POST" && req.url == baseURL + "pretend-s3/2") {
assert.equal(req.requestHeaders["Content-Type"], "application/zip" + fixSinonBug);
// Verify uploaded ZIP file
let tmpZipPath = OS.Path.join(
Zotero.getTempDirectory().path,
Zotero.Utilities.randomString() + '.zip'
);
let deferred = Zotero.Promise.defer();
deferreds.push(deferred);
var reader = new FileReader();
reader.addEventListener("loadend", Zotero.Promise.coroutine(function* () {
try {
let file = yield OS.File.open(tmpZipPath, {
create: true
});
var contents = new Uint8Array(reader.result);
contents = contents.slice(prefix2.length, suffix2.length * -1);
yield file.write(contents);
yield file.close();
var zr = Components.classes["@mozilla.org/libjar/zip-reader;1"]
.createInstance(Components.interfaces.nsIZipReader);
zr.open(Zotero.File.pathToFile(tmpZipPath));
zr.test(null);
var entries = zr.findEntries('*');
var entryNames = [];
while (entries.hasMore()) {
entryNames.push(entries.getNext());
}
assert.equal(entryNames.length, 2);
assert.sameMembers(entryNames, ['index.html', 'img.gif']);
assert.equal(zr.getEntry('index.html').realSize, size2);
assert.equal(zr.getEntry('img.gif').realSize, 42);
deferred.resolve();
}
catch (e) {
deferred.reject(e);
}
}));
reader.readAsArrayBuffer(req.requestBody);
req.respond(201, {}, "");
}
// Register single-file upload
else if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item1.key}/file`
&& req.requestBody.indexOf('upload=') != -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
let parts = req.requestBody.split('&');
let params = {};
for (let part of parts) {
let [key, val] = part.split('=');
params[key] = decodeURIComponent(val);
}
assert.equal(params.upload, uploadKey1);
req.respond(
204,
{
"Last-Modified-Version": 10
},
""
);
}
// Register multi-file upload
else if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item2.key}/file`
&& req.requestBody.indexOf('upload=') != -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
let parts = req.requestBody.split('&');
let params = {};
for (let part of parts) {
let [key, val] = part.split('=');
params[key] = decodeURIComponent(val);
}
assert.equal(params.upload, uploadKey2);
req.respond(
204,
{
"Last-Modified-Version": 15
},
""
);
}
})
var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
setResponse({
method: "POST",
url: "users/1/laststoragesync",
status: 200,
text: "" + newStorageSyncTime
});
// TODO: One-step uploads
/*// https://github.com/cjohansen/Sinon.JS/issues/607
let fixSinonBug = ";charset=utf-8";
server.respond(function (req) {
if (req.method == "POST" && req.url == `${baseURL}users/1/items/${item.key}/file`) {
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/json" + fixSinonBug
);
let params = JSON.parse(req.requestBody);
assert.equal(params.md5, hash);
assert.equal(params.mtime, mtime);
assert.equal(params.filename, filename);
assert.equal(params.size, size);
assert.equal(params.contentType, contentType);
req.respond(
200,
{
"Content-Type": "application/json"
},
JSON.stringify({
url: baseURL + "pretend-s3",
headers: {
"Content-Type": contentType,
"Content-MD5": hash,
//"Content-Length": params.size, process but don't return
//"x-amz-meta-"
},
uploadKey
})
);
}
else if (req.method == "PUT" && req.url == baseURL + "pretend-s3") {
assert.equal(req.requestHeaders["Content-Type"], contentType + fixSinonBug);
assert.instanceOf(req.requestBody, File);
req.respond(201, {}, "");
}
})*/
var result = yield engine.start();
yield Zotero.Promise.all(deferreds.map(d => d.promise));
assert.isTrue(result.localChanges);
assert.isTrue(result.remoteChanges);
assert.isFalse(result.syncRequired);
// Check local objects
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item1.id)), mtime1);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item1.id)), hash1);
assert.equal(item1.version, 10);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item2.id)), mtime2);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item2.id)), hash2);
assert.equal(item2.version, 15);
// Check last sync time
assert.equal(Zotero.Libraries.userLibrary.lastStorageSync, newStorageSyncTime);
})
it("should update local info for file that already exists on the server", function* () {
var { engine, client, caller } = yield setup();
var file = getTestDataDirectory();
file.append('test.png');
var item = yield Zotero.Attachments.importFromFile({ file: file });
item.version = 5;
yield item.saveTx();
var json = yield item.toJSON();
yield Zotero.Sync.Data.Local.saveCacheObject('item', item.libraryID, json);
var mtime = yield item.attachmentModificationTime;
var hash = yield item.attachmentHash;
var path = item.getFilePath();
var filename = 'test.png';
var size = (yield OS.File.stat(path)).size;
var contentType = 'image/png';
var newVersion = 10;
setResponse({
method: "POST",
url: "users/1/laststoragesync",
status: 200,
text: "" + (Math.round(new Date().getTime() / 1000) - 50000)
});
// https://github.com/cjohansen/Sinon.JS/issues/607
let fixSinonBug = ";charset=utf-8";
server.respond(function (req) {
// Get upload authorization for single file
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item.key}/file`
&& req.requestBody.indexOf('upload=') == -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
req.respond(
200,
{
"Content-Type": "application/json",
"Last-Modified-Version": newVersion
},
JSON.stringify({
exists: 1,
})
);
}
})
var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
setResponse({
method: "POST",
url: "users/1/laststoragesync",
status: 200,
text: "" + newStorageSyncTime
});
// TODO: One-step uploads
var result = yield engine.start();
assert.isTrue(result.localChanges);
assert.isTrue(result.remoteChanges);
assert.isFalse(result.syncRequired);
// Check local objects
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item.id)), mtime);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item.id)), hash);
assert.equal(item.version, newVersion);
// Check last sync time
assert.equal(Zotero.Libraries.userLibrary.lastStorageSync, newStorageSyncTime);
})
})
describe("#_processUploadFile()", function () {
it("should handle 412 with matching version and hash matching local file", function* () {
var { engine, client, caller } = yield setup();
var zfs = new Zotero.Sync.Storage.ZFS_Module({
apiClient: client
})
var filePath = OS.Path.join(getTestDataDirectory().path, 'test.png');
var item = yield Zotero.Attachments.importFromFile({ file: filePath });
item.version = 5;
item.synced = true;
yield item.saveTx();
var itemJSON = yield item.toResponseJSON();
// Set saved hash to a different value, which should be overwritten
//
// We're also testing cases where a hash isn't set for a file (e.g., if the
// storage directory was transferred, the mtime doesn't match, but the file was
// never downloaded), but there's no difference in behavior
var dbHash = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa';
yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, dbHash)
});
server.respond(function (req) {
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item.key}/file`
&& req.requestBody.indexOf('upload=') == -1
&& req.requestHeaders["If-Match"] == dbHash) {
req.respond(
412,
{
"Content-Type": "application/json",
"Last-Modified-Version": 5
},
"ETag does not match current version of file"
);
}
})
setResponse({
method: "GET",
url: `users/1/items?format=json&itemKey=${item.key}&includeTrashed=1`,
status: 200,
text: JSON.stringify([itemJSON])
});
var result = yield zfs._processUploadFile({
name: item.libraryKey
});
yield assert.eventually.equal(
Zotero.Sync.Storage.Local.getSyncedHash(item.id), itemJSON.data.md5
);
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
assert.isFalse(result.fileSyncRequired);
})
it("should handle 412 with matching version and hash not matching local file", function* () {
var { engine, client, caller } = yield setup();
var zfs = new Zotero.Sync.Storage.ZFS_Module({
apiClient: client
})
var filePath = OS.Path.join(getTestDataDirectory().path, 'test.png');
var item = yield Zotero.Attachments.importFromFile({ file: filePath });
item.version = 5;
item.synced = true;
yield item.saveTx();
var fileHash = yield item.attachmentHash;
var itemJSON = yield item.toResponseJSON();
itemJSON.data.md5 = 'aaaaaaaaaaaaaaaaaaaaaaaa'
server.respond(function (req) {
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item.key}/file`
&& req.requestBody.indexOf('upload=') == -1
&& req.requestHeaders["If-None-Match"] == "*") {
req.respond(
412,
{
"Content-Type": "application/json",
"Last-Modified-Version": 5
},
"If-None-Match: * set but file exists"
);
}
})
setResponse({
method: "GET",
url: `users/1/items?format=json&itemKey=${item.key}&includeTrashed=1`,
status: 200,
text: JSON.stringify([itemJSON])
});
var result = yield zfs._processUploadFile({
name: item.libraryKey
});
yield assert.eventually.isNull(Zotero.Sync.Storage.Local.getSyncedHash(item.id));
yield assert.eventually.equal(
Zotero.Sync.Storage.Local.getSyncState(item.id),
Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
);
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
assert.isTrue(result.fileSyncRequired);
})
it("should handle 412 with greater version", function* () {
var { engine, client, caller } = yield setup();
var zfs = new Zotero.Sync.Storage.ZFS_Module({
apiClient: client
})
var file = getTestDataDirectory();
file.append('test.png');
var item = yield Zotero.Attachments.importFromFile({ file });
item.version = 5;
item.synced = true;
yield item.saveTx();
server.respond(function (req) {
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item.key}/file`
&& req.requestBody.indexOf('upload=') == -1
&& req.requestHeaders["If-None-Match"] == "*") {
req.respond(
412,
{
"Content-Type": "application/json",
"Last-Modified-Version": 10
},
"If-None-Match: * set but file exists"
);
}
})
var result = yield zfs._processUploadFile({
name: item.libraryKey
});
assert.equal(item.version, 5);
assert.equal(item.synced, true);
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isTrue(result.syncRequired);
})
})
})
})

View file

@ -31,9 +31,7 @@ describe("Zotero.Sync.Storage.Local", function () {
yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, hash);
yield Zotero.Sync.Storage.Local.setSyncedModificationTime(item.id, mtime);
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
);
yield Zotero.Sync.Storage.Local.setSyncState(item.id, "in_sync");
});
// Update mtime and contents
@ -50,7 +48,7 @@ describe("Zotero.Sync.Storage.Local", function () {
assert.equal(changed, true);
assert.equal(
(yield Zotero.Sync.Storage.Local.getSyncState(item.id)),
Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD
Zotero.Sync.Storage.Local.SYNC_STATE_TO_UPLOAD
);
})
@ -64,9 +62,7 @@ describe("Zotero.Sync.Storage.Local", function () {
yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, hash);
yield Zotero.Sync.Storage.Local.setSyncedModificationTime(item.id, mtime);
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
);
yield Zotero.Sync.Storage.Local.setSyncState(item.id, "in_sync");
});
var libraryID = Zotero.Libraries.userLibraryID;
@ -76,7 +72,7 @@ describe("Zotero.Sync.Storage.Local", function () {
yield item.eraseTx();
assert.isFalse(changed);
assert.equal(syncState, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
assert.equal(syncState, Zotero.Sync.Storage.Local.SYNC_STATE_IN_SYNC);
})
it("should skip a file if mod time has changed but contents haven't", function* () {
@ -91,9 +87,7 @@ describe("Zotero.Sync.Storage.Local", function () {
yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, hash);
yield Zotero.Sync.Storage.Local.setSyncedModificationTime(item.id, mtime);
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
);
yield Zotero.Sync.Storage.Local.setSyncState(item.id, "in_sync");
});
// Update mtime, but not contents
@ -109,7 +103,7 @@ describe("Zotero.Sync.Storage.Local", function () {
yield item.eraseTx();
assert.isFalse(changed);
assert.equal(syncState, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
assert.equal(syncState, Zotero.Sync.Storage.Local.SYNC_STATE_IN_SYNC);
assert.equal(syncedModTime, newModTime);
})
})
@ -217,12 +211,8 @@ describe("Zotero.Sync.Storage.Local", function () {
json3.mtime = now - 20000;
yield Zotero.Sync.Data.Local.saveCacheObjects('item', libraryID, [json1, json3]);
yield Zotero.Sync.Storage.Local.setSyncState(
item1.id, Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
);
yield Zotero.Sync.Storage.Local.setSyncState(
item3.id, Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
);
yield Zotero.Sync.Storage.Local.setSyncState(item1.id, "in_conflict");
yield Zotero.Sync.Storage.Local.setSyncState(item3.id, "in_conflict");
var conflicts = yield Zotero.Sync.Storage.Local.getConflicts(libraryID);
assert.lengthOf(conflicts, 2);
@ -269,10 +259,10 @@ describe("Zotero.Sync.Storage.Local", function () {
yield Zotero.Sync.Data.Local.saveCacheObjects('item', libraryID, [json1, json3]);
yield Zotero.Sync.Storage.Local.setSyncState(
item1.id, Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
item1.id, "in_conflict"
);
yield Zotero.Sync.Storage.Local.setSyncState(
item3.id, Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
item3.id, "in_conflict"
);
var promise = waitForWindow('chrome://zotero/content/merge.xul', function (dialog) {
@ -317,11 +307,11 @@ describe("Zotero.Sync.Storage.Local", function () {
yield assert.eventually.equal(
Zotero.Sync.Storage.Local.getSyncState(item1.id),
Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD
Zotero.Sync.Storage.Local.SYNC_STATE_FORCE_UPLOAD
);
yield assert.eventually.equal(
Zotero.Sync.Storage.Local.getSyncState(item3.id),
Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD
Zotero.Sync.Storage.Local.SYNC_STATE_FORCE_DOWNLOAD
);
})
})

View file

@ -107,11 +107,6 @@ describe("Zotero.Sync.Data.Engine", function () {
yield Zotero.Users.setCurrentUserID(1);
yield Zotero.Users.setCurrentUsername("testuser");
})
after(function* () {
yield resetDB({
thisArg: this
});
})
describe("Syncing", function () {
it("should download items into a new library", function* () {
@ -415,6 +410,70 @@ describe("Zotero.Sync.Data.Engine", function () {
}
})
it("should upload synced storage properties", function* () {
({ engine, client, caller } = yield setup());
var libraryID = Zotero.Libraries.userLibraryID;
var lastLibraryVersion = 2;
yield Zotero.Libraries.setVersion(libraryID, lastLibraryVersion);
var item = new Zotero.Item('attachment');
item.attachmentLinkMode = 'imported_file';
item.attachmentFilename = 'test1.txt';
yield item.saveTx();
var mtime = new Date().getTime();
var md5 = '57f8a4fda823187b91e1191487b87fe6';
yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncedModificationTime(item.id, mtime);
yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, md5);
});
var itemResponseJSON = yield item.toResponseJSON();
itemResponseJSON.version = itemResponseJSON.data.version = lastLibraryVersion;
itemResponseJSON.data.mtime = mtime;
itemResponseJSON.data.md5 = md5;
server.respond(function (req) {
if (req.method == "POST") {
if (req.url == baseURL + "users/1/items") {
let json = JSON.parse(req.requestBody);
assert.lengthOf(json, 1);
let itemJSON = json[0];
assert.equal(itemJSON.key, item.key);
assert.equal(itemJSON.version, 0);
assert.equal(itemJSON.mtime, mtime);
assert.equal(itemJSON.md5, md5);
req.respond(
200,
{
"Content-Type": "application/json",
"Last-Modified-Version": lastLibraryVersion
},
JSON.stringify({
successful: {
"0": itemResponseJSON
},
unchanged: {},
failed: {}
})
);
return;
}
}
})
yield engine.start();
// Check data in cache
var json = yield Zotero.Sync.Data.Local.getCacheObject(
'item', libraryID, item.key, lastLibraryVersion
);
assert.equal(json.data.mtime, mtime);
assert.equal(json.data.md5, md5);
})
it("should update local objects with remotely saved version after uploading if necessary", function* () {
({ engine, client, caller } = yield setup());

View file

@ -149,7 +149,7 @@ describe("Zotero.Sync.Data.Local", function() {
var id = Zotero.Items.getIDFromLibraryAndKey(libraryID, key);
assert.equal(
(yield Zotero.Sync.Storage.Local.getSyncState(id)),
Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
Zotero.Sync.Storage.Local.SYNC_STATE_TO_DOWNLOAD
);
})
@ -170,9 +170,7 @@ describe("Zotero.Sync.Data.Local", function() {
yield Zotero.Sync.Storage.Local.setSyncedHash(
item.id, (yield item.attachmentHash)
);
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
);
yield Zotero.Sync.Storage.Local.setSyncState(item.id, "in_sync");
});
// Simulate download of version with updated attachment
@ -191,7 +189,7 @@ describe("Zotero.Sync.Data.Local", function() {
assert.equal(
(yield Zotero.Sync.Storage.Local.getSyncState(item.id)),
Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
Zotero.Sync.Storage.Local.SYNC_STATE_TO_DOWNLOAD
);
})
@ -213,9 +211,7 @@ describe("Zotero.Sync.Data.Local", function() {
yield Zotero.Sync.Storage.Local.setSyncedHash(
item.id, (yield item.attachmentHash)
);
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
);
yield Zotero.Sync.Storage.Local.setSyncState(item.id, "in_sync");
});
// Modify title locally, leaving item unsynced
@ -237,7 +233,7 @@ describe("Zotero.Sync.Data.Local", function() {
assert.equal(item.getField('title'), newTitle);
assert.equal(
(yield Zotero.Sync.Storage.Local.getSyncState(item.id)),
Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
Zotero.Sync.Storage.Local.SYNC_STATE_TO_DOWNLOAD
);
})
})

764
test/tests/webdavTest.js Normal file
View file

@ -0,0 +1,764 @@
"use strict";
describe("Zotero.Sync.Storage.Mode.WebDAV", function () {
//
// Setup
//
Components.utils.import("resource://zotero-unit/httpd.js");
var apiKey = Zotero.Utilities.randomString(24);
var apiPort = 16213;
var apiURL = `http://localhost:${apiPort}/`;
var davScheme = "http";
var davPort = 16214;
var davBasePath = "/webdav/";
var davHostPath = `localhost:${davPort}${davBasePath}`;
var davUsername = "user";
var davPassword = "password";
var davURL = `${davScheme}://${davUsername}:${davPassword}@${davHostPath}`;
var win, controller, server, requestCount;
var responses = {};
function setResponse(response) {
setHTTPResponse(server, davURL, response, responses);
}
function resetRequestCount() {
requestCount = server.requests.filter(r => r.responseHeaders["Fake-Server-Match"]).length;
}
function assertRequestCount(count) {
assert.equal(
server.requests.filter(r => r.responseHeaders["Fake-Server-Match"]).length - requestCount,
count
);
}
function generateLastSyncID() {
return "" + Zotero.Utilities.randomString(controller._lastSyncIDLength);
}
function parseQueryString(str) {
var queryStringParams = str.split('&');
var params = {};
for (let param of queryStringParams) {
let [ key, val ] = param.split('=');
params[key] = decodeURIComponent(val);
}
return params;
}
function assertAPIKey(request) {
assert.equal(request.requestHeaders["Zotero-API-Key"], apiKey);
}
before(function* () {
controller = new Zotero.Sync.Storage.Mode.WebDAV;
Zotero.Prefs.set("sync.storage.scheme", davScheme);
Zotero.Prefs.set("sync.storage.url", davHostPath);
Zotero.Prefs.set("sync.storage.username", davUsername);
controller.password = davPassword;
})
beforeEach(function* () {
yield resetDB({
thisArg: this,
skipBundledFiles: true
});
Zotero.HTTP.mock = sinon.FakeXMLHttpRequest;
server = sinon.fakeServer.create();
server.autoRespond = true;
this.httpd = new HttpServer();
this.httpd.start(davPort);
yield Zotero.Users.setCurrentUserID(1);
yield Zotero.Users.setCurrentUsername("testuser");
Zotero.Sync.Storage.Local.setModeForLibrary(Zotero.Libraries.userLibraryID, 'webdav');
// Set download-on-sync by default
Zotero.Sync.Storage.Local.downloadOnSync(
Zotero.Libraries.userLibraryID, true
);
})
var setup = Zotero.Promise.coroutine(function* (options = {}) {
var engine = new Zotero.Sync.Storage.Engine({
libraryID: options.libraryID || Zotero.Libraries.userLibraryID,
controller,
stopOnError: true
});
if (!controller.verified) {
setResponse({
method: "OPTIONS",
url: "zotero/",
headers: {
DAV: 1
},
status: 200
})
setResponse({
method: "PROPFIND",
url: "zotero/",
status: 207
})
setResponse({
method: "PUT",
url: "zotero/zotero-test-file.prop",
status: 201
})
setResponse({
method: "GET",
url: "zotero/zotero-test-file.prop",
status: 200
})
setResponse({
method: "DELETE",
url: "zotero/zotero-test-file.prop",
status: 200
})
yield controller.checkServer();
yield controller.cacheCredentials();
}
resetRequestCount();
return engine;
})
afterEach(function* () {
var defer = new Zotero.Promise.defer();
this.httpd.stop(() => defer.resolve());
yield defer.promise;
})
after(function* () {
if (win) {
win.close();
}
})
//
// Tests
//
describe("Syncing", function () {
beforeEach(function* () {
win = yield loadZoteroPane();
})
afterEach(function () {
win.close();
})
it("should skip downloads if not marked as needed", function* () {
var engine = yield setup();
var library = Zotero.Libraries.userLibrary;
library.libraryVersion = 5;
yield library.saveTx();
var result = yield engine.start();
assertRequestCount(0);
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
assert.equal(library.storageVersion, library.libraryVersion);
})
it("should ignore a remotely missing file", function* () {
var engine = yield setup();
var library = Zotero.Libraries.userLibrary;
library.libraryVersion = 5;
yield library.saveTx();
library.storageDownloadNeeded = true;
var item = new Zotero.Item("attachment");
item.attachmentLinkMode = 'imported_file';
item.attachmentPath = 'storage:test.txt';
yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState(item.id, "to_download");
setResponse({
method: "GET",
url: `zotero/${item.key}.prop`,
status: 404
});
var result = yield engine.start();
assertRequestCount(1);
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
assert.isFalse(library.storageDownloadNeeded);
assert.equal(library.storageVersion, library.libraryVersion);
})
it("should handle a remotely failing .prop file", function* () {
var engine = yield setup();
var library = Zotero.Libraries.userLibrary;
library.libraryVersion = 5;
yield library.saveTx();
library.storageDownloadNeeded = true;
var item = new Zotero.Item("attachment");
item.attachmentLinkMode = 'imported_file';
item.attachmentPath = 'storage:test.txt';
yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState(item.id, "to_download");
setResponse({
method: "GET",
url: `zotero/${item.key}.prop`,
status: 500
});
// TODO: In stopOnError mode, the promise is rejected.
// This should probably test with stopOnError mode turned off instead.
var e = yield getPromiseError(engine.start());
assert.include(
e.message,
Zotero.getString('sync.storage.error.webdav.requestError', [500, "GET"])
);
assertRequestCount(1);
assert.isTrue(library.storageDownloadNeeded);
assert.equal(library.storageVersion, 0);
})
it("should handle a remotely failing .zip file", function* () {
var engine = yield setup();
var library = Zotero.Libraries.userLibrary;
library.libraryVersion = 5;
yield library.saveTx();
library.storageDownloadNeeded = true;
var item = new Zotero.Item("attachment");
item.attachmentLinkMode = 'imported_file';
item.attachmentPath = 'storage:test.txt';
yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState(item.id, "to_download");
setResponse({
method: "GET",
url: `zotero/${item.key}.prop`,
status: 200,
text: '<properties version="1">'
+ '<mtime>1234567890</mtime>'
+ '<hash>8286300a280f64a4b5cfaac547c21d32</hash>'
+ '</properties>'
});
this.httpd.registerPathHandler(
`${davBasePath}zotero/${item.key}.zip`,
{
handle: function (request, response) {
response.setStatusLine(null, 500, null);
}
}
);
// TODO: In stopOnError mode, the promise is rejected.
// This should probably test with stopOnError mode turned off instead.
var e = yield getPromiseError(engine.start());
assert.include(
e.message,
Zotero.getString('sync.storage.error.webdav.requestError', [500, "GET"])
);
assert.isTrue(library.storageDownloadNeeded);
assert.equal(library.storageVersion, 0);
})
it("should download a missing file", function* () {
var engine = yield setup();
var library = Zotero.Libraries.userLibrary;
library.libraryVersion = 5;
yield library.saveTx();
library.storageDownloadNeeded = true;
var fileName = "test.txt";
var item = new Zotero.Item("attachment");
item.attachmentLinkMode = 'imported_file';
item.attachmentPath = 'storage:' + fileName;
// TODO: Test binary data
var text = Zotero.Utilities.randomString();
yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState(item.id, "to_download");
// Create ZIP file containing above text file
var tmpPath = Zotero.getTempDirectory().path;
var tmpID = "webdav_download_" + Zotero.Utilities.randomString();
var zipDirPath = OS.Path.join(tmpPath, tmpID);
var zipPath = OS.Path.join(tmpPath, tmpID + ".zip");
yield OS.File.makeDir(zipDirPath);
yield Zotero.File.putContentsAsync(OS.Path.join(zipDirPath, fileName), text);
yield Zotero.File.zipDirectory(zipDirPath, zipPath);
yield OS.File.removeDir(zipDirPath);
yield Zotero.Promise.delay(1000);
var zipContents = yield Zotero.File.getBinaryContentsAsync(zipPath);
var mtime = "1441252524905";
var md5 = yield Zotero.Utilities.Internal.md5Async(zipPath);
yield OS.File.remove(zipPath);
setResponse({
method: "GET",
url: `zotero/${item.key}.prop`,
status: 200,
text: '<properties version="1">'
+ `<mtime>${mtime}</mtime>`
+ `<hash>${md5}</hash>`
+ '</properties>'
});
this.httpd.registerPathHandler(
`${davBasePath}zotero/${item.key}.zip`,
{
handle: function (request, response) {
response.setStatusLine(null, 200, "OK");
response.write(zipContents);
}
}
);
var result = yield engine.start();
assert.isTrue(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
var contents = yield Zotero.File.getContentsAsync(yield item.getFilePathAsync());
assert.equal(contents, text);
assert.isFalse(library.storageDownloadNeeded);
assert.equal(library.storageVersion, library.libraryVersion);
})
it("should upload new files", function* () {
var engine = yield setup();
var file = getTestDataDirectory();
file.append('test.png');
var item = yield Zotero.Attachments.importFromFile({ file });
item.synced = true;
yield item.saveTx();
var mtime = yield item.attachmentModificationTime;
var hash = yield item.attachmentHash;
var path = item.getFilePath();
var filename = 'test.png';
var size = (yield OS.File.stat(path)).size;
var contentType = 'image/png';
var fileContents = yield Zotero.File.getContentsAsync(path);
var deferreds = [];
setResponse({
method: "GET",
url: `zotero/${item.key}.prop`,
status: 404
});
// https://github.com/cjohansen/Sinon.JS/issues/607
let fixSinonBug = ";charset=utf-8";
server.respond(function (req) {
if (req.method == "PUT" && req.url == `${davURL}zotero/${item.key}.zip`) {
assert.equal(req.requestHeaders["Content-Type"], "application/zip" + fixSinonBug);
let deferred = Zotero.Promise.defer();
deferreds.push(deferred);
var reader = new FileReader();
reader.addEventListener("loadend", Zotero.Promise.coroutine(function* () {
try {
let tmpZipPath = OS.Path.join(
Zotero.getTempDirectory().path,
Zotero.Utilities.randomString() + '.zip'
);
let file = yield OS.File.open(tmpZipPath, {
create: true
});
var contents = new Uint8Array(reader.result);
yield file.write(contents);
yield file.close();
// Make sure ZIP file contains the necessary entries
var zr = Components.classes["@mozilla.org/libjar/zip-reader;1"]
.createInstance(Components.interfaces.nsIZipReader);
zr.open(Zotero.File.pathToFile(tmpZipPath));
zr.test(null);
var entries = zr.findEntries('*');
var entryNames = [];
while (entries.hasMore()) {
entryNames.push(entries.getNext());
}
assert.equal(entryNames.length, 1);
assert.sameMembers(entryNames, [filename]);
assert.equal(zr.getEntry(filename).realSize, size);
yield OS.File.remove(tmpZipPath);
deferred.resolve();
}
catch (e) {
deferred.reject(e);
}
}));
reader.readAsArrayBuffer(req.requestBody);
req.respond(201, { "Fake-Server-Match": 1 }, "");
}
else if (req.method == "PUT" && req.url == `${davURL}zotero/${item.key}.prop`) {
var parser = Components.classes["@mozilla.org/xmlextras/domparser;1"]
.createInstance(Components.interfaces.nsIDOMParser);
var doc = parser.parseFromString(req.requestBody, "text/xml");
assert.equal(
doc.documentElement.getElementsByTagName('mtime')[0].textContent, mtime
);
assert.equal(
doc.documentElement.getElementsByTagName('hash')[0].textContent, hash
);
req.respond(204, { "Fake-Server-Match": 1 }, "");
}
});
var result = yield engine.start();
yield Zotero.Promise.all(deferreds.map(d => d.promise));
assertRequestCount(3);
assert.isTrue(result.localChanges);
assert.isTrue(result.remoteChanges);
assert.isTrue(result.syncRequired);
// Check local objects
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item.id)), mtime);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item.id)), hash);
assert.isFalse(item.synced);
})
it("should upload an updated file", function* () {
var engine = yield setup();
var file = getTestDataDirectory();
file.append('test.txt');
var item = yield Zotero.Attachments.importFromFile({ file });
item.synced = true;
yield item.saveTx();
yield Zotero.DB.executeTransaction(function* () {
// Set an mtime in the past
yield Zotero.Sync.Storage.Local.setSyncedModificationTime(
item.id,
new Date(Date.now() - 10000)
);
// And a different hash
yield Zotero.Sync.Storage.Local.setSyncedHash(
item.id, "3a2f092dd62178eb8bbfda42e07e64da"
);
});
var mtime = yield item.attachmentModificationTime;
var hash = yield item.attachmentHash;
setResponse({
method: "DELETE",
url: `zotero/${item.key}.prop`,
status: 204
});
setResponse({
method: "PUT",
url: `zotero/${item.key}.zip`,
status: 204
});
setResponse({
method: "PUT",
url: `zotero/${item.key}.prop`,
status: 204
});
var result = yield engine.start();
assertRequestCount(3);
assert.isTrue(result.localChanges);
assert.isTrue(result.remoteChanges);
assert.isTrue(result.syncRequired);
assert.isFalse(result.fileSyncRequired);
// Check local objects
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item.id)), mtime);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item.id)), hash);
assert.isFalse(item.synced);
})
it("should skip upload that already exists on the server", function* () {
var engine = yield setup();
var file = getTestDataDirectory();
file.append('test.png');
var item = yield Zotero.Attachments.importFromFile({ file });
item.synced = true;
yield item.saveTx();
var mtime = yield item.attachmentModificationTime;
var hash = yield item.attachmentHash;
var path = item.getFilePath();
var filename = 'test.png';
var size = (yield OS.File.stat(path)).size;
var contentType = 'image/png';
var fileContents = yield Zotero.File.getContentsAsync(path);
setResponse({
method: "GET",
url: `zotero/${item.key}.prop`,
status: 200,
text: '<properties version="1">'
+ `<mtime>${mtime}</mtime>`
+ `<hash>${hash}</hash>`
+ '</properties>'
});
var result = yield engine.start();
assertRequestCount(1);
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
// Check local object
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item.id)), mtime);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item.id)), hash);
assert.isFalse(item.synced);
})
it("should mark item as in conflict if mod time and hash on storage server don't match synced values", function* () {
var engine = yield setup();
var file = getTestDataDirectory();
file.append('test.png');
var item = yield Zotero.Attachments.importFromFile({ file });
item.synced = true;
yield item.saveTx();
var mtime = yield item.attachmentModificationTime;
var hash = yield item.attachmentHash;
var path = item.getFilePath();
var filename = 'test.png';
var size = (yield OS.File.stat(path)).size;
var contentType = 'image/png';
var fileContents = yield Zotero.File.getContentsAsync(path);
var newModTime = mtime + 5000;
var newHash = "4f69f43d8ac8788190b13ff7f4a0a915";
setResponse({
method: "GET",
url: `zotero/${item.key}.prop`,
status: 200,
text: '<properties version="1">'
+ `<mtime>${newModTime}</mtime>`
+ `<hash>${newHash}</hash>`
+ '</properties>'
});
var result = yield engine.start();
assertRequestCount(1);
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
assert.isTrue(result.fileSyncRequired);
// Check local object
//
// Item should be marked as in conflict
assert.equal(
(yield Zotero.Sync.Storage.Local.getSyncState(item.id)),
Zotero.Sync.Storage.Local.SYNC_STATE_IN_CONFLICT
);
// Synced mod time should have been changed, because that's what's shown in the
// conflict dialog
assert.equal(
(yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item.id)), newModTime
);
assert.isTrue(item.synced);
})
})
describe("#purgeDeletedStorageFiles()", function () {
beforeEach(function () {
resetRequestCount();
})
it("should delete files on storage server that were deleted locally", function* () {
var libraryID = Zotero.Libraries.userLibraryID;
var file = getTestDataDirectory();
file.append('test.png');
var item = yield Zotero.Attachments.importFromFile({ file });
item.synced = true;
yield item.saveTx();
yield item.eraseTx();
assert.lengthOf((yield Zotero.Sync.Storage.Local.getDeletedFiles(libraryID)), 1);
setResponse({
method: "DELETE",
url: `zotero/${item.key}.prop`,
status: 204
});
setResponse({
method: "DELETE",
url: `zotero/${item.key}.zip`,
status: 204
});
var results = yield controller.purgeDeletedStorageFiles(libraryID);
assertRequestCount(2);
assert.lengthOf(results.deleted, 2);
assert.sameMembers(results.deleted, [`${item.key}.prop`, `${item.key}.zip`]);
assert.lengthOf(results.missing, 0);
assert.lengthOf(results.error, 0);
// Storage delete log should be empty
assert.lengthOf((yield Zotero.Sync.Storage.Local.getDeletedFiles(libraryID)), 0);
})
})
describe("#purgeOrphanedStorageFiles()", function () {
beforeEach(function () {
resetRequestCount();
Zotero.Prefs.clear('lastWebDAVOrphanPurge');
})
it("should delete orphaned files more than a week older than the last sync time", function* () {
var library = Zotero.Libraries.userLibrary;
library.updateLastSyncTime();
yield library.saveTx();
const daysBeforeSyncTime = 7;
var beforeTime = new Date(Date.now() - (daysBeforeSyncTime * 86400 * 1000 + 1)).toUTCString();
var currentTime = new Date(Date.now() - 3600000).toUTCString();
setResponse({
method: "PROPFIND",
url: `zotero/`,
status: 207,
headers: {
"Content-Type": 'text/xml; charset="utf-8"'
},
text: '<?xml version="1.0" encoding="utf-8"?>'
+ '<D:multistatus xmlns:D="DAV:" xmlns:ns0="DAV:">'
+ '<D:response xmlns:lp1="DAV:" xmlns:lp2="http://apache.org/dav/props/">'
+ `<D:href>${davBasePath}zotero/</D:href>`
+ '<D:propstat>'
+ '<D:prop>'
+ `<lp1:getlastmodified>${beforeTime}</lp1:getlastmodified>`
+ '</D:prop>'
+ '<D:status>HTTP/1.1 200 OK</D:status>'
+ '</D:propstat>'
+ '</D:response>'
+ '<D:response xmlns:lp1="DAV:" xmlns:lp2="http://apache.org/dav/props/">'
+ `<D:href>${davBasePath}zotero/lastsync.txt</D:href>`
+ '<D:propstat>'
+ '<D:prop>'
+ `<lp1:getlastmodified>${beforeTime}</lp1:getlastmodified>`
+ '</D:prop>'
+ '<D:status>HTTP/1.1 200 OK</D:status>'
+ '</D:propstat>'
+ '</D:response>'
+ '<D:response xmlns:lp1="DAV:" xmlns:lp2="http://apache.org/dav/props/">'
+ `<D:href>${davBasePath}zotero/lastsync</D:href>`
+ '<D:propstat>'
+ '<D:prop>'
+ `<lp1:getlastmodified>${beforeTime}</lp1:getlastmodified>`
+ '</D:prop>'
+ '<D:status>HTTP/1.1 200 OK</D:status>'
+ '</D:propstat>'
+ '</D:response>'
+ '<D:response xmlns:lp1="DAV:" xmlns:lp2="http://apache.org/dav/props/">'
+ `<D:href>${davBasePath}zotero/AAAAAAAA.zip</D:href>`
+ '<D:propstat>'
+ '<D:prop>'
+ `<lp1:getlastmodified>${beforeTime}</lp1:getlastmodified>`
+ '</D:prop>'
+ '<D:status>HTTP/1.1 200 OK</D:status>'
+ '</D:propstat>'
+ '</D:response>'
+ '<D:response xmlns:lp1="DAV:" xmlns:lp2="http://apache.org/dav/props/">'
+ `<D:href>${davBasePath}zotero/AAAAAAAA.prop</D:href>`
+ '<D:propstat>'
+ '<D:prop>'
+ `<lp1:getlastmodified>${beforeTime}</lp1:getlastmodified>`
+ '</D:prop>'
+ '<D:status>HTTP/1.1 200 OK</D:status>'
+ '</D:propstat>'
+ '</D:response>'
+ '<D:response xmlns:lp1="DAV:" xmlns:lp2="http://apache.org/dav/props/">'
+ `<D:href>${davBasePath}zotero/BBBBBBBB.zip</D:href>`
+ '<D:propstat>'
+ '<D:prop>'
+ `<lp1:getlastmodified>${currentTime}</lp1:getlastmodified>`
+ '</D:prop>'
+ '<D:status>HTTP/1.1 200 OK</D:status>'
+ '</D:propstat>'
+ '</D:response>'
+ '<D:response xmlns:lp1="DAV:" xmlns:lp2="http://apache.org/dav/props/">'
+ `<D:href>${davBasePath}zotero/BBBBBBBB.prop</D:href>`
+ '<D:propstat>'
+ '<D:prop>'
+ `<lp1:getlastmodified>${currentTime}</lp1:getlastmodified>`
+ '</D:prop>'
+ '<D:status>HTTP/1.1 200 OK</D:status>'
+ '</D:propstat>'
+ '</D:response>'
+ '</D:multistatus>'
});
setResponse({
method: "DELETE",
url: 'zotero/AAAAAAAA.prop',
status: 204
});
setResponse({
method: "DELETE",
url: 'zotero/AAAAAAAA.zip',
status: 204
});
setResponse({
method: "DELETE",
url: 'zotero/lastsync.txt',
status: 204
});
setResponse({
method: "DELETE",
url: 'zotero/lastsync',
status: 204
});
yield controller.purgeOrphanedStorageFiles();
assertRequestCount(5);
})
it("shouldn't purge if purged recently", function* () {
Zotero.Prefs.set("lastWebDAVOrphanPurge", Math.round(new Date().getTime() / 1000) - 3600);
yield assert.eventually.equal(controller.purgeOrphanedStorageFiles(), false);
assertRequestCount(0);
})
})
})

777
test/tests/zfsTest.js Normal file
View file

@ -0,0 +1,777 @@
"use strict";
describe("Zotero.Sync.Storage.Mode.ZFS", function () {
//
// Setup
//
Components.utils.import("resource://zotero-unit/httpd.js");
var apiKey = Zotero.Utilities.randomString(24);
var port = 16213;
var baseURL = `http://localhost:${port}/`;
var win, server, requestCount;
var responses = {};
function setResponse(response) {
setHTTPResponse(server, baseURL, response, responses);
}
function resetRequestCount() {
requestCount = server.requests.filter(r => r.responseHeaders["Fake-Server-Match"]).length;
}
function assertRequestCount(count) {
assert.equal(
server.requests.filter(r => r.responseHeaders["Fake-Server-Match"]).length - requestCount,
count
);
}
function parseQueryString(str) {
var queryStringParams = str.split('&');
var params = {};
for (let param of queryStringParams) {
let [ key, val ] = param.split('=');
params[key] = decodeURIComponent(val);
}
return params;
}
function assertAPIKey(request) {
assert.equal(request.requestHeaders["Zotero-API-Key"], apiKey);
}
//
// Tests
//
beforeEach(function* () {
yield resetDB({
thisArg: this,
skipBundledFiles: true
});
win = yield loadZoteroPane();
Zotero.HTTP.mock = sinon.FakeXMLHttpRequest;
server = sinon.fakeServer.create();
server.autoRespond = true;
this.httpd = new HttpServer();
this.httpd.start(port);
yield Zotero.Users.setCurrentUserID(1);
yield Zotero.Users.setCurrentUsername("testuser");
Zotero.Sync.Storage.Local.setModeForLibrary(Zotero.Libraries.userLibraryID, 'zfs');
// Set download-on-sync by default
Zotero.Sync.Storage.Local.downloadOnSync(
Zotero.Libraries.userLibraryID, true
);
resetRequestCount();
})
var setup = Zotero.Promise.coroutine(function* (options = {}) {
Components.utils.import("resource://zotero/concurrentCaller.js");
var caller = new ConcurrentCaller(1);
caller.setLogger(msg => Zotero.debug(msg));
caller.stopOnError = true;
Components.utils.import("resource://zotero/config.js");
var client = new Zotero.Sync.APIClient({
baseURL,
apiVersion: options.apiVersion || ZOTERO_CONFIG.API_VERSION,
apiKey,
caller,
background: options.background || true
});
var engine = new Zotero.Sync.Storage.Engine({
libraryID: options.libraryID || Zotero.Libraries.userLibraryID,
controller: new Zotero.Sync.Storage.Mode.ZFS({
apiClient: client
}),
stopOnError: true
});
return { engine, client, caller };
})
afterEach(function* () {
var defer = new Zotero.Promise.defer();
this.httpd.stop(() => defer.resolve());
yield defer.promise;
win.close();
})
after(function* () {
this.timeout(60000);
//yield resetDB();
win.close();
})
describe("Syncing", function () {
it("should skip downloads if not marked as needed", function* () {
var { engine, client, caller } = yield setup();
var library = Zotero.Libraries.userLibrary;
library.libraryVersion = 5;
yield library.saveTx();
var result = yield engine.start();
assertRequestCount(0);
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
assert.equal(library.storageVersion, library.libraryVersion);
})
it("should ignore a remotely missing file", function* () {
var { engine, client, caller } = yield setup();
var library = Zotero.Libraries.userLibrary;
library.libraryVersion = 5;
yield library.saveTx();
library.storageDownloadNeeded = true;
var item = new Zotero.Item("attachment");
item.attachmentLinkMode = 'imported_file';
item.attachmentPath = 'storage:test.txt';
yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState(item.id, "to_download");
this.httpd.registerPathHandler(
`/users/1/items/${item.key}/file`,
{
handle: function (request, response) {
response.setStatusLine(null, 404, null);
}
}
);
var result = yield engine.start();
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
assert.isFalse(library.storageDownloadNeeded);
assert.equal(library.storageVersion, library.libraryVersion);
})
it("should handle a remotely failing file", function* () {
var { engine, client, caller } = yield setup();
var library = Zotero.Libraries.userLibrary;
library.libraryVersion = 5;
yield library.saveTx();
library.storageDownloadNeeded = true;
var item = new Zotero.Item("attachment");
item.attachmentLinkMode = 'imported_file';
item.attachmentPath = 'storage:test.txt';
yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState(item.id, "to_download");
this.httpd.registerPathHandler(
`/users/1/items/${item.key}/file`,
{
handle: function (request, response) {
response.setStatusLine(null, 500, null);
}
}
);
// TODO: In stopOnError mode, this the promise is rejected.
// This should probably test with stopOnError mode turned off instead.
var e = yield getPromiseError(engine.start());
assert.equal(e.message, Zotero.Sync.Storage.defaultError);
assert.isTrue(library.storageDownloadNeeded);
assert.equal(library.storageVersion, 0);
})
it("should download a missing file", function* () {
var { engine, client, caller } = yield setup();
var library = Zotero.Libraries.userLibrary;
library.libraryVersion = 5;
yield library.saveTx();
library.storageDownloadNeeded = true;
var item = new Zotero.Item("attachment");
item.attachmentLinkMode = 'imported_file';
item.attachmentPath = 'storage:test.txt';
// TODO: Test binary data
var text = Zotero.Utilities.randomString();
yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState(item.id, "to_download");
var mtime = "1441252524905";
var md5 = Zotero.Utilities.Internal.md5(text)
var s3Path = `pretend-s3/${item.key}`;
this.httpd.registerPathHandler(
`/users/1/items/${item.key}/file`,
{
handle: function (request, response) {
if (!request.hasHeader('Zotero-API-Key')) {
response.setStatusLine(null, 403, "Forbidden");
return;
}
var key = request.getHeader('Zotero-API-Key');
if (key != apiKey) {
response.setStatusLine(null, 403, "Invalid key");
return;
}
response.setStatusLine(null, 302, "Found");
response.setHeader("Zotero-File-Modification-Time", mtime, false);
response.setHeader("Zotero-File-MD5", md5, false);
response.setHeader("Zotero-File-Compressed", "No", false);
response.setHeader("Location", baseURL + s3Path, false);
}
}
);
this.httpd.registerPathHandler(
"/" + s3Path,
{
handle: function (request, response) {
response.setStatusLine(null, 200, "OK");
response.write(text);
}
}
);
var result = yield engine.start();
assert.isTrue(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
var contents = yield Zotero.File.getContentsAsync(yield item.getFilePathAsync());
assert.equal(contents, text);
assert.isFalse(library.storageDownloadNeeded);
assert.equal(library.storageVersion, library.libraryVersion);
})
it("should upload new files", function* () {
var { engine, client, caller } = yield setup();
// Single file
var file1 = getTestDataDirectory();
file1.append('test.png');
var item1 = yield Zotero.Attachments.importFromFile({ file: file1 });
var mtime1 = yield item1.attachmentModificationTime;
var hash1 = yield item1.attachmentHash;
var path1 = item1.getFilePath();
var filename1 = 'test.png';
var size1 = (yield OS.File.stat(path1)).size;
var contentType1 = 'image/png';
var prefix1 = Zotero.Utilities.randomString();
var suffix1 = Zotero.Utilities.randomString();
var uploadKey1 = Zotero.Utilities.randomString(32, 'abcdef0123456789');
// HTML file with auxiliary image
var file2 = OS.Path.join(getTestDataDirectory().path, 'snapshot', 'index.html');
var parentItem = yield createDataObject('item');
var item2 = yield Zotero.Attachments.importSnapshotFromFile({
file: file2,
url: 'http://example.com/',
parentItemID: parentItem.id,
title: 'Test',
contentType: 'text/html',
charset: 'utf-8'
});
var mtime2 = yield item2.attachmentModificationTime;
var hash2 = yield item2.attachmentHash;
var path2 = item2.getFilePath();
var filename2 = 'index.html';
var size2 = (yield OS.File.stat(path2)).size;
var contentType2 = 'text/html';
var charset2 = 'utf-8';
var prefix2 = Zotero.Utilities.randomString();
var suffix2 = Zotero.Utilities.randomString();
var uploadKey2 = Zotero.Utilities.randomString(32, 'abcdef0123456789');
var deferreds = [];
// https://github.com/cjohansen/Sinon.JS/issues/607
let fixSinonBug = ";charset=utf-8";
server.respond(function (req) {
// Get upload authorization for single file
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item1.key}/file`
&& req.requestBody.indexOf('upload=') == -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
let parts = req.requestBody.split('&');
let params = {};
for (let part of parts) {
let [key, val] = part.split('=');
params[key] = decodeURIComponent(val);
}
assert.equal(params.md5, hash1);
assert.equal(params.mtime, mtime1);
assert.equal(params.filename, filename1);
assert.equal(params.filesize, size1);
assert.equal(params.contentType, contentType1);
req.respond(
200,
{
"Content-Type": "application/json"
},
JSON.stringify({
url: baseURL + "pretend-s3/1",
contentType: contentType1,
prefix: prefix1,
suffix: suffix1,
uploadKey: uploadKey1
})
);
}
// Get upload authorization for multi-file zip
else if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item2.key}/file`
&& req.requestBody.indexOf('upload=') == -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
// Verify ZIP hash
let tmpZipPath = OS.Path.join(
Zotero.getTempDirectory().path,
item2.key + '.zip'
);
deferreds.push({
promise: Zotero.Utilities.Internal.md5Async(tmpZipPath)
.then(function (md5) {
assert.equal(params.zipMD5, md5);
})
});
let parts = req.requestBody.split('&');
let params = {};
for (let part of parts) {
let [key, val] = part.split('=');
params[key] = decodeURIComponent(val);
}
Zotero.debug(params);
assert.equal(params.md5, hash2);
assert.notEqual(params.zipMD5, hash2);
assert.equal(params.mtime, mtime2);
assert.equal(params.filename, filename2);
assert.equal(params.zipFilename, item2.key + ".zip");
assert.isTrue(parseInt(params.filesize) == params.filesize);
assert.equal(params.contentType, contentType2);
assert.equal(params.charset, charset2);
req.respond(
200,
{
"Content-Type": "application/json"
},
JSON.stringify({
url: baseURL + "pretend-s3/2",
contentType: 'application/zip',
prefix: prefix2,
suffix: suffix2,
uploadKey: uploadKey2
})
);
}
// Upload single file to S3
else if (req.method == "POST" && req.url == baseURL + "pretend-s3/1") {
assert.equal(req.requestHeaders["Content-Type"], contentType1 + fixSinonBug);
assert.equal(req.requestBody.size, (new Blob([prefix1, File(file1), suffix1]).size));
req.respond(201, {}, "");
}
// Upload multi-file ZIP to S3
else if (req.method == "POST" && req.url == baseURL + "pretend-s3/2") {
assert.equal(req.requestHeaders["Content-Type"], "application/zip" + fixSinonBug);
// Verify uploaded ZIP file
let tmpZipPath = OS.Path.join(
Zotero.getTempDirectory().path,
Zotero.Utilities.randomString() + '.zip'
);
let deferred = Zotero.Promise.defer();
deferreds.push(deferred);
var reader = new FileReader();
reader.addEventListener("loadend", Zotero.Promise.coroutine(function* () {
try {
let file = yield OS.File.open(tmpZipPath, {
create: true
});
var contents = new Uint8Array(reader.result);
contents = contents.slice(prefix2.length, suffix2.length * -1);
yield file.write(contents);
yield file.close();
var zr = Components.classes["@mozilla.org/libjar/zip-reader;1"]
.createInstance(Components.interfaces.nsIZipReader);
zr.open(Zotero.File.pathToFile(tmpZipPath));
zr.test(null);
var entries = zr.findEntries('*');
var entryNames = [];
while (entries.hasMore()) {
entryNames.push(entries.getNext());
}
assert.equal(entryNames.length, 2);
assert.sameMembers(entryNames, ['index.html', 'img.gif']);
assert.equal(zr.getEntry('index.html').realSize, size2);
assert.equal(zr.getEntry('img.gif').realSize, 42);
deferred.resolve();
}
catch (e) {
deferred.reject(e);
}
}));
reader.readAsArrayBuffer(req.requestBody);
req.respond(201, {}, "");
}
// Register single-file upload
else if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item1.key}/file`
&& req.requestBody.indexOf('upload=') != -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
let parts = req.requestBody.split('&');
let params = {};
for (let part of parts) {
let [key, val] = part.split('=');
params[key] = decodeURIComponent(val);
}
assert.equal(params.upload, uploadKey1);
req.respond(
204,
{
"Last-Modified-Version": 10
},
""
);
}
// Register multi-file upload
else if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item2.key}/file`
&& req.requestBody.indexOf('upload=') != -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
let parts = req.requestBody.split('&');
let params = {};
for (let part of parts) {
let [key, val] = part.split('=');
params[key] = decodeURIComponent(val);
}
assert.equal(params.upload, uploadKey2);
req.respond(
204,
{
"Last-Modified-Version": 15
},
""
);
}
})
// TODO: One-step uploads
/*// https://github.com/cjohansen/Sinon.JS/issues/607
let fixSinonBug = ";charset=utf-8";
server.respond(function (req) {
if (req.method == "POST" && req.url == `${baseURL}users/1/items/${item.key}/file`) {
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/json" + fixSinonBug
);
let params = JSON.parse(req.requestBody);
assert.equal(params.md5, hash);
assert.equal(params.mtime, mtime);
assert.equal(params.filename, filename);
assert.equal(params.size, size);
assert.equal(params.contentType, contentType);
req.respond(
200,
{
"Content-Type": "application/json"
},
JSON.stringify({
url: baseURL + "pretend-s3",
headers: {
"Content-Type": contentType,
"Content-MD5": hash,
//"Content-Length": params.size, process but don't return
//"x-amz-meta-"
},
uploadKey
})
);
}
else if (req.method == "PUT" && req.url == baseURL + "pretend-s3") {
assert.equal(req.requestHeaders["Content-Type"], contentType + fixSinonBug);
assert.instanceOf(req.requestBody, File);
req.respond(201, {}, "");
}
})*/
var result = yield engine.start();
yield Zotero.Promise.all(deferreds.map(d => d.promise));
assert.isTrue(result.localChanges);
assert.isTrue(result.remoteChanges);
assert.isFalse(result.syncRequired);
// Check local objects
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item1.id)), mtime1);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item1.id)), hash1);
assert.equal(item1.version, 10);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item2.id)), mtime2);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item2.id)), hash2);
assert.equal(item2.version, 15);
})
it("should update local info for file that already exists on the server", function* () {
var { engine, client, caller } = yield setup();
var file = getTestDataDirectory();
file.append('test.png');
var item = yield Zotero.Attachments.importFromFile({ file: file });
item.version = 5;
yield item.saveTx();
var json = yield item.toJSON();
yield Zotero.Sync.Data.Local.saveCacheObject('item', item.libraryID, json);
var mtime = yield item.attachmentModificationTime;
var hash = yield item.attachmentHash;
var path = item.getFilePath();
var filename = 'test.png';
var size = (yield OS.File.stat(path)).size;
var contentType = 'image/png';
var newVersion = 10;
// https://github.com/cjohansen/Sinon.JS/issues/607
let fixSinonBug = ";charset=utf-8";
server.respond(function (req) {
// Get upload authorization for single file
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item.key}/file`
&& req.requestBody.indexOf('upload=') == -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
req.respond(
200,
{
"Content-Type": "application/json",
"Last-Modified-Version": newVersion
},
JSON.stringify({
exists: 1,
})
);
}
})
// TODO: One-step uploads
var result = yield engine.start();
assert.isTrue(result.localChanges);
assert.isTrue(result.remoteChanges);
assert.isFalse(result.syncRequired);
// Check local objects
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item.id)), mtime);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item.id)), hash);
assert.equal(item.version, newVersion);
})
})
describe("#_processUploadFile()", function () {
it("should handle 412 with matching version and hash matching local file", function* () {
var { engine, client, caller } = yield setup();
var zfs = new Zotero.Sync.Storage.Mode.ZFS({
apiClient: client
})
var filePath = OS.Path.join(getTestDataDirectory().path, 'test.png');
var item = yield Zotero.Attachments.importFromFile({ file: filePath });
item.version = 5;
item.synced = true;
yield item.saveTx();
var itemJSON = yield item.toResponseJSON();
itemJSON.data.mtime = yield item.attachmentModificationTime;
itemJSON.data.md5 = yield item.attachmentHash;
// Set saved hash to a different value, which should be overwritten
//
// We're also testing cases where a hash isn't set for a file (e.g., if the
// storage directory was transferred, the mtime doesn't match, but the file was
// never downloaded), but there's no difference in behavior
var dbHash = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa';
yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, dbHash)
});
server.respond(function (req) {
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item.key}/file`
&& req.requestBody.indexOf('upload=') == -1
&& req.requestHeaders["If-Match"] == dbHash) {
req.respond(
412,
{
"Content-Type": "application/json",
"Last-Modified-Version": 5
},
"ETag does not match current version of file"
);
}
})
setResponse({
method: "GET",
url: `users/1/items?format=json&itemKey=${item.key}&includeTrashed=1`,
status: 200,
text: JSON.stringify([itemJSON])
});
var result = yield zfs._processUploadFile({
name: item.libraryKey
});
yield assert.eventually.equal(
Zotero.Sync.Storage.Local.getSyncedHash(item.id),
(yield item.attachmentHash)
);
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
assert.isFalse(result.fileSyncRequired);
})
it("should handle 412 with matching version and hash not matching local file", function* () {
var { engine, client, caller } = yield setup();
var zfs = new Zotero.Sync.Storage.Mode.ZFS({
apiClient: client
})
var filePath = OS.Path.join(getTestDataDirectory().path, 'test.png');
var item = yield Zotero.Attachments.importFromFile({ file: filePath });
item.version = 5;
item.synced = true;
yield item.saveTx();
var fileHash = yield item.attachmentHash;
var itemJSON = yield item.toResponseJSON();
itemJSON.data.md5 = 'aaaaaaaaaaaaaaaaaaaaaaaa'
server.respond(function (req) {
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item.key}/file`
&& req.requestBody.indexOf('upload=') == -1
&& req.requestHeaders["If-None-Match"] == "*") {
req.respond(
412,
{
"Content-Type": "application/json",
"Last-Modified-Version": 5
},
"If-None-Match: * set but file exists"
);
}
})
setResponse({
method: "GET",
url: `users/1/items?format=json&itemKey=${item.key}&includeTrashed=1`,
status: 200,
text: JSON.stringify([itemJSON])
});
var result = yield zfs._processUploadFile({
name: item.libraryKey
});
yield assert.eventually.isNull(Zotero.Sync.Storage.Local.getSyncedHash(item.id));
yield assert.eventually.equal(
Zotero.Sync.Storage.Local.getSyncState(item.id),
Zotero.Sync.Storage.Local.SYNC_STATE_IN_CONFLICT
);
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
assert.isTrue(result.fileSyncRequired);
})
it("should handle 412 with greater version", function* () {
var { engine, client, caller } = yield setup();
var zfs = new Zotero.Sync.Storage.Mode.ZFS({
apiClient: client
})
var file = getTestDataDirectory();
file.append('test.png');
var item = yield Zotero.Attachments.importFromFile({ file });
item.version = 5;
item.synced = true;
yield item.saveTx();
server.respond(function (req) {
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item.key}/file`
&& req.requestBody.indexOf('upload=') == -1
&& req.requestHeaders["If-None-Match"] == "*") {
req.respond(
412,
{
"Content-Type": "application/json",
"Last-Modified-Version": 10
},
"If-None-Match: * set but file exists"
);
}
})
var result = yield zfs._processUploadFile({
name: item.libraryKey
});
assert.equal(item.version, 5);
assert.equal(item.synced, true);
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isTrue(result.syncRequired);
})
})
})

View file

@ -148,9 +148,7 @@ describe("ZoteroPane", function() {
// TODO: Test binary data
var text = Zotero.Utilities.randomString();
yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
);
yield Zotero.Sync.Storage.Local.setSyncState(item.id, "to_download");
var mtime = "1441252524000";
var md5 = Zotero.Utilities.Internal.md5(text)