diff --git a/chrome/content/zotero/preferences/preferences_sync.js b/chrome/content/zotero/preferences/preferences_sync.js index ccb607d20a..15f1679a9d 100644 --- a/chrome/content/zotero/preferences/preferences_sync.js +++ b/chrome/content/zotero/preferences/preferences_sync.js @@ -93,21 +93,23 @@ Zotero_Preferences.Sync = { var sql = "INSERT OR IGNORE INTO settings VALUES (?,?,?)"; Zotero.DB.query(sql, ['storage', 'zfsPurge', 'user']); - Zotero.Sync.Storage.ZFS.purgeDeletedStorageFiles(function (success) { - if (success) { - ps.alert( - null, - Zotero.getString("general.success"), - "Attachment files from your personal library have been removed from the Zotero servers." - ); - } - else { - ps.alert( - null, - Zotero.getString("general.error"), - "An error occurred. Please try again later." - ); - } + Zotero.Sync.Storage.ZFS.purgeDeletedStorageFiles() + .then(function () { + ps.alert( + null, + Zotero.getString("general.success"), + "Attachment files from your personal library have been removed from the Zotero servers." + ); + }) + .catch(function (e) { + Zotero.debug(e, 1); + Components.utils.reportError(e); + + ps.alert( + null, + Zotero.getString("general.error"), + "An error occurred. Please try again later." + ); }); } } diff --git a/chrome/content/zotero/xpcom/storage.js b/chrome/content/zotero/xpcom/storage.js index d41045a55b..2f39129f43 100644 --- a/chrome/content/zotero/xpcom/storage.js +++ b/chrome/content/zotero/xpcom/storage.js @@ -326,6 +326,24 @@ Zotero.Sync.Storage = new function () { Zotero.debug("File sync failed for library " + libraryID); finalPromises.push([libraryID, libraryQueues]); } + + // If WebDAV sync enabled, purge deleted and orphaned files + if (libraryID == 0 && Zotero.Sync.Storage.WebDAV.includeUserFiles) { + Zotero.Sync.Storage.WebDAV.purgeDeletedStorageFiles() + .then(function () { + return Zotero.Sync.Storage.WebDAV.purgeOrphanedStorageFiles(); + }) + .catch(function (e) { + Zotero.debug(e, 1); + Components.utils.reportError(e); + }); + } + }); + + Zotero.Sync.Storage.ZFS.purgeDeletedStorageFiles() + .catch(function (e) { + Zotero.debug(e, 1); + Components.utils.reportError(e); }); if (promises.length && !changedLibraries.length) { @@ -1755,20 +1773,11 @@ Zotero.Sync.Storage = new function () { /** * @inner - * @param {Integer} [days=pref:e.z.sync.storage.deleteDelayDays] - * Number of days old files have to be * @return {String[]|FALSE} Array of keys, or FALSE if none */ - this.getDeletedFiles = function (days) { - if (!days) { - days = Zotero.Prefs.get("sync.storage.deleteDelayDays"); - } - - var ts = Zotero.Date.getUnixTimestamp(); - ts = ts - (86400 * days); - - var sql = "SELECT key FROM storageDeleteLog WHERE timestamp i) { - delete files[propIndex]; - i--; - last = (i == files.length - 1); - } + let propFileName = deletePropURI.fileName; // Delete property file return Zotero.HTTP.promise("DELETE", deletePropURI, { successCodes: [200, 204, 404] }) @@ -534,29 +529,40 @@ Zotero.Sync.Storage.WebDAV = (function () { case 204: // IIS 5.1 and Sakai return 200 case 200: - results.deleted.push(fileName); + results.deleted.push(baseName); break; case 404: if (fileDeleted) { - results.deleted.push(fileName); + results.deleted.push(baseName); } else { - results.missing.push(fileName); + results.missing.push(baseName); } break; } }); }) .catch(function (e) { - results.error.push(fileName); - var msg = "An error occurred attempting to delete " - + "'" + fileName - + "' (" + e.status + " " + e.xmlhttp.statusText + ")."; + results.error.push(baseName); + throw e; }); }); + } + + Components.utils.import("resource://zotero/concurrent-caller.js"); + var caller = new ConcurrentCaller(4); + caller.stopOnError = true; + caller.setLogger(function (msg) { + Zotero.debug("[ConcurrentCaller] " + msg); + }); + caller.setErrorLogger(function (msg) { + Components.utils.reportError(msg); + }); + return caller.fcall(funcs) + .then(function () { + return results; }); - return results; } @@ -881,7 +887,8 @@ Zotero.Sync.Storage.WebDAV = (function () { deleteStorageFiles([item.key + ".prop"]) .finally(function (results) { deferred.resolve(false); - }); + }) + .done(); return; } else if (status != 200) { @@ -1457,182 +1464,199 @@ Zotero.Sync.Storage.WebDAV = (function () { /** - * Remove files on storage server that were deleted locally more than - * sync.storage.deleteDelayDays days ago + * Remove files on storage server that were deleted locally * * @param {Function} callback Passed number of files deleted */ obj._purgeDeletedStorageFiles = function () { - if (!this._active) { - return Q(false); - } - - Zotero.debug("Purging deleted storage files"); - var files = Zotero.Sync.Storage.getDeletedFiles(); - if (!files) { - Zotero.debug("No files to delete remotely"); - return Q(false); - } - - // Add .zip extension - var files = files.map(function (file) file + ".zip"); - - return deleteStorageFiles(files) - .then(function (results) { - // Remove deleted and nonexistent files from storage delete log - var toPurge = results.deleted.concat(results.missing); - if (toPurge.length > 0) { - var done = 0; - var maxFiles = 999; - var numFiles = toPurge.length; - - Zotero.DB.beginTransaction(); - - do { - var chunk = toPurge.splice(0, maxFiles); - var sql = "DELETE FROM storageDeleteLog WHERE key IN (" - + chunk.map(function () '?').join() + ")"; - Zotero.DB.query(sql, chunk); - done += chunk.length; - } - while (done < numFiles); - - Zotero.DB.commitTransaction(); + return Q.fcall(function () { + if (!this.includeUserFiles) { + return false; } - return results.deleted.length; - }); + Zotero.debug("Purging deleted storage files"); + var files = Zotero.Sync.Storage.getDeletedFiles(); + if (!files) { + Zotero.debug("No files to delete remotely"); + return false; + } + + // Add .zip extension + var files = files.map(function (file) file + ".zip"); + + return deleteStorageFiles(files) + .then(function (results) { + // Remove deleted and nonexistent files from storage delete log + var toPurge = results.deleted.concat(results.missing); + if (toPurge.length > 0) { + var done = 0; + var maxFiles = 999; + var numFiles = toPurge.length; + + Zotero.DB.beginTransaction(); + + do { + var chunk = toPurge.splice(0, maxFiles); + var sql = "DELETE FROM storageDeleteLog WHERE key IN (" + + chunk.map(function () '?').join() + ")"; + Zotero.DB.query(sql, chunk); + done += chunk.length; + } + while (done < numFiles); + + Zotero.DB.commitTransaction(); + } + + Zotero.debug(results); + + return results.deleted.length; + }); + }.bind(this)); }; /** * Delete orphaned storage files older than a day before last sync time - * - * @param {Function} callback */ - obj._purgeOrphanedStorageFiles = function (callback) { - const daysBeforeSyncTime = 1; - - if (!this._active) { - return false; - } - - // If recently purged, skip - var lastpurge = Zotero.Prefs.get('lastWebDAVOrphanPurge'); - var days = 10; - if (lastpurge && new Date(lastpurge * 1000) > (new Date() - (1000 * 60 * 60 * 24 * days))) { - return false; - } - - Zotero.debug("Purging orphaned storage files"); - - var uri = this.rootURI; - var path = uri.path; - - var xmlstr = "" - + "" - + ""; - - var lastSyncDate = new Date(Zotero.Sync.Server.lastLocalSyncTime * 1000); - - Zotero.HTTP.WebDAV.doProp("PROPFIND", uri, xmlstr, function (req) { - Zotero.debug(req.responseText); - - var funcName = "Zotero.Sync.Storage.purgeOrphanedStorageFiles()"; + obj._purgeOrphanedStorageFiles = function () { + return Q.fcall(function () { + const daysBeforeSyncTime = 1; - var responseNode = req.responseXML.documentElement; - responseNode.xpath = function (path) { - return Zotero.Utilities.xpath(this, path, { D: 'DAV:' }); - }; - - var deleteFiles = []; - var trailingSlash = !!path.match(/\/$/); - for each(var response in responseNode.xpath("response")) { - var href = Zotero.Utilities.xpath(response, "href", { D: 'DAV:' }); - href = href.length ? href[0] : '' - - // Strip trailing slash if there isn't one on the root path - if (!trailingSlash) { - href = href.replace(/\/$/, "") - } - - // Absolute - if (href.match(/^https?:\/\//)) { - var ios = Components.classes["@mozilla.org/network/io-service;1"]. - getService(Components.interfaces.nsIIOService); - var href = ios.newURI(href, null, null); - href = href.path; - } - - // Skip root URI - if (href == path - // Some Apache servers respond with a "/zotero" href - // even for a "/zotero/" request - || (trailingSlash && href + '/' == path) - // Try URL-encoded as well, as above - || decodeURIComponent(href) == path) { - continue; - } - - if (href.indexOf(path) == -1 - // Try URL-encoded as well, in case there's a '~' or similar - // character in the URL and the server (e.g., Sakai) is - // encoding the value - && decodeURIComponent(href).indexOf(path) == -1) { - Zotero.Sync.Storage.EventManager.error( - "DAV:href '" + href + "' does not begin with path '" - + path + "' in " + funcName - ); - } - - var matches = href.match(/[^\/]+$/); - if (!matches) { - Zotero.Sync.Storage.EventManager.error( - "Unexpected href '" + href + "' in " + funcName - ) - } - var file = matches[0]; - - if (file.indexOf('.') == 0) { - Zotero.debug("Skipping hidden file " + file); - continue; - } - if (!file.match(/\.zip$/) && !file.match(/\.prop$/)) { - Zotero.debug("Skipping file " + file); - continue; - } - - var key = file.replace(/\.(zip|prop)$/, ''); - var item = Zotero.Items.getByLibraryAndKey(null, key); - if (item) { - Zotero.debug("Skipping existing file " + file); - continue; - } - - Zotero.debug("Checking orphaned file " + file); - - // TODO: Parse HTTP date properly - var lastModified = Zotero.Utilities.xpath( - response, "//getlastmodified", { D: 'DAV:' } - ); - lastModified = lastModified.length ? lastModified[0] : '' - lastModified = Zotero.Date.strToISO(lastModified); - lastModified = Zotero.Date.sqlToDate(lastModified); - - // Delete files older than a day before last sync time - var days = (lastSyncDate - lastModified) / 1000 / 60 / 60 / 24; - - if (days > daysBeforeSyncTime) { - deleteFiles.push(file); - } + if (!this.includeUserFiles) { + return false; } - deleteStorageFiles(deleteFiles) - .then(function (results) { - Zotero.Prefs.set("lastWebDAVOrphanPurge", Math.round(new Date().getTime() / 1000)) - Zotero.debug(results); - }); - }, { Depth: 1 }); + // If recently purged, skip + var lastpurge = Zotero.Prefs.get('lastWebDAVOrphanPurge'); + var days = 10; + if (lastpurge && new Date(lastpurge * 1000) > (new Date() - (1000 * 60 * 60 * 24 * days))) { + return false; + } + + Zotero.debug("Purging orphaned storage files"); + + var uri = this.rootURI; + var path = uri.path; + + var xmlstr = "" + + "" + + ""; + + var lastSyncDate = new Date(Zotero.Sync.Server.lastLocalSyncTime * 1000); + + var deferred = Q.defer(); + + Zotero.HTTP.WebDAV.doProp("PROPFIND", uri, xmlstr, function (xmlhttp) { + Q.fcall(function () { + Zotero.debug(xmlhttp.responseText); + + var funcName = "Zotero.Sync.Storage.purgeOrphanedStorageFiles()"; + + var responseNode = xmlhttp.responseXML.documentElement; + responseNode.xpath = function (path) { + return Zotero.Utilities.xpath(this, path, { D: 'DAV:' }); + }; + + var deleteFiles = []; + var trailingSlash = !!path.match(/\/$/); + for each(var response in responseNode.xpath("D:response")) { + var href = Zotero.Utilities.xpathText( + response, "D:href", { D: 'DAV:' } + ) || ""; + Zotero.debug(href); + + // Strip trailing slash if there isn't one on the root path + if (!trailingSlash) { + href = href.replace(/\/$/, ""); + } + + // Absolute + if (href.match(/^https?:\/\//)) { + var ios = Components.classes["@mozilla.org/network/io-service;1"]. + getService(Components.interfaces.nsIIOService); + var href = ios.newURI(href, null, null); + href = href.path; + } + + // Skip root URI + if (href == path + // Some Apache servers respond with a "/zotero" href + // even for a "/zotero/" request + || (trailingSlash && href + '/' == path) + // Try URL-encoded as well, as above + || decodeURIComponent(href) == path) { + continue; + } + + if (href.indexOf(path) == -1 + // Try URL-encoded as well, in case there's a '~' or similar + // character in the URL and the server (e.g., Sakai) is + // encoding the value + && decodeURIComponent(href).indexOf(path) == -1) { + throw new Error( + "DAV:href '" + href + "' does not begin with path '" + + path + "' in " + funcName + ); + } + + var matches = href.match(/[^\/]+$/); + if (!matches) { + throw new Error( + "Unexpected href '" + href + "' in " + funcName + ); + } + var file = matches[0]; + + if (file.indexOf('.') == 0) { + Zotero.debug("Skipping hidden file " + file); + continue; + } + if (!file.match(/\.zip$/) && !file.match(/\.prop$/)) { + Zotero.debug("Skipping file " + file); + continue; + } + + var key = file.replace(/\.(zip|prop)$/, ''); + var item = Zotero.Items.getByLibraryAndKey(null, key); + if (item) { + Zotero.debug("Skipping existing file " + file); + continue; + } + + Zotero.debug("Checking orphaned file " + file); + + // TODO: Parse HTTP date properly + Zotero.debug(response.innerHTML); + var lastModified = Zotero.Utilities.xpathText( + response, ".//D:getlastmodified", { D: 'DAV:' } + ); + lastModified = Zotero.Date.strToISO(lastModified); + lastModified = Zotero.Date.sqlToDate(lastModified); + + // Delete files older than a day before last sync time + var days = (lastSyncDate - lastModified) / 1000 / 60 / 60 / 24; + + if (days > daysBeforeSyncTime) { + deleteFiles.push(file); + } + } + + return deleteStorageFiles(deleteFiles) + .then(function (results) { + Zotero.Prefs.set("lastWebDAVOrphanPurge", Math.round(new Date().getTime() / 1000)) + Zotero.debug(results); + }); + }) + .catch(function (e) { + deferred.reject(e); + }) + .then(function () { + deferred.resolve(); + }); + }, { Depth: 1 }); + + return deferred.promise; + }.bind(this)); }; return obj; diff --git a/chrome/content/zotero/xpcom/storage/zfs.js b/chrome/content/zotero/xpcom/storage/zfs.js index e32c1d8171..a93c4c784d 100644 --- a/chrome/content/zotero/xpcom/storage/zfs.js +++ b/chrome/content/zotero/xpcom/storage/zfs.js @@ -1006,7 +1006,7 @@ Zotero.Sync.Storage.ZFS = (function () { Zotero.debug("Credentials are cached"); _cachedCredentials = true; }) - .fail(function (e) { + .catch(function (e) { if (e instanceof Zotero.HTTP.UnexpectedStatusException) { if (e.status == 401) { var msg = "File sync login failed\n\n" @@ -1030,57 +1030,49 @@ Zotero.Sync.Storage.ZFS = (function () { /** * Remove all synced files from the server */ - obj._purgeDeletedStorageFiles = function (callback) { - // If we don't have a user id we've never synced and don't need to bother - if (!Zotero.userID) { - return false; - } - - var sql = "SELECT value FROM settings WHERE setting=? AND key=?"; - var values = Zotero.DB.columnQuery(sql, ['storage', 'zfsPurge']); - if (!values) { - return false; - } - - // TODO: promisify - - Zotero.debug("Unlinking synced files on ZFS"); - - var uri = this.userURI; - uri.spec += "removestoragefiles?"; - // Unused - for each(var value in values) { - switch (value) { - case 'user': - uri.spec += "user=1&"; - break; - - case 'group': - uri.spec += "group=1&"; - break; - - default: - throw "Invalid zfsPurge value '" + value - + "' in ZFS purgeDeletedStorageFiles()"; + obj._purgeDeletedStorageFiles = function () { + return Q.fcall(function () { + // If we don't have a user id we've never synced and don't need to bother + if (!Zotero.userID) { + return false; } - } - uri.spec = uri.spec.substr(0, uri.spec.length - 1); - - Zotero.HTTP.doPost(uri, "", function (xmlhttp) { - if (xmlhttp.status != 204) { - if (callback) { - callback(false); + + var sql = "SELECT value FROM settings WHERE setting=? AND key=?"; + var values = Zotero.DB.columnQuery(sql, ['storage', 'zfsPurge']); + if (!values) { + return false; + } + + // TODO: promisify + + Zotero.debug("Unlinking synced files on ZFS"); + + var uri = this.userURI; + uri.spec += "removestoragefiles?"; + // Unused + for each(var value in values) { + switch (value) { + case 'user': + uri.spec += "user=1&"; + break; + + case 'group': + uri.spec += "group=1&"; + break; + + default: + throw "Invalid zfsPurge value '" + value + + "' in ZFS purgeDeletedStorageFiles()"; } - throw "Unexpected status code " + xmlhttp.status + " purging ZFS files"; } + uri.spec = uri.spec.substr(0, uri.spec.length - 1); - var sql = "DELETE FROM settings WHERE setting=? AND key=?"; - Zotero.DB.query(sql, ['storage', 'zfsPurge']); - - if (callback) { - callback(true); - } - }); + return Zotero.HTTP.promise("POST", uri, "") + .then(function (req) { + var sql = "DELETE FROM settings WHERE setting=? AND key=?"; + Zotero.DB.query(sql, ['storage', 'zfsPurge']); + }); + }.bind(this)); }; return obj; diff --git a/chrome/content/zotero/xpcom/sync.js b/chrome/content/zotero/xpcom/sync.js index 71180f5dac..3e9161afa1 100644 --- a/chrome/content/zotero/xpcom/sync.js +++ b/chrome/content/zotero/xpcom/sync.js @@ -429,7 +429,7 @@ Zotero.Sync.EventListener = new function () { var sql = "REPLACE INTO syncDeleteLog VALUES (?, ?, ?, ?)"; var syncStatement = Zotero.DB.getStatement(sql); - if (isItem && Zotero.Sync.Storage.WebDAV.active) { + if (isItem && Zotero.Sync.Storage.WebDAV.includeUserFiles) { var storageEnabled = true; var sql = "INSERT INTO storageDeleteLog VALUES (?, ?, ?)"; var storageStatement = Zotero.DB.getStatement(sql); diff --git a/chrome/content/zotero/xpcom/zotero.js b/chrome/content/zotero/xpcom/zotero.js index 2781cedf99..75e012d8cd 100644 --- a/chrome/content/zotero/xpcom/zotero.js +++ b/chrome/content/zotero/xpcom/zotero.js @@ -1844,15 +1844,6 @@ Components.utils.import("resource://gre/modules/Services.jsm"); Zotero.Items.purge(); // DEBUG: this might not need to be permanent Zotero.Relations.purge(); - - if (!skipStoragePurge && Math.random() < 1/10) { - Zotero.Sync.Storage.ZFS.purgeDeletedStorageFiles(); - Zotero.Sync.Storage.WebDAV.purgeDeletedStorageFiles(); - } - - if (!skipStoragePurge) { - Zotero.Sync.Storage.WebDAV.purgeOrphanedStorageFiles(); - } }