Fix WebDAV file purging

Deleted files are purged at the end of every sync, without any delay.
(If there's a conflict, it will be resolved before the file is deleted.)

Orphaned files are deleted once every 10 days, since it's a potentially
expensive operation for the server.

chrome/content/zotero/preferences/preferences_sync.js

@@ -93,21 +93,23 @@ Zotero_Preferences.Sync = {
 					var sql = "INSERT OR IGNORE INTO settings VALUES (?,?,?)";
 					Zotero.DB.query(sql, ['storage', 'zfsPurge', 'user']);
 					
-					Zotero.Sync.Storage.ZFS.purgeDeletedStorageFiles(function (success) {
+					Zotero.Sync.Storage.ZFS.purgeDeletedStorageFiles()
-						if (success) {
+					.then(function () {
 						ps.alert(
 							null,
 							Zotero.getString("general.success"),
 							"Attachment files from your personal library have been removed from the Zotero servers."
 						);
-						}
+					})
-						else {
+					.catch(function (e) {
+						Zotero.debug(e, 1);
+						Components.utils.reportError(e);
+						
 						ps.alert(
 							null,
 							Zotero.getString("general.error"),
 							"An error occurred. Please try again later."
 						);
-						}
 					});
 				}
 			}

chrome/content/zotero/xpcom/storage.js

@@ -326,6 +326,24 @@ Zotero.Sync.Storage = new function () {
 					Zotero.debug("File sync failed for library " + libraryID);
 					finalPromises.push([libraryID, libraryQueues]);
 				}
+				
+				// If WebDAV sync enabled, purge deleted and orphaned files
+				if (libraryID == 0 && Zotero.Sync.Storage.WebDAV.includeUserFiles) {
+					Zotero.Sync.Storage.WebDAV.purgeDeletedStorageFiles()
+					.then(function () {
+						return Zotero.Sync.Storage.WebDAV.purgeOrphanedStorageFiles();
+					})
+					.catch(function (e) {
+						Zotero.debug(e, 1);
+						Components.utils.reportError(e);
+					});
+				}
+			});
+			
+			Zotero.Sync.Storage.ZFS.purgeDeletedStorageFiles()
+			.catch(function (e) {
+				Zotero.debug(e, 1);
+				Components.utils.reportError(e);
 			});
 			
 			if (promises.length && !changedLibraries.length) {
@@ -1755,20 +1773,11 @@ Zotero.Sync.Storage = new function () {
 	
 	/**
 	 * @inner
-	 * @param	{Integer}	[days=pref:e.z.sync.storage.deleteDelayDays]
-	 *									Number of days old files have to be
 	 * @return	{String[]|FALSE}			Array of keys, or FALSE if none
 	 */
-	this.getDeletedFiles = function (days) {
+	this.getDeletedFiles = function () {
-		if (!days) {
+		var sql = "SELECT key FROM storageDeleteLog";
-			days = Zotero.Prefs.get("sync.storage.deleteDelayDays");
+		return Zotero.DB.columnQuery(sql);
-		}
-		
-		var ts = Zotero.Date.getUnixTimestamp();
-		ts = ts - (86400 * days);
-		
-		var sql = "SELECT key FROM storageDeleteLog WHERE timestamp<?";
-		return Zotero.DB.columnQuery(sql, ts);
 	}
 	
 	

chrome/content/zotero/xpcom/storage/webdav.js

@@ -473,21 +473,21 @@ Zotero.Sync.Storage.WebDAV = (function () {
 		};
 		
 		if (files.length == 0) {
-			return Q.resolve(results);
+			return Q(results);
 		}
 		
 		let deleteURI = _rootURI.clone();
 		// This should never happen, but let's be safe
 		if (!deleteURI.spec.match(/\/$/)) {
-			throw new Error(
+			return Q.reject("Root URI does not end in slash in "
-				"Root URI does not end in slash in "
+				+ "Zotero.Sync.Storage.WebDAV.deleteStorageFiles()");
-				+ "Zotero.Sync.Storage.WebDAV.deleteStorageFiles()"
-			);
 		}
 		
-		results = Q.resolve(results);
+		var funcs = [];
-		files.forEach(function (fileName) {
+		for (let i=0; i<files.length; i++) {
-			results = results.then(function (results) {
+			let fileName = files[i];
+			let baseName = fileName.match(/^([^\.]+)/)[1];
+			funcs.push(function () {
 				let deleteURI = _rootURI.clone();
 				deleteURI.QueryInterface(Components.interfaces.nsIURL);
 				deleteURI.fileName = fileName;
@@ -502,30 +502,25 @@ Zotero.Sync.Storage.WebDAV = (function () {
 							break;
 						
 						case 404:
-							var fileDeleted = false;
+							var fileDeleted = true;
 							break;
 					}
 					
 					// If an item file URI, get the property URI
 					var deletePropURI = getPropertyURIFromItemURI(deleteURI);
-					if (!deletePropURI) {
+					
+					// If we already deleted the prop file, skip it
+					if (!deletePropURI || results.deleted.indexOf(deletePropURI.fileName) != -1) {
 						if (fileDeleted) {
-							results.deleted.push(fileName);
+							results.deleted.push(baseName);
 						}
 						else {
-							results.missing.push(fileName);
+							results.missing.push(baseName);
 						}
-						return results;
+						return;
 					}
 					
-					// If property file appears separately in delete queue,
+					let propFileName = deletePropURI.fileName;
-					// remove it, since we're taking care of it here
-					var propIndex = files.indexOf(deletePropURI.fileName);
-					if (propIndex > i) {
-						delete files[propIndex];
-						i--;
-						last = (i == files.length - 1);
-					}
 					
 					// Delete property file
 					return Zotero.HTTP.promise("DELETE", deletePropURI, { successCodes: [200, 204, 404] })
@@ -534,29 +529,40 @@ Zotero.Sync.Storage.WebDAV = (function () {
 							case 204:
 							// IIS 5.1 and Sakai return 200
 							case 200:
-								results.deleted.push(fileName);
+								results.deleted.push(baseName);
 								break;
 							
 							case 404:
 								if (fileDeleted) {
-									results.deleted.push(fileName);
+									results.deleted.push(baseName);
 								}
 								else {
-									results.missing.push(fileName);
+									results.missing.push(baseName);
 								}
 								break;
 						}
 					});
 				})
 				.catch(function (e) {
-					results.error.push(fileName);
+					results.error.push(baseName);
-					var msg = "An error occurred attempting to delete "
+					throw e;
-						+ "'" + fileName
-						+ "' (" + e.status + " " + e.xmlhttp.statusText + ").";
 				});
 			});
+		}
+		
+		Components.utils.import("resource://zotero/concurrent-caller.js");
+		var caller = new ConcurrentCaller(4);
+		caller.stopOnError = true;
+		caller.setLogger(function (msg) {
+			Zotero.debug("[ConcurrentCaller] " + msg);
 		});
+		caller.setErrorLogger(function (msg) {
+			Components.utils.reportError(msg);
+		});
+		return caller.fcall(funcs)
+		.then(function () {
 			return results;
+		});
 	}
 	
 	
@@ -881,7 +887,8 @@ Zotero.Sync.Storage.WebDAV = (function () {
 								deleteStorageFiles([item.key + ".prop"])
 								.finally(function (results) {
 									deferred.resolve(false);
-								});
+								})
+								.done();
 								return;
 							}
 							else if (status != 200) {
@@ -1457,21 +1464,21 @@ Zotero.Sync.Storage.WebDAV = (function () {
 	
 	
 	/**
-	 * Remove files on storage server that were deleted locally more than
+	 * Remove files on storage server that were deleted locally
-	 * sync.storage.deleteDelayDays days ago
 	 *
 	 * @param	{Function}	callback		Passed number of files deleted
 	 */
 	obj._purgeDeletedStorageFiles = function () {
-		if (!this._active) {
+		return Q.fcall(function () {
-			return Q(false);
+			if (!this.includeUserFiles) {
+				return false;
 			}
 			
 			Zotero.debug("Purging deleted storage files");
 			var files = Zotero.Sync.Storage.getDeletedFiles();
 			if (!files) {
 				Zotero.debug("No files to delete remotely");
-			return Q(false);
+				return false;
 			}
 			
 			// Add .zip extension
@@ -1500,20 +1507,22 @@ Zotero.Sync.Storage.WebDAV = (function () {
 					Zotero.DB.commitTransaction();
 				}
 				
+				Zotero.debug(results);
+				
 				return results.deleted.length;
 			});
+		}.bind(this));
 	};
 	
 	
 	/**
 	 * Delete orphaned storage files older than a day before last sync time
-	 *
-	 * @param	{Function}	callback
 	 */
-	obj._purgeOrphanedStorageFiles = function (callback) {
+	obj._purgeOrphanedStorageFiles = function () {
+		return Q.fcall(function () {
 			const daysBeforeSyncTime = 1;
 			
-		if (!this._active) {
+			if (!this.includeUserFiles) {
 				return false;
 			}
 			
@@ -1535,25 +1544,30 @@ Zotero.Sync.Storage.WebDAV = (function () {
 			
 			var lastSyncDate = new Date(Zotero.Sync.Server.lastLocalSyncTime * 1000);
 			
-		Zotero.HTTP.WebDAV.doProp("PROPFIND", uri, xmlstr, function (req) {
+			var deferred = Q.defer();
-			Zotero.debug(req.responseText);
+			
+			Zotero.HTTP.WebDAV.doProp("PROPFIND", uri, xmlstr, function (xmlhttp) {
+				Q.fcall(function () {
+					Zotero.debug(xmlhttp.responseText);
 					
 					var funcName = "Zotero.Sync.Storage.purgeOrphanedStorageFiles()";
 					
-			var responseNode = req.responseXML.documentElement;
+					var responseNode = xmlhttp.responseXML.documentElement;
 					responseNode.xpath = function (path) {
 						return Zotero.Utilities.xpath(this, path, { D: 'DAV:' });
 					};
 					
 					var deleteFiles = [];
 					var trailingSlash = !!path.match(/\/$/);
-			for each(var response in responseNode.xpath("response")) {
+					for each(var response in responseNode.xpath("D:response")) {
-				var href = Zotero.Utilities.xpath(response, "href", { D: 'DAV:' });
+						var href = Zotero.Utilities.xpathText(
-				href = href.length ? href[0] : ''
+							response, "D:href", { D: 'DAV:' }
+						) || "";
+						Zotero.debug(href);
 						
 						// Strip trailing slash if there isn't one on the root path
 						if (!trailingSlash) {
-					href = href.replace(/\/$/, "")
+							href = href.replace(/\/$/, "");
 						}
 						
 						// Absolute
@@ -1579,7 +1593,7 @@ Zotero.Sync.Storage.WebDAV = (function () {
 								// character in the URL and the server (e.g., Sakai) is
 								// encoding the value
 								&& decodeURIComponent(href).indexOf(path) == -1) {
-					Zotero.Sync.Storage.EventManager.error(
+							throw new Error(
 								"DAV:href '" + href + "' does not begin with path '"
 									+ path + "' in " + funcName
 							);
@@ -1587,9 +1601,9 @@ Zotero.Sync.Storage.WebDAV = (function () {
 						
 						var matches = href.match(/[^\/]+$/);
 						if (!matches) {
-					Zotero.Sync.Storage.EventManager.error(
+							throw new Error(
 								"Unexpected href '" + href + "' in " + funcName
-					)
+							);
 						}
 						var file = matches[0];
 						
@@ -1612,10 +1626,10 @@ Zotero.Sync.Storage.WebDAV = (function () {
 						Zotero.debug("Checking orphaned file " + file);
 						
 						// TODO: Parse HTTP date properly
-				var lastModified = Zotero.Utilities.xpath(
+						Zotero.debug(response.innerHTML);
-					response, "//getlastmodified", { D: 'DAV:' }
+						var lastModified = Zotero.Utilities.xpathText(
+							response, ".//D:getlastmodified", { D: 'DAV:' }
 						);
-				lastModified = lastModified.length ? lastModified[0] : ''
 						lastModified = Zotero.Date.strToISO(lastModified);
 						lastModified = Zotero.Date.sqlToDate(lastModified);
 						
@@ -1627,12 +1641,22 @@ Zotero.Sync.Storage.WebDAV = (function () {
 						}
 					}
 					
-			deleteStorageFiles(deleteFiles)
+					return deleteStorageFiles(deleteFiles)
 					.then(function (results) {
 						Zotero.Prefs.set("lastWebDAVOrphanPurge", Math.round(new Date().getTime() / 1000))
 						Zotero.debug(results);
 					});
+				})
+				.catch(function (e) {
+					deferred.reject(e);
+				})
+				.then(function () {
+					deferred.resolve();
+				});
 			}, { Depth: 1 });
+			
+			return deferred.promise;
+		}.bind(this));
 	};
 	
 	return obj;

chrome/content/zotero/xpcom/storage/zfs.js

@@ -1006,7 +1006,7 @@ Zotero.Sync.Storage.ZFS = (function () {
 				Zotero.debug("Credentials are cached");
 				_cachedCredentials = true;
 			})
-			.fail(function (e) {
+			.catch(function (e) {
 				if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
 					if (e.status == 401) {
 						var msg = "File sync login failed\n\n"
@@ -1030,7 +1030,8 @@ Zotero.Sync.Storage.ZFS = (function () {
 	/**
 	 * Remove all synced files from the server
 	 */
-	obj._purgeDeletedStorageFiles = function (callback) {
+	obj._purgeDeletedStorageFiles = function () {
+		return Q.fcall(function () {
 			// If we don't have a user id we've never synced and don't need to bother
 			if (!Zotero.userID) {
 				return false;
@@ -1066,21 +1067,12 @@ Zotero.Sync.Storage.ZFS = (function () {
 			}
 			uri.spec = uri.spec.substr(0, uri.spec.length - 1);
 			
-		Zotero.HTTP.doPost(uri, "", function (xmlhttp) {
+			return Zotero.HTTP.promise("POST", uri, "")
-			if (xmlhttp.status != 204) {
+			.then(function (req) {
-				if (callback) {
-					callback(false);
-				}
-				throw "Unexpected status code " + xmlhttp.status + " purging ZFS files";
-			}
-			
 				var sql = "DELETE FROM settings WHERE setting=? AND key=?";
 				Zotero.DB.query(sql, ['storage', 'zfsPurge']);
-			
-			if (callback) {
-				callback(true);
-			}
 			});
+		}.bind(this));
 	};
 	
 	return obj;

chrome/content/zotero/xpcom/sync.js

@@ -429,7 +429,7 @@ Zotero.Sync.EventListener = new function () {
 			var sql = "REPLACE INTO syncDeleteLog VALUES (?, ?, ?, ?)";
 			var syncStatement = Zotero.DB.getStatement(sql);
 			
-			if (isItem && Zotero.Sync.Storage.WebDAV.active) {
+			if (isItem && Zotero.Sync.Storage.WebDAV.includeUserFiles) {
 				var storageEnabled = true;
 				var sql = "INSERT INTO storageDeleteLog VALUES (?, ?, ?)";
 				var storageStatement = Zotero.DB.getStatement(sql);

chrome/content/zotero/xpcom/zotero.js

@@ -1844,15 +1844,6 @@ Components.utils.import("resource://gre/modules/Services.jsm");
 		Zotero.Items.purge();
 		// DEBUG: this might not need to be permanent
 		Zotero.Relations.purge();
-		
-		if (!skipStoragePurge && Math.random() < 1/10) {
-			Zotero.Sync.Storage.ZFS.purgeDeletedStorageFiles();
-			Zotero.Sync.Storage.WebDAV.purgeDeletedStorageFiles();
-		}
-		
-		if (!skipStoragePurge) {
-			Zotero.Sync.Storage.WebDAV.purgeOrphanedStorageFiles();
-		}
 	}