File sync optimizations
This commit is contained in:
Dan Stillman
commit
84d9963e14
4 changed files with 449 additions and 313 deletions
|
|
@ -27,7 +27,7 @@ Zotero.Notifier = new function(){
|
|||
var _observers = {};
|
||||
var _disabled = false;
|
||||
var _types = [
|
||||
'collection', 'creator', 'search', 'share', 'share-items', 'item',
|
||||
'collection', 'creator', 'search', 'share', 'share-items', 'item', 'file',
|
||||
'collection-item', 'item-tag', 'tag', 'setting', 'group', 'trash', 'bucket', 'relation'
|
||||
];
|
||||
var _inTransaction;
|
||||
|
|
|
|||
|
|
@ -75,17 +75,23 @@ Zotero.Sync.Storage = new function () {
|
|||
}
|
||||
}
|
||||
|
||||
Zotero.Notifier.registerObserver(this, ['file']);
|
||||
|
||||
|
||||
//
|
||||
// Private properties
|
||||
//
|
||||
var _maxCheckAgeInSeconds = 10800; // maximum age for upload modification check (3 hours)
|
||||
var _syncInProgress;
|
||||
var _updatesInProgress;
|
||||
var _itemDownloadPercentages = {};
|
||||
var _uploadCheckFiles = [];
|
||||
var _lastFullFileCheck = {};
|
||||
|
||||
|
||||
this.sync = function (libraries) {
|
||||
if (libraries) {
|
||||
Zotero.debug("Starting file sync for libraries " + libraries);
|
||||
this.sync = function (options) {
|
||||
if (options.libraries) {
|
||||
Zotero.debug("Starting file sync for libraries " + options.libraries);
|
||||
}
|
||||
else {
|
||||
Zotero.debug("Starting file sync");
|
||||
|
|
@ -100,7 +106,7 @@ Zotero.Sync.Storage = new function () {
|
|||
return Q.fcall(function () {
|
||||
// TODO: Make sure modes are active
|
||||
|
||||
if (libraries && libraries.indexOf(0) == -1) {
|
||||
if (options.libraries && options.libraries.indexOf(0) == -1) {
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -116,7 +122,7 @@ Zotero.Sync.Storage = new function () {
|
|||
if (Zotero.Sync.Storage.ZFS.includeGroupFiles) {
|
||||
var groups = Zotero.Groups.getAll();
|
||||
for each(var group in groups) {
|
||||
if (libraries && libraries.indexOf(group.libraryID) == -1) {
|
||||
if (options.libraries && options.libraries.indexOf(group.libraryID) == -1) {
|
||||
continue;
|
||||
}
|
||||
// TODO: if library file syncing enabled
|
||||
|
|
@ -214,8 +220,37 @@ Zotero.Sync.Storage = new function () {
|
|||
});
|
||||
|
||||
// Check for updated files to upload in each library
|
||||
return Q.all([self.checkForUpdatedFiles(null, parseInt(libraryID))
|
||||
for (libraryID in librarySyncTimes)])
|
||||
var promises = [];
|
||||
for (let libraryID in librarySyncTimes) {
|
||||
let promise;
|
||||
libraryID = parseInt(libraryID);
|
||||
|
||||
if (!Zotero.Libraries.isFilesEditable(libraryID)) {
|
||||
Zotero.debug("No file editing access -- skipping file "
|
||||
+ "modification check for library " + libraryID);
|
||||
continue;
|
||||
}
|
||||
// If this is a background sync, it's not the first sync of
|
||||
// the session, the library has had at least one full check
|
||||
// this session, and it's been less than _maxCheckAgeInSeconds
|
||||
// since the last full check of this library, check only files
|
||||
// that were previously modified or opened recently
|
||||
else if (options.background
|
||||
&& !options.firstInSession
|
||||
&& _lastFullFileCheck[libraryID]
|
||||
&& (_lastFullFileCheck[libraryID] + (_maxCheckAgeInSeconds * 1000))
|
||||
> new Date().getTime()) {
|
||||
let itemIDs = _getFilesToCheck(libraryID);
|
||||
promise = self.checkForUpdatedFiles(libraryID, itemIDs);
|
||||
}
|
||||
// Otherwise check all files in the library
|
||||
else {
|
||||
_lastFullFileCheck[libraryID] = new Date().getTime();
|
||||
promise = self.checkForUpdatedFiles(libraryID);
|
||||
}
|
||||
promises.push(promise);
|
||||
}
|
||||
return Q.all(promises)
|
||||
.then(function () {
|
||||
// Queue files to download and upload from each library
|
||||
for (let libraryID in librarySyncTimes) {
|
||||
|
|
@ -646,18 +681,25 @@ Zotero.Sync.Storage = new function () {
|
|||
* Scans local files and marks any that have changed for uploading
|
||||
* and any that are missing for downloading
|
||||
*
|
||||
* @param {Integer} [libraryID]
|
||||
* @param {Integer[]} [itemIDs]
|
||||
* @param {Object} [itemModTimes] Item mod times indexed by item ids;
|
||||
* items with stored mod times
|
||||
* that differ from the provided
|
||||
* time but file mod times
|
||||
* matching the stored time will
|
||||
* be marked for download
|
||||
* @param {Boolean} [includePersonalItems=false]
|
||||
* @param {Boolean} [includeGroupItems=false]
|
||||
* @return {Promise} Promise resolving to TRUE if any items changed state,
|
||||
* FALSE otherwise
|
||||
*/
|
||||
this.checkForUpdatedFiles = function (itemModTimes, libraryID) {
|
||||
this.checkForUpdatedFiles = function (libraryID, itemIDs, itemModTimes) {
|
||||
libraryID = parseInt(libraryID);
|
||||
if (isNaN(libraryID)) {
|
||||
libraryID = false;
|
||||
}
|
||||
|
||||
Components.utils.import("resource://gre/modules/Task.jsm");
|
||||
return Q(Task.spawn(function () {
|
||||
var msg = "Checking for locally changed attachment files";
|
||||
|
||||
var memmgr = Components.classes["@mozilla.org/memory-reporter-manager;1"]
|
||||
|
|
@ -665,22 +707,39 @@ Zotero.Sync.Storage = new function () {
|
|||
memmgr.init();
|
||||
Zotero.debug("Memory usage: " + memmgr.resident);
|
||||
|
||||
if (typeof libraryID != 'undefined') {
|
||||
msg += " in library " + libraryID;
|
||||
if (itemModTimes) {
|
||||
throw new Error("libraryID is not allowed when itemModTimes is set");
|
||||
if (libraryID !== false) {
|
||||
if (itemIDs) {
|
||||
if (!itemIDs.length) {
|
||||
var msg = "No files to check for local changes in library " + libraryID;
|
||||
Zotero.debug(msg);
|
||||
throw new Task.Result(false);
|
||||
}
|
||||
}
|
||||
if (itemModTimes) {
|
||||
throw new Error("itemModTimes is not allowed when libraryID is set");
|
||||
}
|
||||
|
||||
msg += " in library " + libraryID;
|
||||
}
|
||||
else if (itemIDs) {
|
||||
throw new Error("libraryID not provided");
|
||||
}
|
||||
else if (itemModTimes) {
|
||||
if (!Object.keys(itemModTimes).length) {
|
||||
throw new Task.Result(false);
|
||||
}
|
||||
msg += " in download-marking mode";
|
||||
}
|
||||
else {
|
||||
if (!itemModTimes) {
|
||||
return Q(false);
|
||||
}
|
||||
throw new Error("libraryID, itemIDs, or itemModTimes must be provided");
|
||||
}
|
||||
Zotero.debug(msg);
|
||||
|
||||
var changed = false;
|
||||
|
||||
var itemIDs = Object.keys(itemModTimes ? itemModTimes : {});
|
||||
if (!itemIDs) {
|
||||
itemIDs = Object.keys(itemModTimes ? itemModTimes : {});
|
||||
}
|
||||
|
||||
// Can only handle 999 bound parameters at a time
|
||||
var numIDs = itemIDs.length;
|
||||
|
|
@ -702,7 +761,7 @@ Zotero.Sync.Storage = new function () {
|
|||
Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD,
|
||||
Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
|
||||
);
|
||||
if (typeof libraryID != 'undefined') {
|
||||
if (libraryID !== false) {
|
||||
sql += " AND libraryID=?";
|
||||
params.push(libraryID == 0 ? null : libraryID);
|
||||
}
|
||||
|
|
@ -724,15 +783,15 @@ Zotero.Sync.Storage = new function () {
|
|||
// we don't need to do anything
|
||||
if (!rows.length) {
|
||||
var msg = "No in-sync or to-upload files found";
|
||||
if (typeof libraryID != 'undefined') {
|
||||
if (libraryID !== false) {
|
||||
msg += " in library " + libraryID;
|
||||
}
|
||||
Zotero.debug(msg);
|
||||
return Q(changed);
|
||||
throw new Task.Result(changed);
|
||||
}
|
||||
|
||||
// Index attachment data by item id
|
||||
var itemIDs = [];
|
||||
itemIDs = [];
|
||||
var attachmentData = {};
|
||||
for each(let row in rows) {
|
||||
var id = row.itemID;
|
||||
|
|
@ -747,11 +806,19 @@ Zotero.Sync.Storage = new function () {
|
|||
}
|
||||
rows = null;
|
||||
|
||||
var t = new Date();
|
||||
var items = Zotero.Items.get(itemIDs);
|
||||
var numItems = items.length;
|
||||
var updatedStates = {};
|
||||
|
||||
// OS.File didn't work reliably before Firefox 23, so use the old code
|
||||
if (Zotero.platformMajorVersion < 23) {
|
||||
var updatedStates = {};
|
||||
var items = Zotero.Items.get(itemIDs);
|
||||
Zotero.debug("Performing synchronous file update check");
|
||||
|
||||
for each(var item in items) {
|
||||
// Spin the event loop during synchronous file access
|
||||
yield Q.delay(1);
|
||||
|
||||
Zotero.debug("Memory usage: " + memmgr.resident);
|
||||
|
||||
let row = attachmentData[item.id];
|
||||
|
|
@ -858,14 +925,13 @@ Zotero.Sync.Storage = new function () {
|
|||
Zotero.debug("No synced files have changed locally");
|
||||
}
|
||||
|
||||
return Q(changed);
|
||||
Zotero.debug("Checked " + numItems + " files in " + (new Date() - t) + "ms");
|
||||
|
||||
throw new Task.Result(changed);
|
||||
}
|
||||
|
||||
Components.utils.import("resource://gre/modules/osfile.jsm")
|
||||
|
||||
var updatedStates = {};
|
||||
var items = Zotero.Items.get(itemIDs);
|
||||
|
||||
let checkItems = function () {
|
||||
if (!items.length) return;
|
||||
|
||||
|
|
@ -997,7 +1063,7 @@ Zotero.Sync.Storage = new function () {
|
|||
});
|
||||
};
|
||||
|
||||
return checkItems()
|
||||
throw new Task.Result(checkItems()
|
||||
.then(function () {
|
||||
for (let itemID in updatedStates) {
|
||||
Zotero.Sync.Storage.setSyncState(itemID, updatedStates[itemID]);
|
||||
|
|
@ -1008,9 +1074,12 @@ Zotero.Sync.Storage = new function () {
|
|||
Zotero.debug("No synced files have changed locally");
|
||||
}
|
||||
|
||||
Zotero.debug("Checked " + numItems + " files in " + (new Date() - t) + "ms");
|
||||
|
||||
return changed;
|
||||
});
|
||||
}
|
||||
}));
|
||||
}));
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
|
|
@ -1310,6 +1379,20 @@ Zotero.Sync.Storage = new function () {
|
|||
}
|
||||
|
||||
|
||||
this.notify = function(event, type, ids, extraData) {
|
||||
if (event == 'open' && type == 'file') {
|
||||
let timestamp = new Date().getTime();
|
||||
|
||||
for each(let id in ids) {
|
||||
_uploadCheckFiles.push({
|
||||
itemID: id,
|
||||
timestamp: timestamp
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//
|
||||
// Private methods
|
||||
//
|
||||
|
|
@ -1940,6 +2023,36 @@ Zotero.Sync.Storage = new function () {
|
|||
}
|
||||
|
||||
|
||||
/**
|
||||
* Get files to check for local modifications for uploading
|
||||
*
|
||||
* This includes files previously modified and files opened externally
|
||||
* via Zotero within _maxCheckAgeInSeconds.
|
||||
*/
|
||||
function _getFilesToCheck(libraryID) {
|
||||
var minTime = new Date().getTime() - (_maxCheckAgeInSeconds * 1000);
|
||||
|
||||
// Get files by modification time
|
||||
var sql = "SELECT itemID FROM itemAttachments JOIN items USING (itemID) "
|
||||
+ "WHERE libraryID=? AND linkMode IN (?,?) AND syncState IN (?) AND "
|
||||
+ "storageModTime>=?";
|
||||
var params = [
|
||||
libraryID == 0 ? null : libraryID,
|
||||
Zotero.Attachments.LINK_MODE_IMPORTED_FILE,
|
||||
Zotero.Attachments.LINK_MODE_IMPORTED_URL,
|
||||
Zotero.Sync.Storage.SYNC_STATE_IN_SYNC,
|
||||
minTime
|
||||
];
|
||||
var itemIDs = Zotero.DB.columnQuery(sql, params) || [];
|
||||
|
||||
// Get files by open time
|
||||
_uploadCheckFiles.filter(function (x) x.timestamp >= minTime);
|
||||
itemIDs = itemIDs.concat([x.itemID for each(x in _uploadCheckFiles)])
|
||||
|
||||
return Zotero.Utilities.arrayUnique(itemIDs);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @inner
|
||||
* @return {String[]|FALSE} Array of keys, or FALSE if none
|
||||
|
|
|
|||
|
|
@ -520,6 +520,7 @@ Zotero.Sync.Runner = new function () {
|
|||
var _autoSyncTimer;
|
||||
var _queue;
|
||||
var _background;
|
||||
var _firstInSession = true;
|
||||
|
||||
var _lastSyncStatus;
|
||||
var _currentSyncStatusLabel;
|
||||
|
|
@ -533,7 +534,13 @@ Zotero.Sync.Runner = new function () {
|
|||
this.IdleListener.init();
|
||||
}
|
||||
|
||||
this.sync = function (background) {
|
||||
this.sync = function (options) {
|
||||
if (!options) options = {};
|
||||
if (_firstInSession) {
|
||||
options.firstInSession = true;
|
||||
_firstInSession = false;
|
||||
}
|
||||
|
||||
_warning = null;
|
||||
|
||||
if (Zotero.HTTP.browserIsOffline()){
|
||||
|
|
@ -549,7 +556,7 @@ Zotero.Sync.Runner = new function () {
|
|||
// Purge deleted objects so they don't cause sync errors (e.g., long tags)
|
||||
Zotero.purgeDataObjects(true);
|
||||
|
||||
_background = !!background;
|
||||
_background = !!options.background;
|
||||
this.setSyncIcon('animate');
|
||||
|
||||
var finalCallbacks = {
|
||||
|
|
@ -562,7 +569,7 @@ Zotero.Sync.Runner = new function () {
|
|||
var storageSync = function () {
|
||||
Zotero.Sync.Runner.setSyncStatus(Zotero.getString('sync.status.syncingFiles'));
|
||||
|
||||
Zotero.Sync.Storage.sync()
|
||||
Zotero.Sync.Storage.sync(options)
|
||||
.then(function (results) {
|
||||
Zotero.debug("File sync is finished");
|
||||
|
||||
|
|
@ -692,7 +699,9 @@ Zotero.Sync.Runner = new function () {
|
|||
return;
|
||||
}
|
||||
|
||||
Zotero.Sync.Runner.sync(background);
|
||||
Zotero.Sync.Runner.sync({
|
||||
background: background
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1143,8 +1152,10 @@ Zotero.Sync.Runner.IdleListener = {
|
|||
|
||||
Zotero.debug("Beginning idle sync");
|
||||
|
||||
Zotero.Sync.Runner.sync(true);
|
||||
Zotero.Sync.Runner.setSyncTimeout(this._idleTimeout, true);
|
||||
Zotero.Sync.Runner.sync({
|
||||
background: true
|
||||
});
|
||||
Zotero.Sync.Runner.setSyncTimeout(this._idleTimeout, true, true);
|
||||
},
|
||||
|
||||
_backObserver: {
|
||||
|
|
@ -1160,7 +1171,9 @@ Zotero.Sync.Runner.IdleListener = {
|
|||
return;
|
||||
}
|
||||
Zotero.debug("Beginning return-from-idle sync");
|
||||
Zotero.Sync.Runner.sync(true);
|
||||
Zotero.Sync.Runner.sync({
|
||||
background: true
|
||||
});
|
||||
}
|
||||
},
|
||||
|
||||
|
|
@ -2021,7 +2034,9 @@ Zotero.Sync.Server = new function () {
|
|||
|
||||
Zotero.Sync.Server.resetClient();
|
||||
Zotero.Sync.Server.canAutoResetClient = false;
|
||||
Zotero.Sync.Runner.sync(background);
|
||||
Zotero.Sync.Runner.sync({
|
||||
background: background
|
||||
});
|
||||
}, 1);
|
||||
break;
|
||||
|
||||
|
|
@ -2124,7 +2139,9 @@ Zotero.Sync.Server = new function () {
|
|||
Zotero.Sync.Server.canAutoResetClient = false;
|
||||
}
|
||||
|
||||
Zotero.Sync.Runner.sync(background);
|
||||
Zotero.Sync.Runner.sync({
|
||||
background: background
|
||||
});
|
||||
}, 1);
|
||||
break;
|
||||
|
||||
|
|
@ -2359,7 +2376,9 @@ Zotero.Sync.Server = new function () {
|
|||
}
|
||||
Zotero.Sync.Server.resetClient();
|
||||
Zotero.Sync.Server.canAutoResetClient = false;
|
||||
Zotero.Sync.Runner.sync(background);
|
||||
Zotero.Sync.Runner.sync({
|
||||
background: background
|
||||
});
|
||||
}, 1);
|
||||
break;
|
||||
|
||||
|
|
@ -3430,7 +3449,7 @@ Zotero.Sync.Server.Data = new function() {
|
|||
// Check mod times and hashes of updated items against stored values to see
|
||||
// if they've been updated elsewhere and mark for download if so
|
||||
if (type == 'item' && Object.keys(itemStorageModTimes).length) {
|
||||
yield Zotero.Sync.Storage.checkForUpdatedFiles(itemStorageModTimes);
|
||||
yield Zotero.Sync.Storage.checkForUpdatedFiles(null, null, itemStorageModTimes);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -433,7 +433,9 @@ var ZoteroPane = new function()
|
|||
return;
|
||||
}
|
||||
|
||||
Zotero.Sync.Runner.sync(true);
|
||||
Zotero.Sync.Runner.sync({
|
||||
background: true
|
||||
});
|
||||
})
|
||||
.done();
|
||||
}
|
||||
|
|
@ -3492,6 +3494,7 @@ var ZoteroPane = new function()
|
|||
this.loadURI(url, event);
|
||||
}
|
||||
else {
|
||||
Zotero.Notifier.trigger('open', 'file', itemID);
|
||||
Zotero.launchFile(file);
|
||||
}
|
||||
}
|
||||
|
|
@ -3613,6 +3616,7 @@ var ZoteroPane = new function()
|
|||
var parent = file.parent.QueryInterface(Components.interfaces.nsILocalFile);
|
||||
Zotero.launchFile(parent);
|
||||
}
|
||||
Zotero.Notifier.trigger('open', 'file', attachment.id);
|
||||
}
|
||||
else {
|
||||
this.showAttachmentNotFoundDialog(attachment.id, noLocateOnMissing)
|
||||
|
|
|
|||
Loading…
Reference in a new issue