mirrors/zotero
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

- Use millisecond resolution for file sync timestamps (fixes problem with slow sync initialization on some systems)

Browse source 
- Fix errors syncing files with filenames containing extended characters

This will not sync with the server until a server update is pushed.
This commit is contained in:
Dan Stillman 2010-02-03 04:43:20 +00:00
parent 673a22430b
commit 4c30e6738f
6 changed files with 184 additions and 92 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
chrome/content/zotero/xpcom/schema.js
View file

@ -1056,10 +1056,6 @@ Zotero.Schema = new function(){
Zotero.DB.query(sql);
var sql = "INSERT INTO itemData VALUES (1, 1, 2)";
Zotero.DB.query(sql);
var sql = "INSERT INTO itemDataValues VALUES (3, CURRENT_TIMESTAMP)";
Zotero.DB.query(sql);
var sql = "INSERT INTO itemData VALUES (1, 27, 3)";
Zotero.DB.query(sql);
// CHNM as creator
var sql = "INSERT INTO creatorData VALUES (1, '', 'Center for History and New Media', '', 1, NULL)";
@ -2793,6 +2789,10 @@ Zotero.Schema = new function(){
Zotero.DB.query("UPDATE itemData SET fieldID=121 WHERE itemID IN (SELECT itemID FROM items WHERE itemTypeID=19) AND fieldID=14");
}
if (i==71) {
Zotero.DB.query("UPDATE itemAttachments SET storageModTime=storageModTime*1000 WHERE storageModTime<10000000000");
}
Zotero.wait();
}

119
chrome/content/zotero/xpcom/storage.js
View file

@ -218,7 +218,7 @@ Zotero.Sync.Storage = new function () {
/**
* @param {Integer} itemID
* @return {Integer|NULL} Mod time as Unix timestamp,
* @return {Integer|NULL} Mod time as timestamp in ms,
* or NULL if never synced
*/
this.getSyncedModificationTime = function (itemID) {
@ -235,7 +235,7 @@ Zotero.Sync.Storage = new function () {
/**
* @param {Integer} itemID
* @param {Integer} mtime File modification time as
* Unix timestamp
* timestamp in ms
* @param {Boolean} [updateItem=FALSE] Update dateModified field of
* attachment item
*/
@ -459,10 +459,10 @@ Zotero.Sync.Storage = new function () {
continue;
}
var fileModTime = Math.round(file.lastModifiedTime / 1000);
var fmtime = file.lastModifiedTime;
//Zotero.debug("Stored mtime is " + attachmentData[item.id].mtime);
//Zotero.debug("File mtime is " + fileModTime);
//Zotero.debug("File mtime is " + fmtime);
// Download-marking mode
if (itemModTimes) {
@ -481,8 +481,29 @@ Zotero.Sync.Storage = new function () {
continue;
}
var mtime = attachmentData[item.id].mtime;
// If stored time matches file, it hasn't changed
if (attachmentData[item.id].mtime == fileModTime) {
if (mtime == fmtime) {
continue;
}
// Allow floored timestamps for filesystems that don't support
// millisecond precision (e.g., HFS+)
if (Math.floor(mtime / 1000) * 1000 == fmtime || Math.floor(fmtime / 1000) * 1000 == mtime) {
Zotero.debug("File mod times are within one-second precision (" + fmtime + " ≅ " + mtime + ") "
+ "for " + file.leafName + " -- ignoring");
continue;
}
// Allow timestamp to be exactly one hour off to get around
// time zone issues -- there may be a proper way to fix this
if (Math.abs(fmtime - mtime) == 3600000
// And check with one-second precision as well
|| Math.abs(fmtime - Math.floor(mtime / 1000) * 1000) == 3600000
|| Math.abs(Math.floor(fmtime / 1000) * 1000 - mtime) == 3600000) {
Zotero.debug("File mod time (" + fmtime + ") is exactly one hour off remote file (" + mtime + ") "
+ "-- assuming time zone issue and skipping upload");
continue;
}
@ -501,13 +522,13 @@ Zotero.Sync.Storage = new function () {
}
var fileHash = item.attachmentHash;
if (attachmentData[item.id].hash && attachmentData[item.id].hash == fileHash) {
Zotero.debug("Mod time didn't match (" + fileModTime + "!=" + attachmentData[item.id].mtime + ") "
Zotero.debug("Mod time didn't match (" + fmtime + "!=" + mtime + ") "
+ "but hash did for " + file.leafName + " -- ignoring");
continue;
}
Zotero.debug("Marking attachment " + item.id + " as changed ("
+ attachmentData[item.id].mtime + " != " + fileModTime + ")");
+ mtime + " != " + fmtime + ")");
updatedStates[item.id] = Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD;
}
@ -603,7 +624,7 @@ Zotero.Sync.Storage = new function () {
}
if (!data.compressed && !data.syncHash) {
_error("|data.storageHash| is required if |data.compressed| is false in " + funcName);
_error("|data.syncHash| is required if |data.compressed| is false in " + funcName);
}
var item = data.item;
@ -658,12 +679,12 @@ Zotero.Sync.Storage = new function () {
Zotero.Sync.Storage.resyncOnFinish = true;
}
else {
file.lastModifiedTime = syncModTime * 1000;
// Only save hash if file isn't compressed
if (!data.compressed) {
Zotero.Sync.Storage.setSyncedHash(item.id, syncHash);
file.lastModifiedTime = syncModTime;
// If hash not provided (e.g., WebDAV), calculate it now
if (!syncHash) {
syncHash = item.attachmentHash;
}
Zotero.Sync.Storage.setSyncedHash(item.id, syncHash);
Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
}
@ -817,23 +838,16 @@ Zotero.Sync.Storage = new function () {
throw ("Empty path for item " + item.key + " in " + funcName);
}
var itemFileName = item.getFilename();
var fileName = file.leafName;
var renamed = false;
// If file doesn't match the known filename, use that name
if (itemFileName != fileName) {
Zotero.debug("Renaming file '" + fileName + "' to known filename '" + itemFileName + "'");
fileName = itemFileName;
renamed = true;
}
// Make sure the new filename is valid, in case an invalid character made it over
// (e.g., from before we checked for them)
var filteredName = Zotero.File.getValidFileName(fileName);
if (filteredName != fileName) {
Zotero.debug("Filtering filename '" + fileName + "' to '" + filteredName + "'");
fileName = filteredName;
file.leafName = fileName;
renamed = true;
}
@ -842,9 +856,7 @@ Zotero.Sync.Storage = new function () {
tempFile.moveTo(parentDir, fileName);
}
catch (e) {
var destFile = parentDir.clone();
destFile.QueryInterface(Components.interfaces.nsILocalFile);
destFile.setRelativeDescriptor(parentDir, fileName);
var destFile = file.clone();
var windowsLength = false;
var nameLength = false;
@ -910,10 +922,7 @@ Zotero.Sync.Storage = new function () {
var returnFile = null;
// processDownload() needs to know that we're renaming the file
if (renamed) {
destFile = parentDir.clone();
destFile.QueryInterface(Components.interfaces.nsILocalFile);
destFile.setRelativeDescriptor(parentDir, fileName);
returnFile = destFile;
var returnFile = file.clone();
}
return returnFile;
@ -966,7 +975,6 @@ Zotero.Sync.Storage = new function () {
var entries = zipReader.findEntries(null);
while (entries.hasMore()) {
var renamed = false;
count++;
var entryName = entries.getNext();
var b64re = /%ZB64$/;
@ -984,35 +992,54 @@ Zotero.Sync.Storage = new function () {
continue;
}
Zotero.debug("Extracting " + fileName);
var primaryFile = false;
var filtered = false;
var renamed = false;
// Get the old filename
var itemFileName = item.getFilename();
// Make sure the new filename is valid, in case an invalid character
// somehow make it into the ZIP (e.g., from before we checked for them)
//
// Do this before trying to use the relative descriptor, since otherwise
// it might fail silently and select the parent directory
var filteredName = Zotero.File.getValidFileName(fileName);
if (filteredName != fileName) {
Zotero.debug("Filtering filename '" + fileName + "' to '" + filteredName + "'");
fileName = filteredName;
filtered = true;
}
// Name in ZIP is a relative descriptor, so file has to be reconstructed
// using setRelativeDescriptor()
var destFile = parentDir.clone();
destFile.QueryInterface(Components.interfaces.nsILocalFile);
destFile.setRelativeDescriptor(parentDir, fileName);
fileName = destFile.leafName;
// If only one file in zip and it doesn't match the known filename,
// take our chances and use that name
if (count == 1 && !entries.hasMore()) {
// May not be necessary, but let's be safe
itemFileName = Zotero.File.getValidFileName(itemFileName);
if (itemFileName != fileName) {
Zotero.debug("Renaming single file '" + fileName + "' in ZIP to known filename '" + itemFileName + "'");
Zotero.debug("Renaming single file '" + fileName + "' in ZIP to known filename '" + itemFileName + "'", 2);
Components.utils.reportError("Renaming single file '" + fileName + "' in ZIP to known filename '" + itemFileName + "'");
fileName = itemFileName;
destFile.leafName = fileName;
renamed = true;
}
}
var primaryFile = itemFileName == fileName;
// Make sure the new filename is valid, in case an invalid character
// somehow make it into the ZIP (e.g., from before we checked for them)
var filteredName = Zotero.File.getValidFileName(fileName);
if (filteredName != fileName) {
Zotero.debug("Filtering filename '" + fileName + "' to '" + filteredName + "'");
fileName = filteredName;
if (primaryFile) {
renamed = true;
}
if (primaryFile && filtered) {
renamed = true;
}
Zotero.debug("Extracting " + fileName);
var destFile = parentDir.clone();
destFile.QueryInterface(Components.interfaces.nsILocalFile);
destFile.setRelativeDescriptor(parentDir, fileName);
if (destFile.exists()) {
var msg = "ZIP entry '" + fileName + "' " + "already exists";
Zotero.debug(msg, 2);
@ -1680,10 +1707,10 @@ Zotero.Sync.Storage.QueueManager = new function () {
var item = Zotero.Sync.Storage.getItemFromRequestName(conflict.name);
var item1 = item.clone(false, false, true);
item1.setField('dateModified',
Zotero.Date.dateToSQL(new Date(conflict.localData.modTime * 1000), true));
Zotero.Date.dateToSQL(new Date(conflict.localData.modTime), true));
var item2 = item.clone(false, false, true);
item2.setField('dateModified',
Zotero.Date.dateToSQL(new Date(conflict.remoteData.modTime * 1000), true));
Zotero.Date.dateToSQL(new Date(conflict.remoteData.modTime), true));
objectPairs.push([item1, item2]);
}

96
chrome/content/zotero/xpcom/storage/webdav.js
View file

@ -241,27 +241,52 @@ Zotero.Sync.Storage.Session.WebDAV.prototype._getStorageModificationTime = funct
var xml = new XML(req.responseText);
}
catch (e) {
Zotero.debug(e);
var xml = null;
}
if (xml && xml.childNodes.length()) {
if (xml) {
Zotero.debug(xml.children().length());
}
if (xml && xml.children().length()) {
// TODO: other stuff, but this makes us forward-compatible
mtime = xml.mtime.toString();
var seconds = false;
}
else {
mtime = req.responseText;
var seconds = true;
}
var invalid = false;
// Unix timestamps need to be converted to ms-based timestamps
if (seconds) {
if (mtime.match(/^[0-9]{1,10}$/)) {
Zotero.debug("Converting Unix timestamp '" + mtime + "' to milliseconds");
mtime = mtime * 1000;
}
else {
invalid = true;
}
}
else if (!mtime.match(/^[0-9]{1,13}$/)) {
invalid = true;
}
// Delete invalid .prop files
if (!(mtime + '').match(/^[0-9]{10}$/)) {
var msg = "Invalid mod date '" + Zotero.Utilities.prototype.ellipsize(mtime, 20)
+ "' for item " + Zotero.Items.getLibraryKeyHash(item) + " in " + funcName;
if (invalid) {
var msg = "An error occurred during file syncing. Try the sync again.\n\n"
+ "Invalid mod date '" + Zotero.Utilities.prototype.ellipsize(mtime, 20)
+ "' for item " + Zotero.Items.getLibraryKeyHash(item);
Zotero.debug(msg, 1);
self._deleteStorageFiles([item.key + ".prop"], null, self);
self.onError(msg);
return;
}
var mdate = new Date(mtime * 1000);
var mdate = new Date(parseInt(mtime));
callback(item, mdate);
});
}
@ -271,12 +296,20 @@ Zotero.Sync.Storage.Session.WebDAV.prototype._getStorageModificationTime = funct
* Set mod time of file on storage server
*
* @param {Zotero.Item} item
* @param {Function} callback Callback f(item, mtime)
* @param {Function} callback Callback f(item, props)
*/
Zotero.Sync.Storage.Session.WebDAV.prototype._setStorageModificationTime = function (item, callback) {
var uri = this._getItemPropertyURI(item);
Zotero.Utilities.HTTP.WebDAV.doPut(uri, item.attachmentModificationTime + '', function (req) {
var mtime = item.attachmentModificationTime;
var hash = item.attachmentHash;
var prop = <properties version="1">
<mtime>{mtime}</mtime>
<hash>{hash}</hash>
</properties>;
Zotero.Utilities.HTTP.WebDAV.doPut(uri, prop.toXMLString(), function (req) {
switch (req.status) {
case 200:
case 201:
@ -288,7 +321,7 @@ Zotero.Sync.Storage.Session.WebDAV.prototype._setStorageModificationTime = funct
throw ("Unexpected status code " + req.status + " in "
+ "Zotero.Sync.Storage._setStorageModificationTime()");
}
callback(item, item.attachmentModificationTime);
callback(item, { mtime: mtime, hash: hash });
});
}
@ -324,12 +357,11 @@ Zotero.Sync.Storage.Session.WebDAV.prototype.downloadFile = function (request) {
}
try {
var syncModTime = Zotero.Date.toUnixTimestamp(mdate);
var syncModTime = mdate.getTime();
// Skip download if local file exists and matches mod time
var file = item.getFile();
if (file && file.exists()
&& syncModTime == Math.round(file.lastModifiedTime / 1000)) {
if (file && file.exists() && syncModTime == file.lastModifiedTime) {
Zotero.debug("File mod time matches remote file -- skipping download");
Zotero.DB.beginTransaction();
@ -461,21 +493,35 @@ Zotero.Sync.Storage.Session.WebDAV.prototype._processUploadFile = function (data
!= Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD) {
if (mdate) {
// Remote prop time
var mtime = Zotero.Date.toUnixTimestamp(mdate);
var mtime = mdate.getTime();
// Local file time
var fmtime = item.attachmentModificationTime;
var same = false;
if (fmtime == mtime) {
same = true;
Zotero.debug("File mod time matches remote file -- skipping upload");
}
// Allow floored timestamps for filesystems that don't support
// millisecond precision (e.g., HFS+)
else if (Math.floor(mtime / 1000) * 1000 == fmtime || Math.floor(fmtime / 1000) * 1000 == mtime) {
same = true;
Zotero.debug("File mod times are within one-second precision (" + fmtime + " ≅ " + mtime + ") "
+ "-- skipping upload");
}
// Allow timestamp to be exactly one hour off to get around
// time zone issues -- there may be a proper way to fix this
if (fmtime == mtime || Math.abs(fmtime - mtime) == 3600) {
if (fmtime == mtime) {
Zotero.debug("File mod time matches remote file -- skipping upload");
}
else {
Zotero.debug("File mod time (" + fmtime + ") is exactly one hour off remote file (" + mtime + ") "
+ "-- assuming time zone issue and skipping upload");
}
else if (Math.abs(fmtime - mtime) == 3600000
// And check with one-second precision as well
|| Math.abs(fmtime - Math.floor(mtime / 1000) * 1000) == 3600000
|| Math.abs(Math.floor(fmtime / 1000) * 1000 - mtime) == 3600000) {
same = true;
Zotero.debug("File mod time (" + fmtime + ") is exactly one hour off remote file (" + mtime + ") "
+ "-- assuming time zone issue and skipping upload");
}
if (same) {
Zotero.DB.beginTransaction();
var syncState = Zotero.Sync.Storage.getSyncState(item.id);
Zotero.Sync.Storage.setSyncedModificationTime(item.id, fmtime, true);
@ -592,7 +638,7 @@ Zotero.Sync.Storage.Session.WebDAV.prototype._onUploadComplete = function (httpR
var self = this;
this._setStorageModificationTime(item, function (item, mtime) {
this._setStorageModificationTime(item, function (item, props) {
if (!request.isRunning()) {
Zotero.debug("Upload request '" + request.name
+ "' is no longer running after getting mod time");
@ -602,10 +648,8 @@ Zotero.Sync.Storage.Session.WebDAV.prototype._onUploadComplete = function (httpR
Zotero.DB.beginTransaction();
Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
Zotero.Sync.Storage.setSyncedModificationTime(item.id, mtime, true);
var hash = item.attachmentHash;
Zotero.Sync.Storage.setSyncedHash(item.id, hash);
Zotero.Sync.Storage.setSyncedModificationTime(item.id, props.mtime, true);
Zotero.Sync.Storage.setSyncedHash(item.id, props.hash);
Zotero.DB.commitTransaction();

41
chrome/content/zotero/xpcom/storage/zfs.js
View file

@ -130,7 +130,8 @@ Zotero.Sync.Storage.Session.ZFS.prototype._getStorageFileInfo = function (item,
var info = {};
info.hash = req.getResponseHeader('ETag');
info.filename = req.getResponseHeader('X-Zotero-Filename');
info.mtime = req.getResponseHeader('X-Zotero-Modification-Time');
var mtime = req.getResponseHeader('X-Zotero-Modification-Time');
info.mtime = parseInt(mtime);
info.compressed = req.getResponseHeader('X-Zotero-Compressed') == 'Yes';
Zotero.debug(info);
@ -184,7 +185,7 @@ Zotero.Sync.Storage.Session.ZFS.prototype.downloadFile = function (request) {
var file = item.getFile();
// Skip download if local file exists and matches mod time
if (file && file.exists()) {
if (syncModTime == Math.round(file.lastModifiedTime / 1000)) {
if (syncModTime == file.lastModifiedTime) {
Zotero.debug("File mod time matches remote file -- skipping download");
Zotero.DB.beginTransaction();
@ -346,20 +347,34 @@ Zotero.Sync.Storage.Session.ZFS.prototype._processUploadFile = function (data) {
if (info) {
// Remote mod time
var mtime = info.mtime;
// Local file time
var fmtime = item.attachmentModificationTime;
var same = false;
if (fmtime == mtime) {
same = true;
Zotero.debug("File mod time matches remote file -- skipping upload");
}
// Allow floored timestamps for filesystems that don't support
// millisecond precision (e.g., HFS+)
else if (Math.floor(mtime / 1000) * 1000 == fmtime || Math.floor(fmtime / 1000) * 1000 == mtime) {
same = true;
Zotero.debug("File mod times are within one-second precision (" + fmtime + " ≅ " + mtime + ") "
+ "-- skipping upload");
}
// Allow timestamp to be exactly one hour off to get around
// time zone issues -- there may be a proper way to fix this
if (fmtime == mtime || Math.abs(fmtime - mtime) == 3600) {
if (fmtime == mtime) {
Zotero.debug("File mod time matches remote file -- skipping upload");
}
else {
Zotero.debug("File mod time (" + fmtime + ") is exactly one hour off remote file (" + mtime + ") "
+ "-- assuming time zone issue and skipping upload");
}
else if (Math.abs(fmtime - mtime) == 3600000
// And check with one-second precision as well
|| Math.abs(fmtime - Math.floor(mtime / 1000) * 1000) == 3600000
|| Math.abs(Math.floor(fmtime / 1000) * 1000 - mtime) == 3600000) {
same = true;
Zotero.debug("File mod time (" + fmtime + ") is exactly one hour off remote file (" + mtime + ") "
+ "-- assuming time zone issue and skipping upload");
}
if (same) {
Zotero.debug(Zotero.Sync.Storage.getSyncedModificationTime(item.id));
Zotero.DB.beginTransaction();
@ -804,7 +819,7 @@ Zotero.Sync.Storage.Session.ZFS.prototype.getLastSyncTime = function (callback)
if (req.status == 200) {
var ts = req.responseText;
var date = new Date(req.responseText * 1000);
var date = new Date(ts * 1000);
Zotero.debug("Last successful storage sync was " + date);
self._lastSyncTime = ts;
}
@ -934,7 +949,7 @@ Zotero.Sync.Storage.Session.ZFS.prototype.purgeDeletedStorageFiles = function (c
*/
Zotero.Sync.Storage.Session.ZFS.prototype._getItemURI = function (item) {
var uri = this.rootURI;
uri.spec += Zotero.URI.getItemPath(item) + '/file?auth=1&iskey=1';
uri.spec += Zotero.URI.getItemPath(item) + '/file?auth=1&iskey=1&version=1';
return uri;
}

10
chrome/content/zotero/xpcom/sync.js
View file

@ -1133,7 +1133,7 @@ Zotero.Sync.Server = new function () {
this.canAutoResetClient = true;
this.manualSyncRequired = false;
this.nextLocalSyncDate = false;
this.apiVersion = 7;
this.apiVersion = 8;
default xml namespace = '';
@ -2450,7 +2450,8 @@ Zotero.Sync.Server.Data = new function() {
continue;
case 'item':
var diff = obj.diff(remoteObj, false, ["dateModified"]);
var diff = obj.diff(remoteObj, false, ["dateAdded", "dateModified"]);
Zotero.debug('Diff:');
Zotero.debug(diff);
if (!diff) {
// Check if creators changed
@ -2655,6 +2656,11 @@ Zotero.Sync.Server.Data = new function() {
var mtime = xmlNode.@storageModTime.toString();
if (mtime) {
var lk = Zotero.Items.getLibraryKeyHash(obj)
// Convert previously used Unix timestamps to ms-based timestamps
if (mtime < 10000000000) {
Zotero.debug("Converting Unix timestamp '" + mtime + "' to milliseconds");
mtime = mtime * 1000;
}
itemStorageModTimes[lk] = parseInt(mtime);
}
}

2
userdata.sql
View file

@ -1,4 +1,4 @@
-- 70
-- 71
-- Copyright (c) 2009 Center for History and New Media
-- George Mason University, Fairfax, Virginia, USA