mirrors/zotero
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Overhaul object downloading/processing during data syncs

Browse source 
Previously, objects were first downloaded and saved to the sync cache,
which was then processed separately to create/update local objects. This
meant that a server bug could result in invalid data in the sync cache
that would never be processed. Now, objects are saved as they're
downloaded and only added to the sync cache after being successfully
saved. The keys of objects that fail are added to a queue, and those
objects are refetched and retried on a backoff schedule or when a new
client version is installed (in case of a client bug or a client with
outdated data model support).

An alternative would be to save to the sync cache first and evict
objects that fail and add them to the queue, but that requires more
complicated logic, and it probably makes more sense just to buffer a few
downloads ahead so that processing is never waiting for downloads to
finish.
This commit is contained in:
Dan Stillman 2016-02-29 04:23:00 -05:00
parent 6ac35c75c1
commit a1ce85decb
17 changed files with 1251 additions and 485 deletions
Download patch file Download diff file Expand all files Collapse all files

26
test/tests/schemaTest.js
View file

@ -1,4 +1,30 @@
describe("Zotero.Schema", function() {
describe("#initializeSchema()", function () {
it("should set last client version", function* () {
yield resetDB({
thisArg: this,
skipBundledFiles: true
});
var sql = "SELECT value FROM settings WHERE setting='client' AND key='lastVersion'";
var lastVersion = yield Zotero.DB.valueQueryAsync(sql);
yield assert.eventually.equal(Zotero.DB.valueQueryAsync(sql), Zotero.version);
});
});
describe("#updateSchema()", function () {
it("should set last client version", function* () {
var sql = "REPLACE INTO settings (setting, key, value) VALUES ('client', 'lastVersion', ?)";
return Zotero.DB.queryAsync(sql, "5.0old");
yield Zotero.Schema.updateSchema();
var sql = "SELECT value FROM settings WHERE setting='client' AND key='lastVersion'";
var lastVersion = yield Zotero.DB.valueQueryAsync(sql);
yield assert.eventually.equal(Zotero.DB.valueQueryAsync(sql), Zotero.version);
});
});
describe("#integrityCheck()", function () {
before(function* () {
yield resetDB({

8
test/tests/storageLocalTest.js
View file

@ -144,12 +144,8 @@ describe("Zotero.Sync.Storage.Local", function () {
md5,
mtime
};
yield Zotero.Sync.Data.Local.saveCacheObjects(
'item', libraryID, [json]
);
yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
libraryID, 'item', { stopOnError: true }
);
yield Zotero.Sync.Data.Local.processObjectsFromJSON('item', libraryID, [json]);
var item = yield Zotero.Items.getByLibraryAndKeyAsync(libraryID, key);
yield Zotero.Sync.Storage.Local.processDownload({
item,

29
test/tests/syncAPIClientTest.js
View file

@ -12,6 +12,10 @@ describe("Zotero.Sync.APIClient", function () {
}
before(function () {
Zotero.HTTP.mock = sinon.FakeXMLHttpRequest;
});
beforeEach(function () {
Components.utils.import("resource://zotero/concurrentCaller.js");
var caller = new ConcurrentCaller(1);
caller.setLogger(msg => Zotero.debug(msg));
@ -24,17 +28,13 @@ describe("Zotero.Sync.APIClient", function () {
}
};
Zotero.HTTP.mock = sinon.FakeXMLHttpRequest;
client = new Zotero.Sync.APIClient({
baseURL,
apiVersion: ZOTERO_CONFIG.API_VERSION,
apiKey,
caller
})
})
beforeEach(function () {
server = sinon.fakeServer.create();
server.autoRespond = true;
})
@ -44,16 +44,29 @@ describe("Zotero.Sync.APIClient", function () {
})
describe("#_checkConnection()", function () {
it("should catch an error with an empty response", function* () {
var spy;
beforeEach(function () {
client.failureDelayIntervals = [10];
client.failureDelayMax = 15;
});
afterEach(function () {
if (spy) {
spy.restore();
}
});
it("should retry on 500 error", function* () {
setResponse({
method: "GET",
url: "error",
status: 500,
text: ""
})
var spy = sinon.spy(Zotero.HTTP, "request");
var e = yield getPromiseError(client.makeRequest("GET", baseURL + "error"));
assert.ok(e);
assert.isTrue(e.message.startsWith(Zotero.getString('sync.error.emptyResponseServer')));
assert.instanceOf(e, Zotero.HTTP.UnexpectedStatusException);
assert.isTrue(spy.calledTwice);
})
it("should catch an interrupted connection", function* () {

219
test/tests/syncEngineTest.js
View file

@ -46,7 +46,8 @@ describe("Zotero.Sync.Data.Engine", function () {
data: {
key: options.key,
version: options.version,
name: options.name
name: options.name,
parentCollection: options.parentCollection
}
};
}
@ -93,6 +94,14 @@ describe("Zotero.Sync.Data.Engine", function () {
item: makeItemJSON
};
var assertInCache = Zotero.Promise.coroutine(function* (obj) {
var cacheObject = yield Zotero.Sync.Data.Local.getCacheObject(
obj.objectType, obj.libraryID, obj.key, obj.version
);
assert.isObject(cacheObject);
assert.propertyVal(cacheObject, 'key', obj.key);
});
//
// Tests
//
@ -250,23 +259,27 @@ describe("Zotero.Sync.Data.Engine", function () {
assert.equal(obj.name, 'A');
assert.equal(obj.version, 1);
assert.isTrue(obj.synced);
yield assertInCache(obj);
obj = yield Zotero.Searches.getByLibraryAndKeyAsync(userLibraryID, "AAAAAAAA");
assert.equal(obj.name, 'A');
assert.equal(obj.version, 2);
assert.isTrue(obj.synced);
yield assertInCache(obj);
obj = yield Zotero.Items.getByLibraryAndKeyAsync(userLibraryID, "AAAAAAAA");
assert.equal(obj.getField('title'), 'A');
assert.equal(obj.version, 3);
assert.isTrue(obj.synced);
var parentItemID = obj.id;
yield assertInCache(obj);
obj = yield Zotero.Items.getByLibraryAndKeyAsync(userLibraryID, "BBBBBBBB");
assert.equal(obj.getNote(), 'This is a note.');
assert.equal(obj.parentItemID, parentItemID);
assert.equal(obj.version, 3);
assert.isTrue(obj.synced);
yield assertInCache(obj);
})
it("should upload new full items and subsequent patches", function* () {
@ -651,10 +664,211 @@ describe("Zotero.Sync.Data.Engine", function () {
});
yield engine.start();
})
it("should ignore errors when saving downloaded objects", function* () {
({ engine, client, caller } = yield setup());
engine.stopOnError = false;
var headers = {
"Last-Modified-Version": 3
};
setResponse({
method: "GET",
url: "users/1/settings",
status: 200,
headers: headers,
json: {}
});
setResponse({
method: "GET",
url: "users/1/collections?format=versions",
status: 200,
headers: headers,
json: {
"AAAAAAAA": 1,
"BBBBBBBB": 1,
"CCCCCCCC": 1
}
});
setResponse({
method: "GET",
url: "users/1/searches?format=versions",
status: 200,
headers: headers,
json: {
"DDDDDDDD": 2,
"EEEEEEEE": 2,
"FFFFFFFF": 2
}
});
setResponse({
method: "GET",
url: "users/1/items/top?format=versions&includeTrashed=1",
status: 200,
headers: headers,
json: {
"GGGGGGGG": 3,
"HHHHHHHH": 3
}
});
setResponse({
method: "GET",
url: "users/1/items?format=versions&includeTrashed=1",
status: 200,
headers: headers,
json: {
"GGGGGGGG": 3,
"HHHHHHHH": 3,
"JJJJJJJJ": 3
}
});
setResponse({
method: "GET",
url: "users/1/collections?format=json&collectionKey=AAAAAAAA%2CBBBBBBBB%2CCCCCCCCC",
status: 200,
headers: headers,
json: [
makeCollectionJSON({
key: "AAAAAAAA",
version: 1,
name: "A"
}),
makeCollectionJSON({
key: "BBBBBBBB",
version: 1,
name: "B",
// Missing parent -- collection should be queued
parentCollection: "ZZZZZZZZ"
}),
makeCollectionJSON({
key: "CCCCCCCC",
version: 1,
name: "C",
// Unknown field -- should be ignored
unknownField: 5
})
]
});
setResponse({
method: "GET",
url: "users/1/searches?format=json&searchKey=DDDDDDDD%2CEEEEEEEE%2CFFFFFFFF",
status: 200,
headers: headers,
json: [
makeSearchJSON({
key: "DDDDDDDD",
version: 2,
name: "D",
conditions: [
{
condition: "title",
operator: "is",
value: "a"
}
]
}),
makeSearchJSON({
key: "EEEEEEEE",
version: 2,
name: "E",
conditions: [
{
// Unknown search condition -- search should be queued
condition: "unknownCondition",
operator: "is",
value: "a"
}
]
}),
makeSearchJSON({
key: "FFFFFFFF",
version: 2,
name: "F",
conditions: [
{
condition: "title",
// Unknown search operator -- search should be queued
operator: "unknownOperator",
value: "a"
}
]
})
]
});
setResponse({
method: "GET",
url: "users/1/items?format=json&itemKey=GGGGGGGG%2CHHHHHHHH&includeTrashed=1",
status: 200,
headers: headers,
json: [
makeItemJSON({
key: "GGGGGGGG",
version: 3,
itemType: "book",
title: "G",
// Unknown item field -- should be ignored
unknownField: "B"
}),
makeItemJSON({
key: "HHHHHHHH",
version: 3,
// Unknown item type -- item should be queued
itemType: "unknownItemType",
title: "H"
})
]
});
setResponse({
method: "GET",
url: "users/1/items?format=json&itemKey=JJJJJJJJ&includeTrashed=1",
status: 200,
headers: headers,
json: [
makeItemJSON({
key: "JJJJJJJJ",
version: 3,
itemType: "note",
// Parent that couldn't be saved -- item should be queued
parentItem: "HHHHHHHH",
note: "This is a note."
})
]
});
setResponse({
method: "GET",
url: "users/1/deleted?since=0",
status: 200,
headers: headers,
json: {}
});
var spy = sinon.spy(engine, "onError");
yield engine.start();
var userLibraryID = Zotero.Libraries.userLibraryID;
// Library version should have been updated
assert.equal(Zotero.Libraries.getVersion(userLibraryID), 3);
// Check for saved objects
yield assert.eventually.ok(Zotero.Collections.getByLibraryAndKeyAsync(userLibraryID, "AAAAAAAA"));
yield assert.eventually.ok(Zotero.Searches.getByLibraryAndKeyAsync(userLibraryID, "DDDDDDDD"));
yield assert.eventually.ok(Zotero.Items.getByLibraryAndKeyAsync(userLibraryID, "GGGGGGGG"));
var keys = yield Zotero.Sync.Data.Local.getObjectsFromSyncQueue('collection', userLibraryID);
assert.sameMembers(keys, ['BBBBBBBB']);
var keys = yield Zotero.Sync.Data.Local.getObjectsFromSyncQueue('search', userLibraryID);
assert.sameMembers(keys, ['EEEEEEEE', 'FFFFFFFF']);
var keys = yield Zotero.Sync.Data.Local.getObjectsFromSyncQueue('item', userLibraryID);
assert.sameMembers(keys, ['HHHHHHHH', 'JJJJJJJJ']);
assert.equal(spy.callCount, 3);
});
})
describe("#_startDownload()", function () {
it("shouldn't redownload objects already in the cache", function* () {
it("shouldn't redownload objects that are already up to date", function* () {
var userLibraryID = Zotero.Libraries.userLibraryID;
//yield Zotero.Libraries.setVersion(userLibraryID, 5);
({ engine, client, caller } = yield setup());
@ -1230,6 +1444,7 @@ describe("Zotero.Sync.Data.Engine", function () {
let obj = objectsClass.getByLibraryAndKey(userLibraryID, objectJSON[type][0].key);
assert.equal(obj.version, 20);
assert.isTrue(obj.synced);
yield assertInCache(obj);
// JSON objects 2 should be marked as unsynced, with their version reset to 0
assert.equal(objects[type][1].version, 0);

312
test/tests/syncLocalTest.js
View file

@ -89,10 +89,10 @@ describe("Zotero.Sync.Data.Local", function() {
})
describe("#processSyncCacheForObjectType()", function () {
describe("#processObjectsFromJSON()", function () {
var types = Zotero.DataObjectUtilities.getTypes();
before(function* () {
beforeEach(function* () {
yield resetDB({
thisArg: this,
skipBundledFiles: true
@ -113,11 +113,8 @@ describe("Zotero.Sync.Data.Local", function() {
version: 10,
data: data
};
yield Zotero.Sync.Data.Local.saveCacheObjects(
type, libraryID, [json]
);
yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
libraryID, type, { stopOnError: true }
yield Zotero.Sync.Data.Local.processObjectsFromJSON(
type, libraryID, [json], { stopOnError: true }
);
let localObj = objectsClass.getByLibraryAndKey(libraryID, obj.key);
assert.equal(localObj.version, 10);
@ -148,12 +145,8 @@ describe("Zotero.Sync.Data.Local", function() {
version: 10,
data: data
};
yield Zotero.Sync.Data.Local.saveCacheObjects(
type, libraryID, [json]
);
yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
libraryID, type, { stopOnError: true }
yield Zotero.Sync.Data.Local.processObjectsFromJSON(
type, libraryID, [json], { stopOnError: true }
);
assert.equal(obj.version, 10);
assert.equal(obj.getField('title'), changedTitle);
@ -164,23 +157,19 @@ describe("Zotero.Sync.Data.Local", function() {
var libraryID = Zotero.Libraries.userLibraryID;
for (let type of types) {
let objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(type);
// Save original version
let obj = yield createDataObject(type, { version: 5 });
let data = obj.toJSON();
yield Zotero.Sync.Data.Local.saveCacheObjects(
type, libraryID, [data]
);
}
for (let type of types) {
let objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(type);
let obj = yield createDataObject(type, { version: 10 });
let data = obj.toJSON();
yield Zotero.Sync.Data.Local.saveCacheObjects(
type, libraryID, [data]
);
yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
libraryID, type, { stopOnError: true }
// Save newer version
data.version = 10;
yield Zotero.Sync.Data.Local.processObjectsFromJSON(
type, libraryID, [data], { stopOnError: true }
);
let localObj = objectsClass.getByLibraryAndKey(libraryID, obj.key);
@ -195,7 +184,36 @@ describe("Zotero.Sync.Data.Local", function() {
"should have only latest version of " + type + " in cache"
);
}
})
});
it("should delete object from sync queue after processing", function* () {
var objectType = 'item';
var libraryID = Zotero.Libraries.userLibraryID;
var key = Zotero.DataObjectUtilities.generateKey();
yield Zotero.Sync.Data.Local.addObjectsToSyncQueue(objectType, libraryID, [key]);
var versions = yield Zotero.Sync.Data.Local.getObjectsFromSyncQueue(objectType, libraryID);
assert.include(versions, key);
var json = {
key,
version: 10,
data: {
key,
version: 10,
itemType: "book",
title: "Test"
}
};
yield Zotero.Sync.Data.Local.processObjectsFromJSON(
objectType, libraryID, [json], { stopOnError: true }
);
var versions = yield Zotero.Sync.Data.Local.getObjectsFromSyncQueue(objectType, libraryID);
assert.notInclude(versions, key);
});
it("should mark new attachment items for download", function* () {
var libraryID = Zotero.Libraries.userLibraryID;
@ -216,11 +234,8 @@ describe("Zotero.Sync.Data.Local", function() {
}
};
yield Zotero.Sync.Data.Local.saveCacheObjects(
'item', Zotero.Libraries.userLibraryID, [json]
);
yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
libraryID, 'item', { stopOnError: true }
yield Zotero.Sync.Data.Local.processObjectsFromJSON(
'item', libraryID, [json], { stopOnError: true }
);
var item = Zotero.Items.getByLibraryAndKey(libraryID, key);
assert.equal(item.attachmentSyncState, Zotero.Sync.Storage.Local.SYNC_STATE_TO_DOWNLOAD);
@ -247,12 +262,8 @@ describe("Zotero.Sync.Data.Local", function() {
json.data.version = 10;
json.data.md5 = '57f8a4fda823187b91e1191487b87fe6';
json.data.mtime = new Date().getTime() + 10000;
yield Zotero.Sync.Data.Local.saveCacheObjects(
'item', Zotero.Libraries.userLibraryID, [json]
);
yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
libraryID, 'item', { stopOnError: true }
yield Zotero.Sync.Data.Local.processObjectsFromJSON(
'item', libraryID, [json], { stopOnError: true }
);
assert.equal(item.attachmentSyncState, Zotero.Sync.Storage.Local.SYNC_STATE_TO_DOWNLOAD);
@ -284,17 +295,188 @@ describe("Zotero.Sync.Data.Local", function() {
json.data.version = 10;
json.data.md5 = '57f8a4fda823187b91e1191487b87fe6';
json.data.mtime = new Date().getTime() + 10000;
yield Zotero.Sync.Data.Local.saveCacheObjects('item', libraryID, [json]);
yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
libraryID, 'item', { stopOnError: true }
yield Zotero.Sync.Data.Local.processObjectsFromJSON(
'item', libraryID, [json], { stopOnError: true }
);
assert.equal(item.getField('title'), newTitle);
assert.equal(item.attachmentSyncState, Zotero.Sync.Storage.Local.SYNC_STATE_TO_DOWNLOAD);
})
it("should roll back partial object changes on error", function* () {
var libraryID = Zotero.Libraries.publicationsLibraryID;
var key1 = "AAAAAAAA";
var key2 = "BBBBBBBB";
var json = [
{
key: key1,
version: 1,
data: {
key: key1,
version: 1,
itemType: "book",
title: "Test A"
}
},
{
key: key2,
version: 1,
data: {
key: key2,
version: 1,
itemType: "journalArticle",
title: "Test B",
deleted: true // Not allowed in My Publications
}
}
];
yield Zotero.Sync.Data.Local.processObjectsFromJSON('item', libraryID, json);
// Shouldn't roll back the successful item
yield assert.eventually.equal(Zotero.DB.valueQueryAsync(
"SELECT COUNT(*) FROM items WHERE libraryID=? AND key=?", [libraryID, key1]
), 1);
// Should rollback the unsuccessful item
yield assert.eventually.equal(Zotero.DB.valueQueryAsync(
"SELECT COUNT(*) FROM items WHERE libraryID=? AND key=?", [libraryID, key2]
), 0);
});
})
describe("Sync Queue", function () {
var lib1, lib2;
before(function* () {
lib1 = Zotero.Libraries.userLibraryID;
lib2 = Zotero.Libraries.publicationsLibraryID;
});
beforeEach(function* () {
yield Zotero.DB.queryAsync("DELETE FROM syncQueue");
});
after(function* () {
yield Zotero.DB.queryAsync("DELETE FROM syncQueue");
});
describe("#addObjectsToSyncQueue()", function () {
it("should add new objects and update lastCheck and tries for existing objects", function* () {
var objectType = 'item';
var syncObjectTypeID = Zotero.Sync.Data.Utilities.getSyncObjectTypeID(objectType);
var now = Zotero.Date.getUnixTimestamp();
var key1 = Zotero.DataObjectUtilities.generateKey();
var key2 = Zotero.DataObjectUtilities.generateKey();
var key3 = Zotero.DataObjectUtilities.generateKey();
var key4 = Zotero.DataObjectUtilities.generateKey();
yield Zotero.DB.queryAsync(
"INSERT INTO syncQueue (libraryID, key, syncObjectTypeID, lastCheck, tries) "
+ "VALUES (?, ?, ?, ?, ?), (?, ?, ?, ?, ?), (?, ?, ?, ?, ?)",
[
lib1, key1, syncObjectTypeID, now - 3700, 0,
lib1, key2, syncObjectTypeID, now - 7000, 1,
lib2, key3, syncObjectTypeID, now - 86400, 2
]
);
yield Zotero.Sync.Data.Local.addObjectsToSyncQueue(objectType, lib1, [key1, key2]);
yield Zotero.Sync.Data.Local.addObjectsToSyncQueue(objectType, lib2, [key4]);
var sql = "SELECT lastCheck, tries FROM syncQueue WHERE libraryID=? "
+ `AND syncObjectTypeID=${syncObjectTypeID} AND key=?`;
var row;
// key1
row = yield Zotero.DB.rowQueryAsync(sql, [lib1, key1]);
assert.approximately(row.lastCheck, now, 1);
assert.equal(row.tries, 1);
// key2
row = yield Zotero.DB.rowQueryAsync(sql, [lib1, key2]);
assert.approximately(row.lastCheck, now, 1);
assert.equal(row.tries, 2);
// key3
row = yield Zotero.DB.rowQueryAsync(sql, [lib2, key3]);
assert.equal(row.lastCheck, now - 86400);
assert.equal(row.tries, 2);
// key4
row = yield Zotero.DB.rowQueryAsync(sql, [lib2, key4]);
assert.approximately(row.lastCheck, now, 1);
assert.equal(row.tries, 0);
});
});
describe("#getObjectsToTryFromSyncQueue()", function () {
it("should get objects that should be retried", function* () {
var objectType = 'item';
var syncObjectTypeID = Zotero.Sync.Data.Utilities.getSyncObjectTypeID(objectType);
var now = Zotero.Date.getUnixTimestamp();
var key1 = Zotero.DataObjectUtilities.generateKey();
var key2 = Zotero.DataObjectUtilities.generateKey();
var key3 = Zotero.DataObjectUtilities.generateKey();
var key4 = Zotero.DataObjectUtilities.generateKey();
yield Zotero.DB.queryAsync(
"INSERT INTO syncQueue (libraryID, key, syncObjectTypeID, lastCheck, tries) "
+ "VALUES (?, ?, ?, ?, ?), (?, ?, ?, ?, ?), (?, ?, ?, ?, ?)",
[
lib1, key1, syncObjectTypeID, now - (30 * 60) - 10, 0, // more than half an hour, so should be retried
lib1, key2, syncObjectTypeID, now - (16 * 60 * 60) + 10, 4, // less than 16 hours, shouldn't be retried
lib2, key3, syncObjectTypeID, now - 86400 * 7, 20 // more than 64 hours, so should be retried
]
);
var keys = yield Zotero.Sync.Data.Local.getObjectsToTryFromSyncQueue('item', lib1);
assert.sameMembers(keys, [key1]);
var keys = yield Zotero.Sync.Data.Local.getObjectsToTryFromSyncQueue('item', lib2);
assert.sameMembers(keys, [key3]);
});
});
describe("#removeObjectsFromSyncQueue()", function () {
it("should remove objects from the sync queue", function* () {
var objectType = 'item';
var syncObjectTypeID = Zotero.Sync.Data.Utilities.getSyncObjectTypeID(objectType);
var now = Zotero.Date.getUnixTimestamp();
var key1 = Zotero.DataObjectUtilities.generateKey();
var key2 = Zotero.DataObjectUtilities.generateKey();
var key3 = Zotero.DataObjectUtilities.generateKey();
yield Zotero.DB.queryAsync(
"INSERT INTO syncQueue (libraryID, key, syncObjectTypeID, lastCheck, tries) "
+ "VALUES (?, ?, ?, ?, ?), (?, ?, ?, ?, ?), (?, ?, ?, ?, ?)",
[
lib1, key1, syncObjectTypeID, now, 0,
lib1, key2, syncObjectTypeID, now, 4,
lib2, key3, syncObjectTypeID, now, 20
]
);
yield Zotero.Sync.Data.Local.removeObjectsFromSyncQueue('item', lib1, [key1]);
var sql = "SELECT COUNT(*) FROM syncQueue WHERE libraryID=? "
+ `AND syncObjectTypeID=${syncObjectTypeID} AND key=?`;
assert.notOk(yield Zotero.DB.valueQueryAsync(sql, [lib1, key1]));
assert.ok(yield Zotero.DB.valueQueryAsync(sql, [lib1, key2]));
assert.ok(yield Zotero.DB.valueQueryAsync(sql, [lib2, key3]));
})
});
describe("#resetSyncQueueTries", function () {
var spy;
after(function () {
if (spy) {
spy.restore();
}
})
it("should be run on version upgrade", function* () {
var sql = "REPLACE INTO settings (setting, key, value) VALUES ('client', 'lastVersion', ?)";
yield Zotero.DB.queryAsync(sql, "5.0foo");
spy = sinon.spy(Zotero.Sync.Data.Local, "resetSyncQueueTries");
yield Zotero.Schema.updateSchema();
assert.ok(spy.called);
});
});
});
describe("Conflict Resolution", function () {
beforeEach(function* () {
yield Zotero.DB.queryAsync("DELETE FROM syncCache");
@ -311,6 +493,7 @@ describe("Zotero.Sync.Data.Local", function() {
var objects = [];
var values = [];
var dateAdded = Date.now() - 86400000;
var downloadedJSON = [];
for (let i = 0; i < 2; i++) {
values.push({
left: {},
@ -339,14 +522,15 @@ describe("Zotero.Sync.Data.Local", function() {
};
yield Zotero.Sync.Data.Local.saveCacheObjects(type, libraryID, [json]);
// Create new version in cache, simulating a download
json.version = jsonData.version = 15;
// Create updated JSON, simulating a download
values[i].right.title = jsonData.title = Zotero.Utilities.randomString();
yield Zotero.Sync.Data.Local.saveCacheObjects(type, libraryID, [json]);
values[i].right.version = json.version = jsonData.version = 15;
downloadedJSON.push(json);
// Modify object locally
yield modifyDataObject(obj, undefined, { skipDateModifiedUpdate: true });
values[i].left.title = obj.getField('title');
values[i].left.version = obj.getField('version');
}
waitForWindow('chrome://zotero/content/merge.xul', function (dialog) {
@ -373,12 +557,14 @@ describe("Zotero.Sync.Data.Local", function() {
}
wizard.getButton('finish').click();
})
yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
libraryID, type, { stopOnError: true }
yield Zotero.Sync.Data.Local.processObjectsFromJSON(
type, libraryID, downloadedJSON, { stopOnError: true }
);
assert.equal(objects[0].getField('title'), values[0].right.title);
assert.equal(objects[1].getField('title'), values[1].left.title);
assert.equal(objects[0].getField('version'), values[0].right.version);
assert.equal(objects[1].getField('version'), values[1].left.version);
})
it("should resolve all remaining conflicts with one side", function* () {
@ -388,6 +574,7 @@ describe("Zotero.Sync.Data.Local", function() {
var objects = [];
var values = [];
var downloadedJSON = [];
var dateAdded = Date.now() - 86400000;
for (let i = 0; i < 3; i++) {
values.push({
@ -418,13 +605,15 @@ describe("Zotero.Sync.Data.Local", function() {
yield Zotero.Sync.Data.Local.saveCacheObjects(type, libraryID, [json]);
// Create new version in cache, simulating a download
json.version = jsonData.version = 15;
values[i].right.title = jsonData.title = Zotero.Utilities.randomString();
values[i].right.version = json.version = jsonData.version = 15;
yield Zotero.Sync.Data.Local.saveCacheObjects(type, libraryID, [json]);
downloadedJSON.push(json);
// Modify object locally
yield modifyDataObject(obj, undefined, { skipDateModifiedUpdate: true });
values[i].left.title = obj.getField('title');
values[i].left.version = obj.getField('version');
}
waitForWindow('chrome://zotero/content/merge.xul', function (dialog) {
@ -460,13 +649,16 @@ describe("Zotero.Sync.Data.Local", function() {
}
wizard.getButton('finish').click();
})
yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
libraryID, type, { stopOnError: true }
yield Zotero.Sync.Data.Local.processObjectsFromJSON(
type, libraryID, downloadedJSON, { stopOnError: true }
);
assert.equal(objects[0].getField('title'), values[0].right.title);
assert.equal(objects[0].getField('version'), values[0].right.version);
assert.equal(objects[1].getField('title'), values[1].left.title);
assert.equal(objects[1].getField('version'), values[1].left.version);
assert.equal(objects[2].getField('title'), values[2].left.title);
assert.equal(objects[2].getField('version'), values[2].left.version);
})
it("should handle local item deletion, keeping deletion", function* () {
@ -475,6 +667,8 @@ describe("Zotero.Sync.Data.Local", function() {
var type = 'item';
var objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(type);
var downloadedJSON = [];
// Create object, generate JSON, and delete
var obj = yield createDataObject(type, { version: 10 });
var jsonData = obj.toJSON();
@ -491,7 +685,7 @@ describe("Zotero.Sync.Data.Local", function() {
// Create new version in cache, simulating a download
json.version = jsonData.version = 15;
jsonData.title = Zotero.Utilities.randomString();
yield Zotero.Sync.Data.Local.saveCacheObjects(type, libraryID, [json]);
downloadedJSON.push(json);
var windowOpened = false;
waitForWindow('chrome://zotero/content/merge.xul', function (dialog) {
@ -508,8 +702,8 @@ describe("Zotero.Sync.Data.Local", function() {
mergeGroup.leftpane.pane.click();
wizard.getButton('finish').click();
})
yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
libraryID, type, { stopOnError: true }
yield Zotero.Sync.Data.Local.processObjectsFromJSON(
type, libraryID, downloadedJSON, { stopOnError: true }
);
assert.isTrue(windowOpened);
@ -523,6 +717,8 @@ describe("Zotero.Sync.Data.Local", function() {
var type = 'item';
var objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(type);
var downloadedJSON = [];
// Create object, generate JSON, and delete
var obj = yield createDataObject(type, { version: 10 });
var jsonData = obj.toJSON();
@ -538,7 +734,7 @@ describe("Zotero.Sync.Data.Local", function() {
// Create new version in cache, simulating a download
json.version = jsonData.version = 15;
jsonData.title = Zotero.Utilities.randomString();
yield Zotero.Sync.Data.Local.saveCacheObjects(type, libraryID, [json]);
downloadedJSON.push(json);
waitForWindow('chrome://zotero/content/merge.xul', function (dialog) {
var doc = dialog.document;
@ -551,8 +747,8 @@ describe("Zotero.Sync.Data.Local", function() {
assert.equal(mergeGroup.rightpane.getAttribute('selected'), 'true');
wizard.getButton('finish').click();
})
yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
libraryID, type, { stopOnError: true }
yield Zotero.Sync.Data.Local.processObjectsFromJSON(
type, libraryID, downloadedJSON, { stopOnError: true }
);
obj = objectsClass.getByLibraryAndKey(libraryID, key);
@ -562,9 +758,9 @@ describe("Zotero.Sync.Data.Local", function() {
it("should handle note conflict", function* () {
var libraryID = Zotero.Libraries.userLibraryID;
var type = 'item';
var objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(type);
var downloadedJSON = [];
var noteText1 = "<p>A</p>";
var noteText2 = "<p>B</p>";
@ -586,7 +782,7 @@ describe("Zotero.Sync.Data.Local", function() {
// Create new version in cache, simulating a download
json.version = jsonData.version = 15;
json.data.note = noteText2;
yield Zotero.Sync.Data.Local.saveCacheObjects(type, libraryID, [json]);
downloadedJSON.push(json);
// Delete object locally
obj.setNote(noteText1);
@ -600,8 +796,8 @@ describe("Zotero.Sync.Data.Local", function() {
assert.equal(mergeGroup.rightpane.getAttribute('selected'), 'true');
wizard.getButton('finish').click();
})
yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
libraryID, type, { stopOnError: true }
yield Zotero.Sync.Data.Local.processObjectsFromJSON(
type, libraryID, downloadedJSON, { stopOnError: true }
);
obj = objectsClass.getByLibraryAndKey(libraryID, key);

98
test/tests/syncRunnerTest.js
View file

@ -105,14 +105,31 @@ describe("Zotero.Sync.Runner", function () {
//
// Helper functions
//
var setup = Zotero.Promise.coroutine(function* (options = {}) {
yield Zotero.DB.queryAsync("DELETE FROM settings WHERE setting='account'");
yield Zotero.Users.init();
function setResponse(response) {
setHTTPResponse(server, baseURL, response, responses);
}
//
// Tests
//
beforeEach(function* () {
yield resetDB({
thisArg: this,
skipBundledFiles: true
});
var runner = new Zotero.Sync.Runner_Module({ baseURL, apiKey });
userLibraryID = Zotero.Libraries.userLibraryID;
publicationsLibraryID = Zotero.Libraries.publicationsLibraryID;
Zotero.HTTP.mock = sinon.FakeXMLHttpRequest;
server = sinon.fakeServer.create();
server.autoRespond = true;
runner = new Zotero.Sync.Runner_Module({ baseURL, apiKey });
Components.utils.import("resource://zotero/concurrentCaller.js");
var caller = new ConcurrentCaller(1);
caller = new ConcurrentCaller(1);
caller.setLogger(msg => Zotero.debug(msg));
caller.stopOnError = true;
caller.onError = function (e) {
@ -126,29 +143,6 @@ describe("Zotero.Sync.Runner", function () {
}
};
return { runner, caller };
})
function setResponse(response) {
setHTTPResponse(server, baseURL, response, responses);
}
//
// Tests
//
before(function* () {
userLibraryID = Zotero.Libraries.userLibraryID;
publicationsLibraryID = Zotero.Libraries.publicationsLibraryID;
})
beforeEach(function* () {
Zotero.HTTP.mock = sinon.FakeXMLHttpRequest;
server = sinon.fakeServer.create();
server.autoRespond = true;
({ runner, caller } = yield setup());
yield Zotero.Users.setCurrentUserID(1);
yield Zotero.Users.setCurrentUsername("A");
})
@ -404,27 +398,7 @@ describe("Zotero.Sync.Runner", function () {
})
describe("#sync()", function () {
var spy;
before(function* () {
yield resetDB({
thisArg: this,
skipBundledFiles: true
});
yield Zotero.Libraries.init();
})
afterEach(function () {
if (spy) {
spy.restore();
}
});
it("should perform a sync across all libraries and update library versions", function* () {
yield Zotero.Users.setCurrentUserID(1);
yield Zotero.Users.setCurrentUsername("A");
setResponse('keyInfo.fullAccess');
setResponse('userGroups.groupVersions');
setResponse('groups.ownerGroup');
@ -675,11 +649,11 @@ describe("Zotero.Sync.Runner", function () {
// Check local library versions
assert.equal(
Zotero.Libraries.getVersion(Zotero.Libraries.userLibraryID),
Zotero.Libraries.getVersion(userLibraryID),
5
);
assert.equal(
Zotero.Libraries.getVersion(Zotero.Libraries.publicationsLibraryID),
Zotero.Libraries.getVersion(publicationsLibraryID),
10
);
assert.equal(
@ -699,8 +673,9 @@ describe("Zotero.Sync.Runner", function () {
it("should show the sync error icon on error", function* () {
yield Zotero.Users.setCurrentUserID(1);
yield Zotero.Users.setCurrentUsername("A");
let pubLib = Zotero.Libraries.get(publicationsLibraryID);
pubLib.libraryVersion = 5;
yield pubLib.save();
setResponse('keyInfo.fullAccess');
setResponse('userGroups.groupVersionsEmpty');
@ -716,6 +691,25 @@ describe("Zotero.Sync.Runner", function () {
INVALID: true // TODO: Find a cleaner error
}
});
// No publications changes
setResponse({
method: "GET",
url: "users/1/publications/settings?since=5",
status: 304,
headers: {
"Last-Modified-Version": 5
},
json: {}
});
setResponse({
method: "GET",
url: "users/1/publications/fulltext",
status: 200,
headers: {
"Last-Modified-Version": 5
},
json: {}
});
spy = sinon.spy(runner, "updateIcons");
yield runner.sync();