Overhaul object downloading/processing during data syncs
Previously, objects were first downloaded and saved to the sync cache, which was then processed separately to create/update local objects. This meant that a server bug could result in invalid data in the sync cache that would never be processed. Now, objects are saved as they're downloaded and only added to the sync cache after being successfully saved. The keys of objects that fail are added to a queue, and those objects are refetched and retried on a backoff schedule or when a new client version is installed (in case of a client bug or a client with outdated data model support). An alternative would be to save to the sync cache first and evict objects that fail and add them to the queue, but that requires more complicated logic, and it probably makes more sense just to buffer a few downloads ahead so that processing is never waiting for downloads to finish.
This commit is contained in:
Dan Stillman
parent
6ac35c75c1
commit
a1ce85decb
17 changed files with 1251 additions and 485 deletions
|
|
@ -46,7 +46,8 @@ describe("Zotero.Sync.Data.Engine", function () {
|
|||
data: {
|
||||
key: options.key,
|
||||
version: options.version,
|
||||
name: options.name
|
||||
name: options.name,
|
||||
parentCollection: options.parentCollection
|
||||
}
|
||||
};
|
||||
}
|
||||
|
|
@ -93,6 +94,14 @@ describe("Zotero.Sync.Data.Engine", function () {
|
|||
item: makeItemJSON
|
||||
};
|
||||
|
||||
var assertInCache = Zotero.Promise.coroutine(function* (obj) {
|
||||
var cacheObject = yield Zotero.Sync.Data.Local.getCacheObject(
|
||||
obj.objectType, obj.libraryID, obj.key, obj.version
|
||||
);
|
||||
assert.isObject(cacheObject);
|
||||
assert.propertyVal(cacheObject, 'key', obj.key);
|
||||
});
|
||||
|
||||
//
|
||||
// Tests
|
||||
//
|
||||
|
|
@ -250,23 +259,27 @@ describe("Zotero.Sync.Data.Engine", function () {
|
|||
assert.equal(obj.name, 'A');
|
||||
assert.equal(obj.version, 1);
|
||||
assert.isTrue(obj.synced);
|
||||
yield assertInCache(obj);
|
||||
|
||||
obj = yield Zotero.Searches.getByLibraryAndKeyAsync(userLibraryID, "AAAAAAAA");
|
||||
assert.equal(obj.name, 'A');
|
||||
assert.equal(obj.version, 2);
|
||||
assert.isTrue(obj.synced);
|
||||
yield assertInCache(obj);
|
||||
|
||||
obj = yield Zotero.Items.getByLibraryAndKeyAsync(userLibraryID, "AAAAAAAA");
|
||||
assert.equal(obj.getField('title'), 'A');
|
||||
assert.equal(obj.version, 3);
|
||||
assert.isTrue(obj.synced);
|
||||
var parentItemID = obj.id;
|
||||
yield assertInCache(obj);
|
||||
|
||||
obj = yield Zotero.Items.getByLibraryAndKeyAsync(userLibraryID, "BBBBBBBB");
|
||||
assert.equal(obj.getNote(), 'This is a note.');
|
||||
assert.equal(obj.parentItemID, parentItemID);
|
||||
assert.equal(obj.version, 3);
|
||||
assert.isTrue(obj.synced);
|
||||
yield assertInCache(obj);
|
||||
})
|
||||
|
||||
it("should upload new full items and subsequent patches", function* () {
|
||||
|
|
@ -651,10 +664,211 @@ describe("Zotero.Sync.Data.Engine", function () {
|
|||
});
|
||||
yield engine.start();
|
||||
})
|
||||
|
||||
it("should ignore errors when saving downloaded objects", function* () {
|
||||
({ engine, client, caller } = yield setup());
|
||||
engine.stopOnError = false;
|
||||
|
||||
var headers = {
|
||||
"Last-Modified-Version": 3
|
||||
};
|
||||
setResponse({
|
||||
method: "GET",
|
||||
url: "users/1/settings",
|
||||
status: 200,
|
||||
headers: headers,
|
||||
json: {}
|
||||
});
|
||||
setResponse({
|
||||
method: "GET",
|
||||
url: "users/1/collections?format=versions",
|
||||
status: 200,
|
||||
headers: headers,
|
||||
json: {
|
||||
"AAAAAAAA": 1,
|
||||
"BBBBBBBB": 1,
|
||||
"CCCCCCCC": 1
|
||||
}
|
||||
});
|
||||
setResponse({
|
||||
method: "GET",
|
||||
url: "users/1/searches?format=versions",
|
||||
status: 200,
|
||||
headers: headers,
|
||||
json: {
|
||||
"DDDDDDDD": 2,
|
||||
"EEEEEEEE": 2,
|
||||
"FFFFFFFF": 2
|
||||
}
|
||||
});
|
||||
setResponse({
|
||||
method: "GET",
|
||||
url: "users/1/items/top?format=versions&includeTrashed=1",
|
||||
status: 200,
|
||||
headers: headers,
|
||||
json: {
|
||||
"GGGGGGGG": 3,
|
||||
"HHHHHHHH": 3
|
||||
}
|
||||
});
|
||||
setResponse({
|
||||
method: "GET",
|
||||
url: "users/1/items?format=versions&includeTrashed=1",
|
||||
status: 200,
|
||||
headers: headers,
|
||||
json: {
|
||||
"GGGGGGGG": 3,
|
||||
"HHHHHHHH": 3,
|
||||
"JJJJJJJJ": 3
|
||||
}
|
||||
});
|
||||
setResponse({
|
||||
method: "GET",
|
||||
url: "users/1/collections?format=json&collectionKey=AAAAAAAA%2CBBBBBBBB%2CCCCCCCCC",
|
||||
status: 200,
|
||||
headers: headers,
|
||||
json: [
|
||||
makeCollectionJSON({
|
||||
key: "AAAAAAAA",
|
||||
version: 1,
|
||||
name: "A"
|
||||
}),
|
||||
makeCollectionJSON({
|
||||
key: "BBBBBBBB",
|
||||
version: 1,
|
||||
name: "B",
|
||||
// Missing parent -- collection should be queued
|
||||
parentCollection: "ZZZZZZZZ"
|
||||
}),
|
||||
makeCollectionJSON({
|
||||
key: "CCCCCCCC",
|
||||
version: 1,
|
||||
name: "C",
|
||||
// Unknown field -- should be ignored
|
||||
unknownField: 5
|
||||
})
|
||||
]
|
||||
});
|
||||
setResponse({
|
||||
method: "GET",
|
||||
url: "users/1/searches?format=json&searchKey=DDDDDDDD%2CEEEEEEEE%2CFFFFFFFF",
|
||||
status: 200,
|
||||
headers: headers,
|
||||
json: [
|
||||
makeSearchJSON({
|
||||
key: "DDDDDDDD",
|
||||
version: 2,
|
||||
name: "D",
|
||||
conditions: [
|
||||
{
|
||||
condition: "title",
|
||||
operator: "is",
|
||||
value: "a"
|
||||
}
|
||||
]
|
||||
}),
|
||||
makeSearchJSON({
|
||||
key: "EEEEEEEE",
|
||||
version: 2,
|
||||
name: "E",
|
||||
conditions: [
|
||||
{
|
||||
// Unknown search condition -- search should be queued
|
||||
condition: "unknownCondition",
|
||||
operator: "is",
|
||||
value: "a"
|
||||
}
|
||||
]
|
||||
}),
|
||||
makeSearchJSON({
|
||||
key: "FFFFFFFF",
|
||||
version: 2,
|
||||
name: "F",
|
||||
conditions: [
|
||||
{
|
||||
condition: "title",
|
||||
// Unknown search operator -- search should be queued
|
||||
operator: "unknownOperator",
|
||||
value: "a"
|
||||
}
|
||||
]
|
||||
})
|
||||
]
|
||||
});
|
||||
setResponse({
|
||||
method: "GET",
|
||||
url: "users/1/items?format=json&itemKey=GGGGGGGG%2CHHHHHHHH&includeTrashed=1",
|
||||
status: 200,
|
||||
headers: headers,
|
||||
json: [
|
||||
makeItemJSON({
|
||||
key: "GGGGGGGG",
|
||||
version: 3,
|
||||
itemType: "book",
|
||||
title: "G",
|
||||
// Unknown item field -- should be ignored
|
||||
unknownField: "B"
|
||||
}),
|
||||
makeItemJSON({
|
||||
key: "HHHHHHHH",
|
||||
version: 3,
|
||||
// Unknown item type -- item should be queued
|
||||
itemType: "unknownItemType",
|
||||
title: "H"
|
||||
})
|
||||
]
|
||||
});
|
||||
setResponse({
|
||||
method: "GET",
|
||||
url: "users/1/items?format=json&itemKey=JJJJJJJJ&includeTrashed=1",
|
||||
status: 200,
|
||||
headers: headers,
|
||||
json: [
|
||||
makeItemJSON({
|
||||
key: "JJJJJJJJ",
|
||||
version: 3,
|
||||
itemType: "note",
|
||||
// Parent that couldn't be saved -- item should be queued
|
||||
parentItem: "HHHHHHHH",
|
||||
note: "This is a note."
|
||||
})
|
||||
]
|
||||
});
|
||||
setResponse({
|
||||
method: "GET",
|
||||
url: "users/1/deleted?since=0",
|
||||
status: 200,
|
||||
headers: headers,
|
||||
json: {}
|
||||
});
|
||||
var spy = sinon.spy(engine, "onError");
|
||||
yield engine.start();
|
||||
|
||||
var userLibraryID = Zotero.Libraries.userLibraryID;
|
||||
|
||||
// Library version should have been updated
|
||||
assert.equal(Zotero.Libraries.getVersion(userLibraryID), 3);
|
||||
|
||||
// Check for saved objects
|
||||
yield assert.eventually.ok(Zotero.Collections.getByLibraryAndKeyAsync(userLibraryID, "AAAAAAAA"));
|
||||
yield assert.eventually.ok(Zotero.Searches.getByLibraryAndKeyAsync(userLibraryID, "DDDDDDDD"));
|
||||
yield assert.eventually.ok(Zotero.Items.getByLibraryAndKeyAsync(userLibraryID, "GGGGGGGG"));
|
||||
|
||||
var keys = yield Zotero.Sync.Data.Local.getObjectsFromSyncQueue('collection', userLibraryID);
|
||||
assert.sameMembers(keys, ['BBBBBBBB']);
|
||||
|
||||
var keys = yield Zotero.Sync.Data.Local.getObjectsFromSyncQueue('search', userLibraryID);
|
||||
assert.sameMembers(keys, ['EEEEEEEE', 'FFFFFFFF']);
|
||||
|
||||
var keys = yield Zotero.Sync.Data.Local.getObjectsFromSyncQueue('item', userLibraryID);
|
||||
assert.sameMembers(keys, ['HHHHHHHH', 'JJJJJJJJ']);
|
||||
|
||||
assert.equal(spy.callCount, 3);
|
||||
});
|
||||
})
|
||||
|
||||
describe("#_startDownload()", function () {
|
||||
it("shouldn't redownload objects already in the cache", function* () {
|
||||
it("shouldn't redownload objects that are already up to date", function* () {
|
||||
var userLibraryID = Zotero.Libraries.userLibraryID;
|
||||
//yield Zotero.Libraries.setVersion(userLibraryID, 5);
|
||||
({ engine, client, caller } = yield setup());
|
||||
|
|
@ -1230,6 +1444,7 @@ describe("Zotero.Sync.Data.Engine", function () {
|
|||
let obj = objectsClass.getByLibraryAndKey(userLibraryID, objectJSON[type][0].key);
|
||||
assert.equal(obj.version, 20);
|
||||
assert.isTrue(obj.synced);
|
||||
yield assertInCache(obj);
|
||||
|
||||
// JSON objects 2 should be marked as unsynced, with their version reset to 0
|
||||
assert.equal(objects[type][1].version, 0);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue