mirrors/zotero
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Overhaul object downloading/processing during data syncs

Browse source 
Previously, objects were first downloaded and saved to the sync cache,
which was then processed separately to create/update local objects. This
meant that a server bug could result in invalid data in the sync cache
that would never be processed. Now, objects are saved as they're
downloaded and only added to the sync cache after being successfully
saved. The keys of objects that fail are added to a queue, and those
objects are refetched and retried on a backoff schedule or when a new
client version is installed (in case of a client bug or a client with
outdated data model support).

An alternative would be to save to the sync cache first and evict
objects that fail and add them to the queue, but that requires more
complicated logic, and it probably makes more sense just to buffer a few
downloads ahead so that processing is never waiting for downloads to
finish.
This commit is contained in:
Dan Stillman 2016-02-29 04:23:00 -05:00
parent 6ac35c75c1
commit a1ce85decb
17 changed files with 1251 additions and 485 deletions
Download patch file Download diff file Expand all files Collapse all files

98
test/tests/syncRunnerTest.js
View file

@ -105,14 +105,31 @@ describe("Zotero.Sync.Runner", function () {
//
// Helper functions
//
var setup = Zotero.Promise.coroutine(function* (options = {}) {
yield Zotero.DB.queryAsync("DELETE FROM settings WHERE setting='account'");
yield Zotero.Users.init();
function setResponse(response) {
setHTTPResponse(server, baseURL, response, responses);
}
//
// Tests
//
beforeEach(function* () {
yield resetDB({
thisArg: this,
skipBundledFiles: true
});
var runner = new Zotero.Sync.Runner_Module({ baseURL, apiKey });
userLibraryID = Zotero.Libraries.userLibraryID;
publicationsLibraryID = Zotero.Libraries.publicationsLibraryID;
Zotero.HTTP.mock = sinon.FakeXMLHttpRequest;
server = sinon.fakeServer.create();
server.autoRespond = true;
runner = new Zotero.Sync.Runner_Module({ baseURL, apiKey });
Components.utils.import("resource://zotero/concurrentCaller.js");
var caller = new ConcurrentCaller(1);
caller = new ConcurrentCaller(1);
caller.setLogger(msg => Zotero.debug(msg));
caller.stopOnError = true;
caller.onError = function (e) {
@ -126,29 +143,6 @@ describe("Zotero.Sync.Runner", function () {
}
};
return { runner, caller };
})
function setResponse(response) {
setHTTPResponse(server, baseURL, response, responses);
}
//
// Tests
//
before(function* () {
userLibraryID = Zotero.Libraries.userLibraryID;
publicationsLibraryID = Zotero.Libraries.publicationsLibraryID;
})
beforeEach(function* () {
Zotero.HTTP.mock = sinon.FakeXMLHttpRequest;
server = sinon.fakeServer.create();
server.autoRespond = true;
({ runner, caller } = yield setup());
yield Zotero.Users.setCurrentUserID(1);
yield Zotero.Users.setCurrentUsername("A");
})
@ -404,27 +398,7 @@ describe("Zotero.Sync.Runner", function () {
})
describe("#sync()", function () {
var spy;
before(function* () {
yield resetDB({
thisArg: this,
skipBundledFiles: true
});
yield Zotero.Libraries.init();
})
afterEach(function () {
if (spy) {
spy.restore();
}
});
it("should perform a sync across all libraries and update library versions", function* () {
yield Zotero.Users.setCurrentUserID(1);
yield Zotero.Users.setCurrentUsername("A");
setResponse('keyInfo.fullAccess');
setResponse('userGroups.groupVersions');
setResponse('groups.ownerGroup');
@ -675,11 +649,11 @@ describe("Zotero.Sync.Runner", function () {
// Check local library versions
assert.equal(
Zotero.Libraries.getVersion(Zotero.Libraries.userLibraryID),
Zotero.Libraries.getVersion(userLibraryID),
5
);
assert.equal(
Zotero.Libraries.getVersion(Zotero.Libraries.publicationsLibraryID),
Zotero.Libraries.getVersion(publicationsLibraryID),
10
);
assert.equal(
@ -699,8 +673,9 @@ describe("Zotero.Sync.Runner", function () {
it("should show the sync error icon on error", function* () {
yield Zotero.Users.setCurrentUserID(1);
yield Zotero.Users.setCurrentUsername("A");
let pubLib = Zotero.Libraries.get(publicationsLibraryID);
pubLib.libraryVersion = 5;
yield pubLib.save();
setResponse('keyInfo.fullAccess');
setResponse('userGroups.groupVersionsEmpty');
@ -716,6 +691,25 @@ describe("Zotero.Sync.Runner", function () {
INVALID: true // TODO: Find a cleaner error
}
});
// No publications changes
setResponse({
method: "GET",
url: "users/1/publications/settings?since=5",
status: 304,
headers: {
"Last-Modified-Version": 5
},
json: {}
});
setResponse({
method: "GET",
url: "users/1/publications/fulltext",
status: 200,
headers: {
"Last-Modified-Version": 5
},
json: {}
});
spy = sinon.spy(runner, "updateIcons");
yield runner.sync();