Full-text syncing support via API [DB reupgrade]
This commit is contained in:
Dan Stillman
parent
cb81f3febd
commit
62aeb1da32
13 changed files with 855 additions and 130 deletions
324
test/tests/syncFullTextEngineTest.js
Normal file
324
test/tests/syncFullTextEngineTest.js
Normal file
|
|
@ -0,0 +1,324 @@
|
|||
"use strict";
|
||||
|
||||
describe("Zotero.Sync.Data.FullTextEngine", function () {
|
||||
Components.utils.import("resource://zotero/config.js");
|
||||
|
||||
var apiKey = Zotero.Utilities.randomString(24);
|
||||
var baseURL = "http://local.zotero/";
|
||||
var engine, server, client, caller, stub, spy;
|
||||
|
||||
var responses = {};
|
||||
|
||||
var setup = Zotero.Promise.coroutine(function* (options = {}) {
|
||||
server = sinon.fakeServer.create();
|
||||
server.autoRespond = true;
|
||||
|
||||
Components.utils.import("resource://zotero/concurrentCaller.js");
|
||||
var caller = new ConcurrentCaller(1);
|
||||
caller.setLogger(msg => Zotero.debug(msg));
|
||||
caller.stopOnError = true;
|
||||
|
||||
var client = new Zotero.Sync.APIClient({
|
||||
baseURL,
|
||||
apiVersion: options.apiVersion || ZOTERO_CONFIG.API_VERSION,
|
||||
apiKey,
|
||||
caller,
|
||||
background: options.background || true
|
||||
});
|
||||
|
||||
var engine = new Zotero.Sync.Data.FullTextEngine({
|
||||
apiClient: client,
|
||||
libraryID: options.libraryID || Zotero.Libraries.userLibraryID,
|
||||
stopOnError: true
|
||||
});
|
||||
|
||||
return { engine, client, caller };
|
||||
});
|
||||
|
||||
function setResponse(response) {
|
||||
setHTTPResponse(server, baseURL, response, responses);
|
||||
}
|
||||
|
||||
//
|
||||
// Tests
|
||||
//
|
||||
beforeEach(function* () {
|
||||
yield resetDB({
|
||||
thisArg: this,
|
||||
skipBundledFiles: true
|
||||
});
|
||||
|
||||
Zotero.HTTP.mock = sinon.FakeXMLHttpRequest;
|
||||
|
||||
yield Zotero.Users.setCurrentUserID(1);
|
||||
yield Zotero.Users.setCurrentUsername("testuser");
|
||||
})
|
||||
|
||||
describe("Full-Text Syncing", function () {
|
||||
it("should download full-text into a new library and subsequent updates", function* () {
|
||||
({ engine, client, caller } = yield setup());
|
||||
|
||||
var item = yield createDataObject('item');
|
||||
var attachment = new Zotero.Item('attachment');
|
||||
attachment.parentItemID = item.id;
|
||||
attachment.attachmentLinkMode = 'imported_file';
|
||||
attachment.attachmentContentType = 'application/pdf';
|
||||
attachment.attachmentFilename = 'test.pdf';
|
||||
yield attachment.saveTx();
|
||||
|
||||
var content = [Zotero.Utilities.randomString() for (x of new Array(10))].join(" ");
|
||||
var spy = sinon.spy(Zotero.Fulltext, "startContentProcessor")
|
||||
|
||||
var itemFullTextVersion = 10;
|
||||
var libraryFullTextVersion = 15;
|
||||
setResponse({
|
||||
method: "GET",
|
||||
url: "users/1/fulltext",
|
||||
status: 200,
|
||||
headers: {
|
||||
"Last-Modified-Version": libraryFullTextVersion
|
||||
},
|
||||
json: {
|
||||
[attachment.key]: itemFullTextVersion
|
||||
}
|
||||
});
|
||||
setResponse({
|
||||
method: "GET",
|
||||
url: `users/1/items/${attachment.key}/fulltext`,
|
||||
status: 200,
|
||||
headers: {
|
||||
"Last-Modified-Version": itemFullTextVersion
|
||||
},
|
||||
json: {
|
||||
content,
|
||||
indexedPages: 1,
|
||||
totalPages: 1
|
||||
}
|
||||
});
|
||||
yield engine.start();
|
||||
|
||||
var dir = Zotero.Attachments.getStorageDirectory(attachment).path;
|
||||
var unprocessed = OS.Path.join(dir, '.zotero-ft-unprocessed');
|
||||
assert.isTrue(yield OS.File.exists(unprocessed));
|
||||
var data = JSON.parse(yield Zotero.File.getContentsAsync(unprocessed));
|
||||
assert.propertyVal(data, 'text', content);
|
||||
assert.propertyVal(data, 'indexedPages', 1);
|
||||
assert.propertyVal(data, 'totalPages', 1);
|
||||
assert.propertyVal(data, 'version', itemFullTextVersion);
|
||||
yield assert.eventually.equal(
|
||||
Zotero.FullText.getLibraryVersion(item.libraryID),
|
||||
libraryFullTextVersion
|
||||
);
|
||||
|
||||
sinon.assert.calledOnce(spy);
|
||||
spy.restore();
|
||||
|
||||
//
|
||||
// Get new content
|
||||
//
|
||||
({ engine, client, caller } = yield setup());
|
||||
|
||||
item = yield createDataObject('item');
|
||||
attachment = new Zotero.Item('attachment');
|
||||
attachment.parentItemID = item.id;
|
||||
attachment.attachmentLinkMode = 'imported_file';
|
||||
attachment.attachmentContentType = 'application/pdf';
|
||||
attachment.attachmentFilename = 'test.pdf';
|
||||
yield attachment.saveTx();
|
||||
|
||||
content = [Zotero.Utilities.randomString() for (x of new Array(10))].join(" ");
|
||||
spy = sinon.spy(Zotero.Fulltext, "startContentProcessor")
|
||||
|
||||
itemFullTextVersion = 17;
|
||||
var lastLibraryFullTextVersion = libraryFullTextVersion;
|
||||
libraryFullTextVersion = 20;
|
||||
setResponse({
|
||||
method: "GET",
|
||||
url: "users/1/fulltext?since=" + lastLibraryFullTextVersion,
|
||||
status: 200,
|
||||
headers: {
|
||||
"Last-Modified-Version": libraryFullTextVersion
|
||||
},
|
||||
json: {
|
||||
[attachment.key]: itemFullTextVersion
|
||||
}
|
||||
});
|
||||
setResponse({
|
||||
method: "GET",
|
||||
url: `users/1/items/${attachment.key}/fulltext`,
|
||||
status: 200,
|
||||
headers: {
|
||||
"Last-Modified-Version": itemFullTextVersion
|
||||
},
|
||||
json: {
|
||||
content,
|
||||
indexedPages: 1,
|
||||
totalPages: 1
|
||||
}
|
||||
});
|
||||
yield engine.start();
|
||||
|
||||
var dir = Zotero.Attachments.getStorageDirectory(attachment).path;
|
||||
var unprocessed = OS.Path.join(dir, '.zotero-ft-unprocessed');
|
||||
assert.isTrue(yield OS.File.exists(unprocessed));
|
||||
var data = JSON.parse(yield Zotero.File.getContentsAsync(unprocessed));
|
||||
assert.propertyVal(data, 'text', content);
|
||||
assert.propertyVal(data, 'indexedPages', 1);
|
||||
assert.propertyVal(data, 'totalPages', 1);
|
||||
assert.propertyVal(data, 'version', itemFullTextVersion);
|
||||
yield assert.eventually.equal(
|
||||
Zotero.FullText.getLibraryVersion(item.libraryID),
|
||||
libraryFullTextVersion
|
||||
);
|
||||
|
||||
sinon.assert.calledOnce(spy);
|
||||
spy.restore();
|
||||
})
|
||||
|
||||
it("should upload new full-text content and subsequent updates", function* () {
|
||||
// https://github.com/cjohansen/Sinon.JS/issues/607
|
||||
var fixSinonBug = ";charset=utf-8";
|
||||
|
||||
var libraryID = Zotero.Libraries.userLibraryID;
|
||||
yield Zotero.Libraries.setVersion(libraryID, 5);
|
||||
|
||||
({ engine, client, caller } = yield setup());
|
||||
|
||||
var item = yield createDataObject('item');
|
||||
var attachment = new Zotero.Item('attachment');
|
||||
attachment.parentItemID = item.id;
|
||||
attachment.attachmentLinkMode = 'imported_file';
|
||||
attachment.attachmentContentType = 'text/html';
|
||||
attachment.attachmentFilename = 'test.html';
|
||||
attachment.attachmentCharset = 'utf-8';
|
||||
attachment.synced = true;
|
||||
yield attachment.saveTx();
|
||||
yield Zotero.Attachments.createDirectoryForItem(attachment);
|
||||
|
||||
var path = attachment.getFilePath();
|
||||
var content = [Zotero.Utilities.randomString() for (x of new Array(10))].join(" ");
|
||||
var htmlContent = "<html><body>" + content + "</body></html>";
|
||||
yield Zotero.File.putContentsAsync(path, content);
|
||||
yield Zotero.Fulltext.indexItems([attachment.id]);
|
||||
|
||||
var libraryVersion = 15;
|
||||
var previousLibraryVersion = libraryVersion;
|
||||
|
||||
var count = 1;
|
||||
setResponse({
|
||||
method: "GET",
|
||||
url: "users/1/fulltext",
|
||||
status: 200,
|
||||
headers: {
|
||||
"Last-Modified-Version": libraryVersion
|
||||
},
|
||||
json: {}
|
||||
});
|
||||
server.respond(function (req) {
|
||||
if (req.method == "PUT") {
|
||||
if (req.url == `${baseURL}users/1/items/${attachment.key}/fulltext`) {
|
||||
assert.propertyVal(
|
||||
req.requestHeaders,
|
||||
'Content-Type',
|
||||
'application/json' + fixSinonBug
|
||||
);
|
||||
|
||||
let json = JSON.parse(req.requestBody);
|
||||
assert.propertyVal(json, 'content', content);
|
||||
assert.propertyVal(json, 'indexedChars', content.length);
|
||||
assert.propertyVal(json, 'totalChars', content.length);
|
||||
assert.propertyVal(json, 'indexedPages', 0);
|
||||
assert.propertyVal(json, 'totalPages', 0);
|
||||
|
||||
req.respond(
|
||||
204,
|
||||
{
|
||||
"Content-Type": "application/json",
|
||||
"Last-Modified-Version": ++libraryVersion
|
||||
},
|
||||
""
|
||||
);
|
||||
count--;
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
yield engine.start();
|
||||
assert.equal(count, 0);
|
||||
yield assert.eventually.equal(
|
||||
Zotero.FullText.getItemVersion(attachment.id),
|
||||
libraryVersion
|
||||
);
|
||||
|
||||
//
|
||||
// Upload new content
|
||||
//
|
||||
({ engine, client, caller } = yield setup());
|
||||
yield Zotero.Libraries.setVersion(libraryID, libraryVersion);
|
||||
|
||||
item = yield createDataObject('item');
|
||||
attachment = new Zotero.Item('attachment');
|
||||
attachment.parentItemID = item.id;
|
||||
attachment.attachmentLinkMode = 'imported_file';
|
||||
attachment.attachmentContentType = 'text/html';
|
||||
attachment.attachmentFilename = 'test.html';
|
||||
attachment.attachmentCharset = 'utf-8';
|
||||
attachment.synced = true;
|
||||
yield attachment.saveTx();
|
||||
yield Zotero.Attachments.createDirectoryForItem(attachment);
|
||||
|
||||
path = attachment.getFilePath();
|
||||
content = [Zotero.Utilities.randomString() for (x of new Array(10))].join(" ");
|
||||
htmlContent = "<html><body>" + content + "</body></html>";
|
||||
yield Zotero.File.putContentsAsync(path, content);
|
||||
yield Zotero.Fulltext.indexItems([attachment.id]);
|
||||
|
||||
count = 1;
|
||||
setResponse({
|
||||
method: "GET",
|
||||
url: "users/1/fulltext?since=" + previousLibraryVersion,
|
||||
status: 200,
|
||||
headers: {
|
||||
"Last-Modified-Version": libraryVersion
|
||||
},
|
||||
json: {}
|
||||
});
|
||||
server.respond(function (req) {
|
||||
if (req.method == "PUT") {
|
||||
if (req.url == `${baseURL}users/1/items/${attachment.key}/fulltext`) {
|
||||
assert.propertyVal(req.requestHeaders, 'Zotero-API-Key', apiKey);
|
||||
assert.propertyVal(
|
||||
req.requestHeaders,
|
||||
'Content-Type',
|
||||
'application/json' + fixSinonBug
|
||||
);
|
||||
|
||||
let json = JSON.parse(req.requestBody);
|
||||
assert.propertyVal(json, 'content', content);
|
||||
assert.propertyVal(json, 'indexedChars', content.length);
|
||||
assert.propertyVal(json, 'totalChars', content.length);
|
||||
assert.propertyVal(json, 'indexedPages', 0);
|
||||
assert.propertyVal(json, 'totalPages', 0);
|
||||
|
||||
req.respond(
|
||||
204,
|
||||
{
|
||||
"Content-Type": "application/json",
|
||||
"Last-Modified-Version": ++libraryVersion
|
||||
},
|
||||
""
|
||||
);
|
||||
count--;
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
yield engine.start();
|
||||
assert.equal(count, 0);
|
||||
yield assert.eventually.equal(
|
||||
Zotero.FullText.getItemVersion(attachment.id),
|
||||
libraryVersion
|
||||
);
|
||||
})
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue