Full-text syncing support via API [DB reupgrade]
This commit is contained in:
Dan Stillman
parent
cb81f3febd
commit
62aeb1da32
13 changed files with 855 additions and 130 deletions
|
|
@ -1 +1,4 @@
|
|||
This is a test file.
|
||||
Zotero [zoh-TAIR-oh] is a free, easy-to-use tool to help you collect, organize, cite, and share
|
||||
your research sources.
|
||||
|
||||
|
||||
|
|
@ -1,6 +1,9 @@
|
|||
describe("Zotero.Fulltext", function () {
|
||||
describe("#downloadPDFTool()", function () {
|
||||
it("should install the PDF tools", function* () {
|
||||
yield Zotero.Fulltext.uninstallPDFTools();
|
||||
assert.isFalse(Zotero.Fulltext.pdfInfoIsRegistered());
|
||||
|
||||
var version = Zotero.isWin ? '3.02a' : '3.04';
|
||||
var dataDir = Zotero.getZoteroDirectory().path;
|
||||
var execFileName = Zotero.Fulltext.pdfInfoFileName;
|
||||
|
|
@ -54,8 +57,82 @@ describe("Zotero.Fulltext", function () {
|
|||
assert.equal((yield OS.File.stat(scriptPath)).unixMode, 0o755);
|
||||
}
|
||||
|
||||
yield Zotero.Fulltext.uninstallPDFTools();
|
||||
yield uninstallPDFTools();
|
||||
assert.isFalse(Zotero.Fulltext.pdfInfoIsRegistered());
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
describe("#getUnsyncedContent()", function () {
|
||||
before(function* () {
|
||||
yield installPDFTools();
|
||||
})
|
||||
|
||||
after(function* () {
|
||||
yield uninstallPDFTools();
|
||||
})
|
||||
|
||||
it("should get content that hasn't been uploaded", function* () {
|
||||
var toSync = [];
|
||||
var group = yield getGroup();
|
||||
|
||||
var add = Zotero.Promise.coroutine(function* (options = {}) {
|
||||
let item = yield createDataObject('item', { libraryID: options.libraryID });
|
||||
let attachment = new Zotero.Item('attachment');
|
||||
if (options.libraryID) {
|
||||
attachment.libraryID = options.libraryID;
|
||||
}
|
||||
attachment.parentItemID = item.id;
|
||||
attachment.attachmentLinkMode = 'imported_file';
|
||||
attachment.attachmentContentType = 'text/plain';
|
||||
attachment.attachmentCharset = 'utf-8';
|
||||
attachment.attachmentFilename = 'test.txt';
|
||||
if (options.synced) {
|
||||
attachment.synced = true;
|
||||
}
|
||||
yield attachment.saveTx();
|
||||
yield Zotero.Attachments.createDirectoryForItem(attachment);
|
||||
|
||||
let path = attachment.getFilePath();
|
||||
let content = [Zotero.Utilities.randomString() for (x of new Array(10))].join(" ");
|
||||
yield Zotero.File.putContentsAsync(path, content);
|
||||
|
||||
if (!options.skip) {
|
||||
toSync.push({
|
||||
item: attachment,
|
||||
content,
|
||||
indexedChars: content.length,
|
||||
indexedPages: 0
|
||||
});
|
||||
}
|
||||
});
|
||||
yield add({ synced: true });
|
||||
yield add({ synced: true });
|
||||
// Unsynced attachment shouldn't uploaded
|
||||
yield add({ skip: true });
|
||||
// Attachment in another library shouldn't be uploaded
|
||||
yield add({ libraryID: group.libraryID, synced: true, skip: true });
|
||||
// PDF attachment
|
||||
var pdfAttachment = yield importFileAttachment('test.pdf');
|
||||
pdfAttachment.synced = true;
|
||||
yield pdfAttachment.saveTx();
|
||||
toSync.push({
|
||||
item: pdfAttachment,
|
||||
content: "Zotero [zoh-TAIR-oh] is a free, easy-to-use tool to help you collect, "
|
||||
+ "organize, cite, and share your research sources.\n\n",
|
||||
indexedChars: 0,
|
||||
indexedPages: 1
|
||||
});
|
||||
|
||||
yield Zotero.Fulltext.indexItems(toSync.map(x => x.item.id));
|
||||
|
||||
var data = yield Zotero.FullText.getUnsyncedContent(Zotero.Libraries.userLibraryID);
|
||||
assert.lengthOf(data, 3);
|
||||
for (let i = toSync.length - 1; i >= 0 ; i--) {
|
||||
assert.equal(data[i].content, toSync[i].content);
|
||||
assert.equal(data[i].indexedChars, toSync[i].indexedChars);
|
||||
assert.equal(data[i].indexedPages, toSync[i].indexedPages);
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
|
|||
324
test/tests/syncFullTextEngineTest.js
Normal file
324
test/tests/syncFullTextEngineTest.js
Normal file
|
|
@ -0,0 +1,324 @@
|
|||
"use strict";
|
||||
|
||||
describe("Zotero.Sync.Data.FullTextEngine", function () {
|
||||
Components.utils.import("resource://zotero/config.js");
|
||||
|
||||
var apiKey = Zotero.Utilities.randomString(24);
|
||||
var baseURL = "http://local.zotero/";
|
||||
var engine, server, client, caller, stub, spy;
|
||||
|
||||
var responses = {};
|
||||
|
||||
var setup = Zotero.Promise.coroutine(function* (options = {}) {
|
||||
server = sinon.fakeServer.create();
|
||||
server.autoRespond = true;
|
||||
|
||||
Components.utils.import("resource://zotero/concurrentCaller.js");
|
||||
var caller = new ConcurrentCaller(1);
|
||||
caller.setLogger(msg => Zotero.debug(msg));
|
||||
caller.stopOnError = true;
|
||||
|
||||
var client = new Zotero.Sync.APIClient({
|
||||
baseURL,
|
||||
apiVersion: options.apiVersion || ZOTERO_CONFIG.API_VERSION,
|
||||
apiKey,
|
||||
caller,
|
||||
background: options.background || true
|
||||
});
|
||||
|
||||
var engine = new Zotero.Sync.Data.FullTextEngine({
|
||||
apiClient: client,
|
||||
libraryID: options.libraryID || Zotero.Libraries.userLibraryID,
|
||||
stopOnError: true
|
||||
});
|
||||
|
||||
return { engine, client, caller };
|
||||
});
|
||||
|
||||
function setResponse(response) {
|
||||
setHTTPResponse(server, baseURL, response, responses);
|
||||
}
|
||||
|
||||
//
|
||||
// Tests
|
||||
//
|
||||
beforeEach(function* () {
|
||||
yield resetDB({
|
||||
thisArg: this,
|
||||
skipBundledFiles: true
|
||||
});
|
||||
|
||||
Zotero.HTTP.mock = sinon.FakeXMLHttpRequest;
|
||||
|
||||
yield Zotero.Users.setCurrentUserID(1);
|
||||
yield Zotero.Users.setCurrentUsername("testuser");
|
||||
})
|
||||
|
||||
describe("Full-Text Syncing", function () {
|
||||
it("should download full-text into a new library and subsequent updates", function* () {
|
||||
({ engine, client, caller } = yield setup());
|
||||
|
||||
var item = yield createDataObject('item');
|
||||
var attachment = new Zotero.Item('attachment');
|
||||
attachment.parentItemID = item.id;
|
||||
attachment.attachmentLinkMode = 'imported_file';
|
||||
attachment.attachmentContentType = 'application/pdf';
|
||||
attachment.attachmentFilename = 'test.pdf';
|
||||
yield attachment.saveTx();
|
||||
|
||||
var content = [Zotero.Utilities.randomString() for (x of new Array(10))].join(" ");
|
||||
var spy = sinon.spy(Zotero.Fulltext, "startContentProcessor")
|
||||
|
||||
var itemFullTextVersion = 10;
|
||||
var libraryFullTextVersion = 15;
|
||||
setResponse({
|
||||
method: "GET",
|
||||
url: "users/1/fulltext",
|
||||
status: 200,
|
||||
headers: {
|
||||
"Last-Modified-Version": libraryFullTextVersion
|
||||
},
|
||||
json: {
|
||||
[attachment.key]: itemFullTextVersion
|
||||
}
|
||||
});
|
||||
setResponse({
|
||||
method: "GET",
|
||||
url: `users/1/items/${attachment.key}/fulltext`,
|
||||
status: 200,
|
||||
headers: {
|
||||
"Last-Modified-Version": itemFullTextVersion
|
||||
},
|
||||
json: {
|
||||
content,
|
||||
indexedPages: 1,
|
||||
totalPages: 1
|
||||
}
|
||||
});
|
||||
yield engine.start();
|
||||
|
||||
var dir = Zotero.Attachments.getStorageDirectory(attachment).path;
|
||||
var unprocessed = OS.Path.join(dir, '.zotero-ft-unprocessed');
|
||||
assert.isTrue(yield OS.File.exists(unprocessed));
|
||||
var data = JSON.parse(yield Zotero.File.getContentsAsync(unprocessed));
|
||||
assert.propertyVal(data, 'text', content);
|
||||
assert.propertyVal(data, 'indexedPages', 1);
|
||||
assert.propertyVal(data, 'totalPages', 1);
|
||||
assert.propertyVal(data, 'version', itemFullTextVersion);
|
||||
yield assert.eventually.equal(
|
||||
Zotero.FullText.getLibraryVersion(item.libraryID),
|
||||
libraryFullTextVersion
|
||||
);
|
||||
|
||||
sinon.assert.calledOnce(spy);
|
||||
spy.restore();
|
||||
|
||||
//
|
||||
// Get new content
|
||||
//
|
||||
({ engine, client, caller } = yield setup());
|
||||
|
||||
item = yield createDataObject('item');
|
||||
attachment = new Zotero.Item('attachment');
|
||||
attachment.parentItemID = item.id;
|
||||
attachment.attachmentLinkMode = 'imported_file';
|
||||
attachment.attachmentContentType = 'application/pdf';
|
||||
attachment.attachmentFilename = 'test.pdf';
|
||||
yield attachment.saveTx();
|
||||
|
||||
content = [Zotero.Utilities.randomString() for (x of new Array(10))].join(" ");
|
||||
spy = sinon.spy(Zotero.Fulltext, "startContentProcessor")
|
||||
|
||||
itemFullTextVersion = 17;
|
||||
var lastLibraryFullTextVersion = libraryFullTextVersion;
|
||||
libraryFullTextVersion = 20;
|
||||
setResponse({
|
||||
method: "GET",
|
||||
url: "users/1/fulltext?since=" + lastLibraryFullTextVersion,
|
||||
status: 200,
|
||||
headers: {
|
||||
"Last-Modified-Version": libraryFullTextVersion
|
||||
},
|
||||
json: {
|
||||
[attachment.key]: itemFullTextVersion
|
||||
}
|
||||
});
|
||||
setResponse({
|
||||
method: "GET",
|
||||
url: `users/1/items/${attachment.key}/fulltext`,
|
||||
status: 200,
|
||||
headers: {
|
||||
"Last-Modified-Version": itemFullTextVersion
|
||||
},
|
||||
json: {
|
||||
content,
|
||||
indexedPages: 1,
|
||||
totalPages: 1
|
||||
}
|
||||
});
|
||||
yield engine.start();
|
||||
|
||||
var dir = Zotero.Attachments.getStorageDirectory(attachment).path;
|
||||
var unprocessed = OS.Path.join(dir, '.zotero-ft-unprocessed');
|
||||
assert.isTrue(yield OS.File.exists(unprocessed));
|
||||
var data = JSON.parse(yield Zotero.File.getContentsAsync(unprocessed));
|
||||
assert.propertyVal(data, 'text', content);
|
||||
assert.propertyVal(data, 'indexedPages', 1);
|
||||
assert.propertyVal(data, 'totalPages', 1);
|
||||
assert.propertyVal(data, 'version', itemFullTextVersion);
|
||||
yield assert.eventually.equal(
|
||||
Zotero.FullText.getLibraryVersion(item.libraryID),
|
||||
libraryFullTextVersion
|
||||
);
|
||||
|
||||
sinon.assert.calledOnce(spy);
|
||||
spy.restore();
|
||||
})
|
||||
|
||||
it("should upload new full-text content and subsequent updates", function* () {
|
||||
// https://github.com/cjohansen/Sinon.JS/issues/607
|
||||
var fixSinonBug = ";charset=utf-8";
|
||||
|
||||
var libraryID = Zotero.Libraries.userLibraryID;
|
||||
yield Zotero.Libraries.setVersion(libraryID, 5);
|
||||
|
||||
({ engine, client, caller } = yield setup());
|
||||
|
||||
var item = yield createDataObject('item');
|
||||
var attachment = new Zotero.Item('attachment');
|
||||
attachment.parentItemID = item.id;
|
||||
attachment.attachmentLinkMode = 'imported_file';
|
||||
attachment.attachmentContentType = 'text/html';
|
||||
attachment.attachmentFilename = 'test.html';
|
||||
attachment.attachmentCharset = 'utf-8';
|
||||
attachment.synced = true;
|
||||
yield attachment.saveTx();
|
||||
yield Zotero.Attachments.createDirectoryForItem(attachment);
|
||||
|
||||
var path = attachment.getFilePath();
|
||||
var content = [Zotero.Utilities.randomString() for (x of new Array(10))].join(" ");
|
||||
var htmlContent = "<html><body>" + content + "</body></html>";
|
||||
yield Zotero.File.putContentsAsync(path, content);
|
||||
yield Zotero.Fulltext.indexItems([attachment.id]);
|
||||
|
||||
var libraryVersion = 15;
|
||||
var previousLibraryVersion = libraryVersion;
|
||||
|
||||
var count = 1;
|
||||
setResponse({
|
||||
method: "GET",
|
||||
url: "users/1/fulltext",
|
||||
status: 200,
|
||||
headers: {
|
||||
"Last-Modified-Version": libraryVersion
|
||||
},
|
||||
json: {}
|
||||
});
|
||||
server.respond(function (req) {
|
||||
if (req.method == "PUT") {
|
||||
if (req.url == `${baseURL}users/1/items/${attachment.key}/fulltext`) {
|
||||
assert.propertyVal(
|
||||
req.requestHeaders,
|
||||
'Content-Type',
|
||||
'application/json' + fixSinonBug
|
||||
);
|
||||
|
||||
let json = JSON.parse(req.requestBody);
|
||||
assert.propertyVal(json, 'content', content);
|
||||
assert.propertyVal(json, 'indexedChars', content.length);
|
||||
assert.propertyVal(json, 'totalChars', content.length);
|
||||
assert.propertyVal(json, 'indexedPages', 0);
|
||||
assert.propertyVal(json, 'totalPages', 0);
|
||||
|
||||
req.respond(
|
||||
204,
|
||||
{
|
||||
"Content-Type": "application/json",
|
||||
"Last-Modified-Version": ++libraryVersion
|
||||
},
|
||||
""
|
||||
);
|
||||
count--;
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
yield engine.start();
|
||||
assert.equal(count, 0);
|
||||
yield assert.eventually.equal(
|
||||
Zotero.FullText.getItemVersion(attachment.id),
|
||||
libraryVersion
|
||||
);
|
||||
|
||||
//
|
||||
// Upload new content
|
||||
//
|
||||
({ engine, client, caller } = yield setup());
|
||||
yield Zotero.Libraries.setVersion(libraryID, libraryVersion);
|
||||
|
||||
item = yield createDataObject('item');
|
||||
attachment = new Zotero.Item('attachment');
|
||||
attachment.parentItemID = item.id;
|
||||
attachment.attachmentLinkMode = 'imported_file';
|
||||
attachment.attachmentContentType = 'text/html';
|
||||
attachment.attachmentFilename = 'test.html';
|
||||
attachment.attachmentCharset = 'utf-8';
|
||||
attachment.synced = true;
|
||||
yield attachment.saveTx();
|
||||
yield Zotero.Attachments.createDirectoryForItem(attachment);
|
||||
|
||||
path = attachment.getFilePath();
|
||||
content = [Zotero.Utilities.randomString() for (x of new Array(10))].join(" ");
|
||||
htmlContent = "<html><body>" + content + "</body></html>";
|
||||
yield Zotero.File.putContentsAsync(path, content);
|
||||
yield Zotero.Fulltext.indexItems([attachment.id]);
|
||||
|
||||
count = 1;
|
||||
setResponse({
|
||||
method: "GET",
|
||||
url: "users/1/fulltext?since=" + previousLibraryVersion,
|
||||
status: 200,
|
||||
headers: {
|
||||
"Last-Modified-Version": libraryVersion
|
||||
},
|
||||
json: {}
|
||||
});
|
||||
server.respond(function (req) {
|
||||
if (req.method == "PUT") {
|
||||
if (req.url == `${baseURL}users/1/items/${attachment.key}/fulltext`) {
|
||||
assert.propertyVal(req.requestHeaders, 'Zotero-API-Key', apiKey);
|
||||
assert.propertyVal(
|
||||
req.requestHeaders,
|
||||
'Content-Type',
|
||||
'application/json' + fixSinonBug
|
||||
);
|
||||
|
||||
let json = JSON.parse(req.requestBody);
|
||||
assert.propertyVal(json, 'content', content);
|
||||
assert.propertyVal(json, 'indexedChars', content.length);
|
||||
assert.propertyVal(json, 'totalChars', content.length);
|
||||
assert.propertyVal(json, 'indexedPages', 0);
|
||||
assert.propertyVal(json, 'totalPages', 0);
|
||||
|
||||
req.respond(
|
||||
204,
|
||||
{
|
||||
"Content-Type": "application/json",
|
||||
"Last-Modified-Version": ++libraryVersion
|
||||
},
|
||||
""
|
||||
);
|
||||
count--;
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
yield engine.start();
|
||||
assert.equal(count, 0);
|
||||
yield assert.eventually.equal(
|
||||
Zotero.FullText.getItemVersion(attachment.id),
|
||||
libraryVersion
|
||||
);
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
@ -478,7 +478,7 @@ describe("Zotero.Sync.Runner", function () {
|
|||
});
|
||||
})
|
||||
|
||||
it("should perform a sync across all libraries", function* () {
|
||||
it("should perform a sync across all libraries and update library versions", function* () {
|
||||
yield Zotero.Users.setCurrentUserID(1);
|
||||
yield Zotero.Users.setCurrentUsername("A");
|
||||
|
||||
|
|
@ -652,6 +652,43 @@ describe("Zotero.Sync.Runner", function () {
|
|||
},
|
||||
json: []
|
||||
});
|
||||
// Full-text syncing
|
||||
setResponse({
|
||||
method: "GET",
|
||||
url: "users/1/fulltext",
|
||||
status: 200,
|
||||
headers: {
|
||||
"Last-Modified-Version": 5
|
||||
},
|
||||
json: {}
|
||||
});
|
||||
setResponse({
|
||||
method: "GET",
|
||||
url: "users/1/publications/fulltext",
|
||||
status: 200,
|
||||
headers: {
|
||||
"Last-Modified-Version": 10
|
||||
},
|
||||
json: {}
|
||||
});
|
||||
setResponse({
|
||||
method: "GET",
|
||||
url: "groups/1623562/fulltext",
|
||||
status: 200,
|
||||
headers: {
|
||||
"Last-Modified-Version": 15
|
||||
},
|
||||
json: {}
|
||||
});
|
||||
setResponse({
|
||||
method: "GET",
|
||||
url: "groups/2694172/fulltext",
|
||||
status: 200,
|
||||
headers: {
|
||||
"Last-Modified-Version": 20
|
||||
},
|
||||
json: {}
|
||||
});
|
||||
|
||||
yield runner.sync({
|
||||
onError: e => { throw e },
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue