"use strict"; describe("Zotero.Sync.Data.FullTextEngine", function () { Components.utils.import("resource://zotero/config.js"); var apiKey = Zotero.Utilities.randomString(24); var baseURL = "http://local.zotero/"; var engine, server, client, caller, stub, spy; var responses = {}; var setup = Zotero.Promise.coroutine(function* (options = {}) { server = sinon.fakeServer.create(); server.autoRespond = true; Components.utils.import("resource://zotero/concurrentCaller.js"); var caller = new ConcurrentCaller(1); caller.setLogger(msg => Zotero.debug(msg)); caller.stopOnError = true; var client = new Zotero.Sync.APIClient({ baseURL, apiVersion: options.apiVersion || ZOTERO_CONFIG.API_VERSION, apiKey, caller, background: options.background || true }); var engine = new Zotero.Sync.Data.FullTextEngine({ apiClient: client, libraryID: options.libraryID || Zotero.Libraries.userLibraryID, stopOnError: true }); return { engine, client, caller }; }); function setResponse(response) { setHTTPResponse(server, baseURL, response, responses); } // // Tests // beforeEach(function* () { yield resetDB({ thisArg: this, skipBundledFiles: true }); Zotero.HTTP.mock = sinon.FakeXMLHttpRequest; yield Zotero.Users.setCurrentUserID(1); yield Zotero.Users.setCurrentUsername("testuser"); }) describe("Full-Text Syncing", function () { it("should download full-text into a new library and subsequent updates", function* () { ({ engine, client, caller } = yield setup()); var item = yield createDataObject('item'); var attachment = new Zotero.Item('attachment'); attachment.parentItemID = item.id; attachment.attachmentLinkMode = 'imported_file'; attachment.attachmentContentType = 'application/pdf'; attachment.attachmentFilename = 'test.pdf'; yield attachment.saveTx(); var content = [Zotero.Utilities.randomString() for (x of new Array(10))].join(" "); var spy = sinon.spy(Zotero.Fulltext, "startContentProcessor") var itemFullTextVersion = 10; var libraryFullTextVersion = 15; setResponse({ method: "GET", url: "users/1/fulltext", status: 200, headers: { "Last-Modified-Version": libraryFullTextVersion }, json: { [attachment.key]: itemFullTextVersion } }); setResponse({ method: "GET", url: `users/1/items/${attachment.key}/fulltext`, status: 200, headers: { "Last-Modified-Version": itemFullTextVersion }, json: { content, indexedPages: 1, totalPages: 1 } }); yield engine.start(); var dir = Zotero.Attachments.getStorageDirectory(attachment).path; var unprocessed = OS.Path.join(dir, '.zotero-ft-unprocessed'); assert.isTrue(yield OS.File.exists(unprocessed)); var data = JSON.parse(yield Zotero.File.getContentsAsync(unprocessed)); assert.propertyVal(data, 'text', content); assert.propertyVal(data, 'indexedPages', 1); assert.propertyVal(data, 'totalPages', 1); assert.propertyVal(data, 'version', itemFullTextVersion); yield assert.eventually.equal( Zotero.FullText.getLibraryVersion(item.libraryID), libraryFullTextVersion ); sinon.assert.calledOnce(spy); spy.restore(); // // Get new content // ({ engine, client, caller } = yield setup()); item = yield createDataObject('item'); attachment = new Zotero.Item('attachment'); attachment.parentItemID = item.id; attachment.attachmentLinkMode = 'imported_file'; attachment.attachmentContentType = 'application/pdf'; attachment.attachmentFilename = 'test.pdf'; yield attachment.saveTx(); content = [Zotero.Utilities.randomString() for (x of new Array(10))].join(" "); spy = sinon.spy(Zotero.Fulltext, "startContentProcessor") itemFullTextVersion = 17; var lastLibraryFullTextVersion = libraryFullTextVersion; libraryFullTextVersion = 20; setResponse({ method: "GET", url: "users/1/fulltext?since=" + lastLibraryFullTextVersion, status: 200, headers: { "Last-Modified-Version": libraryFullTextVersion }, json: { [attachment.key]: itemFullTextVersion } }); setResponse({ method: "GET", url: `users/1/items/${attachment.key}/fulltext`, status: 200, headers: { "Last-Modified-Version": itemFullTextVersion }, json: { content, indexedPages: 1, totalPages: 1 } }); yield engine.start(); var dir = Zotero.Attachments.getStorageDirectory(attachment).path; var unprocessed = OS.Path.join(dir, '.zotero-ft-unprocessed'); assert.isTrue(yield OS.File.exists(unprocessed)); var data = JSON.parse(yield Zotero.File.getContentsAsync(unprocessed)); assert.propertyVal(data, 'text', content); assert.propertyVal(data, 'indexedPages', 1); assert.propertyVal(data, 'totalPages', 1); assert.propertyVal(data, 'version', itemFullTextVersion); yield assert.eventually.equal( Zotero.FullText.getLibraryVersion(item.libraryID), libraryFullTextVersion ); sinon.assert.calledOnce(spy); spy.restore(); }) it("should upload new full-text content and subsequent updates", function* () { // https://github.com/cjohansen/Sinon.JS/issues/607 var fixSinonBug = ";charset=utf-8"; var libraryID = Zotero.Libraries.userLibraryID; yield Zotero.Libraries.setVersion(libraryID, 5); ({ engine, client, caller } = yield setup()); var item = yield createDataObject('item'); var attachment = new Zotero.Item('attachment'); attachment.parentItemID = item.id; attachment.attachmentLinkMode = 'imported_file'; attachment.attachmentContentType = 'text/html'; attachment.attachmentFilename = 'test.html'; attachment.attachmentCharset = 'utf-8'; attachment.synced = true; yield attachment.saveTx(); yield Zotero.Attachments.createDirectoryForItem(attachment); var path = attachment.getFilePath(); var content = [Zotero.Utilities.randomString() for (x of new Array(10))].join(" "); var htmlContent = "" + content + ""; yield Zotero.File.putContentsAsync(path, content); yield Zotero.Fulltext.indexItems([attachment.id]); var libraryVersion = 15; var previousLibraryVersion = libraryVersion; var count = 1; setResponse({ method: "GET", url: "users/1/fulltext", status: 200, headers: { "Last-Modified-Version": libraryVersion }, json: {} }); server.respond(function (req) { if (req.method == "PUT") { if (req.url == `${baseURL}users/1/items/${attachment.key}/fulltext`) { assert.propertyVal( req.requestHeaders, 'Content-Type', 'application/json' + fixSinonBug ); let json = JSON.parse(req.requestBody); assert.propertyVal(json, 'content', content); assert.propertyVal(json, 'indexedChars', content.length); assert.propertyVal(json, 'totalChars', content.length); assert.propertyVal(json, 'indexedPages', 0); assert.propertyVal(json, 'totalPages', 0); req.respond( 204, { "Content-Type": "application/json", "Last-Modified-Version": ++libraryVersion }, "" ); count--; } } }) yield engine.start(); assert.equal(count, 0); yield assert.eventually.equal( Zotero.FullText.getItemVersion(attachment.id), libraryVersion ); // // Upload new content // ({ engine, client, caller } = yield setup()); yield Zotero.Libraries.setVersion(libraryID, libraryVersion); item = yield createDataObject('item'); attachment = new Zotero.Item('attachment'); attachment.parentItemID = item.id; attachment.attachmentLinkMode = 'imported_file'; attachment.attachmentContentType = 'text/html'; attachment.attachmentFilename = 'test.html'; attachment.attachmentCharset = 'utf-8'; attachment.synced = true; yield attachment.saveTx(); yield Zotero.Attachments.createDirectoryForItem(attachment); path = attachment.getFilePath(); content = [Zotero.Utilities.randomString() for (x of new Array(10))].join(" "); htmlContent = "" + content + ""; yield Zotero.File.putContentsAsync(path, content); yield Zotero.Fulltext.indexItems([attachment.id]); count = 1; setResponse({ method: "GET", url: "users/1/fulltext?since=" + previousLibraryVersion, status: 200, headers: { "Last-Modified-Version": libraryVersion }, json: {} }); server.respond(function (req) { if (req.method == "PUT") { if (req.url == `${baseURL}users/1/items/${attachment.key}/fulltext`) { assert.propertyVal(req.requestHeaders, 'Zotero-API-Key', apiKey); assert.propertyVal( req.requestHeaders, 'Content-Type', 'application/json' + fixSinonBug ); let json = JSON.parse(req.requestBody); assert.propertyVal(json, 'content', content); assert.propertyVal(json, 'indexedChars', content.length); assert.propertyVal(json, 'totalChars', content.length); assert.propertyVal(json, 'indexedPages', 0); assert.propertyVal(json, 'totalPages', 0); req.respond( 204, { "Content-Type": "application/json", "Last-Modified-Version": ++libraryVersion }, "" ); count--; } } }) yield engine.start(); assert.equal(count, 0); yield assert.eventually.equal( Zotero.FullText.getItemVersion(attachment.id), libraryVersion ); }) }) })