From 377f8d0aa884c6811c601017cf756adaac23fd1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adomas=20Ven=C4=8Dkauskas?= Date: Thu, 23 Aug 2018 15:57:59 +0300 Subject: [PATCH] Make Connector Server tests compatible with electron Refactor /detect and /savePage endpoints --- .../xpcom/connector/server_connector.js | 164 +++++-------- .../zotero/xpcom/translation/translator.js | 1 + .../zotero/xpcom/translation/translators.js | 230 +++++++++--------- test/tests/server_connectorTest.js | 118 ++++----- 4 files changed, 239 insertions(+), 274 deletions(-) diff --git a/chrome/content/zotero/xpcom/connector/server_connector.js b/chrome/content/zotero/xpcom/connector/server_connector.js index 455ffacaa3..48834cbc8b 100644 --- a/chrome/content/zotero/xpcom/connector/server_connector.js +++ b/chrome/content/zotero/xpcom/connector/server_connector.js @@ -100,7 +100,7 @@ Zotero.Server.Connector = { case 'C': collection = Zotero.Collections.get(id); library = collection.library; - editable = collection.editable; + editable = collection.library.editable; break; default: @@ -447,59 +447,25 @@ Zotero.Server.Connector.Detect.prototype = { * @param {Object} data POST data or GET query string * @param {Function} sendResponseCallback function to send HTTP response */ - init: function(url, data, sendResponseCallback) { - this.sendResponse = sendResponseCallback; - this._parsedPostData = data; + init: async function({data}) { + var translate = new Zotero.Translate.Web(); - this._translate = new Zotero.Translate("web"); - this._translate.setHandler("translators", function(obj, item) { me._translatorsAvailable(obj, item) }); + var parser = Components.classes["@mozilla.org/xmlextras/domparser;1"] + .createInstance(Components.interfaces.nsIDOMParser); + var doc = parser.parseFromString(`${data.html}`, 'text/html'); + doc = Zotero.HTTP.wrapDocument(doc, data.uri); - Zotero.Server.Connector.Data[this._parsedPostData["uri"]] = ""+this._parsedPostData["html"]+""; - this._browser = Zotero.Browser.createHiddenBrowser(); - - var ioService = Components.classes["@mozilla.org/network/io-service;1"] - .getService(Components.interfaces.nsIIOService); - var uri = ioService.newURI(this._parsedPostData["uri"], "UTF-8", null); - - var pageShowCalled = false; - var me = this; - this._translate.setCookieSandbox(new Zotero.CookieSandbox(this._browser, - this._parsedPostData["uri"], this._parsedPostData["cookie"], url.userAgent)); - this._browser.addEventListener("DOMContentLoaded", function() { - try { - if(me._browser.contentDocument.location.href == "about:blank") return; - if(pageShowCalled) return; - pageShowCalled = true; - delete Zotero.Server.Connector.Data[me._parsedPostData["uri"]]; - - // get translators - me._translate.setDocument(me._browser.contentDocument); - me._translate.setLocation(me._parsedPostData["uri"], me._parsedPostData["uri"]); - me._translate.getTranslators(); - } catch(e) { - sendResponseCallback(500); - throw e; - } - }, false); - - me._browser.loadURI("zotero://connector/"+encodeURIComponent(this._parsedPostData["uri"])); - }, + // get translators + translate.setDocument(doc); - /** - * Callback to be executed when list of translators becomes available. Sends standard - * translator passing properties with proxies where available for translators. - * @param {Zotero.Translate} translate - * @param {Zotero.Translator[]} translators - */ - _translatorsAvailable: function(translate, translators) { + var translators = await translate.getTranslators(); + translators = translators.map(function(translator) { - translator = translator.serialize(TRANSLATOR_PASSING_PROPERTIES.concat('proxy')); + translator = translator.serialize(Zotero.Translator.TRANSLATOR_PASSING_PROPERTIES.concat('proxy')); translator.proxy = translator.proxy ? translator.proxy.toJSON() : null; return translator; }); - this.sendResponse(200, "application/json", JSON.stringify(translators)); - - Zotero.Browser.deleteHiddenBrowser(this._browser); + return [200, "application/json", JSON.stringify(translators)]; } } @@ -532,17 +498,56 @@ Zotero.Server.Connector.SavePage.prototype = { * @param {Object} data POST data or GET query string * @param {Function} sendResponseCallback function to send HTTP response */ - init: function(url, data, sendResponseCallback) { - var { library, collection, editable } = Zotero.Server.Connector.getSaveTarget(); - - // Shouldn't happen as long as My Library exists - if (!library.editable) { - Zotero.logError("Can't add item to read-only library " + library.name); - return sendResponseCallback(500, "application/json", JSON.stringify({ libraryEditable: false })); + init: async function(url, data, sendResponseCallback) { + try { + this.sendResponse = sendResponseCallback; + var { library, collection, editable } = Zotero.Server.Connector.getSaveTarget(); + + // Shouldn't happen as long as My Library exists + if (!library.editable) { + Zotero.logError("Can't add item to read-only library " + library.name); + return sendResponseCallback(500, "application/json", JSON.stringify({ libraryEditable: false })); + } + var libraryID = library.libraryID; + + var translate = new Zotero.Translate.Web(); + + var parser = Components.classes["@mozilla.org/xmlextras/domparser;1"] + .createInstance(Components.interfaces.nsIDOMParser); + var doc = parser.parseFromString(`${data.html}`, 'text/html'); + doc = Zotero.HTTP.wrapDocument(doc, data.uri); + + // get translators + translate.setDocument(doc); + + var translators = await translate.getTranslators(); + + // make sure translatorsAvailable succeded + if(!translators.length) { + return sendResponseCallback(500); + } + + // set handlers for translation + var jsonItems = []; + translate.setHandler("select", function(obj, item, callback) { return me._selectItems(obj, item, callback) }); + translate.setHandler("itemDone", function(obj, item, jsonItem) { + jsonItems.push(jsonItem); + }); + translate.setHandler("done", function(obj, item) { + if(jsonItems.length || me.selectedItems === false) { + sendResponseCallback(201, "application/json", JSON.stringify({items: jsonItems})); + } else { + sendResponseCallback(500); + } + }); + + Zotero.debug(data.translatorID || translators[0]); + translate.setTranslator(data.translatorID || translators[0]); + translate.translate({libraryID, collections: collection ? [collection.id] : false}); + } catch (e) { + Zotero.logError(e); + sendResponseCallback(500); } - - this.sendResponse = sendResponseCallback; - Zotero.Server.Connector.Detect.prototype.init.apply(this, [url, data, sendResponseCallback]) }, /** @@ -567,51 +572,6 @@ Zotero.Server.Connector.SavePage.prototype = { this.sendResponse(300, "application/json", JSON.stringify({selectItems: itemList, instanceID: instanceID, uri: this._parsedPostData.uri})); this.selectedItemsCallback = callback; }, - - /** - * Callback to be executed when list of translators becomes available. Opens progress window, - * selects specified translator, and initiates translation. - * @param {Zotero.Translate} translate - * @param {Zotero.Translator[]} translators - */ - _translatorsAvailable: function(translate, translators) { - // make sure translatorsAvailable succeded - if(!translators.length) { - Zotero.Browser.deleteHiddenBrowser(this._browser); - this.sendResponse(500); - return; - } - - var { library, collection, editable } = Zotero.Server.Connector.getSaveTarget(); - var libraryID = library.libraryID; - - // set handlers for translation - var me = this; - var jsonItems = []; - translate.setHandler("select", function(obj, item, callback) { return me._selectItems(obj, item, callback) }); - translate.setHandler("itemDone", function(obj, item, jsonItem) { - //Zotero.Server.Connector.AttachmentProgressManager.add(jsonItem.attachments); - jsonItems.push(jsonItem); - }); - translate.setHandler("attachmentProgress", function(obj, attachment, progress, error) { - //Zotero.Server.Connector.AttachmentProgressManager.onProgress(attachment, progress, error); - }); - translate.setHandler("done", function(obj, item) { - Zotero.Browser.deleteHiddenBrowser(me._browser); - if(jsonItems.length || me.selectedItems === false) { - me.sendResponse(201, "application/json", JSON.stringify({items: jsonItems})); - } else { - me.sendResponse(500); - } - }); - - if (this._parsedPostData.translatorID) { - translate.setTranslator(this._parsedPostData.translatorID); - } else { - translate.setTranslator(translators[0]); - } - translate.translate({libraryID, collections: collection ? [collection.id] : false}); - } } /** diff --git a/chrome/content/zotero/xpcom/translation/translator.js b/chrome/content/zotero/xpcom/translation/translator.js index 5ad6388b7d..52d0077355 100644 --- a/chrome/content/zotero/xpcom/translation/translator.js +++ b/chrome/content/zotero/xpcom/translation/translator.js @@ -219,3 +219,4 @@ Zotero.Translator.RUN_MODE_ZOTERO_SERVER = 4; Zotero.Translator.TRANSLATOR_TYPES = TRANSLATOR_TYPES; Zotero.Translator.TRANSLATOR_OPTIONAL_PROPERTIES = TRANSLATOR_OPTIONAL_PROPERTIES; Zotero.Translator.TRANSLATOR_REQUIRED_PROPERTIES = TRANSLATOR_REQUIRED_PROPERTIES; +Zotero.Translator.TRANSLATOR_PASSING_PROPERTIES = TRANSLATOR_PASSING_PROPERTIES; diff --git a/chrome/content/zotero/xpcom/translation/translators.js b/chrome/content/zotero/xpcom/translation/translators.js index 3936466127..2ee088572a 100644 --- a/chrome/content/zotero/xpcom/translation/translators.js +++ b/chrome/content/zotero/xpcom/translation/translators.js @@ -81,127 +81,129 @@ Zotero.Translators = new function() { var translatorsDir = Zotero.getTranslatorsDirectory().path; var iterator = new OS.File.DirectoryIterator(translatorsDir); try { - while (true) { - let entries = yield iterator.nextBatch(5); // TODO: adjust as necessary - if (!entries.length) break; - for (let i = 0; i < entries.length; i++) { - let entry = entries[i]; - let path = entry.path; - let fileName = entry.name; - - if (!(/^[^.].*\.js$/.test(fileName))) continue; - - let lastModifiedTime; - if ('winLastWriteDate' in entry) { - lastModifiedTime = entry.winLastWriteDate.getTime(); - } - else { - lastModifiedTime = (yield OS.File.stat(path)).lastModificationDate.getTime(); - } - - // Check passed cache for metadata - let memCacheJSON = false; - if (options.metadataCache && options.metadataCache[fileName]) { - memCacheJSON = options.metadataCache[fileName]; - } - - // Check DB cache - let dbCacheEntry = false; - if (dbCache[fileName]) { - filesInCache[fileName] = true; - if (dbCache[fileName].lastModifiedTime == lastModifiedTime) { - dbCacheEntry = dbCache[fileName]; + yield Zotero.DB.executeTransaction(function* () { + while (true) { + let entries = yield iterator.nextBatch(5); // TODO: adjust as necessary + if (!entries.length) break; + for (let i = 0; i < entries.length; i++) { + let entry = entries[i]; + let path = entry.path; + let fileName = entry.name; + + if (!(/^[^.].*\.js$/.test(fileName))) continue; + + let lastModifiedTime; + if ('winLastWriteDate' in entry) { + lastModifiedTime = entry.winLastWriteDate.getTime(); } - } - - // Get JSON from cache if possible - if (memCacheJSON || dbCacheEntry) { - try { - var translator = Zotero.Translators.load( - memCacheJSON || dbCacheEntry.metadataJSON, path - ); - } - catch (e) { - Zotero.logError(e); - Zotero.debug(memCacheJSON || dbCacheEntry.metadataJSON, 1); - - // If JSON is invalid, clear from cache - yield Zotero.DB.queryAsync( - "DELETE FROM translatorCache WHERE fileName=?", - fileName - ); - continue; - } - } - // Otherwise, load from file - else { - try { - var translator = yield Zotero.Translators.loadFromFile(path); - } - catch (e) { - Zotero.logError(e); - - // If translator file is invalid, delete it and clear the cache entry - // so that the translator is reinstalled the next time it's updated. - // - // TODO: Reinstall the correct translator immediately - yield OS.File.remove(path); - let sql = "DELETE FROM translatorCache WHERE fileName=?"; - yield Zotero.DB.queryAsync(sql, fileName); - continue; - } - } - - // When can this happen? - if (!translator.translatorID) { - Zotero.debug("Translator ID for " + path + " not found"); - continue; - } - - // Check if there's already a cached translator with the same id - if (_translators[translator.translatorID]) { - let existingTranslator = _translators[translator.translatorID]; - // If cached translator is older, delete it - if (existingTranslator.lastUpdated < translator.lastUpdated) { - translator.logError("Deleting older translator " - + existingTranslator.fileName + " with same ID as " - + translator.fileName); - yield OS.File.remove(existingTranslator.path); - delete _translators[translator.translatorID]; - } - // If cached translator is newer or the same, delete the current one else { - translator.logError("Translator " + existingTranslator.fileName - + " with same ID is already loaded -- deleting " - + translator.fileName); - yield OS.File.remove(translator.path); - continue; + lastModifiedTime = (yield OS.File.stat(path)).lastModificationDate.getTime(); } - } - - // add to cache - _translators[translator.translatorID] = translator; - for (let type in Zotero.Translator.TRANSLATOR_TYPES) { - if (translator.translatorType & Zotero.Translator.TRANSLATOR_TYPES[type]) { - _cache[type].push(translator); - if ((translator.translatorType & Zotero.Translator.TRANSLATOR_TYPES.web) && translator.targetAll) { - _cache.webWithTargetAll.push(translator); + + // Check passed cache for metadata + let memCacheJSON = false; + if (options.metadataCache && options.metadataCache[fileName]) { + memCacheJSON = options.metadataCache[fileName]; + } + + // Check DB cache + let dbCacheEntry = false; + if (dbCache[fileName]) { + filesInCache[fileName] = true; + if (dbCache[fileName].lastModifiedTime == lastModifiedTime) { + dbCacheEntry = dbCache[fileName]; } } + + // Get JSON from cache if possible + if (memCacheJSON || dbCacheEntry) { + try { + var translator = Zotero.Translators.load( + memCacheJSON || dbCacheEntry.metadataJSON, path + ); + } + catch (e) { + Zotero.logError(e); + Zotero.debug(memCacheJSON || dbCacheEntry.metadataJSON, 1); + + // If JSON is invalid, clear from cache + yield Zotero.DB.queryAsync( + "DELETE FROM translatorCache WHERE fileName=?", + fileName + ); + continue; + } + } + // Otherwise, load from file + else { + try { + var translator = yield Zotero.Translators.loadFromFile(path); + } + catch (e) { + Zotero.logError(e); + + // If translator file is invalid, delete it and clear the cache entry + // so that the translator is reinstalled the next time it's updated. + // + // TODO: Reinstall the correct translator immediately + yield OS.File.remove(path); + let sql = "DELETE FROM translatorCache WHERE fileName=?"; + yield Zotero.DB.queryAsync(sql, fileName); + continue; + } + } + + // When can this happen? + if (!translator.translatorID) { + Zotero.debug("Translator ID for " + path + " not found"); + continue; + } + + // Check if there's already a cached translator with the same id + if (_translators[translator.translatorID]) { + let existingTranslator = _translators[translator.translatorID]; + // If cached translator is older, delete it + if (existingTranslator.lastUpdated < translator.lastUpdated) { + translator.logError("Deleting older translator " + + existingTranslator.fileName + " with same ID as " + + translator.fileName); + yield OS.File.remove(existingTranslator.path); + delete _translators[translator.translatorID]; + } + // If cached translator is newer or the same, delete the current one + else { + translator.logError("Translator " + existingTranslator.fileName + + " with same ID is already loaded -- deleting " + + translator.fileName); + yield OS.File.remove(translator.path); + continue; + } + } + + // add to cache + _translators[translator.translatorID] = translator; + for (let type in Zotero.Translator.TRANSLATOR_TYPES) { + if (translator.translatorType & Zotero.Translator.TRANSLATOR_TYPES[type]) { + _cache[type].push(translator); + if ((translator.translatorType & Zotero.Translator.TRANSLATOR_TYPES.web) && translator.targetAll) { + _cache.webWithTargetAll.push(translator); + } + } + } + + if (!dbCacheEntry) { + yield Zotero.Translators.cacheInDB( + fileName, + translator.serialize(Zotero.Translator.TRANSLATOR_REQUIRED_PROPERTIES. + concat(Zotero.Translator.TRANSLATOR_OPTIONAL_PROPERTIES)), + lastModifiedTime + ); + } + + numCached++; } - - if (!dbCacheEntry) { - yield Zotero.Translators.cacheInDB( - fileName, - translator.serialize(Zotero.Translator.TRANSLATOR_REQUIRED_PROPERTIES. - concat(Zotero.Translator.TRANSLATOR_OPTIONAL_PROPERTIES)), - lastModifiedTime - ); - } - - numCached++; } - } + }.bind(this)) } finally { iterator.close(); diff --git a/test/tests/server_connectorTest.js b/test/tests/server_connectorTest.js index a698562e3a..9f64b540ae 100644 --- a/test/tests/server_connectorTest.js +++ b/test/tests/server_connectorTest.js @@ -472,7 +472,7 @@ describe("Connector Server", function () { continue; } } - await Zotero.Promise.delay(10); + await Zotero.Promise.delay(100); } // Legacy endpoint should show 100 @@ -579,7 +579,7 @@ describe("Connector Server", function () { } } assert.isFalse(response.done); - await Zotero.Promise.delay(10); + await Zotero.Promise.delay(100); } assert.isTrue(wasZero); @@ -791,62 +791,64 @@ describe("Connector Server", function () { }); it("should save a PDF to the current selected collection and retrieve metadata", async function () { - var collection = await createDataObject('collection'); - await waitForItemsLoad(win); - - var file = getTestDataDirectory(); - file.append('test.pdf'); - httpd.registerFile("/test.pdf", file); - - var promise = waitForItemEvent('add'); - var recognizerPromise = waitForRecognizer(); - - var origRequest = Zotero.HTTP.request.bind(Zotero.HTTP); - var called = 0; - var stub = sinon.stub(Zotero.HTTP, 'request').callsFake(function (method, url, options) { - // Forward saveSnapshot request - if (url.endsWith('saveSnapshot')) { - return origRequest(...arguments); - } + try { + var collection = await createDataObject('collection'); + await waitForItemsLoad(win); - // Fake recognizer response - return Zotero.Promise.resolve({ - getResponseHeader: () => {}, - responseText: JSON.stringify({ - title: 'Test', - authors: [] - }) + var file = getTestDataDirectory(); + file.append('test.pdf'); + httpd.registerFile("/test.pdf", file); + + var promise = waitForItemEvent('add'); + var recognizerPromise = waitForRecognizer(); + + var origRequest = Zotero.HTTP.request.bind(Zotero.HTTP); + var called = 0; + var stub = sinon.stub(Zotero.HTTP, 'request').callsFake(function (method, url, options) { + // Forward saveSnapshot request + if (url.endsWith('saveSnapshot')) { + return origRequest(...arguments); + } + + // Fake recognizer response + return Zotero.Promise.resolve({ + getResponseHeader: () => {}, + responseText: JSON.stringify({ + title: 'Test', + authors: [] + }) + }); }); - }); - - await Zotero.HTTP.request( - 'POST', - connectorServerPath + "/connector/saveSnapshot", - { - headers: { - "Content-Type": "application/json" - }, - body: JSON.stringify({ - url: testServerPath + "/test.pdf", - pdf: true - }) - } - ); - - var ids = await promise; - - assert.lengthOf(ids, 1); - var item = Zotero.Items.get(ids[0]); - assert.isTrue(item.isImportedAttachment()); - assert.equal(item.attachmentContentType, 'application/pdf'); - assert.isTrue(collection.hasItem(item.id)); - - var progressWindow = await recognizerPromise; - progressWindow.close(); - Zotero.RecognizePDF.cancel(); - assert.isFalse(item.isTopLevelItem()); - - stub.restore(); + + await Zotero.HTTP.request( + 'POST', + connectorServerPath + "/connector/saveSnapshot", + { + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify({ + url: testServerPath + "/test.pdf", + pdf: true + }) + } + ); + + var ids = await promise; + + assert.lengthOf(ids, 1); + var item = Zotero.Items.get(ids[0]); + assert.isTrue(item.isImportedAttachment()); + assert.equal(item.attachmentContentType, 'application/pdf'); + assert.isTrue(collection.hasItem(item.id)); + + var progressWindow = await recognizerPromise; + progressWindow.close(); + Zotero.RecognizePDF.cancel(); + assert.isFalse(item.isTopLevelItem()); + } finally { + stub.restore(); + } }); it("should switch to My Library if a read-only library is selected", function* () { @@ -912,7 +914,7 @@ describe("Connector Server", function () { }); it("should translate a page if translators are available", function* () { - var html = Zotero.File.getContentsFromURL(getTestDataUrl('coins.html')); + var html = yield Zotero.File.getContentsFromURLAsync(getTestDataUrl('coins.html')); var promise = waitForItemEvent('add'); var xmlhttp = yield Zotero.HTTP.request( 'POST', @@ -929,12 +931,12 @@ describe("Connector Server", function () { } ); + assert.equal(xmlhttp.status, 201); let ids = yield promise; var item = Zotero.Items.get(ids[0]); var title = "Test Page"; assert.equal(JSON.parse(xmlhttp.responseText).items[0].title, title); assert.equal(item.getField('title'), title); - assert.equal(xmlhttp.status, 201); }); });