Make Connector Server tests compatible with electron

Refactor /detect and /savePage endpoints
This commit is contained in:
Adomas Venčkauskas 2018-08-23 15:57:59 +03:00
parent 57fd05c7e3
commit 377f8d0aa8
4 changed files with 239 additions and 274 deletions

View file

@ -100,7 +100,7 @@ Zotero.Server.Connector = {
case 'C':
collection = Zotero.Collections.get(id);
library = collection.library;
editable = collection.editable;
editable = collection.library.editable;
break;
default:
@ -447,59 +447,25 @@ Zotero.Server.Connector.Detect.prototype = {
* @param {Object} data POST data or GET query string
* @param {Function} sendResponseCallback function to send HTTP response
*/
init: function(url, data, sendResponseCallback) {
this.sendResponse = sendResponseCallback;
this._parsedPostData = data;
init: async function({data}) {
var translate = new Zotero.Translate.Web();
this._translate = new Zotero.Translate("web");
this._translate.setHandler("translators", function(obj, item) { me._translatorsAvailable(obj, item) });
var parser = Components.classes["@mozilla.org/xmlextras/domparser;1"]
.createInstance(Components.interfaces.nsIDOMParser);
var doc = parser.parseFromString(`<html>${data.html}</html>`, 'text/html');
doc = Zotero.HTTP.wrapDocument(doc, data.uri);
Zotero.Server.Connector.Data[this._parsedPostData["uri"]] = "<html>"+this._parsedPostData["html"]+"</html>";
this._browser = Zotero.Browser.createHiddenBrowser();
var ioService = Components.classes["@mozilla.org/network/io-service;1"]
.getService(Components.interfaces.nsIIOService);
var uri = ioService.newURI(this._parsedPostData["uri"], "UTF-8", null);
var pageShowCalled = false;
var me = this;
this._translate.setCookieSandbox(new Zotero.CookieSandbox(this._browser,
this._parsedPostData["uri"], this._parsedPostData["cookie"], url.userAgent));
this._browser.addEventListener("DOMContentLoaded", function() {
try {
if(me._browser.contentDocument.location.href == "about:blank") return;
if(pageShowCalled) return;
pageShowCalled = true;
delete Zotero.Server.Connector.Data[me._parsedPostData["uri"]];
// get translators
me._translate.setDocument(me._browser.contentDocument);
me._translate.setLocation(me._parsedPostData["uri"], me._parsedPostData["uri"]);
me._translate.getTranslators();
} catch(e) {
sendResponseCallback(500);
throw e;
}
}, false);
me._browser.loadURI("zotero://connector/"+encodeURIComponent(this._parsedPostData["uri"]));
},
// get translators
translate.setDocument(doc);
/**
* Callback to be executed when list of translators becomes available. Sends standard
* translator passing properties with proxies where available for translators.
* @param {Zotero.Translate} translate
* @param {Zotero.Translator[]} translators
*/
_translatorsAvailable: function(translate, translators) {
var translators = await translate.getTranslators();
translators = translators.map(function(translator) {
translator = translator.serialize(TRANSLATOR_PASSING_PROPERTIES.concat('proxy'));
translator = translator.serialize(Zotero.Translator.TRANSLATOR_PASSING_PROPERTIES.concat('proxy'));
translator.proxy = translator.proxy ? translator.proxy.toJSON() : null;
return translator;
});
this.sendResponse(200, "application/json", JSON.stringify(translators));
Zotero.Browser.deleteHiddenBrowser(this._browser);
return [200, "application/json", JSON.stringify(translators)];
}
}
@ -532,17 +498,56 @@ Zotero.Server.Connector.SavePage.prototype = {
* @param {Object} data POST data or GET query string
* @param {Function} sendResponseCallback function to send HTTP response
*/
init: function(url, data, sendResponseCallback) {
var { library, collection, editable } = Zotero.Server.Connector.getSaveTarget();
// Shouldn't happen as long as My Library exists
if (!library.editable) {
Zotero.logError("Can't add item to read-only library " + library.name);
return sendResponseCallback(500, "application/json", JSON.stringify({ libraryEditable: false }));
init: async function(url, data, sendResponseCallback) {
try {
this.sendResponse = sendResponseCallback;
var { library, collection, editable } = Zotero.Server.Connector.getSaveTarget();
// Shouldn't happen as long as My Library exists
if (!library.editable) {
Zotero.logError("Can't add item to read-only library " + library.name);
return sendResponseCallback(500, "application/json", JSON.stringify({ libraryEditable: false }));
}
var libraryID = library.libraryID;
var translate = new Zotero.Translate.Web();
var parser = Components.classes["@mozilla.org/xmlextras/domparser;1"]
.createInstance(Components.interfaces.nsIDOMParser);
var doc = parser.parseFromString(`<html>${data.html}</html>`, 'text/html');
doc = Zotero.HTTP.wrapDocument(doc, data.uri);
// get translators
translate.setDocument(doc);
var translators = await translate.getTranslators();
// make sure translatorsAvailable succeded
if(!translators.length) {
return sendResponseCallback(500);
}
// set handlers for translation
var jsonItems = [];
translate.setHandler("select", function(obj, item, callback) { return me._selectItems(obj, item, callback) });
translate.setHandler("itemDone", function(obj, item, jsonItem) {
jsonItems.push(jsonItem);
});
translate.setHandler("done", function(obj, item) {
if(jsonItems.length || me.selectedItems === false) {
sendResponseCallback(201, "application/json", JSON.stringify({items: jsonItems}));
} else {
sendResponseCallback(500);
}
});
Zotero.debug(data.translatorID || translators[0]);
translate.setTranslator(data.translatorID || translators[0]);
translate.translate({libraryID, collections: collection ? [collection.id] : false});
} catch (e) {
Zotero.logError(e);
sendResponseCallback(500);
}
this.sendResponse = sendResponseCallback;
Zotero.Server.Connector.Detect.prototype.init.apply(this, [url, data, sendResponseCallback])
},
/**
@ -567,51 +572,6 @@ Zotero.Server.Connector.SavePage.prototype = {
this.sendResponse(300, "application/json", JSON.stringify({selectItems: itemList, instanceID: instanceID, uri: this._parsedPostData.uri}));
this.selectedItemsCallback = callback;
},
/**
* Callback to be executed when list of translators becomes available. Opens progress window,
* selects specified translator, and initiates translation.
* @param {Zotero.Translate} translate
* @param {Zotero.Translator[]} translators
*/
_translatorsAvailable: function(translate, translators) {
// make sure translatorsAvailable succeded
if(!translators.length) {
Zotero.Browser.deleteHiddenBrowser(this._browser);
this.sendResponse(500);
return;
}
var { library, collection, editable } = Zotero.Server.Connector.getSaveTarget();
var libraryID = library.libraryID;
// set handlers for translation
var me = this;
var jsonItems = [];
translate.setHandler("select", function(obj, item, callback) { return me._selectItems(obj, item, callback) });
translate.setHandler("itemDone", function(obj, item, jsonItem) {
//Zotero.Server.Connector.AttachmentProgressManager.add(jsonItem.attachments);
jsonItems.push(jsonItem);
});
translate.setHandler("attachmentProgress", function(obj, attachment, progress, error) {
//Zotero.Server.Connector.AttachmentProgressManager.onProgress(attachment, progress, error);
});
translate.setHandler("done", function(obj, item) {
Zotero.Browser.deleteHiddenBrowser(me._browser);
if(jsonItems.length || me.selectedItems === false) {
me.sendResponse(201, "application/json", JSON.stringify({items: jsonItems}));
} else {
me.sendResponse(500);
}
});
if (this._parsedPostData.translatorID) {
translate.setTranslator(this._parsedPostData.translatorID);
} else {
translate.setTranslator(translators[0]);
}
translate.translate({libraryID, collections: collection ? [collection.id] : false});
}
}
/**

View file

@ -219,3 +219,4 @@ Zotero.Translator.RUN_MODE_ZOTERO_SERVER = 4;
Zotero.Translator.TRANSLATOR_TYPES = TRANSLATOR_TYPES;
Zotero.Translator.TRANSLATOR_OPTIONAL_PROPERTIES = TRANSLATOR_OPTIONAL_PROPERTIES;
Zotero.Translator.TRANSLATOR_REQUIRED_PROPERTIES = TRANSLATOR_REQUIRED_PROPERTIES;
Zotero.Translator.TRANSLATOR_PASSING_PROPERTIES = TRANSLATOR_PASSING_PROPERTIES;

View file

@ -81,127 +81,129 @@ Zotero.Translators = new function() {
var translatorsDir = Zotero.getTranslatorsDirectory().path;
var iterator = new OS.File.DirectoryIterator(translatorsDir);
try {
while (true) {
let entries = yield iterator.nextBatch(5); // TODO: adjust as necessary
if (!entries.length) break;
for (let i = 0; i < entries.length; i++) {
let entry = entries[i];
let path = entry.path;
let fileName = entry.name;
if (!(/^[^.].*\.js$/.test(fileName))) continue;
let lastModifiedTime;
if ('winLastWriteDate' in entry) {
lastModifiedTime = entry.winLastWriteDate.getTime();
}
else {
lastModifiedTime = (yield OS.File.stat(path)).lastModificationDate.getTime();
}
// Check passed cache for metadata
let memCacheJSON = false;
if (options.metadataCache && options.metadataCache[fileName]) {
memCacheJSON = options.metadataCache[fileName];
}
// Check DB cache
let dbCacheEntry = false;
if (dbCache[fileName]) {
filesInCache[fileName] = true;
if (dbCache[fileName].lastModifiedTime == lastModifiedTime) {
dbCacheEntry = dbCache[fileName];
yield Zotero.DB.executeTransaction(function* () {
while (true) {
let entries = yield iterator.nextBatch(5); // TODO: adjust as necessary
if (!entries.length) break;
for (let i = 0; i < entries.length; i++) {
let entry = entries[i];
let path = entry.path;
let fileName = entry.name;
if (!(/^[^.].*\.js$/.test(fileName))) continue;
let lastModifiedTime;
if ('winLastWriteDate' in entry) {
lastModifiedTime = entry.winLastWriteDate.getTime();
}
}
// Get JSON from cache if possible
if (memCacheJSON || dbCacheEntry) {
try {
var translator = Zotero.Translators.load(
memCacheJSON || dbCacheEntry.metadataJSON, path
);
}
catch (e) {
Zotero.logError(e);
Zotero.debug(memCacheJSON || dbCacheEntry.metadataJSON, 1);
// If JSON is invalid, clear from cache
yield Zotero.DB.queryAsync(
"DELETE FROM translatorCache WHERE fileName=?",
fileName
);
continue;
}
}
// Otherwise, load from file
else {
try {
var translator = yield Zotero.Translators.loadFromFile(path);
}
catch (e) {
Zotero.logError(e);
// If translator file is invalid, delete it and clear the cache entry
// so that the translator is reinstalled the next time it's updated.
//
// TODO: Reinstall the correct translator immediately
yield OS.File.remove(path);
let sql = "DELETE FROM translatorCache WHERE fileName=?";
yield Zotero.DB.queryAsync(sql, fileName);
continue;
}
}
// When can this happen?
if (!translator.translatorID) {
Zotero.debug("Translator ID for " + path + " not found");
continue;
}
// Check if there's already a cached translator with the same id
if (_translators[translator.translatorID]) {
let existingTranslator = _translators[translator.translatorID];
// If cached translator is older, delete it
if (existingTranslator.lastUpdated < translator.lastUpdated) {
translator.logError("Deleting older translator "
+ existingTranslator.fileName + " with same ID as "
+ translator.fileName);
yield OS.File.remove(existingTranslator.path);
delete _translators[translator.translatorID];
}
// If cached translator is newer or the same, delete the current one
else {
translator.logError("Translator " + existingTranslator.fileName
+ " with same ID is already loaded -- deleting "
+ translator.fileName);
yield OS.File.remove(translator.path);
continue;
lastModifiedTime = (yield OS.File.stat(path)).lastModificationDate.getTime();
}
}
// add to cache
_translators[translator.translatorID] = translator;
for (let type in Zotero.Translator.TRANSLATOR_TYPES) {
if (translator.translatorType & Zotero.Translator.TRANSLATOR_TYPES[type]) {
_cache[type].push(translator);
if ((translator.translatorType & Zotero.Translator.TRANSLATOR_TYPES.web) && translator.targetAll) {
_cache.webWithTargetAll.push(translator);
// Check passed cache for metadata
let memCacheJSON = false;
if (options.metadataCache && options.metadataCache[fileName]) {
memCacheJSON = options.metadataCache[fileName];
}
// Check DB cache
let dbCacheEntry = false;
if (dbCache[fileName]) {
filesInCache[fileName] = true;
if (dbCache[fileName].lastModifiedTime == lastModifiedTime) {
dbCacheEntry = dbCache[fileName];
}
}
// Get JSON from cache if possible
if (memCacheJSON || dbCacheEntry) {
try {
var translator = Zotero.Translators.load(
memCacheJSON || dbCacheEntry.metadataJSON, path
);
}
catch (e) {
Zotero.logError(e);
Zotero.debug(memCacheJSON || dbCacheEntry.metadataJSON, 1);
// If JSON is invalid, clear from cache
yield Zotero.DB.queryAsync(
"DELETE FROM translatorCache WHERE fileName=?",
fileName
);
continue;
}
}
// Otherwise, load from file
else {
try {
var translator = yield Zotero.Translators.loadFromFile(path);
}
catch (e) {
Zotero.logError(e);
// If translator file is invalid, delete it and clear the cache entry
// so that the translator is reinstalled the next time it's updated.
//
// TODO: Reinstall the correct translator immediately
yield OS.File.remove(path);
let sql = "DELETE FROM translatorCache WHERE fileName=?";
yield Zotero.DB.queryAsync(sql, fileName);
continue;
}
}
// When can this happen?
if (!translator.translatorID) {
Zotero.debug("Translator ID for " + path + " not found");
continue;
}
// Check if there's already a cached translator with the same id
if (_translators[translator.translatorID]) {
let existingTranslator = _translators[translator.translatorID];
// If cached translator is older, delete it
if (existingTranslator.lastUpdated < translator.lastUpdated) {
translator.logError("Deleting older translator "
+ existingTranslator.fileName + " with same ID as "
+ translator.fileName);
yield OS.File.remove(existingTranslator.path);
delete _translators[translator.translatorID];
}
// If cached translator is newer or the same, delete the current one
else {
translator.logError("Translator " + existingTranslator.fileName
+ " with same ID is already loaded -- deleting "
+ translator.fileName);
yield OS.File.remove(translator.path);
continue;
}
}
// add to cache
_translators[translator.translatorID] = translator;
for (let type in Zotero.Translator.TRANSLATOR_TYPES) {
if (translator.translatorType & Zotero.Translator.TRANSLATOR_TYPES[type]) {
_cache[type].push(translator);
if ((translator.translatorType & Zotero.Translator.TRANSLATOR_TYPES.web) && translator.targetAll) {
_cache.webWithTargetAll.push(translator);
}
}
}
if (!dbCacheEntry) {
yield Zotero.Translators.cacheInDB(
fileName,
translator.serialize(Zotero.Translator.TRANSLATOR_REQUIRED_PROPERTIES.
concat(Zotero.Translator.TRANSLATOR_OPTIONAL_PROPERTIES)),
lastModifiedTime
);
}
numCached++;
}
if (!dbCacheEntry) {
yield Zotero.Translators.cacheInDB(
fileName,
translator.serialize(Zotero.Translator.TRANSLATOR_REQUIRED_PROPERTIES.
concat(Zotero.Translator.TRANSLATOR_OPTIONAL_PROPERTIES)),
lastModifiedTime
);
}
numCached++;
}
}
}.bind(this))
}
finally {
iterator.close();

View file

@ -472,7 +472,7 @@ describe("Connector Server", function () {
continue;
}
}
await Zotero.Promise.delay(10);
await Zotero.Promise.delay(100);
}
// Legacy endpoint should show 100
@ -579,7 +579,7 @@ describe("Connector Server", function () {
}
}
assert.isFalse(response.done);
await Zotero.Promise.delay(10);
await Zotero.Promise.delay(100);
}
assert.isTrue(wasZero);
@ -791,62 +791,64 @@ describe("Connector Server", function () {
});
it("should save a PDF to the current selected collection and retrieve metadata", async function () {
var collection = await createDataObject('collection');
await waitForItemsLoad(win);
var file = getTestDataDirectory();
file.append('test.pdf');
httpd.registerFile("/test.pdf", file);
var promise = waitForItemEvent('add');
var recognizerPromise = waitForRecognizer();
var origRequest = Zotero.HTTP.request.bind(Zotero.HTTP);
var called = 0;
var stub = sinon.stub(Zotero.HTTP, 'request').callsFake(function (method, url, options) {
// Forward saveSnapshot request
if (url.endsWith('saveSnapshot')) {
return origRequest(...arguments);
}
try {
var collection = await createDataObject('collection');
await waitForItemsLoad(win);
// Fake recognizer response
return Zotero.Promise.resolve({
getResponseHeader: () => {},
responseText: JSON.stringify({
title: 'Test',
authors: []
})
var file = getTestDataDirectory();
file.append('test.pdf');
httpd.registerFile("/test.pdf", file);
var promise = waitForItemEvent('add');
var recognizerPromise = waitForRecognizer();
var origRequest = Zotero.HTTP.request.bind(Zotero.HTTP);
var called = 0;
var stub = sinon.stub(Zotero.HTTP, 'request').callsFake(function (method, url, options) {
// Forward saveSnapshot request
if (url.endsWith('saveSnapshot')) {
return origRequest(...arguments);
}
// Fake recognizer response
return Zotero.Promise.resolve({
getResponseHeader: () => {},
responseText: JSON.stringify({
title: 'Test',
authors: []
})
});
});
});
await Zotero.HTTP.request(
'POST',
connectorServerPath + "/connector/saveSnapshot",
{
headers: {
"Content-Type": "application/json"
},
body: JSON.stringify({
url: testServerPath + "/test.pdf",
pdf: true
})
}
);
var ids = await promise;
assert.lengthOf(ids, 1);
var item = Zotero.Items.get(ids[0]);
assert.isTrue(item.isImportedAttachment());
assert.equal(item.attachmentContentType, 'application/pdf');
assert.isTrue(collection.hasItem(item.id));
var progressWindow = await recognizerPromise;
progressWindow.close();
Zotero.RecognizePDF.cancel();
assert.isFalse(item.isTopLevelItem());
stub.restore();
await Zotero.HTTP.request(
'POST',
connectorServerPath + "/connector/saveSnapshot",
{
headers: {
"Content-Type": "application/json"
},
body: JSON.stringify({
url: testServerPath + "/test.pdf",
pdf: true
})
}
);
var ids = await promise;
assert.lengthOf(ids, 1);
var item = Zotero.Items.get(ids[0]);
assert.isTrue(item.isImportedAttachment());
assert.equal(item.attachmentContentType, 'application/pdf');
assert.isTrue(collection.hasItem(item.id));
var progressWindow = await recognizerPromise;
progressWindow.close();
Zotero.RecognizePDF.cancel();
assert.isFalse(item.isTopLevelItem());
} finally {
stub.restore();
}
});
it("should switch to My Library if a read-only library is selected", function* () {
@ -912,7 +914,7 @@ describe("Connector Server", function () {
});
it("should translate a page if translators are available", function* () {
var html = Zotero.File.getContentsFromURL(getTestDataUrl('coins.html'));
var html = yield Zotero.File.getContentsFromURLAsync(getTestDataUrl('coins.html'));
var promise = waitForItemEvent('add');
var xmlhttp = yield Zotero.HTTP.request(
'POST',
@ -929,12 +931,12 @@ describe("Connector Server", function () {
}
);
assert.equal(xmlhttp.status, 201);
let ids = yield promise;
var item = Zotero.Items.get(ids[0]);
var title = "Test Page";
assert.equal(JSON.parse(xmlhttp.responseText).items[0].title, title);
assert.equal(item.getField('title'), title);
assert.equal(xmlhttp.status, 201);
});
});