"use strict"; describe("Connector Server", function () { Components.utils.import("resource://zotero-unit/httpd.js"); var win, connectorServerPath, testServerPath, httpd; var testServerPort = 16213; var snapshotHTML = "TitleBody"; before(function* () { this.timeout(20000); Zotero.Prefs.set("httpServer.enabled", true); yield resetDB({ thisArg: this, skipBundledFiles: true }); yield Zotero.Translators.init(); win = yield loadZoteroPane(); connectorServerPath = 'http://127.0.0.1:' + Zotero.Prefs.get('httpServer.port'); }); beforeEach(function () { // Alternate ports to prevent exceptions not catchable in JS testServerPort += (testServerPort & 1) ? 1 : -1; testServerPath = 'http://127.0.0.1:' + testServerPort; httpd = new HttpServer(); httpd.start(testServerPort); httpd.registerPathHandler( "/snapshot", { handle: function (request, response) { response.setStatusLine(null, 200, "OK"); response.write(snapshotHTML); } } ); }); afterEach(function* () { var defer = new Zotero.Promise.defer(); httpd.stop(() => defer.resolve()); yield defer.promise; }); after(function () { win.close(); }); describe('/connector/getTranslatorCode', function() { it('should respond with translator code', function* () { var code = 'function detectWeb() {}\nfunction doImport() {}'; var translator = buildDummyTranslator(4, code); sinon.stub(Zotero.Translators, 'get').returns(translator); var response = yield Zotero.HTTP.request( 'POST', connectorServerPath + "/connector/getTranslatorCode", { headers: { "Content-Type": "application/json" }, body: JSON.stringify({ translatorID: "dummy-translator", }) } ); assert.isTrue(Zotero.Translators.get.calledWith('dummy-translator')); let translatorCode = yield Zotero.Translators.getCodeForTranslator(translator); assert.equal(response.response, translatorCode); Zotero.Translators.get.restore(); }) }); describe("/connector/detect", function() { it("should return relevant translators with proxies", function* () { var code = 'function detectWeb() {return "newspaperArticle";}\nfunction doWeb() {}'; var translator = buildDummyTranslator("web", code, {target: "https://www.example.com/.*"}); sinon.stub(Zotero.Translators, 'getAllForType').resolves([translator]); var response = yield Zotero.HTTP.request( 'POST', connectorServerPath + "/connector/detect", { headers: { "Content-Type": "application/json" }, body: JSON.stringify({ uri: "https://www-example-com.proxy.example.com/article", html: "Owl

🦉

" }) } ); assert.equal(JSON.parse(response.response)[0].proxy.scheme, 'https://%h.proxy.example.com/%p'); Zotero.Translators.getAllForType.restore(); }); }); describe("/connector/saveItems", function () { // TODO: Test cookies it("should save a translated item to the current selected collection", function* () { var collection = yield createDataObject('collection'); yield waitForItemsLoad(win); var body = { items: [ { itemType: "newspaperArticle", title: "Title", creators: [ { firstName: "First", lastName: "Last", creatorType: "author" } ], attachments: [ { title: "Attachment", url: `${testServerPath}/attachment`, mimeType: "text/html" } ] } ], uri: "http://example.com" }; httpd.registerPathHandler( "/attachment", { handle: function (request, response) { response.setStatusLine(null, 200, "OK"); response.write("TitleBody"); } } ); var promise = waitForItemEvent('add'); var reqPromise = Zotero.HTTP.request( 'POST', connectorServerPath + "/connector/saveItems", { headers: { "Content-Type": "application/json" }, body: JSON.stringify(body) } ); // Check parent item var ids = yield promise; assert.lengthOf(ids, 1); var item = Zotero.Items.get(ids[0]); assert.equal(Zotero.ItemTypes.getName(item.itemTypeID), 'newspaperArticle'); assert.isTrue(collection.hasItem(item.id)); // Check attachment promise = waitForItemEvent('add'); ids = yield promise; assert.lengthOf(ids, 1); item = Zotero.Items.get(ids[0]); assert.isTrue(item.isImportedAttachment()); var req = yield reqPromise; assert.equal(req.status, 201); }); it("should switch to My Library if read-only library is selected", function* () { var group = yield createGroup({ editable: false }); yield selectLibrary(win, group.libraryID); yield waitForItemsLoad(win); var body = { items: [ { itemType: "newspaperArticle", title: "Title", creators: [ { firstName: "First", lastName: "Last", creatorType: "author" } ], attachments: [] } ], uri: "http://example.com" }; var promise = waitForItemEvent('add'); var reqPromise = Zotero.HTTP.request( 'POST', connectorServerPath + "/connector/saveItems", { headers: { "Content-Type": "application/json" }, body: JSON.stringify(body), successCodes: false } ); // My Library be selected, and the item should be in it var ids = yield promise; assert.equal( win.ZoteroPane.collectionsView.getSelectedLibraryID(), Zotero.Libraries.userLibraryID ); assert.lengthOf(ids, 1); var item = Zotero.Items.get(ids[0]); assert.equal(item.libraryID, Zotero.Libraries.userLibraryID); assert.equal(Zotero.ItemTypes.getName(item.itemTypeID), 'newspaperArticle'); var req = yield reqPromise; assert.equal(req.status, 201); }); it("should use the provided proxy to deproxify item url", function* () { yield selectLibrary(win, Zotero.Libraries.userLibraryID); yield waitForItemsLoad(win); var body = { items: [ { itemType: "newspaperArticle", title: "Title", creators: [ { firstName: "First", lastName: "Last", creatorType: "author" } ], attachments: [], url: "https://www-example-com.proxy.example.com/path" } ], uri: "https://www-example-com.proxy.example.com/path", proxy: {scheme: 'https://%h.proxy.example.com/%p'} }; var promise = waitForItemEvent('add'); var req = yield Zotero.HTTP.request( 'POST', connectorServerPath + "/connector/saveItems", { headers: { "Content-Type": "application/json" }, body: JSON.stringify(body) } ); // Check item var ids = yield promise; assert.lengthOf(ids, 1); var item = Zotero.Items.get(ids[0]); assert.equal(item.getField('url'), 'https://www.example.com/path'); }); it("shouldn't return an attachment that isn't being saved", async function () { Zotero.Prefs.set('automaticSnapshots', false); await selectLibrary(win, Zotero.Libraries.userLibraryID); await waitForItemsLoad(win); var body = { items: [ { itemType: "webpage", title: "Title", creators: [], attachments: [ { url: "http://example.com/", mimeType: "text/html" } ], url: "http://example.com/" } ], uri: "http://example.com/" }; var req = await Zotero.HTTP.request( 'POST', connectorServerPath + "/connector/saveItems", { headers: { "Content-Type": "application/json" }, body: JSON.stringify(body), responseType: 'json' } ); Zotero.Prefs.clear('automaticSnapshots'); assert.equal(req.status, 201); assert.lengthOf(req.response.items, 1); assert.lengthOf(req.response.items[0].attachments, 0); }); describe("PDF retrieval", function () { var oaDOI = '10.1111/abcd'; var nonOADOI = '10.2222/bcde'; var pdfURL; var badPDFURL; var stub; before(function () { var origFunc = Zotero.HTTP.request.bind(Zotero.HTTP); stub = sinon.stub(Zotero.HTTP, 'request'); stub.callsFake(function (method, url, options) { // OA PDF lookup if (url.startsWith(ZOTERO_CONFIG.SERVICES_URL)) { let json = JSON.parse(options.body); let response = []; if (json.doi == oaDOI) { response.push({ url: pdfURL, version: 'submittedVersion' }); } return { status: 200, response }; } return origFunc(...arguments); }); }); beforeEach(() => { pdfURL = testServerPath + '/pdf'; badPDFURL = testServerPath + '/badpdf'; httpd.registerFile( pdfURL.substr(testServerPath.length), Zotero.File.pathToFile(OS.Path.join(getTestDataDirectory().path, 'test.pdf')) ); // PDF URL that's actually an HTML page httpd.registerFile( badPDFURL.substr(testServerPath.length), Zotero.File.pathToFile(OS.Path.join(getTestDataDirectory().path, 'test.html')) ); }); afterEach(() => { stub.resetHistory(); }); after(() => { stub.restore(); }); it("should download a translated PDF", async function () { var collection = await createDataObject('collection'); await waitForItemsLoad(win); var sessionID = Zotero.Utilities.randomString(); // Save item var itemAddPromise = waitForItemEvent('add'); var saveItemsReq = await Zotero.HTTP.request( 'POST', connectorServerPath + "/connector/saveItems", { headers: { "Content-Type": "application/json" }, body: JSON.stringify({ sessionID, items: [ { itemType: 'journalArticle', title: 'Title', DOI: nonOADOI, attachments: [ { title: "PDF", url: pdfURL, mimeType: 'application/pdf' } ] } ], uri: 'http://website/article' }), responseType: 'json' } ); assert.equal(saveItemsReq.status, 201); assert.lengthOf(saveItemsReq.response.items, 1); // Translated attachment should show up in the initial response assert.lengthOf(saveItemsReq.response.items[0].attachments, 1); assert.notProperty(saveItemsReq.response.items[0], 'DOI'); assert.notProperty(saveItemsReq.response.items[0].attachments[0], 'progress'); // Check parent item var ids = await itemAddPromise; assert.lengthOf(ids, 1); var item = Zotero.Items.get(ids[0]); assert.equal(Zotero.ItemTypes.getName(item.itemTypeID), 'journalArticle'); assert.isTrue(collection.hasItem(item.id)); // Legacy endpoint should show 0 let attachmentProgressReq = await Zotero.HTTP.request( 'POST', connectorServerPath + "/connector/attachmentProgress", { headers: { "Content-Type": "application/json" }, body: JSON.stringify([saveItemsReq.response.items[0].attachments[0].id]), responseType: 'json' } ); assert.equal(attachmentProgressReq.status, 200); let progress = attachmentProgressReq.response; assert.sameOrderedMembers(progress, [0]); // Wait for the attachment to finish saving itemAddPromise = waitForItemEvent('add'); var i = 0; while (i < 3) { let sessionProgressReq = await Zotero.HTTP.request( 'POST', connectorServerPath + "/connector/sessionProgress", { headers: { "Content-Type": "application/json" }, body: JSON.stringify({ sessionID }), responseType: 'json' } ); assert.equal(sessionProgressReq.status, 200); let response = sessionProgressReq.response; assert.lengthOf(response.items, 1); let item = response.items[0]; if (item.attachments.length) { await Zotero.Promise.delay(10); let attachments = item.attachments; assert.lengthOf(attachments, 1); let attachment = attachments[0]; switch (i) { // Translated PDF in progress case 0: if (attachment.title == "PDF" && Number.isInteger(attachment.progress) && attachment.progress < 100) { assert.isFalse(response.done); i++; } continue; // Translated PDF finished case 1: if (attachment.title == "PDF" && attachment.progress == 100) { i++; } continue; // done: true case 2: if (response.done) { i++; } continue; } } } // Legacy endpoint should show 100 attachmentProgressReq = await Zotero.HTTP.request( 'POST', connectorServerPath + "/connector/attachmentProgress", { headers: { "Content-Type": "application/json" }, body: JSON.stringify([saveItemsReq.response.items[0].attachments[0].id]), responseType: 'json' } ); assert.equal(attachmentProgressReq.status, 200); progress = attachmentProgressReq.response; assert.sameOrderedMembers(progress, [100]); // Check attachment var ids = await itemAddPromise; assert.lengthOf(ids, 1); item = Zotero.Items.get(ids[0]); assert.isTrue(item.isImportedAttachment()); assert.equal(item.getField('title'), 'PDF'); }); it("should download open-access PDF if no PDF provided", async function () { var collection = await createDataObject('collection'); await waitForItemsLoad(win); var sessionID = Zotero.Utilities.randomString(); // Save item var itemAddPromise = waitForItemEvent('add'); var saveItemsReq = await Zotero.HTTP.request( 'POST', connectorServerPath + "/connector/saveItems", { headers: { "Content-Type": "application/json" }, body: JSON.stringify({ sessionID, items: [ { itemType: 'journalArticle', title: 'Title', DOI: oaDOI, attachments: [] } ], uri: 'http://website/article' }), responseType: 'json' } ); assert.equal(saveItemsReq.status, 201); assert.lengthOf(saveItemsReq.response.items, 1); // Attachment shouldn't show up in the initial response assert.lengthOf(saveItemsReq.response.items[0].attachments, 0); // Check parent item var ids = await itemAddPromise; assert.lengthOf(ids, 1); var item = Zotero.Items.get(ids[0]); assert.equal(Zotero.ItemTypes.getName(item.itemTypeID), 'journalArticle'); assert.isTrue(collection.hasItem(item.id)); // Wait for the attachment to finish saving itemAddPromise = waitForItemEvent('add'); var wasZero = false; var was100 = false; while (true) { let sessionProgressReq = await Zotero.HTTP.request( 'POST', connectorServerPath + "/connector/sessionProgress", { headers: { "Content-Type": "application/json" }, body: JSON.stringify({ sessionID }), responseType: 'json' } ); assert.equal(sessionProgressReq.status, 200); let response = sessionProgressReq.response; assert.typeOf(response.items, 'array'); assert.lengthOf(response.items, 1); let item = response.items[0]; if (item.attachments.length) { // 'progress' should have started at 0 if (item.attachments[0].progress === 0) { wasZero = true; } else if (!was100 && item.attachments[0].progress == 100) { if (response.done) { break; } was100 = true; } else if (response.done) { break; } } assert.isFalse(response.done); await Zotero.Promise.delay(10); } assert.isTrue(wasZero); // Check attachment var ids = await itemAddPromise; assert.lengthOf(ids, 1); item = Zotero.Items.get(ids[0]); assert.isTrue(item.isImportedAttachment()); assert.equal(item.getField('title'), Zotero.getString('attachment.submittedVersion')); }); it("should download open-access PDF if a translated PDF fails", async function () { var collection = await createDataObject('collection'); await waitForItemsLoad(win); var sessionID = Zotero.Utilities.randomString(); // Save item var itemAddPromise = waitForItemEvent('add'); var saveItemsReq = await Zotero.HTTP.request( 'POST', connectorServerPath + "/connector/saveItems", { headers: { "Content-Type": "application/json" }, body: JSON.stringify({ sessionID, items: [ { itemType: 'journalArticle', title: 'Title', DOI: oaDOI, attachments: [ { title: "PDF", url: badPDFURL, mimeType: 'application/pdf' } ] } ], uri: 'http://website/article' }), responseType: 'json' } ); assert.equal(saveItemsReq.status, 201); assert.lengthOf(saveItemsReq.response.items, 1); // Translated attachment should show up in the initial response assert.lengthOf(saveItemsReq.response.items[0].attachments, 1); assert.notProperty(saveItemsReq.response.items[0], 'DOI'); assert.notProperty(saveItemsReq.response.items[0].attachments[0], 'progress'); // Check parent item var ids = await itemAddPromise; assert.lengthOf(ids, 1); var item = Zotero.Items.get(ids[0]); assert.equal(Zotero.ItemTypes.getName(item.itemTypeID), 'journalArticle'); assert.isTrue(collection.hasItem(item.id)); // Legacy endpoint should show 0 let attachmentProgressReq = await Zotero.HTTP.request( 'POST', connectorServerPath + "/connector/attachmentProgress", { headers: { "Content-Type": "application/json" }, body: JSON.stringify([saveItemsReq.response.items[0].attachments[0].id]), responseType: 'json' } ); assert.equal(attachmentProgressReq.status, 200); let progress = attachmentProgressReq.response; assert.sameOrderedMembers(progress, [0]); // Wait for the attachment to finish saving itemAddPromise = waitForItemEvent('add'); var i = 0; while (i < 4) { let sessionProgressReq = await Zotero.HTTP.request( 'POST', connectorServerPath + "/connector/sessionProgress", { headers: { "Content-Type": "application/json" }, body: JSON.stringify({ sessionID }), responseType: 'json' } ); assert.equal(sessionProgressReq.status, 200); let response = sessionProgressReq.response; assert.lengthOf(response.items, 1); let item = response.items[0]; if (item.attachments.length) { let attachments = item.attachments; assert.lengthOf(attachments, 1); let attachment = attachments[0]; switch (i) { // Translated PDF in progress case 0: if (attachment.title == "PDF" && Number.isInteger(attachment.progress) && attachment.progress < 100) { assert.isFalse(response.done); i++; } continue; // OA PDF in progress case 1: if (attachment.title == Zotero.getString('findPDF.openAccessPDF') && Number.isInteger(attachment.progress) && attachment.progress < 100) { assert.isFalse(response.done); i++; } continue; // OA PDF finished case 2: if (attachment.progress === 100) { assert.equal(attachment.title, Zotero.getString('findPDF.openAccessPDF')); i++; } continue; // done: true case 3: if (response.done) { i++; } continue; } } await Zotero.Promise.delay(10); } // Legacy endpoint should show 100 attachmentProgressReq = await Zotero.HTTP.request( 'POST', connectorServerPath + "/connector/attachmentProgress", { headers: { "Content-Type": "application/json" }, body: JSON.stringify([saveItemsReq.response.items[0].attachments[0].id]), responseType: 'json' } ); assert.equal(attachmentProgressReq.status, 200); progress = attachmentProgressReq.response; assert.sameOrderedMembers(progress, [100]); // Check attachment var ids = await itemAddPromise; assert.lengthOf(ids, 1); item = Zotero.Items.get(ids[0]); assert.isTrue(item.isImportedAttachment()); assert.equal(item.getField('title'), Zotero.getString('attachment.submittedVersion')); }); }); }); describe("/connector/saveSingleFile", function () { it("should save a webpage item with /saveSnapshot", async function () { var collection = await createDataObject('collection'); await waitForItemsLoad(win); // Promise for item save let promise = waitForItemEvent('add'); let testDataDirectory = getTestDataDirectory().path; let indexPath = OS.Path.join(testDataDirectory, 'snapshot', 'index.html'); let title = Zotero.Utilities.randomString(); let sessionID = Zotero.Utilities.randomString(); let payload = { sessionID, url: "http://example.com/test", title, singleFile: true }; await Zotero.HTTP.request( 'POST', connectorServerPath + "/connector/saveSnapshot", { headers: { "Content-Type": "application/json" }, body: JSON.stringify(payload) } ); // Await item save let parentIDs = await promise; // Check parent item assert.lengthOf(parentIDs, 1); var item = Zotero.Items.get(parentIDs[0]); assert.equal(Zotero.ItemTypes.getName(item.itemTypeID), 'webpage'); assert.isTrue(collection.hasItem(item.id)); assert.equal(item.getField('title'), title); // Promise for attachment save promise = waitForItemEvent('add'); let body = JSON.stringify(Object.assign(payload, { snapshotContent: await Zotero.File.getContentsAsync(indexPath) })); await Zotero.HTTP.request( 'POST', connectorServerPath + "/connector/saveSingleFile", { headers: { "Content-Type": "application/json" }, body } ); // Await attachment save let attachmentIDs = await promise; // Check attachment assert.lengthOf(attachmentIDs, 1); item = Zotero.Items.get(attachmentIDs[0]); assert.isTrue(item.isImportedAttachment()); assert.equal(item.getField('title'), title); // Check attachment html file let attachmentDirectory = Zotero.Attachments.getStorageDirectory(item).path; let path = OS.Path.join(attachmentDirectory, 'test.html'); assert.isTrue(await OS.File.exists(path)); let contents = await Zotero.File.getContentsAsync(path); let expectedContents = await Zotero.File.getContentsAsync(indexPath); assert.equal(contents, expectedContents); }); it("should save a webpage item with /saveItems", async function () { let collection = await createDataObject('collection'); await waitForItemsLoad(win); let title = Zotero.Utilities.randomString(); let sessionID = Zotero.Utilities.randomString(); let payload = { sessionID: sessionID, items: [ { itemType: "newspaperArticle", title: title, creators: [ { firstName: "First", lastName: "Last", creatorType: "author" } ], attachments: [ { title: "Snapshot", url: `${testServerPath}/attachment`, mimeType: "text/html", singleFile: true } ] } ], uri: "http://example.com" }; let promise = waitForItemEvent('add'); let req = await Zotero.HTTP.request( 'POST', connectorServerPath + "/connector/saveItems", { headers: { "Content-Type": "application/json" }, body: JSON.stringify(payload) } ); assert.equal(req.status, 201); // Check parent item let itemIDs = await promise; assert.lengthOf(itemIDs, 1); let item = Zotero.Items.get(itemIDs[0]); assert.equal(Zotero.ItemTypes.getName(item.itemTypeID), 'newspaperArticle'); assert.isTrue(collection.hasItem(item.id)); // Promise for attachment save promise = waitForItemEvent('add'); let testDataDirectory = getTestDataDirectory().path; let indexPath = OS.Path.join(testDataDirectory, 'snapshot', 'index.html'); let body = JSON.stringify(Object.assign(payload, { snapshotContent: await Zotero.File.getContentsAsync(indexPath) })); req = await Zotero.HTTP.request( 'POST', connectorServerPath + "/connector/saveSingleFile", { headers: { "Content-Type": "application/json" }, body } ); assert.equal(req.status, 201); // Await attachment save let attachmentIDs = await promise; // Check attachment assert.lengthOf(attachmentIDs, 1); item = Zotero.Items.get(attachmentIDs[0]); assert.isTrue(item.isImportedAttachment()); assert.equal(item.getField('title'), 'Snapshot'); // Check attachment html file let attachmentDirectory = Zotero.Attachments.getStorageDirectory(item).path; let path = OS.Path.join(attachmentDirectory, 'attachment.html'); assert.isTrue(await OS.File.exists(path)); let contents = await Zotero.File.getContentsAsync(path); let expectedContents = await Zotero.File.getContentsAsync(indexPath); assert.equal(contents, expectedContents); }); it("should override SingleFileZ from old connector in /saveSnapshot", async function () { Components.utils.import("resource://gre/modules/FileUtils.jsm"); var collection = await createDataObject('collection'); await waitForItemsLoad(win); // Promise for item save let promise = waitForItemEvent('add'); let testDataDirectory = getTestDataDirectory().path; let indexPath = OS.Path.join(testDataDirectory, 'snapshot', 'index.html'); let prefix = '/' + Zotero.Utilities.randomString() + '/'; let uri = OS.Path.join(getTestDataDirectory().path, 'snapshot'); httpd.registerDirectory(prefix, new FileUtils.File(uri)); let title = Zotero.Utilities.randomString(); let sessionID = Zotero.Utilities.randomString(); let payload = { sessionID, url: testServerPath + prefix + 'index.html', title, singleFile: true }; await Zotero.HTTP.request( 'POST', connectorServerPath + "/connector/saveSnapshot", { headers: { "Content-Type": "application/json" }, body: JSON.stringify(payload) } ); // Await item save let parentIDs = await promise; // Check parent item assert.lengthOf(parentIDs, 1); var item = Zotero.Items.get(parentIDs[0]); assert.equal(Zotero.ItemTypes.getName(item.itemTypeID), 'webpage'); assert.isTrue(collection.hasItem(item.id)); assert.equal(item.getField('title'), title); // Promise for attachment save promise = waitForItemEvent('add'); let body = new FormData(); let uuid = 'binary-' + Zotero.Utilities.randomString(); body.append("payload", JSON.stringify(Object.assign(payload, { pageData: { content: await Zotero.File.getContentsAsync(indexPath), resources: { images: [ { name: "img.gif", content: uuid, binary: true } ] } } }))); await Zotero.HTTP.request( 'POST', connectorServerPath + "/connector/saveSingleFile", { headers: { "Content-Type": "multipart/form-data", "zotero-allowed-request": "true" }, body } ); // Await attachment save let attachmentIDs = await promise; // Check attachment assert.lengthOf(attachmentIDs, 1); item = Zotero.Items.get(attachmentIDs[0]); assert.isTrue(item.isImportedAttachment()); assert.equal(item.getField('title'), title); // Check attachment html file let attachmentDirectory = Zotero.Attachments.getStorageDirectory(item).path; let path = OS.Path.join(attachmentDirectory, item.attachmentFilename); assert.isTrue(await OS.File.exists(path)); let contents = await Zotero.File.getContentsAsync(path); assert.match(contents, /^