From 747c11c917fa96ecf5c30d63e8276600bdc4cce5 Mon Sep 17 00:00:00 2001 From: Adomas Ven Date: Mon, 12 Dec 2016 14:29:59 +0200 Subject: [PATCH] Improves proxy support (#1129) Improves proxy support - Automatically detect and dehyphenise https proxies which use EZProxy HttpsHyphens - Web translators now pass around Zotero.Proxy instances which can proxify/deproxify urls passed to `translate.setLocation()` before calling `translate.getTranslators()`/ translate.detect()`. The proxy passing is done within connector background/injected processes and between standalone and connectors. - Proxy protocol unified with connectors. Connectors can now pass proxies to `/connector/save_items`. The proxies will be used to resolve true item and attachment urls when saving. Closes zotero/zotero#578, zotero/zotero#721 Relevant zotero/zotero#34, zotero/zotero#556 --- .../zotero/xpcom/connector/connector.js | 11 +- .../zotero/xpcom/connector/translate_item.js | 22 ++- .../zotero/xpcom/connector/translator.js | 55 +------ chrome/content/zotero/xpcom/db.js | 7 + chrome/content/zotero/xpcom/proxy.js | 149 +++++++++++++++--- .../content/zotero/xpcom/server_connector.js | 21 +-- .../zotero/xpcom/translation/translate.js | 51 +++--- .../xpcom/translation/translate_item.js | 15 ++ .../zotero/xpcom/translation/translator.js | 3 + .../zotero/xpcom/translation/translators.js | 15 +- .../zotero/xpcom/utilities_translate.js | 16 +- components/zotero-service.js | 2 +- test/content/support.js | 2 +- test/tests/proxyTest.js | 31 ++++ test/tests/server_connectorTest.js | 73 ++++++++- test/tests/translateTest.js | 43 +++++ 16 files changed, 392 insertions(+), 124 deletions(-) create mode 100644 test/tests/proxyTest.js diff --git a/chrome/content/zotero/xpcom/connector/connector.js b/chrome/content/zotero/xpcom/connector/connector.js index 2c07575a8a..0cc2c11d1c 100644 --- a/chrome/content/zotero/xpcom/connector/connector.js +++ b/chrome/content/zotero/xpcom/connector/connector.js @@ -157,14 +157,11 @@ Zotero.Connector = new function() { options = {method: options}; } var method = options.method; - var sendRequest = (data === null || data === undefined) - ? Zotero.HTTP.doGet.bind(Zotero.HTTP) - : Zotero.HTTP.doPost.bind(Zotero.HTTP); var headers = Object.assign({ "Content-Type":"application/json", "X-Zotero-Version":Zotero.version, "X-Zotero-Connector-API-Version":CONNECTOR_API_VERSION - }, options.headers); + }, options.headers || {}); var queryString = options.queryString ? ("?" + options.queryString) : ""; var newCallback = function(req) { @@ -224,7 +221,11 @@ Zotero.Connector = new function() { if (headers["Content-Type"] == 'application/json') { data = JSON.stringify(data); } - sendRequest(uri, data, newCallback, headers); + if (data == null || data == undefined) { + Zotero.HTTP.doGet(uri, newCallback, headers); + } else { + Zotero.HTTP.doPost(uri, data, newCallback, headers); + } } }, diff --git a/chrome/content/zotero/xpcom/connector/translate_item.js b/chrome/content/zotero/xpcom/connector/translate_item.js index a3b9339493..9aeeadb1b6 100644 --- a/chrome/content/zotero/xpcom/connector/translate_item.js +++ b/chrome/content/zotero/xpcom/connector/translate_item.js @@ -26,9 +26,6 @@ /** * Save translator items. * - * In the connector these options are actually irrelevent. We're just passing the items to standalone or - * saving to server. - * * @constructor * @param {Object} options *
  • libraryID - ID of library in which items should be saved
    • @@ -36,11 +33,14 @@ *
  • attachmentMode - One of Zotero.Translate.ItemSaver.ATTACHMENT_* specifying how attachments should be saved
    • *
  • forceTagType - Force tags to specified tag type
    • *
  • cookieSandbox - Cookie sandbox for attachment requests
    • + *
  • proxy - A proxy to deproxify item URLs
    • *
  • baseURI - URI to which attachment paths should be relative
    • * */ Zotero.Translate.ItemSaver = function(options) { this.newItems = []; + this._proxy = options.proxy; + this._baseURI = options.baseURI; // Add listener for callbacks, but only for Safari or the bookmarklet. In Chrome, we // (have to) save attachments from the inject page. @@ -80,7 +80,12 @@ Zotero.Translate.ItemSaver.prototype = { saveItems: function (items, attachmentCallback) { var deferred = Zotero.Promise.defer(); // first try to save items via connector - var payload = {"items":items}; + var payload = { items, uri: this._baseURI }; + if (Zotero.isSafari) { + // This is the best in terms of cookies we can do in Safari + payload.cookie = document.cookie; + } + payload.proxy = this._proxy && this._proxy.toJSON(); Zotero.Connector.setCookiesThenSaveItems(payload, function(data, status) { if(data !== false) { Zotero.debug("Translate: Save via Standalone succeeded"); @@ -179,6 +184,10 @@ Zotero.Translate.ItemSaver.prototype = { for(var i=0, n=items.length; i www.nature.com) - var m = /^(https?:\/\/)([^\/]+)/i.exec(URI); - if (m) { - // First, drop the 0- if it exists (this is an III invention) - var host = m[2]; - if(host.substr(0, 2) === "0-") host = host.substr(2); - var hostnames = host.split("."); - for (var i=1; i} */ this.newProxyFromRow = Zotero.Promise.coroutine(function* (row) { - var proxy = new Zotero.Proxy; - yield proxy._loadFromRow(row); + var proxy = new Zotero.Proxy(row); + yield proxy.loadHosts(); return proxy; }); @@ -367,6 +367,65 @@ Zotero.Proxies = new function() { return (onlyReturnIfProxied ? false : url); } + /** + * Check the url for potential proxies and deproxify, providing a scheme to build + * a proxy object. + * + * @param URL + * @returns {Object} Unproxied url to proxy object + */ + this.getPotentialProxies = function(URL) { + var urlToProxy = {}; + // If it's a known proxied URL just return it + if (Zotero.Proxies.transparent) { + for (var proxy of Zotero.Proxies.proxies) { + if (proxy.regexp) { + var m = proxy.regexp.exec(URL); + if (m) { + let proper = proxy.toProper(m); + urlToProxy[proper] = proxy.toJSON(); + return urlToProxy; + } + } + } + } + urlToProxy[URL] = null; + + // if there is a subdomain that is also a TLD, also test against URI with the domain + // dropped after the TLD + // (i.e., www.nature.com.mutex.gmu.edu => www.nature.com) + var m = /^(https?:\/\/)([^\/]+)/i.exec(URL); + if (m) { + // First, drop the 0- if it exists (this is an III invention) + var host = m[2]; + if (host.substr(0, 2) === "0-") host = host.substr(2); + var hostnameParts = [host.split(".")]; + if (m[1] == 'https://' && host.replace(/-/g, '.') != host) { + // try replacing hyphens with dots for https protocol + // to account for EZProxy HttpsHypens mode + hostnameParts.push(host.replace(/-/g, '.').split('.')); + } + + for (let i=0; i < hostnameParts.length; i++) { + let parts = hostnameParts[i]; + // If hostnameParts has two entries, then the second one is with replaced hyphens + let dotsToHyphens = i == 1; + // skip the lowest level subdomain, domain and TLD + for (let j=1; j=0; i--) { var param = this.parameters[i]; var value = ""; if(param == "%h") { - value = uri.hostPort; + value = this.dotsToHyphens ? uri.hostPort.replace(/-/g, '.') : uri.hostPort; } else if(param == "%p") { value = uri.path.substr(1); } else if(param == "%d") { @@ -756,19 +863,13 @@ Zotero.Proxy.prototype.toProxy = function(uri) { return proxyURL; } -/** - * Loads a proxy object from a DB row - * @private - */ -Zotero.Proxy.prototype._loadFromRow = Zotero.Promise.coroutine(function* (row) { - this.proxyID = row.proxyID; - this.multiHost = !!row.multiHost; - this.autoAssociate = !!row.autoAssociate; - this.scheme = row.scheme; +Zotero.Proxy.prototype.loadHosts = Zotero.Promise.coroutine(function* () { + if (!this.proxyID) { + throw Error("Cannot load hosts without a proxyID") + } this.hosts = yield Zotero.DB.columnQueryAsync( - "SELECT hostname FROM proxyHosts WHERE proxyID = ? ORDER BY hostname", row.proxyID + "SELECT hostname FROM proxyHosts WHERE proxyID = ? ORDER BY hostname", this.proxyID ); - this.compileRegexp(); }); /** diff --git a/chrome/content/zotero/xpcom/server_connector.js b/chrome/content/zotero/xpcom/server_connector.js index 6a945d05fc..83e3427c07 100644 --- a/chrome/content/zotero/xpcom/server_connector.js +++ b/chrome/content/zotero/xpcom/server_connector.js @@ -177,17 +177,18 @@ Zotero.Server.Connector.Detect.prototype = { }, /** - * Callback to be executed when list of translators becomes available. Sends response with - * item types, translator IDs, labels, and icons for available translators. + * Callback to be executed when list of translators becomes available. Sends standard + * translator passing properties with proxies where available for translators. * @param {Zotero.Translate} translate * @param {Zotero.Translator[]} translators */ - _translatorsAvailable: function(obj, translators) { - var jsons = []; - for (let translator of translators) { - jsons.push(translator.serialize(TRANSLATOR_PASSING_PROPERTIES)); - } - this.sendResponse(200, "application/json", JSON.stringify(jsons)); + _translatorsAvailable: function(translate, translators) { + translators = translators.map(function(translator) { + translator = translator.serialize(TRANSLATOR_PASSING_PROPERTIES.concat('proxy')); + translator.proxy = translator.proxy ? translator.proxy.toJSON() : null; + return translator; + }); + this.sendResponse(200, "application/json", JSON.stringify(translators)); Zotero.Browser.deleteHiddenBrowser(this._browser); } @@ -371,13 +372,15 @@ Zotero.Server.Connector.SaveItem.prototype = { Zotero.Server.Connector.AttachmentProgressManager.add(data.items[i].attachments); } + let proxy = data.proxy && new Zotero.Proxy(data.proxy); // save items var itemSaver = new Zotero.Translate.ItemSaver({ libraryID, collections: collection ? [collection.id] : undefined, attachmentMode: Zotero.Translate.ItemSaver.ATTACHMENT_MODE_DOWNLOAD, forceTagType: 1, - cookieSandbox + cookieSandbox, + proxy }); try { let items = yield itemSaver.saveItems( diff --git a/chrome/content/zotero/xpcom/translation/translate.js b/chrome/content/zotero/xpcom/translation/translate.js index d2c560f003..7e01065501 100644 --- a/chrome/content/zotero/xpcom/translation/translate.js +++ b/chrome/content/zotero/xpcom/translation/translate.js @@ -1120,18 +1120,21 @@ Zotero.Translate.Base.prototype = { // if detection returns immediately, return found translators return potentialTranslators.then(function(result) { var allPotentialTranslators = result[0]; - var properToProxyFunctions = result[1]; + var proxies = result[1]; // this gets passed out by Zotero.Translators.getWebTranslatorsForLocation() because it is // specific for each translator, but we want to avoid making a copy of a translator whenever // possible. - this._properToProxyFunctions = properToProxyFunctions ? properToProxyFunctions : null; + this._proxies = proxies ? [] : null; this._waitingForRPC = false; for(var i=0, n=allPotentialTranslators.length; iattachmentMode - One of Zotero.Translate.ItemSaver.ATTACHMENT_* specifying how attachments should be saved *
  • forceTagType - Force tags to specified tag type
    • *
  • cookieSandbox - Cookie sandbox for attachment requests
    • + *
  • proxy - A proxy to deproxify item URLs
    • *
  • baseURI - URI to which attachment paths should be relative
    • */ Zotero.Translate.ItemSaver = function(options) { @@ -53,6 +54,7 @@ Zotero.Translate.ItemSaver = function(options) { Zotero.Translate.ItemSaver.ATTACHMENT_MODE_IGNORE; this._forceTagType = options.forceTagType; this._cookieSandbox = options.cookieSandbox; + this._proxy = options.proxy; // the URI to which other URIs are assumed to be relative if(typeof baseURI === "object" && baseURI instanceof Components.interfaces.nsIURI) { @@ -109,6 +111,13 @@ Zotero.Translate.ItemSaver.prototype = { }; newItem.fromJSON(this._deleteIrrelevantFields(item)); + // deproxify url + if (this._proxy && item.url) { + let url = this._proxy.toProper(item.url); + Zotero.debug(`Deproxifying item url ${item.url} with scheme ${this._proxy.scheme} to ${url}`, 5); + newItem.setField('url', url); + } + if (this._collections) { newItem.setCollections(this._collections); } @@ -253,6 +262,12 @@ Zotero.Translate.ItemSaver.prototype = { } if (!newAttachment) return false; // attachmentCallback should not have been called in this case + + // deproxify url + let url = newAttachment.getField('url'); + if (this._proxy && url) { + newAttachment.setField('url', this._proxy.toProper(url)); + } // save fields if (attachment.accessDate) newAttachment.setField("accessDate", attachment.accessDate); diff --git a/chrome/content/zotero/xpcom/translation/translator.js b/chrome/content/zotero/xpcom/translation/translator.js index 1b6a754812..b8e1ab1784 100644 --- a/chrome/content/zotero/xpcom/translation/translator.js +++ b/chrome/content/zotero/xpcom/translation/translator.js @@ -23,6 +23,9 @@ ***** END LICENSE BLOCK ***** */ +// Enumeration of types of translators +var TRANSLATOR_TYPES = {"import":1, "export":2, "web":4, "search":8}; + // Properties required for every translator var TRANSLATOR_REQUIRED_PROPERTIES = ["translatorID", "translatorType", "label", "creator", "target", "priority", "lastUpdated"]; diff --git a/chrome/content/zotero/xpcom/translation/translators.js b/chrome/content/zotero/xpcom/translation/translators.js index 62defd6381..52423f6e98 100644 --- a/chrome/content/zotero/xpcom/translation/translators.js +++ b/chrome/content/zotero/xpcom/translation/translators.js @@ -25,9 +25,6 @@ "use strict"; -// Enumeration of types of translators -var TRANSLATOR_TYPES = {"import":1, "export":2, "web":4, "search":8}; - /** * Singleton to handle loading and caching of translators * @namespace @@ -297,10 +294,10 @@ Zotero.Translators = new function() { return this.getAllForType(type).then(function(allTranslators) { var potentialTranslators = []; - var converterFunctions = []; + var proxies = []; - var rootSearchURIs = this.getSearchURIs(rootURI); - var frameSearchURIs = isFrame ? this.getSearchURIs(URI) : rootSearchURIs; + var rootSearchURIs = Zotero.Proxies.getPotentialProxies(rootURI); + var frameSearchURIs = isFrame ? Zotero.Proxies.getPotentialProxies(URI) : rootSearchURIs; Zotero.debug("Translators: Looking for translators for "+Object.keys(frameSearchURIs).join(', ')); @@ -316,7 +313,7 @@ Zotero.Translators = new function() { if (frameURIMatches) { potentialTranslators.push(translator); - converterFunctions.push(frameSearchURIs[frameSearchURI]); + proxies.push(frameSearchURIs[frameSearchURI]); // prevent adding the translator multiple times break rootURIsLoop; } @@ -324,13 +321,13 @@ Zotero.Translators = new function() { } else if(!isFrame && (isGeneric || rootURIMatches)) { potentialTranslators.push(translator); - converterFunctions.push(rootSearchURIs[rootSearchURI]); + proxies.push(rootSearchURIs[rootSearchURI]); break; } } } - return [potentialTranslators, converterFunctions]; + return [potentialTranslators, proxies]; }.bind(this)); }, diff --git a/chrome/content/zotero/xpcom/utilities_translate.js b/chrome/content/zotero/xpcom/utilities_translate.js index 54ec773e76..19c53cce95 100644 --- a/chrome/content/zotero/xpcom/utilities_translate.js +++ b/chrome/content/zotero/xpcom/utilities_translate.js @@ -253,8 +253,8 @@ Zotero.Utilities.Translate.prototype.processDocuments = function(urls, processor } for(var i=0; iOwl

    🦉

    " + }) + } + ); + + assert.equal(JSON.parse(response.response)[0].proxy.scheme, 'https://%h.proxy.example.com/%p'); + + Zotero.Translators.getAllForType.restore(); + }); + }); + + describe("/connector/saveItems", function () { // TODO: Test cookies it("should save a translated item to the current selected collection", function* () { @@ -185,6 +213,49 @@ describe("Connector Server", function () { win.ZoteroPane.collectionsView.getSelectedLibraryID(), Zotero.Libraries.userLibraryID ); }); + + it("should use the provided proxy to deproxify item url", function* () { + yield selectLibrary(win, Zotero.Libraries.userLibraryID); + yield waitForItemsLoad(win); + + var body = { + items: [ + { + itemType: "newspaperArticle", + title: "Title", + creators: [ + { + firstName: "First", + lastName: "Last", + creatorType: "author" + } + ], + attachments: [], + url: "https://www-example-com.proxy.example.com/path" + } + ], + uri: "https://www-example-com.proxy.example.com/path", + proxy: {scheme: 'https://%h.proxy.example.com/%p', dotsToHyphens: true} + }; + + var promise = waitForItemEvent('add'); + var req = yield Zotero.HTTP.request( + 'POST', + connectorServerPath + "/connector/saveItems", + { + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify(body) + } + ); + + // Check item + var ids = yield promise; + assert.lengthOf(ids, 1); + var item = Zotero.Items.get(ids[0]); + assert.equal(item.getField('url'), 'https://www.example.com/path'); + }); }); describe("/connector/saveSnapshot", function () { diff --git a/test/tests/translateTest.js b/test/tests/translateTest.js index 35edfaa5f6..b648b1c446 100644 --- a/test/tests/translateTest.js +++ b/test/tests/translateTest.js @@ -680,6 +680,49 @@ describe("Zotero.Translate", function() { assert.isNumber(translation.newItems[0].id); assert.ok(collection.hasItem(translation.newItems[0].id)); }); + + }); + describe('#saveItems', function() { + it("should deproxify item and attachment urls when proxy provided", function* (){ + var itemID; + var item = loadSampleData('journalArticle'); + item = item.journalArticle; + item.url = 'https://www-example-com.proxy.example.com/'; + item.attachments = [{ + url: 'https://www-example-com.proxy.example.com/pdf.pdf', + mimeType: 'application/pdf', + title: 'Example PDF'}]; + var itemSaver = new Zotero.Translate.ItemSaver({ + libraryID: Zotero.Libraries.userLibraryID, + attachmentMode: Zotero.Translate.ItemSaver.ATTACHMENT_MODE_FILE, + proxy: new Zotero.Proxy({scheme: 'https://%h.proxy.example.com/%p', dotsToHyphens: true}) + }); + var itemDeferred = Zotero.Promise.defer(); + var attachmentDeferred = Zotero.Promise.defer(); + itemSaver.saveItems([item], Zotero.Promise.coroutine(function* (attachment, progressPercentage) { + // ItemSaver returns immediately without waiting for attachments, so we use the callback + // to test attachments + if (progressPercentage != 100) return; + try { + yield itemDeferred.promise; + let item = Zotero.Items.get(itemID); + attachment = Zotero.Items.get(item.getAttachments()[0]); + assert.equal(attachment.getField('url'), 'https://www.example.com/pdf.pdf'); + attachmentDeferred.resolve(); + } catch (e) { + attachmentDeferred.reject(e); + } + })).then(function(items) { + try { + assert.equal(items[0].getField('url'), 'https://www.example.com/'); + itemID = items[0].id; + itemDeferred.resolve(); + } catch (e) { + itemDeferred.reject(e); + } + }); + yield Zotero.Promise.all([itemDeferred.promise, attachmentDeferred.promise]); + }); }); }); });