Improves proxy support (#1129)
Improves proxy support - Automatically detect and dehyphenise https proxies which use EZProxy HttpsHyphens - Web translators now pass around Zotero.Proxy instances which can proxify/deproxify urls passed to `translate.setLocation()` before calling `translate.getTranslators()`/ translate.detect()`. The proxy passing is done within connector background/injected processes and between standalone and connectors. - Proxy protocol unified with connectors. Connectors can now pass proxies to `/connector/save_items`. The proxies will be used to resolve true item and attachment urls when saving. Closes zotero/zotero#578, zotero/zotero#721 Relevant zotero/zotero#34, zotero/zotero#556
This commit is contained in:
Adomas Ven
GitHub
parent
c2ebcc9dbc
commit
747c11c917
16 changed files with 392 additions and 124 deletions
|
|
@ -157,14 +157,11 @@ Zotero.Connector = new function() {
|
|||
options = {method: options};
|
||||
}
|
||||
var method = options.method;
|
||||
var sendRequest = (data === null || data === undefined)
|
||||
? Zotero.HTTP.doGet.bind(Zotero.HTTP)
|
||||
: Zotero.HTTP.doPost.bind(Zotero.HTTP);
|
||||
var headers = Object.assign({
|
||||
"Content-Type":"application/json",
|
||||
"X-Zotero-Version":Zotero.version,
|
||||
"X-Zotero-Connector-API-Version":CONNECTOR_API_VERSION
|
||||
}, options.headers);
|
||||
}, options.headers || {});
|
||||
var queryString = options.queryString ? ("?" + options.queryString) : "";
|
||||
|
||||
var newCallback = function(req) {
|
||||
|
|
@ -224,7 +221,11 @@ Zotero.Connector = new function() {
|
|||
if (headers["Content-Type"] == 'application/json') {
|
||||
data = JSON.stringify(data);
|
||||
}
|
||||
sendRequest(uri, data, newCallback, headers);
|
||||
if (data == null || data == undefined) {
|
||||
Zotero.HTTP.doGet(uri, newCallback, headers);
|
||||
} else {
|
||||
Zotero.HTTP.doPost(uri, data, newCallback, headers);
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
|
|
|
|||
|
|
@ -26,9 +26,6 @@
|
|||
/**
|
||||
* Save translator items.
|
||||
*
|
||||
* In the connector these options are actually irrelevent. We're just passing the items to standalone or
|
||||
* saving to server.
|
||||
*
|
||||
* @constructor
|
||||
* @param {Object} options
|
||||
* <li>libraryID - ID of library in which items should be saved</li>
|
||||
|
|
@ -36,11 +33,14 @@
|
|||
* <li>attachmentMode - One of Zotero.Translate.ItemSaver.ATTACHMENT_* specifying how attachments should be saved</li>
|
||||
* <li>forceTagType - Force tags to specified tag type</li>
|
||||
* <li>cookieSandbox - Cookie sandbox for attachment requests</li>
|
||||
* <li>proxy - A proxy to deproxify item URLs</li>
|
||||
* <li>baseURI - URI to which attachment paths should be relative</li>
|
||||
*
|
||||
*/
|
||||
Zotero.Translate.ItemSaver = function(options) {
|
||||
this.newItems = [];
|
||||
this._proxy = options.proxy;
|
||||
this._baseURI = options.baseURI;
|
||||
|
||||
// Add listener for callbacks, but only for Safari or the bookmarklet. In Chrome, we
|
||||
// (have to) save attachments from the inject page.
|
||||
|
|
@ -80,7 +80,12 @@ Zotero.Translate.ItemSaver.prototype = {
|
|||
saveItems: function (items, attachmentCallback) {
|
||||
var deferred = Zotero.Promise.defer();
|
||||
// first try to save items via connector
|
||||
var payload = {"items":items};
|
||||
var payload = { items, uri: this._baseURI };
|
||||
if (Zotero.isSafari) {
|
||||
// This is the best in terms of cookies we can do in Safari
|
||||
payload.cookie = document.cookie;
|
||||
}
|
||||
payload.proxy = this._proxy && this._proxy.toJSON();
|
||||
Zotero.Connector.setCookiesThenSaveItems(payload, function(data, status) {
|
||||
if(data !== false) {
|
||||
Zotero.debug("Translate: Save via Standalone succeeded");
|
||||
|
|
@ -179,6 +184,10 @@ Zotero.Translate.ItemSaver.prototype = {
|
|||
|
||||
for(var i=0, n=items.length; i<n; i++) {
|
||||
var item = items[i];
|
||||
// deproxify url
|
||||
if (this._proxy && item.url) {
|
||||
item.url = this._proxy.toProper(item.url);
|
||||
}
|
||||
itemIndices[i] = newItems.length;
|
||||
newItems = newItems.concat(Zotero.Utilities.itemToServerJSON(item));
|
||||
if(typedArraysSupported) {
|
||||
|
|
@ -239,7 +248,6 @@ Zotero.Translate.ItemSaver.prototype = {
|
|||
* on failure or attachmentCallback(attachment, progressPercent) periodically during saving.
|
||||
*/
|
||||
"_saveAttachmentsToServer":function(itemKey, baseName, attachments, prefs, attachmentCallback) {
|
||||
Zotero.debug("saveattachmentstoserver");
|
||||
var me = this,
|
||||
uploadAttachments = [],
|
||||
retrieveHeadersForAttachments = attachments.length;
|
||||
|
|
@ -255,6 +263,10 @@ Zotero.Translate.ItemSaver.prototype = {
|
|||
var attachmentPayload = [];
|
||||
for(var i=0; i<uploadAttachments.length; i++) {
|
||||
var attachment = uploadAttachments[i];
|
||||
// deproxify url
|
||||
if (this._proxy && attachment.url) {
|
||||
attachment.url = this._proxy.toProper(attachment.url);
|
||||
}
|
||||
attachmentPayload.push({
|
||||
"itemType":"attachment",
|
||||
"parentItem":itemKey,
|
||||
|
|
|
|||
|
|
@ -146,10 +146,10 @@ Zotero.Translators = new function() {
|
|||
if(!_initialized) Zotero.Translators.init();
|
||||
var allTranslators = _cache["web"];
|
||||
var potentialTranslators = [];
|
||||
var converterFunctions = [];
|
||||
var proxies = [];
|
||||
|
||||
var rootSearchURIs = this.getSearchURIs(rootURI);
|
||||
var frameSearchURIs = isFrame ? this.getSearchURIs(URI) : rootSearchURIs;
|
||||
var rootSearchURIs = Zotero.Proxies.getPotentialProxies(rootURI);
|
||||
var frameSearchURIs = isFrame ? Zotero.Proxies.getPotentialProxies(URI) : rootSearchURIs;
|
||||
|
||||
Zotero.debug("Translators: Looking for translators for "+Object.keys(frameSearchURIs).join(', '));
|
||||
|
||||
|
|
@ -174,14 +174,14 @@ Zotero.Translators = new function() {
|
|||
|
||||
if (frameURIMatches) {
|
||||
potentialTranslators.push(translator);
|
||||
converterFunctions.push(frameSearchURIs[frameSearchURI]);
|
||||
proxies.push(frameSearchURIs[frameSearchURI]);
|
||||
// prevent adding the translator multiple times
|
||||
break rootURIsLoop;
|
||||
}
|
||||
}
|
||||
} else if(!isFrame && (isGeneric || rootURIMatches)) {
|
||||
potentialTranslators.push(translator);
|
||||
converterFunctions.push(rootSearchURIs[rootSearchURI]);
|
||||
proxies.push(rootSearchURIs[rootSearchURI]);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
@ -189,51 +189,10 @@ Zotero.Translators = new function() {
|
|||
|
||||
var codeGetter = new Zotero.Translators.CodeGetter(potentialTranslators);
|
||||
return codeGetter.getAll().then(function () {
|
||||
return [potentialTranslators, converterFunctions];
|
||||
return [potentialTranslators, proxies];
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Get the array of searchURIs and related proxy converter functions
|
||||
*
|
||||
* @param {String} URI to get searchURIs and converterFunctions for
|
||||
*/
|
||||
this.getSearchURIs = function(URI) {
|
||||
var searchURIs = {};
|
||||
searchURIs[URI] = null;
|
||||
|
||||
// if there is a subdomain that is also a TLD, also test against URI with the domain
|
||||
// dropped after the TLD
|
||||
// (i.e., www.nature.com.mutex.gmu.edu => www.nature.com)
|
||||
var m = /^(https?:\/\/)([^\/]+)/i.exec(URI);
|
||||
if (m) {
|
||||
// First, drop the 0- if it exists (this is an III invention)
|
||||
var host = m[2];
|
||||
if(host.substr(0, 2) === "0-") host = host.substr(2);
|
||||
var hostnames = host.split(".");
|
||||
for (var i=1; i<hostnames.length-2; i++) {
|
||||
if (TLDS[hostnames[i].toLowerCase()]) {
|
||||
var properHost = hostnames.slice(0, i+1).join(".");
|
||||
var proxyHost = hostnames.slice(i+1).join(".");
|
||||
var searchURI = m[1]+properHost+URI.substr(m[0].length);
|
||||
if(Zotero.isBrowserExt || Zotero.isSafari) {
|
||||
// in Chrome/Safari, the converterFunction needs to be passed as JSON, so
|
||||
// just push an array with the proper and proxyHosts
|
||||
searchURIs[searchURI] = [properHost, proxyHost];
|
||||
} else {
|
||||
// in Firefox, add a converterFunction
|
||||
searchURIs[searchURI] = new function() {
|
||||
var re = new RegExp('^https?://(?:[^/]+\\.)?'+Zotero.Utilities.quotemeta(properHost)+'(?=/)', "gi");
|
||||
var _proxyHost = proxyHost.replace(/\$/g, "$$$$");
|
||||
return function(uri) { return uri.replace(re, "$&."+_proxyHost) };
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return searchURIs;
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* Converts translators to JSON-serializable objects
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -674,6 +674,13 @@ Zotero.DBConnection.prototype.queryAsync = Zotero.Promise.coroutine(function* (s
|
|||
Zotero.debug(msg, 1);
|
||||
throw new Error(msg);
|
||||
}
|
||||
},
|
||||
has: function(target, name) {
|
||||
try {
|
||||
return !!target.getResultByName(name);
|
||||
} catch (e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
};
|
||||
for (let i=0, len=rows.length; i<len; i++) {
|
||||
|
|
|
|||
|
|
@ -75,8 +75,8 @@ Zotero.Proxies = new function() {
|
|||
* @return {Promise<Zotero.Proxy>}
|
||||
*/
|
||||
this.newProxyFromRow = Zotero.Promise.coroutine(function* (row) {
|
||||
var proxy = new Zotero.Proxy;
|
||||
yield proxy._loadFromRow(row);
|
||||
var proxy = new Zotero.Proxy(row);
|
||||
yield proxy.loadHosts();
|
||||
return proxy;
|
||||
});
|
||||
|
||||
|
|
@ -367,6 +367,65 @@ Zotero.Proxies = new function() {
|
|||
return (onlyReturnIfProxied ? false : url);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check the url for potential proxies and deproxify, providing a scheme to build
|
||||
* a proxy object.
|
||||
*
|
||||
* @param URL
|
||||
* @returns {Object} Unproxied url to proxy object
|
||||
*/
|
||||
this.getPotentialProxies = function(URL) {
|
||||
var urlToProxy = {};
|
||||
// If it's a known proxied URL just return it
|
||||
if (Zotero.Proxies.transparent) {
|
||||
for (var proxy of Zotero.Proxies.proxies) {
|
||||
if (proxy.regexp) {
|
||||
var m = proxy.regexp.exec(URL);
|
||||
if (m) {
|
||||
let proper = proxy.toProper(m);
|
||||
urlToProxy[proper] = proxy.toJSON();
|
||||
return urlToProxy;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
urlToProxy[URL] = null;
|
||||
|
||||
// if there is a subdomain that is also a TLD, also test against URI with the domain
|
||||
// dropped after the TLD
|
||||
// (i.e., www.nature.com.mutex.gmu.edu => www.nature.com)
|
||||
var m = /^(https?:\/\/)([^\/]+)/i.exec(URL);
|
||||
if (m) {
|
||||
// First, drop the 0- if it exists (this is an III invention)
|
||||
var host = m[2];
|
||||
if (host.substr(0, 2) === "0-") host = host.substr(2);
|
||||
var hostnameParts = [host.split(".")];
|
||||
if (m[1] == 'https://' && host.replace(/-/g, '.') != host) {
|
||||
// try replacing hyphens with dots for https protocol
|
||||
// to account for EZProxy HttpsHypens mode
|
||||
hostnameParts.push(host.replace(/-/g, '.').split('.'));
|
||||
}
|
||||
|
||||
for (let i=0; i < hostnameParts.length; i++) {
|
||||
let parts = hostnameParts[i];
|
||||
// If hostnameParts has two entries, then the second one is with replaced hyphens
|
||||
let dotsToHyphens = i == 1;
|
||||
// skip the lowest level subdomain, domain and TLD
|
||||
for (let j=1; j<parts.length-2; j++) {
|
||||
// if a part matches a TLD, everything up to it is probably the true URL
|
||||
if (TLDS[parts[j].toLowerCase()]) {
|
||||
var properHost = parts.slice(0, j+1).join(".");
|
||||
// protocol + properHost + /path
|
||||
var properURL = m[1]+properHost+URL.substr(m[0].length);
|
||||
var proxyHost = parts.slice(j+1).join('.');
|
||||
urlToProxy[properURL] = {scheme: m[1] + '%h.' + proxyHost + '/%p', dotsToHyphens};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return urlToProxy;
|
||||
};
|
||||
|
||||
/**
|
||||
* Determines whether a host is blacklisted, i.e., whether we should refuse to save transparent
|
||||
* proxy entries for this host. This is necessary because EZProxy offers to proxy all Google and
|
||||
|
|
@ -521,9 +580,35 @@ Zotero.Proxies = new function() {
|
|||
* @constructor
|
||||
* @class Represents an individual proxy server
|
||||
*/
|
||||
Zotero.Proxy = function () {
|
||||
Zotero.Proxy = function (row) {
|
||||
this.hosts = [];
|
||||
this.multiHost = false;
|
||||
this._loadFromRow(row);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a proxy object from a DB row
|
||||
* @private
|
||||
*/
|
||||
Zotero.Proxy.prototype._loadFromRow = function (row) {
|
||||
this.proxyID = row.proxyID;
|
||||
this.multiHost = row.scheme && row.scheme.indexOf('%h') != -1 || !!row.multiHost;
|
||||
this.autoAssociate = !!row.autoAssociate;
|
||||
this.scheme = row.scheme;
|
||||
// Database query results will throw as this option is only present when the proxy comes along with the translator
|
||||
if ('dotsToHyphens' in row) {
|
||||
this.dotsToHyphens = !!row.dotsToHyphens;
|
||||
}
|
||||
|
||||
if (this.scheme) {
|
||||
this.compileRegexp();
|
||||
}
|
||||
};
|
||||
|
||||
Zotero.Proxy.prototype.toJSON = function() {
|
||||
if (!this.scheme) {
|
||||
throw Error('Cannot convert proxy to JSON - no scheme');
|
||||
}
|
||||
return {id: this.id, scheme: this.scheme, dotsToHyphens: this.dotsToHyphens};
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -556,7 +641,7 @@ const Zotero_Proxy_schemeParameterRegexps = {
|
|||
Zotero.Proxy.prototype.compileRegexp = function() {
|
||||
// take host only if flagged as multiHost
|
||||
var parametersToCheck = Zotero_Proxy_schemeParameters;
|
||||
if(this.multiHost) parametersToCheck["%h"] = "([a-zA-Z0-9]+\\.[a-zA-Z0-9\.]+)";
|
||||
if(this.multiHost) parametersToCheck["%h"] = "([a-zA-Z0-9]+[.\\-][a-zA-Z0-9.\\-]+)";
|
||||
|
||||
var indices = this.indices = {};
|
||||
this.parameters = [];
|
||||
|
|
@ -686,7 +771,8 @@ Zotero.Proxy.prototype.save = Zotero.Promise.coroutine(function* (transparent) {
|
|||
Zotero.Proxy.prototype.revert = Zotero.Promise.coroutine(function* () {
|
||||
if (!this.proxyID) throw new Error("Cannot revert an unsaved proxy");
|
||||
var row = yield Zotero.DB.rowQueryAsync("SELECT * FROM proxies WHERE proxyID = ?", [this.proxyID]);
|
||||
yield this._loadFromRow(row);
|
||||
this._loadFromRow(row);
|
||||
yield this.loadHosts();
|
||||
});
|
||||
|
||||
/**
|
||||
|
|
@ -706,14 +792,29 @@ Zotero.Proxy.prototype.erase = Zotero.Promise.coroutine(function* () {
|
|||
/**
|
||||
* Converts a proxied URL to an unproxied URL using this proxy
|
||||
*
|
||||
* @param m {Array} The match from running this proxy's regexp against a URL spec
|
||||
* @type String
|
||||
* @param m {String|Array} The URL or the match from running this proxy's regexp against a URL spec
|
||||
* @return {String} The unproxified URL if was proxified or the unchanged URL
|
||||
*/
|
||||
Zotero.Proxy.prototype.toProper = function(m) {
|
||||
if (!Array.isArray(m)) {
|
||||
let match = this.regexp.exec(m);
|
||||
if (!match) {
|
||||
return m
|
||||
} else {
|
||||
m = match;
|
||||
}
|
||||
}
|
||||
let scheme = this.scheme.indexOf('https') == -1 ? 'http://' : 'https://';
|
||||
if(this.multiHost) {
|
||||
var properURL = "http://"+m[this.parameters.indexOf("%h")+1]+"/";
|
||||
var properURL = scheme+m[this.parameters.indexOf("%h")+1]+"/";
|
||||
} else {
|
||||
var properURL = "http://"+this.hosts[0]+"/";
|
||||
var properURL = scheme+this.hosts[0]+"/";
|
||||
}
|
||||
|
||||
// Replace `-` with `.` in https to support EZProxy HttpsHyphens.
|
||||
// Potentially troublesome with domains that contain dashes
|
||||
if (this.dotsToHyphens) {
|
||||
properURL = properURL.replace(/-/g, '.');
|
||||
}
|
||||
|
||||
if(this.indices["%p"]) {
|
||||
|
|
@ -731,17 +832,23 @@ Zotero.Proxy.prototype.toProper = function(m) {
|
|||
/**
|
||||
* Converts an unproxied URL to a proxied URL using this proxy
|
||||
*
|
||||
* @param {nsIURI} uri The nsIURI corresponding to the unproxied URL
|
||||
* @type String
|
||||
* @param {String|nsIURI} uri The URL as a string or the nsIURI corresponding to the unproxied URL
|
||||
* @return {String} The proxified URL if was unproxified or the unchanged url
|
||||
*/
|
||||
Zotero.Proxy.prototype.toProxy = function(uri) {
|
||||
if (typeof uri == "string") {
|
||||
uri = Services.io.newURI(uri, null, null);
|
||||
}
|
||||
if (this.regexp.exec(uri.spec)) {
|
||||
return uri.spec;
|
||||
}
|
||||
var proxyURL = this.scheme;
|
||||
|
||||
for(var i=this.parameters.length-1; i>=0; i--) {
|
||||
var param = this.parameters[i];
|
||||
var value = "";
|
||||
if(param == "%h") {
|
||||
value = uri.hostPort;
|
||||
value = this.dotsToHyphens ? uri.hostPort.replace(/-/g, '.') : uri.hostPort;
|
||||
} else if(param == "%p") {
|
||||
value = uri.path.substr(1);
|
||||
} else if(param == "%d") {
|
||||
|
|
@ -756,19 +863,13 @@ Zotero.Proxy.prototype.toProxy = function(uri) {
|
|||
return proxyURL;
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a proxy object from a DB row
|
||||
* @private
|
||||
*/
|
||||
Zotero.Proxy.prototype._loadFromRow = Zotero.Promise.coroutine(function* (row) {
|
||||
this.proxyID = row.proxyID;
|
||||
this.multiHost = !!row.multiHost;
|
||||
this.autoAssociate = !!row.autoAssociate;
|
||||
this.scheme = row.scheme;
|
||||
Zotero.Proxy.prototype.loadHosts = Zotero.Promise.coroutine(function* () {
|
||||
if (!this.proxyID) {
|
||||
throw Error("Cannot load hosts without a proxyID")
|
||||
}
|
||||
this.hosts = yield Zotero.DB.columnQueryAsync(
|
||||
"SELECT hostname FROM proxyHosts WHERE proxyID = ? ORDER BY hostname", row.proxyID
|
||||
"SELECT hostname FROM proxyHosts WHERE proxyID = ? ORDER BY hostname", this.proxyID
|
||||
);
|
||||
this.compileRegexp();
|
||||
});
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -177,17 +177,18 @@ Zotero.Server.Connector.Detect.prototype = {
|
|||
},
|
||||
|
||||
/**
|
||||
* Callback to be executed when list of translators becomes available. Sends response with
|
||||
* item types, translator IDs, labels, and icons for available translators.
|
||||
* Callback to be executed when list of translators becomes available. Sends standard
|
||||
* translator passing properties with proxies where available for translators.
|
||||
* @param {Zotero.Translate} translate
|
||||
* @param {Zotero.Translator[]} translators
|
||||
*/
|
||||
_translatorsAvailable: function(obj, translators) {
|
||||
var jsons = [];
|
||||
for (let translator of translators) {
|
||||
jsons.push(translator.serialize(TRANSLATOR_PASSING_PROPERTIES));
|
||||
}
|
||||
this.sendResponse(200, "application/json", JSON.stringify(jsons));
|
||||
_translatorsAvailable: function(translate, translators) {
|
||||
translators = translators.map(function(translator) {
|
||||
translator = translator.serialize(TRANSLATOR_PASSING_PROPERTIES.concat('proxy'));
|
||||
translator.proxy = translator.proxy ? translator.proxy.toJSON() : null;
|
||||
return translator;
|
||||
});
|
||||
this.sendResponse(200, "application/json", JSON.stringify(translators));
|
||||
|
||||
Zotero.Browser.deleteHiddenBrowser(this._browser);
|
||||
}
|
||||
|
|
@ -371,13 +372,15 @@ Zotero.Server.Connector.SaveItem.prototype = {
|
|||
Zotero.Server.Connector.AttachmentProgressManager.add(data.items[i].attachments);
|
||||
}
|
||||
|
||||
let proxy = data.proxy && new Zotero.Proxy(data.proxy);
|
||||
// save items
|
||||
var itemSaver = new Zotero.Translate.ItemSaver({
|
||||
libraryID,
|
||||
collections: collection ? [collection.id] : undefined,
|
||||
attachmentMode: Zotero.Translate.ItemSaver.ATTACHMENT_MODE_DOWNLOAD,
|
||||
forceTagType: 1,
|
||||
cookieSandbox
|
||||
cookieSandbox,
|
||||
proxy
|
||||
});
|
||||
try {
|
||||
let items = yield itemSaver.saveItems(
|
||||
|
|
|
|||
|
|
@ -1120,18 +1120,21 @@ Zotero.Translate.Base.prototype = {
|
|||
// if detection returns immediately, return found translators
|
||||
return potentialTranslators.then(function(result) {
|
||||
var allPotentialTranslators = result[0];
|
||||
var properToProxyFunctions = result[1];
|
||||
var proxies = result[1];
|
||||
|
||||
// this gets passed out by Zotero.Translators.getWebTranslatorsForLocation() because it is
|
||||
// specific for each translator, but we want to avoid making a copy of a translator whenever
|
||||
// possible.
|
||||
this._properToProxyFunctions = properToProxyFunctions ? properToProxyFunctions : null;
|
||||
this._proxies = proxies ? [] : null;
|
||||
this._waitingForRPC = false;
|
||||
|
||||
for(var i=0, n=allPotentialTranslators.length; i<n; i++) {
|
||||
var translator = allPotentialTranslators[i];
|
||||
if(translator.runMode === Zotero.Translator.RUN_MODE_IN_BROWSER) {
|
||||
this._potentialTranslators.push(translator);
|
||||
if (proxies) {
|
||||
this._proxies.push(proxies[i]);
|
||||
}
|
||||
} else if (this instanceof Zotero.Translate.Web && Zotero.Connector) {
|
||||
this._waitingForRPC = true;
|
||||
}
|
||||
|
|
@ -1166,6 +1169,7 @@ Zotero.Translate.Base.prototype = {
|
|||
for(var i=0, n=rpcTranslators.length; i<n; i++) {
|
||||
rpcTranslators[i] = new Zotero.Translator(rpcTranslators[i]);
|
||||
rpcTranslators[i].runMode = Zotero.Translator.RUN_MODE_ZOTERO_STANDALONE;
|
||||
rpcTranslators[i].proxy = rpcTranslators[i].proxy ? new Zotero.Proxy(rpcTranslators[i].proxy) : null;
|
||||
}
|
||||
this._foundTranslators = this._foundTranslators.concat(rpcTranslators);
|
||||
}
|
||||
|
|
@ -1378,8 +1382,8 @@ Zotero.Translate.Base.prototype = {
|
|||
|
||||
// convert proxy to proper if applicable
|
||||
if(!dontUseProxy && this.translator && this.translator[0]
|
||||
&& this.translator[0].properToProxy) {
|
||||
var proxiedURL = this.translator[0].properToProxy(resolved);
|
||||
&& this._proxy) {
|
||||
var proxiedURL = this._proxy.toProxy(resolved);
|
||||
if (proxiedURL != resolved) {
|
||||
Zotero.debug("Translate: proxified to " + proxiedURL);
|
||||
}
|
||||
|
|
@ -1440,13 +1444,13 @@ Zotero.Translate.Base.prototype = {
|
|||
if(this._currentState === "detect") {
|
||||
if(this._potentialTranslators.length) {
|
||||
var lastTranslator = this._potentialTranslators.shift();
|
||||
var lastProperToProxyFunction = this._properToProxyFunctions ? this._properToProxyFunctions.shift() : null;
|
||||
var lastProxy = this._proxies ? this._proxies.shift() : null;
|
||||
|
||||
if(returnValue) {
|
||||
var dupeTranslator = {"properToProxy":lastProperToProxyFunction};
|
||||
if (returnValue) {
|
||||
var dupeTranslator = {proxy: lastProxy ? new Zotero.Proxy(lastProxy) : null};
|
||||
|
||||
for(var i in lastTranslator) dupeTranslator[i] = lastTranslator[i];
|
||||
if(Zotero.isBookmarklet && returnValue === "server") {
|
||||
for (var i in lastTranslator) dupeTranslator[i] = lastTranslator[i];
|
||||
if (Zotero.isBookmarklet && returnValue === "server") {
|
||||
// In the bookmarklet, the return value from detectWeb can be "server" to
|
||||
// indicate the translator should be run on the Zotero server
|
||||
dupeTranslator.runMode = Zotero.Translator.RUN_MODE_ZOTERO_SERVER;
|
||||
|
|
@ -1689,6 +1693,13 @@ Zotero.Translate.Base.prototype = {
|
|||
}
|
||||
|
||||
this._currentTranslator = translator;
|
||||
|
||||
// Pass on the proxy of the parent translate
|
||||
if (this._parentTranslator) {
|
||||
this._proxy = this._parentTranslator._proxy;
|
||||
} else {
|
||||
this._proxy = translator.proxy;
|
||||
}
|
||||
this._runningAsyncProcesses = 0;
|
||||
this._returnValue = undefined;
|
||||
this._aborted = false;
|
||||
|
|
@ -1950,12 +1961,13 @@ Zotero.Translate.Web.prototype._getParameters = function() {
|
|||
*/
|
||||
Zotero.Translate.Web.prototype._prepareTranslation = Zotero.Promise.method(function () {
|
||||
this._itemSaver = new Zotero.Translate.ItemSaver({
|
||||
"libraryID":this._libraryID,
|
||||
"collections": this._collections,
|
||||
"attachmentMode":Zotero.Translate.ItemSaver[(this._saveAttachments ? "ATTACHMENT_MODE_DOWNLOAD" : "ATTACHMENT_MODE_IGNORE")],
|
||||
"forceTagType":1,
|
||||
"cookieSandbox":this._cookieSandbox,
|
||||
"baseURI":this.location
|
||||
libraryID: this._libraryID,
|
||||
collections: this._collections,
|
||||
attachmentMode: Zotero.Translate.ItemSaver[(this._saveAttachments ? "ATTACHMENT_MODE_DOWNLOAD" : "ATTACHMENT_MODE_IGNORE")],
|
||||
forceTagType: 1,
|
||||
cookieSandbox: this._cookieSandbox,
|
||||
proxy: this._proxy,
|
||||
baseURI: this.location
|
||||
});
|
||||
this.newItems = [];
|
||||
});
|
||||
|
|
@ -1987,11 +1999,12 @@ Zotero.Translate.Web.prototype._translateTranslatorLoaded = function() {
|
|||
(runMode === Zotero.Translator.RUN_MODE_ZOTERO_SERVER && Zotero.Connector.isOnline)) {
|
||||
var me = this;
|
||||
Zotero.Connector.callMethod("savePage", {
|
||||
"uri":this.location.toString(),
|
||||
"translatorID":(typeof this.translator[0] === "object"
|
||||
uri: this.location.toString(),
|
||||
translatorID: (typeof this.translator[0] === "object"
|
||||
? this.translator[0].translatorID : this.translator[0]),
|
||||
"cookie":this.document.cookie,
|
||||
"html":this.document.documentElement.innerHTML
|
||||
cookie: this.document.cookie,
|
||||
proxy: this._proxy ? this._proxy.toJSON() : null,
|
||||
html: this.document.documentElement.innerHTML
|
||||
}, function(obj) { me._translateRPCComplete(obj) });
|
||||
} else if(runMode === Zotero.Translator.RUN_MODE_ZOTERO_SERVER) {
|
||||
var me = this;
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@
|
|||
* <li>attachmentMode - One of Zotero.Translate.ItemSaver.ATTACHMENT_* specifying how attachments should be saved</li>
|
||||
* <li>forceTagType - Force tags to specified tag type</li>
|
||||
* <li>cookieSandbox - Cookie sandbox for attachment requests</li>
|
||||
* <li>proxy - A proxy to deproxify item URLs</li>
|
||||
* <li>baseURI - URI to which attachment paths should be relative</li>
|
||||
*/
|
||||
Zotero.Translate.ItemSaver = function(options) {
|
||||
|
|
@ -53,6 +54,7 @@ Zotero.Translate.ItemSaver = function(options) {
|
|||
Zotero.Translate.ItemSaver.ATTACHMENT_MODE_IGNORE;
|
||||
this._forceTagType = options.forceTagType;
|
||||
this._cookieSandbox = options.cookieSandbox;
|
||||
this._proxy = options.proxy;
|
||||
|
||||
// the URI to which other URIs are assumed to be relative
|
||||
if(typeof baseURI === "object" && baseURI instanceof Components.interfaces.nsIURI) {
|
||||
|
|
@ -109,6 +111,13 @@ Zotero.Translate.ItemSaver.prototype = {
|
|||
};
|
||||
newItem.fromJSON(this._deleteIrrelevantFields(item));
|
||||
|
||||
// deproxify url
|
||||
if (this._proxy && item.url) {
|
||||
let url = this._proxy.toProper(item.url);
|
||||
Zotero.debug(`Deproxifying item url ${item.url} with scheme ${this._proxy.scheme} to ${url}`, 5);
|
||||
newItem.setField('url', url);
|
||||
}
|
||||
|
||||
if (this._collections) {
|
||||
newItem.setCollections(this._collections);
|
||||
}
|
||||
|
|
@ -253,6 +262,12 @@ Zotero.Translate.ItemSaver.prototype = {
|
|||
}
|
||||
|
||||
if (!newAttachment) return false; // attachmentCallback should not have been called in this case
|
||||
|
||||
// deproxify url
|
||||
let url = newAttachment.getField('url');
|
||||
if (this._proxy && url) {
|
||||
newAttachment.setField('url', this._proxy.toProper(url));
|
||||
}
|
||||
|
||||
// save fields
|
||||
if (attachment.accessDate) newAttachment.setField("accessDate", attachment.accessDate);
|
||||
|
|
|
|||
|
|
@ -23,6 +23,9 @@
|
|||
***** END LICENSE BLOCK *****
|
||||
*/
|
||||
|
||||
// Enumeration of types of translators
|
||||
var TRANSLATOR_TYPES = {"import":1, "export":2, "web":4, "search":8};
|
||||
|
||||
// Properties required for every translator
|
||||
var TRANSLATOR_REQUIRED_PROPERTIES = ["translatorID", "translatorType", "label", "creator",
|
||||
"target", "priority", "lastUpdated"];
|
||||
|
|
|
|||
|
|
@ -25,9 +25,6 @@
|
|||
|
||||
"use strict";
|
||||
|
||||
// Enumeration of types of translators
|
||||
var TRANSLATOR_TYPES = {"import":1, "export":2, "web":4, "search":8};
|
||||
|
||||
/**
|
||||
* Singleton to handle loading and caching of translators
|
||||
* @namespace
|
||||
|
|
@ -297,10 +294,10 @@ Zotero.Translators = new function() {
|
|||
|
||||
return this.getAllForType(type).then(function(allTranslators) {
|
||||
var potentialTranslators = [];
|
||||
var converterFunctions = [];
|
||||
var proxies = [];
|
||||
|
||||
var rootSearchURIs = this.getSearchURIs(rootURI);
|
||||
var frameSearchURIs = isFrame ? this.getSearchURIs(URI) : rootSearchURIs;
|
||||
var rootSearchURIs = Zotero.Proxies.getPotentialProxies(rootURI);
|
||||
var frameSearchURIs = isFrame ? Zotero.Proxies.getPotentialProxies(URI) : rootSearchURIs;
|
||||
|
||||
Zotero.debug("Translators: Looking for translators for "+Object.keys(frameSearchURIs).join(', '));
|
||||
|
||||
|
|
@ -316,7 +313,7 @@ Zotero.Translators = new function() {
|
|||
|
||||
if (frameURIMatches) {
|
||||
potentialTranslators.push(translator);
|
||||
converterFunctions.push(frameSearchURIs[frameSearchURI]);
|
||||
proxies.push(frameSearchURIs[frameSearchURI]);
|
||||
// prevent adding the translator multiple times
|
||||
break rootURIsLoop;
|
||||
}
|
||||
|
|
@ -324,13 +321,13 @@ Zotero.Translators = new function() {
|
|||
}
|
||||
else if(!isFrame && (isGeneric || rootURIMatches)) {
|
||||
potentialTranslators.push(translator);
|
||||
converterFunctions.push(rootSearchURIs[rootSearchURI]);
|
||||
proxies.push(rootSearchURIs[rootSearchURI]);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return [potentialTranslators, converterFunctions];
|
||||
return [potentialTranslators, proxies];
|
||||
}.bind(this));
|
||||
},
|
||||
|
||||
|
|
|
|||
|
|
@ -253,8 +253,8 @@ Zotero.Utilities.Translate.prototype.processDocuments = function(urls, processor
|
|||
}
|
||||
|
||||
for(var i=0; i<urls.length; i++) {
|
||||
if(this._translate.document && this._translate.document.location
|
||||
&& this._translate.document.location.toString() === urls[i]) {
|
||||
if(translate.document && translate.document.location
|
||||
&& translate.document.location.toString() === urls[i]) {
|
||||
// Document is attempting to reload itself
|
||||
Zotero.debug("Translate: Attempted to load the current document using processDocuments; using loaded document instead");
|
||||
// This fixes document permissions issues in translation-server when translators call
|
||||
|
|
@ -374,6 +374,18 @@ Zotero.Utilities.Translate.prototype.doPost = function(url, body, onDone, header
|
|||
}, headers, responseCharset, translate.cookieSandbox ? translate.cookieSandbox : undefined);
|
||||
}
|
||||
|
||||
Zotero.Utilities.Translate.prototype.urlToProxy = function(url) {
|
||||
var proxy = this._translate._proxy;
|
||||
if (proxy) return proxy.toProxy(url);
|
||||
return url;
|
||||
};
|
||||
|
||||
Zotero.Utilities.Translate.prototype.urlToProper = function(url) {
|
||||
var proxy = this._translate._proxy;
|
||||
if (proxy) return proxy.toProper(url);
|
||||
return url;
|
||||
};
|
||||
|
||||
Zotero.Utilities.Translate.prototype.__exposedProps__ = {"HTTP":"r"};
|
||||
for(var j in Zotero.Utilities.Translate.prototype) {
|
||||
if(typeof Zotero.Utilities.Translate.prototype[j] === "function" && j[0] !== "_" && j != "Translate") {
|
||||
|
|
|
|||
|
|
@ -44,6 +44,7 @@ const xpcomFilesAll = [
|
|||
'ipc',
|
||||
'profile',
|
||||
'progressWindow',
|
||||
'proxy',
|
||||
'translation/translate',
|
||||
'translation/translate_firefox',
|
||||
'translation/translator',
|
||||
|
|
@ -98,7 +99,6 @@ const xpcomFilesLocal = [
|
|||
'locateManager',
|
||||
'mime',
|
||||
'notifier',
|
||||
'proxy',
|
||||
'quickCopy',
|
||||
'report',
|
||||
'router',
|
||||
|
|
|
|||
|
|
@ -788,7 +788,7 @@ function buildDummyTranslator(translatorType, code, info={}) {
|
|||
"lastUpdated":"0000-00-00 00:00:00",
|
||||
}, info);
|
||||
let translator = new Zotero.Translator(info);
|
||||
translator.code = code;
|
||||
translator.code = JSON.stringify(info) + "\n" + code;
|
||||
return translator;
|
||||
}
|
||||
|
||||
|
|
|
|||
31
test/tests/proxyTest.js
Normal file
31
test/tests/proxyTest.js
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
"use strict";
|
||||
|
||||
describe("Zotero.Proxies", function(){
|
||||
describe("#getPotentialProxies", function() {
|
||||
it("should return the provided url mapped to null when url is not proxied", function() {
|
||||
let url = "http://www.example.com";
|
||||
let proxies = Zotero.Proxies.getPotentialProxies(url);
|
||||
let expectedProxies = {};
|
||||
expectedProxies[url] = null;
|
||||
assert.deepEqual(proxies, expectedProxies);
|
||||
});
|
||||
|
||||
it("should return the provided url and deproxied url", function() {
|
||||
let url = "https://www.example.com.proxy.example.com";
|
||||
let proxies = Zotero.Proxies.getPotentialProxies(url);
|
||||
let expectedProxies = {};
|
||||
expectedProxies[url] = null;
|
||||
expectedProxies["https://www.example.com"] = {scheme: "https://%h.proxy.example.com/%p", dotsToHyphens: false};
|
||||
assert.deepEqual(proxies, expectedProxies);
|
||||
});
|
||||
|
||||
it("should return the provided url and deproxied url with replaced hyphens", function() {
|
||||
let url = "https://www-example-com.proxy.example.com";
|
||||
let proxies = Zotero.Proxies.getPotentialProxies(url);
|
||||
let expectedProxies = {};
|
||||
expectedProxies[url] = null;
|
||||
expectedProxies["https://www.example.com"] = {scheme: "https://%h.proxy.example.com/%p", dotsToHyphens: true};
|
||||
assert.deepEqual(proxies, expectedProxies);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -57,13 +57,41 @@ describe("Connector Server", function () {
|
|||
);
|
||||
|
||||
assert.isTrue(Zotero.Translators.get.calledWith('dummy-translator'));
|
||||
assert.equal(response.response, code);
|
||||
let translatorCode = yield translator.getCode();
|
||||
assert.equal(response.response, translatorCode);
|
||||
|
||||
Zotero.Translators.get.restore();
|
||||
})
|
||||
});
|
||||
|
||||
|
||||
describe("/connector/detect", function() {
|
||||
it("should return relevant translators with proxies", function* () {
|
||||
var code = 'function detectWeb() {return "newspaperArticle";}\nfunction doWeb() {}';
|
||||
var translator = buildDummyTranslator("web", code, {target: "https://www.example.com/.*"});
|
||||
sinon.stub(Zotero.Translators, 'getAllForType').resolves([translator]);
|
||||
|
||||
var response = yield Zotero.HTTP.request(
|
||||
'POST',
|
||||
connectorServerPath + "/connector/detect",
|
||||
{
|
||||
headers: {
|
||||
"Content-Type": "application/json"
|
||||
},
|
||||
body: JSON.stringify({
|
||||
uri: "https://www-example-com.proxy.example.com/article",
|
||||
html: "<head><title>Owl</title></head><body><p>🦉</p></body>"
|
||||
})
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(JSON.parse(response.response)[0].proxy.scheme, 'https://%h.proxy.example.com/%p');
|
||||
|
||||
Zotero.Translators.getAllForType.restore();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
describe("/connector/saveItems", function () {
|
||||
// TODO: Test cookies
|
||||
it("should save a translated item to the current selected collection", function* () {
|
||||
|
|
@ -185,6 +213,49 @@ describe("Connector Server", function () {
|
|||
win.ZoteroPane.collectionsView.getSelectedLibraryID(), Zotero.Libraries.userLibraryID
|
||||
);
|
||||
});
|
||||
|
||||
it("should use the provided proxy to deproxify item url", function* () {
|
||||
yield selectLibrary(win, Zotero.Libraries.userLibraryID);
|
||||
yield waitForItemsLoad(win);
|
||||
|
||||
var body = {
|
||||
items: [
|
||||
{
|
||||
itemType: "newspaperArticle",
|
||||
title: "Title",
|
||||
creators: [
|
||||
{
|
||||
firstName: "First",
|
||||
lastName: "Last",
|
||||
creatorType: "author"
|
||||
}
|
||||
],
|
||||
attachments: [],
|
||||
url: "https://www-example-com.proxy.example.com/path"
|
||||
}
|
||||
],
|
||||
uri: "https://www-example-com.proxy.example.com/path",
|
||||
proxy: {scheme: 'https://%h.proxy.example.com/%p', dotsToHyphens: true}
|
||||
};
|
||||
|
||||
var promise = waitForItemEvent('add');
|
||||
var req = yield Zotero.HTTP.request(
|
||||
'POST',
|
||||
connectorServerPath + "/connector/saveItems",
|
||||
{
|
||||
headers: {
|
||||
"Content-Type": "application/json"
|
||||
},
|
||||
body: JSON.stringify(body)
|
||||
}
|
||||
);
|
||||
|
||||
// Check item
|
||||
var ids = yield promise;
|
||||
assert.lengthOf(ids, 1);
|
||||
var item = Zotero.Items.get(ids[0]);
|
||||
assert.equal(item.getField('url'), 'https://www.example.com/path');
|
||||
});
|
||||
});
|
||||
|
||||
describe("/connector/saveSnapshot", function () {
|
||||
|
|
|
|||
|
|
@ -680,6 +680,49 @@ describe("Zotero.Translate", function() {
|
|||
assert.isNumber(translation.newItems[0].id);
|
||||
assert.ok(collection.hasItem(translation.newItems[0].id));
|
||||
});
|
||||
|
||||
});
|
||||
describe('#saveItems', function() {
|
||||
it("should deproxify item and attachment urls when proxy provided", function* (){
|
||||
var itemID;
|
||||
var item = loadSampleData('journalArticle');
|
||||
item = item.journalArticle;
|
||||
item.url = 'https://www-example-com.proxy.example.com/';
|
||||
item.attachments = [{
|
||||
url: 'https://www-example-com.proxy.example.com/pdf.pdf',
|
||||
mimeType: 'application/pdf',
|
||||
title: 'Example PDF'}];
|
||||
var itemSaver = new Zotero.Translate.ItemSaver({
|
||||
libraryID: Zotero.Libraries.userLibraryID,
|
||||
attachmentMode: Zotero.Translate.ItemSaver.ATTACHMENT_MODE_FILE,
|
||||
proxy: new Zotero.Proxy({scheme: 'https://%h.proxy.example.com/%p', dotsToHyphens: true})
|
||||
});
|
||||
var itemDeferred = Zotero.Promise.defer();
|
||||
var attachmentDeferred = Zotero.Promise.defer();
|
||||
itemSaver.saveItems([item], Zotero.Promise.coroutine(function* (attachment, progressPercentage) {
|
||||
// ItemSaver returns immediately without waiting for attachments, so we use the callback
|
||||
// to test attachments
|
||||
if (progressPercentage != 100) return;
|
||||
try {
|
||||
yield itemDeferred.promise;
|
||||
let item = Zotero.Items.get(itemID);
|
||||
attachment = Zotero.Items.get(item.getAttachments()[0]);
|
||||
assert.equal(attachment.getField('url'), 'https://www.example.com/pdf.pdf');
|
||||
attachmentDeferred.resolve();
|
||||
} catch (e) {
|
||||
attachmentDeferred.reject(e);
|
||||
}
|
||||
})).then(function(items) {
|
||||
try {
|
||||
assert.equal(items[0].getField('url'), 'https://www.example.com/');
|
||||
itemID = items[0].id;
|
||||
itemDeferred.resolve();
|
||||
} catch (e) {
|
||||
itemDeferred.reject(e);
|
||||
}
|
||||
});
|
||||
yield Zotero.Promise.all([itemDeferred.promise, attachmentDeferred.promise]);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in a new issue