Add ability for Scaffold to provide alternative translators

Zotero.Translate::setTranslatorProviderMethods(methods) can be used to
provide custom 'get' and 'getAllForType' methods that override the
default Zotero.Translators methods.

chrome/content/zotero/tools/testTranslators/translatorTester.js

@@ -193,13 +193,15 @@ var Zotero_TranslatorTesters = new function() {
  * @param {String} type The type of tests to run (web, import, export, or search)
  * @param {Function} [debugCallback] A function to call to write debug output. If not present,
  *     Zotero.debug will be used.
+ * @param {Object} [translatorProvider] Used by Scaffold to override Zotero.Translators
  */
-var Zotero_TranslatorTester = function(translator, type, debugCallback) {
+var Zotero_TranslatorTester = function(translator, type, debugCallback, translatorProvider) {
 	this.type = type;
 	this.translator = translator;
 	this.output = "";
 	this.isSupported = this.translator.runMode === Zotero.Translator.RUN_MODE_IN_BROWSER;
 	this.translator.runMode = Zotero.Translator.RUN_MODE_IN_BROWSER;
+	this.translatorProvider = translatorProvider;
 	
 	this.tests = [];
 	this.pending = [];
@@ -441,7 +443,9 @@ Zotero_TranslatorTester.prototype.runTest = function(test, doc, testDoneCallback
 	
 	var me = this;
 	var translate = Zotero.Translate.newInstance(this.type);
-	
+	if (this.translatorProvider) {
+		translate.setTranslatorProvider(this.translatorProvider);
+	}
 	if(this.type === "web") {
 		translate.setDocument(doc);
 	} else if(this.type === "import") {
@@ -602,6 +606,9 @@ Zotero_TranslatorTester.prototype.newTest = function(doc, testReadyCallback) {
 	
 	var me = this;
 	var translate = Zotero.Translate.newInstance(this.type);
+	if (this.translatorProvider) {
+		translate.setTranslatorProvider(this.translatorProvider);
+	}
 	translate.setDocument(doc);
 	translate.setTranslator(this.translator);
 	translate.setHandler("debug", this._debug);

chrome/content/zotero/xpcom/file.js

@@ -583,13 +583,14 @@ Zotero.File = new function(){
 	
 	/**
 	 * Run a generator with an OS.File.DirectoryIterator, closing the
-	 * iterator when done
+	 * iterator when done. Promises yielded by the generator are awaited.
 	 *
 	 * The DirectoryIterator is passed as the first parameter to the generator.
 	 *
 	 * Zotero.File.iterateDirectory(path, function* (iterator) {
 	 *    while (true) {
-	 *        var entry = yield iterator.next();
+	 *        let entry = yield iterator.next();
+	 *        let contents = yield Zotero.File.getContentsAsync(entry.path);
 	 *        [...]
 	 *    }
 	 * })

chrome/content/zotero/xpcom/translation/translate.js

@@ -330,6 +330,7 @@ Zotero.Translate.Sandbox = {
 			Zotero.debug("Translate: Creating translate instance of type "+type+" in sandbox");
 			var translation = Zotero.Translate.newInstance(type);
 			translation._parentTranslator = translate;
+			translation.setTranslatorProvider(translate._translatorProvider);
 			
 			if(translation instanceof Zotero.Translate.Export && !(translation instanceof Zotero.Translate.Export)) {
 				throw(new Error("Only export translators may call other export translators"));
@@ -435,7 +436,9 @@ Zotero.Translate.Sandbox = {
 				}
 				
 				var translator = translation.translator[0];
-				translator = typeof translator === "object" ? translator : Zotero.Translators.get(translator);
+				translator = typeof translator === "object"
+					? translator
+					: translation._translatorProvider.get(translator);
 				// Zotero.Translators.get returns a value in the client and a promise in connectors
 				// so we normalize the value to a promise here
 				Zotero.Promise.resolve(translator)
@@ -948,6 +951,7 @@ Zotero.Translate.Base.prototype = {
 		this._handlers = [];
 		this._currentState = null;
 		this._translatorInfo = null;
+		this._translatorProvider = Zotero.Translators;
 		this.document = null;
 		this.location = null;
 	},
@@ -1071,6 +1075,17 @@ Zotero.Translate.Base.prototype = {
 		if(handlerIndex !== -1) this._handlers[type].splice(handlerIndex, 1);
 	},
 	
+	/**
+	 * Set custom translator provider, as returned by Zotero.Translators.makeTranslatorProvider()
+	 *
+	 * Used by Scaffold to substitute external translator files
+	 *
+	 * @param {Object} translatorProvider
+	 */
+	setTranslatorProvider: function (translatorProvider) {
+		this._translatorProvider = translatorProvider;
+	},
+	
 	/**
 	 * Indicates that a new async process is running
 	 */
@@ -1177,7 +1192,7 @@ Zotero.Translate.Base.prototype = {
 			var t;
 			for(var i=0, n=this.translator.length; i<n; i++) {
 				if(typeof(this.translator[i]) == 'string') {
-					t = Zotero.Translators.get(this.translator[i]);
+					t = this._translatorProvider.get(this.translator[i]);
 					if(!t) Zotero.debug("getTranslators: could not retrieve translator '" + this.translator[i] + "'");
 				} else {
 					t = this.translator[i];
@@ -1268,9 +1283,9 @@ Zotero.Translate.Base.prototype = {
 	 * Get all potential translators (without running detect)
 	 * @return {Promise} Promise for an array of {@link Zotero.Translator} objects
 	 */
-	 "_getTranslatorsGetPotentialTranslators":function() {
+	 _getTranslatorsGetPotentialTranslators: async function () {
-		return Zotero.Translators.getAllForType(this.type).
+		var translators = await this._translatorProvider.getAllForType(this.type);
-		then(function(translators) { return [translators] });
+		return [translators];
 	 },
 
 	/**
@@ -1347,7 +1362,7 @@ Zotero.Translate.Base.prototype = {
 		
 		// need to get translator first
 		if (typeof this.translator[0] !== "object") {
-			this.translator[0] = Zotero.Translators.get(this.translator[0]);
+			this.translator[0] = this._translatorProvider.get(this.translator[0]);
 		}
 		
 		// Zotero.Translators.get() returns a promise in the connectors, but we don't expect it to
@@ -2095,7 +2110,7 @@ Zotero.Translate.Web.prototype.setLocation = function(location, rootLocation) {
  * Get potential web translators
  */
 Zotero.Translate.Web.prototype._getTranslatorsGetPotentialTranslators = function() {
-	return Zotero.Translators.getWebTranslatorsForLocation(this.location, this.rootLocation);
+	return this._translatorProvider.getWebTranslatorsForLocation(this.location, this.rootLocation);
 }
 
 /**
@@ -2357,11 +2372,11 @@ Zotero.Translate.Import.prototype.complete = function(returnValue, error) {
 /**
  * Get all potential import translators, ordering translators with the right file extension first
  */
-Zotero.Translate.Import.prototype._getTranslatorsGetPotentialTranslators = function() {
+Zotero.Translate.Import.prototype._getTranslatorsGetPotentialTranslators = async function () {
-	return (this.location ?
+	var translators = await (this.location
-	        Zotero.Translators.getImportTranslatorsForLocation(this.location) :
+		? this._translatorProvider.getImportTranslatorsForLocation(this.location)
-	        Zotero.Translators.getAllForType(this.type)).
+		: this._translatorProvider.getAllForType(this.type));
-	then(function(translators) { return [translators] });;
+	return [translators];
 }
 
 /**
@@ -2373,7 +2388,7 @@ Zotero.Translate.Import.prototype.getTranslators = function() {
 		if(this._currentState === "detect") throw new Error("getTranslators: detection is already running");
 		this._currentState = "detect";
 		var me = this;
-		return Zotero.Translators.getAllForType(this.type).
+		return this._translatorProvider.getAllForType(this.type).
 		then(function(translators) {
 			me._potentialTranslators = [];
 			me._foundTranslators = translators;
@@ -2538,7 +2553,7 @@ Zotero.Translate.Export.prototype.getTranslators = function() {
 		return Zotero.Promise.reject(new Error("getTranslators: detection is already running"));
 	}
 	var me = this;
-	return Zotero.Translators.getAllForType(this.type).then(function(translators) {
+	return this._translatorProvider.getAllForType(this.type).then(function(translators) {
 		me._currentState = "detect";
 		me._foundTranslators = translators;
 		me._potentialTranslators = [];

chrome/content/zotero/xpcom/translation/translator.js

@@ -107,7 +107,7 @@ Zotero.Translator.prototype.init = function(info) {
 		delete this.importRegexp;
 	}
 	
-	this.cacheCode = Zotero.isConnector;
+	this.cacheCode = Zotero.isConnector || info.cacheCode;
 	if (this.translatorType & TRANSLATOR_TYPES["web"]) {
 		// compile web regexp
 		this.cacheCode |= !this.target;

chrome/content/zotero/xpcom/translation/translators.js

@@ -117,7 +117,7 @@ Zotero.Translators = new function() {
 					// Get JSON from cache if possible
 					if (memCacheJSON || dbCacheEntry) {
 						try {
-							var translator = Zotero.Translators.load(
+							var translator = this.load(
 								memCacheJSON || dbCacheEntry.metadataJSON, path
 							);
 						}
@@ -136,7 +136,7 @@ Zotero.Translators = new function() {
 					// Otherwise, load from file
 					else {
 						try {
-							var translator = yield Zotero.Translators.loadFromFile(path);
+							var translator = yield this.loadFromFile(path);
 						}
 						catch (e) {
 							Zotero.logError(e);
@@ -191,7 +191,7 @@ Zotero.Translators = new function() {
 					}
 					
 					if (!dbCacheEntry) {
-						yield Zotero.Translators.cacheInDB(
+						yield this.cacheInDB(
 							fileName,
 							translator.serialize(Zotero.Translator.TRANSLATOR_REQUIRED_PROPERTIES.
 												 concat(Zotero.Translator.TRANSLATOR_OPTIONAL_PROPERTIES)),
@@ -263,15 +263,15 @@ Zotero.Translators = new function() {
 	 *
 	 * @param {String} file - Path to translator file
 	 */
-	this.loadFromFile = function(path) {
+	this.loadFromFile = async function (path) {
 		const infoRe = /^\s*{[\S\s]*?}\s*?[\r\n]/;
-		return Zotero.File.getContentsAsync(path)
+		try {
-		.then(function(source) {
+			let source = await Zotero.File.getContentsAsync(path);
-			return Zotero.Translators.load(infoRe.exec(source)[0], path, source);
+			return this.load(infoRe.exec(source)[0], path, source);
-		})
+		}
-		.catch(function() {
+		catch (e) {
 			throw new Error("Invalid or missing translator metadata JSON object in " + OS.Path.basename(path));
-		});
+		}
 	}
 	
 	/**
@@ -403,7 +403,7 @@ Zotero.Translators = new function() {
 	 *     otherwise true
 	 */
 	this.getImportTranslatorsForLocation = function(location, callback) {	
-		return Zotero.Translators.getAllForType("import").then(function(allTranslators) {
+		return this.getAllForType("import").then(function(allTranslators) {
 			var tier1Translators = [];
 			var tier2Translators = [];
 			
@@ -438,6 +438,10 @@ Zotero.Translators = new function() {
 		return fileName;
 	}
 	
+	this.getTranslatorsDirectory = function () {
+		return Zotero.getTranslatorsDirectory().path;
+	};
+	
 	/**
 	 * @param	{String}		metadata
 	 * @param	{String}		metadata.translatorID		Translator GUID
@@ -490,10 +494,10 @@ Zotero.Translators = new function() {
 			throw new Error("code not provided");
 		}
 		
-		var fileName = Zotero.Translators.getFileNameFromLabel(
+		var fileName = this.getFileNameFromLabel(
 			metadata.label, metadata.translatorID
 		);
-		var destFile = OS.Path.join(Zotero.getTranslatorsDirectory().path, fileName);
+		var destFile = OS.Path.join(this.getTranslatorsDirectory(), fileName);
 		
 		// JSON.stringify has the benefit of indenting JSON
 		var metadataJSON = JSON.stringify(metadata, null, "\t");
@@ -505,7 +509,7 @@ Zotero.Translators = new function() {
 			str += '\n';
 		}
 		
-		var translator = Zotero.Translators.get(metadata.translatorID);
+		var translator = this.get(metadata.translatorID);
 		var sameFile = translator && destFile == translator.path;
 		
 		var exists = yield OS.File.exists(destFile);
@@ -528,4 +532,21 @@ Zotero.Translators = new function() {
 			[fileName, JSON.stringify(metadataJSON), lastModifiedTime]
 		);
 	}
+	
+	this.makeTranslatorProvider = function (methods) {
+		var requiredMethods = [
+			'get',
+			'getAllForType'
+		];
+		for (let method of requiredMethods) {
+			if (!(method in methods)) {
+				throw new Error(`Translator provider method ${method} not provided`);
+			}
+		}
+		return Object.assign(
+			{},
+			this,
+			methods
+		);
+	}
 }

test/tests/translateTest.js

@@ -959,6 +959,160 @@ describe("Zotero.Translate", function() {
 	});
 	
 	
+	describe("#setTranslatorProvider()", function () {
+		var url = "http://127.0.0.1:23119/test/translate/test.html";
+		var doc;
+		
+		beforeEach(function* () {
+			// This is the main processDocuments, not the translation sandbox one being tested
+			doc = (yield Zotero.HTTP.processDocuments(url, doc => doc))[0];
+		});
+		
+		it("should set a custom version of Zotero.Translators", async function () {
+			// Create a dummy translator to be returned by the stub methods
+			var info = {
+				translatorID: "e6111720-1f6c-42b0-a487-99b9fa50b8a1",
+				label: "Test",
+				creator: "Creator",
+				target: "^http:\/\/127.0.0.1:23119\/test",
+				minVersion: "5.0",
+				maxVersion: "",
+				priority: 100,
+				translatorType: 4,
+				browserSupport: "gcsibv",
+				lastUpdated: "2019-07-10 05:50:39",
+				cacheCode: true
+			};
+			info.code = JSON.stringify(info, null, '\t') + "\n\n"
+				+ "function detectWeb(doc, url) {"
+				+ "return 'journalArticle';"
+				+ "}\n"
+				+ "function doWeb(doc, url) {"
+				+ "var item = new Zotero.Item('journalArticle');"
+				+ "item.title = 'Test';"
+				+ "item.complete();"
+				+ "}\n";
+			var translator = new Zotero.Translator(info);
+			
+			var translate = new Zotero.Translate.Web();
+			var provider = Zotero.Translators.makeTranslatorProvider({
+				get: function (translatorID) {
+					if (translatorID == info.translatorID) {
+						return translator;
+					}
+					return false;
+				},
+				
+				getAllForType: async function (type) {
+					var translators = [];
+					if (type == 'web') {
+						translators.push(translator);
+					}
+					return translators;
+				}
+			});
+			translate.setTranslatorProvider(provider);
+			translate.setDocument(doc);
+			var translators = await translate.getTranslators();
+			translate.setTranslator(translators[0]);
+			var newItems = await translate.translate();
+			assert.equal(newItems.length, 1);
+			
+			var item = newItems[0];
+			assert.equal(item.getField('title'), 'Test');
+		});
+		
+		it("should set a custom version of Zotero.Translators in a child translator", async function () {
+			// Create dummy translators to be returned by the stub methods
+			var info1 = {
+				translatorID: "e6111720-1f6c-42b0-a487-99b9fa50b8a1",
+				label: "Test",
+				creator: "Creator",
+				target: "^http:\/\/127.0.0.1:23119\/test",
+				minVersion: "5.0",
+				maxVersion: "",
+				priority: 100,
+				translatorType: 4,
+				browserSupport: "gcsibv",
+				lastUpdated: "2019-07-10 05:50:39",
+				cacheCode: true
+			};
+			info1.code = JSON.stringify(info1, null, '\t') + "\n\n"
+				+ "function detectWeb(doc, url) {"
+					+ "return 'journalArticle';"
+				+ "}\n"
+				+ "function doWeb(doc, url) {"
+					+ "var translator = Zotero.loadTranslator('import');"
+					+ "translator.setTranslator('86e58f50-4e2d-4ee8-8a20-bafa225381fa');"
+					+ "translator.setString('foo\\n');"
+					+ "translator.setHandler('itemDone', function(obj, item) {"
+						+ "item.complete();"
+					+ "});"
+					+ "translator.translate();"
+				+ "}\n";
+			var translator1 = new Zotero.Translator(info1);
+			
+			var info2 = {
+				translatorID: "86e58f50-4e2d-4ee8-8a20-bafa225381fa",
+				label: "Child Test",
+				creator: "Creator",
+				target: "",
+				minVersion: "5.0",
+				maxVersion: "",
+				priority: 100,
+				translatorType: 3,
+				browserSupport: "gcsibv",
+				lastUpdated: "2019-07-19 06:22:21",
+				cacheCode: true
+			};
+			info2.code = JSON.stringify(info2, null, '\t') + "\n\n"
+				+ "function detectImport() {"
+					+ "return true;"
+				+ "}\n"
+				+ "function doImport() {"
+					+ "var item = new Zotero.Item('journalArticle');"
+					+ "item.title = 'Test';"
+					+ "item.complete();"
+				+ "}\n";
+			var translator2 = new Zotero.Translator(info2);
+			
+			var translate = new Zotero.Translate.Web();
+			var provider = Zotero.Translators.makeTranslatorProvider({
+				get: function (translatorID) {
+					switch (translatorID) {
+						case info1.translatorID:
+							return translator1;
+						
+						case info2.translatorID:
+							return translator2;
+					}
+					return false;
+				},
+				
+				getAllForType: async function (type) {
+					var translators = [];
+					if (type == 'web') {
+						translators.push(translator1);
+					}
+					if (type == 'import') {
+						translators.push(translator2);
+					}
+					return translators;
+				}
+			});
+			translate.setTranslatorProvider(provider);
+			translate.setDocument(doc);
+			var translators = await translate.getTranslators();
+			translate.setTranslator(translators[0]);
+			var newItems = await translate.translate();
+			assert.equal(newItems.length, 1);
+			
+			var item = newItems[0];
+			assert.equal(item.getField('title'), 'Test');
+		});
+	});
+	
+	
 	describe("Translators", function () {
 		it("should round-trip child attachment via BibTeX", function* () {
 			var item = yield createDataObject('item');