Merge pull request #1293 from zotero/xhr-processDocuments

XMLHttpRequest() processDocuments

.babelrc

@@ -27,13 +27,6 @@
     "syntax-object-rest-spread",
     "transform-react-jsx",
     "transform-react-display-name",
-    [
-      "transform-async-to-module-method",
-      {
-        "module": "resource://zotero/bluebird.js",
-        "method": "coroutine"
-      }
-    ],
     [
       "transform-es2015-modules-commonjs",
       {

chrome/content/zotero/tools/testTranslators/translatorTester.js

@@ -63,7 +63,7 @@ var Zotero_TranslatorTesters = new function() {
 					try {
 						for(var i=0; i<translators.length; i++) {
 							if (includeTranslators.length
-									&& !includeTranslators.includes(translators[i].label)) continue;
+									&& !includeTranslators.some(x => translators[i].label.includes(x))) continue;
 							if (skipTranslators && skipTranslators[translators[i].translatorID]) continue;
 							testers.push(new Zotero_TranslatorTester(translators[i], type));
 						};
@@ -299,9 +299,17 @@ Zotero_TranslatorTester._sanitizeItem = function(item, testItem, keepValidFields
 	// remove fields to be ignored
 	if(!keepValidFields && "accessDate" in item) delete item.accessDate;
 	
-	//sort tags, if they're still there
+	// Sort tags
-	if(item.tags && typeof item.tags === "object" && "sort" in item.tags) {
+	if (item.tags && Array.isArray(item.tags)) {
-		item.tags = Zotero.Utilities.arrayUnique(item.tags).sort();
+		// Normalize tags -- necessary until tests are updated for 5.0
+		if (testItem) {
+			item.tags = Zotero.Translate.Base.prototype._cleanTags(item.tags);
+		}
+		item.tags.sort((a, b) => {
+			if (a.tag < b.tag) return -1;
+			if (b.tag < a.tag) return 1;
+			return 0;
+		});
 	}
 	
 	return item;
@@ -391,54 +399,25 @@ Zotero_TranslatorTester.prototype._runTestsRecursively = function(testDoneCallba
 
 /**
  * Fetches the page for a given test and runs it
+ *
  * This function is only applicable in Firefox; it is overridden in translator_global.js in Chrome
- * and Safari
+ * and Safari.
- * @param {Object} test Test to execute
+ *
- * @param {Document} doc DOM document to test against
+ * @param {Object} test - Test to execute
- * @param {Function} testDoneCallback A callback to be executed when test is complete
+ * @param {Function} testDoneCallback - A callback to be executed when test is complete
  */
-Zotero_TranslatorTester.prototype.fetchPageAndRunTest = function(test, testDoneCallback) {
+Zotero_TranslatorTester.prototype.fetchPageAndRunTest = function (test, testDoneCallback) {
-	var timer = Components.classes["@mozilla.org/timer;1"].
+	Zotero.HTTP.processDocuments(
-		createInstance(Components.interfaces.nsITimer);
+		test.url,
-	timer.initWithCallback({"notify":function() {
+		(doc) => {
-		try {
+			this.runTest(test, doc, function (obj, test, status, message) {
-			if (hiddenBrowser) Zotero.Browser.deleteHiddenBrowser(hiddenBrowser);
-		} catch(e) {}
-	}}, TEST_RUN_TIMEOUT, Components.interfaces.nsITimer.TYPE_ONE_SHOT);
-	
-	var me = this;
-	var runTest = function(doc) {
-		me.runTest(test, doc, function(obj, test, status, message) {
-			try {
-				timer.cancel();
-			} catch(e) {};
-			if(hiddenBrowser) Zotero.Browser.deleteHiddenBrowser(hiddenBrowser);
 				testDoneCallback(obj, test, status, message);
 			});
-	};
-	var hiddenBrowser = Zotero.HTTP.processDocuments(test.url,
-		function(doc) {
-			if(test.defer) {
-				me._debug(this, "TranslatorTesting: Waiting "
-					+ (Zotero_TranslatorTester.DEFER_DELAY/1000)
-					+ " second(s) for page content to settle"
-				);
-				Zotero.setTimeout(() => runTest(doc), Zotero_TranslatorTester.DEFER_DELAY);
-			} else {
-				runTest(doc);
-			}
-		},
-		null,
-		function(e) {
-			testDoneCallback(this, test, "failed", "Translation failed to initialize: "+e);
-		},
-		true
-	);
-	
-	// No hidden browser returned from translation-server processDocuments()
-	if (hiddenBrowser) {
-		hiddenBrowser.docShell.allowMetaRedirects = true;
 		}
+	)
+	.catch(function (e) {
+		testDoneCallback(this, test, "failed", "Translation failed to initialize: " + e);
+	}.bind(this))
 };
 
 /**
@@ -531,7 +510,9 @@ Zotero_TranslatorTester.prototype._runTestTranslate = function(translate, transl
 	}
 	
 	translate.setTranslator(this.translator);
-	translate.translate(false);
+	translate.translate({
+		libraryID: false
+	});
 };
 
 /**
@@ -628,7 +609,9 @@ Zotero_TranslatorTester.prototype.newTest = function(doc, testReadyCallback) {
 	});
 	translate.setHandler("done", function(obj, returnValue) { me._createTest(obj, multipleMode, returnValue, testReadyCallback) });
 	translate.capitalizeTitles = false;
-	translate.translate(false);
+	translate.translate({
+		libraryID: false
+	});
 };
 
 /**

chrome/content/zotero/xpcom/attachments.js

@@ -265,7 +265,7 @@ Zotero.Attachments = new function(){
 		// Save using a hidden browser
 		var nativeHandlerImport = function () {
 			return new Zotero.Promise(function (resolve, reject) {
-				var browser = Zotero.HTTP.processDocuments(
+				var browser = Zotero.HTTP.loadDocuments(
 					url,
 					Zotero.Promise.coroutine(function* () {
 						let channel = browser.docShell.currentDocumentChannel;
@@ -591,7 +591,9 @@ Zotero.Attachments = new function(){
 				}
 			}
 			
-			if (contentType === 'text/html' || contentType === 'application/xhtml+xml') {
+			if ((contentType === 'text/html' || contentType === 'application/xhtml+xml')
+					// Documents from XHR don't work here
+					&& document instanceof Ci.nsIDOMDocument) {
 				Zotero.debug('Saving document with saveDocument()');
 				yield Zotero.Utilities.Internal.saveDocument(document, tmpFile);
 			}

chrome/content/zotero/xpcom/data/feedItem.js

@@ -234,10 +234,12 @@ Zotero.FeedItem.prototype.translate = Zotero.Promise.coroutine(function* (librar
 	}
 	
 	// Load document
-	let hiddenBrowser = Zotero.HTTP.processDocuments(
+	let hiddenBrowser = Zotero.HTTP.loadDocuments(
 		this.getField('url'),
-		item => deferred.resolve(item),
+		doc => deferred.resolve(doc),
-		()=>{}, error, true
+		() => {},
+		error,
+		true
 	);
 	let doc = yield deferred.promise;
 

chrome/content/zotero/xpcom/http.js

@@ -65,7 +65,10 @@ Zotero.HTTP = new function() {
 			this[i] = options[i];
 		}
 	};
-	this.SecurityException.prototype = Object.create(Zotero.Error.prototype);
+	this.SecurityException.prototype = Object.create(
+		// Zotero.Error not available in the connector
+		Zotero.Error ? Zotero.Error.prototype : Error.prototype
+	);
 	
 	
 	this.promise = function () {
@@ -797,20 +800,81 @@ Zotero.HTTP = new function() {
 			.getService(Components.interfaces.nsIIOService).offline;
 	}
 	
+	
+	/**
+	 * Load one or more documents using XMLHttpRequest
+	 *
+	 * This should stay in sync with the equivalent function in the connector
+	 *
+	 * @param {String|String[]} urls URL(s) of documents to load
+	 * @param {Function} processor - Callback to be executed for each document loaded; if function returns
+	 *     a promise, it's waited for before continuing
+	 * @param {Zotero.CookieSandbox} [cookieSandbox] Cookie sandbox object
+	 * @return {Promise<Array>} - A promise for an array of results from the processor runs
+	 */
+	this.processDocuments = async function (urls, processor, cookieSandbox) {
+		// Handle old signature: urls, processor, onDone, onError, dontDelete, cookieSandbox
+		if (arguments.length > 3) {
+			Zotero.debug("Zotero.HTTP.processDocuments() now takes only 3 arguments -- update your code");
+			var onDone = arguments[2];
+			var onError = arguments[3];
+			var cookieSandbox = arguments[5];
+		}
+		
+		if (typeof urls == "string") urls = [urls];
+		var funcs = urls.map(url => () => {
+			return Zotero.HTTP.request(
+				"GET",
+				url,
+				{
+					responseType: 'document'
+				}
+			)
+			.then((xhr) => {
+				var doc = this.wrapDocument(xhr.response, url);
+				return processor(doc, url);
+			});
+		});
+		
+		// Run processes serially
+		// TODO: Add some concurrency?
+		var f;
+		var results = [];
+		while (f = funcs.shift()) {
+			try {
+				results.push(await f());
+			}
+			catch (e) {
+				if (onError) {
+					onError(e);
+				}
+				throw e;
+			}
+		}
+		
+		// Deprecated
+		if (onDone) {
+			onDone();
+		}
+		
+		return results;
+	};
+	
+	
 	/**
 	 * Load one or more documents in a hidden browser
 	 *
 	 * @param {String|String[]} urls URL(s) of documents to load
 	 * @param {Function} processor - Callback to be executed for each document loaded; if function returns
 	 *     a promise, it's waited for before continuing
-	 * @param {Function} done Callback to be executed after all documents have been loaded
+	 * @param {Function} onDone - Callback to be executed after all documents have been loaded
-	 * @param {Function} exception Callback to be executed if an exception occurs
+	 * @param {Function} onError - Callback to be executed if an error occurs
 	 * @param {Boolean} dontDelete Don't delete the hidden browser upon completion; calling function
 	 *                             must call deleteHiddenBrowser itself.
 	 * @param {Zotero.CookieSandbox} [cookieSandbox] Cookie sandbox object
 	 * @return {browser} Hidden browser used for loading
 	 */
-	this.processDocuments = function(urls, processor, done, exception, dontDelete, cookieSandbox) {
+	this.loadDocuments = function (urls, processor, onDone, onError, dontDelete, cookieSandbox) {
 		// (Approximately) how many seconds to wait if the document is left in the loading state and
 		// pageshow is called before we call pageshow with an incomplete document
 		const LOADING_STATE_TIMEOUT = 120;
@@ -827,11 +891,11 @@ Zotero.HTTP = new function() {
 				firedLoadEvent = 0;
 				currentURL++;
 				try {
-					Zotero.debug("Zotero.HTTP.processDocuments: Loading "+url);
+					Zotero.debug("Zotero.HTTP.loadDocuments: Loading " + url);
 					hiddenBrowser.loadURI(url);
 				} catch(e) {
-					if(exception) {
+					if (onError) {
-						exception(e);
+						onError(e);
 						return;
 					} else {
 						if(!dontDelete) Zotero.Browser.deleteHiddenBrowser(hiddenBrowsers);
@@ -840,7 +904,7 @@ Zotero.HTTP = new function() {
 				}
 			} else {
 				if(!dontDelete) Zotero.Browser.deleteHiddenBrowser(hiddenBrowsers);
-				if(done) done();
+				if (onDone) onDone();
 			}
 		};
 		
@@ -861,7 +925,7 @@ Zotero.HTTP = new function() {
 				return;
 			}
 			
-			Zotero.debug("Zotero.HTTP.processDocuments: "+url+" loaded");
+			Zotero.debug("Zotero.HTTP.loadDocuments: " + url + " loaded");
 			hiddenBrowser.removeEventListener("pageshow", onLoad, true);
 			hiddenBrowser.zotero_loaded = true;
 			
@@ -878,8 +942,8 @@ Zotero.HTTP = new function() {
 			if (maybePromise && maybePromise.then) {
 				maybePromise.then(() => doLoad())
 				.catch(e => {
-					if (exception) {
+					if (onError) {
-						exception(e);
+						onError(e);
 					}
 					else {
 						throw e;
@@ -890,8 +954,8 @@ Zotero.HTTP = new function() {
 			
 			try {
 				if (error) {
-					if (exception) {
+					if (onError) {
-						exception(error);
+						onError(error);
 					}
 					else {
 						throw error;

chrome/content/zotero/xpcom/server_connector.js

@@ -505,7 +505,7 @@ Zotero.Server.Connector.SaveSnapshot.prototype = {
 		}
 		else {
 			let deferred = Zotero.Promise.defer();
-			Zotero.HTTP.processDocuments(
+			Zotero.HTTP.loadDocuments(
 				["zotero://connector/" + encodeURIComponent(data.url)],
 				Zotero.Promise.coroutine(function* (doc) {
 					delete Zotero.Server.Connector.Data[data.url];

chrome/content/zotero/xpcom/translation/translate.js

@@ -86,7 +86,7 @@ Zotero.Translate.Sandbox = {
 				&& translate.translator[0].configOptions
 				&& translate.translator[0].configOptions.async;
 			
-			var run = function (resolve) {
+			var run = async function (async) {
 				Zotero.debug("Translate: Saving item");
 				
 				// warn if itemDone called after translation completed
@@ -152,7 +152,7 @@ Zotero.Translate.Sandbox = {
 				if(translate._libraryID === false || translate._parentTranslator) {
 					translate.newItems.push(item);
 					if(translate._parentTranslator && Zotero.isFx && !Zotero.isBookmarklet) {
-						// Copy object so it is accessible to child translator
+						// Copy object so it is accessible to parent translator
 						item = translate._sandboxManager.copyObject(item);
 						item.complete = oldItem.complete;
 					}
@@ -216,12 +216,9 @@ Zotero.Translate.Sandbox = {
 				
 				// For synchronous import (when Promise isn't available in the sandbox or the do*
 				// function doesn't use it) and web translators, queue saves
-				if (!resolve || !asyncTranslator) {
+				if (!async || !asyncTranslator) {
 					Zotero.debug("Translate: Saving via queue");
 					translate.saveQueue.push(item);
-					if (resolve) {
-						resolve();
-					}
 				}
 				// For async import, save items immediately
 				else {
@@ -240,11 +237,9 @@ Zotero.Translate.Sandbox = {
 			
 			return new translate._sandboxManager.sandbox.Promise(function (resolve, reject) {
 				try {
-					let maybePromise = run(resolve);
+					run(true).then(
-					if (maybePromise) {
+						resolve,
-						maybePromise
+						function (e) {
-						.then(resolve)
-						.catch(function (e) {
 							// Fix wrapping error from sandbox when error is thrown from _saveItems()
 							if (Zotero.isFx) {
 								reject(translate._sandboxManager.copyObject(e));
@@ -252,8 +247,8 @@ Zotero.Translate.Sandbox = {
 							else {
 								reject(e);
 							}
-						});
 						}
+					);
 				}
 				catch (e) {
 					reject(e);
@@ -1106,7 +1101,9 @@ Zotero.Translate.Base.prototype = {
 			
 			var handlers = this._handlers[type].slice();
 			for(var i=0, n=handlers.length; i<n; i++) {
-				Zotero.debug("Translate: Running handler "+i+" for "+type, 5);
+				if (type != 'debug') {
+					Zotero.debug(`Translate: Running handler ${i} for ${type}`, 5);
+				}
 				try {
 					returnValue = handlers[i].apply(null, args);
 				} catch(e) {

chrome/content/zotero/xpcom/utilities_translate.js

@@ -191,13 +191,22 @@ Zotero.Utilities.Translate.prototype.loadDocument = function(url, succeeded, fai
 }
 
 /**
- * Already documented in Zotero.HTTP
+ * Already documented in Zotero.HTTP, except this optionally takes noCompleteOnError, which prevents
+ * the translation process from being cancelled automatically on error, as it is normally. The promise
+ * is still rejected on error for handling by the calling function.
  * @ignore
  */
-Zotero.Utilities.Translate.prototype.processDocuments = function(urls, processor, done, exception) {
+Zotero.Utilities.Translate.prototype.processDocuments = async function (urls, processor, noCompleteOnError) {
+	// Handle old signature: urls, processor, onDone, onError
+	if (arguments.length > 3 || typeof arguments[2] == 'function') {
+		Zotero.debug("ZU.processDocuments() now takes only 3 arguments -- update your code");
+		var onDone = arguments[2];
+		var onError = arguments[3];
+	}
+	
 	var translate = this._translate;
 
-	if(typeof(urls) == "string") {
+	if (typeof urls == "string") {
 		urls = [translate.resolveURL(urls)];
 	} else {
 		for(var i in urls) {
@@ -205,109 +214,89 @@ Zotero.Utilities.Translate.prototype.processDocuments = function(urls, processor
 		}
 	}
 	
-	// Unless the translator has proposed some way to handle an error, handle it
+	var processDoc = function (doc) {
-	// by throwing a "scraping error" message
+		if (Zotero.isFx) {
-	if(exception) {
+			let newLoc = doc.location;
-		var myException = function(e) {
+			let url = Services.io.newURI(newLoc.href, null, null);
-			var browserDeleted;
+			return processor(
-			try {
+				// Rewrap document for the sandbox
-				exception(e);
-			} catch(e) {
-				if(hiddenBrowser) {
-					try {
-						Zotero.Browser.deleteHiddenBrowser(hiddenBrowser);
-					} catch(e) {}
-				}
-				browserDeleted = true;
-				translate.complete(false, e);
-			}
-			
-			if(!browserDeleted) {
-				try {
-					Zotero.Browser.deleteHiddenBrowser(hiddenBrowser);
-				} catch(e) {}
-			}
-		}
-	} else {
-		var myException = function(e) {
-			if(hiddenBrowser) {
-				try {
-					Zotero.Browser.deleteHiddenBrowser(hiddenBrowser);
-				} catch(e) {}
-			}
-			translate.complete(false, e);
-		}
-	}
-	
-	if(Zotero.isFx) {
-		if(typeof translate._sandboxLocation === "object") {
-			var protocol = translate._sandboxLocation.location.protocol,
-				host = translate._sandboxLocation.location.host;
-        } else {
-			var url = Components.classes["@mozilla.org/network/io-service;1"] 
-					.getService(Components.interfaces.nsIIOService)
-					.newURI(translate._sandboxLocation, null, null),
-				protocol = url.scheme+":",
-				host = url.host;
-		}
-	}
-	
-	for(var i=0; i<urls.length; i++) {
-		if(translate.document && translate.document.location
-				&& translate.document.location.toString() === urls[i]) {
-			// Document is attempting to reload itself
-			Zotero.debug("Translate: Attempted to load the current document using processDocuments; using loaded document instead");
-			// This fixes document permissions issues in translation-server when translators call
-			// processDocuments() on the original URL (e.g., AOSIC)
-			// DEBUG: Why is this necessary? (see below also)
-			if (Zotero.isServer) {
-				processor(
 				translate._sandboxManager.wrap(
-						Zotero.Translate.DOMWrapper.unwrap(
+					Zotero.Translate.DOMWrapper.unwrap(doc),
-							this._translate.document
+					null,
-						)
+					// Duplicate overrides from Zotero.HTTP.wrapDocument()
+					{
+						documentURI: newLoc.spec,
+						URL: newLoc.spec,
+						location: new Zotero.HTTP.Location(url),
+						defaultView: new Zotero.HTTP.Window(url)
+					}
 				),
-					urls[i]
+				newLoc.href
 			);
 		}
-			else {
+		
-				processor(this._translate.document, urls[i]);
+		return processor(doc, doc.location.href);
-			}
+	};
+	
+	var funcs = [];
+	// If current URL passed, use loaded document instead of reloading
+	for (var i = 0; i < urls.length; i++) {
+		if(translate.document && translate.document.location
+				&& translate.document.location.toString() === urls[i]) {
+			Zotero.debug("Translate: Attempted to load the current document using processDocuments; using loaded document instead");
+			funcs.push(() => processDoc(this._translate.document, urls[i]));
 			urls.splice(i, 1);
 			i--;
 		}
 	}
 	
 	translate.incrementAsyncProcesses("Zotero.Utilities.Translate#processDocuments");
-	var hiddenBrowser = Zotero.HTTP.processDocuments(urls, function (doc, url) {
-		if(!processor) return;
 	
-		var newLoc = doc.location;
+	if (urls.length) {
-		if((Zotero.isFx && !Zotero.isBookmarklet && (protocol != newLoc.protocol || host != newLoc.host))
+		funcs.push(
-				// This fixes document permissions issues in translation-server when translators call
+			() => Zotero.HTTP.processDocuments(
-				// processDocuments() on same-domain URLs (e.g., some of the Code4Lib tests).
+				urls,
-				// DEBUG: Is there a better fix for this?
+				function (doc) {
-				|| Zotero.isServer) {
+					if (!processor) return;
-			// Cross-site; need to wrap
+					return processDoc(doc);
-			processor(translate._sandboxManager.wrap(doc), url);
-		} else {
-			// Not cross-site; no need to wrap
-			processor(doc, url);
-		}
 				},
-	function() {
+				translate.cookieSandbox
-		if(done) done();
+			)
-		var handler = function() {
+		);
-			if(hiddenBrowser) {
-				try {
-					Zotero.Browser.deleteHiddenBrowser(hiddenBrowser);
-				} catch(e) {}
 	}
-			translate.removeHandler("done", handler);
+	
-		};
+	var f;
-		translate.setHandler("done", handler);
+	while (f = funcs.shift()) {
+		try {
+			let maybePromise = f();
+			// The processor may or may not return a promise
+			if (maybePromise) {
+				await maybePromise;
+			}
+		}
+		catch (e) {
+			if (onError) {
+				try {
+					onError(e);
+				}
+				catch (e) {
+					translate.complete(false, e);
+				}
+			}
+			// Unless instructed otherwise, end the translation on error
+			else if (!noCompleteOnError) {
+				translate.complete(false, e);
+			}
+			throw e;
+		}
+	}
+	
+	// Deprecated
+	if (onDone) {
+		onDone();
+	}
+	
 	translate.decrementAsyncProcesses("Zotero.Utilities.Translate#processDocuments");
-	}, myException, true, translate.cookieSandbox);
 }
 
 /**

chrome/content/zotero/zoteroPane.js

@@ -3885,18 +3885,17 @@ var ZoteroPane = new function()
 			var deferred = Zotero.Promise.defer();
 			
 			var processor = function (doc) {
-				ZoteroPane_Local.addItemFromDocument(doc, itemType, saveSnapshot, row)
+				return ZoteroPane_Local.addItemFromDocument(doc, itemType, saveSnapshot, row)
 				.then(function () {
 					deferred.resolve()
-				})
+				});
 			};
-			// TODO: processDocuments should wait for the processor promise to be resolved
 			var done = function () {}
 			var exception = function (e) {
 				Zotero.debug(e, 1);
 				deferred.reject(e);
 			}
-			Zotero.HTTP.processDocuments([url], processor, done, exception);
+			Zotero.HTTP.loadDocuments([url], processor, done, exception);
 			
 			return deferred.promise;
 		}

package.json

@@ -30,7 +30,6 @@
     "babel-plugin-syntax-flow": "^6.13.0",
     "babel-plugin-syntax-jsx": "^6.13.0",
     "babel-plugin-syntax-object-rest-spread": "^6.13.0",
-    "babel-plugin-transform-async-to-module-method": "^6.16.0",
     "babel-plugin-transform-es2015-modules-commonjs": "^6.18.0",
     "babel-preset-react": "^6.16.0",
     "browserify": "^14.3.0",

test/tests/httpTest.js

@@ -0,0 +1,76 @@
+describe("Zotero.HTTP", function () {
+	var httpd;
+	var port = 16213;
+	
+	before(function* () {
+		Components.utils.import("resource://zotero-unit/httpd.js");
+		
+		httpd = new HttpServer();
+		httpd.start(port);
+		httpd.registerPathHandler(
+			'/test.html',
+			{
+				handle: function (request, response) {
+					response.setStatusLine(null, 200, "OK");
+					response.write("<html><body><p>Test</p><p>Test 2</p></body></html>");
+				}
+			}
+		);
+	});
+	
+	after(function* () {
+		var defer = new Zotero.Promise.defer();
+		httpd.stop(() => defer.resolve());
+		yield defer.promise;
+	});
+	
+	describe("#processDocuments()", function () {
+		it("should provide a document object", function* () {
+			var called = false;
+			var url = `http://127.0.0.1:${port}/test.html`;
+			yield Zotero.HTTP.processDocuments(
+				url,
+				function (doc) {
+					assert.equal(doc.location.href, url);
+					assert.equal(doc.querySelector('p').textContent, 'Test');
+					var p = doc.evaluate('//p', doc, null, XPathResult.ANY_TYPE, null).iterateNext();
+					assert.equal(p.textContent, 'Test');
+					called = true;
+				}
+			);
+			assert.isTrue(called);
+		});
+	});
+	
+	describe("#loadDocuments()", function () {
+		var win;
+		
+		before(function* () {
+			// TEMP: createHiddenBrowser currently needs a parent window
+			win = yield loadBrowserWindow();
+		});
+		
+		after(function* () {
+			win.close();
+		});
+		
+		it("should provide a document object", function* () {
+			var called = false;
+			var url = `http://127.0.0.1:${port}/test.html`;
+			yield new Zotero.Promise((resolve) => {
+				Zotero.HTTP.loadDocuments(
+					url,
+					function (doc) {
+						assert.equal(doc.location.href, url);
+						assert.equal(doc.querySelector('p').textContent, 'Test');
+						var p = doc.evaluate('//p', doc, null, XPathResult.ANY_TYPE, null).iterateNext();
+						assert.equal(p.textContent, 'Test');
+						called = true;
+					},
+					resolve
+				);
+			});
+			assert.isTrue(called);
+		});
+	});
+});

test/tests/translateTest.js

@@ -486,11 +486,15 @@ describe("Zotero.Translate", function() {
 			checkTestTags(pdf, true);
 		});
 
-		it('web translators should save attachment from document', function* () {
+		it('web translators should save attachment from browser document', function* () {
 			let deferred = Zotero.Promise.defer();
-			let browser = Zotero.HTTP.processDocuments("http://127.0.0.1:23119/test/translate/test.html",
+			let browser = Zotero.HTTP.loadDocuments(
-				                                       function (doc) { deferred.resolve(doc) }, undefined,
+				"http://127.0.0.1:23119/test/translate/test.html",
-				                                       undefined, true);
+				doc => deferred.resolve(doc),
+				undefined,
+				undefined,
+				true
+			);
 			let doc = yield deferred.promise;
 
 			let translate = new Zotero.Translate.Web();
@@ -510,7 +514,7 @@ describe("Zotero.Translate", function() {
 				'}'));
 			let newItems = yield translate.translate();
 			assert.equal(newItems.length, 1);
-			let containedAttachments = yield Zotero.Items.getAsync(newItems[0].getAttachments());
+			let containedAttachments = Zotero.Items.get(newItems[0].getAttachments());
 			assert.equal(containedAttachments.length, 1);
 
 			let snapshot = containedAttachments[0];
@@ -523,6 +527,40 @@ describe("Zotero.Translate", function() {
 			Zotero.Browser.deleteHiddenBrowser(browser);
 		});
 		
+		it('web translators should save attachment from non-browser document', function* () {
+			return Zotero.HTTP.processDocuments(
+				"http://127.0.0.1:23119/test/translate/test.html",
+				async function (doc) {
+					let translate = new Zotero.Translate.Web();
+					translate.setDocument(doc);
+					translate.setTranslator(buildDummyTranslator(4,
+						'function detectWeb() {}\n'+
+						'function doWeb(doc) {\n'+
+						'	var item = new Zotero.Item("book");\n'+
+						'	item.title = "Container Item";\n'+
+						'	item.attachments = [{\n'+
+						'		"document":doc,\n'+
+						'		"title":"Snapshot from Document",\n'+
+						'		"note":"attachment note",\n'+
+						'		"tags":'+JSON.stringify(TEST_TAGS)+'\n'+
+						'	}];\n'+
+						'	item.complete();\n'+
+						'}'));
+					let newItems = await translate.translate();
+					assert.equal(newItems.length, 1);
+					let containedAttachments = Zotero.Items.get(newItems[0].getAttachments());
+					assert.equal(containedAttachments.length, 1);
+		
+					let snapshot = containedAttachments[0];
+					assert.equal(snapshot.getField("url"), "http://127.0.0.1:23119/test/translate/test.html");
+					assert.equal(snapshot.getNote(), "attachment note");
+					assert.equal(snapshot.attachmentLinkMode, Zotero.Attachments.LINK_MODE_IMPORTED_URL);
+					assert.equal(snapshot.attachmentContentType, "text/html");
+					checkTestTags(snapshot, true);
+				}
+			);
+		});
+
 		it('web translators should ignore attachments that return error codes', function* () {
 			this.timeout(60000);
 			let myItems = [
@@ -629,6 +667,117 @@ describe("Zotero.Translate", function() {
 	});
 	
 	
+	describe("#processDocuments()", function () {
+		var url = "http://127.0.0.1:23119/test/translate/test.html";
+		var doc;
+		
+		beforeEach(function* () {
+			// This is the main processDocuments, not the translation sandbox one being tested
+			doc = (yield Zotero.HTTP.processDocuments(url, doc => doc))[0];
+		});
+		
+		it("should provide document object", async function () {
+			var translate = new Zotero.Translate.Web();
+			translate.setDocument(doc);
+			translate.setTranslator(
+				buildDummyTranslator(
+					4,
+					`function detectWeb() {}
+					function doWeb(doc) {
+						ZU.processDocuments(
+							doc.location.href + '?t',
+							function (doc) {
+								var item = new Zotero.Item("book");
+								item.title = "Container Item";
+								// document.location
+								item.url = doc.location.href;
+								// document.evaluate()
+								item.extra = doc
+									.evaluate('//p', doc, null, XPathResult.ANY_TYPE, null)
+									.iterateNext()
+									.textContent;
+								item.attachments = [{
+									document: doc,
+									title: "Snapshot from Document",
+									note: "attachment note",
+									tags: ${JSON.stringify(TEST_TAGS)}
+								}];
+								item.complete();
+							}
+						);
+					}`
+				)
+			);
+			var newItems = await translate.translate();
+			assert.equal(newItems.length, 1);
+			
+			var item = newItems[0];
+			assert.equal(item.getField('url'), url + '?t');
+			assert.include(item.getField('extra'), 'your research sources');
+			
+			var containedAttachments = Zotero.Items.get(newItems[0].getAttachments());
+			assert.equal(containedAttachments.length, 1);
+			
+			var snapshot = containedAttachments[0];
+			assert.equal(snapshot.getField("url"), url + '?t');
+			assert.equal(snapshot.getNote(), "attachment note");
+			assert.equal(snapshot.attachmentLinkMode, Zotero.Attachments.LINK_MODE_IMPORTED_URL);
+			assert.equal(snapshot.attachmentContentType, "text/html");
+			checkTestTags(snapshot, true);
+		});
+		
+		it("should use loaded document instead of reloading if possible", function* () {
+			var translate = new Zotero.Translate.Web();
+			translate.setDocument(doc);
+			translate.setTranslator(
+				buildDummyTranslator(
+					4,
+					`function detectWeb() {}
+					function doWeb(doc) {
+						ZU.processDocuments(
+							doc.location.href,
+							function (doc) {
+								var item = new Zotero.Item("book");
+								item.title = "Container Item";
+								// document.location
+								item.url = doc.location.href;
+								// document.evaluate()
+								item.extra = doc
+									.evaluate('//p', doc, null, XPathResult.ANY_TYPE, null)
+									.iterateNext()
+									.textContent;
+								item.attachments = [{
+									document: doc,
+									title: "Snapshot from Document",
+									note: "attachment note",
+									tags: ${JSON.stringify(TEST_TAGS)}
+								}];
+								item.complete();
+							}
+						);
+					}`
+				)
+			);
+			var newItems = yield translate.translate();
+			assert.equal(newItems.length, 1);
+			
+			var item = newItems[0];
+			assert.equal(item.getField('url'), url);
+			assert.include(item.getField('extra'), 'your research sources');
+			
+			var containedAttachments = Zotero.Items.get(newItems[0].getAttachments());
+			assert.equal(containedAttachments.length, 1);
+			
+			var snapshot = containedAttachments[0];
+			assert.equal(snapshot.getField("url"), url);
+			assert.equal(snapshot.getNote(), "attachment note");
+			assert.equal(snapshot.attachmentLinkMode, Zotero.Attachments.LINK_MODE_IMPORTED_URL);
+			assert.equal(snapshot.attachmentContentType, "text/html");
+			checkTestTags(snapshot, true);
+		});
+	});
+	
+	
 	describe("Translators", function () {
 		it("should round-trip child attachment via BibTeX", function* () {
 			var item = yield createDataObject('item');