From cd856efef8bc72c48c74e3e41f5abb1ddf539927 Mon Sep 17 00:00:00 2001 From: Adomas Ven Date: Thu, 19 Jun 2025 08:01:52 +0300 Subject: [PATCH] Connector attachment saving server changes for 7.0 (#5345) --- .../content/zotero/ZoteroProtocolHandler.jsm | 94 - chrome/content/zotero/advancedSearch.js | 1 - chrome/content/zotero/collectionTree.jsx | 7 + chrome/content/zotero/xpcom/attachments.js | 92 +- .../content/zotero/xpcom/collectionTreeRow.js | 4 +- .../xpcom/connector/server_connector.js | 1947 ------ chrome/content/zotero/xpcom/file.js | 62 + .../{connector => }/httpIntegrationClient.js | 0 chrome/content/zotero/xpcom/prompt.js | 2 +- .../content/zotero/xpcom/recognizeDocument.js | 16 +- .../zotero/xpcom/server/saveSession.js | 283 + .../zotero/xpcom/{ => server}/server.js | 581 +- .../zotero/xpcom/server/server_connector.js | 1240 ++++ .../server_connectorIntegration.js | 0 .../xpcom/{ => server}/server_integration.js | 0 .../{localAPI => server}/server_localAPI.js | 0 .../xpcom/translation/translate_item.js | 566 +- chrome/content/zotero/zotero.mjs | 13 +- test/content/support.js | 3 +- test/resource/httpd.js | 5618 ----------------- test/tests/HiddenBrowserTest.js | 5 +- test/tests/attachmentsTest.js | 2 +- test/tests/collectionTreeTest.js | 2 +- test/tests/fileTest.js | 2 +- test/tests/httpTest.js | 2 +- test/tests/itemTreeTest.js | 2 +- test/tests/mendeleyImportTest.js | 2 +- test/tests/recognizeDocumentTest.js | 14 +- test/tests/serverTest.js | 48 +- test/tests/server_connectorTest.js | 2385 ++----- test/tests/server_integrationTest.js | 5 +- test/tests/translateTest.js | 2 +- test/tests/webdavTest.js | 6 +- 33 files changed, 2991 insertions(+), 10015 deletions(-) delete mode 100644 chrome/content/zotero/xpcom/connector/server_connector.js rename chrome/content/zotero/xpcom/{connector => }/httpIntegrationClient.js (100%) create mode 100644 chrome/content/zotero/xpcom/server/saveSession.js rename chrome/content/zotero/xpcom/{ => server}/server.js (52%) create mode 100644 chrome/content/zotero/xpcom/server/server_connector.js rename chrome/content/zotero/xpcom/{connector => server}/server_connectorIntegration.js (100%) rename chrome/content/zotero/xpcom/{ => server}/server_integration.js (100%) rename chrome/content/zotero/xpcom/{localAPI => server}/server_localAPI.js (100%) delete mode 100644 test/resource/httpd.js diff --git a/chrome/content/zotero/ZoteroProtocolHandler.jsm b/chrome/content/zotero/ZoteroProtocolHandler.jsm index 1776335682..27fb164f7d 100644 --- a/chrome/content/zotero/ZoteroProtocolHandler.jsm +++ b/chrome/content/zotero/ZoteroProtocolHandler.jsm @@ -969,99 +969,6 @@ function ZoteroProtocolHandler() { } }; - var ConnectorChannel = function(uri, data) { - var secMan = Components.classes["@mozilla.org/scriptsecuritymanager;1"] - .getService(Components.interfaces.nsIScriptSecurityManager); - - this.name = uri; - this.URI = ios.newURI(uri, "UTF-8", null); - this.owner = (secMan.getCodebasePrincipal || secMan.getSimpleCodebasePrincipal)(this.URI); - this._isPending = true; - - var converter = Components.classes["@mozilla.org/intl/scriptableunicodeconverter"]. - createInstance(Components.interfaces.nsIScriptableUnicodeConverter); - converter.charset = "UTF-8"; - this._stream = converter.convertToInputStream(data); - this.contentLength = this._stream.available(); - } - - ConnectorChannel.prototype.contentCharset = "UTF-8"; - ConnectorChannel.prototype.contentType = "text/html"; - ConnectorChannel.prototype.notificationCallbacks = null; - ConnectorChannel.prototype.securityInfo = null; - ConnectorChannel.prototype.status = 0; - ConnectorChannel.prototype.loadGroup = null; - ConnectorChannel.prototype.loadFlags = 393216; - - ConnectorChannel.prototype.__defineGetter__("originalURI", function() { return this.URI }); - ConnectorChannel.prototype.__defineSetter__("originalURI", function() { }); - - ConnectorChannel.prototype.asyncOpen = function(streamListener) { - if(this.loadGroup) this.loadGroup.addRequest(this, null); - streamListener.onStartRequest(this); - streamListener.onDataAvailable(this, this._stream, 0, this.contentLength); - streamListener.onStopRequest(this, this.status); - this._isPending = false; - if(this.loadGroup) this.loadGroup.removeRequest(this, null, 0); - } - - ConnectorChannel.prototype.isPending = function() { - return this._isPending; - } - - ConnectorChannel.prototype.cancel = function(status) { - this.status = status; - this._isPending = false; - if(this._stream) this._stream.close(); - } - - ConnectorChannel.prototype.suspend = function() {} - - ConnectorChannel.prototype.resume = function() {} - - ConnectorChannel.prototype.open = function() { - return this._stream; - } - - ConnectorChannel.prototype.QueryInterface = function(iid) { - if (!iid.equals(Components.interfaces.nsIChannel) && !iid.equals(Components.interfaces.nsIRequest) && - !iid.equals(Components.interfaces.nsISupports)) { - throw Components.results.NS_ERROR_NO_INTERFACE; - } - return this; - } - - /** - * zotero://connector/ - * - * URI spoofing for transferring page data across boundaries - */ - var ConnectorExtension = new function() { - this.loadAsChrome = false; - - this.newChannel = function(uri) { - var secMan = Components.classes["@mozilla.org/scriptsecuritymanager;1"] - .getService(Components.interfaces.nsIScriptSecurityManager); - var Zotero = Components.classes["@zotero.org/Zotero;1"] - .getService(Components.interfaces.nsISupports) - .wrappedJSObject; - - try { - var originalURI = uri.pathQueryRef.substr('zotero://connector/'.length); - originalURI = decodeURIComponent(originalURI); - if(!Zotero.Server.Connector.Data[originalURI]) { - return null; - } else { - return new ConnectorChannel(originalURI, Zotero.Server.Connector.Data[originalURI]); - } - } catch(e) { - Zotero.debug(e); - throw e; - } - } - }; - - /* zotero://pdf.js/viewer.html zotero://pdf.js/pdf/1/ABCD5678 @@ -1249,7 +1156,6 @@ function ZoteroProtocolHandler() { this._extensions[ZOTERO_SCHEME + "://timeline"] = TimelineExtension; this._extensions[ZOTERO_SCHEME + "://select"] = SelectExtension; this._extensions[ZOTERO_SCHEME + "://debug"] = DebugExtension; - this._extensions[ZOTERO_SCHEME + "://connector"] = ConnectorExtension; this._extensions[ZOTERO_SCHEME + "://pdf.js"] = PDFJSExtension; this._extensions[ZOTERO_SCHEME + "://open"] = OpenExtension; this._extensions[ZOTERO_SCHEME + "://open-pdf"] = OpenExtension; diff --git a/chrome/content/zotero/advancedSearch.js b/chrome/content/zotero/advancedSearch.js index 26d5d93132..8f6da1fd89 100644 --- a/chrome/content/zotero/advancedSearch.js +++ b/chrome/content/zotero/advancedSearch.js @@ -89,7 +89,6 @@ var ZoteroAdvancedSearch = new function() { isTrash: () => false }; - this.itemsView.changeCollectionTreeRow(collectionTreeRow); // Focus the first field in the window Services.focus.moveFocus(window, null, Services.focus.MOVEFOCUS_FORWARD, 0); } diff --git a/chrome/content/zotero/collectionTree.jsx b/chrome/content/zotero/collectionTree.jsx index 739533b712..8b3a4c004a 100644 --- a/chrome/content/zotero/collectionTree.jsx +++ b/chrome/content/zotero/collectionTree.jsx @@ -1326,6 +1326,13 @@ var CollectionTree = class CollectionTree extends LibraryTree { return this.getRow(this.selection.focused).editable; } + /** + * Returns TRUE if the underlying view is editable + */ + get filesEditable() { + return this.getRow(this.selection.focused).filesEditable; + } + getRowString(index) { // During filtering, context rows return an empty string to not be selectable // with key-based navigation diff --git a/chrome/content/zotero/xpcom/attachments.js b/chrome/content/zotero/xpcom/attachments.js index a562ad2f84..079f354289 100644 --- a/chrome/content/zotero/xpcom/attachments.js +++ b/chrome/content/zotero/xpcom/attachments.js @@ -681,7 +681,7 @@ Zotero.Attachments = new function () { */ this.createURLAttachmentFromTemporaryStorageDirectory = async function (options) { if (!options.directory) throw new Error("'directory' not provided"); - if (!options.libraryID) throw new Error("'libraryID' not provided"); + if (!options.libraryID && !options.parentItemID) throw new Error("'libraryID' or 'parentItemID' not provided"); if (!options.filename) throw new Error("'filename' not provided"); if (!options.url) throw new Error("'directory' not provided"); if (!options.contentType) throw new Error("'contentType' not provided"); @@ -977,6 +977,96 @@ Zotero.Attachments = new function () { return attachmentItem; }); + + /** + * Save an attachment from a nsIInputStream + * + * @param {Object} options + * @param {String} options.url + * @param {nsIStream} options.stream - Stream with data + * @param {Integer} options.byteCount - Number of bytes in the stream, usually from the + * 'Content-Length' HTTP header. + * @param {String} options.contentType - Expected content type + * @param {Integer} [options.libraryID] Parent item ID if child attachment + * @param {Integer} [options.parentItemID] Parent item ID if child attachment + * Either options.libraryID or options.parentItemID are mandatory + * @param {Array} [options.collections] Collection ids or keys + * @param {String} [options.title] + * @param {Object} [options.saveOptions] - Options to pass to Zotero.Item::save() + * @return {Promise} - A promise for the created attachment item + */ + this.importFromNetworkStream = async (options) => { + if (!options.url) throw new Error("'url' not provided"); + if (!options.stream) throw new Error("'stream' not provided"); + if (!options.byteCount) throw new Error("'byteCount' not provided"); + if (!options.contentType) throw new Error("'contentType' not provided"); + Zotero.debug("Importing attachment item from network stream"); + + let url = options.url; + let stream = options.stream; + let contentType = options.contentType; + let libraryID = options.libraryID; + let parentItemID = options.parentItemID; + let collections = options.collections; + let title = options.title; + let saveOptions = options.saveOptions; + + if (parentItemID && collections) { + throw new Error("parentItemID and collections cannot both be provided"); + } + + // Create a temporary file + let filename; + if (parentItemID) { + let parentItem = Zotero.Items.get(parentItemID); + let fileBaseName = this.getFileBaseNameFromItem(parentItem, { attachmentTitle: title }); + let ext = this._getExtensionFromURL(url, contentType); + filename = fileBaseName + (ext != '' ? '.' + ext : ''); + } + else { + filename = Zotero.File.truncateFileName(this._getFileNameFromURL(url, contentType), 100); + } + + let tmpDirectory = (await this.createTemporaryStorageDirectory()).path; + let destDirectory; + let attachmentItem; + try { + let tmpFile = OS.Path.join(tmpDirectory, filename); + await Zotero.File.putNetworkStream(tmpFile, stream, options.byteCount); + + attachmentItem = await this.createURLAttachmentFromTemporaryStorageDirectory({ + directory: tmpDirectory, + libraryID, + parentItemID, + title, + filename, + url, + contentType, + collections, + saveOptions + }); + } + catch (e) { + Zotero.debug(e, 1); + + // Clean up + try { + if (tmpDirectory) { + await OS.File.removeDir(tmpDirectory, { ignoreAbsent: true }); + } + if (destDirectory) { + await OS.File.removeDir(destDirectory, { ignoreAbsent: true }); + } + } + catch (e) { + Zotero.debug(e, 1); + } + + throw e; + } + + return attachmentItem; + }; /** diff --git a/chrome/content/zotero/xpcom/collectionTreeRow.js b/chrome/content/zotero/xpcom/collectionTreeRow.js index 6d76d0003e..73cf0d8bb2 100644 --- a/chrome/content/zotero/xpcom/collectionTreeRow.js +++ b/chrome/content/zotero/xpcom/collectionTreeRow.js @@ -208,14 +208,14 @@ Zotero.CollectionTreeRow.prototype.__defineGetter__('filesEditable', function () } var libraryID = this.ref.libraryID; if (this.isGroup()) { - return this.ref.filesEditable; + return this.ref.editable && this.ref.filesEditable; } if (this.isCollection() || this.isSearch() || this.isDuplicates() || this.isUnfiled() || this.isRetracted()) { var type = Zotero.Libraries.get(libraryID).libraryType; if (type == 'group') { var groupID = Zotero.Groups.getGroupIDFromLibraryID(libraryID); var group = Zotero.Groups.get(groupID); - return group.filesEditable; + return group.editable && group.filesEditable; } throw ("Unknown library type '" + type + "' in Zotero.CollectionTreeRow.filesEditable"); } diff --git a/chrome/content/zotero/xpcom/connector/server_connector.js b/chrome/content/zotero/xpcom/connector/server_connector.js deleted file mode 100644 index 68d7a968f5..0000000000 --- a/chrome/content/zotero/xpcom/connector/server_connector.js +++ /dev/null @@ -1,1947 +0,0 @@ -/* - ***** BEGIN LICENSE BLOCK ***** - - Copyright © 2011 Center for History and New Media - George Mason University, Fairfax, Virginia, USA - http://zotero.org - - This file is part of Zotero. - - Zotero is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - Zotero is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with Zotero. If not, see . - - ***** END LICENSE BLOCK ***** -*/ -const CONNECTOR_API_VERSION = 2; - -Zotero.Server.Connector = { - _waitingForSelection: {}, - - getSaveTarget: function (allowReadOnly) { - var zp = Zotero.getActiveZoteroPane(); - var library = null; - var collection = null; - var editable = null; - - if (zp && zp.collectionsView) { - if (zp.collectionsView.editable || allowReadOnly) { - library = Zotero.Libraries.get(zp.getSelectedLibraryID()); - collection = zp.getSelectedCollection(); - editable = zp.collectionsView.editable; - } - // If not editable, switch to My Library if it exists and is editable - else { - let userLibrary = Zotero.Libraries.userLibrary; - if (userLibrary && userLibrary.editable) { - Zotero.debug("Save target isn't editable -- switching to My Library"); - - // Don't wait for this, because we don't want to slow down all conenctor - // requests by making this function async - zp.collectionsView.selectByID(userLibrary.treeViewID); - - library = userLibrary; - collection = null; - editable = true; - } - } - } - else { - let id = Zotero.Prefs.get('lastViewedFolder'); - if (id) { - ({ library, collection, editable } = this.resolveTarget(id)); - if (!editable && !allowReadOnly) { - let userLibrary = Zotero.Libraries.userLibrary; - if (userLibrary && userLibrary.editable) { - Zotero.debug("Save target isn't editable -- switching lastViewedFolder to My Library"); - let treeViewID = userLibrary.treeViewID; - Zotero.Prefs.set('lastViewedFolder', treeViewID); - ({ library, collection, editable } = this.resolveTarget(treeViewID)); - } - } - } - } - - // Default to My Library if present if pane not yet opened - // (which should never be the case anymore) - if (!library) { - let userLibrary = Zotero.Libraries.userLibrary; - if (userLibrary && userLibrary.editable) { - library = userLibrary; - } - } - - return { library, collection, editable }; - }, - - resolveTarget: function (targetID) { - var library; - var collection; - var editable; - - var type = targetID[0]; - var id = parseInt(('' + targetID).substr(1)); - - switch (type) { - case 'L': - library = Zotero.Libraries.get(id); - editable = library.editable; - break; - - case 'C': - collection = Zotero.Collections.get(id); - library = collection.library; - editable = collection.editable; - break; - - default: - throw new Error(`Unsupported target type '${type}'`); - } - - return { library, collection, editable }; - } -}; -Zotero.Server.Connector.Data = {}; - -Zotero.Server.Connector.SessionManager = { - _sessions: new Map(), - - get: function (id) { - return this._sessions.get(id); - }, - - create: function (id, action, requestData) { - // Legacy connector - if (!id) { - Zotero.debug("No session id provided by client", 2); - id = Zotero.Utilities.randomString(); - } - if (this._sessions.has(id)) { - throw new Error(`Session ID ${id} exists`); - } - Zotero.debug("Creating connector save session " + id); - var session = new Zotero.Server.Connector.SaveSession(id, action, requestData); - this._sessions.set(id, session); - this.gc(); - return session; - }, - - gc: function () { - // Delete sessions older than 10 minutes, or older than 1 minute if more than 10 sessions - var ttl = this._sessions.size >= 10 ? 60 : 600; - var deleteBefore = new Date() - ttl * 1000; - - for (let session of this._sessions) { - if (session.created < deleteBefore) { - this._session.delete(session.id); - } - } - } -}; - - -Zotero.Server.Connector.SaveSession = function (id, action, requestData) { - this.id = id; - this.created = new Date(); - this.savingDone = false; - this.pendingAttachments = []; - this._action = action; - this._requestData = requestData; - this._items = new Set(); - - this._progressItems = {}; - this._orderedProgressItems = []; -}; - - -Zotero.Server.Connector.SaveSession.prototype.addSnapshotContent = function (snapshotContent) { - this._requestData.data.snapshotContent = snapshotContent; -}; - - -Zotero.Server.Connector.SaveSession.prototype.onProgress = function (item, progress, error) { - if (item.id === null || item.id === undefined) { - throw new Error("ID not provided"); - } - - // Child item - if (item.parent) { - let progressItem = this._progressItems[item.parent]; - if (!progressItem) { - throw new Error(`Parent progress item ${item.parent} not found ` - + `for attachment ${item.id}`); - } - let a = progressItem.attachments.find(a => a.id == item.id); - if (!a) { - a = { - id: item.id - }; - progressItem.attachments.push(a); - } - a.title = item.title; - a.contentType = item.mimeType; - a.progress = progress; - return; - } - - // Top-level item - var o = this._progressItems[item.id]; - if (!o) { - o = { - id: item.id - }; - this._progressItems[item.id] = o; - this._orderedProgressItems.push(item.id); - } - o.title = item.title; - // PDF being converted to a top-level item after recognition - if (o.itemType == 'attachment' && item.itemType != 'attachment') { - delete o.progress; - delete o.contentType; - } - if (o.itemType === item.itemType) { - o.progress = progress; - return; - } - o.itemType = item.itemType; - o.attachments = item.attachments; -}; - -Zotero.Server.Connector.SaveSession.prototype.isSavingDone = function () { - return this.savingDone - || Object.values(this._progressItems).every(i => i.progress === 100 || typeof i.progress !== "number") - && Object.values(this._progressItems).every((i) => { - return !i.attachments || i.attachments.every(a => a.progress === 100 || typeof i.progress !== "number"); - }); -}; - -Zotero.Server.Connector.SaveSession.prototype.getProgressItem = function (id) { - return this._progressItems[id]; -}; - -Zotero.Server.Connector.SaveSession.prototype.getAllProgress = function () { - return this._orderedProgressItems.map(id => this._progressItems[id]); -}; - -Zotero.Server.Connector.SaveSession.prototype.addItem = async function (item) { - return this.addItems([item]); -}; - -Zotero.Server.Connector.SaveSession.prototype.addItems = async function (items) { - for (let item of items) { - this._items.add(item); - } - - // Update the items with the current target data, in case it changed since the save began - await this._updateItems(items); -}; - -Zotero.Server.Connector.SaveSession.prototype.remove = function () { - delete Zotero.Server.Connector.SessionManager._sessions[this.id]; -} - -/** - * Change the target data for this session and update any items that have already been saved - */ -Zotero.Server.Connector.SaveSession.prototype.update = async function (targetID, tags) { - var previousTargetID = this._currentTargetID; - this._currentTargetID = targetID; - this._currentTags = tags || ""; - - // Select new destination in collections pane - var zp = Zotero.getActiveZoteroPane(); - if (zp && zp.collectionsView) { - await zp.collectionsView.selectByID(targetID); - } - // If window is closed, select target collection re-open - else { - Zotero.Prefs.set('lastViewedFolder', targetID); - } - - // If moving from a non-filesEditable library to a filesEditable library, resave from - // original data, since there might be files that weren't saved or were removed - if (previousTargetID && previousTargetID != targetID) { - let { library: oldLibrary } = Zotero.Server.Connector.resolveTarget(previousTargetID); - let { library: newLibrary } = Zotero.Server.Connector.resolveTarget(targetID); - if (oldLibrary != newLibrary && !oldLibrary.filesEditable && newLibrary.filesEditable) { - Zotero.debug("Resaving items to filesEditable library"); - if (this._action == 'saveItems' || this._action == 'saveSnapshot') { - // Delete old items - for (let item of this._items) { - await item.eraseTx(); - } - let actionUC = Zotero.Utilities.capitalize(this._action); - // saveItems has a different signature with the session as the first argument - let params = [targetID, this._requestData]; - if (this._action == 'saveItems') { - params.unshift(this); - } - let newItems = await Zotero.Server.Connector[actionUC].prototype[this._action].apply( - Zotero.Server.Connector[actionUC], params - ); - // saveSnapshot only returns a single item - if (this._action == 'saveSnapshot') { - newItems = [newItems]; - } - this._items = new Set(newItems); - } - } - } - - await this._updateItems(this._items); - - // If a single item was saved, select it (or its parent, if it now has one) - if (zp && zp.collectionsView && this._items.size == 1) { - let item = Array.from(this._items)[0]; - item = item.isTopLevelItem() ? item : item.parentItem; - // Don't select if in trash - if (!item.deleted) { - await zp.selectItem(item.id); - } - } -}; - -/** - * Update the passed items with the current target and tags - */ -Zotero.Server.Connector.SaveSession.prototype._updateItems = Zotero.serial(async function (items) { - if (items.length == 0) { - return; - } - - var { library, collection, editable } = Zotero.Server.Connector.resolveTarget(this._currentTargetID); - var libraryID = library.libraryID; - - var tags = this._currentTags.trim(); - tags = tags ? tags.split(/\s*,\s*/).filter(x => x) : []; - - Zotero.debug("Updating items for connector save session " + this.id); - - for (let item of items) { - let newLibrary = Zotero.Libraries.get(library.libraryID); - - if (item.libraryID != libraryID) { - let newItem = await item.moveToLibrary(libraryID); - // Check pending attachments and switch parent ID - for (let i = 0; i < this.pendingAttachments.length; ++i) { - if (this.pendingAttachments[i][0] === item.id) { - this.pendingAttachments[i][0] = newItem.id; - } - } - // Replace item in session - this._items.delete(item); - this._items.add(newItem); - } - - // If the item is now a child item (e.g., from Retrieve Metadata), update the - // parent item instead - if (!item.isTopLevelItem()) { - item = item.parentItem; - } - // Skip deleted items - if (!Zotero.Items.exists(item.id)) { - Zotero.debug(`Item ${item.id} in save session no longer exists`); - continue; - } - // Keep automatic tags - let originalTags = item.getTags().filter(tag => tag.type == 1); - item.setTags(originalTags.concat(tags)); - item.setCollections(collection ? [collection.id] : []); - await item.saveTx(); - } - - this._updateRecents(); -}); - - -Zotero.Server.Connector.SaveSession.prototype._updateRecents = function () { - var targetID = this._currentTargetID; - try { - let numRecents = 7; - let recents = Zotero.Prefs.get('recentSaveTargets') || '[]'; - recents = JSON.parse(recents); - // If there's already a target from this session in the list, update it - for (let recent of recents) { - if (recent.sessionID == this.id) { - recent.id = targetID; - break; - } - } - // If a session is found with the same target, move it to the end without changing - // the sessionID. This could be the current session that we updated above or a different - // one. (We need to leave the old sessionID for the same target or we'll end up removing - // the previous target from the history if it's changed in the current one.) - let pos = recents.findIndex(r => r.id == targetID); - if (pos != -1) { - recents = [ - ...recents.slice(0, pos), - ...recents.slice(pos + 1), - recents[pos] - ]; - } - // Otherwise just add this one to the end - else { - recents = recents.concat([{ - id: targetID, - sessionID: this.id - }]); - } - recents = recents.slice(-1 * numRecents); - Zotero.Prefs.set('recentSaveTargets', JSON.stringify(recents)); - } - catch (e) { - Zotero.logError(e); - Zotero.Prefs.clear('recentSaveTargets'); - } -}; - - -/** - * Lists all available translators, including code for translators that should be run on every page - * - * Accepts: - * Nothing - * Returns: - * Array of Zotero.Translator objects - */ -Zotero.Server.Connector.GetTranslators = function() {}; -Zotero.Server.Endpoints["/connector/getTranslators"] = Zotero.Server.Connector.GetTranslators; -Zotero.Server.Connector.GetTranslators.prototype = { - supportedMethods: ["POST"], - supportedDataTypes: ["application/json"], - permitBookmarklet: true, - - /** - * Gets available translator list and other important data - * @param {Object} data POST data or GET query string - * @param {Function} sendResponseCallback function to send HTTP response - */ - init: function(data, sendResponseCallback) { - // Translator data - var me = this; - if(data.url) { - Zotero.Translators.getWebTranslatorsForLocation(data.url, data.url).then(function(data) { - sendResponseCallback(200, "application/json", - JSON.stringify(me._serializeTranslators(data[0]))); - }); - } else { - Zotero.Translators.getAll().then(function(translators) { - var responseData = me._serializeTranslators(translators); - sendResponseCallback(200, "application/json", JSON.stringify(responseData)); - }).catch(function(e) { - sendResponseCallback(500); - throw e; - }); - } - }, - - _serializeTranslators: function(translators) { - var responseData = []; - let properties = ["translatorID", "translatorType", "label", "creator", "target", "targetAll", - "minVersion", "maxVersion", "priority", "browserSupport", "inRepository", "lastUpdated"]; - for (var translator of translators) { - responseData.push(translator.serialize(properties)); - } - return responseData; - } -} - -/** - * Detects whether there is an available translator to handle a given page - * - * Accepts: - * uri - The URI of the page to be saved - * html - document.innerHTML or equivalent - * cookie - document.cookie or equivalent - * - * Returns a list of available translators as an array - */ -Zotero.Server.Connector.Detect = function() {}; -Zotero.Server.Endpoints["/connector/detect"] = Zotero.Server.Connector.Detect; -Zotero.Server.Connector.Detect.prototype = { - supportedMethods: ["POST"], - supportedDataTypes: ["application/json"], - permitBookmarklet: true, - - /** - * Loads HTML into a hidden browser and initiates translator detection - */ - init: async function(requestData) { - try { - var translators = await this.getTranslators(requestData); - } catch (e) { - Zotero.logError(e); - return 500; - } - - translators = translators.map(function(translator) { - return translator.serialize(TRANSLATOR_PASSING_PROPERTIES); - }); - return [200, "application/json", JSON.stringify(translators)]; - }, - - async getTranslators(requestData) { - var data = requestData.data; - var cookieSandbox = data.uri - ? new Zotero.CookieSandbox( - null, - data.uri, - data.cookie || "", - requestData.headers["User-Agent"] - ) - : null; - - var parser = new DOMParser(); - var doc = parser.parseFromString(`${data.html}`, 'text/html'); - doc = Zotero.HTTP.wrapDocument(doc, data.uri); - - let translate = this._translate = new Zotero.Translate.Web(); - translate.setDocument(doc); - cookieSandbox && translate.setCookieSandbox(cookieSandbox); - - return await translate.getTranslators(); - }, -} - -/** - * Performs translation of a given page - * - * Accepts: - * uri - The URI of the page to be saved - * html - document.innerHTML or equivalent - * cookie - document.cookie or equivalent - * translatorID [optional] - a translator ID as returned by /connector/detect - * - * Returns: - * If a single item, sends response code 201 with item in body. - * If multiple items, sends response code 300 with the following content: - * items - list of items in the format typically passed to the selectItems handler - * instanceID - an ID that must be maintained for the subsequent Zotero.Connector.Select call - * uri - the URI of the page for which multiple items are available - */ -Zotero.Server.Connector.SavePage = function() {}; -Zotero.Server.Endpoints["/connector/savePage"] = Zotero.Server.Connector.SavePage; -Zotero.Server.Connector.SavePage.prototype = { - supportedMethods: ["POST"], - supportedDataTypes: ["application/json"], - permitBookmarklet: true, - - /** - * Either loads HTML into a hidden browser and initiates translation, or saves items directly - * to the database - */ - init: function(requestData) { - return new Zotero.Promise(async function(resolve) { - function sendResponseCallback() { - if (arguments.length > 1) { - return resolve(arguments); - } - return resolve(arguments[0]); - } - var data = requestData.data; - var { library, collection, editable } = Zotero.Server.Connector.getSaveTarget(); - var libraryID = library.libraryID; - var targetID = collection ? collection.treeViewID : library.treeViewID; - - if (Zotero.Server.Connector.SessionManager.get(data.sessionID)) { - return sendResponseCallback(409, "application/json", JSON.stringify({ error: "SESSION_EXISTS" })); - } - - // Shouldn't happen as long as My Library exists - if (!library.editable) { - Zotero.logError("Can't add item to read-only library " + library.name); - return sendResponseCallback(500, "application/json", JSON.stringify({ libraryEditable: false })); - } - - var session = Zotero.Server.Connector.SessionManager.create(data.sessionID); - await session.update(targetID); - - this.sendResponse = sendResponseCallback; - this._parsedPostData = data; - - try { - var translators = await Zotero.Server.Connector.Detect.prototype.getTranslators.call(this, requestData); - } catch(e) { - Zotero.logError(e); - session.remove(); - return sendResponseCallback(500); - } - - if(!translators.length) { - Zotero.debug(`No translators available for /connector/savePage ${data.uri}`); - session.remove(); - return this.sendResponse(500); - } - - // set handlers for translation - var me = this; - var translate = this._translate; - translate.setHandler("select", function(obj, item, callback) { return me._selectItems(obj, item, callback) }); - let attachmentTitleData = {}; - translate.setHandler("itemsDone", function(obj, items) { - if(items.length || me.selectedItems === false) { - items = items.map((item) => { - let o = { - id: item.id, - title: item.title, - itemType: item.itemType, - contentType: item.mimeType, - mimeType: item.mimeType, // TODO: Remove - }; - if (item.attachments) { - let id = 0; - for (let attachment of item.attachments) { - attachment.parent = item.id; - attachment.id = id++; - } - o.attachments = item.attachments.map((attachment) => { - // Retaining id and parent info for session progress management - attachmentTitleData[attachment.title] = {id: attachment.id, parent: item.id}; - return { - id: session.id + '_' + attachment.id, // TODO: Remove prefix - title: attachment.title, - contentType: attachment.contentType, - mimeType: attachment.mimeType, // TODO: Remove - }; - }); - }; - session.onProgress(item, 100); - return o; - }); - me.sendResponse(201, "application/json", JSON.stringify({items})); - } else { - me.sendResponse(500); - session.remove(); - } - }); - - translate.setHandler("attachmentProgress", function(obj, attachment, progress, error) { - if (attachmentTitleData[attachment.title]) { - session.onProgress(Object.assign( - {}, - attachment, - attachmentTitleData[attachment.title], - ), progress, error); - } - }); - - translate.setHandler("error", function(obj, err) { - Zotero.logError(err); - sendResponseCallback(500); - session.remove(); - }); - - if (this._parsedPostData.translatorID) { - translate.setTranslator(this._parsedPostData.translatorID); - } else { - translate.setTranslator(translators[0]); - } - let items = await translate.translate({libraryID, collections: collection ? [collection.id] : false}); - session.addItems(items); - }.bind(this)); - }, - - /** - * Callback to be executed when items must be selected - * @param {Zotero.Translate} translate - * @param {Object} itemList ID=>text pairs representing available items - */ - _selectItems: function(translate, itemList, callback) { - var instanceID = Zotero.randomString(); - Zotero.Server.Connector._waitingForSelection[instanceID] = this; - - // Fix for translators that don't create item lists as objects - if(itemList.push && typeof itemList.push === "function") { - var newItemList = {}; - for(var item in itemList) { - newItemList[item] = itemList[item]; - } - itemList = newItemList; - } - - // Send "Multiple Choices" HTTP response - this.sendResponse(300, "application/json", JSON.stringify({selectItems: itemList, instanceID: instanceID, uri: this._parsedPostData.uri})); - this.selectedItemsCallback = callback; - } -} - -/** - * Handle item selection - * - * Accepts: - * selectedItems - a list of items to translate in ID => text format as returned by a selectItems handler - * instanceID - as returned by savePage call - * Returns: - * 201 response code with empty body - */ -Zotero.Server.Connector.SelectItems = function() {}; -Zotero.Server.Endpoints["/connector/selectItems"] = Zotero.Server.Connector.SelectItems; -Zotero.Server.Connector.SelectItems.prototype = { - supportedMethods: ["POST"], - supportedDataTypes: ["application/json"], - permitBookmarklet: true, - - /** - * Finishes up translation when item selection is complete - * @param {String} data POST data or GET query string - * @param {Function} sendResponseCallback function to send HTTP response - */ - init: function(data, sendResponseCallback) { - var saveInstance = Zotero.Server.Connector._waitingForSelection[data.instanceID]; - saveInstance.sendResponse = sendResponseCallback; - - var selectedItems = false; - for(var i in data.selectedItems) { - selectedItems = data.selectedItems; - break; - } - saveInstance.selectedItemsCallback(selectedItems); - } -} - -/** - * Saves items to DB - * - * Accepts: - * items - an array of JSON format items - * Returns: - * 201 response code with item in body. - */ -Zotero.Server.Connector.SaveItems = function() {}; -Zotero.Server.Endpoints["/connector/saveItems"] = Zotero.Server.Connector.SaveItems; -Zotero.Server.Connector.SaveItems.prototype = { - supportedMethods: ["POST"], - supportedDataTypes: ["application/json"], - permitBookmarklet: true, - - /** - * Either loads HTML into a hidden browser and initiates translation, or saves items directly - * to the database - */ - init: Zotero.Promise.coroutine(function* (requestData) { - var data = requestData.data; - - var { library, collection, editable } = Zotero.Server.Connector.getSaveTarget(); - var libraryID = library.libraryID; - var targetID = collection ? collection.treeViewID : library.treeViewID; - - try { - var session = Zotero.Server.Connector.SessionManager.create( - data.sessionID, - 'saveItems', - requestData - ); - } - catch (e) { - return [409, "application/json", JSON.stringify({ error: "SESSION_EXISTS" })]; - } - yield session.update(targetID); - - // Shouldn't happen as long as My Library exists - if (!library.editable) { - Zotero.logError("Can't add item to read-only library " + library.name); - return [500, "application/json", JSON.stringify({ libraryEditable: false })]; - } - - return new Zotero.Promise((resolve) => { - try { - this.saveItems( - session, - targetID, - requestData, - function (jsonItems, items) { - session.addItems(items); - let singleFile = false; - // Only return the properties the connector needs - jsonItems = jsonItems.map((item) => { - let o = { - id: item.id, - title: item.title, - itemType: item.itemType, - contentType: item.mimeType, - mimeType: item.mimeType, // TODO: Remove - }; - if (item.attachments) { - o.attachments = item.attachments.map((attachment) => { - if (attachment.singleFile) { - singleFile = true; - } - return { - id: session.id + '_' + attachment.id, // TODO: Remove prefix - title: attachment.title, - contentType: attachment.contentType, - mimeType: attachment.mimeType, // TODO: Remove - }; - }); - }; - return o; - }); - resolve([201, "application/json", JSON.stringify({ items: jsonItems, singleFile: singleFile })]); - } - ) - // Add items to session once all attachments have been saved - .then(function (items) { - session.addItems(items); - }); - } - catch (e) { - Zotero.logError(e); - session.remove(); - resolve(500); - } - }); - }), - - saveItems: async function (session, target, requestData, onTopLevelItemsDone) { - var { library, collection, editable } = Zotero.Server.Connector.resolveTarget(target); - var data = requestData.data; - var cookieSandbox = data.uri - ? new Zotero.CookieSandbox( - null, - data.uri, - data.detailedCookies ? "" : data.cookie || "", - requestData.headers["User-Agent"] - ) - : null; - if (cookieSandbox && data.detailedCookies) { - cookieSandbox.addCookiesFromHeader(data.detailedCookies); - } - - var id = 1; - for (let item of data.items) { - if (!item.id) { - item.id = id++; - } - - if (item.attachments) { - for (let attachment of item.attachments) { - attachment.id = id++; - attachment.parent = item.id; - } - } - - // Add parent item to session progress without attachments, which are added later if - // they're saved. - let progressItem = Object.assign( - {}, - item, - { - attachments: [] - } - ); - session.onProgress(progressItem, 0); - } - - var proxy = data.proxy && new Zotero.Proxy(data.proxy); - - // Save items - var itemSaver = new Zotero.Translate.ItemSaver({ - libraryID: library.libraryID, - collections: collection ? [collection.id] : undefined, - attachmentMode: Zotero.Translate.ItemSaver.ATTACHMENT_MODE_DOWNLOAD, - forceTagType: 1, - referrer: data.uri, - cookieSandbox, - proxy - }); - // This is a bit tricky. When saving items, the callback `onTopLevelItemsDone` will - // return the HTTP request to the connector. Then it may spend some time fetching - // PDFs. In the meantime, the connector will create a snapshot and send it along to - // the `saveSingleFile` endpoint, which quickly adds the data to the session and - // then saves the pending attachments, without removing them (we need them in case - // the session switches libraries and we need to save again). So the pending - // attachments exist and have already been saved by the time this `saveItems` - // promise resolves and we continue executing. So we save the number of existing - // attachments before that to prevent double saving. - let hadPendingAttachments = session.pendingAttachments.length > 0; - if (hadPendingAttachments) { - // If we have pending attachments then we are saving again by switching to - // a `filesEditable` library. So we clear the pendingAttachments since they - // get added again right below here in `saveItems` - session.pendingAttachments = []; - } - let items = await itemSaver.saveItems( - data.items, - function (attachment, progress, error) { - session.onProgress(attachment, progress, error); - }, - (itemsJSON, items) => { - itemsJSON.forEach(item => session.onProgress(item, 100)); - if (onTopLevelItemsDone) onTopLevelItemsDone(itemsJSON, items); - }, - function (parentItemID, attachment) { - session.pendingAttachments.push([parentItemID, attachment]); - } - ); - if (hadPendingAttachments) { - // If the session has snapshotContent already (from switching to a `filesEditable` library - // then we can save `pendingAttachments` now - if (data.snapshotContent) { - await itemSaver.saveSnapshotAttachments( - session.pendingAttachments, - data.snapshotContent, - function (attachment, progress, error) { - session.onProgress(attachment, progress, error); - }, - ); - } - // This means SingleFile in the Connector failed and we need to just go - // ahead and do our fallback save - else if (data.singleFile === false) { - itemSaver.saveSnapshotAttachments( - session.pendingAttachments, - false, - function (attachment, progress, error) { - session.onProgress(attachment, progress, error); - }, - ); - } - // Otherwise we are still waiting for SingleFile in Connector to finish - } - return items; - } -} - -/** - * Saves a snapshot to the DB - * - * Accepts: - * uri - The URI of the page to be saved - * html - document.innerHTML or equivalent - * cookie - document.cookie or equivalent - * Returns: - * Nothing (200 OK response) - */ -Zotero.Server.Connector.SaveSingleFile = function () {}; -Zotero.Server.Endpoints["/connector/saveSingleFile"] = Zotero.Server.Connector.SaveSingleFile; -Zotero.Server.Connector.SaveSingleFile.prototype = { - supportedMethods: ["POST"], - supportedDataTypes: ["application/json", "multipart/form-data"], - permitBookmarklet: true, - - /** - * Save SingleFile snapshot to pending attachments - */ - init: async function (requestData) { - const { HiddenBrowser } = ChromeUtils.import('chrome://zotero/content/HiddenBrowser.jsm'); - - // Retrieve payload - let data = requestData.data; - - // For a brief while the connector used SingleFileZ to save web pages, but we - // switched to using SingleFile. If the user is using that connector, the request - // will be multipart, which results in an array being passed in. In that case we - // want to mark this a legacySnapshot so we can ignore the snapshot content we have - // been given and save our own. This results in a double save, which takes a long - // time and is not ideal, but hopefully with auto-update of extensions it will be - // not be in use for many people for long. - let legacySnapshot = false; - if (Array.isArray(data)) { - legacySnapshot = true; - data = JSON.parse(Zotero.Utilities.Internal.decodeUTF8( - requestData.data.find(e => e.params.name === "payload").body - )); - } - - if (!data.sessionID) { - return [400, "application/json", JSON.stringify({ error: "SESSION_ID_NOT_PROVIDED" })]; - } - - let session = Zotero.Server.Connector.SessionManager.get(data.sessionID); - if (!session) { - Zotero.debug("Can't find session " + data.sessionID, 1); - return [400, "application/json", JSON.stringify({ error: "SESSION_NOT_FOUND" })]; - } - - let snapshotContent; - if (legacySnapshot) { - // Retrieve our snapshot content inside a hidden browser - let cookieSandbox = data.uri - ? new Zotero.CookieSandbox( - null, - data.uri, - data.detailedCookies ? "" : data.cookie || "", - requestData.headers["User-Agent"] - ) - : null; - if (cookieSandbox && data.detailedCookies) { - cookieSandbox.addCookiesFromHeader(data.detailedCookies); - } - - // Get the URL from the first pending attachment - if (!session.pendingAttachments.length) { - session.savingDone = true; - - return [200, 'text/plain', 'Legacy snapshot has no pending attachments.']; - } - - let url = session.pendingAttachments[0][1].url; - - let browser = new HiddenBrowser({ - docShell: { - allowImages: true - }, - cookieSandbox, - }); - await browser.load(url, { requireSuccessfulStatus: true }); - try { - snapshotContent = await browser.snapshot(); - } - finally { - browser.destroy(); - } - } - else { - snapshotContent = data.snapshotContent; - } - - if (!snapshotContent) { - // Connector SingleFile has failed so if we re-save attachments (via - // updateSession) then we want to inform saveItems and saveSnapshot that they - // do not need to use pendingAttachments because those have failed. - session._requestData.data.singleFile = false; - - for (let [_parentItemID, attachment] of session.pendingAttachments) { - session.onProgress(attachment, false); - } - - return [200, 'text/plain', 'No snapshot content attached.']; - } - - // Add to session data, in case `saveSnapshot` is called again by the session - session.addSnapshotContent(snapshotContent); - - // We do this after adding to session because if we switch to a `filesEditable` - // library we need to have access to the snapshotContent. - let { library, collection } = Zotero.Server.Connector.getSaveTarget(); - if (!library.filesEditable) { - session.savingDone = true; - - return [200, 'text/plain', 'Library is not editable.']; - } - - // Retrieve all items in the session that need a snapshot - if (session._action === 'saveSnapshot') { - await Zotero.Promise.all( - session.pendingAttachments.map((pendingAttachment) => { - return Zotero.Attachments.importFromSnapshotContent({ - title: data.title, - url: data.url, - parentItemID: pendingAttachment[0], - snapshotContent - }); - }) - ); - } - else if (session._action === 'saveItems') { - var cookieSandbox = data.uri - ? new Zotero.CookieSandbox( - null, - data.uri, - data.detailedCookies ? "" : data.cookie || "", - requestData.headers["User-Agent"] - ) - : null; - if (cookieSandbox && data.detailedCookies) { - cookieSandbox.addCookiesFromHeader(data.detailedCookies); - } - - let proxy = data.proxy && new Zotero.Proxy(data.proxy); - - let itemSaver = new Zotero.Translate.ItemSaver({ - libraryID: library.libraryID, - collections: collection ? [collection.id] : undefined, - attachmentMode: Zotero.Translate.ItemSaver.ATTACHMENT_MODE_DOWNLOAD, - forceTagType: 1, - referrer: data.uri, - cookieSandbox, - proxy - }); - - await itemSaver.saveSnapshotAttachments( - session.pendingAttachments, - snapshotContent, - function (attachment, progress, error) { - session.onProgress(attachment, progress, error); - }, - ); - } - - return 201; - } -}; - -/** - * Saves a snapshot to the DB - * - * Accepts: - * uri - The URI of the page to be saved - * html - document.innerHTML or equivalent - * cookie - document.cookie or equivalent - * Returns: - * Nothing (200 OK response) - */ -Zotero.Server.Connector.SaveSnapshot = function() {}; -Zotero.Server.Endpoints["/connector/saveSnapshot"] = Zotero.Server.Connector.SaveSnapshot; -Zotero.Server.Connector.SaveSnapshot.prototype = { - supportedMethods: ["POST"], - supportedDataTypes: ["application/json"], - permitBookmarklet: true, - - /** - * Save snapshot - */ - init: async function (requestData) { - var data = requestData.data; - - var { library, collection, editable } = Zotero.Server.Connector.getSaveTarget(); - var targetID = collection ? collection.treeViewID : library.treeViewID; - - try { - var session = Zotero.Server.Connector.SessionManager.create( - data.sessionID, - 'saveSnapshot', - requestData - ); - } - catch (e) { - return [409, "application/json", JSON.stringify({ error: "SESSION_EXISTS" })]; - } - await session.update(collection ? collection.treeViewID : library.treeViewID); - - // Shouldn't happen as long as My Library exists - if (!library.editable) { - Zotero.logError("Can't add item to read-only library " + library.name); - return [500, "application/json", JSON.stringify({ libraryEditable: false })]; - } - - try { - var item = await this.saveSnapshot(targetID, requestData); - await session.addItem(item); - } - catch (e) { - Zotero.logError(e); - return 500; - } - - let attachments = []; - let hasAttachments = !item.isAttachment() && item.getAttachments().length; - if (hasAttachments) { - attachments = [{mimeType: "text/html", title: data.title, url: data.url}]; - } - - return [201, - "application/json", - JSON.stringify({ saveSingleFile: !data.skipSnapshot && !data.pdf && data.singleFile, attachments })]; - }, - - /* - * Perform saving the snapshot - * - * Note: this function signature cannot change because it can also be called by - * updateSession (`Zotero.Server.Connector.SaveSession.prototype.update`). - */ - saveSnapshot: async function (target, requestData) { - var { library, collection, editable } = Zotero.Server.Connector.resolveTarget(target); - var libraryID = library.libraryID; - var data = requestData.data; - - var cookieSandbox = data.url - ? new Zotero.CookieSandbox( - null, - data.url, - data.detailedCookies ? "" : data.cookie || "", - requestData.headers["User-Agent"] - ) - : null; - if (cookieSandbox && data.detailedCookies) { - cookieSandbox.addCookiesFromHeader(data.detailedCookies); - } - - if (data.pdf && library.filesEditable) { - let item = await Zotero.Attachments.importFromURL({ - libraryID, - url: data.url, - referrer: data.referrer, - collections: collection ? [collection.id] : undefined, - contentType: "application/pdf", - cookieSandbox - }); - - // Automatically recognize PDF/EPUB - Zotero.RecognizeDocument.autoRecognizeItems([item]); - - return item; - } - - if (data.html) { - var parser = new DOMParser(); - var doc = parser.parseFromString(`${data.html}`, 'text/html'); - doc = Zotero.HTTP.wrapDocument(doc, data.url); - var title = doc.title; - } else { - title = data.title || data.url; - } - - // Create new webpage item - let item = new Zotero.Item("webpage"); - item.libraryID = libraryID; - item.setField("title", title); - item.setField("url", data.url); - item.setField("accessDate", "CURRENT_TIMESTAMP"); - if (collection) { - item.setCollections([collection.id]); - } - var itemID = await item.saveTx(); - - // Save snapshot - if (!data.skipSnapshot) { - // If called from session update, requestData may already have SingleFile data - if (library.filesEditable && data.snapshotContent) { - await Zotero.Attachments.importFromSnapshotContent({ - title: data.title, - url: data.url, - parentItemID: itemID, - snapshotContent: data.snapshotContent - }); - } - // Otherwise, connector will POST SingleFile data at later time - // We want this data regardless of `library.filesEditable` because if we - // start on a non-filesEditable library and switch to one, we won't have a - // pending attachment - else if (data.hasOwnProperty('singleFile')) { - let session = Zotero.Server.Connector.SessionManager.get(data.sessionID); - session.pendingAttachments = [ - [itemID, { title: data.title, url: data.url }] - ]; - } - else if (library.filesEditable) { - // Old connector will not use SingleFile so importFromURL now - await Zotero.Attachments.importFromURL({ - libraryID, - url: data.url, - referrer: data.referrer, - title, - parentItemID: itemID, - contentType: "text/html", - cookieSandbox - }); - } - } - - return item; - } -}; - -/** - * - * - * Accepts: - * sessionID - A session ID previously passed to /saveItems - * target - A treeViewID (L1, C23, etc.) for the library or collection to save to - * tags - A string of tags separated by commas - * - * Returns: - * 200 response on successful change - * 400 on error with 'error' property in JSON - */ -Zotero.Server.Connector.UpdateSession = function() {}; -Zotero.Server.Endpoints["/connector/updateSession"] = Zotero.Server.Connector.UpdateSession; -Zotero.Server.Connector.UpdateSession.prototype = { - supportedMethods: ["POST"], - supportedDataTypes: ["application/json"], - permitBookmarklet: true, - - init: async function (requestData) { - var data = requestData.data - - if (!data.sessionID) { - return [400, "application/json", JSON.stringify({ error: "SESSION_ID_NOT_PROVIDED" })]; - } - - var session = Zotero.Server.Connector.SessionManager.get(data.sessionID); - if (!session) { - Zotero.debug("Can't find session " + data.sessionID, 1); - return [400, "application/json", JSON.stringify({ error: "SESSION_NOT_FOUND" })]; - } - - // Parse treeViewID - var [type, id] = [data.target[0], parseInt(data.target.substr(1))]; - var tags = data.tags; - - if (type == 'C') { - let collection = await Zotero.Collections.getAsync(id); - if (!collection) { - return [400, "application/json", JSON.stringify({ error: "COLLECTION_NOT_FOUND" })]; - } - } - - await session.update(data.target, tags); - - return [200, "application/json", JSON.stringify({})]; - } -}; - -Zotero.Server.Connector.SessionProgress = function() {}; -Zotero.Server.Endpoints["/connector/sessionProgress"] = Zotero.Server.Connector.SessionProgress; -Zotero.Server.Connector.SessionProgress.prototype = { - supportedMethods: ["POST"], - supportedDataTypes: ["application/json"], - permitBookmarklet: true, - - init: async function (requestData) { - var data = requestData.data - - if (!data.sessionID) { - return [400, "application/json", JSON.stringify({ error: "SESSION_ID_NOT_PROVIDED" })]; - } - - var session = Zotero.Server.Connector.SessionManager.get(data.sessionID); - if (!session) { - Zotero.debug("Can't find session " + data.sessionID, 1); - return [400, "application/json", JSON.stringify({ error: "SESSION_NOT_FOUND" })]; - } - - return [ - 200, - "application/json", - JSON.stringify({ - items: session.getAllProgress() - .map((item) => { - var newItem = Object.assign({}, item); - if (item.attachments) { - newItem.attachments = item.attachments.map((attachment) => { - return Object.assign( - {}, - attachment, - // Prefix id with 'sessionID_' - // TODO: Remove this once support for /attachmentProgress is - // removed and we stop prefixing the ids in the /saveItems - // response - { - id: session.id + '_' + attachment.id - } - ); - }); - } - return newItem; - }), - done: session.isSavingDone() - }) - ]; - } -}; - -Zotero.Server.Connector.DelaySync = function () {}; -Zotero.Server.Endpoints["/connector/delaySync"] = Zotero.Server.Connector.DelaySync; -Zotero.Server.Connector.DelaySync.prototype = { - supportedMethods: ["POST"], - permitBookmarklet: true, - - init: function (requestData) { - Zotero.Sync.Runner.delaySync(10000); - return 204; - } -}; - -/** - * Gets progress for an attachment that is currently being saved - * - * Accepts: - * Array of attachment IDs returned by savePage, saveItems, or saveSnapshot - * Returns: - * 200 response code with current progress in body. Progress is either a number - * between 0 and 100 or "false" to indicate that saving failed. - */ -Zotero.Server.Connector.Progress = function() {}; -Zotero.Server.Endpoints["/connector/attachmentProgress"] = Zotero.Server.Connector.Progress; -Zotero.Server.Connector.Progress.prototype = { - supportedMethods: ["POST"], - supportedDataTypes: ["application/json"], - permitBookmarklet: true, - - /** - * @param {String} data POST data or GET query string - * @param {Function} sendResponseCallback function to send HTTP response - */ - init: function(data, sendResponseCallback) { - sendResponseCallback( - 200, - "application/json", - JSON.stringify( - data.map((id) => { - var [sessionID, progressID] = id.split('_'); - var session = Zotero.Server.Connector.SessionManager.get(sessionID); - var items = session.getAllProgress(); - for (let item of items) { - for (let attachment of item.attachments) { - // TODO: Change to progressID instead of id once we stop prepending - // the sessionID to support older connector versions - if (attachment.id == progressID) { - // TODO: Remove - return attachment.progress == -1 ? false : attachment.progress; - //return attachment.progress; - } - } - } - return null; - }) - ) - ); - } -}; - -/** - * Translates resources using import translators - * - * Returns: - * - Object[Item] an array of imported items - */ - -Zotero.Server.Connector.Import = function() {}; -Zotero.Server.Endpoints["/connector/import"] = Zotero.Server.Connector.Import; -Zotero.Server.Connector.Import.prototype = { - supportedMethods: ["POST"], - supportedDataTypes: '*', - permitBookmarklet: false, - - init: async function (requestData) { - let translate = new Zotero.Translate.Import(); - translate.setString(requestData.data); - let translators = await translate.getTranslators(); - if (!translators || !translators.length) { - return 400; - } - translate.setTranslator(translators[0]); - var { library, collection, editable } = Zotero.Server.Connector.getSaveTarget(); - var libraryID = library.libraryID; - - // Shouldn't happen as long as My Library exists - if (!library.editable) { - Zotero.logError("Can't import into read-only library " + library.name); - return [500, "application/json", JSON.stringify({ libraryEditable: false })]; - } - - try { - var session = Zotero.Server.Connector.SessionManager.create(requestData.searchParams.get('session')); - } - catch (e) { - return [409, "application/json", JSON.stringify({ error: "SESSION_EXISTS" })]; - } - await session.update(collection ? collection.treeViewID : library.treeViewID); - - let items = await translate.translate({ - libraryID, - collections: collection ? [collection.id] : null, - forceTagType: 1, - // Import translation skips selection by default, so force it to occur - saveOptions: { - skipSelect: false - } - }); - session.addItems(items); - - return [201, "application/json", JSON.stringify(items)]; - } -} - -/** - * Install CSL styles - * - * Returns: - * - {name: styleName} - */ - -Zotero.Server.Connector.InstallStyle = function() {}; -Zotero.Server.Endpoints["/connector/installStyle"] = Zotero.Server.Connector.InstallStyle; -Zotero.Server.Connector.InstallStyle.prototype = { - supportedMethods: ["POST"], - supportedDataTypes: '*', - permitBookmarklet: false, - - init: Zotero.Promise.coroutine(function* (requestData) { - try { - var { styleTitle, styleID } = yield Zotero.Styles.install( - requestData.data, requestData.searchParams.get('origin') || null, true - ); - } catch (e) { - return [400, "text/plain", e.message]; - } - return [201, "application/json", JSON.stringify({name: styleTitle})]; - }) -}; - -/** - * Get code for a translator - * - * Accepts: - * translatorID - * Returns: - * code - translator code - */ -Zotero.Server.Connector.GetTranslatorCode = function() {}; -Zotero.Server.Endpoints["/connector/getTranslatorCode"] = Zotero.Server.Connector.GetTranslatorCode; -Zotero.Server.Connector.GetTranslatorCode.prototype = { - supportedMethods: ["POST"], - supportedDataTypes: ["application/json"], - permitBookmarklet: true, - - /** - * Returns a 200 response to say the server is alive - * @param {String} data POST data or GET query string - * @param {Function} sendResponseCallback function to send HTTP response - */ - init: function(postData, sendResponseCallback) { - var translator = Zotero.Translators.get(postData.translatorID); - Zotero.Translators.getCodeForTranslator(translator).then(function(code) { - sendResponseCallback(200, "application/javascript", code); - }); - } -} - -/** - * Get selected collection - * - * Accepts: - * Nothing - * Returns: - * libraryID - * libraryName - * collectionID - * collectionName - */ -Zotero.Server.Connector.GetSelectedCollection = function() {}; -Zotero.Server.Endpoints["/connector/getSelectedCollection"] = Zotero.Server.Connector.GetSelectedCollection; -Zotero.Server.Connector.GetSelectedCollection.prototype = { - supportedMethods: ["POST"], - supportedDataTypes: ["application/json"], - permitBookmarklet: true, - - /** - * Returns a 200 response to say the server is alive - * @param {String} data POST data or GET query string - * @param {Function} sendResponseCallback function to send HTTP response - */ - init: function(postData, sendResponseCallback) { - var { library, collection, editable } = Zotero.Server.Connector.getSaveTarget(true); - var response = { - libraryID: library.libraryID, - libraryName: library.name, - libraryEditable: library.editable, - editable - }; - - if(collection && collection.id) { - response.id = collection.id; - response.name = collection.name; - } else { - response.id = null; - response.name = response.libraryName; - } - - // Get list of editable libraries and collections - var collections = []; - var originalLibraryID = library.libraryID; - for (let library of Zotero.Libraries.getAll()) { - if (!library.editable) continue; - - // Add recent: true for recent targets - - collections.push( - { - id: library.treeViewID, - name: library.name, - level: 0 - }, - ...Zotero.Collections.getByLibrary(library.libraryID, true).map(c => ({ - id: c.treeViewID, - name: c.name, - level: c.level + 1 || 1 // Added by Zotero.Collections._getByContainer() - })) - ); - } - response.targets = collections; - - // Mark recent targets - try { - let recents = Zotero.Prefs.get('recentSaveTargets'); - if (recents) { - recents = new Set(JSON.parse(recents).map(o => o.id)); - for (let target of response.targets) { - if (recents.has(target.id)) { - target.recent = true; - } - } - } - } - catch (e) { - Zotero.logError(e); - Zotero.Prefs.clear('recentSaveTargets'); - } - - sendResponseCallback( - 200, - "application/json", - JSON.stringify(response), - { - // Filter out collection names in debug output - logFilter: function (str) { - try { - let json = JSON.parse(str.match(/^{"libraryID"[^]+/m)[0]); - json.targets.forEach(t => t.name = "\u2026"); - return JSON.stringify(json); - } - catch (e) { - return str; - } - } - } - ); - } -} - -/** - * Get a list of client hostnames (reverse local IP DNS) - * - * Accepts: - * Nothing - * Returns: - * {Array} hostnames - */ -Zotero.Server.Connector.GetClientHostnames = {}; -Zotero.Server.Connector.GetClientHostnames = function() {}; -Zotero.Server.Endpoints["/connector/getClientHostnames"] = Zotero.Server.Connector.GetClientHostnames; -Zotero.Server.Connector.GetClientHostnames.prototype = { - supportedMethods: ["POST"], - supportedDataTypes: ["application/json"], - permitBookmarklet: false, - - /** - * Returns a 200 response to say the server is alive - */ - init: Zotero.Promise.coroutine(function* (requestData) { - try { - var hostnames = yield Zotero.Proxies.DNS.getHostnames(); - } catch(e) { - return 500; - } - return [200, "application/json", JSON.stringify(hostnames)]; - }) -}; - -/** - * Get a list of stored proxies - * - * Accepts: - * Nothing - * Returns: - * {Array} hostnames - */ -Zotero.Server.Connector.Proxies = {}; -Zotero.Server.Connector.Proxies = function() {}; -Zotero.Server.Endpoints["/connector/proxies"] = Zotero.Server.Connector.Proxies; -Zotero.Server.Connector.Proxies.prototype = { - supportedMethods: ["POST"], - supportedDataTypes: ["application/json"], - permitBookmarklet: false, - - /** - * Returns a 200 response to say the server is alive - */ - init: Zotero.Promise.coroutine(function* () { - let proxies = Zotero.Proxies.proxies.map((p) => Object.assign(p.toJSON(), {hosts: p.hosts})); - return [200, "application/json", JSON.stringify(proxies)]; - }) -}; - - -/** - * Test connection - * - * Accepts: - * Nothing - * Returns: - * Nothing (200 OK response) - */ -Zotero.Server.Connector.Ping = function() {}; -Zotero.Server.Endpoints["/connector/ping"] = Zotero.Server.Connector.Ping; -Zotero.Server.Connector.Ping.prototype = { - supportedMethods: ["GET", "POST"], - supportedDataTypes: ["application/json", "text/plain"], - permitBookmarklet: true, - - /** - * Sends 200 and HTML status on GET requests - * @param data {Object} request information defined in connector.js - */ - init: async function (req) { - if (req.method == 'GET') { - return [200, "text/html", '' - + 'Zotero is running']; - } else { - // Store the active URL so it can be used for site-specific Quick Copy - if (req.data.activeURL) { - //Zotero.debug("Setting active URL to " + req.data.activeURL); - Zotero.QuickCopy.lastActiveURL = req.data.activeURL; - } - let translatorsHash = await Zotero.Translators.getTranslatorsHash(false); - let sortedTranslatorHash = await Zotero.Translators.getTranslatorsHash(true); - - let response = { - prefs: { - automaticSnapshots: Zotero.Prefs.get('automaticSnapshots'), - googleDocsAddNoteEnabled: true, - translatorsHash, - sortedTranslatorHash - } - }; - if (Zotero.QuickCopy.hasSiteSettings()) { - response.prefs.reportActiveURL = true; - } - - this.versionWarning(req); - - return [200, 'application/json', JSON.stringify(response)]; - } - }, - - - /** - * Warn on outdated connector version - * - * We can remove this once the connector checks and warns on its own and most people are on - * a version that does that. - */ - versionWarning: function (req) { - try { - if (!Zotero.Prefs.get('showConnectorVersionWarning')) return; - if (!req.headers) return; - - var minVersion = ZOTERO_CONFIG.CONNECTOR_MIN_VERSION; - var appName = ZOTERO_CONFIG.CLIENT_NAME; - var domain = ZOTERO_CONFIG.DOMAIN_NAME; - var origin = req.headers.Origin; - - var browser; - var message; - var showDownloadButton = false; - // Legacy Safari extension - if (origin && origin.startsWith('safari-extension')) { - browser = 'safari'; - message = `An update is available for the ${appName} Connector for Safari.\n\n` - + 'You can upgrade from the Extensions pane of the Safari preferences.'; - } - else if (origin && origin.startsWith('chrome-extension')) { - browser = 'chrome'; - message = `An update is available for the ${appName} Connector for Chrome.\n\n` - + `You can upgrade to the latest version from ${domain}.`; - showDownloadButton = true; - } - else if (req.headers['User-Agent'] && req.headers['User-Agent'].includes('Firefox/')) { - browser = 'firefox'; - message = `An update is available for the ${appName} Connector for Firefox.\n\n` - + `You can upgrade to the latest version from ${domain}.`; - showDownloadButton = true; - } - // Safari App Extension is always up to date - else if (req.headers['User-Agent'] && req.headers['User-Agent'].includes('Safari/')) { - return; - } - else { - Zotero.debug("Unknown browser"); - return; - } - - if (Zotero.Server.Connector['skipVersionWarning-' + browser]) return; - - var version = req.headers['X-Zotero-Version']; - if (!version || version == '4.999.0') return; - - // If connector is up to date, bail - if (Services.vc.compare(version, minVersion) >= 0) return; - - var showNextPref = `nextConnectorVersionWarning.${browser}`; - var showNext = Zotero.Prefs.get(showNextPref); - if (showNext && new Date() < new Date(showNext * 1000)) return; - - // Don't show again for this browser until restart - Zotero.Server.Connector['skipVersionWarning-' + browser] = true; - var ps = Services.prompt; - var buttonFlags; - if (showDownloadButton) { - buttonFlags = ps.BUTTON_POS_0 * ps.BUTTON_TITLE_IS_STRING - + ps.BUTTON_POS_1 * ps.BUTTON_TITLE_IS_STRING; - } - else { - buttonFlags = ps.BUTTON_POS_0 * ps.BUTTON_TITLE_OK; - } - setTimeout(function () { - var dontShow = {}; - var index = ps.confirmEx(null, - Zotero.getString('general.updateAvailable'), - message, - buttonFlags, - showDownloadButton ? Zotero.getString('general.upgrade') : null, - showDownloadButton ? Zotero.getString('general.notNow') : null, - null, - "Don\u0027t show again for a month", - dontShow - ); - - var nextShowDays; - if (dontShow.value) { - nextShowDays = 30; - } - // Don't show again for at least a day, even after a restart - else { - nextShowDays = 1; - } - Zotero.Prefs.set(showNextPref, Math.round(Date.now() / 1000) + 86400 * nextShowDays); - - if (showDownloadButton && index == 0) { - Zotero.launchURL(ZOTERO_CONFIG.CONNECTORS_URL); - } - }, 500); - } - catch (e) { - Zotero.debug(e, 2); - } - } -} - -/** - * IE messaging hack - * - * Accepts: - * Nothing - * Returns: - * Static Response - */ -Zotero.Server.Connector.IEHack = function() {}; -Zotero.Server.Endpoints["/connector/ieHack"] = Zotero.Server.Connector.IEHack; -Zotero.Server.Connector.IEHack.prototype = { - supportedMethods: ["GET"], - permitBookmarklet: true, - - /** - * Sends a fixed webpage - * @param {String} data POST data or GET query string - * @param {Function} sendResponseCallback function to send HTTP response - */ - init: function(postData, sendResponseCallback) { - sendResponseCallback(200, "text/html", - ''+ - ''+ - ''+ - ''); - } -} - -/** - * Make an HTTP request from the client. Accepts {@link Zotero.HTTP.request} options and returns a minimal response - * object with the same form as the one returned from {@link Zotero.Utilities.Translate#request}. - * - * Accepts: - * method - The request method ('GET', 'POST', etc.) - * url - The URL to make the request to. Must be an absolute HTTP(S) URL. - * options - See Zotero.HTTP.request() documentation. Differences: - * - responseType is always set to 'text' - * - successCodes is always set to false (non-2xx status codes will not trigger an error) - * Returns: - * Response code is always 200. Body contains: - * status - The response status code, as a number - * headers - An object mapping header names to values - * body - The response body, as a string - */ -Zotero.Server.Connector.Request = function () {}; - -/** - * The list of allowed hosts. Intentionally hardcoded. - */ -Zotero.Server.Connector.Request.allowedHosts = ['www.worldcat.org']; - -/** - * For testing: allow disabling validation so we can make requests to the server. - */ -Zotero.Server.Connector.Request.enableValidation = false; - -Zotero.Server.Endpoints["/connector/request"] = Zotero.Server.Connector.Request; -Zotero.Server.Connector.Request.prototype = { - supportedMethods: ["POST"], - supportedDataTypes: ["application/json"], - - init: async function (req) { - let { method, url, options } = req.data; - - if (typeof method !== 'string' || typeof url !== 'string') { - return [400, 'text/plain', 'method and url are required and must be strings']; - } - - let uri; - try { - uri = Services.io.newURI(url); - } - catch (e) { - return [400, 'text/plain', 'Invalid URL']; - } - - if (uri.scheme != 'http' && uri.scheme != 'https') { - return [400, 'text/plain', 'Unsupported scheme']; - } - - if (Zotero.Server.Connector.Request.enableValidation) { - if (!Zotero.Server.Connector.Request.allowedHosts.includes(uri.host)) { - return [ - 400, - 'text/plain', - 'Unsupported URL' - ]; - } - - if (!req.headers['User-Agent'] || !req.headers['User-Agent'].startsWith('Mozilla/')) { - return [400, 'text/plain', 'Unsupported User-Agent']; - } - } - - options = options || {}; - options.responseType = 'text'; - options.successCodes = false; - - let xhr; - try { - xhr = await Zotero.HTTP.request(req.data.method, req.data.url, options); - } - catch (e) { - if (e instanceof Zotero.HTTP.BrowserOfflineException) { - return [503, 'text/plain', 'Client is offline']; - } - else { - throw e; - } - } - - let status = xhr.status; - let headers = {}; - xhr.getAllResponseHeaders() - .trim() - .split(/[\r\n]+/) - .map(line => line.split(': ')) - .forEach(parts => headers[parts.shift()] = parts.join(': ')); - let body = xhr.response; - - return [200, 'application/json', JSON.stringify({ - status, - headers, - body - })]; - } -}; diff --git a/chrome/content/zotero/xpcom/file.js b/chrome/content/zotero/xpcom/file.js index 3427a6e2cc..f59120414a 100644 --- a/chrome/content/zotero/xpcom/file.js +++ b/chrome/content/zotero/xpcom/file.js @@ -446,6 +446,68 @@ Zotero.File = new function(){ }); }; + /** + * Asynchronously writes data from an nsIAsyncInputStream to a file. + * + * Designed to handle input streams where data may not be + * immediately or fully available, such as network streams. + * + * @param {nsIInputStream} inputStream - The input stream to read from. This + * stream should implement nsIAsyncInputStream. + * @param {string} path - The file path where the data will be written. + * @param {number} byteCount - The expected number of bytes to write. + * + * @returns {Promise} A promise that resolves with the number of bytes + * written when the operation is complete, or rejects with an error + * if any issues occur during reading or writing. + */ + this.putNetworkStream = async function (path, stream, byteCount) { + return new Promise((resolve, reject) => { + let bytesRead = 0; + var os = FileUtils.openSafeFileOutputStream(new FileUtils.File(path)); + + let binaryInputStream = Cc["@mozilla.org/binaryinputstream;1"].createInstance(Ci.nsIBinaryInputStream); + binaryInputStream.setInputStream(stream); + + let readNextChunk = () => { + stream.asyncWait({ + onInputStreamReady: (input) => { + try { + // Check available data in the stream + let available = input.available(); + if (available > 0) { + os.write(binaryInputStream.readBytes(available), available); + bytesRead += available; + + if (bytesRead < byteCount) { + // Continue reading + readNextChunk(); + } + else { + // Finished writing all expected bytes + FileUtils.closeSafeFileOutputStream(os); + resolve(bytesRead); + } + } + else { + // No more data, finish the stream + FileUtils.closeSafeFileOutputStream(os); + resolve(bytesRead); + } + } + catch (e) { + os.close(); + reject(new Components.Exception("File write operation failed", e)); + } + } + }, 0, 0, null); + }; + + // Start reading the first chunk of data + readNextChunk(); + }); + }; + this.download = async function (uri, path) { var uriStr = uri.spec || uri; diff --git a/chrome/content/zotero/xpcom/connector/httpIntegrationClient.js b/chrome/content/zotero/xpcom/httpIntegrationClient.js similarity index 100% rename from chrome/content/zotero/xpcom/connector/httpIntegrationClient.js rename to chrome/content/zotero/xpcom/httpIntegrationClient.js diff --git a/chrome/content/zotero/xpcom/prompt.js b/chrome/content/zotero/xpcom/prompt.js index d2e0bf2571..cd4dcfa59d 100644 --- a/chrome/content/zotero/xpcom/prompt.js +++ b/chrome/content/zotero/xpcom/prompt.js @@ -73,7 +73,7 @@ Zotero.Prompt = { Zotero.warn("Zotero.Prompt.confirm() option 'delayButtons' is deprecated -- use 'buttonDelay'"); buttonDelay = true; } - let flags = (buttonDelay && !Zotero.automatedTest) ? Services.prompt.BUTTON_DELAY_ENABLE : 0; + let flags = (buttonDelay && !Zotero.test) ? Services.prompt.BUTTON_DELAY_ENABLE : 0; if (typeof button0 == 'number') flags += Services.prompt.BUTTON_POS_0 * button0; else if (typeof button0 == 'string') flags += Services.prompt.BUTTON_POS_0 * Services.prompt.BUTTON_TITLE_IS_STRING; if (typeof button1 == 'number') flags += Services.prompt.BUTTON_POS_1 * button1; diff --git a/chrome/content/zotero/xpcom/recognizeDocument.js b/chrome/content/zotero/xpcom/recognizeDocument.js index 3c65974df3..c6275fc847 100644 --- a/chrome/content/zotero/xpcom/recognizeDocument.js +++ b/chrome/content/zotero/xpcom/recognizeDocument.js @@ -58,8 +58,8 @@ Zotero.RecognizeDocument = new function () { async function _processQueue() { await Zotero.Schema.schemaUpdatePromise; - if (_queueProcessing) return; - _queueProcessing = true; + if (_queueProcessing) return _queueProcessing.promise; + _queueProcessing = Zotero.Promise.defer(); while (1) { // While all current progress queue usages are related with @@ -99,6 +99,7 @@ Zotero.RecognizeDocument = new function () { } } + _queueProcessing.resolve(); _queueProcessing = false; _processingItemID = null; } @@ -253,6 +254,7 @@ Zotero.RecognizeDocument = new function () { * @return {Promise} A promise that resolves to a newly created, recognized parent item */ async function _processItem(attachment) { + Zotero.debug(`RecognizeDocument: Recognizing attachment ${attachment.getDisplayTitle()}`); // Make sure the attachment still doesn't have a parent if (attachment.parentItemID) { throw new Error('Already has parent'); @@ -268,10 +270,12 @@ Zotero.RecognizeDocument = new function () { } } - let parentItem = await _recognize(attachment); + let parentItem = await Zotero.RecognizeDocument._recognize(attachment); if (!parentItem) { + Zotero.debug(`RecognizeDocument: No matches for attachment ${attachment.getDisplayTitle()}`); throw new Zotero.Exception.Alert("recognizePDF.noMatches"); } + Zotero.debug(`RecognizeDocument: Recognized attachment ${attachment.getDisplayTitle()}`); // Put new item in same collections as the old one let collections = attachment.getCollections(); @@ -388,11 +392,7 @@ Zotero.RecognizeDocument = new function () { * @param {Zotero.Item} item * @return {Promise} - New item */ - async function _recognize(item) { - if (Zotero.RecognizeDocument.recognizeStub) { - return Zotero.RecognizeDocument.recognizeStub(item); - } - + this._recognize = async function (item) { let filePath = await item.getFilePath(); if (!filePath || !await OS.File.exists(filePath)) throw new Zotero.Exception.Alert('recognizePDF.fileNotFound'); diff --git a/chrome/content/zotero/xpcom/server/saveSession.js b/chrome/content/zotero/xpcom/server/saveSession.js new file mode 100644 index 0000000000..ae0905a263 --- /dev/null +++ b/chrome/content/zotero/xpcom/server/saveSession.js @@ -0,0 +1,283 @@ +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2024 Corporation for Digital Scholarship + Vienna, Virginia, USA + http://zotero.org + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + +Zotero.Server.Connector.SessionManager = { + _sessions: new Map(), + + get: function (id) { + return this._sessions.get(id); + }, + + create: function (id, action, requestData) { + if (typeof id === 'undefined') { + id = Zotero.Utilities.randomString(); + } + if (this._sessions.has(id)) { + throw new Error(`Session ID ${id} exists`); + } + Zotero.debug(`Creating connector save session ${id}`); + var session = new Zotero.Server.Connector.SaveSession(id, action, requestData); + this._sessions.set(id, session); + this.gc(); + return session; + }, + + gc: function () { + // Delete sessions older than 10 minutes, or older than 1 minute if more than 10 sessions + var ttl = this._sessions.size >= 10 ? 60 : 600; + var deleteBefore = new Date() - ttl * 1000; + + for (let session of this._sessions) { + if (session.created < deleteBefore) { + this._session.delete(session.id); + } + } + } +}; + + + +Zotero.Server.Connector.SaveSession = class { + constructor(id, action, requestData) { + this.id = id; + this.created = new Date(); + this._action = action; + this._requestData = requestData; + this._items = {}; + + this._progressItems = {}; + this._orderedProgressItems = []; + } + + async saveItems(target) { + var { library, collection } = Zotero.Server.Connector.resolveTarget(target); + var data = this._requestData.data; + var headers = this._requestData.headers; + var cookieSandbox = data.uri + ? new Zotero.CookieSandbox( + null, + data.uri, + data.detailedCookies ? "" : data.cookie || "", + headers["User-Agent"] + ) + : null; + if (cookieSandbox && data.detailedCookies) { + cookieSandbox.addCookiesFromHeader(data.detailedCookies); + } + + var proxy = data.proxy && new Zotero.Proxy(data.proxy); + + this.itemSaver = new Zotero.Translate.ItemSaver({ + libraryID: library.libraryID, + collections: collection ? [collection.id] : undefined, + // All attachments come from the Connector + attachmentMode: Zotero.Translate.ItemSaver.ATTACHMENT_MODE_IGNORE, + forceTagType: 1, + referrer: data.uri, + cookieSandbox, + proxy + }); + let items = await this.itemSaver.saveItems(data.items, () => 0, () => 0); + // If more itemSaver calls are made, it means we are saving attachments explicitly (like + // a snapshot) and we don't want to ignore those. + this.itemSaver.attachmentMode = Zotero.Translate.ItemSaver.ATTACHMENT_MODE_DOWNLOAD; + items.forEach((item, index) => { + this.addItem(data.items[index].id, item); + }); + + return items; + } + + async saveSnapshot(target) { + var { library, collection } = Zotero.Server.Connector.resolveTarget(target); + var libraryID = library.libraryID; + var data = this._requestData.data; + + let title = data.title || data.url; + + // Create new webpage item + let item = new Zotero.Item("webpage"); + item.libraryID = libraryID; + item.setField("title", title); + item.setField("url", data.url); + item.setField("accessDate", "CURRENT_TIMESTAMP"); + if (collection) { + item.setCollections([collection.id]); + } + await item.saveTx(); + + // SingleFile snapshot may be coming later + this.addItem(data.url, item); + + return item; + } + + async addItem(key, item) { + return this.addItems({ [key]: item }); + } + + async addItems(items) { + this._items = Object.assign(this._items, items); + + // Update the items with the current target data, in case it changed since the save began + await this._updateItems(items); + } + + getItemByConnectorKey(key) { + return this._items[key]; + } + + // documentRecognizer doesn't return recognized items and it's complicated to make it + // do it, so we just retrieve the parent item which is a little hacky but does the job + getRecognizedItem() { + try { + return Object.values(this._items)[0].parentItem; + } + catch (_) {} + } + + remove() { + delete Zotero.Server.Connector.SessionManager._sessions[this.id]; + } + + /** + * Change the target data for this session and update any items that have already been saved + */ + async update(targetID, tags) { + var previousTargetID = this._currentTargetID; + this._currentTargetID = targetID; + this._currentTags = tags || ""; + + // Select new destination in collections pane + var zp = Zotero.getActiveZoteroPane(); + if (zp && zp.collectionsView) { + await zp.collectionsView.selectByID(targetID); + } + // If window is closed, select target collection re-open + else { + Zotero.Prefs.set('lastViewedFolder', targetID); + } + + await this._updateItems(this._items); + + // If a single item was saved, select it (or its parent, if it now has one) + if (zp && zp.collectionsView && Object.values(this._items).length == 1) { + let item = Object.values(this._items)[0]; + item = item.isTopLevelItem() ? item : item.parentItem; + // Don't select if in trash + if (!item.deleted) { + await zp.selectItem(item.id); + } + } + } + + /** + * Update the passed items with the current target and tags + */ + _updateItems = Zotero.serial(async function (items) { + if (Object.values(items).length == 0) { + return; + } + + var { library, collection } = Zotero.Server.Connector.resolveTarget(this._currentTargetID); + var libraryID = library.libraryID; + + var tags = this._currentTags.trim(); + tags = tags ? tags.split(/\s*,\s*/).filter(x => x) : []; + + Zotero.debug("Updating items for connector save session " + this.id); + + for (let key in items) { + let item = items[key]; + + // If the item is now a child item (e.g., from Retrieve Metadata), update the + // parent item instead + if (!item.isTopLevelItem()) { + item = item.parentItem; + } + + // Skip deleted items + if (!Zotero.Items.exists(item.id)) { + Zotero.debug(`Item ${item.id} in save session no longer exists`); + continue; + } + + if (item.libraryID != libraryID) { + let newItem = await item.moveToLibrary(libraryID); + this._items[key] = newItem; + } + + // Keep automatic tags + let originalTags = item.getTags().filter(tag => tag.type == 1); + item.setTags(originalTags.concat(tags)); + item.setCollections(collection ? [collection.id] : []); + await item.saveTx(); + } + + this._updateRecents(); + }); + + + _updateRecents() { + var targetID = this._currentTargetID; + try { + let numRecents = 7; + let recents = Zotero.Prefs.get('recentSaveTargets') || '[]'; + recents = JSON.parse(recents); + // If there's already a target from this session in the list, update it + for (let recent of recents) { + if (recent.sessionID == this.id) { + recent.id = targetID; + break; + } + } + // If a session is found with the same target, move it to the end without changing + // the sessionID. This could be the current session that we updated above or a different + // one. (We need to leave the old sessionID for the same target or we'll end up removing + // the previous target from the history if it's changed in the current one.) + let pos = recents.findIndex(r => r.id == targetID); + if (pos != -1) { + recents = [ + ...recents.slice(0, pos), + ...recents.slice(pos + 1), + recents[pos] + ]; + } + // Otherwise just add this one to the end + else { + recents = recents.concat([{ + id: targetID, + sessionID: this.id + }]); + } + recents = recents.slice(-1 * numRecents); + Zotero.Prefs.set('recentSaveTargets', JSON.stringify(recents)); + } + catch (e) { + Zotero.logError(e); + Zotero.Prefs.clear('recentSaveTargets'); + } + } +}; \ No newline at end of file diff --git a/chrome/content/zotero/xpcom/server.js b/chrome/content/zotero/xpcom/server/server.js similarity index 52% rename from chrome/content/zotero/xpcom/server.js rename to chrome/content/zotero/xpcom/server/server.js index bcdfac7ec6..3c4087de0e 100755 --- a/chrome/content/zotero/xpcom/server.js +++ b/chrome/content/zotero/xpcom/server/server.js @@ -23,6 +23,9 @@ ***** END LICENSE BLOCK ***** */ +var { HttpServer } = ChromeUtils.import("chrome://remote/content/server/HTTPD.jsm"); +Components.utils.import("resource://gre/modules/NetUtil.jsm"); + Zotero.Server = new function() { var _onlineObserverRegistered, serv; this.responseCodes = { @@ -42,50 +45,55 @@ Zotero.Server = new function() { 504:"Gateway Timeout" }; + Object.defineProperty(this, 'port', { + get() { + if (!serv) { + throw new Error('Server not initialized'); + } + return serv.identity.primaryPort; + } + }); + /** * initializes a very rudimentary web server */ - this.init = function(port, bindAllAddr, maxConcurrentConnections) { - if (Zotero.HTTP.browserIsOffline()) { - Zotero.debug('Browser is offline -- not initializing HTTP server'); - _registerOnlineObserver(); - return; - } - + this.init = function (port) { if(serv) { Zotero.debug("Already listening on port " + serv.port); return; } - // start listening on socket - serv = Components.classes["@mozilla.org/network/server-socket;1"] - .createInstance(Components.interfaces.nsIServerSocket); + port = port || Zotero.Prefs.get('httpServer.port'); try { - // bind to a random port on loopback only - serv.init(port ? port : Zotero.Prefs.get('httpServer.port'), !bindAllAddr, -1); - serv.asyncListen(Zotero.Server.SocketListener); - - Zotero.debug("HTTP server listening on "+(bindAllAddr ? "*": " 127.0.0.1")+":"+serv.port); + serv = new HttpServer(); + serv.registerPrefixHandler('/', this.handleRequest) + serv.start(port); + Zotero.debug(`HTTP server listening on 127.0.0.1:${serv.identity.primaryPort}`); + // Close port on Zotero shutdown (doesn't apply to translation-server) if (Zotero.addShutdownListener) { Zotero.addShutdownListener(this.close.bind(this)); } - } catch(e) { + } + catch (e) { Zotero.logError(e); Zotero.debug("Not initializing HTTP server"); serv = undefined; } - - _registerOnlineObserver() + }; + + this.handleRequest = function (request, response) { + let requestHandler = new Zotero.Server.RequestHandler(request, response); + return requestHandler.handleRequest(); } /** * releases bound port */ - this.close = function() { - if(!serv) return; - serv.close(); + this.close = function () { + if (!serv) return; + serv.stop(); serv = undefined; }; @@ -102,294 +110,100 @@ Zotero.Server = new function() { } return decodedData; } - - function _registerOnlineObserver() { - if (_onlineObserverRegistered) { - return; - } - - // Observer to enable the integration when we go online - var observer = { - observe: function(subject, topic, data) { - if (data == 'online') { - Zotero.Server.init(); +} + + +// A proxy headers class to make header retrieval case-insensitive +Zotero.Server.Headers = class { + constructor() { + return new Proxy(this, { + get(target, name, receiver) { + if (typeof name !== 'string') { + return Reflect.get(target, name, receiver); } + return Reflect.get(target, name.toLowerCase(), receiver); + }, + has(target, name, receiver) { + if (typeof name !== 'string') { + return Reflect.has(target, name, receiver); + } + return Reflect.has(target, name.toLowerCase(), receiver); + }, + set(target, name, value, receiver) { + return Reflect.set(target, name.toLowerCase(), value, receiver); } - }; - - var observerService = - Components.classes["@mozilla.org/observer-service;1"] - .getService(Components.interfaces.nsIObserverService); - observerService.addObserver(observer, "network:offline-status-changed", false); - - _onlineObserverRegistered = true; + }); } -} +}; -Zotero.Server.SocketListener = new function() { - this.onSocketAccepted = onSocketAccepted; - this.onStopListening = onStopListening; - - /* - * called when a socket is opened - */ - function onSocketAccepted(socket, transport) { - // get an input stream - var iStream = transport.openInputStream(0, 0, 0); - var oStream = transport.openOutputStream(Components.interfaces.nsITransport.OPEN_BLOCKING, 0, 0); - - var dataListener = new Zotero.Server.DataListener(iStream, oStream); - var pump = Components.classes["@mozilla.org/network/input-stream-pump;1"] - .createInstance(Components.interfaces.nsIInputStreamPump); - try { - pump.init(iStream, 0, 0, false); - } - catch (e) { - pump.init(iStream, -1, -1, 0, 0, false); - } - pump.asyncRead(dataListener, null); - } - - function onStopListening(serverSocket, status) { - Zotero.debug("HTTP server going offline"); - } -} -/* - * handles the actual acquisition of data - */ -Zotero.Server.DataListener = function(iStream, oStream) { - Components.utils.import("resource://gre/modules/NetUtil.jsm"); - this.header = ""; - this.headerFinished = false; - +Zotero.Server.networkStreamToString = function (stream, length) { + let data = NetUtil.readInputStreamToString(stream, length); + return Zotero.Utilities.Internal.decodeUTF8(data); +}; + + +Zotero.Server.RequestHandler = function (request, response) { this.body = ""; this.bodyLength = 0; - this.iStream = iStream; - this.oStream = oStream; - this.foundReturn = false; -} - -/* - * called when a request begins (although the request should have begun before - * the DataListener was generated) - */ -Zotero.Server.DataListener.prototype.onStartRequest = function(request) {} - -/* - * called when a request stops - */ -Zotero.Server.DataListener.prototype.onStopRequest = function(request, status) { - this.iStream.close(); - this.oStream.close(); -} - -/* - * called when new data is available - */ -Zotero.Server.DataListener.prototype.onDataAvailable = function (request, inputStream, offset, count) { - var readData = NetUtil.readInputStreamToString(inputStream, count); - - if(this.headerFinished) { // reading body - this.body += readData; - // check to see if data is done - this._bodyData(); - } else { // reading header - // see if there's a magic double return - var lineBreakIndex = readData.indexOf("\r\n\r\n"); - if(lineBreakIndex != -1) { - if(lineBreakIndex != 0) { - this.header += readData.substr(0, lineBreakIndex+4); - this.body = readData.substr(lineBreakIndex+4); - } - - this._headerFinished(); - return; - } - var lineBreakIndex = readData.indexOf("\n\n"); - if(lineBreakIndex != -1) { - if(lineBreakIndex != 0) { - this.header += readData.substr(0, lineBreakIndex+2); - this.body = readData.substr(lineBreakIndex+2); - } - - this._headerFinished(); - return; - } - if(this.header && this.header[this.header.length-1] == "\n" && - (readData[0] == "\n" || readData[0] == "\r")) { - if(readData.length > 1 && readData[1] == "\n") { - this.header += readData.substr(0, 2); - this.body = readData.substr(2); - } else { - this.header += readData[0]; - this.body = readData.substr(1); - } - - this._headerFinished(); - return; - } - this.header += readData; - } -} - -/* - * processes an HTTP header and decides what to do - */ -Zotero.Server.DataListener.prototype._headerFinished = function() { - this.headerFinished = true; - - Zotero.debug(this.header, 5); - - // Parse headers into this.headers with lowercase names - this.headers = {}; - var headerLines = this.header.trim().split(/\r\n/); - for (let line of headerLines) { - line = line.trim(); - let pos = line.indexOf(':'); - if (pos == -1) { - continue; - } - let k = line.substr(0, pos).toLowerCase(); - let v = line.substr(pos + 1).trim(); - this.headers[k] = v; - } - - if (this.headers.origin) { - this.origin = this.headers.origin; - } - else if (this.headers['zotero-bookmarklet']) { - this.origin = "https://www.zotero.org"; - } - - if (!Zotero.isServer) { - // Make sure the Host header is set to localhost/127.0.0.1 to prevent DNS rebinding attacks - const hostRe = /^(localhost|127\.0\.0\.1)(:[0-9]+)?$/i; - if (!hostRe.test(this.headers.host)) { - this._requestFinished(this._generateResponse(400, "text/plain", "Invalid Host header\n")); - return; - } - } - - // get first line of request - const methodRe = /^([A-Z]+) ([^ \r\n?]+)(\?[^ \r\n]+)?/; - var method = methodRe.exec(this.header); - - // get content-type - var contentType = this.headers['content-type']; - if (contentType) { - let splitContentType = contentType.split(/\s*;/); - this.contentType = splitContentType[0]; - } - - if(!method) { - this._requestFinished(this._generateResponse(400, "text/plain", "Invalid method specified\n")); - return; - } - - this.pathParams = {}; - if (Zotero.Server.Endpoints[method[2]]) { - this.endpoint = Zotero.Server.Endpoints[method[2]]; - } - else { - let router = new Zotero.Router(this.pathParams); - for (let [potentialTemplate, endpoint] of Object.entries(Zotero.Server.Endpoints)) { - if (!potentialTemplate.includes(':')) continue; - router.add(potentialTemplate, () => { - this.pathParams._endpoint = endpoint; - }, true, /* Do not allow missing params */ false); - } - if (router.run(method[2].split('?')[0])) { // Don't let parser handle query params - we do that already - this.endpoint = this.pathParams._endpoint; - delete this.pathParams._endpoint; - delete this.pathParams.url; - } - else { - this._requestFinished(this._generateResponse(404, "text/plain", "No endpoint found\n")); - return; - } - } - this.pathname = method[2]; - this.query = method[3]; - - if(method[1] == "HEAD" || method[1] == "OPTIONS") { - this._requestFinished(this._generateResponse(200)); - } else if(method[1] == "GET") { - this._processEndpoint("GET", null); // async - } else if(method[1] == "POST") { - const contentLengthRe = /^([0-9]+)$/; - - // parse content length - var m = contentLengthRe.exec(this.headers['content-length']); - if(!m) { - this._requestFinished(this._generateResponse(400, "text/plain", "Content-length not provided\n")); - return; - } - - this.bodyLength = parseInt(m[1]); - this._bodyData(); - } else { - this._requestFinished(this._generateResponse(501, "text/plain", "Method not implemented\n")); - return; - } + this.request = request; + this.response = response; } /* * checks to see if Content-Length bytes of body have been read and, if so, processes the body */ -Zotero.Server.DataListener.prototype._bodyData = function() { - if(this.body.length >= this.bodyLength) { - let logContentTypes = [ - 'text/plain', - 'application/json' - ]; +Zotero.Server.RequestHandler.prototype._bodyData = function () { + const PLAIN_TEXT_CONTENT_TYPES = new Set([ + 'text/plain', + 'application/json', + 'application/x-www-form-urlencoded' + ]); + + let data = null; + if (this.bodyLength > 0) { + if (PLAIN_TEXT_CONTENT_TYPES.has(this.contentType)) { + this.body = data = Zotero.Server.networkStreamToString(this.request.bodyInputStream, this.bodyLength); + } + else if (this.contentType === 'multipart/form-data') { + data = NetUtil.readInputStreamToString(this.request.bodyInputStream, this.bodyLength); + try { + data = this._decodeMultipartData(data); + } + catch (e) { + return this._requestFinished(this._generateResponse(400, "text/plain", "Invalid multipart/form-data provided\n")); + } + } + } + if (this.body.length >= this.bodyLength) { let noLogEndpoints = [ '/connector/saveSingleFile' ]; if (this.body != '{}' - && logContentTypes.includes(this.contentType) + && PLAIN_TEXT_CONTENT_TYPES.has(this.contentType) && !noLogEndpoints.includes(this.pathname)) { Zotero.debug(Zotero.Utilities.ellipsize(this.body, 1000, false, true), 5); } - // handle envelope - this._processEndpoint("POST", this.body); // async } + // handle envelope + this._processEndpoint("POST", data); // async } /** * Generates the response to an HTTP request */ -Zotero.Server.DataListener.prototype._generateResponse = function (status, contentTypeOrHeaders, body) { +Zotero.Server.RequestHandler.prototype._generateResponse = function (status, contentTypeOrHeaders, body) { var response = "HTTP/1.0 "+status+" "+Zotero.Server.responseCodes[status]+"\r\n"; - - // Translation server - if (Zotero.isServer) { - // Add CORS headers if Origin header matches the allowed origins - if (this.origin) { - let allowedOrigins = Zotero.Prefs.get('httpServer.allowedOrigins') - .split(/, */).filter(x => x); - let allAllowed = allowedOrigins.includes('*'); - if (allAllowed || allowedOrigins.includes(this.origin)) { - response += "Access-Control-Allow-Origin: " + (allAllowed ? '*' : this.origin) + "\r\n"; - response += "Access-Control-Allow-Methods: POST, GET, OPTIONS\r\n"; - response += "Access-Control-Allow-Headers: Content-Type\r\n"; - response += "Access-Control-Expose-Headers: Link\r\n"; - } - } - } - // Client - else { - response += "X-Zotero-Version: "+Zotero.version+"\r\n"; - response += "X-Zotero-Connector-API-Version: "+CONNECTOR_API_VERSION+"\r\n"; + response += "X-Zotero-Version: "+Zotero.version+"\r\n"; + response += "X-Zotero-Connector-API-Version: "+CONNECTOR_API_VERSION+"\r\n"; - if (this.origin === ZOTERO_CONFIG.BOOKMARKLET_ORIGIN) { - response += "Access-Control-Allow-Origin: " + this.origin + "\r\n"; - response += "Access-Control-Allow-Methods: POST, GET, OPTIONS\r\n"; - response += "Access-Control-Allow-Headers: Content-Type,X-Zotero-Connector-API-Version,X-Zotero-Version\r\n"; - } + if (this.origin === ZOTERO_CONFIG.BOOKMARKLET_ORIGIN) { + response += "Access-Control-Allow-Origin: " + this.origin + "\r\n"; + response += "Access-Control-Allow-Methods: POST, GET, OPTIONS\r\n"; + response += "Access-Control-Allow-Headers: Content-Type,X-Zotero-Connector-API-Version,X-Zotero-Version\r\n"; } if (contentTypeOrHeaders) { @@ -403,7 +217,7 @@ Zotero.Server.DataListener.prototype._generateResponse = function (status, conte } } - if(body) { + if (body) { response += "\r\n"+body; } else { response += "Content-Length: 0\r\n\r\n"; @@ -412,34 +226,98 @@ Zotero.Server.DataListener.prototype._generateResponse = function (status, conte return response; } +Zotero.Server.RequestHandler.prototype.handleRequest = async function () { + const request = this.request; + const response = this.response; + // Tell httpd that we will be constructing our own response + // without its custom methods, asynchronously + response.seizePower(); + + let requestDebug = `${request.method} ${request.path} HTTP/${request.httpVersion}\n` + // Parse headers into this.headers with lowercase names + this.headers = new Zotero.Server.Headers(); + for (let { data: name } of request.headers) { + requestDebug += `${name}: ${request.getHeader(name)}\n`; + this.headers[name.toLowerCase()] = request.getHeader(name); + } + + Zotero.debug(requestDebug, 5); + + if (this.headers.origin) { + this.origin = this.headers.origin; + } + + this.pathname = request.path; + this.query = request.queryString; + + // get content-type + var contentType = this.headers['content-type']; + if (contentType) { + let splitContentType = contentType.split(/\s*;/); + this.contentType = splitContentType[0]; + } + + this.pathParams = {}; + if (Zotero.Server.Endpoints[this.pathname]) { + this.endpoint = Zotero.Server.Endpoints[this.pathname]; + } + else { + let router = new Zotero.Router(this.pathParams); + for (let [potentialTemplate, endpoint] of Object.entries(Zotero.Server.Endpoints)) { + if (!potentialTemplate.includes(':')) continue; + router.add(potentialTemplate, () => { + this.pathParams._endpoint = endpoint; + }, true, /* Do not allow missing params */ false); + } + if (router.run(this.pathname)) { + this.endpoint = this.pathParams._endpoint; + delete this.pathParams._endpoint; + delete this.pathParams.url; + } + else { + this._requestFinished(this._generateResponse(404, "text/plain", "No endpoint found\n")); + return; + } + } + + if (request.method == "HEAD" || request.method == "OPTIONS") { + this._requestFinished(this._generateResponse(200)); + } + else if (request.method == "GET") { + this._processEndpoint("GET", null); // async + } + else if (request.method == "POST") { + const contentLengthRe = /^([0-9]+)$/; + + // parse content length + var m = contentLengthRe.exec(this.headers['content-length']); + if(!m) { + this._requestFinished(this._generateResponse(400, "text/plain", "Content-length not provided\n")); + return; + } + + this.bodyLength = parseInt(m[1]); + this._bodyData(); + } else { + this._requestFinished(this._generateResponse(501, "text/plain", "Method not implemented\n")); + } +} + /** * Generates a response based on calling the function associated with the endpoint * * Note: postData contains raw bytes and should be decoded before use */ -Zotero.Server.DataListener.prototype._processEndpoint = Zotero.Promise.coroutine(function* (method, postData) { +Zotero.Server.RequestHandler.prototype._processEndpoint = async function (method, postData) { try { var endpoint = new this.endpoint; // Check that endpoint supports method - if(endpoint.supportedMethods && endpoint.supportedMethods.indexOf(method) === -1) { + if (endpoint.supportedMethods && endpoint.supportedMethods.indexOf(method) === -1) { this._requestFinished(this._generateResponse(400, "text/plain", "Endpoint does not support method\n")); return; } - // Check that endpoint supports bookmarklet - if(this.origin) { - var isBookmarklet = this.origin === "https://www.zotero.org" || this.origin === "http://www.zotero.org"; - // Disallow bookmarklet origins to access endpoints without permitBookmarklet - // set. We allow other origins to access these endpoints because they have to - // be privileged to avoid being blocked by our headers. - if(isBookmarklet && !endpoint.permitBookmarklet) { - this._requestFinished(this._generateResponse(403, "text/plain", "Access forbidden to bookmarklet\n")); - return; - } - } - - // Reject browser-based requests that don't require a CORS preflight request [1] if they // don't come from the connector or include Zotero-Allowed-Request // @@ -471,53 +349,44 @@ Zotero.Server.DataListener.prototype._processEndpoint = Zotero.Promise.coroutine return; } - var decodedData = null; - if(postData && this.contentType) { + var data = null; + if (method === 'POST' && this.contentType) { // check that endpoint supports contentType var supportedDataTypes = endpoint.supportedDataTypes; - if(supportedDataTypes && supportedDataTypes != '*' + if (supportedDataTypes && supportedDataTypes != '*' && supportedDataTypes.indexOf(this.contentType) === -1) { this._requestFinished(this._generateResponse(400, "text/plain", "Endpoint does not support content-type\n")); return; } // decode content-type post data - if(this.contentType === "application/json") { + if (this.contentType === "application/json") { try { - postData = Zotero.Utilities.Internal.decodeUTF8(postData); - decodedData = JSON.parse(postData); - } catch(e) { + data = JSON.parse(postData); + } + catch(e) { this._requestFinished(this._generateResponse(400, "text/plain", "Invalid JSON provided\n")); return; } - } else if(this.contentType === "application/x-www-form-urlencoded") { - postData = Zotero.Utilities.Internal.decodeUTF8(postData); - decodedData = Zotero.Server.decodeQueryString(postData); - } else if(this.contentType === "multipart/form-data") { - let boundary = /boundary=([^\s]*)/i.exec(this.header); - if (!boundary) { - Zotero.debug('Invalid boundary: ' + this.header, 1); - return this._requestFinished(this._generateResponse(400, "text/plain", "Invalid multipart/form-data provided\n")); - } - boundary = '--' + boundary[1]; - try { - decodedData = this._decodeMultipartData(postData, boundary); - } catch(e) { - return this._requestFinished(this._generateResponse(400, "text/plain", "Invalid multipart/form-data provided\n")); - } - } else { - postData = Zotero.Utilities.Internal.decodeUTF8(postData); - decodedData = postData; + } + else if (this.contentType === "application/x-www-form-urlencoded") { + data = Zotero.Server.decodeQueryString(postData); + } + else if (postData) { + data = postData; + } + else { + data = this.request.bodyInputStream; } } // set up response callback - var sendResponseCallback = function (code, contentTypeOrHeaders, arg, options) { + var sendResponseCallback = (code, contentTypeOrHeaders, arg, options) => { this._requestFinished( this._generateResponse(code, contentTypeOrHeaders, arg), options ); - }.bind(this); + }; // Pass to endpoint // @@ -528,30 +397,18 @@ Zotero.Server.DataListener.prototype._processEndpoint = Zotero.Promise.coroutine if (endpoint.init.length === 1 // Return value from Zotero.Promise.coroutine() || endpoint.init.length === 0) { - let headers = {}; - let headerLines = this.header.trim().split(/\r\n/); - for (let line of headerLines) { - line = line.trim(); - let pos = line.indexOf(':'); - if (pos == -1) { - continue; - } - let k = line.substr(0, pos); - let v = line.substr(pos + 1).trim(); - headers[k] = v; - } let maybePromise = endpoint.init({ method, pathname: this.pathname, pathParams: this.pathParams, - searchParams: new URLSearchParams(this.query ? this.query.substring(1) : ''), - headers, - data: decodedData + searchParams: new URLSearchParams(this.query || ''), + headers: this.headers, + data }); let result; if (maybePromise.then) { - result = yield maybePromise; + result = await maybePromise; } else { result = maybePromise; @@ -565,71 +422,75 @@ Zotero.Server.DataListener.prototype._processEndpoint = Zotero.Promise.coroutine } // Two-parameter endpoint takes data and a callback else if (endpoint.init.length === 2) { - endpoint.init(decodedData, sendResponseCallback); + endpoint.init(data, sendResponseCallback); } // Three-parameter endpoint takes a URL, data, and a callback else { - const uaRe = /[\r\n]User-Agent: +([^\r\n]+)/i; - var m = uaRe.exec(this.header); var url = { pathname: this.pathname, - searchParams: new URLSearchParams(this.query ? this.query.substring(1) : ''), - userAgent: m && m[1] + searchParams: new URLSearchParams(this.query || ''), + userAgent: this.headers['user-agent'] }; - endpoint.init(url, decodedData, sendResponseCallback); + endpoint.init(url, data, sendResponseCallback); } } catch(e) { Zotero.debug(e); this._requestFinished(this._generateResponse(500), "text/plain", "An error occurred\n"); throw e; } -}); +}; /* * returns HTTP data from a request */ -Zotero.Server.DataListener.prototype._requestFinished = function (response, options) { - if(this._responseSent) { +Zotero.Server.RequestHandler.prototype._requestFinished = function (responseBody, options) { + if (this._responseSent) { Zotero.debug("Request already finished; not sending another response"); return; } this._responseSent = true; - // close input stream - this.iStream.close(); - // open UTF-8 converter for output stream var intlStream = Components.classes["@mozilla.org/intl/converter-output-stream;1"] .createInstance(Components.interfaces.nsIConverterOutputStream); // write try { - intlStream.init(this.oStream, "UTF-8", 1024, "?".charCodeAt(0)); + intlStream.init(this.response.bodyOutputStream, "UTF-8", 1024, "?".charCodeAt(0)); // Filter logged response if (Zotero.Debug.enabled) { let maxLogLength = 2000; - let str = response; + let str = responseBody; if (options && options.logFilter) { str = options.logFilter(str); } if (str.length > maxLogLength) { - str = str.substr(0, maxLogLength) + `\u2026 (${response.length} chars)`; + str = str.substr(0, maxLogLength) + `\u2026 (${responseBody.length} chars)`; } Zotero.debug(str, 5); } - intlStream.writeString(response); - } finally { - intlStream.close(); + intlStream.writeString(responseBody); + } + finally { + this.response.finish(); } } -Zotero.Server.DataListener.prototype._decodeMultipartData = function(data, boundary) { - var contentDispositionRe = /^Content-Disposition:\s*(.*)$/i; - let contentTypeRe = /^Content-Type:\s*(.*)$/i +Zotero.Server.RequestHandler.prototype._decodeMultipartData = function(data) { + const contentDispositionRe = /^Content-Disposition:\s*(.*)$/i; + const contentTypeRe = /^Content-Type:\s*(.*)$/i - var results = []; + let results = []; + + let boundary = /boundary=([^\s]*)/i.exec(this.headers['content-type']); + if (!boundary) { + Zotero.debug('Invalid boundary: ' + this.headers['content-type'], 1); + return this._requestFinished(this._generateResponse(400, "text/plain", "Invalid multipart/form-data provided\n")); + } + boundary = '--' + boundary[1]; + data = data.split(boundary); // Ignore pre first boundary and post last boundary data = data.slice(1, data.length-1); diff --git a/chrome/content/zotero/xpcom/server/server_connector.js b/chrome/content/zotero/xpcom/server/server_connector.js new file mode 100644 index 0000000000..99fe70e960 --- /dev/null +++ b/chrome/content/zotero/xpcom/server/server_connector.js @@ -0,0 +1,1240 @@ +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2011 Center for History and New Media + George Mason University, Fairfax, Virginia, USA + http://zotero.org + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ +const CONNECTOR_API_VERSION = 2; + +Zotero.Server.Connector = { + _waitingForSelection: {}, + + getSaveTarget: function (allowReadOnly, allowFilesReadOnly=true) { + var zp = Zotero.getActiveZoteroPane(); + var library = null; + var collection = null; + var editable = null; + + if (zp && zp.collectionsView) { + if (allowReadOnly || zp.collectionsView.editable && allowFilesReadOnly || zp.collectionsView.filesEditable) { + library = Zotero.Libraries.get(zp.getSelectedLibraryID()); + collection = zp.getSelectedCollection(); + editable = zp.collectionsView.editable; + } + // If not editable, switch to My Library if it exists and is editable + else { + let userLibrary = Zotero.Libraries.userLibrary; + if (userLibrary && userLibrary.editable) { + Zotero.debug("Save target isn't editable -- switching to My Library"); + + // Don't wait for this, because we don't want to slow down all conenctor + // requests by making this function async + zp.collectionsView.selectByID(userLibrary.treeViewID); + + library = userLibrary; + collection = null; + editable = true; + } + } + } + else { + let id = Zotero.Prefs.get('lastViewedFolder'); + if (id) { + ({ library, collection, editable } = this.resolveTarget(id)); + if (!editable && !allowReadOnly) { + let userLibrary = Zotero.Libraries.userLibrary; + if (userLibrary && userLibrary.editable) { + Zotero.debug("Save target isn't editable -- switching lastViewedFolder to My Library"); + let treeViewID = userLibrary.treeViewID; + Zotero.Prefs.set('lastViewedFolder', treeViewID); + ({ library, collection, editable } = this.resolveTarget(treeViewID)); + } + } + } + } + + // Default to My Library if present if pane not yet opened + // (which should never be the case anymore) + if (!library) { + let userLibrary = Zotero.Libraries.userLibrary; + if (userLibrary && userLibrary.editable) { + library = userLibrary; + } + } + + return { library, collection, editable }; + }, + + resolveTarget: function (targetID) { + var library; + var collection; + var editable; + + var type = targetID[0]; + var id = parseInt(('' + targetID).substr(1)); + + switch (type) { + case 'L': + library = Zotero.Libraries.get(id); + editable = library.editable; + break; + + case 'C': + collection = Zotero.Collections.get(id); + library = collection.library; + editable = collection.editable; + break; + + default: + throw new Error(`Unsupported target type '${type}'`); + } + + return { library, collection, editable }; + } +}; + +/** + * Lists all available translators, including code for translators that should be run on every page + * + * Accepts: + * Nothing + * Returns: + * Array of Zotero.Translator objects + */ +Zotero.Server.Connector.GetTranslators = function() {}; +Zotero.Server.Endpoints["/connector/getTranslators"] = Zotero.Server.Connector.GetTranslators; +Zotero.Server.Connector.GetTranslators.prototype = { + supportedMethods: ["POST"], + supportedDataTypes: ["application/json"], + permitBookmarklet: true, + + /** + * Gets available translator list and other important data + * @param {Object} data POST data or GET query string + * @param {Function} sendResponseCallback function to send HTTP response + */ + init: function(data, sendResponseCallback) { + // Translator data + var me = this; + if(data.url) { + Zotero.Translators.getWebTranslatorsForLocation(data.url, data.url).then(function(data) { + sendResponseCallback(200, "application/json", + JSON.stringify(me._serializeTranslators(data[0]))); + }); + } else { + Zotero.Translators.getAll().then(function(translators) { + var responseData = me._serializeTranslators(translators); + sendResponseCallback(200, "application/json", JSON.stringify(responseData)); + }).catch(function(e) { + sendResponseCallback(500); + throw e; + }); + } + }, + + _serializeTranslators: function(translators) { + var responseData = []; + let properties = ["translatorID", "translatorType", "label", "creator", "target", "targetAll", + "minVersion", "maxVersion", "priority", "browserSupport", "inRepository", "lastUpdated"]; + for (var translator of translators) { + responseData.push(translator.serialize(properties)); + } + return responseData; + } +} + +/** + * Detects whether there is an available translator to handle a given page + * + * Accepts: + * uri - The URI of the page to be saved + * html - document.innerHTML or equivalent + * cookie - document.cookie or equivalent + * + * Returns a list of available translators as an array + */ +Zotero.Server.Connector.Detect = function() {}; +Zotero.Server.Endpoints["/connector/detect"] = Zotero.Server.Connector.Detect; +Zotero.Server.Connector.Detect.prototype = { + supportedMethods: ["POST"], + supportedDataTypes: ["application/json"], + permitBookmarklet: true, + + /** + * Loads HTML into a hidden browser and initiates translator detection + */ + init: async function(requestData) { + try { + var translators = await this.getTranslators(requestData); + } catch (e) { + Zotero.logError(e); + return 500; + } + + translators = translators.map(function(translator) { + return translator.serialize(TRANSLATOR_PASSING_PROPERTIES); + }); + return [200, "application/json", JSON.stringify(translators)]; + }, + + async getTranslators(requestData) { + var data = requestData.data; + var cookieSandbox = data.uri + ? new Zotero.CookieSandbox( + null, + data.uri, + data.cookie || "", + requestData.headers["User-Agent"] + ) + : null; + + var parser = new DOMParser(); + var doc = parser.parseFromString(`${data.html}`, 'text/html'); + doc = Zotero.HTTP.wrapDocument(doc, data.uri); + + let translate = this._translate = new Zotero.Translate.Web(); + translate.setDocument(doc); + cookieSandbox && translate.setCookieSandbox(cookieSandbox); + + return await translate.getTranslators(); + }, +} + +/** + * Saves items to DB + * + * Accepts: + * items - an array of JSON format items + * Returns: + * 201 response code with item in body. + */ +Zotero.Server.Connector.SaveItems = function() {}; +Zotero.Server.Endpoints["/connector/saveItems"] = Zotero.Server.Connector.SaveItems; +Zotero.Server.Connector.SaveItems.prototype = { + supportedMethods: ["POST"], + supportedDataTypes: ["application/json"], + permitBookmarklet: true, + + /** + * Either loads HTML into a hidden browser and initiates translation, or saves items directly + * to the database + */ + init: async function (requestData) { + var data = requestData.data; + + var { library, collection, editable } = Zotero.Server.Connector.getSaveTarget(); + var libraryID = library.libraryID; + var targetID = collection ? collection.treeViewID : library.treeViewID; + + try { + var session = Zotero.Server.Connector.SessionManager.create( + data.sessionID, + 'saveItems', + requestData + ); + } + catch (e) { + Zotero.debug(e); + return [409, "application/json", JSON.stringify({ error: "SESSION_EXISTS" })]; + } + await session.update(targetID); + + // Shouldn't happen as long as My Library exists + if (!library.editable) { + Zotero.logError("Can't add item to read-only library " + library.name); + return [500, "application/json", JSON.stringify({ libraryEditable: false })]; + } + + try { + await session.saveItems(targetID); + return [201, "application/json"]; + } + catch (e) { + Zotero.logError(e); + session.remove(); + return 500; + } + }, +} + +/** + * Gets the top-level item created for a standalone attachment + * + * Accepts: + * sessionID - A session ID previously passed to /saveItems + * Returns: + * 200 + */ +Zotero.Server.Connector.GetRecognizedItem = function () {}; +Zotero.Server.Endpoints["/connector/getRecognizedItem"] = Zotero.Server.Connector.GetRecognizedItem; +Zotero.Server.Connector.GetRecognizedItem.prototype = { + supportedMethods: ["POST"], + supportedDataTypes: ["*"], + permitBookmarklet: true, + + init: async function (requestData) { + const sessionID = requestData.data.sessionID; + if (!sessionID) { + return [400, "application/json", JSON.stringify({ error: "SESSION_ID_NOT_PROVIDED" })]; + } + + const session = Zotero.Server.Connector.SessionManager.get(sessionID); + if (!session) { + Zotero.debug("Can't find session " + sessionID, 1); + return [400, "application/json", JSON.stringify({ error: "SESSION_NOT_FOUND" })]; + } + + await session.autoRecognizePromise; + let item = session.getRecognizedItem(); + if (!item) { + return 204; + } + let jsonItem = { + title: item.getDisplayTitle(), + itemType: item.itemType, + }; + return [200, "application/json", JSON.stringify({ ...jsonItem })]; + } +}; + + +/** + * Saves a standalone attachment + * + * URI params: + * sessionID + * Expected headers: + * X-Metadata: + * - parentItemID + * - title + * - url + * Returns: + * 400 - Bad params + * 200 - Non-writable library + * 201 - Created + */ +Zotero.Server.Connector.SaveStandaloneAttachment = function () {}; +Zotero.Server.Endpoints["/connector/saveStandaloneAttachment"] = Zotero.Server.Connector.SaveStandaloneAttachment; +Zotero.Server.Connector.SaveStandaloneAttachment.prototype = { + supportedMethods: ["POST"], + supportedDataTypes: ["*"], + permitBookmarklet: true, + + init: async function (requestData) { + // Retrieve payload + if (!requestData.headers['X-Metadata']) { + return [400, "application/json", JSON.stringify({ error: "METADATA_NOT_PROVIDED" })]; + } + const metadata = JSON.parse(requestData.headers['X-Metadata']); + + const sessionID = metadata.sessionID || requestData.searchParams.get('sessionID'); + if (!sessionID) { + return [400, "application/json", JSON.stringify({ error: "SESSION_ID_NOT_PROVIDED" })]; + } + var { library, collection } = Zotero.Server.Connector.getSaveTarget(false, false); + var libraryID = library.libraryID; + var targetID = collection ? collection.treeViewID : library.treeViewID; + + try { + var session = Zotero.Server.Connector.SessionManager.create( + sessionID, + 'saveStandaloneAttachment', + requestData + ); + } + catch (e) { + return [409, "application/json", JSON.stringify({ error: "SESSION_EXISTS" })]; + } + await session.update(targetID); + + // Save standalone attachment from stream + let item = await Zotero.Attachments.importFromNetworkStream({ + url: metadata.url, + libraryID, + collections: collection ? [collection.id] : undefined, + title: metadata.title, + contentType: requestData.headers['Content-Type'], + stream: requestData.data, + byteCount: requestData.headers['Content-Length'], + }); + session.addItem(metadata.url, item); + + let canRecognize = Zotero.RecognizeDocument.canRecognize(item); + if (canRecognize) { + // Automatically recognize PDF/EPUB + session.autoRecognizePromise = Zotero.RecognizeDocument.autoRecognizeItems([item]); + } + return [201, "application/json", JSON.stringify({ canRecognize })]; + } +}; + +/** + * Attaches an PDF/EPUB attachment to an item saved with /saveItems or /saveSnapshot + * + * URI params: + * sessionID + * Expected headers: + * X-Metadata: + * - parentItemID + * - title + * - url + * Returns: + * 400 - Bad params + * 200 - Non-writable library + * 201 - Created + */ +Zotero.Server.Connector.SaveAttachment = function () {}; +Zotero.Server.Endpoints["/connector/saveAttachment"] = Zotero.Server.Connector.SaveAttachment; +Zotero.Server.Connector.SaveAttachment.prototype = { + supportedMethods: ["POST"], + supportedDataTypes: ["*"], + permitBookmarklet: true, + + init: async function (requestData) { + // Retrieve payload + if (!requestData.headers['X-Metadata']) { + return [400, "application/json", JSON.stringify({ error: "METADATA_NOT_PROVIDED" })]; + } + const metadata = JSON.parse(requestData.headers['X-Metadata']); + + const sessionID = metadata.sessionID || requestData.searchParams.get('sessionID'); + if (!sessionID) { + return [400, "application/json", JSON.stringify({ error: "SESSION_ID_NOT_PROVIDED" })]; + } + + let session = Zotero.Server.Connector.SessionManager.get(sessionID); + if (!session) { + Zotero.debug("Can't find session " + sessionID, 1); + return [400, "application/json", JSON.stringify({ error: "SESSION_NOT_FOUND" })]; + } + + let { library } = Zotero.Server.Connector.getSaveTarget(); + if (!library.filesEditable) { + return [200, 'text/plain', 'Library files are not editable.']; + } + + // Save attachment based on provided parent id from stream + let parentItem = session.getItemByConnectorKey(metadata.parentItemID); + await Zotero.Attachments.importFromNetworkStream({ + url: metadata.url, + parentItemID: parentItem.id, + title: metadata.title, + contentType: requestData.headers['Content-Type'], + stream: requestData.data, + byteCount: requestData.headers['Content-Length'], + }); + + return 201; + } +}; + + +/** + * Attaches a singlefile attachment to an item saved with /saveItems or /saveSnapshot + * If data.snapshotContent is empty, it means the save failed in the Connector + * And we fallback to saving in Zotero + * + * Accepts: + * sessionID + * snapshotContent + * url - The URI of the page to be saved + * title + * cookie - document.cookie or equivalent + * detailedCookies + * proxy + * Returns: + * Nothing (200 OK response) + */ +Zotero.Server.Connector.SaveSingleFile = function () {}; +Zotero.Server.Endpoints["/connector/saveSingleFile"] = Zotero.Server.Connector.SaveSingleFile; +Zotero.Server.Connector.SaveSingleFile.prototype = { + supportedMethods: ["POST"], + supportedDataTypes: ["application/json", "multipart/form-data"], + permitBookmarklet: true, + + /** + * Save SingleFile snapshot to pending attachments + */ + init: async function (requestData) { + // Retrieve payload + let data = requestData.data; + + if (!data.sessionID) { + return [400, "application/json", JSON.stringify({ error: "SESSION_ID_NOT_PROVIDED" })]; + } + + let session = Zotero.Server.Connector.SessionManager.get(data.sessionID); + if (!session) { + Zotero.debug("Can't find session " + data.sessionID, 1); + return [400, "application/json", JSON.stringify({ error: "SESSION_NOT_FOUND" })]; + } + + let { library } = Zotero.Server.Connector.getSaveTarget(); + if (!library.filesEditable) { + return [200, 'text/plain', 'Library files are not editable.']; + } + + // We only save the snapshot in single-item cases + if (session._action === 'saveSnapshot') { + const parentItemID = session.getItemByConnectorKey(data.url).id; + // Just saves the snapshot straight up + await Zotero.Attachments.importFromSnapshotContent({ + title: data.title, + url: data.url, + parentItemID, + snapshotContent: data.snapshotContent + }); + } + else if (session._action === 'saveItems') { + const parentItemID = session.getItemByConnectorKey(data.items[0].id).id; + // Deproxifies and does some other attachment preprocessing + await session.itemSaver.saveSnapshotAttachments({ + title: data.title, + url: data.url, + parentItemID, + snapshotContent: data.snapshotContent + }); + } + + return 201; + } +}; + +/** + * Creates a webpage item top-level item in Zotero + * Called by the Connector when no translators are detected on the page + * + * Accepts: + * uri - The URI of the page to be saved + * html - document.innerHTML or equivalent + * cookie - document.cookie or equivalent + * Returns: + * Nothing (200 OK response) + */ +Zotero.Server.Connector.SaveSnapshot = function() {}; +Zotero.Server.Endpoints["/connector/saveSnapshot"] = Zotero.Server.Connector.SaveSnapshot; +Zotero.Server.Connector.SaveSnapshot.prototype = { + supportedMethods: ["POST"], + supportedDataTypes: ["application/json"], + permitBookmarklet: true, + + /** + * Save snapshot + */ + init: async function (requestData) { + var data = requestData.data; + + var { library, collection } = Zotero.Server.Connector.getSaveTarget(); + var targetID = collection ? collection.treeViewID : library.treeViewID; + + try { + var session = Zotero.Server.Connector.SessionManager.create( + data.sessionID, + 'saveSnapshot', + requestData + ); + } + catch (e) { + Zotero.debug(e); + return [409, "application/json", JSON.stringify({ error: "SESSION_EXISTS" })]; + } + await session.update(collection ? collection.treeViewID : library.treeViewID); + + // Shouldn't happen as long as My Library exists + if (!library.editable) { + Zotero.logError("Can't add item to read-only library " + library.name); + return [500, "application/json", JSON.stringify({ libraryEditable: false })]; + } + + try { + await session.saveSnapshot(targetID); + } + catch (e) { + Zotero.logError(e); + return 500; + } + + return [201, "application/json"]; + } +}; + + +/** + * Checks if the item has OA attachments (in case PDF saving in connector failed). + * Also checks custom resolvers. + * + * Accepts: + * sessionID - A session ID previously passed to /saveItems + * itemID - The ID of the item to save alternative attachment for + */ +Zotero.Server.Connector.HasAttachmentResolvers = function () {}; +Zotero.Server.Endpoints["/connector/hasAttachmentResolvers"] = Zotero.Server.Connector.HasAttachmentResolvers; +Zotero.Server.Connector.HasAttachmentResolvers.prototype = { + supportedMethods: ["POST"], + supportedDataTypes: ["application/json"], + permitBookmarklet: true, + init: async function (requestData) { + let data = requestData.data; + let session = Zotero.Server.Connector.SessionManager.get(data.sessionID); + if (!session) { + Zotero.debug("Can't find session " + data.sessionID, 1); + return [400, "application/json", JSON.stringify({ error: "SESSION_NOT_FOUND" })]; + } + let item = session.getItemByConnectorKey(data.itemID); + let resolvers = Zotero.Attachments.getFileResolvers(item, ['oa', 'custom'], true); + return [200, "application/json", JSON.stringify(resolvers.length > 0)]; + } +} + + +/** + * Accepts: + * sessionID - A session ID previously passed to /saveItems + * itemID - The ID of the item to save alternative attachment for + * + * Returns: + * 400 - Bad params + * 201 - Created and attachment title + * 500 - Failed to save + */ +Zotero.Server.Connector.SaveAttachmentFromResolver = function () {}; +Zotero.Server.Endpoints["/connector/saveAttachmentFromResolver"] = Zotero.Server.Connector.SaveAttachmentFromResolver; +Zotero.Server.Connector.SaveAttachmentFromResolver.prototype = { + supportedMethods: ["POST"], + supportedDataTypes: ["application/json"], + permitBookmarklet: true, + init: async function (requestData) { + let data = requestData.data; + let session = Zotero.Server.Connector.SessionManager.get(data.sessionID); + if (!session) { + Zotero.debug("Can't find session " + data.sessionID, 1); + return [400, "application/json", JSON.stringify({ error: "SESSION_NOT_FOUND" })]; + } + let item = session.getItemByConnectorKey(data.itemID); + let resolvers = Zotero.Attachments.getFileResolvers(item, ['oa', 'custom'], true); + + let attachment = await Zotero.Attachments.addFileFromURLs(item, resolvers); + + if (attachment) { + return [201, "text/plain", attachment.getDisplayTitle()]; + } + else { + return [500, "text/plain", "Failed to save an attachment"]; + } + } +} + +/** + * + * + * Accepts: + * sessionID - A session ID previously passed to /saveItems + * target - A treeViewID (L1, C23, etc.) for the library or collection to save to + * tags - A string of tags separated by commas + * + * Returns: + * 200 response on successful change + * 400 on error with 'error' property in JSON + */ +Zotero.Server.Connector.UpdateSession = function() {}; +Zotero.Server.Endpoints["/connector/updateSession"] = Zotero.Server.Connector.UpdateSession; +Zotero.Server.Connector.UpdateSession.prototype = { + supportedMethods: ["POST"], + supportedDataTypes: ["application/json"], + permitBookmarklet: true, + + init: async function (requestData) { + var data = requestData.data + + if (!data.sessionID) { + return [400, "application/json", JSON.stringify({ error: "SESSION_ID_NOT_PROVIDED" })]; + } + + var session = Zotero.Server.Connector.SessionManager.get(data.sessionID); + if (!session) { + Zotero.debug("Can't find session " + data.sessionID, 1); + return [400, "application/json", JSON.stringify({ error: "SESSION_NOT_FOUND" })]; + } + + // Parse treeViewID + var [type, id] = [data.target[0], parseInt(data.target.substr(1))]; + var tags = data.tags; + + if (type == 'C') { + let collection = await Zotero.Collections.getAsync(id); + if (!collection) { + return [400, "application/json", JSON.stringify({ error: "COLLECTION_NOT_FOUND" })]; + } + } + + await session.update(data.target, tags); + + return [200, "application/json", JSON.stringify({})]; + } +}; + + +Zotero.Server.Connector.DelaySync = function () {}; +Zotero.Server.Endpoints["/connector/delaySync"] = Zotero.Server.Connector.DelaySync; +Zotero.Server.Connector.DelaySync.prototype = { + supportedMethods: ["POST"], + permitBookmarklet: true, + + init: function (requestData) { + Zotero.Sync.Runner.delaySync(10000); + return 204; + } +}; + +/** + * Translates resources using import translators + * + * Returns: + * - Object[Item] an array of imported items + */ + +Zotero.Server.Connector.Import = function() {}; +Zotero.Server.Endpoints["/connector/import"] = Zotero.Server.Connector.Import; +Zotero.Server.Connector.Import.prototype = { + supportedMethods: ["POST"], + supportedDataTypes: '*', + permitBookmarklet: false, + + init: async function (requestData) { + let dataString = requestData.data; + if (requestData.data instanceof Ci.nsIInputStream) { + dataString = Zotero.Server.networkStreamToString(dataString, requestData.headers['content-length']); + } + let translate = new Zotero.Translate.Import(); + translate.setString(dataString); + let translators = await translate.getTranslators(); + if (!translators || !translators.length) { + return 400; + } + translate.setTranslator(translators[0]); + var { library, collection, editable } = Zotero.Server.Connector.getSaveTarget(); + var libraryID = library.libraryID; + + // Shouldn't happen as long as My Library exists + if (!library.editable) { + Zotero.logError("Can't import into read-only library " + library.name); + return [500, "application/json", JSON.stringify({ libraryEditable: false })]; + } + + try { + var session = Zotero.Server.Connector.SessionManager.create(requestData.searchParams.get('session')); + } + catch (e) { + Zotero.debug(e); + return [409, "application/json", JSON.stringify({ error: "SESSION_EXISTS" })]; + } + await session.update(collection ? collection.treeViewID : library.treeViewID); + + let items = await translate.translate({ + libraryID, + collections: collection ? [collection.id] : null, + forceTagType: 1, + // Import translation skips selection by default, so force it to occur + saveOptions: { + skipSelect: false + } + }); + items.forEach((item, index) => { + session.addItem(items[index].id, item); + }); + + return [201, "application/json", JSON.stringify(items)]; + } +} + +/** + * Install CSL styles + * + * Returns: + * - {name: styleName} + */ + +Zotero.Server.Connector.InstallStyle = function() {}; +Zotero.Server.Endpoints["/connector/installStyle"] = Zotero.Server.Connector.InstallStyle; +Zotero.Server.Connector.InstallStyle.prototype = { + supportedMethods: ["POST"], + supportedDataTypes: '*', + permitBookmarklet: false, + + init: Zotero.Promise.coroutine(function* (requestData) { + let dataString = requestData.data; + if (requestData.data instanceof Ci.nsIInputStream) { + dataString = Zotero.Server.networkStreamToString(dataString, requestData.headers['content-length']); + } + try { + var { styleTitle } = yield Zotero.Styles.install( + dataString, requestData.searchParams.get('origin') || null, true + ); + } catch (e) { + return [400, "text/plain", e.message]; + } + return [201, "application/json", JSON.stringify({name: styleTitle})]; + }) +}; + +/** + * Get code for a translator + * + * Accepts: + * translatorID + * Returns: + * code - translator code + */ +Zotero.Server.Connector.GetTranslatorCode = function() {}; +Zotero.Server.Endpoints["/connector/getTranslatorCode"] = Zotero.Server.Connector.GetTranslatorCode; +Zotero.Server.Connector.GetTranslatorCode.prototype = { + supportedMethods: ["POST"], + supportedDataTypes: ["application/json"], + permitBookmarklet: true, + + /** + * Returns a 200 response to say the server is alive + * @param {String} data POST data or GET query string + * @param {Function} sendResponseCallback function to send HTTP response + */ + init: function(postData, sendResponseCallback) { + var translator = Zotero.Translators.get(postData.translatorID); + Zotero.Translators.getCodeForTranslator(translator).then(function(code) { + sendResponseCallback(200, "application/javascript", code); + }); + } +} + +/** + * Returns the full serialized collection tree (excluding non-editable libraries) + * and the selected collection tree item. + * + * Accepts: + * Nothing + * Returns: + * libraryID + * libraryName + * collectionID + * collectionName + */ +Zotero.Server.Connector.GetSelectedCollection = function() {}; +Zotero.Server.Endpoints["/connector/getSelectedCollection"] = Zotero.Server.Connector.GetSelectedCollection; +Zotero.Server.Connector.GetSelectedCollection.prototype = { + supportedMethods: ["POST"], + supportedDataTypes: ["application/json"], + permitBookmarklet: true, + + /** + * Returns a 200 response to say the server is alive + * @param {String} data POST data or GET query string + * @param {Function} sendResponseCallback function to send HTTP response + */ + init: function(postData, sendResponseCallback) { + var { library, collection, editable } = Zotero.Server.Connector.getSaveTarget(true); + var response = { + libraryID: library.libraryID, + libraryName: library.name, + libraryEditable: library.editable, + filesEditable: library.filesEditable, + editable + }; + + if(collection && collection.id) { + response.id = collection.id; + response.name = collection.name; + } else { + response.id = null; + response.name = response.libraryName; + } + + // Get list of editable libraries and collections + var collections = []; + var originalLibraryID = library.libraryID; + for (let library of Zotero.Libraries.getAll()) { + if (!library.editable) continue; + + // Add recent: true for recent targets + + collections.push( + { + id: library.treeViewID, + name: library.name, + filesEditable: library.filesEditable, + level: 0 + }, + ...Zotero.Collections.getByLibrary(library.libraryID, true).map(c => ({ + id: c.treeViewID, + name: c.name, + filesEditable: library.filesEditable, + level: c.level + 1 || 1 // Added by Zotero.Collections._getByContainer() + })) + ); + } + response.targets = collections; + + // Mark recent targets + try { + let recents = Zotero.Prefs.get('recentSaveTargets'); + if (recents) { + recents = new Set(JSON.parse(recents).map(o => o.id)); + for (let target of response.targets) { + if (recents.has(target.id)) { + target.recent = true; + } + } + } + } + catch (e) { + Zotero.logError(e); + Zotero.Prefs.clear('recentSaveTargets'); + } + + sendResponseCallback( + 200, + "application/json", + JSON.stringify(response), + { + // Filter out collection names in debug output + logFilter: function (str) { + try { + let json = JSON.parse(str.match(/^{"libraryID"[^]+/m)[0]); + json.targets.forEach(t => t.name = "\u2026"); + return JSON.stringify(json); + } + catch (e) { + return str; + } + } + } + ); + } +} + +/** + * Get a list of client hostnames (reverse local IP DNS) + * + * Accepts: + * Nothing + * Returns: + * {Array} hostnames + */ +Zotero.Server.Connector.GetClientHostnames = {}; +Zotero.Server.Connector.GetClientHostnames = function() {}; +Zotero.Server.Endpoints["/connector/getClientHostnames"] = Zotero.Server.Connector.GetClientHostnames; +Zotero.Server.Connector.GetClientHostnames.prototype = { + supportedMethods: ["POST"], + supportedDataTypes: ["application/json"], + permitBookmarklet: false, + + /** + * Returns a 200 response to say the server is alive + */ + init: Zotero.Promise.coroutine(function* (requestData) { + try { + var hostnames = yield Zotero.Proxies.DNS.getHostnames(); + } catch(e) { + return 500; + } + return [200, "application/json", JSON.stringify(hostnames)]; + }) +}; + +/** + * Get a list of stored proxies + * + * Accepts: + * Nothing + * Returns: + * {Array} hostnames + */ +Zotero.Server.Connector.Proxies = {}; +Zotero.Server.Connector.Proxies = function() {}; +Zotero.Server.Endpoints["/connector/proxies"] = Zotero.Server.Connector.Proxies; +Zotero.Server.Connector.Proxies.prototype = { + supportedMethods: ["POST"], + supportedDataTypes: ["application/json"], + permitBookmarklet: false, + + /** + * Returns a 200 response to say the server is alive + */ + init: Zotero.Promise.coroutine(function* () { + let proxies = Zotero.Proxies.proxies.map((p) => Object.assign(p.toJSON(), {hosts: p.hosts})); + return [200, "application/json", JSON.stringify(proxies)]; + }) +}; + + +/** + * Test connection + * + * Accepts: + * Nothing + * Returns: + * Nothing (200 OK response) + */ +Zotero.Server.Connector.Ping = function() {}; +Zotero.Server.Endpoints["/connector/ping"] = Zotero.Server.Connector.Ping; +Zotero.Server.Connector.Ping.prototype = { + supportedMethods: ["GET", "POST"], + supportedDataTypes: ["application/json", "text/plain"], + permitBookmarklet: true, + + /** + * Sends 200 and HTML status on GET requests + * @param data {Object} request information defined in connector.js + */ + init: async function (req) { + if (req.method == 'GET') { + return [200, "text/html", '' + + 'Zotero is running']; + } else { + // Store the active URL so it can be used for site-specific Quick Copy + if (req.data.activeURL) { + //Zotero.debug("Setting active URL to " + req.data.activeURL); + Zotero.QuickCopy.lastActiveURL = req.data.activeURL; + } + let translatorsHash = await Zotero.Translators.getTranslatorsHash(false); + let sortedTranslatorHash = await Zotero.Translators.getTranslatorsHash(true); + + let response = { + prefs: { + automaticSnapshots: Zotero.Prefs.get('automaticSnapshots'), + downloadAssociatedFiles: Zotero.Prefs.get("downloadAssociatedFiles"), + supportsAttachmentUpload: true, + googleDocsAddNoteEnabled: true, + translatorsHash, + sortedTranslatorHash + } + }; + if (Zotero.QuickCopy.hasSiteSettings()) { + response.prefs.reportActiveURL = true; + } + + this.versionWarning(req); + + return [200, 'application/json', JSON.stringify(response)]; + } + }, + + + /** + * Warn on outdated connector version + * + * We can remove this once the connector checks and warns on its own and most people are on + * a version that does that. + */ + versionWarning: function (req) { + try { + if (!Zotero.Prefs.get('showConnectorVersionWarning')) return; + if (!req.headers) return; + + var minVersion = ZOTERO_CONFIG.CONNECTOR_MIN_VERSION; + var appName = ZOTERO_CONFIG.CLIENT_NAME; + var domain = ZOTERO_CONFIG.DOMAIN_NAME; + var origin = req.headers.Origin; + + var browser; + var message; + var showDownloadButton = false; + // Legacy Safari extension + if (origin && origin.startsWith('safari-extension')) { + browser = 'safari'; + message = `An update is available for the ${appName} Connector for Safari.\n\n` + + 'You can upgrade from the Extensions pane of the Safari preferences.'; + } + else if (origin && origin.startsWith('chrome-extension')) { + browser = 'chrome'; + message = `An update is available for the ${appName} Connector for Chrome.\n\n` + + `You can upgrade to the latest version from ${domain}.`; + showDownloadButton = true; + } + else if (req.headers['User-Agent'] && req.headers['User-Agent'].includes('Firefox/')) { + browser = 'firefox'; + message = `An update is available for the ${appName} Connector for Firefox.\n\n` + + `You can upgrade to the latest version from ${domain}.`; + showDownloadButton = true; + } + // Safari App Extension is always up to date + else if (req.headers['User-Agent'] && req.headers['User-Agent'].includes('Safari/')) { + return; + } + else { + Zotero.debug("Unknown browser"); + return; + } + + if (Zotero.Server.Connector['skipVersionWarning-' + browser]) return; + + var version = req.headers['X-Zotero-Version']; + if (!version || version == '4.999.0') return; + + // If connector is up to date, bail + if (Services.vc.compare(version, minVersion) >= 0) return; + + var showNextPref = `nextConnectorVersionWarning.${browser}`; + var showNext = Zotero.Prefs.get(showNextPref); + if (showNext && new Date() < new Date(showNext * 1000)) return; + + // Don't show again for this browser until restart + Zotero.Server.Connector['skipVersionWarning-' + browser] = true; + var ps = Services.prompt; + var buttonFlags; + if (showDownloadButton) { + buttonFlags = ps.BUTTON_POS_0 * ps.BUTTON_TITLE_IS_STRING + + ps.BUTTON_POS_1 * ps.BUTTON_TITLE_IS_STRING; + } + else { + buttonFlags = ps.BUTTON_POS_0 * ps.BUTTON_TITLE_OK; + } + setTimeout(function () { + var dontShow = {}; + var index = ps.confirmEx(null, + Zotero.getString('general.updateAvailable'), + message, + buttonFlags, + showDownloadButton ? Zotero.getString('general.upgrade') : null, + showDownloadButton ? Zotero.getString('general.notNow') : null, + null, + "Don\u0027t show again for a month", + dontShow + ); + + var nextShowDays; + if (dontShow.value) { + nextShowDays = 30; + } + // Don't show again for at least a day, even after a restart + else { + nextShowDays = 1; + } + Zotero.Prefs.set(showNextPref, Math.round(Date.now() / 1000) + 86400 * nextShowDays); + + if (showDownloadButton && index == 0) { + Zotero.launchURL(ZOTERO_CONFIG.CONNECTORS_URL); + } + }, 500); + } + catch (e) { + Zotero.debug(e, 2); + } + } +} + +/** + * Make an HTTP request from the client. Accepts {@link Zotero.HTTP.request} options and returns a minimal response + * object with the same form as the one returned from {@link Zotero.Utilities.Translate#request}. + * + * Accepts: + * method - The request method ('GET', 'POST', etc.) + * url - The URL to make the request to. Must be an absolute HTTP(S) URL. + * options - See Zotero.HTTP.request() documentation. Differences: + * - responseType is always set to 'text' + * - successCodes is always set to false (non-2xx status codes will not trigger an error) + * Returns: + * Response code is always 200. Body contains: + * status - The response status code, as a number + * headers - An object mapping header names to values + * body - The response body, as a string + */ +Zotero.Server.Connector.Request = function () {}; + +/** + * The list of allowed hosts. Intentionally hardcoded. + */ +Zotero.Server.Connector.Request.allowedHosts = ['www.worldcat.org']; + +/** + * For testing: allow disabling validation so we can make requests to the server. + */ +Zotero.Server.Connector.Request.enableValidation = false; + +Zotero.Server.Endpoints["/connector/request"] = Zotero.Server.Connector.Request; +Zotero.Server.Connector.Request.prototype = { + supportedMethods: ["POST"], + supportedDataTypes: ["application/json"], + + init: async function (req) { + let { method, url, options } = req.data; + + if (typeof method !== 'string' || typeof url !== 'string') { + return [400, 'text/plain', 'method and url are required and must be strings']; + } + + let uri; + try { + uri = Services.io.newURI(url); + } + catch (e) { + return [400, 'text/plain', 'Invalid URL']; + } + + if (uri.scheme != 'http' && uri.scheme != 'https') { + return [400, 'text/plain', 'Unsupported scheme']; + } + + if (Zotero.Server.Connector.Request.enableValidation) { + if (!Zotero.Server.Connector.Request.allowedHosts.includes(uri.host)) { + return [ + 400, + 'text/plain', + 'Unsupported URL' + ]; + } + Zotero.debug(`${JSON.stringify(req.headers)}`, 1); + if (!req.headers['User-Agent'] || !req.headers['User-Agent'].startsWith('Mozilla/')) { + return [400, 'text/plain', 'Unsupported User-Agent']; + } + } + + options = options || {}; + options.responseType = 'text'; + options.successCodes = false; + + let xhr; + try { + xhr = await Zotero.HTTP.request(req.data.method, req.data.url, options); + } + catch (e) { + if (e instanceof Zotero.HTTP.BrowserOfflineException) { + return [503, 'text/plain', 'Client is offline']; + } + else { + throw e; + } + } + + let status = xhr.status; + let headers = {}; + xhr.getAllResponseHeaders() + .trim() + .split(/[\r\n]+/) + .map(line => line.split(': ')) + .forEach(parts => headers[parts.shift()] = parts.join(': ')); + let body = xhr.response; + + return [200, 'application/json', JSON.stringify({ + status, + headers, + body + })]; + } +}; diff --git a/chrome/content/zotero/xpcom/connector/server_connectorIntegration.js b/chrome/content/zotero/xpcom/server/server_connectorIntegration.js similarity index 100% rename from chrome/content/zotero/xpcom/connector/server_connectorIntegration.js rename to chrome/content/zotero/xpcom/server/server_connectorIntegration.js diff --git a/chrome/content/zotero/xpcom/server_integration.js b/chrome/content/zotero/xpcom/server/server_integration.js similarity index 100% rename from chrome/content/zotero/xpcom/server_integration.js rename to chrome/content/zotero/xpcom/server/server_integration.js diff --git a/chrome/content/zotero/xpcom/localAPI/server_localAPI.js b/chrome/content/zotero/xpcom/server/server_localAPI.js similarity index 100% rename from chrome/content/zotero/xpcom/localAPI/server_localAPI.js rename to chrome/content/zotero/xpcom/server/server_localAPI.js diff --git a/chrome/content/zotero/xpcom/translation/translate_item.js b/chrome/content/zotero/xpcom/translation/translate_item.js index ba9fef7d5c..8b76bfe4d5 100644 --- a/chrome/content/zotero/xpcom/translation/translate_item.js +++ b/chrome/content/zotero/xpcom/translation/translate_item.js @@ -59,6 +59,7 @@ Zotero.Translate.ItemSaver = function(options) { this._referrer = options.referrer; this._cookieSandbox = options.cookieSandbox; this._proxy = options.proxy; + this._itemToJSONItem = new Map(); // the URI to which other URIs are assumed to be relative if(typeof options.baseURI === "object" && options.baseURI instanceof Components.interfaces.nsIURI) { @@ -82,6 +83,7 @@ Zotero.Translate.ItemSaver.PRIMARY_ATTACHMENT_TYPES = new Set([ ]); Zotero.Translate.ItemSaver.prototype = { + /** * Saves items to Standalone or the server * @param {Object[]} jsonItems - Items in Zotero.Item.toArray() format @@ -90,21 +92,17 @@ Zotero.Translate.ItemSaver.prototype = { * on failure or attachmentCallback(attachment, progressPercent) periodically during saving. * @param {Function} [itemsDoneCallback] A callback that is called once all top-level items are * done saving with a list of items. Will include saved notes, but exclude attachments. - * @param {Function} [pendingAttachmentsCallback] A callback that is called for every - * pending attachment to an item. pendingAttachmentsCallback(parentItemID, jsonAttachment) */ - saveItems: async function (jsonItems, attachmentCallback, itemsDoneCallback, pendingAttachmentsCallback) { + saveItems: async function (jsonItems, attachmentCallback, itemsDoneCallback) { var items = []; var standaloneAttachments = []; var childAttachments = []; - var jsonByItem = new Map(); await Zotero.DB.executeTransaction(async function () { for (let jsonItem of jsonItems) { jsonItem = Object.assign({}, jsonItem); let item; - let itemID; // Type defaults to "webpage" let type = jsonItem.itemType || "webpage"; @@ -121,84 +119,25 @@ Zotero.Translate.ItemSaver.prototype = { continue; } else { - item = new Zotero.Item(type); - item.libraryID = this._libraryID; - if (jsonItem.creators) this._cleanCreators(jsonItem.creators); - if (jsonItem.tags) jsonItem.tags = this._cleanTags(jsonItem.tags); - - if (jsonItem.accessDate == 'CURRENT_TIMESTAMP') { - jsonItem.accessDate = Zotero.Date.dateToISO(new Date()); - } - - item.fromJSON(this._copyJSONItemForImport(jsonItem)); - - // deproxify url - if (this._proxy && jsonItem.url) { - let url = this._proxy.toProper(jsonItem.url); - Zotero.debug(`Deproxifying item url ${jsonItem.url} with scheme ${this._proxy.scheme} to ${url}`, 5); - item.setField('url', url); - } - - if (this._collections) { - item.setCollections(this._collections); - } - - // save item - itemID = await item.save(this._saveOptions); - - // handle notes - if (jsonItem.notes) { - for (let note of jsonItem.notes) { - await this._saveNote(note, itemID); - } - } + item = await this._saveItem(jsonItem, type); - // handle attachments - if (jsonItem.attachments) { - let attachmentsToSave = []; - let foundPrimary = false; - for (let jsonAttachment of jsonItem.attachments) { - if (!this._canSaveAttachment(jsonAttachment)) { - continue; - } - - // The first PDF/EPUB is the primary one. If that one fails to download, - // we might check for an open-access PDF below. - if (Zotero.Translate.ItemSaver.PRIMARY_ATTACHMENT_TYPES.has(jsonAttachment.mimeType) - && !foundPrimary) { - jsonAttachment.isPrimary = true; - foundPrimary = true; - } - attachmentsToSave.push(jsonAttachment); - attachmentCallback(jsonAttachment, 0); - if (jsonAttachment.singleFile) { - // SingleFile attachments are saved in 'saveSingleFile' - // connector endpoint - if (pendingAttachmentsCallback) { - pendingAttachmentsCallback(itemID, jsonAttachment); - } - continue; - } - childAttachments.push([jsonAttachment, itemID]); - } - jsonItem.attachments = attachmentsToSave; - } - - // handle see also - this._handleRelated(jsonItem, item); + // process attachments + let attachments = this._processChildAttachments(jsonItem, attachmentCallback); + attachments.forEach(attachment => childAttachments.push([attachment, item.id])); } // Add to new item list items.push(item); - jsonByItem.set(item, jsonItem); + this._itemToJSONItem.set(item, jsonItem); } }.bind(this)); + // Done saving top-level items. Call the callback so that UI code can update if (itemsDoneCallback) { - itemsDoneCallback(items.map(item => jsonByItem.get(item)), items); + itemsDoneCallback(items.map(item => this._itemToJSONItem.get(item)), items); } - // Save standalone attachments + // Download standalone attachments for (let jsonItem of standaloneAttachments) { let item = await this._saveAttachment(jsonItem, null, attachmentCallback); if (item) { @@ -210,45 +149,21 @@ Zotero.Translate.ItemSaver.prototype = { // open-access PDFs. There's no guarantee that either translated PDFs or OA PDFs will // successfully download, but this lets us update the progress window sooner with // possible downloads. - // + this._openAccessPDFURLs = new Map(); + // TODO: Separate pref? var shouldDownloadOAPDF = this.attachmentMode == Zotero.Translate.ItemSaver.ATTACHMENT_MODE_DOWNLOAD - && Zotero.Prefs.get('downloadAssociatedFiles'); - var openAccessPDFURLs = new Map(); + && Zotero.Prefs.get('downloadAssociatedFiles'); if (shouldDownloadOAPDF) { for (let item of items) { - let jsonItem = jsonByItem.get(item); - - // Skip items with translated PDF attachments - if (jsonItem.attachments - && jsonItem.attachments.some(x => Zotero.Translate.ItemSaver.PRIMARY_ATTACHMENT_TYPES.has(x.mimeType))) { - continue; - } - - try { - let resolvers = Zotero.Attachments.getPDFResolvers(item, ['oa']); - if (!resolvers.length) { - openAccessPDFURLs.set(item, []); - continue; - } - let urlObjects = await resolvers[0](); - openAccessPDFURLs.set(item, urlObjects); - // If there are possible URLs, create a status line for the PDF - if (urlObjects.length) { - let title = Zotero.getString('findPDF.openAccessPDF'); - let jsonAttachment = this._makeJSONAttachment(jsonItem.id, title); - if (!jsonItem.attachments) jsonItem.attachments = []; - jsonItem.attachments.push(jsonAttachment); - attachmentCallback(jsonAttachment, 0); - } - } - catch (e) { - Zotero.logError(e); + let urlObjects = await this._getOpenAccessPDFURLs(item, attachmentCallback); + if (urlObjects) { + this._openAccessPDFURLs.set(item, urlObjects); } } } - // Save translated child attachments, and keep track of whether the save was successful + // Save translated child attachments var itemIDsWithPrimaryAttachments = new Set(); for (let [jsonAttachment, parentItemID] of childAttachments) { let attachment = await this._saveAttachment( @@ -267,93 +182,15 @@ Zotero.Translate.ItemSaver.prototype = { } } - // If a translated PDF attachment wasn't saved successfully, either because there wasn't - // one or there was but it failed, look for another PDF (if enabled) if (shouldDownloadOAPDF) { + // If a translated PDF attachment wasn't saved successfully, either because there wasn't + // one or there was but it failed, look for another PDF (if enabled) for (let item of items) { // Already have a primary attachment from translation if (itemIDsWithPrimaryAttachments.has(item.id)) { continue; } - - let jsonItem = jsonByItem.get(item); - // Reuse the existing status line if there is one. This could be a failed - // translator attachment or a possible OA PDF found above. - // Explicitly check that the attachment is a PDF, not just any primary type, - // since we're reusing it for a PDF attachment. - let jsonAttachment = jsonItem.attachments && jsonItem.attachments.find( - x => x.mimeType == 'application/pdf' && x.isPrimary - ); - - // If no translated, no OA, and no custom, don't show a line - // If no translated and potential OA, show "Open-Access PDF" - // If no translated, no OA, but custom, show custom when it starts - // If translated fails and potential OA, show "Open-Access PDF" - // If translated fails, no OA, no custom, fail original - // If translated fails, no OA, but custom, change to custom when it starts - let resolvers = openAccessPDFURLs.get(item); - // No translated PDF, so we checked for OA PDFs above - if (resolvers) { - // Add custom resolvers - resolvers.push(...Zotero.Attachments.getPDFResolvers(item, ['custom'], true)); - - // No translated, no OA, no custom, no status line - if (!resolvers.length) { - continue; - } - - // No translated, no OA, just potential custom, so create a status line - if (!jsonAttachment) { - jsonAttachment = this._makeJSONAttachment( - jsonItem.id, Zotero.getString('findPDF.searchingForAvailableFiles') - ); - } - } - // There was a translated PDF, so we didn't check for OA PDFs yet and didn't - // update the status line - else { - // Look for OA PDFs now - resolvers = Zotero.Attachments.getPDFResolvers(item, ['oa']); - if (resolvers.length) { - resolvers = await resolvers[0](); - } - - // Add custom resolvers - resolvers.push(...Zotero.Attachments.getPDFResolvers(item, ['custom'], true)); - - // Failed translated, no OA, no custom, so fail the existing translator line - if (!resolvers.length) { - attachmentCallback(jsonAttachment, false); - continue; - } - } - - let attachment; - try { - attachment = await Zotero.Attachments.addFileFromURLs( - item, - resolvers, - { - // When a new access method starts, update the status line - onAccessMethodStart: (method) => { - jsonAttachment.title = this._getPDFTitleForAccessMethod(method); - attachmentCallback(jsonAttachment, 0); - } - } - ); - } - catch (e) { - Zotero.logError(e); - attachmentCallback(jsonAttachment, false, e); - continue; - } - - if (attachment) { - attachmentCallback(jsonAttachment, 100); - } - else { - attachmentCallback(jsonAttachment, false, "PDF not found"); - } + await this.saveOpenAccessAttachment(item, attachmentCallback); } } @@ -364,23 +201,233 @@ Zotero.Translate.ItemSaver.prototype = { /** * Save pending snapshot attachments to disk and library * - * @param {Array} pendingAttachments - A list of snapshot attachments - * @param {Object} content - Snapshot content from SingleFile - * @param {Function} attachmentCallback - Callback with progress of attachments + * @param {Object} options - A list of snapshot attachments + * - title {String} + * - url {String} + * - parentItemID {Number} + * - snapshotContent {String} */ - saveSnapshotAttachments: Zotero.Promise.coroutine(function* (pendingAttachments, snapshotContent, attachmentCallback) { - for (let [parentItemID, attachment] of pendingAttachments) { - Zotero.debug('Saving pending attachment: ' + JSON.stringify(attachment)); - if (snapshotContent) { - attachment.snapshotContent = snapshotContent; - } - yield this._saveAttachment( + saveSnapshotAttachments: async function (options) { + let { title, url, parentItemID, snapshotContent } = options; + let attachment = { title, url }; + Zotero.debug('Saving pending attachment: ' + JSON.stringify(attachment)); + if (snapshotContent) { + attachment.snapshotContent = snapshotContent; + } + await new Promise(async (resolve, reject) => { + await this._saveAttachment( attachment, parentItemID, - attachmentCallback + (attachment, progress, e) => { + if (e) reject(e); + if (progress === 100) { + resolve(progress); + } + } + ); + }); + }, + + + async _saveItem(jsonItem, type) { + let itemID; + let item = new Zotero.Item(type); + item.libraryID = this._libraryID; + if (jsonItem.creators) this._cleanCreators(jsonItem.creators); + if (jsonItem.tags) jsonItem.tags = this._cleanTags(jsonItem.tags); + + if (jsonItem.accessDate == 'CURRENT_TIMESTAMP') { + jsonItem.accessDate = Zotero.Date.dateToISO(new Date()); + } + + item.fromJSON(this._copyJSONItemForImport(jsonItem)); + + // deproxify url + if (this._proxy && jsonItem.url) { + let url = this._proxy.toProper(jsonItem.url); + Zotero.debug(`Deproxifying item url ${jsonItem.url} with scheme ${this._proxy.scheme} to ${url}`, 5); + item.setField('url', url); + } + + // save item + if (this._collections) { + item.setCollections(this._collections); + } + itemID = await item.save(this._saveOptions); + + // handle notes + if (jsonItem.notes) { + for (let note of jsonItem.notes) { + await this._saveNote(note, itemID); + } + } + + // handle see also + this._handleRelated(jsonItem, item); + return item; + }, + + + /** + * Processes attachments to be saved either via Zotero or externally (Connector) + * + * Calls attachment callbacks for initial attachment progress (0) + */ + _processChildAttachments(jsonItem, attachmentCallback) { + let childAttachments = []; + + let foundPrimary = false; + // Attachments to be saved within Zotero + if (jsonItem.attachments) { + let attachmentsToSave = []; + for (let jsonAttachment of jsonItem.attachments) { + if (!this._canSaveAttachment(jsonAttachment)) { + continue; + } + + // The first PDF/EPUB is the primary one. If that one fails to download, + // we might check for an open-access PDF below. + if (Zotero.Translate.ItemSaver.PRIMARY_ATTACHMENT_TYPES.has(jsonAttachment.mimeType) + && !foundPrimary) { + jsonAttachment.isPrimary = true; + foundPrimary = true; + } + attachmentsToSave.push(jsonAttachment); + attachmentCallback(jsonAttachment, 0); + childAttachments.push(jsonAttachment); + } + + jsonItem.attachments = attachmentsToSave; + } + return childAttachments; + }, + + /** + * Gets a list of OA PDF URLs for items that did not receive a PDF attachment + * from the translator + * + * Calls attachmentCallback to update UI + * @param items + * @param attachmentCallback + * @returns {Promise>} + * @private + */ + async _getOpenAccessPDFURLs(item, attachmentCallback) { + let jsonItem = this._itemToJSONItem.get(item); + let urlObjects = []; + + // Has a primary attachment or a pending (from Connector) primary attachment + if (jsonItem.attachments?.some(x => Zotero.Translate.ItemSaver.PRIMARY_ATTACHMENT_TYPES.has(x.mimeType)) + || jsonItem.pendingPrimaryAttachment) { + return null; + } + + // If no primary attachments available look for an OA one and call attachmentCallback to update UI + try { + let resolvers = Zotero.Attachments.getPDFResolvers(item, ['oa']); + if (!resolvers.length) { + return urlObjects; + } + urlObjects = await resolvers[0](); + // If there are possible URLs, create a status line for the PDF + if (urlObjects.length) { + let title = Zotero.getString('findPDF.openAccessPDF'); + let jsonAttachment = this._makeJSONAttachment(jsonItem.id, title); + if (!jsonItem.attachments) jsonItem.attachments = []; + jsonItem.attachments.push(jsonAttachment); + attachmentCallback(jsonAttachment, 0); + } + } + catch (e) { + Zotero.logError(e); + } + return urlObjects; + }, + + async saveOpenAccessAttachment(item, attachmentCallback) { + let jsonItem = this._itemToJSONItem.get(item); + // Reuse the existing status line if there is one. This could be a failed + // translator attachment or a possible OA PDF found above. + // Explicitly check that the attachment is a PDF, not just any primary type, + // since we're reusing it for a PDF attachment. + let jsonAttachment = jsonItem.attachments && jsonItem.attachments.find( + x => x.mimeType == 'application/pdf' && x.isPrimary + ); + + + // If no translated, no OA, no custom, don't show a line + // If translated fails, no OA, no custom, fail original + + // If no translated, potential OA, show "Open-Access PDF" (set in _getOpenAccessPDFURLs()) + // If translated fails, potential OA, show "Open-Access PDF" (set here) + + // If no translated + // or translated fails, no OA, but custom, show custom when it starts + + let resolvers = this._openAccessPDFURLs.get(item); + // We checked for OA PDFs in _getOpenAccessPDFURLs() so there was no translated pdf + if (resolvers) { + // Add custom resolvers + resolvers.push(...Zotero.Attachments.getPDFResolvers(item, ['custom'], true)); + + // No translated, no OA, no custom, no status line + if (!resolvers.length) { + return; + } + + // No translated, no OA, just potential custom, so create a status line + if (!jsonAttachment) { + jsonAttachment = this._makeJSONAttachment( + jsonItem.id, Zotero.getString('findPDF.searchingForAvailableFiles') + ); + } + } + else { + // Translated attachment failed, so we didn't check for OA PDFs yet and didn't + // update the status line + // Look for OA PDFs now + resolvers = Zotero.Attachments.getPDFResolvers(item, ['oa']); + if (resolvers.length) { + resolvers = await resolvers[0](); + } + + // Add custom resolvers + resolvers.push(...Zotero.Attachments.getPDFResolvers(item, ['custom'], true)); + + // Failed translated, no OA, no custom, so fail the existing translator line + if (!resolvers.length) { + attachmentCallback(jsonAttachment, false); + return + } + } + + let attachment; + try { + attachment = await Zotero.Attachments.addFileFromURLs( + item, + resolvers, + { + // When a new access method starts, update the status line + onAccessMethodStart: (method) => { + jsonAttachment.title = this._getPDFTitleForAccessMethod(method); + attachmentCallback(jsonAttachment, 0); + } + } ); } - }), + catch (e) { + Zotero.logError(e); + attachmentCallback(jsonAttachment, false, e); + return; + } + + if (attachment) { + attachmentCallback(jsonAttachment, 100); + } + else { + attachmentCallback(jsonAttachment, false, "PDF not found"); + } + }, _makeJSONAttachment: function (parentID, title) { @@ -534,16 +581,23 @@ Zotero.Translate.ItemSaver.prototype = { try { let newAttachment; + const isSinglefileSnapshot = !!attachment.snapshotContent; // determine whether to save files and attachments - var isLink = Zotero.MIME.isWebPageType(attachment.mimeType) - // .snapshot coming from most translators, .linkMode coming from RDF - && (attachment.snapshot === false || attachment.linkMode == Zotero.Attachments.LINK_MODE_LINKED_URL); - if (isLink || this.attachmentMode == Zotero.Translate.ItemSaver.ATTACHMENT_MODE_DOWNLOAD) { + // .snapshot coming from most translators, .linkMode coming from RDF + var isLink = attachment.snapshot === false + || attachment.linkMode == Zotero.Attachments.LINK_MODE_LINKED_URL; + + if (isLink || this.attachmentMode === Zotero.Translate.ItemSaver.ATTACHMENT_MODE_IGNORE) { + newAttachment = yield this._saveAttachmentLink.apply(this, arguments); + } + else if (isSinglefileSnapshot || this.attachmentMode == Zotero.Translate.ItemSaver.ATTACHMENT_MODE_DOWNLOAD) { newAttachment = yield this._saveAttachmentDownload.apply(this, arguments); - } else if (this.attachmentMode == Zotero.Translate.ItemSaver.ATTACHMENT_MODE_FILE) { + } + else if (this.attachmentMode == Zotero.Translate.ItemSaver.ATTACHMENT_MODE_FILE) { newAttachment = yield this._saveAttachmentFile.apply(this, arguments); - } else { - Zotero.debug('Translate: Ignoring attachment due to ATTACHMENT_MODE_IGNORE'); + } + else { + Zotero.debug(`Translate: Ignoring attachment ${attachment.title} due to ATTACHMENT_MODE_IGNORE`); } if (!newAttachment) return false; // attachmentCallback should not have been called in this case @@ -821,6 +875,50 @@ Zotero.Translate.ItemSaver.prototype = { return false; }, + _saveAttachmentLink: async function(attachment, parentItemID, attachmentCallback) { + attachment.linkMode = "linked_url"; + let url, mimeType; + if(attachment.document) { + url = attachment.document.location.href; + mimeType = attachment.mimeType || attachment.document.contentType; + } else { + url = attachment.url + mimeType = attachment.mimeType || undefined; + } + + // If no title provided, use "Attachment" as title for progress UI (but not for item) + let title = attachment.title || null; + if(!attachment.title) { + attachment.title = Zotero.getString("itemTypes.attachment"); + } + + if(!mimeType || !title) { + Zotero.debug("Translate: mimeType or title is missing; attaching link to URL will be slower"); + } + + let cleanURI = Zotero.Attachments.cleanAttachmentURI(url); + if (!cleanURI) { + throw new Error("Translate: Invalid attachment URL specified <" + url + ">"); + } + url = Components.classes["@mozilla.org/network/io-service;1"] + .getService(Components.interfaces.nsIIOService) + .newURI(cleanURI, null, null); // This cannot fail, since we check above + + // Only HTTP/HTTPS links are allowed + if(url.scheme != "http" && url.scheme != "https") { + throw new Error("Translate: " + url.scheme + " protocol is not allowed for attachments from translators."); + } + + return Zotero.Attachments.linkFromURL({ + url: cleanURI, + parentItemID, + contentType: mimeType, + title, + collections: !parentItemID ? this._collections : undefined, + saveOptions: this._saveOptions, + }); + }, + _saveAttachmentDownload: Zotero.Promise.coroutine(function* (attachment, parentItemID, attachmentCallback) { Zotero.debug("Translate: Adding attachment", 4); @@ -839,51 +937,11 @@ Zotero.Translate.ItemSaver.prototype = { // Commit to saving attachmentCallback(attachment, 0); - var isLink = attachment.snapshot === false - || attachment.linkMode == Zotero.Attachments.LINK_MODE_LINKED_URL; - if (isLink || this.attachmentMode === Zotero.Translate.ItemSaver.ATTACHMENT_MODE_IGNORE) { - // if snapshot is explicitly set to false, attach as link - attachment.linkMode = "linked_url"; - let url, mimeType; - if(attachment.document) { - url = attachment.document.location.href; - mimeType = attachment.mimeType || attachment.document.contentType; - } else { - url = attachment.url - mimeType = attachment.mimeType || undefined; - } - - if(!mimeType || !title) { - Zotero.debug("Translate: mimeType or title is missing; attaching link to URL will be slower"); - } - - let cleanURI = Zotero.Attachments.cleanAttachmentURI(url); - if (!cleanURI) { - throw new Error("Translate: Invalid attachment URL specified <" + url + ">"); - } - url = Components.classes["@mozilla.org/network/io-service;1"] - .getService(Components.interfaces.nsIIOService) - .newURI(cleanURI, null, null); // This cannot fail, since we check above - - // Only HTTP/HTTPS links are allowed - if(url.scheme != "http" && url.scheme != "https") { - throw new Error("Translate: " + url.scheme + " protocol is not allowed for attachments from translators."); - } - - return Zotero.Attachments.linkFromURL({ - url: cleanURI, - parentItemID, - contentType: mimeType, - title, - collections: !parentItemID ? this._collections : undefined, - saveOptions: this._saveOptions, - }); - } - // Snapshot is not explicitly set to false, import as file attachment + attachment.linkMode = "imported_url"; // Import from document - if(attachment.document) { + if (attachment.document) { Zotero.debug('Importing attachment from document'); attachment.linkMode = "imported_url"; @@ -897,17 +955,6 @@ Zotero.Translate.ItemSaver.prototype = { }); } - let mimeType = attachment.mimeType ? attachment.mimeType : null; - let fileBaseName; - if (parentItemID) { - let parentItem = yield Zotero.Items.getAsync(parentItemID); - fileBaseName = Zotero.Attachments.getFileBaseNameFromItem(parentItem, { attachmentTitle: title }); - } - - attachment.linkMode = "imported_url"; - - attachmentCallback(attachment, 0); - // Import from SingleFile content if (attachment.snapshotContent) { Zotero.debug('Importing attachment from SingleFile'); @@ -924,6 +971,13 @@ Zotero.Translate.ItemSaver.prototype = { } // Import from URL + let mimeType = attachment.mimeType ? attachment.mimeType : null; + let fileBaseName; + if (parentItemID) { + let parentItem = yield Zotero.Items.getAsync(parentItemID); + fileBaseName = Zotero.Attachments.getFileBaseNameFromItem(parentItem, { attachmentTitle: title }); + } + Zotero.debug('Importing attachment from URL'); return Zotero.Attachments.importFromURL({ libraryID: this._libraryID, diff --git a/chrome/content/zotero/zotero.mjs b/chrome/content/zotero/zotero.mjs index 483e8c6495..b257d8f47a 100644 --- a/chrome/content/zotero/zotero.mjs +++ b/chrome/content/zotero/zotero.mjs @@ -105,6 +105,7 @@ const xpcomFilesLocal = [ 'feedReader', 'fileDragDataProvider', 'fulltext', + 'httpIntegrationClient', 'id', 'integration', 'locale', @@ -124,8 +125,12 @@ const xpcomFilesLocal = [ 'retractions', 'router', 'schema', - 'server', - 'server_integration', + 'server/server', + 'server/server_integration', + 'server/server_connector', + 'server/server_connectorIntegration', + 'server/server_localAPI', + 'server/saveSession', 'session', 'streamer', 'style', @@ -152,10 +157,6 @@ const xpcomFilesLocal = [ 'users', 'translation/translate_item', 'translation/translators', - 'connector/httpIntegrationClient', - 'connector/server_connector', - 'connector/server_connectorIntegration', - 'localAPI/server_localAPI', ]; Components.utils.import("resource://gre/modules/ComponentUtils.jsm"); diff --git a/test/content/support.js b/test/content/support.js index 2d1aa5a9fb..a41320cbd3 100644 --- a/test/content/support.js +++ b/test/content/support.js @@ -1177,7 +1177,8 @@ async function startHTTPServer(port = null) { if (!port) { port = httpdServerPort; } - Components.utils.import("resource://zotero-unit/httpd.js"); + + var { HttpServer } = ChromeUtils.import("chrome://remote/content/server/HTTPD.jsm");; var httpd = new HttpServer(); while (true) { try { diff --git a/test/resource/httpd.js b/test/resource/httpd.js deleted file mode 100644 index 9a31e915f0..0000000000 --- a/test/resource/httpd.js +++ /dev/null @@ -1,5618 +0,0 @@ -/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */ -/* vim:set ts=2 sw=2 sts=2 et: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/* - * An implementation of an HTTP server both as a loadable script and as an XPCOM - * component. See the accompanying README file for user documentation on - * httpd.js. - */ - -var EXPORTED_SYMBOLS = [ - "HTTP_400", - "HTTP_401", - "HTTP_402", - "HTTP_403", - "HTTP_404", - "HTTP_405", - "HTTP_406", - "HTTP_407", - "HTTP_408", - "HTTP_409", - "HTTP_410", - "HTTP_411", - "HTTP_412", - "HTTP_413", - "HTTP_414", - "HTTP_415", - "HTTP_417", - "HTTP_500", - "HTTP_501", - "HTTP_502", - "HTTP_503", - "HTTP_504", - "HTTP_505", - "HttpError", - "HttpServer", - "NodeServer", -]; - -const CC = Components.Constructor; - -const PR_UINT32_MAX = Math.pow(2, 32) - 1; - -/** True if debugging output is enabled, false otherwise. */ -var DEBUG = false; // non-const *only* so tweakable in server tests - -/** True if debugging output should be timestamped. */ -var DEBUG_TIMESTAMP = false; // non-const so tweakable in server tests - -const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm"); -const { AppConstants } = ChromeUtils.import( - "resource://gre/modules/AppConstants.jsm" -); - -/** - * Asserts that the given condition holds. If it doesn't, the given message is - * dumped, a stack trace is printed, and an exception is thrown to attempt to - * stop execution (which unfortunately must rely upon the exception not being - * accidentally swallowed by the code that uses it). - */ -function NS_ASSERT(cond, msg) { - if (DEBUG && !cond) { - dumpn("###!!!"); - dumpn("###!!! ASSERTION" + (msg ? ": " + msg : "!")); - dumpn("###!!! Stack follows:"); - - var stack = new Error().stack.split(/\n/); - dumpn( - stack - .map(function(val) { - return "###!!! " + val; - }) - .join("\n") - ); - - throw Components.Exception("", Cr.NS_ERROR_ABORT); - } -} - -/** Constructs an HTTP error object. */ -function HttpError(code, description) { - this.code = code; - this.description = description; -} -HttpError.prototype = { - toString() { - return this.code + " " + this.description; - }, -}; - -/** - * Errors thrown to trigger specific HTTP server responses. - */ -var HTTP_400 = new HttpError(400, "Bad Request"); -var HTTP_401 = new HttpError(401, "Unauthorized"); -var HTTP_402 = new HttpError(402, "Payment Required"); -var HTTP_403 = new HttpError(403, "Forbidden"); -var HTTP_404 = new HttpError(404, "Not Found"); -var HTTP_405 = new HttpError(405, "Method Not Allowed"); -var HTTP_406 = new HttpError(406, "Not Acceptable"); -var HTTP_407 = new HttpError(407, "Proxy Authentication Required"); -var HTTP_408 = new HttpError(408, "Request Timeout"); -var HTTP_409 = new HttpError(409, "Conflict"); -var HTTP_410 = new HttpError(410, "Gone"); -var HTTP_411 = new HttpError(411, "Length Required"); -var HTTP_412 = new HttpError(412, "Precondition Failed"); -var HTTP_413 = new HttpError(413, "Request Entity Too Large"); -var HTTP_414 = new HttpError(414, "Request-URI Too Long"); -var HTTP_415 = new HttpError(415, "Unsupported Media Type"); -var HTTP_417 = new HttpError(417, "Expectation Failed"); - -var HTTP_500 = new HttpError(500, "Internal Server Error"); -var HTTP_501 = new HttpError(501, "Not Implemented"); -var HTTP_502 = new HttpError(502, "Bad Gateway"); -var HTTP_503 = new HttpError(503, "Service Unavailable"); -var HTTP_504 = new HttpError(504, "Gateway Timeout"); -var HTTP_505 = new HttpError(505, "HTTP Version Not Supported"); - -/** Creates a hash with fields corresponding to the values in arr. */ -function array2obj(arr) { - var obj = {}; - for (var i = 0; i < arr.length; i++) { - obj[arr[i]] = arr[i]; - } - return obj; -} - -/** Returns an array of the integers x through y, inclusive. */ -function range(x, y) { - var arr = []; - for (var i = x; i <= y; i++) { - arr.push(i); - } - return arr; -} - -/** An object (hash) whose fields are the numbers of all HTTP error codes. */ -const HTTP_ERROR_CODES = array2obj(range(400, 417).concat(range(500, 505))); - -/** - * The character used to distinguish hidden files from non-hidden files, a la - * the leading dot in Apache. Since that mechanism also hides files from - * easy display in LXR, ls output, etc. however, we choose instead to use a - * suffix character. If a requested file ends with it, we append another - * when getting the file on the server. If it doesn't, we just look up that - * file. Therefore, any file whose name ends with exactly one of the character - * is "hidden" and available for use by the server. - */ -const HIDDEN_CHAR = "^"; - -/** - * The file name suffix indicating the file containing overridden headers for - * a requested file. - */ -const HEADERS_SUFFIX = HIDDEN_CHAR + "headers" + HIDDEN_CHAR; -const INFORMATIONAL_RESPONSE_SUFFIX = - HIDDEN_CHAR + "informationalResponse" + HIDDEN_CHAR; - -/** Type used to denote SJS scripts for CGI-like functionality. */ -const SJS_TYPE = "sjs"; - -/** Base for relative timestamps produced by dumpn(). */ -var firstStamp = 0; - -/** dump(str) with a trailing "\n" -- only outputs if DEBUG. */ -function dumpn(str) { - if (DEBUG) { - var prefix = "HTTPD-INFO | "; - if (DEBUG_TIMESTAMP) { - if (firstStamp === 0) { - firstStamp = Date.now(); - } - - var elapsed = Date.now() - firstStamp; // milliseconds - var min = Math.floor(elapsed / 60000); - var sec = (elapsed % 60000) / 1000; - - if (sec < 10) { - prefix += min + ":0" + sec.toFixed(3) + " | "; - } else { - prefix += min + ":" + sec.toFixed(3) + " | "; - } - } - - dump(prefix + str + "\n"); - } -} - -/** Dumps the current JS stack if DEBUG. */ -function dumpStack() { - // peel off the frames for dumpStack() and Error() - var stack = new Error().stack.split(/\n/).slice(2); - stack.forEach(dumpn); -} - -/** The XPCOM thread manager. */ -var gThreadManager = null; - -/** - * JavaScript constructors for commonly-used classes; precreating these is a - * speedup over doing the same from base principles. See the docs at - * http://developer.mozilla.org/en/docs/Components.Constructor for details. - */ -const ServerSocket = CC( - "@mozilla.org/network/server-socket;1", - "nsIServerSocket", - "init" -); -const ServerSocketIPv6 = CC( - "@mozilla.org/network/server-socket;1", - "nsIServerSocket", - "initIPv6" -); -const ScriptableInputStream = CC( - "@mozilla.org/scriptableinputstream;1", - "nsIScriptableInputStream", - "init" -); -const Pipe = CC("@mozilla.org/pipe;1", "nsIPipe", "init"); -const FileInputStream = CC( - "@mozilla.org/network/file-input-stream;1", - "nsIFileInputStream", - "init" -); -const ConverterInputStream = CC( - "@mozilla.org/intl/converter-input-stream;1", - "nsIConverterInputStream", - "init" -); -const WritablePropertyBag = CC( - "@mozilla.org/hash-property-bag;1", - "nsIWritablePropertyBag2" -); -const SupportsString = CC( - "@mozilla.org/supports-string;1", - "nsISupportsString" -); - -/* These two are non-const only so a test can overwrite them. */ -var BinaryInputStream = CC( - "@mozilla.org/binaryinputstream;1", - "nsIBinaryInputStream", - "setInputStream" -); -var BinaryOutputStream = CC( - "@mozilla.org/binaryoutputstream;1", - "nsIBinaryOutputStream", - "setOutputStream" -); - -/** - * Returns the RFC 822/1123 representation of a date. - * - * @param date : Number - * the date, in milliseconds from midnight (00:00:00), January 1, 1970 GMT - * @returns string - * the representation of the given date - */ -function toDateString(date) { - // - // rfc1123-date = wkday "," SP date1 SP time SP "GMT" - // date1 = 2DIGIT SP month SP 4DIGIT - // ; day month year (e.g., 02 Jun 1982) - // time = 2DIGIT ":" 2DIGIT ":" 2DIGIT - // ; 00:00:00 - 23:59:59 - // wkday = "Mon" | "Tue" | "Wed" - // | "Thu" | "Fri" | "Sat" | "Sun" - // month = "Jan" | "Feb" | "Mar" | "Apr" - // | "May" | "Jun" | "Jul" | "Aug" - // | "Sep" | "Oct" | "Nov" | "Dec" - // - - const wkdayStrings = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"]; - const monthStrings = [ - "Jan", - "Feb", - "Mar", - "Apr", - "May", - "Jun", - "Jul", - "Aug", - "Sep", - "Oct", - "Nov", - "Dec", - ]; - - /** - * Processes a date and returns the encoded UTC time as a string according to - * the format specified in RFC 2616. - * - * @param date : Date - * the date to process - * @returns string - * a string of the form "HH:MM:SS", ranging from "00:00:00" to "23:59:59" - */ - function toTime(date) { - var hrs = date.getUTCHours(); - var rv = hrs < 10 ? "0" + hrs : hrs; - - var mins = date.getUTCMinutes(); - rv += ":"; - rv += mins < 10 ? "0" + mins : mins; - - var secs = date.getUTCSeconds(); - rv += ":"; - rv += secs < 10 ? "0" + secs : secs; - - return rv; - } - - /** - * Processes a date and returns the encoded UTC date as a string according to - * the date1 format specified in RFC 2616. - * - * @param date : Date - * the date to process - * @returns string - * a string of the form "HH:MM:SS", ranging from "00:00:00" to "23:59:59" - */ - function toDate1(date) { - var day = date.getUTCDate(); - var month = date.getUTCMonth(); - var year = date.getUTCFullYear(); - - var rv = day < 10 ? "0" + day : day; - rv += " " + monthStrings[month]; - rv += " " + year; - - return rv; - } - - date = new Date(date); - - const fmtString = "%wkday%, %date1% %time% GMT"; - var rv = fmtString.replace("%wkday%", wkdayStrings[date.getUTCDay()]); - rv = rv.replace("%time%", toTime(date)); - return rv.replace("%date1%", toDate1(date)); -} - -/** - * Prints out a human-readable representation of the object o and its fields, - * omitting those whose names begin with "_" if showMembers != true (to ignore - * "private" properties exposed via getters/setters). - */ -function printObj(o, showMembers) { - var s = "******************************\n"; - s += "o = {\n"; - for (var i in o) { - if (typeof i != "string" || showMembers || (i.length > 0 && i[0] != "_")) { - s += " " + i + ": " + o[i] + ",\n"; - } - } - s += " };\n"; - s += "******************************"; - dumpn(s); -} - -/** - * Instantiates a new HTTP server. - */ -function nsHttpServer() { - if (!gThreadManager) { - gThreadManager = Cc["@mozilla.org/thread-manager;1"].getService(); - } - - /** The port on which this server listens. */ - this._port = undefined; - - /** The socket associated with this. */ - this._socket = null; - - /** The handler used to process requests to this server. */ - this._handler = new ServerHandler(this); - - /** Naming information for this server. */ - this._identity = new ServerIdentity(); - - /** - * Indicates when the server is to be shut down at the end of the request. - */ - this._doQuit = false; - - /** - * True if the socket in this is closed (and closure notifications have been - * sent and processed if the socket was ever opened), false otherwise. - */ - this._socketClosed = true; - - /** - * Used for tracking existing connections and ensuring that all connections - * are properly cleaned up before server shutdown; increases by 1 for every - * new incoming connection. - */ - this._connectionGen = 0; - - /** - * Hash of all open connections, indexed by connection number at time of - * creation. - */ - this._connections = {}; -} -nsHttpServer.prototype = { - // NSISERVERSOCKETLISTENER - - /** - * Processes an incoming request coming in on the given socket and contained - * in the given transport. - * - * @param socket : nsIServerSocket - * the socket through which the request was served - * @param trans : nsISocketTransport - * the transport for the request/response - * @see nsIServerSocketListener.onSocketAccepted - */ - onSocketAccepted(socket, trans) { - dumpn("*** onSocketAccepted(socket=" + socket + ", trans=" + trans + ")"); - - dumpn(">>> new connection on " + trans.host + ":" + trans.port); - - const SEGMENT_SIZE = 8192; - const SEGMENT_COUNT = 1024; - try { - var input = trans - .openInputStream(0, SEGMENT_SIZE, SEGMENT_COUNT) - .QueryInterface(Ci.nsIAsyncInputStream); - var output = trans.openOutputStream(0, 0, 0); - } catch (e) { - dumpn("*** error opening transport streams: " + e); - trans.close(Cr.NS_BINDING_ABORTED); - return; - } - - var connectionNumber = ++this._connectionGen; - - try { - var conn = new Connection( - input, - output, - this, - socket.port, - trans.port, - connectionNumber, - trans - ); - var reader = new RequestReader(conn); - - // XXX add request timeout functionality here! - - // Note: must use main thread here, or we might get a GC that will cause - // threadsafety assertions. We really need to fix XPConnect so that - // you can actually do things in multi-threaded JS. :-( - input.asyncWait(reader, 0, 0, gThreadManager.mainThread); - } catch (e) { - // Assume this connection can't be salvaged and bail on it completely; - // don't attempt to close it so that we can assert that any connection - // being closed is in this._connections. - dumpn("*** error in initial request-processing stages: " + e); - trans.close(Cr.NS_BINDING_ABORTED); - return; - } - - this._connections[connectionNumber] = conn; - dumpn("*** starting connection " + connectionNumber); - }, - - /** - * Called when the socket associated with this is closed. - * - * @param socket : nsIServerSocket - * the socket being closed - * @param status : nsresult - * the reason the socket stopped listening (NS_BINDING_ABORTED if the server - * was stopped using nsIHttpServer.stop) - * @see nsIServerSocketListener.onStopListening - */ - onStopListening(socket, status) { - dumpn(">>> shutting down server on port " + socket.port); - for (var n in this._connections) { - if (!this._connections[n]._requestStarted) { - this._connections[n].close(); - } - } - this._socketClosed = true; - if (this._hasOpenConnections()) { - dumpn("*** open connections!!!"); - } - if (!this._hasOpenConnections()) { - dumpn("*** no open connections, notifying async from onStopListening"); - - // Notify asynchronously so that any pending teardown in stop() has a - // chance to run first. - var self = this; - var stopEvent = { - run() { - dumpn("*** _notifyStopped async callback"); - self._notifyStopped(); - }, - }; - gThreadManager.currentThread.dispatch( - stopEvent, - Ci.nsIThread.DISPATCH_NORMAL - ); - } - }, - - // NSIHTTPSERVER - - // - // see nsIHttpServer.start - // - start(port) { - this._start(port, "localhost"); - }, - - // - // see nsIHttpServer.start_ipv6 - // - start_ipv6(port) { - this._start(port, "[::1]"); - }, - - _start(port, host) { - if (this._socket) { - throw Components.Exception("", Cr.NS_ERROR_ALREADY_INITIALIZED); - } - - this._port = port; - this._doQuit = this._socketClosed = false; - - this._host = host; - - // The listen queue needs to be long enough to handle - // network.http.max-persistent-connections-per-server or - // network.http.max-persistent-connections-per-proxy concurrent - // connections, plus a safety margin in case some other process is - // talking to the server as well. - var maxConnections = - 5 + - Math.max( - Services.prefs.getIntPref( - "network.http.max-persistent-connections-per-server" - ), - Services.prefs.getIntPref( - "network.http.max-persistent-connections-per-proxy" - ) - ); - - try { - var loopback = true; - if ( - this._host != "127.0.0.1" && - this._host != "localhost" && - this._host != "[::1]" - ) { - loopback = false; - } - - // When automatically selecting a port, sometimes the chosen port is - // "blocked" from clients. We don't want to use these ports because - // tests will intermittently fail. So, we simply keep trying to to - // get a server socket until a valid port is obtained. We limit - // ourselves to finite attempts just so we don't loop forever. - var socket; - for (var i = 100; i; i--) { - var temp = null; - if (this._host.includes(":")) { - temp = new ServerSocketIPv6( - this._port, - loopback, // true = localhost, false = everybody - maxConnections - ); - } else { - temp = new ServerSocket( - this._port, - loopback, // true = localhost, false = everybody - maxConnections - ); - } - - var allowed = Services.io.allowPort(temp.port, "http"); - if (!allowed) { - dumpn( - ">>>Warning: obtained ServerSocket listens on a blocked " + - "port: " + - temp.port - ); - } - - if (!allowed && this._port == -1) { - dumpn(">>>Throwing away ServerSocket with bad port."); - temp.close(); - continue; - } - - socket = temp; - break; - } - - if (!socket) { - throw new Error( - "No socket server available. Are there no available ports?" - ); - } - - socket.asyncListen(this); - this._port = socket.port; - this._identity._initialize(socket.port, host, true); - this._socket = socket; - dumpn( - ">>> listening on port " + - socket.port + - ", " + - maxConnections + - " pending connections" - ); - } catch (e) { - dump("\n!!! could not start server on port " + port + ": " + e + "\n\n"); - throw Components.Exception("", Cr.NS_ERROR_NOT_AVAILABLE); - } - }, - - // - // see nsIHttpServer.stop - // - stop(callback) { - if (!this._socket) { - throw Components.Exception("", Cr.NS_ERROR_UNEXPECTED); - } - - // If no argument was provided to stop, return a promise. - let returnValue = undefined; - if (!callback) { - returnValue = new Promise(resolve => { - callback = resolve; - }); - } - - this._stopCallback = - typeof callback === "function" - ? callback - : function() { - callback.onStopped(); - }; - - dumpn(">>> stopping listening on port " + this._socket.port); - this._socket.close(); - this._socket = null; - - // We can't have this identity any more, and the port on which we're running - // this server now could be meaningless the next time around. - this._identity._teardown(); - - this._doQuit = false; - - // socket-close notification and pending request completion happen async - - return returnValue; - }, - - // - // see nsIHttpServer.registerFile - // - registerFile(path, file, handler) { - if (file && (!file.exists() || file.isDirectory())) { - throw Components.Exception("", Cr.NS_ERROR_INVALID_ARG); - } - - this._handler.registerFile(path, file, handler); - }, - - // - // see nsIHttpServer.registerDirectory - // - registerDirectory(path, directory) { - // XXX true path validation! - if ( - path.charAt(0) != "/" || - path.charAt(path.length - 1) != "/" || - (directory && (!directory.exists() || !directory.isDirectory())) - ) { - throw Components.Exception("", Cr.NS_ERROR_INVALID_ARG); - } - - // XXX determine behavior of nonexistent /foo/bar when a /foo/bar/ mapping - // exists! - - this._handler.registerDirectory(path, directory); - }, - - // - // see nsIHttpServer.registerPathHandler - // - registerPathHandler(path, handler) { - this._handler.registerPathHandler(path, handler); - }, - - // - // see nsIHttpServer.registerPrefixHandler - // - registerPrefixHandler(prefix, handler) { - this._handler.registerPrefixHandler(prefix, handler); - }, - - // - // see nsIHttpServer.registerErrorHandler - // - registerErrorHandler(code, handler) { - this._handler.registerErrorHandler(code, handler); - }, - - // - // see nsIHttpServer.setIndexHandler - // - setIndexHandler(handler) { - this._handler.setIndexHandler(handler); - }, - - // - // see nsIHttpServer.registerContentType - // - registerContentType(ext, type) { - this._handler.registerContentType(ext, type); - }, - - get connectionNumber() { - return this._connectionGen; - }, - - // - // see nsIHttpServer.serverIdentity - // - get identity() { - return this._identity; - }, - - // - // see nsIHttpServer.getState - // - getState(path, k) { - return this._handler._getState(path, k); - }, - - // - // see nsIHttpServer.setState - // - setState(path, k, v) { - return this._handler._setState(path, k, v); - }, - - // - // see nsIHttpServer.getSharedState - // - getSharedState(k) { - return this._handler._getSharedState(k); - }, - - // - // see nsIHttpServer.setSharedState - // - setSharedState(k, v) { - return this._handler._setSharedState(k, v); - }, - - // - // see nsIHttpServer.getObjectState - // - getObjectState(k) { - return this._handler._getObjectState(k); - }, - - // - // see nsIHttpServer.setObjectState - // - setObjectState(k, v) { - return this._handler._setObjectState(k, v); - }, - - get wrappedJSObject() { - return this; - }, - - // NSISUPPORTS - - // - // see nsISupports.QueryInterface - // - QueryInterface: ChromeUtils.generateQI([ - "nsIHttpServer", - "nsIServerSocketListener", - ]), - - // NON-XPCOM PUBLIC API - - /** - * Returns true iff this server is not running (and is not in the process of - * serving any requests still to be processed when the server was last - * stopped after being run). - */ - isStopped() { - return this._socketClosed && !this._hasOpenConnections(); - }, - - // PRIVATE IMPLEMENTATION - - /** True if this server has any open connections to it, false otherwise. */ - _hasOpenConnections() { - // - // If we have any open connections, they're tracked as numeric properties on - // |this._connections|. The non-standard __count__ property could be used - // to check whether there are any properties, but standard-wise, even - // looking forward to ES5, there's no less ugly yet still O(1) way to do - // this. - // - for (var n in this._connections) { - return true; - } - return false; - }, - - /** Calls the server-stopped callback provided when stop() was called. */ - _notifyStopped() { - NS_ASSERT(this._stopCallback !== null, "double-notifying?"); - NS_ASSERT(!this._hasOpenConnections(), "should be done serving by now"); - - // - // NB: We have to grab this now, null out the member, *then* call the - // callback here, or otherwise the callback could (indirectly) futz with - // this._stopCallback by starting and immediately stopping this, at - // which point we'd be nulling out a field we no longer have a right to - // modify. - // - var callback = this._stopCallback; - this._stopCallback = null; - try { - callback(); - } catch (e) { - // not throwing because this is specified as being usually (but not - // always) asynchronous - dump("!!! error running onStopped callback: " + e + "\n"); - } - }, - - /** - * Notifies this server that the given connection has been closed. - * - * @param connection : Connection - * the connection that was closed - */ - _connectionClosed(connection) { - NS_ASSERT( - connection.number in this._connections, - "closing a connection " + - this + - " that we never added to the " + - "set of open connections?" - ); - NS_ASSERT( - this._connections[connection.number] === connection, - "connection number mismatch? " + this._connections[connection.number] - ); - delete this._connections[connection.number]; - - // Fire a pending server-stopped notification if it's our responsibility. - if (!this._hasOpenConnections() && this._socketClosed) { - this._notifyStopped(); - } - }, - - /** - * Requests that the server be shut down when possible. - */ - _requestQuit() { - dumpn(">>> requesting a quit"); - dumpStack(); - this._doQuit = true; - }, -}; - -var HttpServer = nsHttpServer; - -class NodeServer { - // Executes command in the context of a node server. - // See handler in moz-http2.js - // - // Example use: - // let id = NodeServer.fork(); // id is a random string - // await NodeServer.execute(id, `"hello"`) - // > "hello" - // await NodeServer.execute(id, `(() => "hello")()`) - // > "hello" - // await NodeServer.execute(id, `(() => var_defined_on_server)()`) - // > "0" - // await NodeServer.execute(id, `var_defined_on_server`) - // > "0" - // function f(param) { if (param) return param; return "bla"; } - // await NodeServer.execute(id, f); // Defines the function on the server - // await NodeServer.execute(id, `f()`) // executes defined function - // > "bla" - // let result = await NodeServer.execute(id, `f("test")`); - // > "test" - // await NodeServer.kill(id); // shuts down the server - - // Forks a new node server using moz-http2-child.js as a starting point - static fork() { - return this.sendCommand("", "/fork"); - } - // Executes command in the context of the node server indicated by `id` - static execute(id, command) { - return this.sendCommand(command, `/execute/${id}`); - } - // Shuts down the server - static kill(id) { - return this.sendCommand("", `/kill/${id}`); - } - - // Issues a request to the node server (handler defined in moz-http2.js) - // This method should not be called directly. - static sendCommand(command, path) { - let env = Cc["@mozilla.org/process/environment;1"].getService( - Ci.nsIEnvironment - ); - let h2Port = env.get("MOZNODE_EXEC_PORT"); - if (!h2Port) { - throw new Error("Could not find MOZNODE_EXEC_PORT"); - } - - let req = new XMLHttpRequest(); - const serverIP = - AppConstants.platform == "android" ? "10.0.2.2" : "127.0.0.1"; - req.open("POST", `http://${serverIP}:${h2Port}${path}`); - - // Passing a function to NodeServer.execute will define that function - // in node. It can be called in a later execute command. - let isFunction = function(obj) { - return !!(obj && obj.constructor && obj.call && obj.apply); - }; - let payload = command; - if (isFunction(command)) { - payload = `${command.name} = ${command.toString()};`; - } - - return new Promise((resolve, reject) => { - req.onload = () => { - let x = null; - - if (req.statusText != "OK") { - reject(`XHR request failed: ${req.statusText}`); - return; - } - - try { - x = JSON.parse(req.responseText); - } catch (e) { - reject(`Failed to parse ${req.responseText} - ${e}`); - return; - } - - if (x.error) { - let e = new Error(x.error, "", 0); - e.stack = x.errorStack; - reject(e); - return; - } - resolve(x.result); - }; - req.onerror = e => { - reject(e); - }; - - req.send(payload.toString()); - }); - } -} - -// -// RFC 2396 section 3.2.2: -// -// host = hostname | IPv4address -// hostname = *( domainlabel "." ) toplabel [ "." ] -// domainlabel = alphanum | alphanum *( alphanum | "-" ) alphanum -// toplabel = alpha | alpha *( alphanum | "-" ) alphanum -// IPv4address = 1*digit "." 1*digit "." 1*digit "." 1*digit -// - -const HOST_REGEX = new RegExp( - "^(?:" + - // *( domainlabel "." ) - "(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)*" + - // toplabel - "[a-z](?:[a-z0-9-]*[a-z0-9])?" + - "|" + - // IPv4 address - "\\d+\\.\\d+\\.\\d+\\.\\d+" + - ")$", - "i" -); - -/** - * Represents the identity of a server. An identity consists of a set of - * (scheme, host, port) tuples denoted as locations (allowing a single server to - * serve multiple sites or to be used behind both HTTP and HTTPS proxies for any - * host/port). Any incoming request must be to one of these locations, or it - * will be rejected with an HTTP 400 error. One location, denoted as the - * primary location, is the location assigned in contexts where a location - * cannot otherwise be endogenously derived, such as for HTTP/1.0 requests. - * - * A single identity may contain at most one location per unique host/port pair; - * other than that, no restrictions are placed upon what locations may - * constitute an identity. - */ -function ServerIdentity() { - /** The scheme of the primary location. */ - this._primaryScheme = "http"; - - /** The hostname of the primary location. */ - this._primaryHost = "127.0.0.1"; - - /** The port number of the primary location. */ - this._primaryPort = -1; - - /** - * The current port number for the corresponding server, stored so that a new - * primary location can always be set if the current one is removed. - */ - this._defaultPort = -1; - - /** - * Maps hosts to maps of ports to schemes, e.g. the following would represent - * https://example.com:789/ and http://example.org/: - * - * { - * "xexample.com": { 789: "https" }, - * "xexample.org": { 80: "http" } - * } - * - * Note the "x" prefix on hostnames, which prevents collisions with special - * JS names like "prototype". - */ - this._locations = { xlocalhost: {} }; -} -ServerIdentity.prototype = { - // NSIHTTPSERVERIDENTITY - - // - // see nsIHttpServerIdentity.primaryScheme - // - get primaryScheme() { - if (this._primaryPort === -1) { - throw Components.Exception("", Cr.NS_ERROR_NOT_INITIALIZED); - } - return this._primaryScheme; - }, - - // - // see nsIHttpServerIdentity.primaryHost - // - get primaryHost() { - if (this._primaryPort === -1) { - throw Components.Exception("", Cr.NS_ERROR_NOT_INITIALIZED); - } - return this._primaryHost; - }, - - // - // see nsIHttpServerIdentity.primaryPort - // - get primaryPort() { - if (this._primaryPort === -1) { - throw Components.Exception("", Cr.NS_ERROR_NOT_INITIALIZED); - } - return this._primaryPort; - }, - - // - // see nsIHttpServerIdentity.add - // - add(scheme, host, port) { - this._validate(scheme, host, port); - - var entry = this._locations["x" + host]; - if (!entry) { - this._locations["x" + host] = entry = {}; - } - - entry[port] = scheme; - }, - - // - // see nsIHttpServerIdentity.remove - // - remove(scheme, host, port) { - this._validate(scheme, host, port); - - var entry = this._locations["x" + host]; - if (!entry) { - return false; - } - - var present = port in entry; - delete entry[port]; - - if ( - this._primaryScheme == scheme && - this._primaryHost == host && - this._primaryPort == port && - this._defaultPort !== -1 - ) { - // Always keep at least one identity in existence at any time, unless - // we're in the process of shutting down (the last condition above). - this._primaryPort = -1; - this._initialize(this._defaultPort, host, false); - } - - return present; - }, - - // - // see nsIHttpServerIdentity.has - // - has(scheme, host, port) { - this._validate(scheme, host, port); - - return ( - "x" + host in this._locations && - scheme === this._locations["x" + host][port] - ); - }, - - // - // see nsIHttpServerIdentity.has - // - getScheme(host, port) { - this._validate("http", host, port); - - var entry = this._locations["x" + host]; - if (!entry) { - return ""; - } - - return entry[port] || ""; - }, - - // - // see nsIHttpServerIdentity.setPrimary - // - setPrimary(scheme, host, port) { - this._validate(scheme, host, port); - - this.add(scheme, host, port); - - this._primaryScheme = scheme; - this._primaryHost = host; - this._primaryPort = port; - }, - - // NSISUPPORTS - - // - // see nsISupports.QueryInterface - // - QueryInterface: ChromeUtils.generateQI(["nsIHttpServerIdentity"]), - - // PRIVATE IMPLEMENTATION - - /** - * Initializes the primary name for the corresponding server, based on the - * provided port number. - */ - _initialize(port, host, addSecondaryDefault) { - this._host = host; - if (this._primaryPort !== -1) { - this.add("http", host, port); - } else { - this.setPrimary("http", "localhost", port); - } - this._defaultPort = port; - - // Only add this if we're being called at server startup - if (addSecondaryDefault && host != "127.0.0.1") { - if (host.includes(":")) { - this.add("http", "[::1]", port); - } else { - this.add("http", "127.0.0.1", port); - } - } - }, - - /** - * Called at server shutdown time, unsets the primary location only if it was - * the default-assigned location and removes the default location from the - * set of locations used. - */ - _teardown() { - if (this._host != "127.0.0.1") { - // Not the default primary location, nothing special to do here - this.remove("http", "127.0.0.1", this._defaultPort); - } - - // This is a *very* tricky bit of reasoning here; make absolutely sure the - // tests for this code pass before you commit changes to it. - if ( - this._primaryScheme == "http" && - this._primaryHost == this._host && - this._primaryPort == this._defaultPort - ) { - // Make sure we don't trigger the readding logic in .remove(), then remove - // the default location. - var port = this._defaultPort; - this._defaultPort = -1; - this.remove("http", this._host, port); - - // Ensure a server start triggers the setPrimary() path in ._initialize() - this._primaryPort = -1; - } else { - // No reason not to remove directly as it's not our primary location - this.remove("http", this._host, this._defaultPort); - } - }, - - /** - * Ensures scheme, host, and port are all valid with respect to RFC 2396. - * - * @throws NS_ERROR_ILLEGAL_VALUE - * if any argument doesn't match the corresponding production - */ - _validate(scheme, host, port) { - if (scheme !== "http" && scheme !== "https") { - dumpn("*** server only supports http/https schemes: '" + scheme + "'"); - dumpStack(); - throw Components.Exception("", Cr.NS_ERROR_ILLEGAL_VALUE); - } - if (!HOST_REGEX.test(host) && host != "[::1]") { - dumpn("*** unexpected host: '" + host + "'"); - throw Components.Exception("", Cr.NS_ERROR_ILLEGAL_VALUE); - } - if (port < 0 || port > 65535) { - dumpn("*** unexpected port: '" + port + "'"); - throw Components.Exception("", Cr.NS_ERROR_ILLEGAL_VALUE); - } - }, -}; - -/** - * Represents a connection to the server (and possibly in the future the thread - * on which the connection is processed). - * - * @param input : nsIInputStream - * stream from which incoming data on the connection is read - * @param output : nsIOutputStream - * stream to write data out the connection - * @param server : nsHttpServer - * the server handling the connection - * @param port : int - * the port on which the server is running - * @param outgoingPort : int - * the outgoing port used by this connection - * @param number : uint - * a serial number used to uniquely identify this connection - */ -function Connection( - input, - output, - server, - port, - outgoingPort, - number, - transport -) { - dumpn("*** opening new connection " + number + " on port " + outgoingPort); - - /** Stream of incoming data. */ - this.input = input; - - /** Stream for outgoing data. */ - this.output = output; - - /** The server associated with this request. */ - this.server = server; - - /** The port on which the server is running. */ - this.port = port; - - /** The outgoing poort used by this connection. */ - this._outgoingPort = outgoingPort; - - /** The serial number of this connection. */ - this.number = number; - - /** Reference to the underlying transport. */ - this.transport = transport; - - /** - * The request for which a response is being generated, null if the - * incoming request has not been fully received or if it had errors. - */ - this.request = null; - - /** This allows a connection to disambiguate between a peer initiating a - * close and the socket being forced closed on shutdown. - */ - this._closed = false; - - /** State variable for debugging. */ - this._processed = false; - - /** whether or not 1st line of request has been received */ - this._requestStarted = false; -} -Connection.prototype = { - /** Closes this connection's input/output streams. */ - close() { - if (this._closed) { - return; - } - - dumpn( - "*** closing connection " + this.number + " on port " + this._outgoingPort - ); - - this.input.close(); - this.output.close(); - this._closed = true; - - var server = this.server; - server._connectionClosed(this); - - // If an error triggered a server shutdown, act on it now - if (server._doQuit) { - server.stop(function() { - /* not like we can do anything better */ - }); - } - }, - - /** - * Initiates processing of this connection, using the data in the given - * request. - * - * @param request : Request - * the request which should be processed - */ - process(request) { - NS_ASSERT(!this._closed && !this._processed); - - this._processed = true; - - this.request = request; - this.server._handler.handleResponse(this); - }, - - /** - * Initiates processing of this connection, generating a response with the - * given HTTP error code. - * - * @param code : uint - * an HTTP code, so in the range [0, 1000) - * @param request : Request - * incomplete data about the incoming request (since there were errors - * during its processing - */ - processError(code, request) { - NS_ASSERT(!this._closed && !this._processed); - - this._processed = true; - this.request = request; - this.server._handler.handleError(code, this); - }, - - /** Converts this to a string for debugging purposes. */ - toString() { - return ( - "" - ); - }, - - requestStarted() { - this._requestStarted = true; - }, -}; - -/** Returns an array of count bytes from the given input stream. */ -function readBytes(inputStream, count) { - return new BinaryInputStream(inputStream).readByteArray(count); -} - -/** Request reader processing states; see RequestReader for details. */ -const READER_IN_REQUEST_LINE = 0; -const READER_IN_HEADERS = 1; -const READER_IN_BODY = 2; -const READER_FINISHED = 3; - -/** - * Reads incoming request data asynchronously, does any necessary preprocessing, - * and forwards it to the request handler. Processing occurs in three states: - * - * READER_IN_REQUEST_LINE Reading the request's status line - * READER_IN_HEADERS Reading headers in the request - * READER_IN_BODY Reading the body of the request - * READER_FINISHED Entire request has been read and processed - * - * During the first two stages, initial metadata about the request is gathered - * into a Request object. Once the status line and headers have been processed, - * we start processing the body of the request into the Request. Finally, when - * the entire body has been read, we create a Response and hand it off to the - * ServerHandler to be given to the appropriate request handler. - * - * @param connection : Connection - * the connection for the request being read - */ -function RequestReader(connection) { - /** Connection metadata for this request. */ - this._connection = connection; - - /** - * A container providing line-by-line access to the raw bytes that make up the - * data which has been read from the connection but has not yet been acted - * upon (by passing it to the request handler or by extracting request - * metadata from it). - */ - this._data = new LineData(); - - /** - * The amount of data remaining to be read from the body of this request. - * After all headers in the request have been read this is the value in the - * Content-Length header, but as the body is read its value decreases to zero. - */ - this._contentLength = 0; - - /** The current state of parsing the incoming request. */ - this._state = READER_IN_REQUEST_LINE; - - /** Metadata constructed from the incoming request for the request handler. */ - this._metadata = new Request(connection.port); - - /** - * Used to preserve state if we run out of line data midway through a - * multi-line header. _lastHeaderName stores the name of the header, while - * _lastHeaderValue stores the value we've seen so far for the header. - * - * These fields are always either both undefined or both strings. - */ - this._lastHeaderName = this._lastHeaderValue = undefined; -} -RequestReader.prototype = { - // NSIINPUTSTREAMCALLBACK - - /** - * Called when more data from the incoming request is available. This method - * then reads the available data from input and deals with that data as - * necessary, depending upon the syntax of already-downloaded data. - * - * @param input : nsIAsyncInputStream - * the stream of incoming data from the connection - */ - onInputStreamReady(input) { - dumpn( - "*** onInputStreamReady(input=" + - input + - ") on thread " + - gThreadManager.currentThread + - " (main is " + - gThreadManager.mainThread + - ")" - ); - dumpn("*** this._state == " + this._state); - - // Handle cases where we get more data after a request error has been - // discovered but *before* we can close the connection. - var data = this._data; - if (!data) { - return; - } - - try { - data.appendBytes(readBytes(input, input.available())); - } catch (e) { - if (streamClosed(e)) { - dumpn( - "*** WARNING: unexpected error when reading from socket; will " + - "be treated as if the input stream had been closed" - ); - dumpn("*** WARNING: actual error was: " + e); - } - - // We've lost a race -- input has been closed, but we're still expecting - // to read more data. available() will throw in this case, and since - // we're dead in the water now, destroy the connection. - dumpn( - "*** onInputStreamReady called on a closed input, destroying " + - "connection" - ); - this._connection.close(); - return; - } - - switch (this._state) { - default: - NS_ASSERT(false, "invalid state: " + this._state); - break; - - case READER_IN_REQUEST_LINE: - if (!this._processRequestLine()) { - break; - } - /* fall through */ - - case READER_IN_HEADERS: - if (!this._processHeaders()) { - break; - } - /* fall through */ - - case READER_IN_BODY: - this._processBody(); - } - - if (this._state != READER_FINISHED) { - input.asyncWait(this, 0, 0, gThreadManager.currentThread); - } - }, - - // - // see nsISupports.QueryInterface - // - QueryInterface: ChromeUtils.generateQI(["nsIInputStreamCallback"]), - - // PRIVATE API - - /** - * Processes unprocessed, downloaded data as a request line. - * - * @returns boolean - * true iff the request line has been fully processed - */ - _processRequestLine() { - NS_ASSERT(this._state == READER_IN_REQUEST_LINE); - - // Servers SHOULD ignore any empty line(s) received where a Request-Line - // is expected (section 4.1). - var data = this._data; - var line = {}; - var readSuccess; - while ((readSuccess = data.readLine(line)) && line.value == "") { - dumpn("*** ignoring beginning blank line..."); - } - - // if we don't have a full line, wait until we do - if (!readSuccess) { - return false; - } - - // we have the first non-blank line - try { - this._parseRequestLine(line.value); - this._state = READER_IN_HEADERS; - this._connection.requestStarted(); - return true; - } catch (e) { - this._handleError(e); - return false; - } - }, - - /** - * Processes stored data, assuming it is either at the beginning or in - * the middle of processing request headers. - * - * @returns boolean - * true iff header data in the request has been fully processed - */ - _processHeaders() { - NS_ASSERT(this._state == READER_IN_HEADERS); - - // XXX things to fix here: - // - // - need to support RFC 2047-encoded non-US-ASCII characters - - try { - var done = this._parseHeaders(); - if (done) { - var request = this._metadata; - - // XXX this is wrong for requests with transfer-encodings applied to - // them, particularly chunked (which by its nature can have no - // meaningful Content-Length header)! - this._contentLength = request.hasHeader("Content-Length") - ? parseInt(request.getHeader("Content-Length"), 10) - : 0; - dumpn("_processHeaders, Content-length=" + this._contentLength); - - this._state = READER_IN_BODY; - } - return done; - } catch (e) { - this._handleError(e); - return false; - } - }, - - /** - * Processes stored data, assuming it is either at the beginning or in - * the middle of processing the request body. - * - * @returns boolean - * true iff the request body has been fully processed - */ - _processBody() { - NS_ASSERT(this._state == READER_IN_BODY); - - // XXX handle chunked transfer-coding request bodies! - - try { - if (this._contentLength > 0) { - var data = this._data.purge(); - var count = Math.min(data.length, this._contentLength); - dumpn( - "*** loading data=" + - data + - " len=" + - data.length + - " excess=" + - (data.length - count) - ); - data.length = count; - - var bos = new BinaryOutputStream(this._metadata._bodyOutputStream); - bos.writeByteArray(data); - this._contentLength -= count; - } - - dumpn("*** remaining body data len=" + this._contentLength); - if (this._contentLength == 0) { - this._validateRequest(); - this._state = READER_FINISHED; - this._handleResponse(); - return true; - } - - return false; - } catch (e) { - this._handleError(e); - return false; - } - }, - - /** - * Does various post-header checks on the data in this request. - * - * @throws : HttpError - * if the request was malformed in some way - */ - _validateRequest() { - NS_ASSERT(this._state == READER_IN_BODY); - - dumpn("*** _validateRequest"); - - var metadata = this._metadata; - var headers = metadata._headers; - - // 19.6.1.1 -- servers MUST report 400 to HTTP/1.1 requests w/o Host header - var identity = this._connection.server.identity; - if (metadata._httpVersion.atLeast(nsHttpVersion.HTTP_1_1)) { - if (!headers.hasHeader("Host")) { - dumpn("*** malformed HTTP/1.1 or greater request with no Host header!"); - throw HTTP_400; - } - - // If the Request-URI wasn't absolute, then we need to determine our host. - // We have to determine what scheme was used to access us based on the - // server identity data at this point, because the request just doesn't - // contain enough data on its own to do this, sadly. - if (!metadata._host) { - var host, port; - var hostPort = headers.getHeader("Host"); - var colon = hostPort.lastIndexOf(":"); - if (hostPort.lastIndexOf("]") > colon) { - colon = -1; - } - if (colon < 0) { - host = hostPort; - port = ""; - } else { - host = hostPort.substring(0, colon); - port = hostPort.substring(colon + 1); - } - - // NB: We allow an empty port here because, oddly, a colon may be - // present even without a port number, e.g. "example.com:"; in this - // case the default port applies. - if ( - (!HOST_REGEX.test(host) && host != "[::1]") || - !/^\d*$/.test(port) - ) { - dumpn( - "*** malformed hostname (" + - hostPort + - ") in Host " + - "header, 400 time" - ); - throw HTTP_400; - } - - // If we're not given a port, we're stuck, because we don't know what - // scheme to use to look up the correct port here, in general. Since - // the HTTPS case requires a tunnel/proxy and thus requires that the - // requested URI be absolute (and thus contain the necessary - // information), let's assume HTTP will prevail and use that. - port = +port || 80; - - var scheme = identity.getScheme(host, port); - if (!scheme) { - dumpn( - "*** unrecognized hostname (" + - hostPort + - ") in Host " + - "header, 400 time" - ); - throw HTTP_400; - } - - metadata._scheme = scheme; - metadata._host = host; - metadata._port = port; - } - } else { - NS_ASSERT( - metadata._host === undefined, - "HTTP/1.0 doesn't allow absolute paths in the request line!" - ); - - metadata._scheme = identity.primaryScheme; - metadata._host = identity.primaryHost; - metadata._port = identity.primaryPort; - } - - NS_ASSERT( - identity.has(metadata._scheme, metadata._host, metadata._port), - "must have a location we recognize by now!" - ); - }, - - /** - * Handles responses in case of error, either in the server or in the request. - * - * @param e - * the specific error encountered, which is an HttpError in the case where - * the request is in some way invalid or cannot be fulfilled; if this isn't - * an HttpError we're going to be paranoid and shut down, because that - * shouldn't happen, ever - */ - _handleError(e) { - // Don't fall back into normal processing! - this._state = READER_FINISHED; - - var server = this._connection.server; - if (e instanceof HttpError) { - var code = e.code; - } else { - dumpn( - "!!! UNEXPECTED ERROR: " + - e + - (e.lineNumber ? ", line " + e.lineNumber : "") - ); - - // no idea what happened -- be paranoid and shut down - code = 500; - server._requestQuit(); - } - - // make attempted reuse of data an error - this._data = null; - - this._connection.processError(code, this._metadata); - }, - - /** - * Now that we've read the request line and headers, we can actually hand off - * the request to be handled. - * - * This method is called once per request, after the request line and all - * headers and the body, if any, have been received. - */ - _handleResponse() { - NS_ASSERT(this._state == READER_FINISHED); - - // We don't need the line-based data any more, so make attempted reuse an - // error. - this._data = null; - - this._connection.process(this._metadata); - }, - - // PARSING - - /** - * Parses the request line for the HTTP request associated with this. - * - * @param line : string - * the request line - */ - _parseRequestLine(line) { - NS_ASSERT(this._state == READER_IN_REQUEST_LINE); - - dumpn("*** _parseRequestLine('" + line + "')"); - - var metadata = this._metadata; - - // clients and servers SHOULD accept any amount of SP or HT characters - // between fields, even though only a single SP is required (section 19.3) - var request = line.split(/[ \t]+/); - if (!request || request.length != 3) { - dumpn("*** No request in line"); - throw HTTP_400; - } - - metadata._method = request[0]; - - // get the HTTP version - var ver = request[2]; - var match = ver.match(/^HTTP\/(\d+\.\d+)$/); - if (!match) { - dumpn("*** No HTTP version in line"); - throw HTTP_400; - } - - // determine HTTP version - try { - metadata._httpVersion = new nsHttpVersion(match[1]); - if (!metadata._httpVersion.atLeast(nsHttpVersion.HTTP_1_0)) { - throw new Error("unsupported HTTP version"); - } - } catch (e) { - // we support HTTP/1.0 and HTTP/1.1 only - throw HTTP_501; - } - - var fullPath = request[1]; - - if (metadata._method == "CONNECT") { - metadata._path = "CONNECT"; - metadata._scheme = "https"; - [metadata._host, metadata._port] = fullPath.split(":"); - return; - } - - var serverIdentity = this._connection.server.identity; - var scheme, host, port; - - if (fullPath.charAt(0) != "/") { - // No absolute paths in the request line in HTTP prior to 1.1 - if (!metadata._httpVersion.atLeast(nsHttpVersion.HTTP_1_1)) { - dumpn("*** Metadata version too low"); - throw HTTP_400; - } - - try { - var uri = Services.io.newURI(fullPath); - fullPath = uri.pathQueryRef; - scheme = uri.scheme; - host = metadata._host = uri.asciiHost; - port = uri.port; - if (port === -1) { - if (scheme === "http") { - port = 80; - } else if (scheme === "https") { - port = 443; - } else { - dumpn("*** Unknown scheme: " + scheme); - throw HTTP_400; - } - } - } catch (e) { - // If the host is not a valid host on the server, the response MUST be a - // 400 (Bad Request) error message (section 5.2). Alternately, the URI - // is malformed. - dumpn("*** Threw when dealing with URI: " + e); - throw HTTP_400; - } - - if ( - !serverIdentity.has(scheme, host, port) || - fullPath.charAt(0) != "/" - ) { - dumpn("*** serverIdentity unknown or path does not start with '/'"); - throw HTTP_400; - } - } - - var splitter = fullPath.indexOf("?"); - if (splitter < 0) { - // _queryString already set in ctor - metadata._path = fullPath; - } else { - metadata._path = fullPath.substring(0, splitter); - metadata._queryString = fullPath.substring(splitter + 1); - } - - metadata._scheme = scheme; - metadata._host = host; - metadata._port = port; - }, - - /** - * Parses all available HTTP headers in this until the header-ending CRLFCRLF, - * adding them to the store of headers in the request. - * - * @throws - * HTTP_400 if the headers are malformed - * @returns boolean - * true if all headers have now been processed, false otherwise - */ - _parseHeaders() { - NS_ASSERT(this._state == READER_IN_HEADERS); - - dumpn("*** _parseHeaders"); - - var data = this._data; - - var headers = this._metadata._headers; - var lastName = this._lastHeaderName; - var lastVal = this._lastHeaderValue; - - var line = {}; - while (true) { - dumpn("*** Last name: '" + lastName + "'"); - dumpn("*** Last val: '" + lastVal + "'"); - NS_ASSERT( - !((lastVal === undefined) ^ (lastName === undefined)), - lastName === undefined - ? "lastVal without lastName? lastVal: '" + lastVal + "'" - : "lastName without lastVal? lastName: '" + lastName + "'" - ); - - if (!data.readLine(line)) { - // save any data we have from the header we might still be processing - this._lastHeaderName = lastName; - this._lastHeaderValue = lastVal; - return false; - } - - var lineText = line.value; - dumpn("*** Line text: '" + lineText + "'"); - var firstChar = lineText.charAt(0); - - // blank line means end of headers - if (lineText == "") { - // we're finished with the previous header - if (lastName) { - try { - headers.setHeader(lastName, lastVal, true); - } catch (e) { - dumpn("*** setHeader threw on last header, e == " + e); - throw HTTP_400; - } - } else { - // no headers in request -- valid for HTTP/1.0 requests - } - - // either way, we're done processing headers - this._state = READER_IN_BODY; - return true; - } else if (firstChar == " " || firstChar == "\t") { - // multi-line header if we've already seen a header line - if (!lastName) { - dumpn("We don't have a header to continue!"); - throw HTTP_400; - } - - // append this line's text to the value; starts with SP/HT, so no need - // for separating whitespace - lastVal += lineText; - } else { - // we have a new header, so set the old one (if one existed) - if (lastName) { - try { - headers.setHeader(lastName, lastVal, true); - } catch (e) { - dumpn("*** setHeader threw on a header, e == " + e); - throw HTTP_400; - } - } - - var colon = lineText.indexOf(":"); // first colon must be splitter - if (colon < 1) { - dumpn("*** No colon or missing header field-name"); - throw HTTP_400; - } - - // set header name, value (to be set in the next loop, usually) - lastName = lineText.substring(0, colon); - lastVal = lineText.substring(colon + 1); - } // empty, continuation, start of header - } // while (true) - }, -}; - -/** The character codes for CR and LF. */ -const CR = 0x0d, - LF = 0x0a; - -/** - * Calculates the number of characters before the first CRLF pair in array, or - * -1 if the array contains no CRLF pair. - * - * @param array : Array - * an array of numbers in the range [0, 256), each representing a single - * character; the first CRLF is the lowest index i where - * |array[i] == "\r".charCodeAt(0)| and |array[i+1] == "\n".charCodeAt(0)|, - * if such an |i| exists, and -1 otherwise - * @param start : uint - * start index from which to begin searching in array - * @returns int - * the index of the first CRLF if any were present, -1 otherwise - */ -function findCRLF(array, start) { - for (var i = array.indexOf(CR, start); i >= 0; i = array.indexOf(CR, i + 1)) { - if (array[i + 1] == LF) { - return i; - } - } - return -1; -} - -/** - * A container which provides line-by-line access to the arrays of bytes with - * which it is seeded. - */ -function LineData() { - /** An array of queued bytes from which to get line-based characters. */ - this._data = []; - - /** Start index from which to search for CRLF. */ - this._start = 0; -} -LineData.prototype = { - /** - * Appends the bytes in the given array to the internal data cache maintained - * by this. - */ - appendBytes(bytes) { - var count = bytes.length; - var quantum = 262144; // just above half SpiderMonkey's argument-count limit - if (count < quantum) { - Array.prototype.push.apply(this._data, bytes); - return; - } - - // Large numbers of bytes may cause Array.prototype.push to be called with - // more arguments than the JavaScript engine supports. In that case append - // bytes in fixed-size amounts until all bytes are appended. - for (var start = 0; start < count; start += quantum) { - var slice = bytes.slice(start, Math.min(start + quantum, count)); - Array.prototype.push.apply(this._data, slice); - } - }, - - /** - * Removes and returns a line of data, delimited by CRLF, from this. - * - * @param out - * an object whose "value" property will be set to the first line of text - * present in this, sans CRLF, if this contains a full CRLF-delimited line - * of text; if this doesn't contain enough data, the value of the property - * is undefined - * @returns boolean - * true if a full line of data could be read from the data in this, false - * otherwise - */ - readLine(out) { - var data = this._data; - var length = findCRLF(data, this._start); - if (length < 0) { - this._start = data.length; - - // But if our data ends in a CR, we have to back up one, because - // the first byte in the next packet might be an LF and if we - // start looking at data.length we won't find it. - if (data.length > 0 && data[data.length - 1] === CR) { - --this._start; - } - - return false; - } - - // Reset for future lines. - this._start = 0; - - // - // We have the index of the CR, so remove all the characters, including - // CRLF, from the array with splice, and convert the removed array - // (excluding the trailing CRLF characters) into the corresponding string. - // - var leading = data.splice(0, length + 2); - var quantum = 262144; - var line = ""; - for (var start = 0; start < length; start += quantum) { - var slice = leading.slice(start, Math.min(start + quantum, length)); - line += String.fromCharCode.apply(null, slice); - } - - out.value = line; - return true; - }, - - /** - * Removes the bytes currently within this and returns them in an array. - * - * @returns Array - * the bytes within this when this method is called - */ - purge() { - var data = this._data; - this._data = []; - return data; - }, -}; - -/** - * Creates a request-handling function for an nsIHttpRequestHandler object. - */ -function createHandlerFunc(handler) { - return function(metadata, response) { - handler.handle(metadata, response); - }; -} - -/** - * The default handler for directories; writes an HTML response containing a - * slightly-formatted directory listing. - */ -function defaultIndexHandler(metadata, response) { - response.setHeader("Content-Type", "text/html;charset=utf-8", false); - - var path = htmlEscape(decodeURI(metadata.path)); - - // - // Just do a very basic bit of directory listings -- no need for too much - // fanciness, especially since we don't have a style sheet in which we can - // stick rules (don't want to pollute the default path-space). - // - - var body = - "\ - \ - " + - path + - "\ - \ - \ -

" + - path + - '

\ -
    '; - - var directory = metadata.getProperty("directory"); - NS_ASSERT(directory && directory.isDirectory()); - - var fileList = []; - var files = directory.directoryEntries; - while (files.hasMoreElements()) { - var f = files.nextFile; - let name = f.leafName; - if ( - !f.isHidden() && - (name.charAt(name.length - 1) != HIDDEN_CHAR || - name.charAt(name.length - 2) == HIDDEN_CHAR) - ) { - fileList.push(f); - } - } - - fileList.sort(fileSort); - - for (var i = 0; i < fileList.length; i++) { - var file = fileList[i]; - try { - let name = file.leafName; - if (name.charAt(name.length - 1) == HIDDEN_CHAR) { - name = name.substring(0, name.length - 1); - } - var sep = file.isDirectory() ? "/" : ""; - - // Note: using " to delimit the attribute here because encodeURIComponent - // passes through '. - var item = - '
  1. ' + - htmlEscape(name) + - sep + - "
    2. "; - - body += item; - } catch (e) { - /* some file system error, ignore the file */ - } - } - - body += "
\ - \ - "; - - response.bodyOutputStream.write(body, body.length); -} - -/** - * Sorts a and b (nsIFile objects) into an aesthetically pleasing order. - */ -function fileSort(a, b) { - var dira = a.isDirectory(), - dirb = b.isDirectory(); - - if (dira && !dirb) { - return -1; - } - if (dirb && !dira) { - return 1; - } - - var namea = a.leafName.toLowerCase(), - nameb = b.leafName.toLowerCase(); - return nameb > namea ? -1 : 1; -} - -/** - * Converts an externally-provided path into an internal path for use in - * determining file mappings. - * - * @param path - * the path to convert - * @param encoded - * true if the given path should be passed through decodeURI prior to - * conversion - * @throws URIError - * if path is incorrectly encoded - */ -function toInternalPath(path, encoded) { - if (encoded) { - path = decodeURI(path); - } - - var comps = path.split("/"); - for (var i = 0, sz = comps.length; i < sz; i++) { - var comp = comps[i]; - if (comp.charAt(comp.length - 1) == HIDDEN_CHAR) { - comps[i] = comp + HIDDEN_CHAR; - } - } - return comps.join("/"); -} - -const PERMS_READONLY = (4 << 6) | (4 << 3) | 4; - -/** - * Adds custom-specified headers for the given file to the given response, if - * any such headers are specified. - * - * @param file - * the file on the disk which is to be written - * @param metadata - * metadata about the incoming request - * @param response - * the Response to which any specified headers/data should be written - * @throws HTTP_500 - * if an error occurred while processing custom-specified headers - */ -function maybeAddHeadersInternal( - file, - metadata, - response, - informationalResponse -) { - var name = file.leafName; - if (name.charAt(name.length - 1) == HIDDEN_CHAR) { - name = name.substring(0, name.length - 1); - } - - var headerFile = file.parent; - if (!informationalResponse) { - headerFile.append(name + HEADERS_SUFFIX); - } else { - headerFile.append(name + INFORMATIONAL_RESPONSE_SUFFIX); - } - - if (!headerFile.exists()) { - return; - } - - const PR_RDONLY = 0x01; - var fis = new FileInputStream( - headerFile, - PR_RDONLY, - PERMS_READONLY, - Ci.nsIFileInputStream.CLOSE_ON_EOF - ); - - try { - var lis = new ConverterInputStream(fis, "UTF-8", 1024, 0x0); - lis.QueryInterface(Ci.nsIUnicharLineInputStream); - - var line = { value: "" }; - var more = lis.readLine(line); - - if (!more && line.value == "") { - return; - } - - // request line - - var status = line.value; - if (status.indexOf("HTTP ") == 0) { - status = status.substring(5); - var space = status.indexOf(" "); - var code, description; - if (space < 0) { - code = status; - description = ""; - } else { - code = status.substring(0, space); - description = status.substring(space + 1, status.length); - } - - if (!informationalResponse) { - response.setStatusLine( - metadata.httpVersion, - parseInt(code, 10), - description - ); - } else { - response.setInformationalResponseStatusLine( - metadata.httpVersion, - parseInt(code, 10), - description - ); - } - - line.value = ""; - more = lis.readLine(line); - } else if (informationalResponse) { - // An informational response must have a status line. - return; - } - - // headers - while (more || line.value != "") { - var header = line.value; - var colon = header.indexOf(":"); - - if (!informationalResponse) { - response.setHeader( - header.substring(0, colon), - header.substring(colon + 1, header.length), - false - ); // allow overriding server-set headers - } else { - response.setInformationalResponseHeader( - header.substring(0, colon), - header.substring(colon + 1, header.length), - false - ); // allow overriding server-set headers - } - - line.value = ""; - more = lis.readLine(line); - } - } catch (e) { - dumpn("WARNING: error in headers for " + metadata.path + ": " + e); - throw HTTP_500; - } finally { - fis.close(); - } -} - -function maybeAddHeaders(file, metadata, response) { - maybeAddHeadersInternal(file, metadata, response, false); -} - -function maybeAddInformationalResponse(file, metadata, response) { - maybeAddHeadersInternal(file, metadata, response, true); -} - -/** - * An object which handles requests for a server, executing default and - * overridden behaviors as instructed by the code which uses and manipulates it. - * Default behavior includes the paths / and /trace (diagnostics), with some - * support for HTTP error pages for various codes and fallback to HTTP 500 if - * those codes fail for any reason. - * - * @param server : nsHttpServer - * the server in which this handler is being used - */ -function ServerHandler(server) { - // FIELDS - - /** - * The nsHttpServer instance associated with this handler. - */ - this._server = server; - - /** - * A FileMap object containing the set of path->nsIFile mappings for - * all directory mappings set in the server (e.g., "/" for /var/www/html/, - * "/foo/bar/" for /local/path/, and "/foo/bar/baz/" for /local/path2). - * - * Note carefully: the leading and trailing "/" in each path (not file) are - * removed before insertion to simplify the code which uses this. You have - * been warned! - */ - this._pathDirectoryMap = new FileMap(); - - /** - * Custom request handlers for the server in which this resides. Path-handler - * pairs are stored as property-value pairs in this property. - * - * @see ServerHandler.prototype._defaultPaths - */ - this._overridePaths = {}; - - /** - * Custom request handlers for the path prefixes on the server in which this - * resides. Path-handler pairs are stored as property-value pairs in this - * property. - * - * @see ServerHandler.prototype._defaultPaths - */ - this._overridePrefixes = {}; - - /** - * Custom request handlers for the error handlers in the server in which this - * resides. Path-handler pairs are stored as property-value pairs in this - * property. - * - * @see ServerHandler.prototype._defaultErrors - */ - this._overrideErrors = {}; - - /** - * Maps file extensions to their MIME types in the server, overriding any - * mapping that might or might not exist in the MIME service. - */ - this._mimeMappings = {}; - - /** - * The default handler for requests for directories, used to serve directories - * when no index file is present. - */ - this._indexHandler = defaultIndexHandler; - - /** Per-path state storage for the server. */ - this._state = {}; - - /** Entire-server state storage. */ - this._sharedState = {}; - - /** Entire-server state storage for nsISupports values. */ - this._objectState = {}; -} -ServerHandler.prototype = { - // PUBLIC API - - /** - * Handles a request to this server, responding to the request appropriately - * and initiating server shutdown if necessary. - * - * This method never throws an exception. - * - * @param connection : Connection - * the connection for this request - */ - handleResponse(connection) { - var request = connection.request; - var response = new Response(connection); - - var path = request.path; - dumpn("*** path == " + path); - - try { - try { - if (path in this._overridePaths) { - // explicit paths first, then files based on existing directory mappings, - // then (if the file doesn't exist) built-in server default paths - dumpn("calling override for " + path); - this._overridePaths[path](request, response); - } else { - var longestPrefix = ""; - for (let prefix in this._overridePrefixes) { - if ( - prefix.length > longestPrefix.length && - path.substr(0, prefix.length) == prefix - ) { - longestPrefix = prefix; - } - } - if (longestPrefix.length > 0) { - dumpn("calling prefix override for " + longestPrefix); - this._overridePrefixes[longestPrefix](request, response); - } else { - this._handleDefault(request, response); - } - } - } catch (e) { - if (response.partiallySent()) { - response.abort(e); - return; - } - - if (!(e instanceof HttpError)) { - dumpn("*** unexpected error: e == " + e); - throw HTTP_500; - } - if (e.code !== 404) { - throw e; - } - - dumpn("*** default: " + (path in this._defaultPaths)); - - response = new Response(connection); - if (path in this._defaultPaths) { - this._defaultPaths[path](request, response); - } else { - throw HTTP_404; - } - } - } catch (e) { - if (response.partiallySent()) { - response.abort(e); - return; - } - - var errorCode = "internal"; - - try { - if (!(e instanceof HttpError)) { - throw e; - } - - errorCode = e.code; - dumpn("*** errorCode == " + errorCode); - - response = new Response(connection); - if (e.customErrorHandling) { - e.customErrorHandling(response); - } - this._handleError(errorCode, request, response); - return; - } catch (e2) { - dumpn( - "*** error handling " + - errorCode + - " error: " + - "e2 == " + - e2 + - ", shutting down server" - ); - - connection.server._requestQuit(); - response.abort(e2); - return; - } - } - - response.complete(); - }, - - // - // see nsIHttpServer.registerFile - // - registerFile(path, file, handler) { - if (!file) { - dumpn("*** unregistering '" + path + "' mapping"); - delete this._overridePaths[path]; - return; - } - - dumpn("*** registering '" + path + "' as mapping to " + file.path); - file = file.clone(); - - var self = this; - this._overridePaths[path] = function(request, response) { - if (!file.exists()) { - throw HTTP_404; - } - - dumpn("*** responding '" + path + "' as mapping to " + file.path); - - response.setStatusLine(request.httpVersion, 200, "OK"); - if (typeof handler === "function") { - handler(request, response); - } - self._writeFileResponse(request, file, response, 0, file.fileSize); - }; - }, - - // - // see nsIHttpServer.registerPathHandler - // - registerPathHandler(path, handler) { - if (path.length == 0) { - throw Components.Exception( - "Handler path cannot be empty", - Cr.NS_ERROR_INVALID_ARG - ); - } - - // XXX true path validation! - if (path.charAt(0) != "/" && path != "CONNECT") { - throw Components.Exception("", Cr.NS_ERROR_INVALID_ARG); - } - - this._handlerToField(handler, this._overridePaths, path); - }, - - // - // see nsIHttpServer.registerPrefixHandler - // - registerPrefixHandler(path, handler) { - // XXX true path validation! - if (path.charAt(0) != "/" || path.charAt(path.length - 1) != "/") { - throw Components.Exception("", Cr.NS_ERROR_INVALID_ARG); - } - - this._handlerToField(handler, this._overridePrefixes, path); - }, - - // - // see nsIHttpServer.registerDirectory - // - registerDirectory(path, directory) { - // strip off leading and trailing '/' so that we can use lastIndexOf when - // determining exactly how a path maps onto a mapped directory -- - // conditional is required here to deal with "/".substring(1, 0) being - // converted to "/".substring(0, 1) per the JS specification - var key = path.length == 1 ? "" : path.substring(1, path.length - 1); - - // the path-to-directory mapping code requires that the first character not - // be "/", or it will go into an infinite loop - if (key.charAt(0) == "/") { - throw Components.Exception("", Cr.NS_ERROR_INVALID_ARG); - } - - key = toInternalPath(key, false); - - if (directory) { - dumpn("*** mapping '" + path + "' to the location " + directory.path); - this._pathDirectoryMap.put(key, directory); - } else { - dumpn("*** removing mapping for '" + path + "'"); - this._pathDirectoryMap.put(key, null); - } - }, - - // - // see nsIHttpServer.registerErrorHandler - // - registerErrorHandler(err, handler) { - if (!(err in HTTP_ERROR_CODES)) { - dumpn( - "*** WARNING: registering non-HTTP/1.1 error code " + - "(" + - err + - ") handler -- was this intentional?" - ); - } - - this._handlerToField(handler, this._overrideErrors, err); - }, - - // - // see nsIHttpServer.setIndexHandler - // - setIndexHandler(handler) { - if (!handler) { - handler = defaultIndexHandler; - } else if (typeof handler != "function") { - handler = createHandlerFunc(handler); - } - - this._indexHandler = handler; - }, - - // - // see nsIHttpServer.registerContentType - // - registerContentType(ext, type) { - if (!type) { - delete this._mimeMappings[ext]; - } else { - this._mimeMappings[ext] = headerUtils.normalizeFieldValue(type); - } - }, - - // PRIVATE API - - /** - * Sets or remove (if handler is null) a handler in an object with a key. - * - * @param handler - * a handler, either function or an nsIHttpRequestHandler - * @param dict - * The object to attach the handler to. - * @param key - * The field name of the handler. - */ - _handlerToField(handler, dict, key) { - // for convenience, handler can be a function if this is run from xpcshell - if (typeof handler == "function") { - dict[key] = handler; - } else if (handler) { - dict[key] = createHandlerFunc(handler); - } else { - delete dict[key]; - } - }, - - /** - * Handles a request which maps to a file in the local filesystem (if a base - * path has already been set; otherwise the 404 error is thrown). - * - * @param metadata : Request - * metadata for the incoming request - * @param response : Response - * an uninitialized Response to the given request, to be initialized by a - * request handler - * @throws HTTP_### - * if an HTTP error occurred (usually HTTP_404); note that in this case the - * calling code must handle post-processing of the response - */ - _handleDefault(metadata, response) { - dumpn("*** _handleDefault()"); - - response.setStatusLine(metadata.httpVersion, 200, "OK"); - - var path = metadata.path; - NS_ASSERT(path.charAt(0) == "/", "invalid path: <" + path + ">"); - - // determine the actual on-disk file; this requires finding the deepest - // path-to-directory mapping in the requested URL - var file = this._getFileForPath(path); - - // the "file" might be a directory, in which case we either serve the - // contained index.html or make the index handler write the response - if (file.exists() && file.isDirectory()) { - file.append("index.html"); // make configurable? - if (!file.exists() || file.isDirectory()) { - metadata._ensurePropertyBag(); - metadata._bag.setPropertyAsInterface("directory", file.parent); - this._indexHandler(metadata, response); - return; - } - } - - // alternately, the file might not exist - if (!file.exists()) { - throw HTTP_404; - } - - var start, end; - if ( - metadata._httpVersion.atLeast(nsHttpVersion.HTTP_1_1) && - metadata.hasHeader("Range") && - this._getTypeFromFile(file) !== SJS_TYPE - ) { - var rangeMatch = metadata - .getHeader("Range") - .match(/^bytes=(\d+)?-(\d+)?$/); - if (!rangeMatch) { - dumpn( - "*** Range header bogosity: '" + metadata.getHeader("Range") + "'" - ); - throw HTTP_400; - } - - if (rangeMatch[1] !== undefined) { - start = parseInt(rangeMatch[1], 10); - } - - if (rangeMatch[2] !== undefined) { - end = parseInt(rangeMatch[2], 10); - } - - if (start === undefined && end === undefined) { - dumpn( - "*** More Range header bogosity: '" + - metadata.getHeader("Range") + - "'" - ); - throw HTTP_400; - } - - // No start given, so the end is really the count of bytes from the - // end of the file. - if (start === undefined) { - start = Math.max(0, file.fileSize - end); - end = file.fileSize - 1; - } - - // start and end are inclusive - if (end === undefined || end >= file.fileSize) { - end = file.fileSize - 1; - } - - if (start !== undefined && start >= file.fileSize) { - var HTTP_416 = new HttpError(416, "Requested Range Not Satisfiable"); - HTTP_416.customErrorHandling = function(errorResponse) { - maybeAddHeaders(file, metadata, errorResponse); - }; - throw HTTP_416; - } - - if (end < start) { - response.setStatusLine(metadata.httpVersion, 200, "OK"); - start = 0; - end = file.fileSize - 1; - } else { - response.setStatusLine(metadata.httpVersion, 206, "Partial Content"); - var contentRange = "bytes " + start + "-" + end + "/" + file.fileSize; - response.setHeader("Content-Range", contentRange); - } - } else { - start = 0; - end = file.fileSize - 1; - } - - // finally... - dumpn( - "*** handling '" + - path + - "' as mapping to " + - file.path + - " from " + - start + - " to " + - end + - " inclusive" - ); - this._writeFileResponse(metadata, file, response, start, end - start + 1); - }, - - /** - * Writes an HTTP response for the given file, including setting headers for - * file metadata. - * - * @param metadata : Request - * the Request for which a response is being generated - * @param file : nsIFile - * the file which is to be sent in the response - * @param response : Response - * the response to which the file should be written - * @param offset: uint - * the byte offset to skip to when writing - * @param count: uint - * the number of bytes to write - */ - _writeFileResponse(metadata, file, response, offset, count) { - const PR_RDONLY = 0x01; - - var type = this._getTypeFromFile(file); - if (type === SJS_TYPE) { - let fis = new FileInputStream( - file, - PR_RDONLY, - PERMS_READONLY, - Ci.nsIFileInputStream.CLOSE_ON_EOF - ); - - try { - // If you update the list of imports, please update the list in - // tools/lint/eslint/eslint-plugin-mozilla/lib/environments/sjs.js - // as well. - var s = Cu.Sandbox(globalThis); - s.importFunction(dump, "dump"); - s.importFunction(atob, "atob"); - s.importFunction(btoa, "btoa"); - s.importFunction(ChromeUtils, "ChromeUtils"); - - // Define a basic key-value state-preservation API across requests, with - // keys initially corresponding to the empty string. - var self = this; - var path = metadata.path; - s.importFunction(function getState(k) { - return self._getState(path, k); - }); - s.importFunction(function setState(k, v) { - self._setState(path, k, v); - }); - s.importFunction(function getSharedState(k) { - return self._getSharedState(k); - }); - s.importFunction(function setSharedState(k, v) { - self._setSharedState(k, v); - }); - s.importFunction(function getObjectState(k, callback) { - callback(self._getObjectState(k)); - }); - s.importFunction(function setObjectState(k, v) { - self._setObjectState(k, v); - }); - s.importFunction(function registerPathHandler(p, h) { - self.registerPathHandler(p, h); - }); - - // Make it possible for sjs files to access their location - this._setState(path, "__LOCATION__", file.path); - - try { - // Alas, the line number in errors dumped to console when calling the - // request handler is simply an offset from where we load the SJS file. - // Work around this in a reasonably non-fragile way by dynamically - // getting the line number where we evaluate the SJS file. Don't - // separate these two lines! - var line = new Error().lineNumber; - let uri = Services.io.newFileURI(file); - Services.scriptloader.loadSubScript(uri.spec, s); - } catch (e) { - dumpn("*** syntax error in SJS at " + file.path + ": " + e); - throw HTTP_500; - } - - try { - s.handleRequest(metadata, response); - } catch (e) { - dump( - "*** error running SJS at " + - file.path + - ": " + - e + - " on line " + - (e instanceof Error - ? e.lineNumber + " in httpd.js" - : e.lineNumber - line) + - "\n" - ); - throw HTTP_500; - } - } finally { - fis.close(); - } - } else { - try { - response.setHeader( - "Last-Modified", - toDateString(file.lastModifiedTime), - false - ); - } catch (e) { - /* lastModifiedTime threw, ignore */ - } - - response.setHeader("Content-Type", type, false); - maybeAddInformationalResponse(file, metadata, response); - maybeAddHeaders(file, metadata, response); - response.setHeader("Content-Length", "" + count, false); - - let fis = new FileInputStream( - file, - PR_RDONLY, - PERMS_READONLY, - Ci.nsIFileInputStream.CLOSE_ON_EOF - ); - - offset = offset || 0; - count = count || file.fileSize; - NS_ASSERT(offset === 0 || offset < file.fileSize, "bad offset"); - NS_ASSERT(count >= 0, "bad count"); - NS_ASSERT(offset + count <= file.fileSize, "bad total data size"); - - try { - if (offset !== 0) { - // Seek (or read, if seeking isn't supported) to the correct offset so - // the data sent to the client matches the requested range. - if (fis instanceof Ci.nsISeekableStream) { - fis.seek(Ci.nsISeekableStream.NS_SEEK_SET, offset); - } else { - new ScriptableInputStream(fis).read(offset); - } - } - } catch (e) { - fis.close(); - throw e; - } - - let writeMore = function() { - gThreadManager.currentThread.dispatch( - writeData, - Ci.nsIThread.DISPATCH_NORMAL - ); - }; - - var input = new BinaryInputStream(fis); - var output = new BinaryOutputStream(response.bodyOutputStream); - var writeData = { - run() { - var chunkSize = Math.min(65536, count); - count -= chunkSize; - NS_ASSERT(count >= 0, "underflow"); - - try { - var data = input.readByteArray(chunkSize); - NS_ASSERT( - data.length === chunkSize, - "incorrect data returned? got " + - data.length + - ", expected " + - chunkSize - ); - output.writeByteArray(data); - if (count === 0) { - fis.close(); - response.finish(); - } else { - writeMore(); - } - } catch (e) { - try { - fis.close(); - } finally { - response.finish(); - } - throw e; - } - }, - }; - - writeMore(); - - // Now that we know copying will start, flag the response as async. - response.processAsync(); - } - }, - - /** - * Get the value corresponding to a given key for the given path for SJS state - * preservation across requests. - * - * @param path : string - * the path from which the given state is to be retrieved - * @param k : string - * the key whose corresponding value is to be returned - * @returns string - * the corresponding value, which is initially the empty string - */ - _getState(path, k) { - var state = this._state; - if (path in state && k in state[path]) { - return state[path][k]; - } - return ""; - }, - - /** - * Set the value corresponding to a given key for the given path for SJS state - * preservation across requests. - * - * @param path : string - * the path from which the given state is to be retrieved - * @param k : string - * the key whose corresponding value is to be set - * @param v : string - * the value to be set - */ - _setState(path, k, v) { - if (typeof v !== "string") { - throw new Error("non-string value passed"); - } - var state = this._state; - if (!(path in state)) { - state[path] = {}; - } - state[path][k] = v; - }, - - /** - * Get the value corresponding to a given key for SJS state preservation - * across requests. - * - * @param k : string - * the key whose corresponding value is to be returned - * @returns string - * the corresponding value, which is initially the empty string - */ - _getSharedState(k) { - var state = this._sharedState; - if (k in state) { - return state[k]; - } - return ""; - }, - - /** - * Set the value corresponding to a given key for SJS state preservation - * across requests. - * - * @param k : string - * the key whose corresponding value is to be set - * @param v : string - * the value to be set - */ - _setSharedState(k, v) { - if (typeof v !== "string") { - throw new Error("non-string value passed"); - } - this._sharedState[k] = v; - }, - - /** - * Returns the object associated with the given key in the server for SJS - * state preservation across requests. - * - * @param k : string - * the key whose corresponding object is to be returned - * @returns nsISupports - * the corresponding object, or null if none was present - */ - _getObjectState(k) { - if (typeof k !== "string") { - throw new Error("non-string key passed"); - } - return this._objectState[k] || null; - }, - - /** - * Sets the object associated with the given key in the server for SJS - * state preservation across requests. - * - * @param k : string - * the key whose corresponding object is to be set - * @param v : nsISupports - * the object to be associated with the given key; may be null - */ - _setObjectState(k, v) { - if (typeof k !== "string") { - throw new Error("non-string key passed"); - } - if (typeof v !== "object") { - throw new Error("non-object value passed"); - } - if (v && !("QueryInterface" in v)) { - throw new Error( - "must pass an nsISupports; use wrappedJSObject to ease " + - "pain when using the server from JS" - ); - } - - this._objectState[k] = v; - }, - - /** - * Gets a content-type for the given file, first by checking for any custom - * MIME-types registered with this handler for the file's extension, second by - * asking the global MIME service for a content-type, and finally by failing - * over to application/octet-stream. - * - * @param file : nsIFile - * the nsIFile for which to get a file type - * @returns string - * the best content-type which can be determined for the file - */ - _getTypeFromFile(file) { - try { - var name = file.leafName; - var dot = name.lastIndexOf("."); - if (dot > 0) { - var ext = name.slice(dot + 1); - if (ext in this._mimeMappings) { - return this._mimeMappings[ext]; - } - } - return Cc["@mozilla.org/mime;1"] - .getService(Ci.nsIMIMEService) - .getTypeFromFile(file); - } catch (e) { - return "application/octet-stream"; - } - }, - - /** - * Returns the nsIFile which corresponds to the path, as determined using - * all registered path->directory mappings and any paths which are explicitly - * overridden. - * - * @param path : string - * the server path for which a file should be retrieved, e.g. "/foo/bar" - * @throws HttpError - * when the correct action is the corresponding HTTP error (i.e., because no - * mapping was found for a directory in path, the referenced file doesn't - * exist, etc.) - * @returns nsIFile - * the file to be sent as the response to a request for the path - */ - _getFileForPath(path) { - // decode and add underscores as necessary - try { - path = toInternalPath(path, true); - } catch (e) { - dumpn("*** toInternalPath threw " + e); - throw HTTP_400; // malformed path - } - - // next, get the directory which contains this path - var pathMap = this._pathDirectoryMap; - - // An example progression of tmp for a path "/foo/bar/baz/" might be: - // "foo/bar/baz/", "foo/bar/baz", "foo/bar", "foo", "" - var tmp = path.substring(1); - while (true) { - // do we have a match for current head of the path? - var file = pathMap.get(tmp); - if (file) { - // XXX hack; basically disable showing mapping for /foo/bar/ when the - // requested path was /foo/bar, because relative links on the page - // will all be incorrect -- we really need the ability to easily - // redirect here instead - if ( - tmp == path.substring(1) && - tmp.length != 0 && - tmp.charAt(tmp.length - 1) != "/" - ) { - file = null; - } else { - break; - } - } - - // if we've finished trying all prefixes, exit - if (tmp == "") { - break; - } - - tmp = tmp.substring(0, tmp.lastIndexOf("/")); - } - - // no mapping applies, so 404 - if (!file) { - throw HTTP_404; - } - - // last, get the file for the path within the determined directory - var parentFolder = file.parent; - var dirIsRoot = parentFolder == null; - - // Strategy here is to append components individually, making sure we - // never move above the given directory; this allows paths such as - // "/foo/../bar" but prevents paths such as "/../base-sibling"; - // this component-wise approach also means the code works even on platforms - // which don't use "/" as the directory separator, such as Windows - var leafPath = path.substring(tmp.length + 1); - var comps = leafPath.split("/"); - for (var i = 0, sz = comps.length; i < sz; i++) { - var comp = comps[i]; - - if (comp == "..") { - file = file.parent; - } else if (comp == "." || comp == "") { - continue; - } else { - file.append(comp); - } - - if (!dirIsRoot && file.equals(parentFolder)) { - throw HTTP_403; - } - } - - return file; - }, - - /** - * Writes the error page for the given HTTP error code over the given - * connection. - * - * @param errorCode : uint - * the HTTP error code to be used - * @param connection : Connection - * the connection on which the error occurred - */ - handleError(errorCode, connection) { - var response = new Response(connection); - - dumpn("*** error in request: " + errorCode); - - this._handleError(errorCode, new Request(connection.port), response); - }, - - /** - * Handles a request which generates the given error code, using the - * user-defined error handler if one has been set, gracefully falling back to - * the x00 status code if the code has no handler, and failing to status code - * 500 if all else fails. - * - * @param errorCode : uint - * the HTTP error which is to be returned - * @param metadata : Request - * metadata for the request, which will often be incomplete since this is an - * error - * @param response : Response - * an uninitialized Response should be initialized when this method - * completes with information which represents the desired error code in the - * ideal case or a fallback code in abnormal circumstances (i.e., 500 is a - * fallback for 505, per HTTP specs) - */ - _handleError(errorCode, metadata, response) { - if (!metadata) { - throw Components.Exception("", Cr.NS_ERROR_NULL_POINTER); - } - - var errorX00 = errorCode - (errorCode % 100); - - try { - if (!(errorCode in HTTP_ERROR_CODES)) { - dumpn("*** WARNING: requested invalid error: " + errorCode); - } - - // RFC 2616 says that we should try to handle an error by its class if we - // can't otherwise handle it -- if that fails, we revert to handling it as - // a 500 internal server error, and if that fails we throw and shut down - // the server - - // actually handle the error - try { - if (errorCode in this._overrideErrors) { - this._overrideErrors[errorCode](metadata, response); - } else { - this._defaultErrors[errorCode](metadata, response); - } - } catch (e) { - if (response.partiallySent()) { - response.abort(e); - return; - } - - // don't retry the handler that threw - if (errorX00 == errorCode) { - throw HTTP_500; - } - - dumpn( - "*** error in handling for error code " + - errorCode + - ", " + - "falling back to " + - errorX00 + - "..." - ); - response = new Response(response._connection); - if (errorX00 in this._overrideErrors) { - this._overrideErrors[errorX00](metadata, response); - } else if (errorX00 in this._defaultErrors) { - this._defaultErrors[errorX00](metadata, response); - } else { - throw HTTP_500; - } - } - } catch (e) { - if (response.partiallySent()) { - response.abort(); - return; - } - - // we've tried everything possible for a meaningful error -- now try 500 - dumpn( - "*** error in handling for error code " + - errorX00 + - ", falling " + - "back to 500..." - ); - - try { - response = new Response(response._connection); - if (500 in this._overrideErrors) { - this._overrideErrors[500](metadata, response); - } else { - this._defaultErrors[500](metadata, response); - } - } catch (e2) { - dumpn("*** multiple errors in default error handlers!"); - dumpn("*** e == " + e + ", e2 == " + e2); - response.abort(e2); - return; - } - } - - response.complete(); - }, - - // FIELDS - - /** - * This object contains the default handlers for the various HTTP error codes. - */ - _defaultErrors: { - 400(metadata, response) { - // none of the data in metadata is reliable, so hard-code everything here - response.setStatusLine("1.1", 400, "Bad Request"); - response.setHeader("Content-Type", "text/plain;charset=utf-8", false); - - var body = "Bad request\n"; - response.bodyOutputStream.write(body, body.length); - }, - 403(metadata, response) { - response.setStatusLine(metadata.httpVersion, 403, "Forbidden"); - response.setHeader("Content-Type", "text/html;charset=utf-8", false); - - var body = - "\ - 403 Forbidden\ - \ -

403 Forbidden

\ - \ - "; - response.bodyOutputStream.write(body, body.length); - }, - 404(metadata, response) { - response.setStatusLine(metadata.httpVersion, 404, "Not Found"); - response.setHeader("Content-Type", "text/html;charset=utf-8", false); - - var body = - "\ - 404 Not Found\ - \ -

404 Not Found

\ -

\ - " + - htmlEscape(metadata.path) + - " was not found.\ -

\ - \ - "; - response.bodyOutputStream.write(body, body.length); - }, - 416(metadata, response) { - response.setStatusLine( - metadata.httpVersion, - 416, - "Requested Range Not Satisfiable" - ); - response.setHeader("Content-Type", "text/html;charset=utf-8", false); - - var body = - "\ - \ - 416 Requested Range Not Satisfiable\ - \ -

416 Requested Range Not Satisfiable

\ -

The byte range was not valid for the\ - requested resource.\ -

\ - \ - "; - response.bodyOutputStream.write(body, body.length); - }, - 500(metadata, response) { - response.setStatusLine( - metadata.httpVersion, - 500, - "Internal Server Error" - ); - response.setHeader("Content-Type", "text/html;charset=utf-8", false); - - var body = - "\ - 500 Internal Server Error\ - \ -

500 Internal Server Error

\ -

Something's broken in this server and\ - needs to be fixed.

\ - \ - "; - response.bodyOutputStream.write(body, body.length); - }, - 501(metadata, response) { - response.setStatusLine(metadata.httpVersion, 501, "Not Implemented"); - response.setHeader("Content-Type", "text/html;charset=utf-8", false); - - var body = - "\ - 501 Not Implemented\ - \ -

501 Not Implemented

\ -

This server is not (yet) Apache.

\ - \ - "; - response.bodyOutputStream.write(body, body.length); - }, - 505(metadata, response) { - response.setStatusLine("1.1", 505, "HTTP Version Not Supported"); - response.setHeader("Content-Type", "text/html;charset=utf-8", false); - - var body = - "\ - 505 HTTP Version Not Supported\ - \ -

505 HTTP Version Not Supported

\ -

This server only supports HTTP/1.0 and HTTP/1.1\ - connections.

\ - \ - "; - response.bodyOutputStream.write(body, body.length); - }, - }, - - /** - * Contains handlers for the default set of URIs contained in this server. - */ - _defaultPaths: { - "/": function(metadata, response) { - response.setStatusLine(metadata.httpVersion, 200, "OK"); - response.setHeader("Content-Type", "text/html;charset=utf-8", false); - - var body = - "\ - httpd.js\ - \ -

httpd.js

\ -

If you're seeing this page, httpd.js is up and\ - serving requests! Now set a base path and serve some\ - files!

\ - \ - "; - - response.bodyOutputStream.write(body, body.length); - }, - - "/trace": function(metadata, response) { - response.setStatusLine(metadata.httpVersion, 200, "OK"); - response.setHeader("Content-Type", "text/plain;charset=utf-8", false); - - var body = - "Request-URI: " + - metadata.scheme + - "://" + - metadata.host + - ":" + - metadata.port + - metadata.path + - "\n\n"; - body += "Request (semantically equivalent, slightly reformatted):\n\n"; - body += metadata.method + " " + metadata.path; - - if (metadata.queryString) { - body += "?" + metadata.queryString; - } - - body += " HTTP/" + metadata.httpVersion + "\r\n"; - - var headEnum = metadata.headers; - while (headEnum.hasMoreElements()) { - var fieldName = headEnum.getNext().QueryInterface(Ci.nsISupportsString) - .data; - body += fieldName + ": " + metadata.getHeader(fieldName) + "\r\n"; - } - - response.bodyOutputStream.write(body, body.length); - }, - }, -}; - -/** - * Maps absolute paths to files on the local file system (as nsILocalFiles). - */ -function FileMap() { - /** Hash which will map paths to nsILocalFiles. */ - this._map = {}; -} -FileMap.prototype = { - // PUBLIC API - - /** - * Maps key to a clone of the nsIFile value if value is non-null; - * otherwise, removes any extant mapping for key. - * - * @param key : string - * string to which a clone of value is mapped - * @param value : nsIFile - * the file to map to key, or null to remove a mapping - */ - put(key, value) { - if (value) { - this._map[key] = value.clone(); - } else { - delete this._map[key]; - } - }, - - /** - * Returns a clone of the nsIFile mapped to key, or null if no such - * mapping exists. - * - * @param key : string - * key to which the returned file maps - * @returns nsIFile - * a clone of the mapped file, or null if no mapping exists - */ - get(key) { - var val = this._map[key]; - return val ? val.clone() : null; - }, -}; - -// Response CONSTANTS - -// token = * -// CHAR = -// CTL = -// separators = "(" | ")" | "<" | ">" | "@" -// | "," | ";" | ":" | "\" | <"> -// | "/" | "[" | "]" | "?" | "=" -// | "{" | "}" | SP | HT -const IS_TOKEN_ARRAY = [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, // 0 - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, // 8 - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, // 16 - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, // 24 - - 0, - 1, - 0, - 1, - 1, - 1, - 1, - 1, // 32 - 0, - 0, - 1, - 1, - 0, - 1, - 1, - 0, // 40 - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, // 48 - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, // 56 - - 0, - 1, - 1, - 1, - 1, - 1, - 1, - 1, // 64 - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, // 72 - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, // 80 - 1, - 1, - 1, - 0, - 0, - 0, - 1, - 1, // 88 - - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, // 96 - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, // 104 - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, // 112 - 1, - 1, - 1, - 0, - 1, - 0, - 1, -]; // 120 - -/** - * Determines whether the given character code is a CTL. - * - * @param code : uint - * the character code - * @returns boolean - * true if code is a CTL, false otherwise - */ -function isCTL(code) { - return (code >= 0 && code <= 31) || code == 127; -} - -/** - * Represents a response to an HTTP request, encapsulating all details of that - * response. This includes all headers, the HTTP version, status code and - * explanation, and the entity itself. - * - * @param connection : Connection - * the connection over which this response is to be written - */ -function Response(connection) { - /** The connection over which this response will be written. */ - this._connection = connection; - - /** - * The HTTP version of this response; defaults to 1.1 if not set by the - * handler. - */ - this._httpVersion = nsHttpVersion.HTTP_1_1; - - /** - * The HTTP code of this response; defaults to 200. - */ - this._httpCode = 200; - - /** - * The description of the HTTP code in this response; defaults to "OK". - */ - this._httpDescription = "OK"; - - /** - * An nsIHttpHeaders object in which the headers in this response should be - * stored. This property is null after the status line and headers have been - * written to the network, and it may be modified up until it is cleared, - * except if this._finished is set first (in which case headers are written - * asynchronously in response to a finish() call not preceded by - * flushHeaders()). - */ - this._headers = new nsHttpHeaders(); - - /** - * Informational response: - * For example 103 Early Hint - **/ - this._informationalResponseHttpVersion = nsHttpVersion.HTTP_1_1; - this._informationalResponseHttpCode = 0; - this._informationalResponseHttpDescription = ""; - this._informationalResponseHeaders = new nsHttpHeaders(); - this._informationalResponseSet = false; - - /** - * Set to true when this response is ended (completely constructed if possible - * and the connection closed); further actions on this will then fail. - */ - this._ended = false; - - /** - * A stream used to hold data written to the body of this response. - */ - this._bodyOutputStream = null; - - /** - * A stream containing all data that has been written to the body of this - * response so far. (Async handlers make the data contained in this - * unreliable as a way of determining content length in general, but auxiliary - * saved information can sometimes be used to guarantee reliability.) - */ - this._bodyInputStream = null; - - /** - * A stream copier which copies data to the network. It is initially null - * until replaced with a copier for response headers; when headers have been - * fully sent it is replaced with a copier for the response body, remaining - * so for the duration of response processing. - */ - this._asyncCopier = null; - - /** - * True if this response has been designated as being processed - * asynchronously rather than for the duration of a single call to - * nsIHttpRequestHandler.handle. - */ - this._processAsync = false; - - /** - * True iff finish() has been called on this, signaling that no more changes - * to this may be made. - */ - this._finished = false; - - /** - * True iff powerSeized() has been called on this, signaling that this - * response is to be handled manually by the response handler (which may then - * send arbitrary data in response, even non-HTTP responses). - */ - this._powerSeized = false; -} -Response.prototype = { - // PUBLIC CONSTRUCTION API - - // - // see nsIHttpResponse.bodyOutputStream - // - get bodyOutputStream() { - if (this._finished) { - throw Components.Exception("", Cr.NS_ERROR_NOT_AVAILABLE); - } - - if (!this._bodyOutputStream) { - var pipe = new Pipe( - true, - false, - Response.SEGMENT_SIZE, - PR_UINT32_MAX, - null - ); - this._bodyOutputStream = pipe.outputStream; - this._bodyInputStream = pipe.inputStream; - if (this._processAsync || this._powerSeized) { - this._startAsyncProcessor(); - } - } - - return this._bodyOutputStream; - }, - - // - // see nsIHttpResponse.write - // - write(data) { - if (this._finished) { - throw Components.Exception("", Cr.NS_ERROR_NOT_AVAILABLE); - } - - var dataAsString = String(data); - this.bodyOutputStream.write(dataAsString, dataAsString.length); - }, - - // - // see nsIHttpResponse.setStatusLine - // - setStatusLineInternal(httpVersion, code, description, informationalResponse) { - if (this._finished || this._powerSeized) { - throw Components.Exception("", Cr.NS_ERROR_NOT_AVAILABLE); - } - - if (!informationalResponse) { - if (!this._headers) { - throw Components.Exception("", Cr.NS_ERROR_NOT_AVAILABLE); - } - } else if (!this._informationalResponseHeaders) { - throw Components.Exception("", Cr.NS_ERROR_NOT_AVAILABLE); - } - this._ensureAlive(); - - if (!(code >= 0 && code < 1000)) { - throw Components.Exception("", Cr.NS_ERROR_INVALID_ARG); - } - - try { - var httpVer; - // avoid version construction for the most common cases - if (!httpVersion || httpVersion == "1.1") { - httpVer = nsHttpVersion.HTTP_1_1; - } else if (httpVersion == "1.0") { - httpVer = nsHttpVersion.HTTP_1_0; - } else { - httpVer = new nsHttpVersion(httpVersion); - } - } catch (e) { - throw Components.Exception("", Cr.NS_ERROR_INVALID_ARG); - } - - // Reason-Phrase = * - // TEXT = - // - // XXX this ends up disallowing octets which aren't Unicode, I think -- not - // much to do if description is IDL'd as string - if (!description) { - description = ""; - } - for (var i = 0; i < description.length; i++) { - if (isCTL(description.charCodeAt(i)) && description.charAt(i) != "\t") { - throw Components.Exception("", Cr.NS_ERROR_INVALID_ARG); - } - } - - // set the values only after validation to preserve atomicity - if (!informationalResponse) { - this._httpDescription = description; - this._httpCode = code; - this._httpVersion = httpVer; - } else { - this._informationalResponseSet = true; - this._informationalResponseHttpDescription = description; - this._informationalResponseHttpCode = code; - this._informationalResponseHttpVersion = httpVer; - } - }, - - // - // see nsIHttpResponse.setStatusLine - // - setStatusLine(httpVersion, code, description) { - this.setStatusLineInternal(httpVersion, code, description, false); - }, - - setInformationalResponseStatusLine(httpVersion, code, description) { - this.setStatusLineInternal(httpVersion, code, description, true); - }, - - // - // see nsIHttpResponse.setHeader - // - setHeader(name, value, merge) { - if (!this._headers || this._finished || this._powerSeized) { - throw Components.Exception("", Cr.NS_ERROR_NOT_AVAILABLE); - } - this._ensureAlive(); - - this._headers.setHeader(name, value, merge); - }, - - setInformationalResponseHeader(name, value, merge) { - if ( - !this._informationalResponseHeaders || - this._finished || - this._powerSeized - ) { - throw Components.Exception("", Cr.NS_ERROR_NOT_AVAILABLE); - } - this._ensureAlive(); - - this._informationalResponseHeaders.setHeader(name, value, merge); - }, - - setHeaderNoCheck(name, value) { - if (!this._headers || this._finished || this._powerSeized) { - throw Components.Exception("", Cr.NS_ERROR_NOT_AVAILABLE); - } - this._ensureAlive(); - - this._headers.setHeaderNoCheck(name, value); - }, - - setInformationalHeaderNoCheck(name, value) { - if (!this._headers || this._finished || this._powerSeized) { - throw Components.Exception("", Cr.NS_ERROR_NOT_AVAILABLE); - } - this._ensureAlive(); - - this._informationalResponseHeaders.setHeaderNoCheck(name, value); - }, - - // - // see nsIHttpResponse.processAsync - // - processAsync() { - if (this._finished) { - throw Components.Exception("", Cr.NS_ERROR_UNEXPECTED); - } - if (this._powerSeized) { - throw Components.Exception("", Cr.NS_ERROR_NOT_AVAILABLE); - } - if (this._processAsync) { - return; - } - this._ensureAlive(); - - dumpn("*** processing connection " + this._connection.number + " async"); - this._processAsync = true; - - /* - * Either the bodyOutputStream getter or this method is responsible for - * starting the asynchronous processor and catching writes of data to the - * response body of async responses as they happen, for the purpose of - * forwarding those writes to the actual connection's output stream. - * If bodyOutputStream is accessed first, calling this method will create - * the processor (when it first is clear that body data is to be written - * immediately, not buffered). If this method is called first, accessing - * bodyOutputStream will create the processor. If only this method is - * called, we'll write nothing, neither headers nor the nonexistent body, - * until finish() is called. Since that delay is easily avoided by simply - * getting bodyOutputStream or calling write(""), we don't worry about it. - */ - if (this._bodyOutputStream && !this._asyncCopier) { - this._startAsyncProcessor(); - } - }, - - // - // see nsIHttpResponse.seizePower - // - seizePower() { - if (this._processAsync) { - throw Components.Exception("", Cr.NS_ERROR_NOT_AVAILABLE); - } - if (this._finished) { - throw Components.Exception("", Cr.NS_ERROR_UNEXPECTED); - } - if (this._powerSeized) { - return; - } - this._ensureAlive(); - - dumpn( - "*** forcefully seizing power over connection " + - this._connection.number + - "..." - ); - - // Purge any already-written data without sending it. We could as easily - // swap out the streams entirely, but that makes it possible to acquire and - // unknowingly use a stale reference, so we require there only be one of - // each stream ever for any response to avoid this complication. - if (this._asyncCopier) { - this._asyncCopier.cancel(Cr.NS_BINDING_ABORTED); - } - this._asyncCopier = null; - if (this._bodyOutputStream) { - var input = new BinaryInputStream(this._bodyInputStream); - var avail; - while ((avail = input.available()) > 0) { - input.readByteArray(avail); - } - } - - this._powerSeized = true; - if (this._bodyOutputStream) { - this._startAsyncProcessor(); - } - }, - - // - // see nsIHttpResponse.finish - // - finish() { - if (!this._processAsync && !this._powerSeized) { - throw Components.Exception("", Cr.NS_ERROR_UNEXPECTED); - } - if (this._finished) { - return; - } - - dumpn("*** finishing connection " + this._connection.number); - this._startAsyncProcessor(); // in case bodyOutputStream was never accessed - if (this._bodyOutputStream) { - this._bodyOutputStream.close(); - } - this._finished = true; - }, - - // NSISUPPORTS - - // - // see nsISupports.QueryInterface - // - QueryInterface: ChromeUtils.generateQI(["nsIHttpResponse"]), - - // POST-CONSTRUCTION API (not exposed externally) - - /** - * The HTTP version number of this, as a string (e.g. "1.1"). - */ - get httpVersion() { - this._ensureAlive(); - return this._httpVersion.toString(); - }, - - /** - * The HTTP status code of this response, as a string of three characters per - * RFC 2616. - */ - get httpCode() { - this._ensureAlive(); - - var codeString = - (this._httpCode < 10 ? "0" : "") + - (this._httpCode < 100 ? "0" : "") + - this._httpCode; - return codeString; - }, - - /** - * The description of the HTTP status code of this response, or "" if none is - * set. - */ - get httpDescription() { - this._ensureAlive(); - - return this._httpDescription; - }, - - /** - * The headers in this response, as an nsHttpHeaders object. - */ - get headers() { - this._ensureAlive(); - - return this._headers; - }, - - // - // see nsHttpHeaders.getHeader - // - getHeader(name) { - this._ensureAlive(); - - return this._headers.getHeader(name); - }, - - /** - * Determines whether this response may be abandoned in favor of a newly - * constructed response. A response may be abandoned only if it is not being - * sent asynchronously and if raw control over it has not been taken from the - * server. - * - * @returns boolean - * true iff no data has been written to the network - */ - partiallySent() { - dumpn("*** partiallySent()"); - return this._processAsync || this._powerSeized; - }, - - /** - * If necessary, kicks off the remaining request processing needed to be done - * after a request handler performs its initial work upon this response. - */ - complete() { - dumpn("*** complete()"); - if (this._processAsync || this._powerSeized) { - NS_ASSERT( - this._processAsync ^ this._powerSeized, - "can't both send async and relinquish power" - ); - return; - } - - NS_ASSERT(!this.partiallySent(), "completing a partially-sent response?"); - - this._startAsyncProcessor(); - - // Now make sure we finish processing this request! - if (this._bodyOutputStream) { - this._bodyOutputStream.close(); - } - }, - - /** - * Abruptly ends processing of this response, usually due to an error in an - * incoming request but potentially due to a bad error handler. Since we - * cannot handle the error in the usual way (giving an HTTP error page in - * response) because data may already have been sent (or because the response - * might be expected to have been generated asynchronously or completely from - * scratch by the handler), we stop processing this response and abruptly - * close the connection. - * - * @param e : Error - * the exception which precipitated this abort, or null if no such exception - * was generated - * @param truncateConnection : Boolean - * ensures that we truncate the connection using an RST packet, so the - * client testing code is aware that an error occurred, otherwise it may - * consider the response as valid. - */ - abort(e, truncateConnection = false) { - dumpn("*** abort(<" + e + ">)"); - - if (truncateConnection) { - dumpn("*** truncate connection"); - this._connection.transport.setLinger(true, 0); - } - - // This response will be ended by the processor if one was created. - var copier = this._asyncCopier; - if (copier) { - // We dispatch asynchronously here so that any pending writes of data to - // the connection will be deterministically written. This makes it easier - // to specify exact behavior, and it makes observable behavior more - // predictable for clients. Note that the correctness of this depends on - // callbacks in response to _waitToReadData in WriteThroughCopier - // happening asynchronously with respect to the actual writing of data to - // bodyOutputStream, as they currently do; if they happened synchronously, - // an event which ran before this one could write more data to the - // response body before we get around to canceling the copier. We have - // tests for this in test_seizepower.js, however, and I can't think of a - // way to handle both cases without removing bodyOutputStream access and - // moving its effective write(data, length) method onto Response, which - // would be slower and require more code than this anyway. - gThreadManager.currentThread.dispatch( - { - run() { - dumpn("*** canceling copy asynchronously..."); - copier.cancel(Cr.NS_ERROR_UNEXPECTED); - }, - }, - Ci.nsIThread.DISPATCH_NORMAL - ); - } else { - this.end(); - } - }, - - /** - * Closes this response's network connection, marks the response as finished, - * and notifies the server handler that the request is done being processed. - */ - end() { - NS_ASSERT(!this._ended, "ending this response twice?!?!"); - - this._connection.close(); - if (this._bodyOutputStream) { - this._bodyOutputStream.close(); - } - - this._finished = true; - this._ended = true; - }, - - // PRIVATE IMPLEMENTATION - - /** - * Sends the status line and headers of this response if they haven't been - * sent and initiates the process of copying data written to this response's - * body to the network. - */ - _startAsyncProcessor() { - dumpn("*** _startAsyncProcessor()"); - - // Handle cases where we're being called a second time. The former case - // happens when this is triggered both by complete() and by processAsync(), - // while the latter happens when processAsync() in conjunction with sent - // data causes abort() to be called. - if (this._asyncCopier || this._ended) { - dumpn("*** ignoring second call to _startAsyncProcessor"); - return; - } - - // Send headers if they haven't been sent already and should be sent, then - // asynchronously continue to send the body. - if (this._headers && !this._powerSeized) { - this._sendHeaders(); - return; - } - - this._headers = null; - this._sendBody(); - }, - - /** - * Signals that all modifications to the response status line and headers are - * complete and then sends that data over the network to the client. Once - * this method completes, a different response to the request that resulted - * in this response cannot be sent -- the only possible action in case of - * error is to abort the response and close the connection. - */ - _sendHeaders() { - dumpn("*** _sendHeaders()"); - - NS_ASSERT(this._headers); - NS_ASSERT(this._informationalResponseHeaders); - NS_ASSERT(!this._powerSeized); - - var preambleData = []; - - // Informational response, e.g. 103 - if (this._informationalResponseSet) { - // request-line - let statusLine = - "HTTP/" + - this._informationalResponseHttpVersion + - " " + - this._informationalResponseHttpCode + - " " + - this._informationalResponseHttpDescription + - "\r\n"; - preambleData.push(statusLine); - - // headers - let headEnum = this._informationalResponseHeaders.enumerator; - while (headEnum.hasMoreElements()) { - let fieldName = headEnum.getNext().QueryInterface(Ci.nsISupportsString) - .data; - let values = this._informationalResponseHeaders.getHeaderValues( - fieldName - ); - for (let i = 0, sz = values.length; i < sz; i++) { - preambleData.push(fieldName + ": " + values[i] + "\r\n"); - } - } - // end request-line/headers - preambleData.push("\r\n"); - } - - // request-line - var statusLine = - "HTTP/" + - this.httpVersion + - " " + - this.httpCode + - " " + - this.httpDescription + - "\r\n"; - - // header post-processing - - var headers = this._headers; - headers.setHeader("Connection", "close", false); - headers.setHeader("Server", "httpd.js", false); - if (!headers.hasHeader("Date")) { - headers.setHeader("Date", toDateString(Date.now()), false); - } - - // Any response not being processed asynchronously must have an associated - // Content-Length header for reasons of backwards compatibility with the - // initial server, which fully buffered every response before sending it. - // Beyond that, however, it's good to do this anyway because otherwise it's - // impossible to test behaviors that depend on the presence or absence of a - // Content-Length header. - if (!this._processAsync) { - dumpn("*** non-async response, set Content-Length"); - - var bodyStream = this._bodyInputStream; - var avail = bodyStream ? bodyStream.available() : 0; - - // XXX assumes stream will always report the full amount of data available - headers.setHeader("Content-Length", "" + avail, false); - } - - // construct and send response - dumpn("*** header post-processing completed, sending response head..."); - - // request-line - preambleData.push(statusLine); - - // headers - var headEnum = headers.enumerator; - while (headEnum.hasMoreElements()) { - var fieldName = headEnum.getNext().QueryInterface(Ci.nsISupportsString) - .data; - var values = headers.getHeaderValues(fieldName); - for (var i = 0, sz = values.length; i < sz; i++) { - preambleData.push(fieldName + ": " + values[i] + "\r\n"); - } - } - - // end request-line/headers - preambleData.push("\r\n"); - - var preamble = preambleData.join(""); - - var responseHeadPipe = new Pipe(true, false, 0, PR_UINT32_MAX, null); - responseHeadPipe.outputStream.write(preamble, preamble.length); - - var response = this; - var copyObserver = { - onStartRequest(request) { - dumpn("*** preamble copying started"); - }, - - onStopRequest(request, statusCode) { - dumpn( - "*** preamble copying complete " + - "[status=0x" + - statusCode.toString(16) + - "]" - ); - - if (!Components.isSuccessCode(statusCode)) { - dumpn( - "!!! header copying problems: non-success statusCode, " + - "ending response" - ); - - response.end(); - } else { - response._sendBody(); - } - }, - - QueryInterface: ChromeUtils.generateQI(["nsIRequestObserver"]), - }; - - this._asyncCopier = new WriteThroughCopier( - responseHeadPipe.inputStream, - this._connection.output, - copyObserver, - null - ); - - responseHeadPipe.outputStream.close(); - - // Forbid setting any more headers or modifying the request line. - this._headers = null; - }, - - /** - * Asynchronously writes the body of the response (or the entire response, if - * seizePower() has been called) to the network. - */ - _sendBody() { - dumpn("*** _sendBody"); - - NS_ASSERT(!this._headers, "still have headers around but sending body?"); - - // If no body data was written, we're done - if (!this._bodyInputStream) { - dumpn("*** empty body, response finished"); - this.end(); - return; - } - - var response = this; - var copyObserver = { - onStartRequest(request) { - dumpn("*** onStartRequest"); - }, - - onStopRequest(request, statusCode) { - dumpn("*** onStopRequest [status=0x" + statusCode.toString(16) + "]"); - - if (statusCode === Cr.NS_BINDING_ABORTED) { - dumpn("*** terminating copy observer without ending the response"); - } else { - if (!Components.isSuccessCode(statusCode)) { - dumpn("*** WARNING: non-success statusCode in onStopRequest"); - } - - response.end(); - } - }, - - QueryInterface: ChromeUtils.generateQI(["nsIRequestObserver"]), - }; - - dumpn("*** starting async copier of body data..."); - this._asyncCopier = new WriteThroughCopier( - this._bodyInputStream, - this._connection.output, - copyObserver, - null - ); - }, - - /** Ensures that this hasn't been ended. */ - _ensureAlive() { - NS_ASSERT(!this._ended, "not handling response lifetime correctly"); - }, -}; - -/** - * Size of the segments in the buffer used in storing response data and writing - * it to the socket. - */ -Response.SEGMENT_SIZE = 8192; - -/** Serves double duty in WriteThroughCopier implementation. */ -function notImplemented() { - throw Components.Exception("", Cr.NS_ERROR_NOT_IMPLEMENTED); -} - -/** Returns true iff the given exception represents stream closure. */ -function streamClosed(e) { - return ( - e === Cr.NS_BASE_STREAM_CLOSED || - (typeof e === "object" && e.result === Cr.NS_BASE_STREAM_CLOSED) - ); -} - -/** Returns true iff the given exception represents a blocked stream. */ -function wouldBlock(e) { - return ( - e === Cr.NS_BASE_STREAM_WOULD_BLOCK || - (typeof e === "object" && e.result === Cr.NS_BASE_STREAM_WOULD_BLOCK) - ); -} - -/** - * Copies data from source to sink as it becomes available, when that data can - * be written to sink without blocking. - * - * @param source : nsIAsyncInputStream - * the stream from which data is to be read - * @param sink : nsIAsyncOutputStream - * the stream to which data is to be copied - * @param observer : nsIRequestObserver - * an observer which will be notified when the copy starts and finishes - * @param context : nsISupports - * context passed to observer when notified of start/stop - * @throws NS_ERROR_NULL_POINTER - * if source, sink, or observer are null - */ -function WriteThroughCopier(source, sink, observer, context) { - if (!source || !sink || !observer) { - throw Components.Exception("", Cr.NS_ERROR_NULL_POINTER); - } - - /** Stream from which data is being read. */ - this._source = source; - - /** Stream to which data is being written. */ - this._sink = sink; - - /** Observer watching this copy. */ - this._observer = observer; - - /** Context for the observer watching this. */ - this._context = context; - - /** - * True iff this is currently being canceled (cancel has been called, the - * callback may not yet have been made). - */ - this._canceled = false; - - /** - * False until all data has been read from input and written to output, at - * which point this copy is completed and cancel() is asynchronously called. - */ - this._completed = false; - - /** Required by nsIRequest, meaningless. */ - this.loadFlags = 0; - /** Required by nsIRequest, meaningless. */ - this.loadGroup = null; - /** Required by nsIRequest, meaningless. */ - this.name = "response-body-copy"; - - /** Status of this request. */ - this.status = Cr.NS_OK; - - /** Arrays of byte strings waiting to be written to output. */ - this._pendingData = []; - - // start copying - try { - observer.onStartRequest(this); - this._waitToReadData(); - this._waitForSinkClosure(); - } catch (e) { - dumpn( - "!!! error starting copy: " + - e + - ("lineNumber" in e ? ", line " + e.lineNumber : "") - ); - dumpn(e.stack); - this.cancel(Cr.NS_ERROR_UNEXPECTED); - } -} -WriteThroughCopier.prototype = { - /* nsISupports implementation */ - - QueryInterface: ChromeUtils.generateQI([ - "nsIInputStreamCallback", - "nsIOutputStreamCallback", - "nsIRequest", - ]), - - // NSIINPUTSTREAMCALLBACK - - /** - * Receives a more-data-in-input notification and writes the corresponding - * data to the output. - * - * @param input : nsIAsyncInputStream - * the input stream on whose data we have been waiting - */ - onInputStreamReady(input) { - if (this._source === null) { - return; - } - - dumpn("*** onInputStreamReady"); - - // - // Ordinarily we'll read a non-zero amount of data from input, queue it up - // to be written and then wait for further callbacks. The complications in - // this method are the cases where we deviate from that behavior when errors - // occur or when copying is drawing to a finish. - // - // The edge cases when reading data are: - // - // Zero data is read - // If zero data was read, we're at the end of available data, so we can - // should stop reading and move on to writing out what we have (or, if - // we've already done that, onto notifying of completion). - // A stream-closed exception is thrown - // This is effectively a less kind version of zero data being read; the - // only difference is that we notify of completion with that result - // rather than with NS_OK. - // Some other exception is thrown - // This is the least kind result. We don't know what happened, so we - // act as though the stream closed except that we notify of completion - // with the result NS_ERROR_UNEXPECTED. - // - - var bytesWanted = 0, - bytesConsumed = -1; - try { - input = new BinaryInputStream(input); - - bytesWanted = Math.min(input.available(), Response.SEGMENT_SIZE); - dumpn("*** input wanted: " + bytesWanted); - - if (bytesWanted > 0) { - var data = input.readByteArray(bytesWanted); - bytesConsumed = data.length; - this._pendingData.push(String.fromCharCode.apply(String, data)); - } - - dumpn("*** " + bytesConsumed + " bytes read"); - - // Handle the zero-data edge case in the same place as all other edge - // cases are handled. - if (bytesWanted === 0) { - throw Components.Exception("", Cr.NS_BASE_STREAM_CLOSED); - } - } catch (e) { - let rv; - if (streamClosed(e)) { - dumpn("*** input stream closed"); - rv = bytesWanted === 0 ? Cr.NS_OK : Cr.NS_ERROR_UNEXPECTED; - } else { - dumpn("!!! unexpected error reading from input, canceling: " + e); - rv = Cr.NS_ERROR_UNEXPECTED; - } - - this._doneReadingSource(rv); - return; - } - - var pendingData = this._pendingData; - - NS_ASSERT(bytesConsumed > 0); - NS_ASSERT(pendingData.length > 0, "no pending data somehow?"); - NS_ASSERT( - pendingData[pendingData.length - 1].length > 0, - "buffered zero bytes of data?" - ); - - NS_ASSERT(this._source !== null); - - // Reading has gone great, and we've gotten data to write now. What if we - // don't have a place to write that data, because output went away just - // before this read? Drop everything on the floor, including new data, and - // cancel at this point. - if (this._sink === null) { - pendingData.length = 0; - this._doneReadingSource(Cr.NS_ERROR_UNEXPECTED); - return; - } - - // Okay, we've read the data, and we know we have a place to write it. We - // need to queue up the data to be written, but *only* if none is queued - // already -- if data's already queued, the code that actually writes the - // data will make sure to wait on unconsumed pending data. - try { - if (pendingData.length === 1) { - this._waitToWriteData(); - } - } catch (e) { - dumpn( - "!!! error waiting to write data just read, swallowing and " + - "writing only what we already have: " + - e - ); - this._doneWritingToSink(Cr.NS_ERROR_UNEXPECTED); - return; - } - - // Whee! We successfully read some data, and it's successfully queued up to - // be written. All that remains now is to wait for more data to read. - try { - this._waitToReadData(); - } catch (e) { - dumpn("!!! error waiting to read more data: " + e); - this._doneReadingSource(Cr.NS_ERROR_UNEXPECTED); - } - }, - - // NSIOUTPUTSTREAMCALLBACK - - /** - * Callback when data may be written to the output stream without blocking, or - * when the output stream has been closed. - * - * @param output : nsIAsyncOutputStream - * the output stream on whose writability we've been waiting, also known as - * this._sink - */ - onOutputStreamReady(output) { - if (this._sink === null) { - return; - } - - dumpn("*** onOutputStreamReady"); - - var pendingData = this._pendingData; - if (pendingData.length === 0) { - // There's no pending data to write. The only way this can happen is if - // we're waiting on the output stream's closure, so we can respond to a - // copying failure as quickly as possible (rather than waiting for data to - // be available to read and then fail to be copied). Therefore, we must - // be done now -- don't bother to attempt to write anything and wrap - // things up. - dumpn("!!! output stream closed prematurely, ending copy"); - - this._doneWritingToSink(Cr.NS_ERROR_UNEXPECTED); - return; - } - - NS_ASSERT(pendingData[0].length > 0, "queued up an empty quantum?"); - - // - // Write out the first pending quantum of data. The possible errors here - // are: - // - // The write might fail because we can't write that much data - // Okay, we've written what we can now, so re-queue what's left and - // finish writing it out later. - // The write failed because the stream was closed - // Discard pending data that we can no longer write, stop reading, and - // signal that copying finished. - // Some other error occurred. - // Same as if the stream were closed, but notify with the status - // NS_ERROR_UNEXPECTED so the observer knows something was wonky. - // - - try { - var quantum = pendingData[0]; - - // XXX |quantum| isn't guaranteed to be ASCII, so we're relying on - // undefined behavior! We're only using this because writeByteArray - // is unusably broken for asynchronous output streams; see bug 532834 - // for details. - var bytesWritten = output.write(quantum, quantum.length); - if (bytesWritten === quantum.length) { - pendingData.shift(); - } else { - pendingData[0] = quantum.substring(bytesWritten); - } - - dumpn("*** wrote " + bytesWritten + " bytes of data"); - } catch (e) { - if (wouldBlock(e)) { - NS_ASSERT( - pendingData.length > 0, - "stream-blocking exception with no data to write?" - ); - NS_ASSERT( - pendingData[0].length > 0, - "stream-blocking exception with empty quantum?" - ); - this._waitToWriteData(); - return; - } - - if (streamClosed(e)) { - dumpn("!!! output stream prematurely closed, signaling error..."); - } else { - dumpn("!!! unknown error: " + e + ", quantum=" + quantum); - } - - this._doneWritingToSink(Cr.NS_ERROR_UNEXPECTED); - return; - } - - // The day is ours! Quantum written, now let's see if we have more data - // still to write. - try { - if (pendingData.length > 0) { - this._waitToWriteData(); - return; - } - } catch (e) { - dumpn("!!! unexpected error waiting to write pending data: " + e); - this._doneWritingToSink(Cr.NS_ERROR_UNEXPECTED); - return; - } - - // Okay, we have no more pending data to write -- but might we get more in - // the future? - if (this._source !== null) { - /* - * If we might, then wait for the output stream to be closed. (We wait - * only for closure because we have no data to write -- and if we waited - * for a specific amount of data, we would get repeatedly notified for no - * reason if over time the output stream permitted more and more data to - * be written to it without blocking.) - */ - this._waitForSinkClosure(); - } else { - /* - * On the other hand, if we can't have more data because the input - * stream's gone away, then it's time to notify of copy completion. - * Victory! - */ - this._sink = null; - this._cancelOrDispatchCancelCallback(Cr.NS_OK); - } - }, - - // NSIREQUEST - - /** Returns true if the cancel observer hasn't been notified yet. */ - isPending() { - return !this._completed; - }, - - /** Not implemented, don't use! */ - suspend: notImplemented, - /** Not implemented, don't use! */ - resume: notImplemented, - - /** - * Cancels data reading from input, asynchronously writes out any pending - * data, and causes the observer to be notified with the given error code when - * all writing has finished. - * - * @param status : nsresult - * the status to pass to the observer when data copying has been canceled - */ - cancel(status) { - dumpn("*** cancel(" + status.toString(16) + ")"); - - if (this._canceled) { - dumpn("*** suppressing a late cancel"); - return; - } - - this._canceled = true; - this.status = status; - - // We could be in the middle of absolutely anything at this point. Both - // input and output might still be around, we might have pending data to - // write, and in general we know nothing about the state of the world. We - // therefore must assume everything's in progress and take everything to its - // final steady state (or so far as it can go before we need to finish - // writing out remaining data). - - this._doneReadingSource(status); - }, - - // PRIVATE IMPLEMENTATION - - /** - * Stop reading input if we haven't already done so, passing e as the status - * when closing the stream, and kick off a copy-completion notice if no more - * data remains to be written. - * - * @param e : nsresult - * the status to be used when closing the input stream - */ - _doneReadingSource(e) { - dumpn("*** _doneReadingSource(0x" + e.toString(16) + ")"); - - this._finishSource(e); - if (this._pendingData.length === 0) { - this._sink = null; - } else { - NS_ASSERT(this._sink !== null, "null output?"); - } - - // If we've written out all data read up to this point, then it's time to - // signal completion. - if (this._sink === null) { - NS_ASSERT(this._pendingData.length === 0, "pending data still?"); - this._cancelOrDispatchCancelCallback(e); - } - }, - - /** - * Stop writing output if we haven't already done so, discard any data that - * remained to be sent, close off input if it wasn't already closed, and kick - * off a copy-completion notice. - * - * @param e : nsresult - * the status to be used when closing input if it wasn't already closed - */ - _doneWritingToSink(e) { - dumpn("*** _doneWritingToSink(0x" + e.toString(16) + ")"); - - this._pendingData.length = 0; - this._sink = null; - this._doneReadingSource(e); - }, - - /** - * Completes processing of this copy: either by canceling the copy if it - * hasn't already been canceled using the provided status, or by dispatching - * the cancel callback event (with the originally provided status, of course) - * if it already has been canceled. - * - * @param status : nsresult - * the status code to use to cancel this, if this hasn't already been - * canceled - */ - _cancelOrDispatchCancelCallback(status) { - dumpn("*** _cancelOrDispatchCancelCallback(" + status + ")"); - - NS_ASSERT(this._source === null, "should have finished input"); - NS_ASSERT(this._sink === null, "should have finished output"); - NS_ASSERT(this._pendingData.length === 0, "should have no pending data"); - - if (!this._canceled) { - this.cancel(status); - return; - } - - var self = this; - var event = { - run() { - dumpn("*** onStopRequest async callback"); - - self._completed = true; - try { - self._observer.onStopRequest(self, self.status); - } catch (e) { - NS_ASSERT( - false, - "how are we throwing an exception here? we control " + - "all the callers! " + - e - ); - } - }, - }; - - gThreadManager.currentThread.dispatch(event, Ci.nsIThread.DISPATCH_NORMAL); - }, - - /** - * Kicks off another wait for more data to be available from the input stream. - */ - _waitToReadData() { - dumpn("*** _waitToReadData"); - this._source.asyncWait( - this, - 0, - Response.SEGMENT_SIZE, - gThreadManager.mainThread - ); - }, - - /** - * Kicks off another wait until data can be written to the output stream. - */ - _waitToWriteData() { - dumpn("*** _waitToWriteData"); - - var pendingData = this._pendingData; - NS_ASSERT(pendingData.length > 0, "no pending data to write?"); - NS_ASSERT(pendingData[0].length > 0, "buffered an empty write?"); - - this._sink.asyncWait( - this, - 0, - pendingData[0].length, - gThreadManager.mainThread - ); - }, - - /** - * Kicks off a wait for the sink to which data is being copied to be closed. - * We wait for stream closure when we don't have any data to be copied, rather - * than waiting to write a specific amount of data. We can't wait to write - * data because the sink might be infinitely writable, and if no data appears - * in the source for a long time we might have to spin quite a bit waiting to - * write, waiting to write again, &c. Waiting on stream closure instead means - * we'll get just one notification if the sink dies. Note that when data - * starts arriving from the sink we'll resume waiting for data to be written, - * dropping this closure-only callback entirely. - */ - _waitForSinkClosure() { - dumpn("*** _waitForSinkClosure"); - - this._sink.asyncWait( - this, - Ci.nsIAsyncOutputStream.WAIT_CLOSURE_ONLY, - 0, - gThreadManager.mainThread - ); - }, - - /** - * Closes input with the given status, if it hasn't already been closed; - * otherwise a no-op. - * - * @param status : nsresult - * status code use to close the source stream if necessary - */ - _finishSource(status) { - dumpn("*** _finishSource(" + status.toString(16) + ")"); - - if (this._source !== null) { - this._source.closeWithStatus(status); - this._source = null; - } - }, -}; - -/** - * A container for utility functions used with HTTP headers. - */ -const headerUtils = { - /** - * Normalizes fieldName (by converting it to lowercase) and ensures it is a - * valid header field name (although not necessarily one specified in RFC - * 2616). - * - * @throws NS_ERROR_INVALID_ARG - * if fieldName does not match the field-name production in RFC 2616 - * @returns string - * fieldName converted to lowercase if it is a valid header, for characters - * where case conversion is possible - */ - normalizeFieldName(fieldName) { - if (fieldName == "") { - dumpn("*** Empty fieldName"); - throw Components.Exception("", Cr.NS_ERROR_INVALID_ARG); - } - - for (var i = 0, sz = fieldName.length; i < sz; i++) { - if (!IS_TOKEN_ARRAY[fieldName.charCodeAt(i)]) { - dumpn(fieldName + " is not a valid header field name!"); - throw Components.Exception("", Cr.NS_ERROR_INVALID_ARG); - } - } - - return fieldName.toLowerCase(); - }, - - /** - * Ensures that fieldValue is a valid header field value (although not - * necessarily as specified in RFC 2616 if the corresponding field name is - * part of the HTTP protocol), normalizes the value if it is, and - * returns the normalized value. - * - * @param fieldValue : string - * a value to be normalized as an HTTP header field value - * @throws NS_ERROR_INVALID_ARG - * if fieldValue does not match the field-value production in RFC 2616 - * @returns string - * fieldValue as a normalized HTTP header field value - */ - normalizeFieldValue(fieldValue) { - // field-value = *( field-content | LWS ) - // field-content = - // TEXT = - // LWS = [CRLF] 1*( SP | HT ) - // - // quoted-string = ( <"> *(qdtext | quoted-pair ) <"> ) - // qdtext = > - // quoted-pair = "\" CHAR - // CHAR = - - // Any LWS that occurs between field-content MAY be replaced with a single - // SP before interpreting the field value or forwarding the message - // downstream (section 4.2); we replace 1*LWS with a single SP - var val = fieldValue.replace(/(?:(?:\r\n)?[ \t]+)+/g, " "); - - // remove leading/trailing LWS (which has been converted to SP) - val = val.replace(/^ +/, "").replace(/ +$/, ""); - - // that should have taken care of all CTLs, so val should contain no CTLs - dumpn("*** Normalized value: '" + val + "'"); - for (var i = 0, len = val.length; i < len; i++) { - if (isCTL(val.charCodeAt(i))) { - dump("*** Char " + i + " has charcode " + val.charCodeAt(i)); - throw Components.Exception("", Cr.NS_ERROR_INVALID_ARG); - } - } - - // XXX disallows quoted-pair where CHAR is a CTL -- will not invalidly - // normalize, however, so this can be construed as a tightening of the - // spec and not entirely as a bug - return val; - }, -}; - -/** - * Converts the given string into a string which is safe for use in an HTML - * context. - * - * @param str : string - * the string to make HTML-safe - * @returns string - * an HTML-safe version of str - */ -function htmlEscape(str) { - // this is naive, but it'll work - var s = ""; - for (var i = 0; i < str.length; i++) { - s += "&#" + str.charCodeAt(i) + ";"; - } - return s; -} - -/** - * Constructs an object representing an HTTP version (see section 3.1). - * - * @param versionString - * a string of the form "#.#", where # is an non-negative decimal integer with - * or without leading zeros - * @throws - * if versionString does not specify a valid HTTP version number - */ -function nsHttpVersion(versionString) { - var matches = /^(\d+)\.(\d+)$/.exec(versionString); - if (!matches) { - throw new Error("Not a valid HTTP version!"); - } - - /** The major version number of this, as a number. */ - this.major = parseInt(matches[1], 10); - - /** The minor version number of this, as a number. */ - this.minor = parseInt(matches[2], 10); - - if ( - isNaN(this.major) || - isNaN(this.minor) || - this.major < 0 || - this.minor < 0 - ) { - throw new Error("Not a valid HTTP version!"); - } -} -nsHttpVersion.prototype = { - /** - * Returns the standard string representation of the HTTP version represented - * by this (e.g., "1.1"). - */ - toString() { - return this.major + "." + this.minor; - }, - - /** - * Returns true if this represents the same HTTP version as otherVersion, - * false otherwise. - * - * @param otherVersion : nsHttpVersion - * the version to compare against this - */ - equals(otherVersion) { - return this.major == otherVersion.major && this.minor == otherVersion.minor; - }, - - /** True if this >= otherVersion, false otherwise. */ - atLeast(otherVersion) { - return ( - this.major > otherVersion.major || - (this.major == otherVersion.major && this.minor >= otherVersion.minor) - ); - }, -}; - -nsHttpVersion.HTTP_1_0 = new nsHttpVersion("1.0"); -nsHttpVersion.HTTP_1_1 = new nsHttpVersion("1.1"); - -/** - * An object which stores HTTP headers for a request or response. - * - * Note that since headers are case-insensitive, this object converts headers to - * lowercase before storing them. This allows the getHeader and hasHeader - * methods to work correctly for any case of a header, but it means that the - * values returned by .enumerator may not be equal case-sensitively to the - * values passed to setHeader when adding headers to this. - */ -function nsHttpHeaders() { - /** - * A hash of headers, with header field names as the keys and header field - * values as the values. Header field names are case-insensitive, but upon - * insertion here they are converted to lowercase. Header field values are - * normalized upon insertion to contain no leading or trailing whitespace. - * - * Note also that per RFC 2616, section 4.2, two headers with the same name in - * a message may be treated as one header with the same field name and a field - * value consisting of the separate field values joined together with a "," in - * their original order. This hash stores multiple headers with the same name - * in this manner. - */ - this._headers = {}; -} -nsHttpHeaders.prototype = { - /** - * Sets the header represented by name and value in this. - * - * @param name : string - * the header name - * @param value : string - * the header value - * @throws NS_ERROR_INVALID_ARG - * if name or value is not a valid header component - */ - setHeader(fieldName, fieldValue, merge) { - var name = headerUtils.normalizeFieldName(fieldName); - var value = headerUtils.normalizeFieldValue(fieldValue); - - // The following three headers are stored as arrays because their real-world - // syntax prevents joining individual headers into a single header using - // ",". See also - if (merge && name in this._headers) { - if ( - name === "www-authenticate" || - name === "proxy-authenticate" || - name === "set-cookie" - ) { - this._headers[name].push(value); - } else { - this._headers[name][0] += "," + value; - NS_ASSERT( - this._headers[name].length === 1, - "how'd a non-special header have multiple values?" - ); - } - } else { - this._headers[name] = [value]; - } - }, - - setHeaderNoCheck(fieldName, fieldValue) { - var name = headerUtils.normalizeFieldName(fieldName); - var value = headerUtils.normalizeFieldValue(fieldValue); - if (name in this._headers) { - this._headers[name].push(value); - } else { - this._headers[name] = [value]; - } - }, - - /** - * Returns the value for the header specified by this. - * - * @throws NS_ERROR_INVALID_ARG - * if fieldName does not constitute a valid header field name - * @throws NS_ERROR_NOT_AVAILABLE - * if the given header does not exist in this - * @returns string - * the field value for the given header, possibly with non-semantic changes - * (i.e., leading/trailing whitespace stripped, whitespace runs replaced - * with spaces, etc.) at the option of the implementation; multiple - * instances of the header will be combined with a comma, except for - * the three headers noted in the description of getHeaderValues - */ - getHeader(fieldName) { - return this.getHeaderValues(fieldName).join("\n"); - }, - - /** - * Returns the value for the header specified by fieldName as an array. - * - * @throws NS_ERROR_INVALID_ARG - * if fieldName does not constitute a valid header field name - * @throws NS_ERROR_NOT_AVAILABLE - * if the given header does not exist in this - * @returns [string] - * an array of all the header values in this for the given - * header name. Header values will generally be collapsed - * into a single header by joining all header values together - * with commas, but certain headers (Proxy-Authenticate, - * WWW-Authenticate, and Set-Cookie) violate the HTTP spec - * and cannot be collapsed in this manner. For these headers - * only, the returned array may contain multiple elements if - * that header has been added more than once. - */ - getHeaderValues(fieldName) { - var name = headerUtils.normalizeFieldName(fieldName); - - if (name in this._headers) { - return this._headers[name]; - } - throw Components.Exception("", Cr.NS_ERROR_NOT_AVAILABLE); - }, - - /** - * Returns true if a header with the given field name exists in this, false - * otherwise. - * - * @param fieldName : string - * the field name whose existence is to be determined in this - * @throws NS_ERROR_INVALID_ARG - * if fieldName does not constitute a valid header field name - * @returns boolean - * true if the header's present, false otherwise - */ - hasHeader(fieldName) { - var name = headerUtils.normalizeFieldName(fieldName); - return name in this._headers; - }, - - /** - * Returns a new enumerator over the field names of the headers in this, as - * nsISupportsStrings. The names returned will be in lowercase, regardless of - * how they were input using setHeader (header names are case-insensitive per - * RFC 2616). - */ - get enumerator() { - var headers = []; - for (var i in this._headers) { - var supports = new SupportsString(); - supports.data = i; - headers.push(supports); - } - - return new nsSimpleEnumerator(headers); - }, -}; - -/** - * Constructs an nsISimpleEnumerator for the given array of items. - * - * @param items : Array - * the items, which must all implement nsISupports - */ -function nsSimpleEnumerator(items) { - this._items = items; - this._nextIndex = 0; -} -nsSimpleEnumerator.prototype = { - hasMoreElements() { - return this._nextIndex < this._items.length; - }, - getNext() { - if (!this.hasMoreElements()) { - throw Components.Exception("", Cr.NS_ERROR_NOT_AVAILABLE); - } - - return this._items[this._nextIndex++]; - }, - [Symbol.iterator]() { - return this._items.values(); - }, - QueryInterface: ChromeUtils.generateQI(["nsISimpleEnumerator"]), -}; - -/** - * A representation of the data in an HTTP request. - * - * @param port : uint - * the port on which the server receiving this request runs - */ -function Request(port) { - /** Method of this request, e.g. GET or POST. */ - this._method = ""; - - /** Path of the requested resource; empty paths are converted to '/'. */ - this._path = ""; - - /** Query string, if any, associated with this request (not including '?'). */ - this._queryString = ""; - - /** Scheme of requested resource, usually http, always lowercase. */ - this._scheme = "http"; - - /** Hostname on which the requested resource resides. */ - this._host = undefined; - - /** Port number over which the request was received. */ - this._port = port; - - var bodyPipe = new Pipe(false, false, 0, PR_UINT32_MAX, null); - - /** Stream from which data in this request's body may be read. */ - this._bodyInputStream = bodyPipe.inputStream; - - /** Stream to which data in this request's body is written. */ - this._bodyOutputStream = bodyPipe.outputStream; - - /** - * The headers in this request. - */ - this._headers = new nsHttpHeaders(); - - /** - * For the addition of ad-hoc properties and new functionality without having - * to change nsIHttpRequest every time; currently lazily created, as its only - * use is in directory listings. - */ - this._bag = null; -} -Request.prototype = { - // SERVER METADATA - - // - // see nsIHttpRequest.scheme - // - get scheme() { - return this._scheme; - }, - - // - // see nsIHttpRequest.host - // - get host() { - return this._host; - }, - - // - // see nsIHttpRequest.port - // - get port() { - return this._port; - }, - - // REQUEST LINE - - // - // see nsIHttpRequest.method - // - get method() { - return this._method; - }, - - // - // see nsIHttpRequest.httpVersion - // - get httpVersion() { - return this._httpVersion.toString(); - }, - - // - // see nsIHttpRequest.path - // - get path() { - return this._path; - }, - - // - // see nsIHttpRequest.queryString - // - get queryString() { - return this._queryString; - }, - - // HEADERS - - // - // see nsIHttpRequest.getHeader - // - getHeader(name) { - return this._headers.getHeader(name); - }, - - // - // see nsIHttpRequest.hasHeader - // - hasHeader(name) { - return this._headers.hasHeader(name); - }, - - // - // see nsIHttpRequest.headers - // - get headers() { - return this._headers.enumerator; - }, - - // - // see nsIPropertyBag.enumerator - // - get enumerator() { - this._ensurePropertyBag(); - return this._bag.enumerator; - }, - - // - // see nsIHttpRequest.headers - // - get bodyInputStream() { - return this._bodyInputStream; - }, - - // - // see nsIPropertyBag.getProperty - // - getProperty(name) { - this._ensurePropertyBag(); - return this._bag.getProperty(name); - }, - - // NSISUPPORTS - - // - // see nsISupports.QueryInterface - // - QueryInterface: ChromeUtils.generateQI(["nsIHttpRequest"]), - - // PRIVATE IMPLEMENTATION - - /** Ensures a property bag has been created for ad-hoc behaviors. */ - _ensurePropertyBag() { - if (!this._bag) { - this._bag = new WritablePropertyBag(); - } - }, -}; - -/** - * Creates a new HTTP server listening for loopback traffic on the given port, - * starts it, and runs the server until the server processes a shutdown request, - * spinning an event loop so that events posted by the server's socket are - * processed. - * - * This method is primarily intended for use in running this script from within - * xpcshell and running a functional HTTP server without having to deal with - * non-essential details. - * - * Note that running multiple servers using variants of this method probably - * doesn't work, simply due to how the internal event loop is spun and stopped. - * - * @note - * This method only works with Mozilla 1.9 (i.e., Firefox 3 or trunk code); - * you should use this server as a component in Mozilla 1.8. - * @param port - * the port on which the server will run, or -1 if there exists no preference - * for a specific port; note that attempting to use some values for this - * parameter (particularly those below 1024) may cause this method to throw or - * may result in the server being prematurely shut down - * @param basePath - * a local directory from which requests will be served (i.e., if this is - * "/home/jwalden/" then a request to /index.html will load - * /home/jwalden/index.html); if this is omitted, only the default URLs in - * this server implementation will be functional - */ -function server(port, basePath) { - if (basePath) { - var lp = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsIFile); - lp.initWithPath(basePath); - } - - // if you're running this, you probably want to see debugging info - DEBUG = true; - - var srv = new nsHttpServer(); - if (lp) { - srv.registerDirectory("/", lp); - } - srv.registerContentType("sjs", SJS_TYPE); - srv.identity.setPrimary("http", "localhost", port); - srv.start(port); - - var thread = gThreadManager.currentThread; - while (!srv.isStopped()) { - thread.processNextEvent(true); - } - - // get rid of any pending requests - while (thread.hasPendingEvents()) { - thread.processNextEvent(true); - } - - DEBUG = false; -} diff --git a/test/tests/HiddenBrowserTest.js b/test/tests/HiddenBrowserTest.js index ddcbb115e9..d36ce628a5 100644 --- a/test/tests/HiddenBrowserTest.js +++ b/test/tests/HiddenBrowserTest.js @@ -1,4 +1,5 @@ describe("HiddenBrowser", function() { + var { HttpServer } = ChromeUtils.import("chrome://remote/content/server/HTTPD.jsm"); const { HiddenBrowser } = ChromeUtils.import( "chrome://zotero/content/HiddenBrowser.jsm" ); @@ -11,7 +12,7 @@ describe("HiddenBrowser", function() { var pngRequested = false; before(function () { - Cu.import("resource://zotero-unit/httpd.js"); + Zotero.debug(HttpServer); httpd = new HttpServer(); httpd.start(port); }); @@ -93,7 +94,6 @@ describe("HiddenBrowser", function() { } before(function () { - Cu.import("resource://zotero-unit/httpd.js"); httpd = new HttpServer(); httpd.start(port); @@ -190,7 +190,6 @@ describe("HiddenBrowser", function() { var baseURL2 = `http://127.0.0.1:${port2}/`; before(function () { - Cu.import("resource://zotero-unit/httpd.js"); // Create two servers with two separate origins httpd1 = new HttpServer(); httpd1.start(port1); diff --git a/test/tests/attachmentsTest.js b/test/tests/attachmentsTest.js index 9968d98ad6..7534a57206 100644 --- a/test/tests/attachmentsTest.js +++ b/test/tests/attachmentsTest.js @@ -1268,7 +1268,7 @@ describe("Zotero.Attachments", function() { var resolvers = [{ name: 'Custom', method: 'get', - // Registered with httpd.js in beforeEach() + // Registered with HTTPD.jsm in beforeEach() url: baseURL + "{doi}", mode: 'html', selector: '#pdf-link', diff --git a/test/tests/collectionTreeTest.js b/test/tests/collectionTreeTest.js index 5574181770..d3237c7438 100644 --- a/test/tests/collectionTreeTest.js +++ b/test/tests/collectionTreeTest.js @@ -1385,7 +1385,7 @@ describe("Zotero.CollectionTree", function() { describe("with feed items", function () { - Components.utils.import("resource://zotero-unit/httpd.js"); + var { HttpServer } = ChromeUtils.import("chrome://remote/content/server/HTTPD.jsm");; const httpdPort = 16214; var httpd; diff --git a/test/tests/fileTest.js b/test/tests/fileTest.js index 78c32e9f90..e99341faf1 100644 --- a/test/tests/fileTest.js +++ b/test/tests/fileTest.js @@ -469,7 +469,7 @@ describe("Zotero.File", function () { before(async function () { // Real HTTP server - Components.utils.import("resource://zotero-unit/httpd.js"); + var { HttpServer } = ChromeUtils.import("chrome://remote/content/server/HTTPD.jsm");; port = 16213; httpd = new HttpServer(); baseURL = `http://127.0.0.1:${port}`; diff --git a/test/tests/httpTest.js b/test/tests/httpTest.js index 4feebfc772..636fa5b56b 100644 --- a/test/tests/httpTest.js +++ b/test/tests/httpTest.js @@ -13,7 +13,7 @@ describe("Zotero.HTTP", function () { before(function* () { // Real HTTP server - Components.utils.import("resource://zotero-unit/httpd.js"); + var { HttpServer } = ChromeUtils.import("chrome://remote/content/server/HTTPD.jsm");; httpd = new HttpServer(); httpd.start(port); httpd.registerPathHandler( diff --git a/test/tests/itemTreeTest.js b/test/tests/itemTreeTest.js index 0ce4799437..96b51fe894 100644 --- a/test/tests/itemTreeTest.js +++ b/test/tests/itemTreeTest.js @@ -1004,7 +1004,7 @@ describe("Zotero.ItemTree", function() { // Serve a PDF to test URL dragging before(function () { - Components.utils.import("resource://zotero-unit/httpd.js"); + var { HttpServer } = ChromeUtils.import("chrome://remote/content/server/HTTPD.jsm");; httpd = new HttpServer(); httpd.start(port); var file = getTestDataDirectory(); diff --git a/test/tests/mendeleyImportTest.js b/test/tests/mendeleyImportTest.js index 56f176923d..2dad8d3fd2 100644 --- a/test/tests/mendeleyImportTest.js +++ b/test/tests/mendeleyImportTest.js @@ -23,7 +23,7 @@ describe('Zotero_Import_Mendeley', function () { Components.utils.import('chrome://zotero/content/import/mendeley/mendeleyImport.js'); // A real HTTP server is used to deliver a Bitcoin PDF so that annotations can be processed during import. - Components.utils.import("resource://zotero-unit/httpd.js"); + var { HttpServer } = ChromeUtils.import("chrome://remote/content/server/HTTPD.jsm");; const port = 16213; httpd = new HttpServer(); httpdURL = `http://127.0.0.1:${port}`; diff --git a/test/tests/recognizeDocumentTest.js b/test/tests/recognizeDocumentTest.js index d6dcd6fa2b..58b87558ca 100644 --- a/test/tests/recognizeDocumentTest.js +++ b/test/tests/recognizeDocumentTest.js @@ -37,7 +37,7 @@ describe("Document Recognition", function() { } queue.cancel(); - Zotero.RecognizeDocument.recognizeStub = null; + Zotero.RecognizeDocument._recognize.restore && Zotero.RecognizeDocument._recognize.restore(); Zotero.Prefs.clear('autoRenameFiles.linked'); }); @@ -223,9 +223,9 @@ describe("Document Recognition", function() { it("should rename a linked file attachment using parent metadata if no existing file attachments and pref enabled", async function () { Zotero.Prefs.set('autoRenameFiles.linked', true); var itemTitle = Zotero.Utilities.randomString(); - Zotero.RecognizeDocument.recognizeStub = async function () { + sinon.stub(Zotero.RecognizeDocument, "_recognize").callsFake(() => { return createDataObject('item', { title: itemTitle }); - }; + }); // Link to the PDF var tempDir = await getTempDirectory(); @@ -263,9 +263,9 @@ describe("Document Recognition", function() { Zotero.Prefs.set('autoRenameFiles.fileTypes', 'x-nonexistent/type'); var itemTitle = Zotero.Utilities.randomString(); - Zotero.RecognizeDocument.recognizeStub = async function () { + sinon.stub(Zotero.RecognizeDocument, "_recognize").callsFake(() => { return createDataObject('item', { title: itemTitle }); - }; + }); var attachment = await importPDFAttachment(); assert.equal(attachment.getField('title'), 'test'); @@ -291,9 +291,9 @@ describe("Document Recognition", function() { it("shouldn't rename a linked file attachment using parent metadata if pref disabled", async function () { Zotero.Prefs.set('autoRenameFiles.linked', false); var itemTitle = Zotero.Utilities.randomString(); - Zotero.RecognizeDocument.recognizeStub = async function () { + sinon.stub(Zotero.RecognizeDocument, "_recognize").callsFake(() => { return createDataObject('item', { title: itemTitle }); - }; + }); // Link to the PDF var tempDir = await getTempDirectory(); diff --git a/test/tests/serverTest.js b/test/tests/serverTest.js index f73ed5b3b7..662838d2d9 100644 --- a/test/tests/serverTest.js +++ b/test/tests/serverTest.js @@ -1,7 +1,8 @@ "use strict"; +Components.utils.import("resource://gre/modules/NetUtil.jsm"); + describe("Zotero.Server", function () { - Components.utils.import("resource://zotero-unit/httpd.js"); var serverPath; before(function* () { @@ -284,10 +285,53 @@ describe("Zotero.Server", function () { } ); + assert.ok(called); + assert.equal(req.status, 204); + }); + }); + describe("application/pdf", function () { + it('should provide a stream', async function () { + let called = false; + let endpoint = "/test/" + Zotero.Utilities.randomString(); + let file = getTestDataDirectory(); + file.append('test.pdf'); + let contents = await Zotero.File.getBinaryContentsAsync(file); + + Zotero.Server.Endpoints[endpoint] = function () {}; + Zotero.Server.Endpoints[endpoint].prototype = { + supportedMethods: ["POST"], + supportedDataTypes: ["application/pdf"], + + init: function (options) { + called = true; + assert.isObject(options); + assert.property(options.headers, "Content-Type"); + assert(options.headers["Content-Type"].startsWith("application/pdf")); + assert.isFunction(options.data.available); + let data = NetUtil.readInputStreamToString(options.data, options.headers['content-length']); + assert.equal(data, contents); + + return 204; + } + }; + + let pdf = await File.createFromFileName(OS.Path.join(getTestDataDirectory().path, 'test.pdf')); + + let req = await Zotero.HTTP.request( + "POST", + serverPath + endpoint, + { + headers: { + "Content-Type": "application/pdf", + }, + body: pdf + } + ); + assert.ok(called); assert.equal(req.status, 204); }); }); }); - }) + }); }); diff --git a/test/tests/server_connectorTest.js b/test/tests/server_connectorTest.js index 5d4823f1f9..b4b3d58dc0 100644 --- a/test/tests/server_connectorTest.js +++ b/test/tests/server_connectorTest.js @@ -1,7 +1,17 @@ "use strict"; +let httpRequest = (method, url, options) => { + if (!options) { + options = {}; + } + if (!('errorDelayMax' in options)) { + options.errorDelayMax = 0; + } + return Zotero.HTTP.request(method, url, options); +} + describe("Connector Server", function () { - Components.utils.import("resource://zotero-unit/httpd.js"); + var { HttpServer } = ChromeUtils.import("chrome://remote/content/server/HTTPD.jsm"); var win, connectorServerPath, testServerPath, httpd; var testServerPort = 16213; var snapshotHTML = "TitleBody"; @@ -54,7 +64,7 @@ describe("Connector Server", function () { var translator = buildDummyTranslator(4, code); sinon.stub(Zotero.Translators, 'get').returns(translator); - var response = yield Zotero.HTTP.request( + var response = yield httpRequest( 'POST', connectorServerPath + "/connector/getTranslatorCode", { @@ -82,7 +92,7 @@ describe("Connector Server", function () { var translator = buildDummyTranslator("web", code, {target: "https://www.example.com/.*"}); sinon.stub(Zotero.Translators, 'getAllForType').resolves([translator]); - var response = yield Zotero.HTTP.request( + var response = yield httpRequest( 'POST', connectorServerPath + "/connector/detect", { @@ -104,7 +114,6 @@ describe("Connector Server", function () { describe("/connector/saveItems", function () { - // TODO: Test cookies it("should save a translated item to the current selected collection", function* () { var collection = yield createDataObject('collection'); yield select(win, collection); @@ -121,30 +130,14 @@ describe("Connector Server", function () { creatorType: "author" } ], - attachments: [ - { - title: "Attachment", - url: `${testServerPath}/attachment`, - mimeType: "text/html" - } - ] } ], uri: "http://example.com" }; - httpd.registerPathHandler( - "/attachment", - { - handle: function (request, response) { - response.setStatusLine(null, 200, "OK"); - response.write("TitleBody"); - } - } - ); var promise = waitForItemEvent('add'); - var reqPromise = Zotero.HTTP.request( + var reqPromise = httpRequest( 'POST', connectorServerPath + "/connector/saveItems", { @@ -162,13 +155,6 @@ describe("Connector Server", function () { assert.equal(Zotero.ItemTypes.getName(item.itemTypeID), 'newspaperArticle'); assert.isTrue(collection.hasItem(item.id)); - // Check attachment - promise = waitForItemEvent('add'); - ids = yield promise; - assert.lengthOf(ids, 1); - item = Zotero.Items.get(ids[0]); - assert.isTrue(item.isImportedAttachment()); - var req = yield reqPromise; assert.equal(req.status, 201); }); @@ -199,7 +185,7 @@ describe("Connector Server", function () { }; var promise = waitForItemEvent('add'); - var reqPromise = Zotero.HTTP.request( + var reqPromise = httpRequest( 'POST', connectorServerPath + "/connector/saveItems", { @@ -251,7 +237,7 @@ describe("Connector Server", function () { }; var promise = waitForItemEvent('add'); - var req = yield Zotero.HTTP.request( + var req = yield httpRequest( 'POST', connectorServerPath + "/connector/saveItems", { @@ -268,493 +254,6 @@ describe("Connector Server", function () { var item = Zotero.Items.get(ids[0]); assert.equal(item.getField('url'), 'https://www.example.com/path'); }); - - it("shouldn't return an attachment that isn't being saved", async function () { - Zotero.Prefs.set('automaticSnapshots', false); - - await selectLibrary(win, Zotero.Libraries.userLibraryID); - await waitForItemsLoad(win); - - var body = { - items: [ - { - itemType: "webpage", - title: "Title", - creators: [], - attachments: [ - { - url: "http://example.com/", - mimeType: "text/html" - } - ], - url: "http://example.com/" - } - ], - uri: "http://example.com/" - }; - - var req = await Zotero.HTTP.request( - 'POST', - connectorServerPath + "/connector/saveItems", - { - headers: { - "Content-Type": "application/json" - }, - body: JSON.stringify(body), - responseType: 'json' - } - ); - - Zotero.Prefs.clear('automaticSnapshots'); - - assert.equal(req.status, 201); - assert.lengthOf(req.response.items, 1); - assert.lengthOf(req.response.items[0].attachments, 0); - }); - - describe("PDF retrieval", function () { - var oaDOI = '10.1111/abcd'; - var nonOADOI = '10.2222/bcde'; - var pdfURL; - var badPDFURL; - var stub; - - before(function () { - var origFunc = Zotero.HTTP.request.bind(Zotero.HTTP); - stub = sinon.stub(Zotero.HTTP, 'request'); - stub.callsFake(function (method, url, options) { - // OA PDF lookup - if (url.startsWith(ZOTERO_CONFIG.SERVICES_URL)) { - let json = JSON.parse(options.body); - let response = []; - if (json.doi == oaDOI) { - response.push({ - url: pdfURL, - version: 'submittedVersion' - }); - } - return { - status: 200, - response - }; - } - - return origFunc(...arguments); - }); - }); - - beforeEach(() => { - pdfURL = testServerPath + '/pdf'; - badPDFURL = testServerPath + '/badpdf'; - - httpd.registerFile( - pdfURL.substr(testServerPath.length), - Zotero.File.pathToFile(OS.Path.join(getTestDataDirectory().path, 'test.pdf')) - ); - // PDF URL that's actually an HTML page - httpd.registerFile( - badPDFURL.substr(testServerPath.length), - Zotero.File.pathToFile(OS.Path.join(getTestDataDirectory().path, 'test.html')) - ); - }); - - afterEach(() => { - stub.resetHistory(); - }); - - after(() => { - stub.restore(); - }); - - - it("should download a translated PDF", async function () { - var collection = await createDataObject('collection'); - await select(win, collection); - - var sessionID = Zotero.Utilities.randomString(); - - // Save item - var itemAddPromise = waitForItemEvent('add'); - var saveItemsReq = await Zotero.HTTP.request( - 'POST', - connectorServerPath + "/connector/saveItems", - { - headers: { - "Content-Type": "application/json" - }, - body: JSON.stringify({ - sessionID, - items: [ - { - itemType: 'journalArticle', - title: 'Title', - DOI: nonOADOI, - attachments: [ - { - title: "PDF", - url: pdfURL, - mimeType: 'application/pdf' - } - ] - } - ], - uri: 'http://website/article' - }), - responseType: 'json' - } - ); - assert.equal(saveItemsReq.status, 201); - assert.lengthOf(saveItemsReq.response.items, 1); - // Translated attachment should show up in the initial response - assert.lengthOf(saveItemsReq.response.items[0].attachments, 1); - assert.notProperty(saveItemsReq.response.items[0], 'DOI'); - assert.notProperty(saveItemsReq.response.items[0].attachments[0], 'progress'); - - // Check parent item - var ids = await itemAddPromise; - assert.lengthOf(ids, 1); - var item = Zotero.Items.get(ids[0]); - assert.equal(Zotero.ItemTypes.getName(item.itemTypeID), 'journalArticle'); - assert.isTrue(collection.hasItem(item.id)); - - // Legacy endpoint should show 0 - let attachmentProgressReq = await Zotero.HTTP.request( - 'POST', - connectorServerPath + "/connector/attachmentProgress", - { - headers: { - "Content-Type": "application/json" - }, - body: JSON.stringify([saveItemsReq.response.items[0].attachments[0].id]), - responseType: 'json' - } - ); - assert.equal(attachmentProgressReq.status, 200); - let progress = attachmentProgressReq.response; - assert.sameOrderedMembers(progress, [0]); - - // Wait for the attachment to finish saving - itemAddPromise = waitForItemEvent('add'); - var i = 0; - while (i < 3) { - let sessionProgressReq = await Zotero.HTTP.request( - 'POST', - connectorServerPath + "/connector/sessionProgress", - { - headers: { - "Content-Type": "application/json" - }, - body: JSON.stringify({ sessionID }), - responseType: 'json' - } - ); - assert.equal(sessionProgressReq.status, 200); - let response = sessionProgressReq.response; - assert.lengthOf(response.items, 1); - let item = response.items[0]; - if (item.attachments.length) { - await Zotero.Promise.delay(10); - let attachments = item.attachments; - assert.lengthOf(attachments, 1); - let attachment = attachments[0]; - switch (i) { - // Translated PDF in progress - case 0: - if (attachment.title == "PDF" - && Number.isInteger(attachment.progress) - && attachment.progress < 100) { - assert.isFalse(response.done); - i++; - } - continue; - - // Translated PDF finished - case 1: - if (attachment.title == "PDF" && attachment.progress == 100) { - i++; - } - continue; - - // done: true - case 2: - if (response.done) { - i++; - } - continue; - } - } - } - - // Legacy endpoint should show 100 - attachmentProgressReq = await Zotero.HTTP.request( - 'POST', - connectorServerPath + "/connector/attachmentProgress", - { - headers: { - "Content-Type": "application/json" - }, - body: JSON.stringify([saveItemsReq.response.items[0].attachments[0].id]), - responseType: 'json' - } - ); - assert.equal(attachmentProgressReq.status, 200); - progress = attachmentProgressReq.response; - assert.sameOrderedMembers(progress, [100]); - - // Check attachment - var ids = await itemAddPromise; - assert.lengthOf(ids, 1); - item = Zotero.Items.get(ids[0]); - assert.isTrue(item.isImportedAttachment()); - assert.equal(item.getField('title'), 'PDF'); - }); - - - it("should download open-access PDF if no PDF provided", async function () { - var collection = await createDataObject('collection'); - await select(win, collection); - - var sessionID = Zotero.Utilities.randomString(); - - // Save item - var itemAddPromise = waitForItemEvent('add'); - var saveItemsReq = await Zotero.HTTP.request( - 'POST', - connectorServerPath + "/connector/saveItems", - { - headers: { - "Content-Type": "application/json" - }, - body: JSON.stringify({ - sessionID, - items: [ - { - itemType: 'journalArticle', - title: 'Title', - DOI: oaDOI, - attachments: [] - } - ], - uri: 'http://website/article' - }), - responseType: 'json' - } - ); - assert.equal(saveItemsReq.status, 201); - assert.lengthOf(saveItemsReq.response.items, 1); - // Attachment shouldn't show up in the initial response - assert.lengthOf(saveItemsReq.response.items[0].attachments, 0); - - // Check parent item - var ids = await itemAddPromise; - assert.lengthOf(ids, 1); - var item = Zotero.Items.get(ids[0]); - assert.equal(Zotero.ItemTypes.getName(item.itemTypeID), 'journalArticle'); - assert.isTrue(collection.hasItem(item.id)); - - // Wait for the attachment to finish saving - itemAddPromise = waitForItemEvent('add'); - var wasZero = false; - var was100 = false; - while (true) { - let sessionProgressReq = await Zotero.HTTP.request( - 'POST', - connectorServerPath + "/connector/sessionProgress", - { - headers: { - "Content-Type": "application/json" - }, - body: JSON.stringify({ sessionID }), - responseType: 'json' - } - ); - assert.equal(sessionProgressReq.status, 200); - let response = sessionProgressReq.response; - assert.typeOf(response.items, 'array'); - assert.lengthOf(response.items, 1); - let item = response.items[0]; - if (item.attachments.length) { - // 'progress' should have started at 0 - if (item.attachments[0].progress === 0) { - wasZero = true; - } - else if (!was100 && item.attachments[0].progress == 100) { - if (response.done) { - break; - } - was100 = true; - } - else if (response.done) { - break; - } - } - assert.isFalse(response.done); - await Zotero.Promise.delay(10); - } - assert.isTrue(wasZero); - - // Check attachment - var ids = await itemAddPromise; - assert.lengthOf(ids, 1); - item = Zotero.Items.get(ids[0]); - assert.isTrue(item.isImportedAttachment()); - assert.equal(item.getField('title'), Zotero.getString('attachment.submittedVersion')); - }); - - - it("should download open-access PDF if a translated PDF fails", async function () { - var collection = await createDataObject('collection'); - await select(win, collection); - - var sessionID = Zotero.Utilities.randomString(); - - // Save item - var itemAddPromise = waitForItemEvent('add'); - var saveItemsReq = await Zotero.HTTP.request( - 'POST', - connectorServerPath + "/connector/saveItems", - { - headers: { - "Content-Type": "application/json" - }, - body: JSON.stringify({ - sessionID, - items: [ - { - itemType: 'journalArticle', - title: 'Title', - DOI: oaDOI, - attachments: [ - { - title: "PDF", - url: badPDFURL, - mimeType: 'application/pdf' - } - ] - } - ], - uri: 'http://website/article' - }), - responseType: 'json' - } - ); - assert.equal(saveItemsReq.status, 201); - assert.lengthOf(saveItemsReq.response.items, 1); - // Translated attachment should show up in the initial response - assert.lengthOf(saveItemsReq.response.items[0].attachments, 1); - assert.notProperty(saveItemsReq.response.items[0], 'DOI'); - assert.notProperty(saveItemsReq.response.items[0].attachments[0], 'progress'); - - // Check parent item - var ids = await itemAddPromise; - assert.lengthOf(ids, 1); - var item = Zotero.Items.get(ids[0]); - assert.equal(Zotero.ItemTypes.getName(item.itemTypeID), 'journalArticle'); - assert.isTrue(collection.hasItem(item.id)); - - // Legacy endpoint should show 0 - let attachmentProgressReq = await Zotero.HTTP.request( - 'POST', - connectorServerPath + "/connector/attachmentProgress", - { - headers: { - "Content-Type": "application/json" - }, - body: JSON.stringify([saveItemsReq.response.items[0].attachments[0].id]), - responseType: 'json' - } - ); - assert.equal(attachmentProgressReq.status, 200); - let progress = attachmentProgressReq.response; - assert.sameOrderedMembers(progress, [0]); - - // Wait for the attachment to finish saving - itemAddPromise = waitForItemEvent('add'); - var i = 0; - while (i < 4) { - let sessionProgressReq = await Zotero.HTTP.request( - 'POST', - connectorServerPath + "/connector/sessionProgress", - { - headers: { - "Content-Type": "application/json" - }, - body: JSON.stringify({ sessionID }), - responseType: 'json' - } - ); - assert.equal(sessionProgressReq.status, 200); - let response = sessionProgressReq.response; - assert.lengthOf(response.items, 1); - let item = response.items[0]; - if (item.attachments.length) { - let attachments = item.attachments; - assert.lengthOf(attachments, 1); - let attachment = attachments[0]; - switch (i) { - // Translated PDF in progress - case 0: - if (attachment.title == "PDF" - && Number.isInteger(attachment.progress) - && attachment.progress < 100) { - assert.isFalse(response.done); - i++; - } - continue; - - // OA PDF in progress - case 1: - if (attachment.title == Zotero.getString('findPDF.openAccessPDF') - && Number.isInteger(attachment.progress) - && attachment.progress < 100) { - assert.isFalse(response.done); - i++; - } - continue; - - // OA PDF finished - case 2: - if (attachment.progress === 100) { - assert.equal(attachment.title, Zotero.getString('findPDF.openAccessPDF')); - i++; - } - continue; - - // done: true - case 3: - if (response.done) { - i++; - } - continue; - } - } - await Zotero.Promise.delay(10); - } - - // Legacy endpoint should show 100 - attachmentProgressReq = await Zotero.HTTP.request( - 'POST', - connectorServerPath + "/connector/attachmentProgress", - { - headers: { - "Content-Type": "application/json" - }, - body: JSON.stringify([saveItemsReq.response.items[0].attachments[0].id]), - responseType: 'json' - } - ); - assert.equal(attachmentProgressReq.status, 200); - progress = attachmentProgressReq.response; - assert.sameOrderedMembers(progress, [100]); - - // Check attachment - var ids = await itemAddPromise; - assert.lengthOf(ids, 1); - item = Zotero.Items.get(ids[0]); - assert.isTrue(item.isImportedAttachment()); - assert.equal(item.getField('title'), Zotero.getString('attachment.submittedVersion')); - }); - }); }); describe("/connector/saveSingleFile", function () { @@ -774,10 +273,9 @@ describe("Connector Server", function () { sessionID, url: "http://example.com/test", title, - singleFile: true }; - await Zotero.HTTP.request( + await httpRequest( 'POST', connectorServerPath + "/connector/saveSnapshot", { @@ -805,7 +303,7 @@ describe("Connector Server", function () { snapshotContent: await Zotero.File.getContentsAsync(indexPath) })); - await Zotero.HTTP.request( + await httpRequest( 'POST', connectorServerPath + "/connector/saveSingleFile", { @@ -827,7 +325,7 @@ describe("Connector Server", function () { // Check attachment html file let attachmentDirectory = Zotero.Attachments.getStorageDirectory(item).path; - let path = OS.Path.join(attachmentDirectory, 'test.html'); + let path = OS.Path.join(attachmentDirectory, item.attachmentFilename); assert.isTrue(await OS.File.exists(path)); let contents = await Zotero.File.getContentsAsync(path); let expectedContents = await Zotero.File.getContentsAsync(indexPath); @@ -852,14 +350,6 @@ describe("Connector Server", function () { lastName: "Last", creatorType: "author" } - ], - attachments: [ - { - title: "Snapshot", - url: `${testServerPath}/attachment`, - mimeType: "text/html", - singleFile: true - } ] } ], @@ -867,7 +357,7 @@ describe("Connector Server", function () { }; let promise = waitForItemEvent('add'); - let req = await Zotero.HTTP.request( + let req = await httpRequest( 'POST', connectorServerPath + "/connector/saveItems", { @@ -893,10 +383,11 @@ describe("Connector Server", function () { let indexPath = OS.Path.join(testDataDirectory, 'snapshot', 'index.html'); let body = JSON.stringify(Object.assign(payload, { + url: `${testServerPath}/attachment`, snapshotContent: await Zotero.File.getContentsAsync(indexPath) })); - req = await Zotero.HTTP.request( + req = await httpRequest( 'POST', connectorServerPath + "/connector/saveSingleFile", { @@ -915,395 +406,27 @@ describe("Connector Server", function () { assert.lengthOf(attachmentIDs, 1); item = Zotero.Items.get(attachmentIDs[0]); assert.isTrue(item.isImportedAttachment()); - assert.equal(item.getField('title'), 'Snapshot'); - - // Check attachment html file - let attachmentDirectory = Zotero.Attachments.getStorageDirectory(item).path; - let path = OS.Path.join(attachmentDirectory, 'attachment.html'); - assert.isTrue(await OS.File.exists(path)); - let contents = await Zotero.File.getContentsAsync(path); - let expectedContents = await Zotero.File.getContentsAsync(indexPath); - assert.equal(contents, expectedContents); - }); - - it("should override SingleFileZ from old connector in /saveSnapshot", async function () { - Components.utils.import("resource://gre/modules/FileUtils.jsm"); - var collection = await createDataObject('collection'); - await select(win, collection); - - // Promise for item save - let promise = waitForItemEvent('add'); - - let testDataDirectory = getTestDataDirectory().path; - let indexPath = OS.Path.join(testDataDirectory, 'snapshot', 'index.html'); - - let prefix = '/' + Zotero.Utilities.randomString() + '/'; - let uri = OS.Path.join(getTestDataDirectory().path, 'snapshot'); - httpd.registerDirectory(prefix, new FileUtils.File(uri)); - - let title = Zotero.Utilities.randomString(); - let sessionID = Zotero.Utilities.randomString(); - let payload = { - sessionID, - url: testServerPath + prefix + 'index.html', - title, - singleFile: true - }; - - await Zotero.HTTP.request( - 'POST', - connectorServerPath + "/connector/saveSnapshot", - { - headers: { - "Content-Type": "application/json" - }, - body: JSON.stringify(payload) - } - ); - - // Await item save - let parentIDs = await promise; - - // Check parent item - assert.lengthOf(parentIDs, 1); - var item = Zotero.Items.get(parentIDs[0]); - assert.equal(Zotero.ItemTypes.getName(item.itemTypeID), 'webpage'); - assert.isTrue(collection.hasItem(item.id)); - assert.equal(item.getField('title'), title); - - // Promise for attachment save - promise = waitForItemEvent('add'); - - let body = new FormData(); - let uuid = 'binary-' + Zotero.Utilities.randomString(); - body.append("payload", JSON.stringify(Object.assign(payload, { - pageData: { - content: await Zotero.File.getContentsAsync(indexPath), - resources: { - images: [ - { - name: "img.gif", - content: uuid, - binary: true - } - ] - } - } - }))); - - await Zotero.HTTP.request( - 'POST', - connectorServerPath + "/connector/saveSingleFile", - { - headers: { - "Content-Type": "multipart/form-data", - "zotero-allowed-request": "true" - }, - body - } - ); - - // Await attachment save - let attachmentIDs = await promise; - - // Check attachment - assert.lengthOf(attachmentIDs, 1); - item = Zotero.Items.get(attachmentIDs[0]); - assert.isTrue(item.isImportedAttachment()); - assert.equal(item.getField('title'), title); + assert.equal(item.getField('title'), 'Test'); // Check attachment html file let attachmentDirectory = Zotero.Attachments.getStorageDirectory(item).path; let path = OS.Path.join(attachmentDirectory, item.attachmentFilename); assert.isTrue(await OS.File.exists(path)); let contents = await Zotero.File.getContentsAsync(path); - assert.match(contents, /^