mirrors/zotero
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

- Improvements to server.js for translation-server

Browse source 
- Optimizations. The biggest of these is to simplify our mechanism of wrapping functions for Fx 4+, which gives us roughly a 3x speed boost in RIS import. However, zotero-node is still ~20% faster than translation-server, and RDF import/export may still be too slow for very large numbers of references. A large part of the RDF overhead seems to come from the number of function calls we make, which numbers in the hundreds of thousands for a 2.5 MB file.
This commit is contained in:
Simon Kornblith 2011-07-16 20:47:17 +00:00
parent 55c331e68b
commit 4c9b5935e8
4 changed files with 144 additions and 69 deletions
Download patch file Download diff file Expand all files Collapse all files

74
chrome/content/zotero/xpcom/server.js
View file

@ -39,7 +39,7 @@ Zotero.Server = new function() {
/**
* initializes a very rudimentary web server
*/
this.init = function() {
this.init = function(port, bindAllAddr, maxConcurrentConnections) {
if (Zotero.HTTP.browserIsOffline()) {
Zotero.debug('Browser is offline -- not initializing HTTP server');
_registerOnlineObserver();
@ -51,10 +51,10 @@ Zotero.Server = new function() {
.createInstance(Components.interfaces.nsIServerSocket);
try {
// bind to a random port on loopback only
serv.init(Zotero.Prefs.get('httpServer.port'), true, -1);
serv.init(port ? port : Zotero.Prefs.get('httpServer.port'), !bindAllAddr, -1);
serv.asyncListen(Zotero.Server.SocketListener);
Zotero.debug("HTTP server listening on 127.0.0.1:"+serv.port);
Zotero.debug("HTTP server listening on "+(bindAllAddr ? "*": " 127.0.0.1")+":"+serv.port);
} catch(e) {
Zotero.debug("Not initializing HTTP server");
}
@ -82,6 +82,20 @@ Zotero.Server = new function() {
return response;
}
/**
* Parses a query string into a key => value object
* @param {String} queryString Query string
*/
this.decodeQueryString = function(queryString) {
var splitData = queryString.split("&");
var decodedData = {};
for each(var variable in splitData) {
var splitIndex = variable.indexOf("=");
decodedData[decodeURIComponent(variable.substr(0, splitIndex))] = decodeURIComponent(variable.substr(splitIndex+1));
}
return decodedData;
}
function _registerOnlineObserver() {
if (_onlineObserverRegistered) {
return;
@ -218,7 +232,7 @@ Zotero.Server.DataListener.prototype.onDataAvailable = function(request, context
Zotero.Server.DataListener.prototype._headerFinished = function() {
this.headerFinished = true;
Zotero.debug(this.header);
Zotero.debug(this.header, 5);
const methodRe = /^([A-Z]+) ([^ \r\n?]+)(\?[^ \r\n]+)?/;
const contentTypeRe = /[\r\n]Content-Type: +([^ \r\n]+)/i;
@ -233,33 +247,35 @@ Zotero.Server.DataListener.prototype._headerFinished = function() {
}
if(!method) {
this._requestFinished(Zotero.Server.generateResponse(400));
this._requestFinished(Zotero.Server.generateResponse(400, "text/plain", "Invalid method specified\n"));
return;
}
if(!Zotero.Server.Endpoints[method[2]]) {
this._requestFinished(Zotero.Server.generateResponse(404));
this._requestFinished(Zotero.Server.generateResponse(404, "text/plain", "No endpoint found\n"));
return;
}
this.pathname = method[2];
this.endpoint = Zotero.Server.Endpoints[method[2]];
this.query = method[3];
if(method[1] == "HEAD" || method[1] == "OPTIONS") {
this._requestFinished(Zotero.Server.generateResponse(200));
} else if(method[1] == "GET") {
this._requestFinished(this._processEndpoint("GET", method[3]));
this._processEndpoint("GET", null);
} else if(method[1] == "POST") {
const contentLengthRe = /[\r\n]Content-Length: +([0-9]+)/i;
// parse content length
var m = contentLengthRe.exec(this.header);
if(!m) {
this._requestFinished(Zotero.Server.generateResponse(400));
this._requestFinished(Zotero.Server.generateResponse(400, "text/plain", "Content-length not provided\n"));
return;
}
this.bodyLength = parseInt(m[1]);
this._bodyData();
} else {
this._requestFinished(Zotero.Server.generateResponse(501));
this._requestFinished(Zotero.Server.generateResponse(501, "text/plain", "Method not implemented\n"));
return;
}
}
@ -297,29 +313,30 @@ Zotero.Server.DataListener.prototype._processEndpoint = function(method, postDat
var endpoint = new this.endpoint;
// check that endpoint supports method
if(endpoint.supportedMethods.indexOf(method) === -1) {
this._requestFinished(Zotero.Server.generateResponse(400));
if(endpoint.supportedMethods && endpoint.supportedMethods.indexOf(method) === -1) {
this._requestFinished(Zotero.Server.generateResponse(400, "text/plain", "Endpoint does not support method\n"));
return;
}
var decodedData = null;
if(postData && this.contentType) {
// check that endpoint supports contentType
if(endpoint.supportedDataTypes.indexOf(this.contentType) === -1) {
this._requestFinished(Zotero.Server.generateResponse(400));
var supportedDataTypes = endpoint.supportedDataTypes;
if(supportedDataTypes && supportedDataTypes.indexOf(this.contentType) === -1) {
this._requestFinished(Zotero.Server.generateResponse(400, "text/plain", "Endpoint does not support content-type\n"));
return;
}
// decode JSON or urlencoded post data, and pass through anything else
if(this.contentType === "application/json") {
decodedData = JSON.parse(postData);
} else if(this.contentType === "application/x-www-urlencoded") {
var splitData = postData.split("&");
decodedData = {};
for each(var variable in splitData) {
var splitIndex = variable.indexOf("=");
data[decodeURIComponent(variable.substr(0, splitIndex))] = decodeURIComponent(variable.substr(splitIndex+1));
if(supportedDataTypes && this.contentType === "application/json") {
try {
decodedData = JSON.parse(postData);
} catch(e) {
this._requestFinished(Zotero.Server.generateResponse(400, "text/plain", "Invalid JSON provided\n"));
return;
}
} else if(supportedDataTypes && this.contentType === "application/x-www-urlencoded") {
decodedData = Zotero.Server.decodeQueryString(postData);
} else {
decodedData = postData;
}
@ -332,10 +349,19 @@ Zotero.Server.DataListener.prototype._processEndpoint = function(method, postDat
}
// pass to endpoint
endpoint.init(decodedData, sendResponseCallback);
if((endpoint.init.length ? endpoint.init.length : endpoint.init.arity) === 3) {
var url = {
"pathname":this.pathname,
"query":this.query ? Zotero.Server.decodeQueryString(this.query.substr(1)) : {}
};
endpoint.init(url, decodedData, sendResponseCallback);
} else {
endpoint.init(decodedData, sendResponseCallback);
}
} catch(e) {
Zotero.debug(e);
this._requestFinished(Zotero.Server.generateResponse(500));
this._requestFinished(Zotero.Server.generateResponse(500), "text/plain", "An error occurred\n");
throw e;
}
}
@ -356,7 +382,7 @@ Zotero.Server.DataListener.prototype._requestFinished = function(response) {
intlStream.init(this.oStream, "UTF-8", 1024, "?".charCodeAt(0));
// write response
Zotero.debug(response);
Zotero.debug(response, 5);
intlStream.writeString(response);
} finally {
intlStream.close();

41
chrome/content/zotero/xpcom/translation/translate.js
View file

@ -81,7 +81,7 @@ Zotero.Translate.Sandbox = {
* @param {SandboxItem} An item created using the Zotero.Item class from the sandbox
*/
"_itemDone":function(translate, item) {
Zotero.debug("Translate: Saving item");
//Zotero.debug("Translate: Saving item");
// warn if itemDone called after translation completed
if(translate._complete) {
@ -1859,7 +1859,7 @@ Zotero.Translate.IO = {
}
if(nodes.getElementsByTagName("parsererror").length) {
throw new Error("DOMParser error: loading data into data store failed");
throw "DOMParser error: loading data into data store failed";
}
return nodes;
@ -1901,8 +1901,14 @@ Zotero.Translate.IO.String.prototype = {
this.RDF = new Zotero.Translate.IO._RDFSandbox(this._dataStore);
if(this._string.length) {
try {
var xml = Zotero.Translate.IO.parseDOMXML(this._string);
} catch(e) {
this._xmlInvalid = true;
throw e;
}
var parser = new Zotero.RDF.AJAW.RDFParser(this._dataStore);
parser.parse(Zotero.Translate.IO.parseDOMXML(this._string), this._uri);
parser.parse(xml, this._uri);
callback(true);
}
},
@ -1931,19 +1937,23 @@ Zotero.Translate.IO.String.prototype = {
var oldPointer = this._stringPointer;
var lfIndex = this._string.indexOf("\n", this._stringPointer);
if(lfIndex != -1) {
if(lfIndex !== -1) {
// in case we have a CRLF
this._stringPointer = lfIndex+1;
if(this._string.length > lfIndex && this._string[lfIndex-1] == "\r") {
if(this._string.length > lfIndex && this._string[lfIndex-1] === "\r") {
lfIndex--;
}
return this._string.substr(oldPointer, lfIndex-oldPointer);
}
var crIndex = this._string.indexOf("\r", this._stringPointer);
if(crIndex != -1) {
this._stringPointer = crIndex+1;
return this._string.substr(oldPointer, crIndex-oldPointer-1);
if(!this._noCR) {
var crIndex = this._string.indexOf("\r", this._stringPointer);
if(crIndex === -1) {
this._noCR = true;
} else {
this._stringPointer = crIndex+1;
return this._string.substr(oldPointer, crIndex-oldPointer-1);
}
}
this._stringPointer = this._string.length;
@ -1957,7 +1967,12 @@ Zotero.Translate.IO.String.prototype = {
"_getXML":function() {
if(this._mode == "xml/dom") {
return Zotero.Translate.IO.parseDOMXML(this._string);
try {
return Zotero.Translate.IO.parseDOMXML(this._string);
} catch(e) {
this._xmlInvalid = true;
throw e;
}
} else {
return this._string.replace(/<\?xml[^>]+\?>/, "");
}
@ -1965,9 +1980,13 @@ Zotero.Translate.IO.String.prototype = {
"init":function(newMode, callback) {
this._stringPointer = 0;
this._noCR = undefined;
this._mode = newMode;
if(Zotero.Translate.IO.rdfDataModes.indexOf(this._mode) !== -1) {
if(newMode && (Zotero.Translate.IO.rdfDataModes.indexOf(newMode) !== -1
|| newMode.substr(0, 3) === "xml") && this._xmlInvalid) {
throw "XML known invalid";
} else if(Zotero.Translate.IO.rdfDataModes.indexOf(this._mode) !== -1) {
this._initRDF(callback);
} else {
callback(true);

46
chrome/content/zotero/xpcom/translation/translate_firefox.js
View file

@ -148,22 +148,44 @@ Zotero.Translate.SandboxManager.prototype = {
if(newExposedProps) newExposedProps[localKey] = "r";
// magical XPCSafeJSObjectWrappers for sandbox
if(typeof object[localKey] === "function" || typeof object[localKey] === "object") {
if(attachTo == this.sandbox) Zotero.debug(localKey);
attachTo[localKey] = function() {
var args = (passAsFirstArgument ? [passAsFirstArgument] : []);
for(var i=0; i<arguments.length; i++) {
args.push((typeof arguments[i] === "object" && arguments[i] !== null)
|| typeof arguments[i] === "function"
? new XPCSafeJSObjectWrapper(arguments[i]) : arguments[i]);
var type = typeof object[localKey];
var isFunction = type === "function";
var isObject = typeof object[localKey] === "object";
if(isFunction || isObject) {
if(isFunction) {
if(Zotero.isFx4) {
if(passAsFirstArgument) {
attachTo[localKey] = object[localKey].bind(object, passAsFirstArgument);
} else {
attachTo[localKey] = object[localKey].bind(object);
}
} else {
attachTo[localKey] = function() {
if(passAsFirstArgument) {
var args = new Array(arguments.length+1);
args[0] = passAsFirstArgument;
var offset = 1;
} else {
var args = new Array(arguments.length);
var offset = 0;
}
for(var i=0, nArgs=arguments.length; i<nArgs; i++) {
args[i+offset] = (((typeof arguments[i] === "object" && arguments[i] !== null)
|| typeof arguments[i] === "function")
? new XPCSafeJSObjectWrapper(arguments[i]) : arguments[i]);
}
return object[localKey].apply(object, args);
};
}
return object[localKey].apply(object, args);
};
} else {
attachTo[localKey] = {};
}
// attach members
if(!(object instanceof Components.interfaces.nsISupports)) {
this.importObject(object[localKey], passAsFirstArgument ? passAsFirstArgument : null, attachTo[localKey]);
this.importObject(object[localKey], passAsFirstArgument, attachTo[localKey]);
}
} else {
attachTo[localKey] = object[localKey];

52
chrome/content/zotero/xpcom/utilities.js
View file

@ -218,12 +218,17 @@ Zotero.Utilities = {
* @type String
*/
"unescapeHTML":function(/**String*/ str) {
// If no tags, no need to unescape
if(str.indexOf("<") === -1 && str.indexOf("&") === -1) return str;
if(Zotero.isFx) {
var nsISUHTML = Components.classes["@mozilla.org/feed-unescapehtml;1"]
.getService(Components.interfaces.nsIScriptableUnescapeHTML);
return nsISUHTML.unescape(str);
if(!Zotero.Utilities._nsISUHTML) {
Zotero.Utilities._nsISUHTML = Components.classes["@mozilla.org/feed-unescapehtml;1"]
.getService(Components.interfaces.nsIScriptableUnescapeHTML);
}
return Zotero.Utilities._nsISUHTML.unescape(str);
} else if(Zotero.isNode) {
var doc = require('jsdom').jsdom(str, null, {
/*var doc = require('jsdom').jsdom(str, null, {
"features":{
"FetchExternalResources":false,
"ProcessExternalResources":false,
@ -232,7 +237,8 @@ Zotero.Utilities = {
}
});
if(!doc.documentElement) return str;
return doc.documentElement.textContent;
return doc.documentElement.textContent;*/
return Zotero.Utilities.cleanTags(str);
} else {
var node = document.createElement("div");
node.innerHTML = str;
@ -847,7 +853,6 @@ Zotero.Utilities = {
* Converts an item from toArray() format to content=json format used by the server
*/
"itemToServerJSON":function(item) {
const IGNORE_FIELDS = ["seeAlso", "attachments", "complete"];
var newItem = {};
var typeID = Zotero.ItemTypes.getID(item.itemType);
@ -857,9 +862,10 @@ Zotero.Utilities = {
typeID = Zotero.ItemTypes.getID(item.itemType);
}
var fieldID;
var fieldID, itemFieldID;
for(var field in item) {
if(IGNORE_FIELDS.indexOf(field) !== -1) continue;
if(field === "complete" || field === "itemID" || field === "attachments"
|| field === "seeAlso") continue;
var val = item[field];
@ -867,8 +873,9 @@ Zotero.Utilities = {
newItem[field] = val;
} else if(field === "creators") {
// normalize creators
var newCreators = newItem.creators = [];
for(var j in val) {
var n = val.length;
var newCreators = newItem.creators = new Array(n);
for(var j=0; j<n; j++) {
var creator = val[j];
// Single-field mode
@ -886,7 +893,6 @@ Zotero.Utilities = {
}
// ensure creatorType is present and valid
newCreator.creatorType = "author";
if(creator.creatorType) {
if(Zotero.CreatorTypes.getID(creator.creatorType)) {
newCreator.creatorType = creator.creatorType;
@ -894,13 +900,15 @@ Zotero.Utilities = {
Zotero.debug("Translate: Invalid creator type "+creator.creatorType+"; falling back to author");
}
}
if(!newCreator.creatorType) newCreator.creatorType = "author";
newCreators.push(newCreator);
newCreators[j] = newCreator;
}
} else if(field === "tags") {
// normalize tags
var newTags = newItem.tags = [];
for(var j in val) {
var n = val.length;
var newTags = newItem.tags = new Array(n);
for(var j=0; j<n; j++) {
var tag = val[j];
if(typeof tag === "object") {
if(tag.tag) {
@ -912,12 +920,13 @@ Zotero.Utilities = {
continue;
}
}
newTags.push({"tag":tag.toString(), "type":1})
newTags[j] = {"tag":tag.toString(), "type":1};
}
} else if(field === "notes") {
// normalize notes
var newNotes = newItem.notes = [];
for(var j in val) {
var n = val.length;
var newNotes = newItem.notes = new Array(n);
for(var j=0; j<n; j++) {
var note = val[j];
if(typeof note === "object") {
if(!note.note) {
@ -926,9 +935,9 @@ Zotero.Utilities = {
}
note = note.note;
}
newNotes.push({"itemType":"note", "note":note.toString()});
newNotes[j] = {"itemType":"note", "note":note.toString()};
}
} else if(fieldID = Zotero.ItemFields.getID(field)) {
} else if((fieldID = Zotero.ItemFields.getID(field))) {
// if content is not a string, either stringify it or delete it
if(typeof val !== "string") {
if(val || val === 0) {
@ -939,8 +948,7 @@ Zotero.Utilities = {
}
// map from base field if possible
var itemFieldID = Zotero.ItemFields.getFieldIDFromTypeAndBase(typeID, fieldID);
if(itemFieldID) {
if((itemFieldID = Zotero.ItemFields.getFieldIDFromTypeAndBase(typeID, fieldID))) {
newItem[Zotero.ItemFields.getName(itemFieldID)] = val;
continue; // already know this is valid
}
@ -951,7 +959,7 @@ Zotero.Utilities = {
} else {
Zotero.debug("Translate: Discarded field "+field+": field not valid for type "+item.itemType, 3);
}
} else if(field !== "complete") {
} else {
Zotero.debug("Translate: Discarded unknown field "+field, 3);
}
}