Introduce a build system

* Add a multi-process, gulp-based build system to support es6 features,
  async/await, jsx and scss
* Add a package.json to support dependency management and allow starting
  the build process via npm
* Replace embedded Bluebird library with npm-installed one
* Add react, react-dom and web-library
* Introduce a custom require() loader in include.js as well as a minimal
  local require() implementation in various other places

.babelrc

@@ -0,0 +1,39 @@
+{
+  "compact": false,
+  "presets": [],
+  "ignore": [
+    "chrome/content/zotero/include.js",
+    "resource/tinymce/tinymce.js",
+    "chrome/content/zotero/xpcom/citeproc.js",
+    "resource/csl-validator.js",
+    "resource/react.js",
+    "resource/react-dom.js"
+  ],
+  "plugins": [
+    "syntax-flow",
+    "syntax-jsx",
+    "syntax-async-generators",
+    "syntax-class-properties",
+    "syntax-decorators",
+    "syntax-do-expressions",
+    "syntax-export-extensions",
+    "syntax-flow",
+    "syntax-jsx",
+    "syntax-object-rest-spread",
+    "transform-react-jsx",
+    "transform-react-display-name",
+    [
+      "transform-async-to-module-method",
+      {
+        "module": "bluebird/bluebird.js",
+        "method": "coroutine"
+      }
+    ],
+    [
+      "transform-es2015-modules-commonjs",
+      {
+        "strictMode": false
+      }
+    ]
+  ]
+}

.gitignore

@@ -0,0 +1,3 @@
+.DS_Store
+node_modules
+build

chrome/content/zotero/include.js

@@ -1,5 +1,30 @@
-var Zotero = Components.classes["@zotero.org/Zotero;1"]
+/* global Components:false */
+/* eslint-disable no-unused-vars */
+
+var Zotero = Components.classes['@zotero.org/Zotero;1']
 				// Currently uses only nsISupports
 				//.getService(Components.interfaces.chnmIZoteroService).
 				.getService(Components.interfaces.nsISupports)
 				.wrappedJSObject;
+
+
+var require = (function() {
+	var { Loader, Require, Module } = Components.utils.import('resource://gre/modules/commonjs/toolkit/loader.js');
+	var requirer = Module('/', '/');
+
+	var loader = Loader({
+		id: 'zotero/require',
+		paths: {
+			'': 'resource://zotero/',
+		},
+		globals: {
+			document,
+			console,
+			navigator,
+			window,
+			Zotero
+		}
+	});
+
+	return Require(loader, requirer);
+})();

chrome/content/zotero/xpcom/zotero.js

@@ -32,6 +32,22 @@ Components.utils.import("resource://gre/modules/PluralForm.jsm");
 
 Services.scriptloader.loadSubScript("resource://zotero/polyfill.js");
 
+var require = (target) => {
+	var { Loader, Require, Module } = Components.utils.import('resource://gre/modules/commonjs/toolkit/loader.js');
+	var requirer = Module('/', '/');
+	var globals = {};
+
+	Components.utils.import("resource://gre/modules/Timer.jsm", globals);
+
+	var loader = Loader({
+		id: 'zotero/requireminimal',
+		globals
+	});
+
+	return (Require(loader, requirer))(target);
+};
+
+
 /*
  * Core functions
  */
@@ -63,7 +79,7 @@ Services.scriptloader.loadSubScript("resource://zotero/polyfill.js");
 	this.isWin;
 	this.initialURL; // used by Schema to show the changelog on upgrades
 	
-	Components.utils.import("resource://zotero/bluebird.js", this);
+	this.Promise = require('resource://zotero/bluebird/bluebird.js');
 	
 	this.getActiveZoteroPane = function() {
 		var win = Services.wm.getMostRecentWindow("navigator:browser");

gulp/babel-worker.js

@@ -0,0 +1,48 @@
+/* global onmessage: true, postMessage: false */
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const babel = require('babel-core');
+const mkdirp = require('mkdirp');
+const options = JSON.parse(fs.readFileSync('.babelrc'));
+
+/* exported onmessage */
+onmessage = (ev) => {
+	const t1 = Date.now();
+	const sourcefile = path.normalize(ev.data);
+	let isError = false;
+	let isSkipped = false;
+	
+	fs.readFile(sourcefile, 'utf8', (err, data) => {
+		var transformed;
+		if(sourcefile === 'resource/react-dom.js') {
+			transformed = data.replace(/ownerDocument\.createElement\((.*?)\)/gi, 'ownerDocument.createElementNS(DOMNamespaces.html, $1)');
+		} else if('ignore' in options && options.ignore.includes(sourcefile)) {
+			transformed = data;
+			isSkipped = true;
+		} else {
+			transformed = babel.transform(data, options).code;
+		}
+
+		const outfile = path.join('build', sourcefile);
+		isError = !!err;
+
+		mkdirp(path.dirname(outfile), err => {
+			isError = !!err;
+
+			fs.writeFile(outfile, transformed, err => {
+				isError = !!err;
+				const t2 = Date.now();
+
+				postMessage({
+					isError,
+					isSkipped,
+					sourcefile,
+					outfile,
+					processingTime: t2 - t1
+				});	
+			});
+		});
+	});	
+};

gulp/gulp-react-patcher.js

@@ -0,0 +1,34 @@
+const path = require('path');
+const gutil = require('gulp-util');
+const through = require('through2');
+const PluginError = gutil.PluginError;
+
+const PLUGIN_NAME = 'gulp-react-patcher';
+
+module.exports = function() {
+	return through.obj(function(file, enc, callback) {
+		if (file.isNull()) {
+			this.push(file);
+			return callback();
+		}
+
+		if(file.isStream()) {
+			this.emit('error', new PluginError(PLUGIN_NAME, 'Streams are not supported!'));
+			return callback();
+		}
+
+		try {
+
+			let filename = path.basename(file.path);
+
+			if(filename === 'react-dom.js') {
+				file.contents = Buffer.from(file.contents.toString().replace(/ownerDocument\.createElement\((.*?)\)/gi, 'ownerDocument.createElementNS(DOMNamespaces.html, $1)'), enc);
+			}
+		} catch(e) {
+			this.emit('error', new PluginError(PLUGIN_NAME, e));
+		}
+
+		this.push(file);
+		callback();
+	});
+};

gulpfile.js

@@ -0,0 +1,145 @@
+'use strict';
+
+const path = require('path');
+const gulp = require('gulp');
+const del = require('del');
+const vfs = require('vinyl-fs');
+const gutil = require('gulp-util');
+const babel = require('gulp-babel');
+const sass = require('gulp-sass');
+const os = require('os');
+const glob = require('glob');
+const Worker = require('tiny-worker');
+const NODE_ENV = process.env.NODE_ENV;
+const reactPatcher = require('./gulp/gulp-react-patcher');
+
+// list of folders from where .js files are compiled and non-js files are symlinked
+const dirs = [
+	'chrome', 'components', 'defaults', 'resource', 'resource/web-library'
+];
+
+// list of folders from where all files are symlinked
+const symlinkDirs = [
+	'styles', 'translators'
+];
+
+// list of files from root folder to symlink
+const symlinkFiles = [
+	'chrome.manifest', 'install.rdf', 'update.rdf'
+];
+
+const jsGlob = `./\{${dirs.join(',')}\}/**/*.js`;
+
+function onError(err) {
+	gutil.log(gutil.colors.red('Error:'), err);
+	this.emit('end');
+}
+
+function onSuccess(msg) {
+	gutil.log(gutil.colors.green('Build:'), msg);
+}
+
+function getJS(source = jsGlob) { 
+	return gulp.src(source, { base: '.' })
+		.pipe(babel())
+		.pipe(reactPatcher())
+		.on('error', onError)
+		.on('data', file => {
+			onSuccess(`[js] ${path.basename(file.path)}`);
+		})
+		.pipe(gulp.dest('./build'));
+}
+
+function getJSParallel(source = jsGlob) {
+	const jsFiles = glob.sync(source);
+	const cpuCount = os.cpus().length;
+	const threadCount = Math.min(cpuCount, jsFiles.length);
+	let threadsActive = threadCount;
+
+	return new Promise((resolve, reject) => {
+		for(let i = 0; i < threadCount; i++) {
+			let worker = new Worker('gulp/babel-worker.js');
+			worker.onmessage = ev => {
+				if(ev.data.isError) {
+					reject(`Failed while processing ${ev.data.sourcefile}`);
+				}
+
+				NODE_ENV == 'debug' && console.log(`process ${i} took ${ev.data.processingTime} ms to process ${ev.data.sourcefile}`);
+				NODE_ENV != 'debug' && onSuccess(`[js] ${path.basename(ev.data.sourcefile)}`);
+				
+				if(ev.data.isSkipped) {
+					NODE_ENV == 'debug' && console.log(`process ${i} SKIPPED ${ev.data.sourcefile}`);
+				}
+				let nextFile = jsFiles.pop();
+
+				if(nextFile) {
+					worker.postMessage(nextFile);
+				} else {
+					NODE_ENV == 'debug' && console.log(`process ${i} has terminated`);
+					worker.terminate();
+					if(!--threadsActive) {
+						resolve();
+					}
+				}
+			};
+			worker.postMessage(jsFiles.pop());
+		}
+		
+		NODE_ENV == 'debug' && console.log(`Started ${threadCount} processes for processing JS`);
+	});
+}
+
+function getSymlinks() {
+	const match = symlinkFiles
+		.concat(dirs.map(d => `${d}/**`))
+		.concat(symlinkDirs.map(d => `${d}/**`))
+		.concat([`!{${dirs.join(',')}}/**/*.js`]);
+
+	return gulp
+		.src(match, { nodir: true, base: '.', read: false })
+		.on('error', onError)
+		.on('data', file => {
+			onSuccess(`[ln] ${path.basename(file.path)}`);
+		})
+		.pipe(vfs.symlink('build/'));
+}
+
+function getSass() {
+	return gulp
+		.src('scss/*.scss')
+		.on('error', onError)
+		.pipe(sass())
+		.pipe(gulp.dest('./build/chrome/skin/default/zotero/components/'));
+}
+
+
+gulp.task('clean', () => {
+	return del('build');
+});
+
+gulp.task('symlink', ['clean'], () => {
+	return getSymlinks();
+});
+
+gulp.task('js', done => {
+	getJSParallel(jsGlob).then(() => done());
+});
+
+gulp.task('sass', () => {
+	return getSass();
+});
+
+gulp.task('build', ['js', 'sass', 'symlink']);
+
+gulp.task('dev', ['clean'], () => {
+	let watcher = gulp.watch(jsGlob);
+
+	watcher.on('change', function(event) {
+		getJS(event.path);
+	});
+
+	gulp.watch('src/styles/*.scss', ['sass']);
+	gulp.start('build');
+});
+
+gulp.task('default', ['dev']);

package.json

@@ -0,0 +1,46 @@
+{
+  "name": "zotero",
+  "private": "private",
+  "version": "5.0.0",
+  "description": "Zotero",
+  "main": "",
+  "scripts": {
+    "start": "./node_modules/.bin/gulp",
+    "build": "./node_modules/.bin/gulp",
+    "sass": "./node_modules/.bin/gulp sass",
+    "clean": "./node_modules/.bin/gulp clean"
+  },
+  "license": "",
+  "dependencies": {
+    "bluebird": "^3.4.6",
+    "zotero-web-library": "next",
+    "react": "^15.3.2",
+    "react-dom": "^15.3.2"
+  },
+  "devDependencies": {
+    "babel-core": "^6.24.1",
+    "babel-plugin-syntax-async-generators": "^6.13.0",
+    "babel-plugin-syntax-class-properties": "^6.13.0",
+    "babel-plugin-syntax-decorators": "^6.13.0",
+    "babel-plugin-syntax-do-expressions": "^6.13.0",
+    "babel-plugin-syntax-export-extensions": "^6.13.0",
+    "babel-plugin-syntax-flow": "^6.13.0",
+    "babel-plugin-syntax-jsx": "^6.13.0",
+    "babel-plugin-syntax-object-rest-spread": "^6.13.0",
+    "babel-plugin-transform-async-to-module-method": "^6.16.0",
+    "babel-plugin-transform-es2015-modules-commonjs": "^6.18.0",
+    "babel-preset-react": "^6.16.0",
+    "del": "^2.2.2",
+    "glob": "^7.1.2",
+    "gulp": "^3.9.1",
+    "gulp-babel": "^6.1.2",
+    "gulp-sass": "^3.1.0",
+    "gulp-util": "^3.0.7",
+    "through2": "^2.0.1",
+    "tiny-worker": "^2.1.1",
+    "vinyl-buffer": "^1.0.0",
+    "vinyl-fs": "^2.4.4",
+    "vinyl-source-stream": "^1.1.0",
+    "watchify": "^3.7.0"
+  }
+}

resource/bluebird

@@ -0,0 +1 @@
+../node_modules/bluebird/js/release

resource/concurrentCaller.js

@@ -24,7 +24,23 @@
 */
 
 EXPORTED_SYMBOLS = ["ConcurrentCaller"];
-Components.utils.import("resource://zotero/bluebird.js");
+
+var require = (target) => {
+	var { Loader, Require, Module } = Components.utils.import('resource://gre/modules/commonjs/toolkit/loader.js');
+	var requirer = Module('/', '/');
+	var globals = {};
+
+	Components.utils.import("resource://gre/modules/Timer.jsm", globals);
+
+	var loader = Loader({
+		id: 'zotero/requireminimal',
+		globals
+	});
+
+	return (Require(loader, requirer))(target);
+};
+
+var Promise = require('resource://zotero/bluebird/bluebird.js');
 
 /**
  * Call a fixed number of functions at once, queueing the rest until slots

resource/react-dom.js

@@ -0,0 +1 @@
+../node_modules/react-dom/dist/react-dom.js

resource/react.js

@@ -0,0 +1 @@
+../node_modules/react/dist/react.js

resource/web-library

@@ -0,0 +1 @@
+../node_modules/zotero-web-library/lib