From f98ccdee46005a0a51c0f0f89fcf824e91a2253f Mon Sep 17 00:00:00 2001 From: Tom Najdek Date: Fri, 2 Jun 2017 14:18:26 +0100 Subject: [PATCH 1/5] Ensure build process exits with non-zero result on failure * During build, error is printed out with stack and process exits * During development, initial build behaves as above, however when watching files, errors are displayed but watch process does not exit allowing fixes without a complete rebuild. --- gulp/babel-worker.js | 22 +++++++---- gulpfile.js | 87 +++++++++++++++++++++++++++++++------------- package.json | 6 +-- 3 files changed, 79 insertions(+), 36 deletions(-) diff --git a/gulp/babel-worker.js b/gulp/babel-worker.js index 8d4899cbe7..f78a49ec69 100644 --- a/gulp/babel-worker.js +++ b/gulp/babel-worker.js @@ -11,9 +11,9 @@ const options = JSON.parse(fs.readFileSync('.babelrc')); onmessage = (ev) => { const t1 = Date.now(); const sourcefile = path.normalize(ev.data); - let isError = false; + let error = null; let isSkipped = false; - + fs.readFile(sourcefile, 'utf8', (err, data) => { var transformed; if(sourcefile === 'resource/react-dom.js') { @@ -22,24 +22,30 @@ onmessage = (ev) => { transformed = data; isSkipped = true; } else { - transformed = babel.transform(data, options).code; + try { + transformed = babel.transform(data, options).code; + } catch(c) { + transformed = data; + isSkipped = true; + error = c.message; + } } const outfile = path.join('build', sourcefile); - isError = !!err; + error = error || err; mkdirp(path.dirname(outfile), err => { - isError = !!err; + error = error || err; fs.writeFile(outfile, transformed, err => { - isError = !!err; - const t2 = Date.now(); + error = error || err; + const t2 = Date.now(); postMessage({ - isError, isSkipped, sourcefile, outfile, + error, processingTime: t2 - t1 }); }); diff --git a/gulpfile.js b/gulpfile.js index d22967a64d..1c5f0aabad 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -1,6 +1,5 @@ 'use strict'; -const path = require('path'); const gulp = require('gulp'); const del = require('del'); const vfs = require('vinyl-fs'); @@ -12,15 +11,23 @@ const glob = require('glob'); const Worker = require('tiny-worker'); const merge = require('merge-stream'); const tap = require('gulp-tap'); -const rename = require("gulp-rename"); +const rename = require('gulp-rename'); const browserify = require('browserify'); const reactPatcher = require('./gulp/gulp-react-patcher'); const NODE_ENV = process.env.NODE_ENV; +const workers = []; +var isExiting = false; const formatDirsforMatcher = dirs => { return dirs.length > 1 ? `{${dirs.join(',')}}` : dirs[0]; }; +const killAllWorkers = () => { + for(let worker of workers) { + worker.terminate(); + } +}; + // list of folders from where .js files are compiled and non-js files are symlinked const dirs = [ 'chrome', @@ -72,16 +79,24 @@ const browserifyConfigs = [ const jsGlob = `./\{${dirs.join(',')}\}/**/*.js`; const jsGlobIgnore = `./\{${symlinkDirs.concat(copyDirs).join(',')}\}/**/*.js`; -function onError(err) { - gutil.log(gutil.colors.red('Error:'), err); - this.emit('end'); +function onError(shouldExit, err) { + if(shouldExit) { + isExiting = true; + killAllWorkers(); + throw new Error(err); + } else { + gutil.log(gutil.colors.red('Error:'), err); + this.emit('end'); + } } function onSuccess(msg) { - gutil.log(gutil.colors.green('Build:'), msg); + if(!isExiting) { + gutil.log(gutil.colors.green('Build:'), msg); + } } -function getBrowserify() { +function getBrowserify(exitOnError = true) { const streams = browserifyConfigs.map(config => { return gulp .src(config.src) @@ -89,39 +104,49 @@ function getBrowserify() { file.contents = browserify(file.path, config.config).bundle(); })) .pipe(rename(config.dest)) + .on('error', function(err) { onError.bind(this)(exitOnError, err); }) + .on('data', file => { + onSuccess(`[browserify] ${file.path}`); + }) .pipe(gulp.dest('build')); }); return merge.apply(merge, streams); } -function getJS(source, sourceIgnore) { +function getJS(source, sourceIgnore, exitOnError = true) { if (sourceIgnore) { source = [source, '!' + sourceIgnore]; } return gulp.src(source, { base: '.' }) .pipe(babel()) - .pipe(reactPatcher()) - .on('error', onError) + .on('error', function(err) { onError.bind(this)(exitOnError, err); }) .on('data', file => { onSuccess(`[js] ${file.path}`); }) + .pipe(reactPatcher()) .pipe(gulp.dest('./build')); } -function getJSParallel(source, sourceIgnore) { +function getJSParallel(source, sourceIgnore, exitOnError = true) { const jsFiles = glob.sync(source, { ignore: sourceIgnore }); const cpuCount = os.cpus().length; const threadCount = Math.min(cpuCount, jsFiles.length); let threadsActive = threadCount; + let isError = false; return new Promise((resolve, reject) => { for(let i = 0; i < threadCount; i++) { let worker = new Worker('gulp/babel-worker.js'); + workers[i] = worker; worker.onmessage = ev => { - if(ev.data.isError) { - reject(`Failed while processing ${ev.data.sourcefile}`); + if(ev.data.error) { + isError = true; + let errorMsg = `Failed while processing ${ev.data.sourcefile}: ${ev.data.error}`; + NODE_ENV == 'debug' && console.log(`process ${i}: ${errorMsg}`); + onError(exitOnError, errorMsg); + reject(errorMsg); } NODE_ENV == 'debug' && console.log(`process ${i} took ${ev.data.processingTime} ms to process ${ev.data.sourcefile}`); @@ -132,11 +157,13 @@ function getJSParallel(source, sourceIgnore) { } let nextFile = jsFiles.pop(); - if(nextFile) { + if(!isError && nextFile) { + NODE_ENV == 'debug' && console.log(`process ${i} scheduled to process ${nextFile}`); worker.postMessage(nextFile); } else { NODE_ENV == 'debug' && console.log(`process ${i} has terminated`); worker.terminate(); + workers.splice(i, 1); if(!--threadsActive) { resolve(); } @@ -149,7 +176,7 @@ function getJSParallel(source, sourceIgnore) { }); } -function getSymlinks() { +function getSymlinks(exitOnError = true) { const match = symlinkFiles .concat(dirs.map(d => `${d}/**`)) .concat(symlinkDirs.map(d => `${d}/**`)) @@ -158,26 +185,27 @@ function getSymlinks() { return gulp .src(match, { nodir: true, base: '.', read: false }) - .on('error', onError) + .on('error', function(err) { onError.bind(this)(exitOnError, err); }) .on('data', file => { onSuccess(`[ln] ${file.path.substr(__dirname.length + 1)}`); }) .pipe(vfs.symlink('build/')); } -function getCopy() { +function getCopy(exitOnError = true) { return gulp .src(copyDirs.map(d => `${d}/**`), { base: '.' }) .on('data', file => { onSuccess(`[cp] ${file.path.substr(__dirname.length + 1)}`); }) + .on('error', function(err) { onError.bind(this)(exitOnError, err); }) .pipe(gulp.dest('build/')); } -function getSass() { +function getSass(exitOnError = true) { return gulp .src('scss/*.scss') - .on('error', onError) + .on('error', function(err) { onError.bind(this)(exitOnError, err); }) .pipe(sass()) .pipe(gulp.dest('./build/chrome/skin/default/zotero/components/')); } @@ -192,7 +220,11 @@ gulp.task('symlink', ['clean'], () => { }); gulp.task('js', done => { - getJSParallel(jsGlob, jsGlobIgnore).then(() => done()); + getJSParallel(jsGlob, jsGlobIgnore) + .then(done) + .catch(errorMsg => { + onError(errorMsg); + }); }); gulp.task('browserify', () => { @@ -209,16 +241,21 @@ gulp.task('sass', () => { gulp.task('build', ['js', 'sass', 'symlink', 'browserify', 'copy']); +gulp.task('build-clean', ['clean'], () => { + gulp.start('build'); +}); + gulp.task('dev', ['clean'], () => { var interval = 750; - let watcher = gulp.watch(jsGlob, { interval }); - - watcher.on('change', function(event) { - getJS(event.path, jsGlobIgnore); + gulp.watch(jsGlob, { interval }).on('change', event => { + getJS(event.path, jsGlobIgnore, false); + }); + + gulp.watch('src/styles/*.scss', { interval }).on('change', () => { + getSass(false); }); - gulp.watch('src/styles/*.scss', { interval }, ['sass']); gulp.start('build'); }); diff --git a/package.json b/package.json index c9a7d82b84..0137e7d7aa 100644 --- a/package.json +++ b/package.json @@ -6,16 +6,16 @@ "main": "", "scripts": { "start": "./node_modules/.bin/gulp", - "build": "./node_modules/.bin/gulp", + "build": "./node_modules/.bin/gulp build-clean", "sass": "./node_modules/.bin/gulp sass", "clean": "./node_modules/.bin/gulp clean" }, "license": "", "dependencies": { "bluebird": "3.4.7", - "zotero-web-library": "next", "react": "^15.3.2", - "react-dom": "^15.3.2" + "react-dom": "^15.3.2", + "zotero-web-library": "next" }, "devDependencies": { "babel-core": "^6.24.1", From c335099465f5587e26cf1804652a8fa985ba8056 Mon Sep 17 00:00:00 2001 From: Tom Najdek Date: Sat, 3 Jun 2017 16:50:10 +0100 Subject: [PATCH 2/5] Add support for glob in babel-worker ignored paths * Also fixes an issue with handling paths on Windows --- gulp/babel-worker.js | 3 ++- package.json | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/gulp/babel-worker.js b/gulp/babel-worker.js index f78a49ec69..d77f01b1f9 100644 --- a/gulp/babel-worker.js +++ b/gulp/babel-worker.js @@ -4,6 +4,7 @@ const fs = require('fs'); const path = require('path'); const babel = require('babel-core'); +const minimatch = require('minimatch') const mkdirp = require('mkdirp'); const options = JSON.parse(fs.readFileSync('.babelrc')); @@ -18,7 +19,7 @@ onmessage = (ev) => { var transformed; if(sourcefile === 'resource/react-dom.js') { transformed = data.replace(/ownerDocument\.createElement\((.*?)\)/gi, 'ownerDocument.createElementNS(DOMNamespaces.html, $1)'); - } else if('ignore' in options && options.ignore.includes(sourcefile)) { + } else if('ignore' in options && options.ignore.some(ignoreGlob => minimatch(sourcefile, ignoreGlob))) { transformed = data; isSkipped = true; } else { diff --git a/package.json b/package.json index 0137e7d7aa..21eeee8b6f 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,7 @@ "gulp-tap": "^1.0.1", "gulp-util": "^3.0.7", "merge-stream": "^1.0.1", + "minimatch": "^3.0.4", "mocha": "^3.4.2", "sinon": "^2.3.1", "through2": "^2.0.1", From e207124f88932a469b6f2633cdf8c76fd3a9084b Mon Sep 17 00:00:00 2001 From: Tom Najdek Date: Sat, 3 Jun 2017 17:48:42 +0100 Subject: [PATCH 3/5] On Windows, copy files instead of symlinking --- gulpfile.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gulpfile.js b/gulpfile.js index 1c5f0aabad..dc2c3426c3 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -14,6 +14,7 @@ const tap = require('gulp-tap'); const rename = require('gulp-rename'); const browserify = require('browserify'); const reactPatcher = require('./gulp/gulp-react-patcher'); +const isWindows = /^win/.test(process.platform); const NODE_ENV = process.env.NODE_ENV; const workers = []; var isExiting = false; @@ -184,12 +185,12 @@ function getSymlinks(exitOnError = true) { .concat([`!./${formatDirsforMatcher(copyDirs)}/**`]); return gulp - .src(match, { nodir: true, base: '.', read: false }) + .src(match, { nodir: true, base: '.', read: isWindows }) .on('error', function(err) { onError.bind(this)(exitOnError, err); }) .on('data', file => { onSuccess(`[ln] ${file.path.substr(__dirname.length + 1)}`); }) - .pipe(vfs.symlink('build/')); + .pipe(isWindows ? gulp.dest('build/') : vfs.symlink('build/')); } function getCopy(exitOnError = true) { From e86bbfbda0454e27f0b783ade8d8a52b9bfcf8c3 Mon Sep 17 00:00:00 2001 From: Tom Najdek Date: Sat, 3 Jun 2017 17:54:06 +0100 Subject: [PATCH 4/5] Tweak test runner to work in MSYS environment on Windows --- test/runtests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/runtests.sh b/test/runtests.sh index 945debef65..0a8e24a81e 100755 --- a/test/runtests.sh +++ b/test/runtests.sh @@ -1,8 +1,8 @@ #!/bin/bash CWD="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -case "$(uname -s)" in - CYGWIN*) IS_CYGWIN=1 ;; +case "$OSTYPE" in + msys*|mingw*|cygwin*) IS_CYGWIN=1 ;; esac function makePath { From 4ca3cf1487cd37d38acef1677fb9f08c81f6939d Mon Sep 17 00:00:00 2001 From: Tom Najdek Date: Sat, 3 Jun 2017 18:25:24 +0100 Subject: [PATCH 5/5] Ignore all bluebird files when processing during the build --- .babelrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.babelrc b/.babelrc index cd6c919bed..6f6f27bfe2 100644 --- a/.babelrc +++ b/.babelrc @@ -11,6 +11,7 @@ "resource/react.js", "resource/react-dom.js", "resource/bluebird.js", + "resource/bluebird/*.js", "test/resource/httpd.js", "test/resource/mocha.js", "test/resource/co-mocha.js"