zotero/scripts/js.js

const globby = require('globby');
const path = require('path');
const os = require('os');
const fs = require('fs-extra');
const cluster = require('cluster');
const { getSignatures, compareSignatures, getFileSignature, writeSignatures, cleanUp, onSuccess, onError, onProgress } = require('./utils');
const { jsFiles, ignoreMask } = require('./config');

const NODE_ENV = process.env.NODE_ENV;
const ROOT = path.resolve(__dirname, '..');

async function getJS(source, options, signatures) {
	const t1 = Date.now();
	const matchingJSFiles = await globby(source, Object.assign({ cwd: ROOT }, options));
	const cpuCount = os.cpus().length;
	const totalCount = matchingJSFiles.length;
	var isError = false;

	cluster.setupMaster({
		exec: path.join(__dirname, 'babel-worker.js')
	});

	// check signatures, collect signatures for files to be processes 
	const newFilesSignatures = {};
	const filesForProcessing = [];
	var f;
	while ((f = matchingJSFiles.pop()) != null) {
		const newFileSignature = await getFileSignature(f);
		const dest = path.join('build', f.replace('.jsx', '.js'));
		f = path.normalize(f);
		if (f in signatures) {
			if (compareSignatures(newFileSignature, signatures[f])) {
				try {
					await fs.access(dest, fs.constants.F_OK);
					continue;
				} catch (_) {
					// file does not exists in build, fallback to browserifing
				}
			}
		}
		filesForProcessing.push(f);
		newFilesSignatures[f] = newFileSignature;
	}

	// shortcut if no files need rebuilding
	if (Object.keys(filesForProcessing).length === 0) {
		const t2 = Date.now();
		return Promise.resolve({
				action: 'js',
				count: 0,
				outFiles: [],
				totalCount,
				processingTime: t2 - t1
		});
	}

	// distribute processing among workers
	const workerCount = Math.min(cpuCount, filesForProcessing.length);
	const outFiles = [];
	var workersActive = workerCount;
	NODE_ENV == 'debug' && console.log(`Will process ${filesForProcessing.length} files using ${workerCount} processes`);
	return new Promise((resolve, reject) => {
		for (let i = 0; i < workerCount; i++) {
			var worker = cluster.fork();

			worker.on('message', function(ev) {
				if (ev.error) {
					isError = true;
					let errorMsg = `Failed while processing ${ev.sourcefile}: ${ev.error}`;
					reject(errorMsg);
				} else {
					signatures[ev.sourcefile] = newFilesSignatures[ev.sourcefile];
					
					if (ev.isSkipped) {
						NODE_ENV == 'debug' && console.log(`process ${this.id} SKIPPED ${ev.sourcefile}`);
					} else {
						NODE_ENV == 'debug' && console.log(`process ${this.id} took ${ev.processingTime} ms to process ${ev.sourcefile} into ${ev.outfile}`);
						NODE_ENV != 'debug' && onProgress(ev.sourcefile, ev.outfile, 'js');
						outFiles.push(ev.outfile);
					}
				}

				let nextFile = filesForProcessing.pop();

				if (!isError && nextFile) {
					NODE_ENV == 'debug' && console.log(`process ${this.id} scheduled to process ${nextFile}`);
					this.send({
						file: nextFile
					});
				} else {
					if (this.isConnected()) {
						this.kill();
					}
					NODE_ENV == 'debug' && console.log(`process ${this.id} has terminated`);
					if (!--workersActive) {
						const t2 = Date.now();
						resolve({
							action: 'js',
							count: outFiles.length,
							outFiles,
							totalCount,
							processingTime: t2 - t1
						});
					}
				}
			});

			let nextFile = filesForProcessing.pop();
			NODE_ENV == 'debug' && console.log(`process ${worker.id} scheduled to process ${nextFile}`);
			worker.send({
				file: nextFile
			});
		}
	});
}

module.exports = getJS;

if (require.main === module) {
	(async () => {
		try {
			const signatures = await getSignatures();
			onSuccess(await getJS(jsFiles, { ignore: ignoreMask }, signatures));
			onSuccess(await cleanUp(signatures));
			await writeSignatures(signatures);
		} catch (err) {
			process.exitCode = 1;
			global.isError = true;
			onError(err);
		}
	})();
}