Merge pull request #1417 from mrtcode/pdftools-test

Fix PDF tools usage in tests

.gitignore

@@ -1,4 +1,5 @@
 .DS_Store
 node_modules
 build
-.signatures.json
+.signatures.json
+tmp

chrome/content/zotero/xpcom/fulltext.js

@@ -26,8 +26,6 @@
 Zotero.Fulltext = Zotero.FullText = new function(){
 	this.isCachedMIMEType = isCachedMIMEType;
 	
-	this.__defineGetter__("pdfConverterName", function() { return 'pdftotext'; });
-	this.__defineGetter__("pdfInfoName", function() { return 'pdfinfo'; });
 	this.__defineGetter__("pdfConverterCacheFile", function () { return '.zotero-ft-cache'; });
 	this.__defineGetter__("pdfInfoCacheFile", function () { return '.zotero-ft-info'; });
 	
@@ -54,11 +52,9 @@ Zotero.Fulltext = Zotero.FullText = new function(){
 	const kWbClassHWKatakanaLetter = 6;
 	const kWbClassThaiLetter =       7;
 	
-	var _pdfConverterFileName = null;
 	var _pdfConverter = null; // nsIFile to executable
-	var _pdfInfoFileName = null;
 	var _pdfInfo = null; // nsIFile to executable
-	var _popplerDatadir = null;
+	var _pdfData = null;
 	
 	var _idleObserverIsRegistered = false;
 	var _idleObserverDelay = 30;
@@ -74,19 +70,19 @@ Zotero.Fulltext = Zotero.FullText = new function(){
 		this.decoder = Components.classes["@mozilla.org/intl/utf8converterservice;1"].
 			getService(Components.interfaces.nsIUTF8ConverterService);
 		
-		_pdfConverterFileName = this.pdfConverterName;
+		let pdfConverterFileName = "pdftotext";
-		_pdfInfoFileName = this.pdfInfoName;
+		let pdfInfoFileName = "pdfinfo";
 		
 		if (Zotero.isWin) {
-			_pdfConverterFileName += '.exe';
+			pdfConverterFileName += '.exe';
-			_pdfInfoFileName += '.exe';
+			pdfInfoFileName += '.exe';
 		}
 		
 		let dir = FileUtils.getFile('AChrom', []).parent;
 		
-		_popplerDatadir = dir.clone();
+		_pdfData = dir.clone();
-		_popplerDatadir.append('poppler-data');
+		_pdfData.append('poppler-data');
-		_popplerDatadir = _popplerDatadir.path;
+		_pdfData = _pdfData.path;
 		
 		_pdfConverter = dir.clone();
 		_pdfInfo = dir.clone();
@@ -99,8 +95,8 @@ Zotero.Fulltext = Zotero.FullText = new function(){
 			_pdfInfo.append('MacOS');
 		}
 
-		_pdfConverter.append(_pdfConverterFileName);
+		_pdfConverter.append(pdfConverterFileName);
-		_pdfInfo.append(_pdfInfoFileName);
+		_pdfInfo.append(pdfInfoFileName);
 		
 		Zotero.uiReadyPromise.delay(30000).then(() => {
 			this.registerContentProcessor();
@@ -135,6 +131,22 @@ Zotero.Fulltext = Zotero.FullText = new function(){
 	});
 	
 	
+	this.setPDFConverterPath = function(path) {
+		_pdfConverter = Zotero.File.pathToFile(path);
+	};
+	
+	
+	this.setPDFInfoPath = function(path) {
+		_pdfInfo = Zotero.File.pathToFile(path);
+		
+	};
+	
+	
+	this.setPDFDataPath = function(path) {
+		_pdfData = path;
+	};
+	
+	
 	this.getLibraryVersion = function (libraryID) {
 		if (!libraryID) throw new Error("libraryID not provided");
 		return Zotero.DB.valueQueryAsync(
@@ -201,7 +213,7 @@ Zotero.Fulltext = Zotero.FullText = new function(){
 	this.getPDFConverterExecAndArgs = function () {
 		return {
 			exec: _pdfConverter,
-			args: ['-datadir', _popplerDatadir]
+			args: ['-datadir', _pdfData]
 		}
 	};
 	

test/content/runtests.js

@@ -293,6 +293,9 @@ if(run) {
 	window.onload = function() {
 		Zotero.spawn(function* () {
 			yield Zotero.Schema.schemaUpdatePromise;
+			
+			initPDFToolsPath();
+			
 			return mocha.run();
 		})
 	};

test/content/support.js

@@ -473,6 +473,37 @@ function getPromiseError(promise) {
 	return promise.thenReturn(false).catch(e => e);
 }
 
+/**
+ * Init paths for PDF tools and data
+ */
+function initPDFToolsPath() {
+	let pdfConvertedFileName = 'pdftotext';
+	let pdfInfoFileName = 'pdfinfo';
+	
+	if (Zotero.isWin) {
+		pdfConvertedFileName += '-win.exe';
+		pdfInfoFileName += '-win.exe';
+	}
+	else if (Zotero.isMac) {
+		pdfConvertedFileName += '-mac';
+		pdfInfoFileName += '-mac';
+	}
+	else {
+		let cpu = Zotero.platform.split(' ')[1];
+		pdfConvertedFileName += '-linux-' + cpu;
+		pdfInfoFileName += '-linux-' + cpu;
+	}
+	
+	let pdfToolsPath = OS.Path.join(Zotero.Profile.dir, 'pdftools');
+	let pdfConverterPath = OS.Path.join(pdfToolsPath, pdfConvertedFileName);
+	let pdfInfoPath = OS.Path.join(pdfToolsPath, pdfInfoFileName);
+	let pdfDataPath = OS.Path.join(pdfToolsPath, 'poppler-data');
+	
+	Zotero.FullText.setPDFConverterPath(pdfConverterPath);
+	Zotero.FullText.setPDFInfoPath(pdfInfoPath);
+	Zotero.FullText.setPDFDataPath(pdfDataPath);
+}
+
 /**
  * Returns the nsIFile corresponding to the test data directory
  * (i.e., test/tests/data)

test/runtests.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
-CWD="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+ROOT_DIR="$( cd "$( dirname "$SCRIPT_DIR" )" && pwd )"
 
 case "$OSTYPE" in
   msys*|mingw*|cygwin*) IS_CYGWIN=1 ;;
@@ -116,7 +117,7 @@ TEMPDIR="`mktemp -d 2>/dev/null || mktemp -d -t 'zotero-unit'`"
 PROFILE="$TEMPDIR/profile"
 mkdir -p "$PROFILE/extensions"
 
-makePath ZOTERO_PATH "`dirname "$CWD"`/build"
+makePath ZOTERO_PATH "$ROOT_DIR/build"
 echo "$ZOTERO_PATH" > "$PROFILE/extensions/zotero@chnm.gmu.edu"
 
 makePath ZOTERO_UNIT_PATH "$ZOTERO_PATH/test"
@@ -125,6 +126,24 @@ echo "$ZOTERO_UNIT_PATH" > "$PROFILE/extensions/zotero-unit@zotero.org"
 # Create data directory
 mkdir "$TEMPDIR/Zotero"
 
+# Download PDF tools if not cached in the source directory and copy to profile directory
+PDF_TOOLS_VERSION="0.0.1"
+PDF_TOOLS_URL="https://zotero-download.s3.amazonaws.com/pdftools/pdftools-$PDF_TOOLS_VERSION.tar.gz"
+PDF_TOOLS_CACHE_DIR="$ROOT_DIR/tmp/pdftools"
+PDF_TOOLS_DIR="$PROFILE/pdftools"
+if [ ! -f "$PDF_TOOLS_CACHE_DIR/$PDF_TOOLS_VERSION" ]; then
+	echo "Fetching PDF tools version $PDF_TOOLS_VERSION"
+	echo
+	rm -rf "$PDF_TOOLS_CACHE_DIR"
+	mkdir -p "$PDF_TOOLS_CACHE_DIR"
+	curl -o "$PDF_TOOLS_CACHE_DIR/pdftools.tar.gz" $PDF_TOOLS_URL
+	tar -zxf "$PDF_TOOLS_CACHE_DIR/pdftools.tar.gz" -C $PDF_TOOLS_CACHE_DIR
+	rm "$PDF_TOOLS_CACHE_DIR/pdftools.tar.gz"
+	touch "$PDF_TOOLS_CACHE_DIR/$PDF_TOOLS_VERSION"
+	echo
+fi
+cp -R $PDF_TOOLS_CACHE_DIR $PDF_TOOLS_DIR
+
 cat <<EOF > "$PROFILE/prefs.js"
 user_pref("app.update.enabled", false);
 user_pref("extensions.autoDisableScopes", 0);
@@ -170,7 +189,7 @@ trap "{ rm -rf \"$TEMPDIR\"; }" EXIT
 if [[ "$TRAVIS" != true ]] && ! ps | grep scripts/build.js | grep -v grep > /dev/null; then
 	echo
 	echo "Running JS build process"
-	cd "$CWD/.."
+	cd "$ROOT_DIR"
 	npm run build || exit $?
 	echo
 fi

test/tests/fulltextTest.js

@@ -5,6 +5,7 @@ describe("Zotero.Fulltext", function () {
 		// Hidden browser, which requires a browser window, needed for charset detection
 		// (until we figure out a better way)
 		win = yield loadBrowserWindow();
+		initPDFToolsPath();
 	});
 	after(function () {
 		if (win) {
@@ -133,7 +134,7 @@ describe("Zotero.Fulltext", function () {
 			toSync.push({
 				item: pdfAttachment,
 				content: "Zotero [zoh-TAIR-oh] is a free, easy-to-use tool to help you collect, "
-					+ "organize, cite, and share your research sources.\n\n",
+					+ "organize, cite, and share\nyour research sources.\n\n",
 				indexedChars: 0,
 				indexedPages: 1
 			});
@@ -175,6 +176,16 @@ describe("Zotero.Fulltext", function () {
 	})
 	
 	describe("#setItemContent()", function () {
+		before(() => {
+			// Disable PDF indexing
+			Zotero.Prefs.set('fulltext.pdfMaxPages', 0);
+		});
+		
+		after(() => {
+			// Re-enable PDF indexing
+			Zotero.Prefs.clear('fulltext.pdfMaxPages');
+		});
+		
 		it("should store data in .zotero-ft-unprocessed file", function* () {
 			var item = yield importFileAttachment('test.pdf');
 			

test/tests/recognizePDFTest.js

@@ -9,6 +9,7 @@ describe("PDF Recognition", function() {
 		yield Zotero.Promise.all([
 			loadZoteroPane().then(w => win = w)
 		]);
+		initPDFToolsPath();
 	});
 	
 	beforeEach(function* () {