Improve PDFRenderer efficiency and add support for ink annotations

chrome/content/zotero/xpcom/pdfWorker/manager.js

@@ -335,7 +335,6 @@ Zotero.PDFWorker = new PDFWorker();
 
 
 // PDF Renderer
-// TODO: Add ink annotations rendering
 class PDFRenderer {
 	constructor() {
 		this._browser = null;
@@ -468,10 +467,11 @@ class PDFRenderer {
 			let attachment = await Zotero.Items.getAsync(itemID);
 			let annotations = [];
 			for (let annotation of attachment.getAnnotations()) {
-				if (annotation.annotationType === 'image'
+				if (['image', 'ink'].includes(annotation.annotationType)
 					&& !await Zotero.Annotations.hasCacheImage(annotation)) {
 					annotations.push({
 						id: annotation.id,
+						color: annotation.annotationColor,
 						position: JSON.parse(annotation.annotationPosition)
 					});
 				}
@@ -505,6 +505,7 @@ class PDFRenderer {
 			buf = new Uint8Array(buf).buffer;
 			let annotations = [{
 				id: annotation.id,
+				color: annotation.annotationColor,
 				position: JSON.parse(annotation.annotationPosition)
 			}];
 			return !!await this._query('renderAnnotations', { buf, annotations }, [buf]);

pdf-reader

@@ -1 +1 @@
-Subproject commit d659a92acdf1450eccb2c9f58cc7808197b76e9f
+Subproject commit a1804bbbd3c03f3984d7c5d0a5f214d7de48643c

resource/pdf-renderer/renderer.js

@@ -23,7 +23,9 @@
     ***** END LICENSE BLOCK *****
 */
 
-const SCALE_FACTOR = 4;
+const SCALE = 4;
+const PATH_BOX_PADDING = 10; // pt
+const MAX_CANVAS_PIXELS = 16777216; // 16 megapixels
 
 window.pdfjsLib.GlobalWorkerOptions.workerSrc = 'resource://zotero/pdf-reader/pdf.worker.js';
 
@@ -49,55 +51,167 @@ async function query(action, data, transfer) {
 async function renderAnnotations(buf, annotations) {
 	let num = 0;
 	let pdfDocument = await window.pdfjsLib.getDocument({ data: buf }).promise;
-	let pages = new Map();
 	for (let annotation of annotations) {
-		let pageIndex = annotation.position.pageIndex;
+		let canvas = await renderImage(pdfDocument, annotation);
-		let page = pages.get(pageIndex) || [];
+		let blob = await new Promise(resolve => canvas.toBlob(resolve, 'image/png'));
-		page.push(annotation);
+		let image = await new Response(blob).arrayBuffer();
-		pages.set(pageIndex, page);
+		await query('renderedAnnotation', { annotation: { id: annotation.id, image } }, [image]);
-	}
+		num++;
-	for (let [pageIndex, annotations] of pages) {
-		let { canvas, viewport } = await renderPage(pdfDocument, pageIndex);
-		for (let annotation of annotations) {
-			let position = p2v(annotation.position, viewport);
-			let rect = position.rects[0];
-			let [left, top, right, bottom] = rect;
-			let width = right - left;
-			let height = bottom - top;
-			let newCanvas = document.createElement('canvas');
-			newCanvas.width = width;
-			newCanvas.height = height;
-			let newCanvasContext = newCanvas.getContext('2d');
-			newCanvasContext.drawImage(canvas, left, top, width, height, 0, 0, width, height);
-			let blob = await new Promise(resolve => newCanvas.toBlob(resolve, 'image/png'));
-			let image = await new Response(blob).arrayBuffer();
-			await query('renderedAnnotation', { annotation: { id: annotation.id, image } }, [image]);
-			num++;
-		}
 	}
 	return num;
 }
 
 function p2v(position, viewport) {
-	return {
+	if (position.rects) {
-		pageIndex: position.pageIndex,
+		return {
-		rects: position.rects.map(rect => {
+			pageIndex: position.pageIndex,
-			let [x1, y2] = viewport.convertToViewportPoint(rect[0], rect[1]);
+			rects: position.rects.map((rect) => {
-			let [x2, y1] = viewport.convertToViewportPoint(rect[2], rect[3]);
+				let [x1, y2] = viewport.convertToViewportPoint(rect[0], rect[1]);
-			return [Math.min(x1, x2), Math.min(y1, y2), Math.max(x1, x2), Math.max(y1, y2)];
+				let [x2, y1] = viewport.convertToViewportPoint(rect[2], rect[3]);
-		})
+				return [
-	};
+					Math.min(x1, x2),
+					Math.min(y1, y2),
+					Math.max(x1, x2),
+					Math.max(y1, y2)
+				];
+			})
+		};
+	}
+	else if (position.paths) {
+		return {
+			pageIndex: position.pageIndex,
+			width: position.width * viewport.scale,
+			paths: position.paths.map((path) => {
+				let vpath = [];
+				for (let i = 0; i < path.length - 1; i += 2) {
+					let x = path[i];
+					let y = path[i + 1];
+					vpath.push(...viewport.convertToViewportPoint(x, y));
+				}
+				return vpath;
+			})
+		};
+	}
 }
 
-async function renderPage(pdfDocument, pageIndex) {
+function fitRectIntoRect(rect, containingRect) {
-	let page = await pdfDocument.getPage(pageIndex + 1);
+	return [
-	var canvas = document.createElement('canvas');
+		Math.max(rect[0], containingRect[0]),
-	var viewport = page.getViewport({ scale: SCALE_FACTOR });
+		Math.max(rect[1], containingRect[1]),
-	var context = canvas.getContext('2d', { alpha: false });
+		Math.min(rect[2], containingRect[2]),
-	canvas.height = viewport.height;
+		Math.min(rect[3], containingRect[3])
-	canvas.width = viewport.width;
+	];
-	await page.render({ canvasContext: context, viewport: viewport }).promise;
+}
-	return { canvas, viewport };
+
+function getPositionBoundingRect(position) {
+	if (position.rects) {
+		return [
+			Math.min(...position.rects.map(x => x[0])),
+			Math.min(...position.rects.map(x => x[1])),
+			Math.max(...position.rects.map(x => x[2])),
+			Math.max(...position.rects.map(x => x[3]))
+		];
+	}
+	else if (position.paths) {
+		let x = position.paths[0][0];
+		let y = position.paths[0][1];
+		let rect = [x, y, x, y];
+		for (let path of position.paths) {
+			for (let i = 0; i < path.length - 1; i += 2) {
+				let x = path[i];
+				let y = path[i + 1];
+				rect[0] = Math.min(rect[0], x);
+				rect[1] = Math.min(rect[1], y);
+				rect[2] = Math.max(rect[2], x);
+				rect[3] = Math.max(rect[3], y);
+			}
+		}
+		return rect;
+	}
+}
+
+async function renderImage(pdfDocument, annotation) {
+	let { position, color } = annotation;
+
+	let page = await pdfDocument.getPage(position.pageIndex + 1);
+
+	// Create a new position that just contains single rect that is a bounding
+	// box of image or ink annotations
+	let expandedPosition = { pageIndex: position.pageIndex };
+	if (position.rects) {
+		// Image annotations have only one rect
+		expandedPosition.rects = position.rects;
+	}
+	// paths
+	else {
+		let rect = getPositionBoundingRect(position);
+		// Add padding
+		expandedPosition.rects = [fitRectIntoRect([
+			rect[0] - PATH_BOX_PADDING,
+			rect[1] - PATH_BOX_PADDING,
+			rect[2] + PATH_BOX_PADDING,
+			rect[3] + PATH_BOX_PADDING
+		], page.view)];
+	}
+
+	let rect = expandedPosition.rects[0];
+	let maxScale = Math.sqrt(
+		MAX_CANVAS_PIXELS
+		/ ((rect[2] - rect[0]) * (rect[3] - rect[1]))
+	);
+	let scale = Math.min(SCALE, maxScale);
+
+	expandedPosition = p2v(expandedPosition, page.getViewport({ scale }));
+	rect = expandedPosition.rects[0];
+
+	let viewport = page.getViewport({ scale, offsetX: -rect[0], offsetY: -rect[1] });
+	position = p2v(position, viewport);
+
+	let canvasWidth = (rect[2] - rect[0]);
+	let canvasHeight = (rect[3] - rect[1]);
+
+	let cropScale = viewport.width / canvasWidth;
+
+	let canvas = document.createElement('canvas');
+	
+	let ctx = canvas.getContext('2d', { alpha: false });
+
+	if (!canvasWidth || !canvasHeight) {
+		return null;
+	}
+
+	canvas.width = canvasWidth;
+	canvas.height = canvasHeight;
+	canvas.style.width = canvasWidth + 'px';
+	canvas.style.height = canvasHeight + 'px';
+
+	let renderContext = {
+		canvasContext: ctx,
+		viewport: viewport
+	};
+
+	await page.render(renderContext).promise;
+
+	if (position.paths) {
+		ctx.lineWidth = position.width * cropScale;
+		ctx.beginPath();
+		ctx.strokeStyle = color;
+		for (let path of position.paths) {
+			for (let i = 0; i < path.length - 1; i += 2) {
+				let x = path[i];
+				let y = path[i + 1];
+
+				if (i === 0) {
+					ctx.moveTo(x, y);
+					continue;
+				}
+				ctx.lineTo(x, y);
+			}
+		}
+		ctx.stroke();
+	}
+
+	return canvas;
 }
 
 window.addEventListener('message', async (event) => {