Upgrade note schema to v2:

- Automatically upgrade all editable v1 notes when note is being opened
- Pull `itemData` from existing citations and annotations into metadata container attribute `data-citation-items`
- Strip `abstract` from `itemData`
- Fill `citationItem` (in citations and annotations) with `itemData` when passing an HTML fragment for further processing in `note-editor`
- Keep only `uri`, `text`, `color`, `pageLabel`, `position`, `citationItem` annotation properties

chrome/content/zotero/bindings/noteeditor.xml

@@ -95,6 +95,13 @@
 						if (this._editorInstance) {
 							this._editorInstance.uninit();
 						}
+
+						// Automatically upgrade editable v1 note before it's loaded
+						// TODO: Remove this at some point
+						if (this.editable) {
+							await Zotero.Notes.upgradeSchemaV1(this._item);
+						}
+
 						this._editorInstance = new Zotero.EditorInstance();
 						await this._editorInstance.init({
 							state,

chrome/content/zotero/xpcom/data/notes.js

@@ -125,41 +125,64 @@ Zotero.Notes = new function() {
 		if (!item.isNote()) {
 			throw new Error('Item is not a note');
 		}
-		var note = item.getNote();
+		let note = item.getNote();
 		
-		var parser = Components.classes["@mozilla.org/xmlextras/domparser;1"]
+		let parser = Components.classes["@mozilla.org/xmlextras/domparser;1"]
 			.createInstance(Components.interfaces.nsIDOMParser);
-		var doc = parser.parseFromString(note, 'text/html');
+		let doc = parser.parseFromString(note, 'text/html');
 		
-		var nodes = doc.querySelectorAll('img[data-attachment-key]');
+		// Make sure this is the new note
-		for (var node of nodes) {
+		let metadataContainer = doc.querySelector('body > div[data-schema-version]');
-			var attachmentKey = node.getAttribute('data-attachment-key');
+		if (metadataContainer) {
-			if (attachmentKey) {
+			// Load base64 image data into src
-				var attachment = Zotero.Items.getByLibraryAndKey(item.libraryID, attachmentKey);
+			let nodes = doc.querySelectorAll('img[data-attachment-key]');
-				if (attachment && attachment.parentID == item.id) {
+			for (let node of nodes) {
-					var dataURI = await attachment.attachmentDataURI;
+				let attachmentKey = node.getAttribute('data-attachment-key');
-					node.setAttribute('src', dataURI);
+				if (attachmentKey) {
-				}
+					let attachment = Zotero.Items.getByLibraryAndKey(item.libraryID, attachmentKey);
-			}
+					if (attachment && attachment.parentID == item.id) {
-			node.removeAttribute('data-attachment-key');
+						let dataURI = await attachment.attachmentDataURI;
-		}
+						node.setAttribute('src', dataURI);
-		
-		var nodes = doc.querySelectorAll('.citation[data-citation]');
-		for (var node of nodes) {
-			var citation = node.getAttribute('data-citation');
-			try {
-				citation = JSON.parse(decodeURIComponent(citation));
-				for (var citationItem of citation.citationItems) {
-					var item = await Zotero.EditorInstance.getItemFromURIs(citationItem.uris);
-					if (item) {
-						citationItem.itemData = Zotero.Cite.System.prototype.retrieveItem(item);
 					}
 				}
-				citation = encodeURIComponent(JSON.stringify(citation));
+				node.removeAttribute('data-attachment-key');
-				node.setAttribute('data-citation', citation);
 			}
-			catch (e) {
+			
-				Zotero.logError(e);
+			// Set itemData for each citation citationItem
+			nodes = doc.querySelectorAll('.citation[data-citation]');
+			for (let node of nodes) {
+				let citation = node.getAttribute('data-citation');
+				try {
+					citation = JSON.parse(decodeURIComponent(citation));
+					for (let citationItem of citation.citationItems) {
+						// Get itemData from existing item
+						let item = await Zotero.EditorInstance.getItemFromURIs(citationItem.uris);
+						if (item) {
+							citationItem.itemData = Zotero.Cite.System.prototype.retrieveItem(item);
+						}
+						// Get itemData from note metadata container
+						else {
+							try {
+								let items = JSON.parse(decodeURIComponent(metadataContainer.getAttribute('data-citation-items')));
+								let item = items.find(item => item.uris.some(uri => citationItem.uris.includes(uri)));
+								if (item) {
+									citationItem.itemData = item.itemData;
+								}
+							}
+							catch (e) {
+							}
+						}
+						if (!citationItem.itemData) {
+							node.replaceWith('(MISSING CITATION)');
+							break;
+						}
+					}
+					citation = encodeURIComponent(JSON.stringify(citation));
+					node.setAttribute('data-citation', citation);
+				}
+				catch (e) {
+					Zotero.logError(e);
+				}
 			}
 		}
 		return doc.body.innerHTML;
@@ -171,6 +194,112 @@ Zotero.Notes = new function() {
 		let doc = parser.parseFromString(note, 'text/html');
 		return !!doc.querySelector('body > div[data-schema-version]');
 	};
+
+	/**
+	 * Upgrade v1 notes:
+	 * - Pull itemData from citations, highlights, images into metadata container
+	 * - Strip abstract field from itemData
+	 * - For `data-annotation` keep only the following fields:
+	 *    - uri
+	 *    - text
+	 *    - color
+	 *    - pageLabel
+	 *    - position
+	 *    - citationItem
+	 * - Increase schema version number
+	 *
+	 * @param item
+	 * @returns {Promise<boolean>}
+	 */
+	this.upgradeSchemaV1 = async function (item) {
+		let note = item.note;
+
+		let parser = Components.classes['@mozilla.org/xmlextras/domparser;1']
+		.createInstance(Components.interfaces.nsIDOMParser);
+		let doc = parser.parseFromString(note, 'text/html');
+
+		let metadataContainer = doc.querySelector('body > div[data-schema-version]');
+		if (!metadataContainer) {
+			return false;
+		}
+
+		let schemaVersion = parseInt(metadataContainer.getAttribute('data-schema-version'));
+		if (schemaVersion !== 1) {
+			return false;
+		}
+		
+		let storedCitationItems = [];
+		try {
+			let data = JSON.parse(decodeURIComponent(metadataContainer.getAttribute('data-citation-items')));
+			if (Array.isArray(data)) {
+				storedCitationItems = data;
+			}
+		} catch (e) {
+		}
+
+		function pullItemData(citationItem) {
+			let { uris, itemData } = citationItem;
+			if (itemData) {
+				delete citationItem.itemData.abstract;
+				delete citationItem.itemData;
+				let item = storedCitationItems.find(item => item.uris.some(uri => uris.includes(uri)));
+				if (!item) {
+					storedCitationItems.push({ uris, itemData });
+				}
+			}
+		}
+		
+		let nodes = doc.querySelectorAll('.citation[data-citation]');
+		for (let node of nodes) {
+			let citation = node.getAttribute('data-citation');
+			try {
+				citation = JSON.parse(decodeURIComponent(citation));
+				citation.citationItems.forEach(citationItem => pullItemData(citationItem));
+				citation = encodeURIComponent(JSON.stringify(citation));
+				node.setAttribute('data-citation', citation);
+			}
+			catch (e) {
+				Zotero.logError(e);
+			}
+		}
+
+		// img[data-annotation] and div.highlight[data-annotation]
+		nodes = doc.querySelectorAll('*[data-annotation]');
+		for (let node of nodes) {
+			let annotation = node.getAttribute('data-annotation');
+			try {
+				annotation = JSON.parse(decodeURIComponent(annotation));
+				if (annotation.citationItem) {
+					pullItemData(annotation.citationItem);
+				}
+				annotation = {
+					uri: annotation.uri,
+					text: annotation.text,
+					color: annotation.color,
+					pageLabel: annotation.pageLabel,
+					position: annotation.position,
+					citationItem: annotation.citationItem
+				};
+				annotation = encodeURIComponent(JSON.stringify(annotation));
+				node.setAttribute('data-annotation', annotation);
+			}
+			catch (e) {
+				Zotero.logError(e);
+			}
+		}
+
+		if (storedCitationItems.length) {
+			storedCitationItems = encodeURIComponent(JSON.stringify(storedCitationItems));
+			metadataContainer.setAttribute('data-citation-items', storedCitationItems);
+		}
+		schemaVersion++;
+		metadataContainer.setAttribute('data-schema-version', schemaVersion);
+		note = doc.body.innerHTML;
+		note = note.trim();
+		item.setNote(note);
+		await item.saveTx({ skipDateModifiedUpdate: true });
+		return true;
+	};
 };
 
 if (typeof process === 'object' && process + '' === '[object process]') {

chrome/content/zotero/xpcom/editorInstance.js

@@ -36,7 +36,7 @@ const DOWNLOADED_IMAGE_TYPE = [
 ];
 
 // Schema version here has to be the same as in note-editor!
-const SCHEMA_VERSION = 1;
+const SCHEMA_VERSION = 2;
 
 class EditorInstance {
 	constructor() {
@@ -159,7 +159,7 @@ class EditorInstance {
 
 	async insertAnnotations(annotations) {
 		await this._ensureNoteCreated();
-		let html = await this._serializeAnnotations(annotations);
+		let [html] = await this._serializeAnnotations(annotations);
 		if (html) {
 			this._postMessage({ action: 'insertHTML', pos: -1, html });
 		}
@@ -197,9 +197,11 @@ class EditorInstance {
 	
 	/**
 	 * @param {Zotero.Item[]} annotations
+	 * @param {Boolean} skipEmbeddingItemData Do not add itemData to citation items
 	 * @return {String} - HTML string
 	 */
-	async _serializeAnnotations(annotations) {
+	async _serializeAnnotations(annotations, skipEmbeddingItemData) {
+		let storedCitationItems = [];
 		let html = '';
 		for (let annotation of annotations) {
 			let attachmentItem = await Zotero.Items.getAsync(annotation.attachmentItemID);
@@ -218,20 +220,36 @@ class EditorInstance {
 			let highlightHTML = '';
 			let commentHTML = '';
 			
-			annotation.uri = Zotero.URI.getItemURI(attachmentItem);
+			let storedAnnotation = {
+				uri: Zotero.URI.getItemURI(attachmentItem),
+				text: annotation.text,
+				color: annotation.color,
+				pageLabel: annotation.pageLabel,
+				position: annotation.position
+			};
 			
 			// Citation
 			let parentItem = attachmentItem.parentID && await Zotero.Items.getAsync(attachmentItem.parentID);
 			if (parentItem) {
+				let uris = [Zotero.URI.getItemURI(parentItem)];
+				let citationItem = {
+					uris,
+					locator: annotation.pageLabel
+				};
+				
 				// TODO: Find a more elegant way to call this
 				let itemData = Zotero.Cite.System.prototype.retrieveItem(parentItem);
 				delete itemData.abstract;
-				let citationItem = {
+				if (!skipEmbeddingItemData) {
-					uris: [Zotero.URI.getItemURI(parentItem)],
+					citationItem.itemData = itemData;
-					itemData,
+				}
-					locator: annotation.pageLabel
+
-				};
+				let item = storedCitationItems.find(item => item.uris.some(uri => uris.includes(uri)));
-				annotation.citationItem = citationItem;
+				if (!item) {
+					storedCitationItems.push({ uris, itemData });
+				}
+				
+				storedAnnotation.citationItem = citationItem;
 				let citation = {
 					citationItems: [citationItem],
 					properties: {}
@@ -255,12 +273,12 @@ class EditorInstance {
 				const PDFJS_DEFAULT_SCALE = 1.25;
 				let width = Math.round(rectWidth * CSS_UNITS * PDFJS_DEFAULT_SCALE);
 				let height = Math.round(rectHeight * width / rectWidth);
-				imageHTML = `<img data-attachment-key="${imageAttachmentKey}" width="${width}" height="${height}" data-annotation="${encodeURIComponent(JSON.stringify(annotation))}"/>`;
+				imageHTML = `<img data-attachment-key="${imageAttachmentKey}" width="${width}" height="${height}" data-annotation="${encodeURIComponent(JSON.stringify(storedAnnotation))}"/>`;
 			}
 
 			// Text
 			if (annotation.text) {
-				highlightHTML = `<span class="highlight" data-annotation="${encodeURIComponent(JSON.stringify(annotation))}">“${annotation.text}”</span>`;
+				highlightHTML = `<span class="highlight" data-annotation="${encodeURIComponent(JSON.stringify(storedAnnotation))}">“${annotation.text}”</span>`;
 			}
 			
 			// Note
@@ -274,7 +292,7 @@ class EditorInstance {
 			}
 			html += '<p>' + imageHTML + otherHTML + '</p>\n';
 		}
-		return html;
+		return [html, storedCitationItems];
 	}
 
 	async _digestItems(ids) {
@@ -299,6 +317,66 @@ class EditorInstance {
 			}
 			else if (item.isNote()) {
 				let note = item.note;
+				
+				let parser = Components.classes['@mozilla.org/xmlextras/domparser;1']
+				.createInstance(Components.interfaces.nsIDOMParser);
+				let doc = parser.parseFromString(note, 'text/html');
+
+				// Get citationItems with itemData from note metadata
+				let storedCitationItems = [];
+				let containerNode = doc.querySelector('body > div[data-schema-version]');
+				if (containerNode) {
+					try {
+						let data = JSON.parse(decodeURIComponent(containerNode.getAttribute('data-citation-items')));
+						if (Array.isArray(data)) {
+							storedCitationItems = data;
+						}
+					}
+					catch (e) {
+					}
+				}
+
+				if (storedCitationItems.length) {
+					let fillWithItemData = (citationItems) => {
+						for (let citationItem of citationItems) {
+							let item = storedCitationItems.find(item => item.uris.some(uri => citationItem.uris.includes(uri)));
+							if (item) {
+								citationItem.itemData = item.itemData;
+							}
+						}
+					};
+
+					let nodes = doc.querySelectorAll('.citation[data-citation]');
+					for (let node of nodes) {
+						let citation = node.getAttribute('data-citation');
+						try {
+							citation = JSON.parse(decodeURIComponent(citation));
+							fillWithItemData(citation.citationItems);
+							citation = encodeURIComponent(JSON.stringify(citation));
+							node.setAttribute('data-citation', citation);
+						}
+						catch (e) {
+							Zotero.logError(e);
+						}
+					}
+					
+					// img[data-annotation] and div.highlight[data-annotation]
+					nodes = doc.querySelectorAll('*[data-annotation]');
+					for (let node of nodes) {
+						let annotation = node.getAttribute('data-annotation');
+						try {
+							annotation = JSON.parse(decodeURIComponent(annotation));
+							fillWithItemData([annotation.citationItem]);
+							annotation = encodeURIComponent(JSON.stringify(annotation));
+							node.setAttribute('data-annotation', annotation);
+						}
+						catch (e) {
+							Zotero.logError(e);
+						}
+					}
+				}
+
+				// Clone all note image attachments and replace keys in the new note
 				let attachments = await Zotero.Items.getAsync(item.getAttachments());
 				for (let attachment of attachments) {
 					let path = await attachment.getFilePathAsync();
@@ -315,9 +393,14 @@ class EditorInstance {
 							}
 						}
 					});
-					note = note.replace(attachment.key, clonedAttachment.key);
+					
+					let node = doc.querySelector(`img[data-attachment-key=${attachment.key}]`);
+					if (node) {
+						node.setAttribute('data-attachment-key', clonedAttachment.key);
+					}
 				}
-				html += `<p></p>${note}<p></p>`;
+				
+				html += `<p></p>${doc.body.innerHTML}<p></p>`;
 			}
 		}
 		return html;
@@ -343,7 +426,7 @@ class EditorInstance {
 					}
 					else if (type === 'zotero/annotation') {
 						let annotations = JSON.parse(data);
-						html = await this._serializeAnnotations(annotations);
+						[html] = await this._serializeAnnotations(annotations);
 					}
 					if (html) {
 						this._postMessage({ action: 'insertHTML', pos, html });
@@ -462,6 +545,7 @@ class EditorInstance {
 							availableCitationItems.push({ ...citationItem, id: item.id });
 						}
 					}
+					// Notice: Citation items that don't exist in the library aren't shown in the popup
 					citation.citationItems = availableCitationItems;
 					let libraryID = this._item.libraryID;
 					this._openQuickFormatDialog(nodeID, citation, [libraryID]);
@@ -857,6 +941,7 @@ class EditorInstance {
 					delete citationItem.id;
 					citationItem.uris = [Zotero.URI.getItemURI(item)];
 					citationItem.itemData = Zotero.Cite.System.prototype.retrieveItem(item);
+					delete citationItem.itemData.abstract;
 				}
 				
 				let formattedCitation = (await that._getFormattedCitationParts(citation)).join(';');
@@ -1019,8 +1104,12 @@ class EditorInstance {
 			jsonAnnotations.push(jsonAnnotation);
 		}
 		let html = `<h1>${Zotero.getString('note.annotationsWithDate', new Date().toLocaleString())}</h1>\n`;
-		html += await editorInstance._serializeAnnotations(jsonAnnotations);
+		let [serializedHTML, storedCitationItems] = await editorInstance._serializeAnnotations(jsonAnnotations, true);
-		html = `<div data-schema-version="${SCHEMA_VERSION}">${html}</div>`;
+		
+		html += serializedHTML;
+		
+		storedCitationItems = encodeURIComponent(JSON.stringify(storedCitationItems));
+		html = `<div data-citation-items="${storedCitationItems}" data-schema-version="${SCHEMA_VERSION}">${html}</div>`;
 		note.setNote(html);
 		await note.saveTx();
 		return note;

note-editor

@@ -1 +1 @@
-Subproject commit f649d1a4e6f74715a8b57ac298c058a07d8ea4e0
+Subproject commit c5dfc982c09915e5a2fb6103edf0395dc17ce728

pdf-reader

@@ -1 +1 @@
-Subproject commit 310698f3a7ebadad8557db1f11e8aa5decaae432
+Subproject commit 047f753389e30e38e967cd48828ab8ea0441575c