Check Extra field for DOIs for PDF retrieval

E.g., a book with a DOI in Extra

Closes #1551

chrome/content/zotero/xpcom/attachments.js

@@ -886,7 +886,7 @@ Zotero.Attachments = new function(){
 	
 	this.canFindPDFForItem = function (item) {
 		return item.isRegularItem()
-			&& (!!item.getField('DOI') || !!item.getField('url'))
+			&& (!!item.getField('DOI') || !!item.getField('url') || !!item.getExtraField('DOI'))
 			&& item.numPDFAttachments() == 0;
 	};
 	
@@ -906,7 +906,7 @@ Zotero.Attachments = new function(){
 		var useCustom = methods.includes('custom');
 		
 		var resolvers = [];
-		var doi = item.getField('DOI');
+		var doi = item.getField('DOI') || item.getExtraField('DOI');
 		doi = Zotero.Utilities.cleanDOI(doi);
 		
 		if (useDOI && doi) {

chrome/content/zotero/xpcom/data/item.js

@@ -282,6 +282,13 @@ Zotero.Item.prototype.getField = function(field, unformatted, includeBaseMapped)
 }
 
 
+Zotero.Item.prototype.getExtraField = function (fieldName) {
+	var fields = Zotero.Utilities.Internal.extractExtraFields(this.getField('extra'));
+	var doi = fields.get(fieldName);
+	return (doi && doi.value) ? doi.value : '';
+};
+
+
 /**
  * @param	{Boolean}				asNames
  * @return	{Integer[]|String[]}

chrome/content/zotero/xpcom/data/itemFields.js

@@ -27,6 +27,7 @@
 Zotero.ItemFields = new function() {
 	// Private members
 	var _fields = {};
+	var _allFields = [];
 	var _fieldsFormats = [];
 	var _fieldsLoaded;
 	var _itemTypeFieldsLoaded;
@@ -87,6 +88,10 @@ Zotero.ItemFields = new function() {
 			};
 			// Store by name as well as id
 			_fields[field['fieldName']] = _fields[field['fieldID']];
+			_allFields.push({
+				id: field.fieldID,
+				name: field.fieldName
+			});
 		}
 		
 		_fieldsLoaded = true;
@@ -123,6 +128,11 @@ Zotero.ItemFields = new function() {
 	}
 	
 	
+	this.getAll = function () {
+		return [..._allFields];
+	};
+	
+	
 	function getLocalizedString(itemType, field) {
 		// unused currently
 		//var typeName = Zotero.ItemTypes.getName(itemType);

chrome/content/zotero/xpcom/utilities_internal.js

@@ -848,6 +848,68 @@ Zotero.Utilities.Internal = {
 	},
 	
 	
+	/**
+	 * Find valid fields in Extra field text
+	 *
+	 * @param {String} str
+	 * @return {Map} - Map of fields to objects with 'originalField', 'field', and 'value'
+	 */
+	extractExtraFields: function (str) {
+		if (!str) {
+			return new Map();
+		}
+		
+		//
+		// Build a Map of normalized field names that might appear in Extra (including CSL variables)
+		// to arrays of built-in fields
+		//
+		// Built-in fields
+		var fieldNames = new Map(Zotero.ItemFields.getAll().map(x => [x.name.toLowerCase(), [x.name]]));
+		// CSL fields
+		for (let map of [CSL_TEXT_MAPPINGS, CSL_DATE_MAPPINGS]) {
+			for (let cslVar in map) {
+				let normalized = cslVar.toLowerCase();
+				let existing = fieldNames.get(normalized) || [];
+				fieldNames.set(normalized, new Set([...existing, ...map[cslVar]]));
+			}
+		}
+		
+		var lines = str.split(/\n+/g);
+		var fields = new Map();
+		for (let line of lines) {
+			let parts = line.match(/^([a-z \-]+):(.+)/i);
+			if (!parts) {
+				continue;
+			}
+			let [_, originalField, value] = parts;
+			
+			let field = originalField.trim().toLowerCase()
+				// Strip spaces
+				.replace(/\s+/g, '')
+				// Old citeproc.js cheater syntax
+				.replace(/{:([^:]+):([^}]+)}/);
+			value = value.trim();
+			let possibleFields = fieldNames.get(field);
+			// No valid fields
+			if (!possibleFields) {
+				continue;
+			}
+			// Create an entry for each possible field, since we don't know what type this is for
+			for (let possibleField of possibleFields) {
+				fields.set(
+					possibleField,
+					{
+						originalField,
+						field: possibleField,
+						value
+					}
+				);
+			}
+		}
+		return fields;
+	},
+	
+	
 	extractIdentifiers: function (text) {
 		var identifiers = [];
 		var foundIDs = new Set(); // keep track of identifiers to avoid duplicates

test/tests/attachmentsTest.js

@@ -510,6 +510,24 @@ describe("Zotero.Attachments", function() {
 			assert.equal(await OS.File.stat(attachment.getFilePath()).size, pdfSize);
 		});
 		
+		it("should add a PDF from a resolved DOI from the Extra field", async function () {
+			var doi = doi1;
+			var item = createUnsavedDataObject('item', { itemType: 'journalArticle' });
+			item.setField('title', 'Test');
+			item.setField('extra', 'DOI: ' + doi);
+			await item.saveTx();
+			var attachment = await Zotero.Attachments.addAvailablePDF(item);
+			
+			assert.isTrue(requestStub.calledOnce);
+			assert.isTrue(requestStub.calledWith('GET', 'https://doi.org/' + doi));
+			assert.ok(attachment);
+			var json = attachment.toJSON();
+			assert.equal(json.url, pdfURL);
+			assert.equal(json.contentType, 'application/pdf');
+			assert.equal(json.filename, 'Test.pdf');
+			assert.equal(await OS.File.stat(attachment.getFilePath()).size, pdfSize);
+		});
+		
 		it("should add a PDF from a URL", async function () {
 			var url = pageURL1;
 			var item = createUnsavedDataObject('item', { itemType: 'journalArticle' });

test/tests/utilities_internalTest.js

@@ -113,6 +113,34 @@ describe("Zotero.Utilities.Internal", function () {
 	});
 	
 	
+	describe("#extractExtraFields()", function () {
+		it("should extract a field", function () {
+			var val = '10.1234/abcdef';
+			var str = `DOI: ${val}`;
+			var fields = Zotero.Utilities.Internal.extractExtraFields(str);
+			assert.equal(fields.size, 1);
+			assert.equal(fields.get('DOI').value, val);
+		});
+		
+		it("should extract a field with different case", function () {
+			var val = '10.1234/abcdef';
+			var str = `doi: ${val}`;
+			var fields = Zotero.Utilities.Internal.extractExtraFields(str);
+			assert.equal(fields.size, 1);
+			assert.equal(fields.get('DOI').value, val);
+		});
+		
+		it("should extract a field with other fields, text, and whitespace", function () {
+			var originalDateVal = '1989';
+			var doiVal = '10.1234/abcdef';
+			var str = `\nOriginal Date: ${originalDateVal}\nDOI: ${doiVal}\n\n`;
+			var fields = Zotero.Utilities.Internal.extractExtraFields(str);
+			assert.equal(fields.size, 1);
+			assert.equal(fields.get('DOI').value, doiVal);
+		});
+	});
+	
+	
 	describe("#extractIdentifiers()", function () {
 		it("should extract ISBN-10", async function () {
 			var id = "0838985890";