Use zotero/translate and zotero/utilities for shared code

.gitmodules

@@ -41,3 +41,11 @@
 	path = note-editor
 	url = https://github.com/zotero/note-editor.git
 	branch = master
+[submodule "chrome/content/zotero/xpcom/utilities"]
+	path = chrome/content/zotero/xpcom/utilities
+	url = https://github.com/zotero/utilities.git
+	shallow = true
+[submodule "chrome/content/zotero/xpcom/translate"]
+	path = chrome/content/zotero/xpcom/translate
+	url = https://github.com/zotero/translate.git
+	shallow = true

chrome/content/zotero/lookup.js

@@ -42,7 +42,7 @@ var Zotero_Lookup = new function () {
 	 * @returns {Promise<boolean>}
 	 */
 	this.addItemsFromIdentifier = async function (textBox, childItem, toggleProgress) {
-		var identifiers = Zotero.Utilities.Internal.extractIdentifiers(textBox.value);
+		var identifiers = Zotero.Utilities.extractIdentifiers(textBox.value);
 		if (!identifiers.length) {
 			Zotero.alert(
 				window,

chrome/content/zotero/tools/testTranslators/testTranslators.js

@@ -556,7 +556,7 @@ function haveTranslators(translators, type) {
 	
 	var promises = [];
 	for(var i in translators) {
-		promises.push(translators[i].getCode());
+		promises.push(Zotero.Translators.getCodeForTranslator(translators[i]));
 	}
 	
 	return Promise.all(promises).then(function(codes) {

chrome/content/zotero/xpcom/cite.js

@@ -318,114 +318,8 @@ Zotero.Cite = {
 	},
 	
 	extraToCSL: function (extra) {
-		return extra.replace(/^([A-Za-z \-]+)(:\s*.+)/gm, function (_, field, value) {
+		Zotero.debug(`Zotero.Cite.extraToCSL() is deprecated -- use Zotero.Utilities.Item.extraToCSL() instead`);
-			var originalField = field;
+		return Zotero.Utilities.Item.extraToCSL(extra);
-			var field = field.toLowerCase().replace(/ /g, '-');
-			// Fields from https://aurimasv.github.io/z2csl/typeMap.xml
-			switch (field) {
-			// Standard fields
-			case 'abstract':
-			case 'accessed':
-			case 'annote':
-			case 'archive':
-			case 'archive-place':
-			case 'author':
-			case 'authority':
-			case 'call-number':
-			case 'chapter-number':
-			case 'citation-label':
-			case 'citation-number':
-			case 'collection-editor':
-			case 'collection-number':
-			case 'collection-title':
-			case 'composer':
-			case 'container':
-			case 'container-author':
-			case 'container-title':
-			case 'container-title-short':
-			case 'dimensions':
-			case 'director':
-			case 'edition':
-			case 'editor':
-			case 'editorial-director':
-			case 'event':
-			case 'event-date':
-			case 'event-place':
-			case 'first-reference-note-number':
-			case 'genre':
-			case 'illustrator':
-			case 'interviewer':
-			case 'issue':
-			case 'issued':
-			case 'jurisdiction':
-			case 'keyword':
-			case 'language':
-			case 'locator':
-			case 'medium':
-			case 'note':
-			case 'number':
-			case 'number-of-pages':
-			case 'number-of-volumes':
-			case 'original-author':
-			case 'original-date':
-			case 'original-publisher':
-			case 'original-publisher-place':
-			case 'original-title':
-			case 'page':
-			case 'page-first':
-			case 'publisher':
-			case 'publisher-place':
-			case 'recipient':
-			case 'references':
-			case 'reviewed-author':
-			case 'reviewed-title':
-			case 'scale':
-			case 'section':
-			case 'source':
-			case 'status':
-			case 'submitted':
-			case 'title':
-			case 'title-short':
-			case 'translator':
-			case 'type':
-			case 'version':
-			case 'volume':
-			case 'year-suffix':
-				break;
-			
-			// Uppercase fields
-			case 'doi':
-			case 'isbn':
-			case 'issn':
-			case 'pmcid':
-			case 'pmid':
-			case 'url':
-				field = field.toUpperCase();
-				break;
-			
-			// Weirdo
-			case 'archive-location':
-				field = 'archive_location';
-				break;
-			
-			default:
-				// See if this is a Zotero field written out (e.g., "Publication Title"), and if so
-				// convert to its associated CSL field
-				var zoteroField = originalField.replace(/ ([A-Z])/, '$1');
-				// If second character is lowercase (so not an acronym), lowercase first letter too
-				if (zoteroField[1] && zoteroField[1] == zoteroField[1].toLowerCase()) {
-					zoteroField = zoteroField[0].toLowerCase() + zoteroField.substr(1);
-				}
-				if (Zotero.Schema.CSL_FIELD_MAPPINGS_REVERSE[zoteroField]) {
-					field = Zotero.Schema.CSL_FIELD_MAPPINGS_REVERSE[zoteroField];
-				}
-				// Don't change other lines
-				else {
-					field = originalField;
-				}
-			}
-			return field + value;
-		});
 	}
 };
 

chrome/content/zotero/xpcom/connector/server_connector.js

@@ -1506,7 +1506,7 @@ Zotero.Server.Connector.GetTranslatorCode.prototype = {
 	 */
 	init: function(postData, sendResponseCallback) {
 		var translator = Zotero.Translators.get(postData.translatorID);
-		translator.getCode().then(function(code) {
+		Zotero.Translators.getCodeForTranslator(translator).then(function(code) {
 			sendResponseCallback(200, "application/javascript", code);
 		});
 	}

chrome/content/zotero/xpcom/data/item.js

@@ -2172,7 +2172,7 @@ Zotero.Item.prototype.setNote = function(text) {
 	
 	this._hasNote = text !== '';
 	this._noteText = text;
-	this._noteTitle = Zotero.Notes.noteToTitle(text);
+	this._noteTitle = Zotero.Utilities.Item.noteToTitle(text);
 	if (this.isNote()) {
 		this._displayTitle = this._noteTitle;
 	}

chrome/content/zotero/xpcom/data/notes.js

@@ -34,34 +34,9 @@ Zotero.Notes = new function() {
 	this._editorInstances = [];
 	this._downloadInProgressPromise = null;
 	
-	/**
-	* Return first line (or first MAX_LENGTH characters) of note content
-	**/
 	this.noteToTitle = function(text) {
-		var origText = text;
+		Zotero.debug(`Zotero.Note.noteToTitle() is deprecated -- use Zotero.Utilities.Item.noteToTitle() instead`);
-		text = text.trim();
+		return Zotero.Utilities.Item.noteToTitle(text);
-		text = text.replace(/<br\s*\/?>/g, ' ');
-		text = Zotero.Utilities.unescapeHTML(text);
-		
-		// If first line is just an opening HTML tag, remove it
-		//
-		// Example:
-		//
-		// <blockquote>
-		// <p>Foo</p>
-		// </blockquote>
-		if (/^<[^>\n]+[^\/]>\n/.test(origText)) {
-			text = text.trim();
-		}
-		
-		var max = this.MAX_TITLE_LENGTH;
-		
-		var t = text.substring(0, max);
-		var ln = t.indexOf("\n");
-		if (ln>-1 && ln<max) {
-			t = t.substring(0, ln);
-		}
-		return t;
 	};
 	
 	this.registerEditorInstance = function(instance) {

chrome/content/zotero/xpcom/quickCopy.js

@@ -469,7 +469,7 @@ Zotero.QuickCopy = new function() {
 				return;
 			}
 			translator.cacheCode = true;
-			await translator.getCode();
+			await Zotero.Translators.getCodeForTranslator(translator);
 		}
 	};
 	

chrome/content/zotero/xpcom/translate

@@ -0,0 +1 @@
+Subproject commit 4eeca1b5a698bb4cf5f1ce9db554a05b82be6bcd

chrome/content/zotero/xpcom/translation/translate_item.js

@@ -1010,13 +1010,13 @@ Zotero.Translate.ItemGetter = function() {
 };
 
 Zotero.Translate.ItemGetter.prototype = {
-	"setItems":function(items) {
+	setItems: function (items) {
 		this._itemsLeft = items;
-		this._itemsLeft.sort(function(a, b) { return a.id - b.id; });
+		this._itemsLeft.sort((a, b) => a.id - b.id);
 		this.numItems = this._itemsLeft.length;
 	},
 	
-	"setCollection": function (collection, getChildCollections) {
+	setCollection: function (collection, getChildCollections) {
 		// get items in this collection
 		var items = new Set(collection.getChildItems());
 		
@@ -1033,12 +1033,16 @@ Zotero.Translate.ItemGetter.prototype = {
 		}
 		
 		this._itemsLeft = Array.from(items.values());
-		this._itemsLeft.sort(function(a, b) { return a.id - b.id; });
+		this._itemsLeft.sort((a, b) => a.id - b.id);
 		this.numItems = this._itemsLeft.length;
 	},
 	
-	"setAll": Zotero.Promise.coroutine(function* (libraryID, getChildCollections) {
+	/**
-		this._itemsLeft = (yield Zotero.Items.getAll(libraryID, true))
+	 * NOTE: This function should use the Zotero.Promise.method wrapper which adds a
+	 * isResolved property to the returned promise for noWait translation.
+	 */
+	setAll: Zotero.Promise.method(async function (libraryID, getChildCollections) {
+		this._itemsLeft = (await Zotero.Items.getAll(libraryID, true))
 			.filter((item) => {
 				// Don't export annotations
 				switch (item.itemType) {
@@ -1052,7 +1056,7 @@ Zotero.Translate.ItemGetter.prototype = {
 			this._collectionsLeft = Zotero.Collections.getByLibrary(libraryID);
 		}
 
-		this._itemsLeft.sort(function(a, b) { return a.id - b.id; });
+		this._itemsLeft.sort((a, b) => a.id - b.id);
 		this.numItems = this._itemsLeft.length;
 	}),
 	

chrome/content/zotero/xpcom/translation/translators.js

@@ -330,27 +330,36 @@ Zotero.Translators = new function() {
 		if (!_initialized) {
 			throw new Zotero.Exception.UnloadedDataException("Translators not yet loaded", 'translators');
 		}
-		return  _translators[id] ? _translators[id] : false
+		return _translators[id] ? _translators[id] : false;
 	}
 	
+	this.getCodeForTranslator = Zotero.Promise.method(function (translator) {
+		if (translator.code) return translator.code;
+		return Zotero.File.getContentsAsync(translator.path).then(function(code) {
+			if (translator.cacheCode) {
+				// See Translator.init() for cache rules
+				translator.code = code;
+			}
+			return code;
+		});
+	});
+	
 	/**
 	 * Gets all translators for a specific type of translation
 	 *
 	 * @param {String} type The type of translators to get (import, export, web, or search)
 	 */
-	this.getAllForType = function(type) {
+	this.getAllForType = async function (type) {
-		return this.init().then(function () {
+		await this.init();
 		return _cache[type].slice();
-		});
 	}
 	
 	/**
 	 * Gets all translators for a specific type of translation
 	 */
-	this.getAll = function() {
+	this.getAll = async function () {
-		return this.init().then(function () {
+		await this.init();
 		return Object.keys(_translators).map(id => _translators[id]);
-		});
 	}
 	
 	/**

chrome/content/zotero/xpcom/utilities

@@ -0,0 +1 @@
+Subproject commit b89bab4ad7b5e118f323c3fd6bad4258681a4e65

chrome/content/zotero/xpcom/utilities_internal.js

@@ -987,8 +987,8 @@ Zotero.Utilities.Internal = {
 
 
 	itemToExportFormat: function () {
-		Zotero.debug(`Zotero.Utilities.Internal.itemToExportFormat() is deprecated -- use Zotero.Utilities.itemToExportFormat() instead`);
+		Zotero.debug(`Zotero.Utilities.Internal.itemToExportFormat() is deprecated -- use Zotero.Utilities.Item.itemToExportFormat() instead`);
-		return Zotero.Utilities.itemToExportFormat.apply(Zotero.Utilities, arguments);
+		return Zotero.Utilities.Item.itemToExportFormat.apply(Zotero.Utilities, arguments);
 	},
 	
 	
@@ -1279,84 +1279,8 @@ Zotero.Utilities.Internal = {
 	
 	
 	extractIdentifiers: function (text) {
-		var identifiers = [];
+		Zotero.debug(`Zotero.Utilities.Internal.extractIdentifiers() is deprecated -- use Zotero.Utilities.extractIdentifiers() instead`);
-		var foundIDs = new Set(); // keep track of identifiers to avoid duplicates
+		return Zotero.Utilities.extractIdentifiers.apply(Zotero.Utilities, arguments);
-		
-		// First look for DOIs
-		var ids = text.split(/[\s\u00A0]+/); // whitespace + non-breaking space
-		var doi;
-		for (let id of ids) {
-			if ((doi = Zotero.Utilities.cleanDOI(id)) && !foundIDs.has(doi)) {
-				identifiers.push({
-					DOI: doi
-				});
-				foundIDs.add(doi);
-			}
-		}
-		
-		// Then try ISBNs
-		if (!identifiers.length) {
-			// First try replacing dashes
-			let ids = text.replace(/[\u002D\u00AD\u2010-\u2015\u2212]+/g, "") // hyphens and dashes
-				.toUpperCase();
-			let ISBN_RE = /(?:\D|^)(97[89]\d{10}|\d{9}[\dX])(?!\d)/g;
-			let isbn;
-			while (isbn = ISBN_RE.exec(ids)) {
-				isbn = Zotero.Utilities.cleanISBN(isbn[1]);
-				if (isbn && !foundIDs.has(isbn)) {
-					identifiers.push({
-						ISBN: isbn
-					});
-					foundIDs.add(isbn);
-				}
-			}
-			
-			// Next try spaces
-			if (!identifiers.length) {
-				ids = ids.replace(/[ \u00A0]+/g, ""); // space + non-breaking space
-				while (isbn = ISBN_RE.exec(ids)) {
-					isbn = Zotero.Utilities.cleanISBN(isbn[1]);
-					if(isbn && !foundIDs.has(isbn)) {
-						identifiers.push({
-							ISBN: isbn
-						});
-						foundIDs.add(isbn);
-					}
-				}
-			}
-		}
-		
-		// Next try arXiv
-		if (!identifiers.length) {
-			// arXiv identifiers are extracted without version number
-			// i.e. 0706.0044v1 is extracted as 0706.0044,
-			// because arXiv OAI API doesn't allow to access individual versions
-			let arXiv_RE = /((?:[^A-Za-z]|^)([\-A-Za-z\.]+\/\d{7})(?:(v[0-9]+)|)(?!\d))|((?:\D|^)(\d{4}\.\d{4,5})(?:(v[0-9]+)|)(?!\d))/g;
-			let m;
-			while ((m = arXiv_RE.exec(text))) {
-				let arXiv = m[2] || m[5];
-				if (arXiv && !foundIDs.has(arXiv)) {
-					identifiers.push({arXiv: arXiv});
-					foundIDs.add(arXiv);
-				}
-			}
-		}
-		
-		// Finally try for PMID
-		if (!identifiers.length) {
-			// PMID; right now, the longest PMIDs are 8 digits, so it doesn't seem like we'll
-			// need to discriminate for a fairly long time
-			let PMID_RE = /(^|\s|,|:)(\d{1,9})(?=\s|,|$)/g;
-			let pmid;
-			while ((pmid = PMID_RE.exec(text)) && !foundIDs.has(pmid)) {
-				identifiers.push({
-					PMID: pmid[2]
-				});
-				foundIDs.add(pmid);
-			}
-		}
-		
-		return identifiers;
 	},
 	
 	

components/zotero-service.js

@@ -38,26 +38,29 @@ const xpcomFilesAll = [
 	'intl',
 	'prefs',
 	'dataDirectory',
-	'date',
 	'debug',
 	'error',
-	'utilities',
+	'utilities/date',
+	'utilities/utilities',
+	'utilities/utilities_item',
+	'utilities/openurl',
+	'utilities/xregexp-all',
+	'utilities/xregexp-unicode-zotero',
 	'utilities_internal',
+	'translate/src/utilities_translate',
 	'file',
 	'http',
 	'mimeTypeHandler',
-	'openurl',
 	'pdfWorker/manager',
 	'ipc',
 	'profile',
 	'progressWindow',
 	'proxy',
-	'translation/translate',
+	'translate/src/translation/translate',
+	'translate/src/translator',
+	'translate/src/tlds',
 	'translation/translate_firefox',
-	'translation/translator',
-	'translation/tlds',
 	'isbn',
-	'utilities_translate'
 ];
 
 /** XPCOM files to be loaded only for local translation and DB access **/
@@ -275,23 +278,11 @@ function makeZoteroContext(isConnector) {
 	// Load XRegExp object into Zotero.XRegExp
 	const xregexpFiles = [
 		/**Core functions**/
-		'xregexp',
+		'xregexp-all',
-	
+		'xregexp-unicode-zotero'				//adds support for some Unicode categories used in Zotero
-		/**Addons**/
-		'addons/build',												//adds ability to "build regular expressions using named subpatterns, for readability and pattern reuse"
-		'addons/matchrecursive',							//adds ability to "match recursive constructs using XRegExp pattern strings as left and right delimiters"
-	
-		/**Unicode support**/
-		'addons/unicode/unicode-base',				//required for all other unicode packages. Adds \p{Letter} category
-	
-		//'addons/unicode/unicode-blocks',			//adds support for all Unicode blocks (e.g. InArabic, InCyrillic_Extended_A, etc.)
-		'addons/unicode/unicode-categories',	//adds support for all Unicode categories (e.g. Punctuation, Lowercase_Letter, etc.)
-		//'addons/unicode/unicode-properties',	//adds Level 1 Unicode properties (e.g. Uppercase, White_Space, etc.)
-		//'addons/unicode/unicode-scripts'			//adds support for all Unicode scripts (e.g. Gujarati, Cyrillic, etc.)
-		'addons/unicode/unicode-zotero'				//adds support for some Unicode categories used in Zotero
 	];
 	for (var i=0; i<xregexpFiles.length; i++) {
-		subscriptLoader.loadSubScript("chrome://zotero/content/xpcom/xregexp/" + xregexpFiles[i] + ".js", zContext, 'utf-8');
+		subscriptLoader.loadSubScript("chrome://zotero/content/xpcom/utilities/" + xregexpFiles[i] + ".js", zContext, 'utf-8');
 	}
 	
 	// Load remaining xpcomFiles
@@ -324,14 +315,13 @@ function makeZoteroContext(isConnector) {
 		'rdf/uri',
 		'rdf/term',
 		'rdf/identity',
-		'rdf/match',
 		'rdf/n3parser',
 		'rdf/rdfparser',
 		'rdf/serialize'
 	];
 	zContext.Zotero.RDF = {Zotero:zContext.Zotero};
 	for (var i=0; i<rdfXpcomFiles.length; i++) {
-		subscriptLoader.loadSubScript("chrome://zotero/content/xpcom/" + rdfXpcomFiles[i] + ".js", zContext.Zotero.RDF, 'utf-8');
+		subscriptLoader.loadSubScript("chrome://zotero/content/xpcom/translate/src/" + rdfXpcomFiles[i] + ".js", zContext.Zotero.RDF, 'utf-8');
 	}
 	
 	if(isStandalone()) {

scripts/config.js

@@ -12,6 +12,7 @@ const dirs = [
 // list of folders that are symlinked
 const symlinkDirs = [
 	'chrome/content/zotero/xpcom/rdf',
+	'chrome/content/zotero/xpcom/translate/src',
 	'styles',
 	'translators'
 ];
@@ -94,7 +95,9 @@ const ignoreMask = [
 	'**/#*',
 	'resource/schema/global/README.md',
 	'resource/schema/global/schema.json.gz',
-	'resource/schema/global/scripts/*'
+	'resource/schema/global/scripts/*',
+	'chrome/content/zotero/xpcom/translate/example/**/*',
+	'chrome/content/zotero/xpcom/translate/README.md',
 ];
 
 const jsFiles = [

test/tests/dataObjectTest.js

@@ -502,7 +502,7 @@ describe("Zotero.DataObject", function() {
 		function makeObjectURI(objectType) {
 			var objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType);
 			return 'http://zotero.org/groups/1/' + objectTypePlural + '/'
-				+ Zotero.Utilities.Internal.generateObjectKey();
+				+ Zotero.Utilities.generateObjectKey();
 		}
 		
 		describe("#addRelation()", function () {

test/tests/server_connectorTest.js

@@ -57,7 +57,7 @@ describe("Connector Server", function () {
 			);
 
 			assert.isTrue(Zotero.Translators.get.calledWith('dummy-translator'));
-			let translatorCode = yield translator.getCode();
+			let translatorCode = yield Zotero.Translators.getCodeForTranslator(translator);
 			assert.equal(response.response, translatorCode);
 
 			Zotero.Translators.get.restore();

test/tests/utilitiesTest.js

@@ -249,266 +249,6 @@ describe("Zotero.Utilities", function() {
 			assert.equal(cleanISSN('<b>ISSN</b>:1234\xA0-\t5679(print)\n<b>eISSN (electronic)</b>:0028-0836'), '1234-5679');
 		});
 	});
-	describe("itemToCSLJSON", function() {
-		it("should accept Zotero.Item and Zotero export item format", Zotero.Promise.coroutine(function* () {
-			let data = yield populateDBWithSampleData(loadSampleData('journalArticle'));
-			let item = yield Zotero.Items.getAsync(data.journalArticle.id);
-			
-			let fromZoteroItem;
-			try {
-				fromZoteroItem = Zotero.Utilities.itemToCSLJSON(item);
-			} catch(e) {
-				assert.fail(e, null, 'accepts Zotero Item');
-			}
-			assert.isObject(fromZoteroItem, 'converts Zotero Item to object');
-			assert.isNotNull(fromZoteroItem, 'converts Zotero Item to non-null object');
-			
-			
-			let fromExportItem;
-			try {
-				fromExportItem = Zotero.Utilities.itemToCSLJSON(
-					Zotero.Utilities.Internal.itemToExportFormat(item)
-				);
-			} catch(e) {
-				assert.fail(e, null, 'accepts Zotero export item');
-			}
-			assert.isObject(fromExportItem, 'converts Zotero export item to object');
-			assert.isNotNull(fromExportItem, 'converts Zotero export item to non-null object');
-			
-			assert.deepEqual(fromZoteroItem, fromExportItem, 'conversion from Zotero Item and from export item are the same');
-		}));
-		it("should convert standalone notes to expected format", Zotero.Promise.coroutine(function* () {
-			let note = new Zotero.Item('note');
-			note.setNote('Some note longer than 50 characters, which will become the title.');
-			yield note.saveTx();
-			
-			let cslJSONNote = Zotero.Utilities.itemToCSLJSON(note);
-			assert.equal(cslJSONNote.type, 'article', 'note is exported as "article"');
-			assert.equal(cslJSONNote.title, note.getNoteTitle(), 'note title is set to Zotero pseudo-title');
-		}));
-		it("should convert standalone attachments to expected format", Zotero.Promise.coroutine(function* () {
-			let file = getTestDataDirectory();
-			file.append("empty.pdf");
-			
-			let attachment = yield Zotero.Attachments.importFromFile({"file":file});
-			attachment.setField('title', 'Empty');
-			attachment.setField('accessDate', '2001-02-03 12:13:14');
-			attachment.setField('url', 'http://example.com');
-			attachment.setNote('Note');
-			
-			yield attachment.saveTx();
-			
-			let cslJSONAttachment = Zotero.Utilities.itemToCSLJSON(attachment);
-			assert.equal(cslJSONAttachment.type, 'article', 'attachment is exported as "article"');
-			assert.equal(cslJSONAttachment.title, 'Empty', 'attachment title is correct');
-			assert.deepEqual(cslJSONAttachment.accessed, {"date-parts":[["2001",2,3]]}, 'attachment access date is mapped correctly');
-		}));
-		it("should refuse to convert unexpected item types", Zotero.Promise.coroutine(function* () {
-			let data = yield populateDBWithSampleData(loadSampleData('journalArticle'));
-			let item = yield Zotero.Items.getAsync(data.journalArticle.id);
-			
-			let exportFormat = Zotero.Utilities.Internal.itemToExportFormat(item);
-			exportFormat.itemType = 'foo';
-			
-			assert.throws(Zotero.Utilities.itemToCSLJSON.bind(Zotero.Utilities, exportFormat), /^Unexpected Zotero Item type ".*"$/, 'throws an error when trying to map invalid item types');
-		}));
-		
-		it("should parse particles in creator names", function* () {
-			let creators = [
-				{
-					// No particles
-					firstName: 'John',
-					lastName: 'Smith',
-					creatorType: 'author',
-					expect: {
-						given: 'John',
-						family: 'Smith'
-					}
-				},
-				{
-					// dropping and non-dropping
-					firstName: 'Jean de',
-					lastName: 'la Fontaine',
-					creatorType: 'author',
-					expect: {
-						given: 'Jean',
-						"dropping-particle": 'de',
-						"non-dropping-particle": 'la',
-						family: 'Fontaine'
-					}
-				},
-				{
-					// only non-dropping
-					firstName: 'Vincent',
-					lastName: 'van Gogh',
-					creatorType: 'author',
-					expect: {
-						given: 'Vincent',
-						"non-dropping-particle": 'van',
-						family: 'Gogh'
-					}
-				},
-				{
-					// only dropping
-					firstName: 'Alexander von',
-					lastName: 'Humboldt',
-					creatorType: 'author',
-					expect: {
-						given: 'Alexander',
-						"dropping-particle": 'von',
-						family: 'Humboldt'
-					}
-				},
-				{
-					// institutional author
-					lastName: 'Jean de la Fontaine',
-					creatorType: 'author',
-					fieldMode: 1,
-					expect: {
-						literal: 'Jean de la Fontaine'
-					}
-				},
-				{
-					// protected last name
-					firstName: 'Jean de',
-					lastName: '"la Fontaine"',
-					creatorType: 'author',
-					expect: {
-						given: 'Jean de',
-						family: 'la Fontaine'
-					}
-				}
-			];
-			
-			let data = yield populateDBWithSampleData({
-				item: {
-					itemType: 'journalArticle',
-					creators: creators
-				}
-			});
-				
-			let item = Zotero.Items.get(data.item.id);
-			let cslCreators = Zotero.Utilities.itemToCSLJSON(item).author;
-			
-			assert.deepEqual(cslCreators[0], creators[0].expect, 'simple name is not parsed');
-			assert.deepEqual(cslCreators[1], creators[1].expect, 'name with dropping and non-dropping particles is parsed');
-			assert.deepEqual(cslCreators[2], creators[2].expect, 'name with only non-dropping particle is parsed');
-			assert.deepEqual(cslCreators[3], creators[3].expect, 'name with only dropping particle is parsed');
-			assert.deepEqual(cslCreators[4], creators[4].expect, 'institutional author is not parsed');
-			assert.deepEqual(cslCreators[5], creators[5].expect, 'protected last name prevents parsing');
-		});
-		
-		it("should convert UTC access date to local time", async function () {
-			var offset = new Date().getTimezoneOffset();
-			var item = new Zotero.Item('webpage');
-			var localDate;
-			if (offset < 0) {
-				localDate = '2019-01-09 00:00:00';
-			}
-			else if (offset > 0) {
-				localDate = '2019-01-09 23:59:59';
-			}
-			// Can't test timezone offset if in UTC
-			else {
-				this.skip();
-				return;
-			}
-			var utcDate = Zotero.Date.sqlToDate(localDate);
-			item.setField('accessDate', Zotero.Date.dateToSQL(utcDate, true));
-			await item.saveTx();
-			let accessed = Zotero.Utilities.itemToCSLJSON(item).accessed;
-			
-			assert.equal(accessed['date-parts'][0][0], 2019);
-			assert.equal(accessed['date-parts'][0][1], 1);
-			assert.equal(accessed['date-parts'][0][2], 9);
-		});
-	});
-	describe("itemFromCSLJSON", function () {
-		it("should stably perform itemToCSLJSON -> itemFromCSLJSON -> itemToCSLJSON", function* () {
-			this.timeout(10000);
-			let data = loadSampleData('citeProcJSExport');
-			
-			for (let i in data) {
-				let json = data[i];
-				
-				// TEMP: https://github.com/zotero/zotero/issues/1667
-				if (i == 'podcast') {
-					delete json['collection-title'];
-				}
-				
-				let item = new Zotero.Item();
-				Zotero.Utilities.itemFromCSLJSON(item, json);
-				yield item.saveTx();
-				
-				let newJSON = Zotero.Utilities.itemToCSLJSON(item);
-				
-				delete newJSON.id;
-				delete json.id;
-				
-				assert.deepEqual(newJSON, json, i + ' export -> import -> export is stable');
-			}
-			
-		});
-		it("should recognize the legacy shortTitle key", function* () {
-			this.timeout(20000);
-
-			let data = loadSampleData('citeProcJSExport');
-
-			var json = data.artwork;
-			var canonicalKeys = Object.keys(json);
-			json.shortTitle = json["title-short"];
-			delete json["title-short"];
-
-			let item = new Zotero.Item();
-			Zotero.Utilities.itemFromCSLJSON(item, json);
-			yield item.saveTx();
-
-			let newJSON = Zotero.Utilities.itemToCSLJSON(item);
-			assert.hasAllKeys(newJSON, canonicalKeys);
-		});
-		it("should import exported standalone note", function* () {
-			let note = new Zotero.Item('note');
-			note.setNote('Some note longer than 50 characters, which will become the title.');
-			yield note.saveTx();
-			
-			let jsonNote = Zotero.Utilities.itemToCSLJSON(note);
-			
-			let item = new Zotero.Item();
-			Zotero.Utilities.itemFromCSLJSON(item, jsonNote);
-			
-			assert.equal(item.getField('title'), jsonNote.title, 'title imported correctly');
-		});
-		it("should import exported standalone attachment", function* () {
-			let attachment = yield importFileAttachment("empty.pdf");
-			attachment.setField('title', 'Empty');
-			attachment.setField('accessDate', '2001-02-03 12:13:14');
-			attachment.setField('url', 'http://example.com');
-			attachment.setNote('Note');
-			yield attachment.saveTx();
-			
-			let jsonAttachment = Zotero.Utilities.itemToCSLJSON(attachment);
-			
-			let item = new Zotero.Item();
-			Zotero.Utilities.itemFromCSLJSON(item, jsonAttachment);
-			
-			assert.equal(item.getField('title'), jsonAttachment.title, 'title imported correctly');
-		});
-		// For Zotero.Item created in translation sandbox in connectors
-		it("should not depend on Zotero.Item existing", function* () {
-			let item = new Zotero.Item;
-			var Item = Zotero.Item;
-			delete Zotero.Item;
-			assert.throws(() => "" instanceof Zotero.Item);
-
-			let data = loadSampleData('citeProcJSExport');
-			assert.doesNotThrow(Zotero.Utilities.itemFromCSLJSON.bind(Zotero.Utilities, item, Object.values(data)[0]));
-			
-			Zotero.Item = Item;
-			assert.doesNotThrow(() => "" instanceof Zotero.Item);
-		})
-	});
-	
-	
 	describe("#ellipsize()", function () {
 		describe("with wordBoundary", function () {
 			it("should truncate at word boundary", function* () {

test/tests/utilities_itemTest.js

@@ -0,0 +1,261 @@
+describe("Zotero.Utilities.Item", function() {
+	describe("itemFromCSLJSON", function () {
+		it("should stably perform itemToCSLJSON -> itemFromCSLJSON -> itemToCSLJSON", function* () {
+			this.timeout(10000);
+			let data = loadSampleData('citeProcJSExport');
+
+			for (let i in data) {
+				let json = data[i];
+
+				// TEMP: https://github.com/zotero/zotero/issues/1667
+				if (i == 'podcast') {
+					delete json['collection-title'];
+				}
+
+				let item = new Zotero.Item();
+				Zotero.Utilities.itemFromCSLJSON(item, json);
+				yield item.saveTx();
+
+				let newJSON = Zotero.Utilities.itemToCSLJSON(item);
+
+				delete newJSON.id;
+				delete json.id;
+
+				assert.deepEqual(newJSON, json, i + ' export -> import -> export is stable');
+			}
+
+		});
+		it("should recognize the legacy shortTitle key", function* () {
+			this.timeout(20000);
+
+			let data = loadSampleData('citeProcJSExport');
+
+			var json = data.artwork;
+			var canonicalKeys = Object.keys(json);
+			json.shortTitle = json["title-short"];
+			delete json["title-short"];
+
+			let item = new Zotero.Item();
+			Zotero.Utilities.itemFromCSLJSON(item, json);
+			yield item.saveTx();
+
+			let newJSON = Zotero.Utilities.itemToCSLJSON(item);
+			assert.hasAllKeys(newJSON, canonicalKeys);
+		});
+		it("should import exported standalone note", function* () {
+			let note = new Zotero.Item('note');
+			note.setNote('Some note longer than 50 characters, which will become the title.');
+			yield note.saveTx();
+
+			let jsonNote = Zotero.Utilities.itemToCSLJSON(note);
+
+			let item = new Zotero.Item();
+			Zotero.Utilities.itemFromCSLJSON(item, jsonNote);
+
+			assert.equal(item.getField('title'), jsonNote.title, 'title imported correctly');
+		});
+		it("should import exported standalone attachment", function* () {
+			let attachment = yield importFileAttachment("empty.pdf");
+			attachment.setField('title', 'Empty');
+			attachment.setField('accessDate', '2001-02-03 12:13:14');
+			attachment.setField('url', 'http://example.com');
+			attachment.setNote('Note');
+			yield attachment.saveTx();
+
+			let jsonAttachment = Zotero.Utilities.itemToCSLJSON(attachment);
+
+			let item = new Zotero.Item();
+			Zotero.Utilities.itemFromCSLJSON(item, jsonAttachment);
+
+			assert.equal(item.getField('title'), jsonAttachment.title, 'title imported correctly');
+		});
+		// For Zotero.Item created in translation sandbox in connectors
+		it("should not depend on Zotero.Item existing", function* () {
+			let item = new Zotero.Item;
+			var Item = Zotero.Item;
+			delete Zotero.Item;
+			assert.throws(() => "" instanceof Zotero.Item);
+
+			let data = loadSampleData('citeProcJSExport');
+			assert.doesNotThrow(Zotero.Utilities.itemFromCSLJSON.bind(Zotero.Utilities, item, Object.values(data)[0]));
+
+			Zotero.Item = Item;
+			assert.doesNotThrow(() => "" instanceof Zotero.Item);
+		})
+	});
+
+	describe("itemToCSLJSON", function() {
+		it("should accept Zotero.Item and Zotero export item format", Zotero.Promise.coroutine(function* () {
+			let data = yield populateDBWithSampleData(loadSampleData('journalArticle'));
+			let item = yield Zotero.Items.getAsync(data.journalArticle.id);
+
+			let fromZoteroItem;
+			try {
+				fromZoteroItem = Zotero.Utilities.itemToCSLJSON(item);
+			} catch(e) {
+				assert.fail(e, null, 'accepts Zotero Item');
+			}
+			assert.isObject(fromZoteroItem, 'converts Zotero Item to object');
+			assert.isNotNull(fromZoteroItem, 'converts Zotero Item to non-null object');
+
+
+			let fromExportItem;
+			try {
+				fromExportItem = Zotero.Utilities.itemToCSLJSON(
+					Zotero.Utilities.Internal.itemToExportFormat(item)
+				);
+			} catch(e) {
+				assert.fail(e, null, 'accepts Zotero export item');
+			}
+			assert.isObject(fromExportItem, 'converts Zotero export item to object');
+			assert.isNotNull(fromExportItem, 'converts Zotero export item to non-null object');
+
+			assert.deepEqual(fromZoteroItem, fromExportItem, 'conversion from Zotero Item and from export item are the same');
+		}));
+		it("should convert standalone notes to expected format", Zotero.Promise.coroutine(function* () {
+			let note = new Zotero.Item('note');
+			note.setNote('Some note longer than 50 characters, which will become the title.');
+			yield note.saveTx();
+
+			let cslJSONNote = Zotero.Utilities.itemToCSLJSON(note);
+			assert.equal(cslJSONNote.type, 'article', 'note is exported as "article"');
+			assert.equal(cslJSONNote.title, note.getNoteTitle(), 'note title is set to Zotero pseudo-title');
+		}));
+		it("should convert standalone attachments to expected format", Zotero.Promise.coroutine(function* () {
+			let file = getTestDataDirectory();
+			file.append("empty.pdf");
+
+			let attachment = yield Zotero.Attachments.importFromFile({"file":file});
+			attachment.setField('title', 'Empty');
+			attachment.setField('accessDate', '2001-02-03 12:13:14');
+			attachment.setField('url', 'http://example.com');
+			attachment.setNote('Note');
+
+			yield attachment.saveTx();
+
+			let cslJSONAttachment = Zotero.Utilities.itemToCSLJSON(attachment);
+			assert.equal(cslJSONAttachment.type, 'article', 'attachment is exported as "article"');
+			assert.equal(cslJSONAttachment.title, 'Empty', 'attachment title is correct');
+			assert.deepEqual(cslJSONAttachment.accessed, {"date-parts":[["2001",2,3]]}, 'attachment access date is mapped correctly');
+		}));
+		it("should refuse to convert unexpected item types", Zotero.Promise.coroutine(function* () {
+			let data = yield populateDBWithSampleData(loadSampleData('journalArticle'));
+			let item = yield Zotero.Items.getAsync(data.journalArticle.id);
+
+			let exportFormat = Zotero.Utilities.Internal.itemToExportFormat(item);
+			exportFormat.itemType = 'foo';
+
+			assert.throws(Zotero.Utilities.itemToCSLJSON.bind(Zotero.Utilities, exportFormat), /^Unexpected Zotero Item type ".*"$/, 'throws an error when trying to map invalid item types');
+		}));
+
+		it("should parse particles in creator names", function* () {
+			let creators = [
+				{
+					// No particles
+					firstName: 'John',
+					lastName: 'Smith',
+					creatorType: 'author',
+					expect: {
+						given: 'John',
+						family: 'Smith'
+					}
+				},
+				{
+					// dropping and non-dropping
+					firstName: 'Jean de',
+					lastName: 'la Fontaine',
+					creatorType: 'author',
+					expect: {
+						given: 'Jean',
+						"dropping-particle": 'de',
+						"non-dropping-particle": 'la',
+						family: 'Fontaine'
+					}
+				},
+				{
+					// only non-dropping
+					firstName: 'Vincent',
+					lastName: 'van Gogh',
+					creatorType: 'author',
+					expect: {
+						given: 'Vincent',
+						"non-dropping-particle": 'van',
+						family: 'Gogh'
+					}
+				},
+				{
+					// only dropping
+					firstName: 'Alexander von',
+					lastName: 'Humboldt',
+					creatorType: 'author',
+					expect: {
+						given: 'Alexander',
+						"dropping-particle": 'von',
+						family: 'Humboldt'
+					}
+				},
+				{
+					// institutional author
+					lastName: 'Jean de la Fontaine',
+					creatorType: 'author',
+					fieldMode: 1,
+					expect: {
+						literal: 'Jean de la Fontaine'
+					}
+				},
+				{
+					// protected last name
+					firstName: 'Jean de',
+					lastName: '"la Fontaine"',
+					creatorType: 'author',
+					expect: {
+						given: 'Jean de',
+						family: 'la Fontaine'
+					}
+				}
+			];
+
+			let data = yield populateDBWithSampleData({
+				item: {
+					itemType: 'journalArticle',
+					creators: creators
+				}
+			});
+
+			let item = Zotero.Items.get(data.item.id);
+			let cslCreators = Zotero.Utilities.itemToCSLJSON(item).author;
+
+			assert.deepEqual(cslCreators[0], creators[0].expect, 'simple name is not parsed');
+			assert.deepEqual(cslCreators[1], creators[1].expect, 'name with dropping and non-dropping particles is parsed');
+			assert.deepEqual(cslCreators[2], creators[2].expect, 'name with only non-dropping particle is parsed');
+			assert.deepEqual(cslCreators[3], creators[3].expect, 'name with only dropping particle is parsed');
+			assert.deepEqual(cslCreators[4], creators[4].expect, 'institutional author is not parsed');
+			assert.deepEqual(cslCreators[5], creators[5].expect, 'protected last name prevents parsing');
+		});
+
+		it("should convert UTC access date to local time", async function () {
+			var offset = new Date().getTimezoneOffset();
+			var item = new Zotero.Item('webpage');
+			var localDate;
+			if (offset < 0) {
+				localDate = '2019-01-09 00:00:00';
+			}
+			else if (offset > 0) {
+				localDate = '2019-01-09 23:59:59';
+			}
+			// Can't test timezone offset if in UTC
+			else {
+				this.skip();
+				return;
+			}
+			var utcDate = Zotero.Date.sqlToDate(localDate);
+			item.setField('accessDate', Zotero.Date.dateToSQL(utcDate, true));
+			await item.saveTx();
+			let accessed = Zotero.Utilities.itemToCSLJSON(item).accessed;
+
+			assert.equal(accessed['date-parts'][0][0], 2019);
+			assert.equal(accessed['date-parts'][0][1], 1);
+			assert.equal(accessed['date-parts'][0][2], 9);
+		});
+	});
+});