From 4b523555d60329def266dd50a254bfe1aea919f0 Mon Sep 17 00:00:00 2001 From: Tom Najdek Date: Sun, 16 Oct 2022 20:10:58 +0200 Subject: [PATCH 01/11] Mendeley Import: Auth using direct login * Importer will now ask user for a login and password via form and will perform sign-in directly using credentials rather than oauth * Signing in this way enables importer to obtain desktop document ID which is now stored for each item * It's possible to switch back to the old method (ouath) by setting `import.mendeleyUseOAuth` pref to `true`. * New option to only import new items. This options only appears if database contains previously imported items. * Importer will now update mendeleyDB:documentUUID on existing items to match value used in Mendeley Desktop if available * Importer will no longer create collections when no new items are imported * Importer will only report number of new items imported on re-import * Importer will now preserve dateAdded on re-import Co-authored-by: Dan Stillman --- chrome/content/zotero/fileInterface.js | 4 +- chrome/content/zotero/import/importWizard.js | 97 +++++++++-- .../content/zotero/import/importWizard.xhtml | 17 +- .../import/mendeley/mendeleyAPIUtils.js | 72 ++++++-- .../zotero/import/mendeley/mendeleyImport.js | 61 +++++-- .../import/mendeley/mendeleyOnlineMappings.js | 1 + chrome/locale/en-US/zotero/zotero.dtd | 1 + chrome/locale/en-US/zotero/zotero.ftl | 9 +- chrome/locale/en-US/zotero/zotero.properties | 2 + scss/components/_import-wizard.scss | 40 +++++ .../items-simple-no-desktop-id.json | 88 ++++++++++ .../tests/data/mendeleyMock/items-simple.json | 3 + .../data/mendeleyMock/items-updated.json | 21 ++- test/tests/mendeleyImportTest.js | 157 ++++++++++++++++-- 14 files changed, 510 insertions(+), 63 deletions(-) create mode 100644 test/tests/data/mendeleyMock/items-simple-no-desktop-id.json diff --git a/chrome/content/zotero/fileInterface.js b/chrome/content/zotero/fileInterface.js index 0d27fe1d33..5caba74f60 100644 --- a/chrome/content/zotero/fileInterface.js +++ b/chrome/content/zotero/fileInterface.js @@ -444,10 +444,12 @@ var Zotero_File_Interface = new function() { var translation; - if (options.mendeleyCode) { + if (options.mendeleyAuth || options.mendeleyCode) { translation = yield _getMendeleyTranslation(); translation.createNewCollection = createNewCollection; + translation.mendeleyAuth = options.mendeleyAuth; translation.mendeleyCode = options.mendeleyCode; + translation.newItemsOnly = options.newItemsOnly; } else if (options.folder) { Components.utils.import("chrome://zotero/content/import/folderImport.js"); diff --git a/chrome/content/zotero/import/importWizard.js b/chrome/content/zotero/import/importWizard.js index 28ea2181d5..3f42467a3d 100644 --- a/chrome/content/zotero/import/importWizard.js +++ b/chrome/content/zotero/import/importWizard.js @@ -1,7 +1,7 @@ /* ***** BEGIN LICENSE BLOCK ***** - Copyright © 2022 Corporation for Digital Scholarship + Copyright © 2023 Corporation for Digital Scholarship Vienna, Virginia, USA https://www.zotero.org @@ -22,23 +22,28 @@ ***** END LICENSE BLOCK ***** */ +/* eslint camelcase: ["error", {allow: ["Zotero_File_Interface", "Zotero_Import_Wizard"]} ] */ +/* global Zotero_File_Interface: false, mendeleyAPIUtils: false */ import FilePicker from 'zotero/filePicker'; import React from 'react'; import ReactDOM from 'react-dom'; import ProgressQueueTable from 'components/progressQueueTable'; -/* eslint camelcase: ["error", {allow: ["Zotero_File_Interface", "Zotero_Import_Wizard"]} ] */ -/* global Zotero_File_Interface: false */ +Components.utils.import("resource://gre/modules/Services.jsm"); +Services.scriptloader.loadSubScript("chrome://zotero/content/import/mendeley/mendeleyAPIUtils.js"); +const { directAuth } = mendeleyAPIUtils; const Zotero_Import_Wizard = { // eslint-disable-line no-unused-vars - wizard: null, - folder: null, file: null, - mendeleyCode: null, + folder: null, libraryID: null, + mendeleyAuth: null, + mendeleyCode: null, + mendeleyHasPreviouslyImported: false, translation: null, + wizard: null, async getShouldCreateCollection() { const sql = "SELECT ROWID FROM collections WHERE libraryID=?1 " @@ -57,15 +62,22 @@ const Zotero_Import_Wizard = { // eslint-disable-line no-unused-vars const { mendeleyCode, libraryID } = window.arguments[0].wrappedJSObject ?? {}; this.libraryID = libraryID; - this.mendeleyCode = mendeleyCode; + + const predicateID = Zotero.RelationPredicates.getID('mendeleyDB:documentUUID'); + if (predicateID) { + const relSQL = 'SELECT ROWID FROM itemRelations WHERE predicateID = ? LIMIT 1'; + this.mendeleyHasPreviouslyImported = !!(await Zotero.DB.valueQueryAsync(relSQL, predicateID)); + } this.wizard = document.getElementById('import-wizard'); this.wizard.getPageById('page-start') .addEventListener('pageadvanced', this.onImportSourceAdvance.bind(this)); + this.wizard.getPageById('page-mendeley-online-intro') + .addEventListener('pageshow', this.onMendeleyOnlineShow.bind(this)); this.wizard.getPageById('page-mendeley-online-intro') .addEventListener('pagerewound', this.goToStart.bind(this)); this.wizard.getPageById('page-mendeley-online-intro') - .addEventListener('pageadvanced', this.openMendeleyAuthWindow.bind(this)); + .addEventListener('pageadvanced', this.onMendeleyOnlineAdvance.bind(this)); this.wizard.getPageById('page-options') .addEventListener('pageshow', this.onOptionsPageShow.bind(this)); this.wizard.getPageById('page-options') @@ -90,6 +102,10 @@ const Zotero_Import_Wizard = { // eslint-disable-line no-unused-vars document .querySelector('#page-done-error > button') .addEventListener('keydown', this.onReportErrorInteract.bind(this)); + document + .getElementById('mendeley-username').addEventListener('keyup', this.onMendeleyAuthKeyUp.bind(this)); + document + .getElementById('mendeley-password').addEventListener('keyup', this.onMendeleyAuthKeyUp.bind(this)); this.wizard.addEventListener('pageshow', this.updateFocus.bind(this)); this.wizard.addEventListener('wizardcancel', this.onCancel.bind(this)); @@ -101,7 +117,8 @@ const Zotero_Import_Wizard = { // eslint-disable-line no-unused-vars this.wizard.shadowRoot .querySelector('.wizard-header-label').style.fontSize = '16px'; - if (mendeleyCode) { + if (mendeleyCode && Zotero.Prefs.get("import.mendeleyUseOAuth")) { + this.mendeleyCode = mendeleyCode; this.wizard.goTo('page-options'); } }, @@ -194,6 +211,50 @@ const Zotero_Import_Wizard = { // eslint-disable-line no-unused-vars this.wizard.goTo('page-options'); }, + async onMendeleyOnlineShow() { + document.getElementById('import-online-intro').l10nId = Zotero.Prefs.get("import.mendeleyUseOAuth") + ? 'import-online-intro' + : 'import-online-form-intro'; + document.getElementById('mendeley-login').style.display = Zotero.Prefs.get("import.mendeleyUseOAuth") ? 'none' : ''; + document.getElementById('mendeley-online-login-feedback').style.display = 'none'; + + // If we use oAuth, form doesn't show and we can advance, otherwise need to fill-in form first so disable + this.wizard.canAdvance = Zotero.Prefs.get("import.mendeleyUseOAuth"); + }, + + async onMendeleyOnlineAdvance(ev) { + ev.preventDefault(); + + if (Zotero.Prefs.get("import.mendeleyUseOAuth")) { + this.openMendeleyAuthWindow(); + } + else { + const userNameEl = document.getElementById('mendeley-username'); + const passwordEl = document.getElementById('mendeley-password'); + userNameEl.disabled = true; + passwordEl.disabled = true; + try { + this.mendeleyAuth = await directAuth(userNameEl.value, passwordEl.value); + this.wizard.goTo('page-options'); + } + catch (e) { + const feedbackEl = document.getElementById('mendeley-online-login-feedback'); + feedbackEl.style.display = ''; + this.wizard.canAdvance = false; // change to either of the inputs will reset thi + } + finally { + userNameEl.disabled = false; + passwordEl.disabled = false; + } + } + }, + + onMendeleyAuthKeyUp() { + document.getElementById('mendeley-online-login-feedback').style.display = 'none'; + this.wizard.canAdvance = document.getElementById('mendeley-username').value.length > 0 + && document.getElementById('mendeley-password').value.length > 0; + }, + async onImportSourceAdvance(ev) { const selectedMode = document.getElementById('import-source-group').selectedItem.value; ev.preventDefault(); @@ -226,13 +287,16 @@ const Zotero_Import_Wizard = { // eslint-disable-line no-unused-vars onOptionsPageShow() { document.getElementById('page-options-folder-import').style.display = this.folder ? 'block' : 'none'; - document.getElementById('page-options-file-handling').style.display = this.mendeleyCode ? 'none' : 'block'; + document.getElementById('page-options-file-handling').style.display = (this.mendeleyCode || this.mendeleyAuth) ? 'none' : 'block'; + const hideExtraMendeleyOptions = !this.mendeleyHasPreviouslyImported || !(this.mendeleyAuth || this.mendeleyCode); + document.getElementById('page-options-mendeley').style.display = hideExtraMendeleyOptions ? 'none' : 'block'; + if (hideExtraMendeleyOptions) { + document.getElementById('new-items-only-checkbox').checked = false; + } this.wizard.canRewind = false; }, - openMendeleyAuthWindow(ev) { - ev.preventDefault(); - + openMendeleyAuthWindow() { const arg = Components.classes["@mozilla.org/supports-string;1"] .createInstance(Components.interfaces.nsISupportsString); arg.data = 'mendeleyImport'; @@ -317,6 +381,7 @@ const Zotero_Import_Wizard = { // eslint-disable-line no-unused-vars const fileTypes = document.getElementById('import-other').checked ? document.getElementById('other-files').value : null; + const newItemsOnly = document.getElementById('new-items-only-checkbox').checked; try { const result = await Zotero_File_Interface.importFile({ @@ -325,8 +390,10 @@ const Zotero_Import_Wizard = { // eslint-disable-line no-unused-vars fileTypes, folder: this.folder, linkFiles, + mendeleyAuth: this.mendeleyAuth, mendeleyCode: this.mendeleyCode, mimeTypes, + newItemsOnly, onBeforeImport: this.onBeforeImport.bind(this), recreateStructure }); @@ -345,12 +412,12 @@ const Zotero_Import_Wizard = { // eslint-disable-line no-unused-vars } catch (e) { if (e.message == 'Encrypted Mendeley database') { - this.skipToDonePage('general.error', [], false, true); + this.skipToDonePage('general-error', [], false, true); } else { const translatorLabel = this.translation?.translator?.[0]?.label; this.skipToDonePage( - 'general.error', + 'general-error', translatorLabel ? ['file-interface-import-error-translator', { translator: translatorLabel }] : 'file-interface-import-error', diff --git a/chrome/content/zotero/import/importWizard.xhtml b/chrome/content/zotero/import/importWizard.xhtml index e25452794c..f912f57dc9 100644 --- a/chrome/content/zotero/import/importWizard.xhtml +++ b/chrome/content/zotero/import/importWizard.xhtml @@ -1,7 +1,7 @@ - + @@ -30,6 +30,7 @@ data-header-label-id="import-online-intro-title" >
@@ -37,6 +38,17 @@ class="mendeley-online-intro" data-l10n-id="import-online-intro2" data-l10n-args='{"targetApp": "Mendeley"}'> +
+
+ + +
+
+ + +
+
+
+
+ +
diff --git a/chrome/content/zotero/import/mendeley/mendeleyAPIUtils.js b/chrome/content/zotero/import/mendeley/mendeleyAPIUtils.js index bfd6982ff8..176c780f19 100644 --- a/chrome/content/zotero/import/mendeley/mendeleyAPIUtils.js +++ b/chrome/content/zotero/import/mendeley/mendeleyAPIUtils.js @@ -1,22 +1,67 @@ // eslint-disable-next-line no-unused-vars var mendeleyAPIUtils = (function () { -const OAUTH_URL = 'https://www.zotero.org/utils/mendeley/oauth'; +const ZOTERO_OAUTH_URL = 'https://www.zotero.org/utils/mendeley/oauth'; +const OAUTH_URL = 'https://api.mendeley.com/oauth/token'; const MENDELEY_API_URL = 'https://api.mendeley.com'; +const CLIENT_ID = '6'; +const CLIENT_NOT_VERY_SECRET = 'JtSAMzFdwC6RAED3RMZU'; +const USER_AGENT = 'Mendeley Desktop/1.18'; -const getTokens = async (codeOrRefreshToken, isRefresh = false) => { - const options = { - body: isRefresh - ? `grant_type=refresh_token&refresh_token=${codeOrRefreshToken}` - : `grant_type=authorization_code&code=${codeOrRefreshToken}`, - headers: { - 'Content-Type': 'application/x-www-form-urlencoded', - } - }; - const response = await Zotero.HTTP.request('POST', OAUTH_URL, options); +const getTokens = async (url, bodyProps, headers = {}, options = {}) => { + const body = Object.entries(bodyProps) + .map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`) + .join('&'); + headers = { ...headers, 'Content-Type': 'application/x-www-form-urlencoded' }; + + if (!Zotero.Prefs.get('import.mendeleyUseOAuth')) { + headers['User-Agent'] = USER_AGENT; + } + + options = { ...options, body, headers, timeout: 30000 }; + const response = await Zotero.HTTP.request('POST', url, options); return JSON.parse(response.responseText); }; +const directAuth = async (username, password, headers = {}, options = {}) => { + const bodyProps = { + client_id: CLIENT_ID, // eslint-disable-line camelcase + client_secret: CLIENT_NOT_VERY_SECRET, // eslint-disable-line camelcase + grant_type: 'password', // eslint-disable-line camelcase + password, + scope: 'all', + username + }; + + return getTokens(OAUTH_URL, bodyProps, headers, options); +}; + +const codeAuth = async (code, headers = {}, options = {}) => { + const bodyProps = { + grant_type: 'authorization_code', // eslint-disable-line camelcase + code, + }; + + return getTokens(ZOTERO_OAUTH_URL, bodyProps, headers, options); +}; + +const refreshAuth = async (refreshToken, headers = {}, options = {}) => { + const bodyProps = { + grant_type: 'refresh_token', // eslint-disable-line camelcase + refresh_token: refreshToken, // eslint-disable-line camelcase + }; + + if (!Zotero.Prefs.get('import.mendeleyUseOAuth')) { + bodyProps.client_id = CLIENT_ID; // eslint-disable-line camelcase + bodyProps.client_secret = CLIENT_NOT_VERY_SECRET; // eslint-disable-line camelcase + } + + return getTokens( + Zotero.Prefs.get('import.mendeleyUseOAuth') ? ZOTERO_OAUTH_URL : OAUTH_URL, + bodyProps, headers, options + ); +}; + const getNextLinkFromResponse = (response) => { let next = null; let links = response.getResponseHeader('link'); @@ -42,7 +87,7 @@ const apiFetchUrl = async (tokens, url, headers = {}, options = {}) => { } catch (e) { if (e.status === 401 || e.status === 403) { - const newTokens = await getTokens(tokens.refresh_token, true); + const newTokens = await refreshAuth(tokens.refresh_token); // update tokens in the tokens object and in the header for next request tokens.access_token = newTokens.access_token; // eslint-disable-line camelcase tokens.refresh_token = newTokens.refresh_token; // eslint-disable-line camelcase @@ -81,6 +126,5 @@ const getAll = async (tokens, endPoint, params = {}, headers = {}, options = {}, return data; }; - -return { getNextLinkFromResponse, getTokens, apiFetch, apiFetchUrl, get, getAll }; +return { codeAuth, directAuth, getNextLinkFromResponse, apiFetch, apiFetchUrl, get, getAll }; })(); diff --git a/chrome/content/zotero/import/mendeley/mendeleyImport.js b/chrome/content/zotero/import/mendeley/mendeleyImport.js index e7f0346324..a67c4917eb 100644 --- a/chrome/content/zotero/import/mendeley/mendeleyImport.js +++ b/chrome/content/zotero/import/mendeley/mendeleyImport.js @@ -10,7 +10,7 @@ Services.scriptloader.loadSubScript("chrome://zotero/content/import/mendeley/men Services.scriptloader.loadSubScript("chrome://zotero/content/import/mendeley/mendeleySchemaMap.js"); const { apiTypeToDBType, apiFieldToDBField } = mendeleyOnlineMappings; -const { apiFetch, get, getAll, getTokens } = mendeleyAPIUtils; +const { apiFetch, codeAuth, directAuth, get, getAll } = mendeleyAPIUtils; const colorMap = new Map(); colorMap.set('rgb(255, 245, 173)', '#ffd400'); @@ -28,7 +28,9 @@ var Zotero_Import_Mendeley = function () { this.createNewCollection = null; this.linkFiles = null; this.newItems = []; - this.mendeleyCode = null; + this.newCollections = []; + this.mendeleyAuth = null; + this.newItemsOnly = false; this._tokens = null; this._db = null; @@ -115,8 +117,11 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) { throw new Error("Not a valid Mendeley database"); } - if (this.mendeleyCode) { - this._tokens = await getTokens(this.mendeleyCode); + if (this.mendeleyAuth) { + this._tokens = this.mendeleyAuth; + } + else if (this.mendeleyCode) { + this._tokens = await codeAuth(this.mendeleyCode); } if (!this._file && !this._tokens) { @@ -303,9 +308,13 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) { annotations.get(document.id) ); } - this.newItems.push(Zotero.Items.get(documentIDMap.get(document.id))); this._interruptChecker(true); } + if (this.newItemsOnly && rootCollectionKey && this.newItems.length === 0) { + Zotero.debug(`Mendeley Import detected no new items, removing import collection containing ${this.newCollections.length} collections created during the import`); + const rootCollection = await Zotero.Collections.getAsync(options.collections[0]); + await rootCollection.eraseTx(this._saveOptions); + } Zotero.debug(`Completed Mendeley import in ${Math.round((Date.now() - this._started) / 1000)}s. (Started: ${this._started})`); } catch (e) { @@ -480,6 +489,7 @@ Zotero_Import_Mendeley.prototype._saveCollections = async function (libraryID, j collection.key = collectionJSON.key; await collection.loadPrimaryData(); } + this.newCollections.push(collection); } // Remove external ids before saving @@ -541,11 +551,19 @@ Zotero_Import_Mendeley.prototype._getDocumentsAPI = async function (groupID) { } - return (await getAll(this._tokens, 'documents', params, headers, {}, this._interruptChecker)).map(d => ({ - ...d, - uuid: d.id, - remoteUuid: d.id - })); + return (await getAll(this._tokens, 'documents', params, headers, {}, this._interruptChecker)).map(d => { + const processedDocument = { ...d, remoteUuid: d.id }; + + try { + const clientData = JSON.parse(d.client_data); + processedDocument.uuid = clientData.desktop_id ? clientData.desktop_id : d.id; + } + catch (_) { + processedDocument.uuid = d.id; + } + + return processedDocument; + }); }; /** @@ -1279,10 +1297,12 @@ Zotero_Import_Mendeley.prototype._saveItems = async function (libraryID, json) { var lastExistingParentItem; for (let i = 0; i < json.length; i++) { let itemJSON = json[i]; + let isMappedToExisting = false; // Check if the item has been previously imported let item = await this._findExistingItem(libraryID, itemJSON, lastExistingParentItem); if (item) { + isMappedToExisting = true; if (item.isRegularItem()) { lastExistingParentItem = item; @@ -1310,8 +1330,25 @@ Zotero_Import_Mendeley.prototype._saveItems = async function (libraryID, json) { // Remove external id before save let toSave = Object.assign({}, itemJSON); delete toSave.documentID; - - item.fromJSON(toSave); + + if ((this.newItemsOnly && !isMappedToExisting) || !this.newItemsOnly) { + if (isMappedToExisting) { + // dateAdded shouldn't change on an updated item. See #2881 + delete toSave.dateAdded; + } + item.fromJSON(toSave); + this.newItems.push(item); + } + else if (isMappedToExisting && toSave.relations) { + const predicate = 'mendeleyDB:documentUUID'; + const existingRels = item.getRelationsByPredicate(predicate); + const newRel = toSave.relations[predicate]; + if (existingRels.length && newRel && existingRels[0] !== newRel) { + Zotero.debug(`Migrating relation ${predicate} for existing item ${item.key} from ${existingRels[0]} to ${newRel}`); + item.removeRelation(predicate, existingRels[0]); + item.addRelation(predicate, newRel); + } + } await item.saveTx({ skipDateModifiedUpdate: true, ...this._saveOptions diff --git a/chrome/content/zotero/import/mendeley/mendeleyOnlineMappings.js b/chrome/content/zotero/import/mendeley/mendeleyOnlineMappings.js index 102437f1fd..9aa138dd4a 100644 --- a/chrome/content/zotero/import/mendeley/mendeleyOnlineMappings.js +++ b/chrome/content/zotero/import/mendeley/mendeleyOnlineMappings.js @@ -28,6 +28,7 @@ var mendeleyOnlineMappings = { arxiv: 'arxivId', accessed: 'dateAccessed', authors: false, // all author types handled separately + client_data: false, // desktop_id extraction handled separately citation_key: 'citationKey', created: 'added', edition: 'edition', diff --git a/chrome/locale/en-US/zotero/zotero.dtd b/chrome/locale/en-US/zotero/zotero.dtd index abec546bbb..5b18a80968 100644 --- a/chrome/locale/en-US/zotero/zotero.dtd +++ b/chrome/locale/en-US/zotero/zotero.dtd @@ -188,6 +188,7 @@ + diff --git a/chrome/locale/en-US/zotero/zotero.ftl b/chrome/locale/en-US/zotero/zotero.ftl index 309ab55edd..32da2f0b54 100644 --- a/chrome/locale/en-US/zotero/zotero.ftl +++ b/chrome/locale/en-US/zotero/zotero.ftl @@ -38,6 +38,10 @@ import-file-handling-store = import-file-handling-link = .label = Link to files in original location import-fileHandling-description = Linked files cannot be synced by { -app-name }. +import-online-new = + .label = Download new items only; don’t update previously imported items +import-mendeley-username = Username +import-mendeley-password = Password general-error = Error file-interface-import-error = An error occurred while trying to import the selected file. Please ensure that the file is valid and try again. @@ -52,11 +56,10 @@ import-mendeley-encrypted = The selected Mendeley database cannot be read, likel file-interface-import-error-translator = An error occurred importing the selected file with “{ $translator }”. Please ensure that the file is valid and try again. -# Variables: -# $targetAppOnline (String) -# $targetApp (String) import-online-intro=In the next step you will be asked to log in to { $targetAppOnline } and grant { -app-name } access. This is necessary to import your { $targetApp } library into { -app-name }. import-online-intro2={ -app-name } will never see or store your { $targetApp } password. +import-online-form-intro = Please enter your credentials to log in to { $targetAppOnline }. This is necessary to import your { $targetApp } library into { -app-name }. +import-online-wrong-credentials = Login to { $targetApp } failed. Please re-enter credentials and try again. report-error = .label = Report Error… diff --git a/chrome/locale/en-US/zotero/zotero.properties b/chrome/locale/en-US/zotero/zotero.properties index 07398f228d..25e339b113 100644 --- a/chrome/locale/en-US/zotero/zotero.properties +++ b/chrome/locale/en-US/zotero/zotero.properties @@ -93,6 +93,8 @@ general.loading = Loading… general.richText = Rich Text general.clearSelection = Clear Selection general.insert = Insert +general.username = Username +general.password = Password general.yellow = Yellow general.red = Red diff --git a/scss/components/_import-wizard.scss b/scss/components/_import-wizard.scss index f485ca8cf4..3794c3bd99 100644 --- a/scss/components/_import-wizard.scss +++ b/scss/components/_import-wizard.scss @@ -106,6 +106,46 @@ } } + #mendeley-login { + padding: 1.5em 1em 0; + font-size: 13px; + display: flex; + flex-direction: column; + align-items: center; + border: none; + + .field { + display: flex; + max-width: 300px; + line-height: 1.5em; + align-items: center; + } + + .field+.field { + margin-top: .5em; + } + + label { + flex: 0 0 90px; + width: 90px; + margin-right: 8px; + text-align: right; + } + + input { + flex: 1 1 auto; + font-size: 13px; + } + } + + #mendeley-online-login-feedback { + text-align: center; + font-size: 13px; + margin-top: 1.3em; + color: red; + font-weight: bold; + } + #import-progress { margin-top: 1em; } diff --git a/test/tests/data/mendeleyMock/items-simple-no-desktop-id.json b/test/tests/data/mendeleyMock/items-simple-no-desktop-id.json new file mode 100644 index 0000000000..ff3aa22494 --- /dev/null +++ b/test/tests/data/mendeleyMock/items-simple-no-desktop-id.json @@ -0,0 +1,88 @@ +[ + { + "authored": false, + "confirmed": true, + "created": "2021-11-02T09:54:28.353Z", + "file_attached": false, + "hidden": false, + "id": "7fea3cb3-f97d-3f16-8fad-f59caaa71688", + "last_modified": "2021-11-02T12:26:30.025Z", + "private_publication": false, + "profile_id": "8dbf0832-8723-4c48-b532-20c0b7f6e01a", + "read": false, + "source": "lorem ipsum", + "identifiers": + { + "doi": "10.1111", + "pmid": "PMID: 11111111", + "arxiv": "1111.2222" + }, + "starred": false, + "title": "Foo Bar", + "authors": + [ + { + "first_name": "Tom", + "last_name": "Najdek" + }, + { + "first_name": "Lorem", + "last_name": "Ipsum" + } + ], + "type": "journal", + "folder_uuids": [ + "8d2f262d-49b3-4dfc-8968-0bb71bcd92ea" + ], + "year": 1987 + }, { + "authored": false, + "confirmed": true, + "created": "2021-11-04T11:53:10.353Z", + "file_attached": false, + "hidden": false, + "id": "07a74c26-28d1-4d9f-a60d-3f3bc5ef76ef", + "last_modified": "2021-11-04T11:53:10.353Z", + "private_publication": false, + "profile_id": "8dbf0832-8723-4c48-b532-20c0b7f6e01a", + "read": false, + "starred": false, + "title": "Sample Report", + "type": "report", + "year": 2002 + }, { + "title": "Item with PDF", + "type": "journal", + "year": 2005, + "source": "Zotero", + "pages": "1-11", + "websites": + [ + "https://zotero.org" + ], + "id": "c54b0c6f-c4ce-4706-8742-bc7d032df862", + "created": "2021-11-09T10:26:15.201Z", + "file_attached": true, + "profile_id": "8dbf0832-8723-4c48-b532-20c0b7f6e01a", + "last_modified": "2021-11-09T10:26:16.303Z", + "read": false, + "starred": false, + "authored": false, + "confirmed": true, + "hidden": false, + "private_publication": false, + "abstract": "Lorem Ipsum. Nostrud elit ullamco laborum cillum.", + "files": + [ + { + "id": "19fb5e5b-1a39-4851-b513-d48441a670e1", + "document_id": "c54b0c6f-c4ce-4706-8742-bc7d032df862", + "mime_type": "application/pdf", + "file_name": "item.pdf", + "size": 123456, + "created": "2021-11-09T10:26:16.292Z", + "filehash": "cc22c6611277df346ff8dc7386ba3880b2bafa15" + } + ] + } +] \ No newline at end of file diff --git a/test/tests/data/mendeleyMock/items-simple.json b/test/tests/data/mendeleyMock/items-simple.json index ff3aa22494..4e0da03e3f 100644 --- a/test/tests/data/mendeleyMock/items-simple.json +++ b/test/tests/data/mendeleyMock/items-simple.json @@ -6,6 +6,7 @@ "file_attached": false, "hidden": false, "id": "7fea3cb3-f97d-3f16-8fad-f59caaa71688", + "client_data": "{\"desktop_id\":\"b5f57b1a-f083-486c-aec7-5d5edd366dd2\"}", "last_modified": "2021-11-02T12:26:30.025Z", "private_publication": false, "profile_id": "8dbf0832-8723-4c48-b532-20c0b7f6e01a", @@ -42,6 +43,7 @@ "file_attached": false, "hidden": false, "id": "07a74c26-28d1-4d9f-a60d-3f3bc5ef76ef", + "client_data": "{\"desktop_id\":\"616ec6d1-8d23-4414-8b6e-7bb129677577\"}", "last_modified": "2021-11-04T11:53:10.353Z", "private_publication": false, "profile_id": "8dbf0832-8723-4c48-b532-20c0b7f6e01a", @@ -61,6 +63,7 @@ "https://zotero.org" ], "id": "c54b0c6f-c4ce-4706-8742-bc7d032df862", + "client_data": "{\"desktop_id\":\"3630a4bf-d97e-46c4-8611-61ec50f840c6\"}", "created": "2021-11-09T10:26:15.201Z", "file_attached": true, "profile_id": "8dbf0832-8723-4c48-b532-20c0b7f6e01a", diff --git a/test/tests/data/mendeleyMock/items-updated.json b/test/tests/data/mendeleyMock/items-updated.json index 2492ac5477..4165e33e37 100644 --- a/test/tests/data/mendeleyMock/items-updated.json +++ b/test/tests/data/mendeleyMock/items-updated.json @@ -2,10 +2,11 @@ { "authored": false, "confirmed": true, - "created": "2021-11-04T11:53:10.353Z", + "created": "2021-12-05T12:00:00.000Z", "file_attached": false, "hidden": false, "id": "07a74c26-28d1-4d9f-a60d-3f3bc5ef76ef", + "client_data": "{\"desktop_id\":\"616ec6d1-8d23-4414-8b6e-7bb129677577\"}", "last_modified": "2021-11-05T11:53:10.353Z", "private_publication": false, "profile_id": "8dbf0832-8723-4c48-b532-20c0b7f6e01a", @@ -15,5 +16,23 @@ "type": "journal", "source": "lorem ipsum", "year": 2002 + }, + { + "authored": false, + "confirmed": true, + "created": "2021-11-05T12:33:18.353Z", + "file_attached": false, + "hidden": false, + "id": "31a8251f-88b0-497b-9d30-1b2516771057", + "client_data": "{\"desktop_id\":\"86e56a00-5ae5-4fe8-a977-9298a03b16d6\"}", + "last_modified": "2021-11-05T12:33:18.353Z", + "private_publication": false, + "profile_id": "8dbf0832-8723-4c48-b532-20c0b7f6e01a", + "read": false, + "starred": true, + "title": "Completely new item", + "type": "book", + "source": "lorem ipsum", + "year": 1999 } ] \ No newline at end of file diff --git a/test/tests/mendeleyImportTest.js b/test/tests/mendeleyImportTest.js index 8eb7936656..d176806ff0 100644 --- a/test/tests/mendeleyImportTest.js +++ b/test/tests/mendeleyImportTest.js @@ -1,12 +1,17 @@ /* global setHTTPResponse:false, sinon: false, Zotero_Import_Mendeley: false, HttpServer: false */ describe('Zotero_Import_Mendeley', function () { - var server, importer, httpd, httpdURL; + var server, httpd, httpdURL, importers; + + const getImporter = () => { + const importer = new Zotero_Import_Mendeley(); + importer.mendeleyAuth = { access_token: 'access_token', refresh_token: 'refresh_token' };// eslint-disable-line camelcase + importers.push(importer); + return importer; + }; before(async () => { Components.utils.import('chrome://zotero/content/import/mendeley/mendeleyImport.js'); - importer = new Zotero_Import_Mendeley(); - importer.mendeleyCode = 'CODE'; // real http server is used to deliver an empty pdf so that annotations can be processed during import Components.utils.import("resource://zotero-unit/httpd.js"); @@ -25,12 +30,15 @@ describe('Zotero_Import_Mendeley', function () { }); beforeEach(async () => { + importers = []; Zotero.HTTP.mock = sinon.FakeXMLHttpRequest; - server = sinon.fakeServer.create(); + server = sinon.fakeServer.create({ + unsafeHeadersEnabled: false + }); server.autoRespond = true; - setHTTPResponse(server, 'https://www.zotero.org/', { + setHTTPResponse(server, 'https://api.mendeley.com/', { method: 'POST', - url: `utils/mendeley/oauth`, + url: `oauth/token`, status: 200, headers: {}, json: { @@ -127,12 +135,21 @@ describe('Zotero_Import_Mendeley', function () { }); }); - afterEach(() => { + afterEach(async () => { + await Promise.all( + importers + .map(importer => ([ + Zotero.Items.erase(Array.from(new Set(importer.newItems)).map(i => i.id)), + Zotero.Collections.erase(Array.from(new Set(importer.newCollections)).map(c => c.id)) + ])) + .reduce((prev, a) => ([...prev, ...a]), []) // .flat() in >= FF62 + ); Zotero.HTTP.mock = null; }); describe('#import', () => { it("should import collections, items, attachments & annotations", async () => { + const importer = getImporter(); await importer.translate({ libraryID: Zotero.Libraries.userLibraryID, collections: null, @@ -140,17 +157,17 @@ describe('Zotero_Import_Mendeley', function () { }); const journal = (await Zotero.Relations - .getByPredicateAndObject('item', 'mendeleyDB:documentUUID', '7fea3cb3-f97d-3f16-8fad-f59caaa71688')) + .getByPredicateAndObject('item', 'mendeleyDB:documentUUID', 'b5f57b1a-f083-486c-aec7-5d5edd366dd2')) .filter(item => item.libraryID == Zotero.Libraries.userLibraryID && !item.deleted) .shift(); const report = (await Zotero.Relations - .getByPredicateAndObject('item', 'mendeleyDB:documentUUID', '07a74c26-28d1-4d9f-a60d-3f3bc5ef76ef')) + .getByPredicateAndObject('item', 'mendeleyDB:documentUUID', '616ec6d1-8d23-4414-8b6e-7bb129677577')) .filter(item => item.libraryID == Zotero.Libraries.userLibraryID && !item.deleted) .shift(); const withpdf = (await Zotero.Relations - .getByPredicateAndObject('item', 'mendeleyDB:documentUUID', 'c54b0c6f-c4ce-4706-8742-bc7d032df862')) + .getByPredicateAndObject('item', 'mendeleyDB:documentUUID', '3630a4bf-d97e-46c4-8611-61ec50f840c6')) .filter(item => item.libraryID == Zotero.Libraries.userLibraryID && !item.deleted) .shift(); @@ -159,10 +176,14 @@ describe('Zotero_Import_Mendeley', function () { .filter(item => item.libraryID == Zotero.Libraries.userLibraryID && !item.deleted) .shift(); + + assert.equal(journal.getRelations()['mendeleyDB:remoteDocumentUUID'], '7fea3cb3-f97d-3f16-8fad-f59caaa71688'); assert.equal(journal.getField('title'), 'Foo Bar'); assert.equal(journal.itemTypeID, Zotero.ItemTypes.getID('journalArticle')); + assert.equal(report.getRelations()['mendeleyDB:remoteDocumentUUID'], '07a74c26-28d1-4d9f-a60d-3f3bc5ef76ef'); assert.equal(report.getField('title'), 'Sample Report'); assert.equal(report.itemTypeID, Zotero.ItemTypes.getID('report')); + assert.equal(withpdf.getRelations()['mendeleyDB:remoteDocumentUUID'], 'c54b0c6f-c4ce-4706-8742-bc7d032df862'); assert.equal(withpdf.getField('title'), 'Item with PDF'); assert.equal(withpdf.itemTypeID, Zotero.ItemTypes.getID('journalArticle')); @@ -237,22 +258,23 @@ describe('Zotero_Import_Mendeley', function () { assert.equal(parentCollection.name, 'folder1'); }); - it("should update previously imported item", async () => { - const importer = new Zotero_Import_Mendeley(); - importer.mendeleyCode = 'CODE'; - await importer.translate({ + it("should update previously imported item, based on config", async () => { + const importer1 = getImporter(); + await importer1.translate({ libraryID: Zotero.Libraries.userLibraryID, collections: null, linkFiles: false, }); const report = (await Zotero.Relations - .getByPredicateAndObject('item', 'mendeleyDB:documentUUID', '07a74c26-28d1-4d9f-a60d-3f3bc5ef76ef')) + .getByPredicateAndObject('item', 'mendeleyDB:documentUUID', '616ec6d1-8d23-4414-8b6e-7bb129677577')) .filter(item => item.libraryID == Zotero.Libraries.userLibraryID && !item.deleted) .shift(); + assert.equal(report.getField('title'), 'Sample Report'); assert.equal(report.getField('year'), '2002'); + assert.equal(report.getField('dateAdded'), '2021-11-04 11:53:10'); assert.equal(report.itemTypeID, Zotero.ItemTypes.getID('report')); assert.lengthOf(report.getTags(), 0); @@ -266,7 +288,9 @@ describe('Zotero_Import_Mendeley', function () { ) }); - await importer.translate({ + const importer2 = getImporter(); + importer2.newItemsOnly = false; + await importer2.translate({ libraryID: Zotero.Libraries.userLibraryID, collections: null, linkFiles: false, @@ -276,6 +300,107 @@ describe('Zotero_Import_Mendeley', function () { assert.equal(report.itemTypeID, Zotero.ItemTypes.getID('journalArticle')); assert.equal(report.getField('year'), '2002'); assert.sameMembers(report.getTags().map(t => t.tag), ['\u2605']); + // dateAdded shouldn't change on an updated item. See #2881 + assert.equal(report.getField('dateAdded'), '2021-11-04 11:53:10'); + }); + + it("shouldn't update previously imported item, based on config", async () => { + const importer1 = getImporter(); + await importer1.translate({ + libraryID: Zotero.Libraries.userLibraryID, + collections: null, + linkFiles: false, + }); + + const report = (await Zotero.Relations + .getByPredicateAndObject('item', 'mendeleyDB:documentUUID', '616ec6d1-8d23-4414-8b6e-7bb129677577')) + .filter(item => item.libraryID == Zotero.Libraries.userLibraryID && !item.deleted) + .shift(); + + const noNewItemHere = await Zotero.Relations.getByPredicateAndObject('item', 'mendeleyDB:documentUUID', '86e56a00-5ae5-4fe8-a977-9298a03b16d6'); + + + assert.equal(report.getField('title'), 'Sample Report'); + assert.equal(report.getField('year'), '2002'); + assert.equal(report.itemTypeID, Zotero.ItemTypes.getID('report')); + assert.lengthOf(report.getTags(), 0); + assert.lengthOf(noNewItemHere, 0); + + setHTTPResponse(server, 'https://api.mendeley.com/', { + method: 'GET', + url: `documents?view=all&limit=500`, + status: 200, + headers: {}, + json: JSON.parse( + await Zotero.File.getContentsFromURLAsync('resource://zotero-unit-tests/data/mendeleyMock/items-updated.json') + ) + }); + + const importer2 = getImporter(); + importer2.newItemsOnly = true; + await importer2.translate({ + libraryID: Zotero.Libraries.userLibraryID, + collections: null, + linkFiles: false, + }); + + assert.equal(report.getField('title'), 'Sample Report'); + assert.equal(report.itemTypeID, Zotero.ItemTypes.getID('report')); + assert.equal(report.getField('year'), '2002'); + assert.lengthOf(report.getTags(), 0); + + const newItem = (await Zotero.Relations + .getByPredicateAndObject('item', 'mendeleyDB:documentUUID', '86e56a00-5ae5-4fe8-a977-9298a03b16d6')) + .filter(item => item.libraryID == Zotero.Libraries.userLibraryID && !item.deleted) + .shift(); + + assert.equal(newItem.getField('title'), 'Completely new item'); + }); + + it("should correct IDs if available on subsequent import", async () => { + setHTTPResponse(server, 'https://api.mendeley.com/', { + method: 'GET', + url: `documents?view=all&limit=500`, + status: 200, + headers: {}, + json: JSON.parse( + await Zotero.File.getContentsFromURLAsync('resource://zotero-unit-tests/data/mendeleyMock/items-simple-no-desktop-id.json') + ) + }); + const importer = getImporter(); + importer.newItemsOnly = true; + await importer.translate({ + libraryID: Zotero.Libraries.userLibraryID, + collections: null, + linkFiles: false, + }); + + const report = (await Zotero.Relations + .getByPredicateAndObject('item', 'mendeleyDB:remoteDocumentUUID', '07a74c26-28d1-4d9f-a60d-3f3bc5ef76ef')) + .filter(item => item.libraryID == Zotero.Libraries.userLibraryID && !item.deleted) + .shift(); + + assert.equal(report.getField('title'), 'Sample Report'); + assert.equal(report.getRelations()['mendeleyDB:documentUUID'], '07a74c26-28d1-4d9f-a60d-3f3bc5ef76ef'); + + setHTTPResponse(server, 'https://api.mendeley.com/', { + method: 'GET', + url: `documents?view=all&limit=500`, + status: 200, + headers: {}, + json: JSON.parse( + await Zotero.File.getContentsFromURLAsync('resource://zotero-unit-tests/data/mendeleyMock/items-simple.json') + ) + }); + + await importer.translate({ + libraryID: Zotero.Libraries.userLibraryID, + collections: null, + linkFiles: false, + }); + + assert.equal(report.getField('title'), 'Sample Report'); + assert.equal(report.getRelations()['mendeleyDB:documentUUID'], '616ec6d1-8d23-4414-8b6e-7bb129677577'); }); }); }); From 1923085068895013201849ac8136b9e342fe758a Mon Sep 17 00:00:00 2001 From: Tom Najdek Date: Wed, 9 Nov 2022 14:19:52 +0100 Subject: [PATCH 02/11] Block Mendeley import if ZotFile installed --- chrome/content/zotero/import/importWizard.js | 11 +++++++++++ chrome/locale/en-US/zotero/zotero.ftl | 1 + 2 files changed, 12 insertions(+) diff --git a/chrome/content/zotero/import/importWizard.js b/chrome/content/zotero/import/importWizard.js index 3f42467a3d..0b4ab7af4b 100644 --- a/chrome/content/zotero/import/importWizard.js +++ b/chrome/content/zotero/import/importWizard.js @@ -38,6 +38,7 @@ const { directAuth } = mendeleyAPIUtils; const Zotero_Import_Wizard = { // eslint-disable-line no-unused-vars file: null, folder: null, + isZotfileInstalled: false, libraryID: null, mendeleyAuth: null, mendeleyCode: null, @@ -69,6 +70,9 @@ const Zotero_Import_Wizard = { // eslint-disable-line no-unused-vars this.mendeleyHasPreviouslyImported = !!(await Zotero.DB.valueQueryAsync(relSQL, predicateID)); } + const extensions = await Zotero.getInstalledExtensions(); + this.isZotfileInstalled = !!extensions.find(extName => extName.match(/^ZotFile((?!disabled).)*$/)); + this.wizard = document.getElementById('import-wizard'); this.wizard.getPageById('page-start') .addEventListener('pageadvanced', this.onImportSourceAdvance.bind(this)); @@ -272,6 +276,13 @@ const Zotero_Import_Wizard = { // eslint-disable-line no-unused-vars case 'mendeleyOnline': this.file = null; this.folder = null; + if (this.isZotfileInstalled) { + this.skipToDonePage( + 'general-error', + ['import-online-blocked-by-plugin', { plugin: 'ZotFile' }] + ); + return; + } this.wizard.goTo('page-mendeley-online-intro'); this.wizard.canRewind = true; break; diff --git a/chrome/locale/en-US/zotero/zotero.ftl b/chrome/locale/en-US/zotero/zotero.ftl index 32da2f0b54..2316d11b62 100644 --- a/chrome/locale/en-US/zotero/zotero.ftl +++ b/chrome/locale/en-US/zotero/zotero.ftl @@ -60,6 +60,7 @@ import-online-intro=In the next step you will be asked to log in to { $targetApp import-online-intro2={ -app-name } will never see or store your { $targetApp } password. import-online-form-intro = Please enter your credentials to log in to { $targetAppOnline }. This is necessary to import your { $targetApp } library into { -app-name }. import-online-wrong-credentials = Login to { $targetApp } failed. Please re-enter credentials and try again. +import-online-blocked-by-plugin = The import cannot continue with { $plugin } installed. Please disable this plugin and try again. report-error = .label = Report Error… From 80bdf51ecb536a5953660204155933e70c932af7 Mon Sep 17 00:00:00 2001 From: Dan Stillman Date: Tue, 13 Dec 2022 23:29:27 -0700 Subject: [PATCH 03/11] Store Mendeley importer version number in database --- chrome/content/zotero/import/mendeley/mendeleyImport.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/chrome/content/zotero/import/mendeley/mendeleyImport.js b/chrome/content/zotero/import/mendeley/mendeleyImport.js index a67c4917eb..532416289c 100644 --- a/chrome/content/zotero/import/mendeley/mendeleyImport.js +++ b/chrome/content/zotero/import/mendeley/mendeleyImport.js @@ -9,6 +9,7 @@ Services.scriptloader.loadSubScript("chrome://zotero/content/import/mendeley/men Services.scriptloader.loadSubScript("chrome://zotero/content/import/mendeley/mendeleyAPIUtils.js"); Services.scriptloader.loadSubScript("chrome://zotero/content/import/mendeley/mendeleySchemaMap.js"); +const importerVersion = 1; const { apiTypeToDBType, apiFieldToDBField } = mendeleyOnlineMappings; const { apiFetch, codeAuth, directAuth, get, getAll } = mendeleyAPIUtils; @@ -315,6 +316,9 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) { const rootCollection = await Zotero.Collections.getAsync(options.collections[0]); await rootCollection.eraseTx(this._saveOptions); } + + await Zotero.DB.queryAsync("REPLACE INTO settings VALUES ('mendeleyImport', 'version', ?)", importerVersion); + Zotero.debug(`Completed Mendeley import in ${Math.round((Date.now() - this._started) / 1000)}s. (Started: ${this._started})`); } catch (e) { From 197d8d1f3b9bc768bf6c2e4138ad419762c21f7e Mon Sep 17 00:00:00 2001 From: Tom Najdek Date: Thu, 15 Dec 2022 19:35:29 +0100 Subject: [PATCH 04/11] Add option to Mendeley importer to relink items New option only appears if importer version is < 1 or not present. It will: * Skip fetching collections and attachments * Skip any new items * Update relations on existing items --- chrome/content/zotero/fileInterface.js | 1 + chrome/content/zotero/import/importWizard.js | 19 ++++++- .../content/zotero/import/importWizard.xhtml | 3 ++ .../zotero/import/mendeley/mendeleyImport.js | 30 +++++++---- chrome/locale/en-US/zotero/zotero.ftl | 2 + scss/components/_import-wizard.scss | 4 ++ test/tests/mendeleyImportTest.js | 51 +++++++++++++++++++ 7 files changed, 99 insertions(+), 11 deletions(-) diff --git a/chrome/content/zotero/fileInterface.js b/chrome/content/zotero/fileInterface.js index 5caba74f60..9afe469e9b 100644 --- a/chrome/content/zotero/fileInterface.js +++ b/chrome/content/zotero/fileInterface.js @@ -450,6 +450,7 @@ var Zotero_File_Interface = new function() { translation.mendeleyAuth = options.mendeleyAuth; translation.mendeleyCode = options.mendeleyCode; translation.newItemsOnly = options.newItemsOnly; + translation.relinkOnly = options.relinkOnly; } else if (options.folder) { Components.utils.import("chrome://zotero/content/import/folderImport.js"); diff --git a/chrome/content/zotero/import/importWizard.js b/chrome/content/zotero/import/importWizard.js index 0b4ab7af4b..eeac19fdc8 100644 --- a/chrome/content/zotero/import/importWizard.js +++ b/chrome/content/zotero/import/importWizard.js @@ -43,6 +43,7 @@ const Zotero_Import_Wizard = { // eslint-disable-line no-unused-vars mendeleyAuth: null, mendeleyCode: null, mendeleyHasPreviouslyImported: false, + mendeleyImporterVersion: 0, translation: null, wizard: null, @@ -72,6 +73,7 @@ const Zotero_Import_Wizard = { // eslint-disable-line no-unused-vars const extensions = await Zotero.getInstalledExtensions(); this.isZotfileInstalled = !!extensions.find(extName => extName.match(/^ZotFile((?!disabled).)*$/)); + this.mendeleyImporterVersion = parseInt((await Zotero.DB.valueQueryAsync("SELECT value FROM settings WHERE setting='mendeleyImport' AND key='version'")) || 0); this.wizard = document.getElementById('import-wizard'); this.wizard.getPageById('page-start') @@ -110,6 +112,8 @@ const Zotero_Import_Wizard = { // eslint-disable-line no-unused-vars .getElementById('mendeley-username').addEventListener('keyup', this.onMendeleyAuthKeyUp.bind(this)); document .getElementById('mendeley-password').addEventListener('keyup', this.onMendeleyAuthKeyUp.bind(this)); + document + .getElementById('relink-only-checkbox').addEventListener('command', this.onRelinkOnlyChange.bind(this)); this.wizard.addEventListener('pageshow', this.updateFocus.bind(this)); this.wizard.addEventListener('wizardcancel', this.onCancel.bind(this)); @@ -301,6 +305,7 @@ const Zotero_Import_Wizard = { // eslint-disable-line no-unused-vars document.getElementById('page-options-file-handling').style.display = (this.mendeleyCode || this.mendeleyAuth) ? 'none' : 'block'; const hideExtraMendeleyOptions = !this.mendeleyHasPreviouslyImported || !(this.mendeleyAuth || this.mendeleyCode); document.getElementById('page-options-mendeley').style.display = hideExtraMendeleyOptions ? 'none' : 'block'; + document.getElementById('page-options-relink-only').style.display = (hideExtraMendeleyOptions || this.mendeleyImporterVersion > 0) ? 'none' : null; if (hideExtraMendeleyOptions) { document.getElementById('new-items-only-checkbox').checked = false; } @@ -354,6 +359,16 @@ const Zotero_Import_Wizard = { // eslint-disable-line no-unused-vars } }, + onRelinkOnlyChange() { + if (document.getElementById('relink-only-checkbox').checked) { + document.getElementById('new-items-only-checkbox').checked = true; + document.getElementById('create-collection').checked = false; + } + + document.getElementById('new-items-only-checkbox').disabled = document.getElementById('relink-only-checkbox').checked; + document.getElementById('create-collection').disabled = document.getElementById('relink-only-checkbox').checked; + }, + onURLInteract(ev) { if (ev.type === 'click' || (ev.type === 'keydown' && ev.key === ' ')) { Zotero.launchURL(ev.currentTarget.getAttribute('href')); @@ -393,6 +408,7 @@ const Zotero_Import_Wizard = { // eslint-disable-line no-unused-vars ? document.getElementById('other-files').value : null; const newItemsOnly = document.getElementById('new-items-only-checkbox').checked; + const relinkOnly = document.getElementById('relink-only-checkbox').checked; try { const result = await Zotero_File_Interface.importFile({ @@ -406,7 +422,8 @@ const Zotero_Import_Wizard = { // eslint-disable-line no-unused-vars mimeTypes, newItemsOnly, onBeforeImport: this.onBeforeImport.bind(this), - recreateStructure + recreateStructure, + relinkOnly }); // Cancelled by user or due to error diff --git a/chrome/content/zotero/import/importWizard.xhtml b/chrome/content/zotero/import/importWizard.xhtml index f912f57dc9..836bd172e9 100644 --- a/chrome/content/zotero/import/importWizard.xhtml +++ b/chrome/content/zotero/import/importWizard.xhtml @@ -54,6 +54,9 @@ pageid="page-options" data-header-label-id="import-options" > +
+ +
diff --git a/chrome/content/zotero/import/mendeley/mendeleyImport.js b/chrome/content/zotero/import/mendeley/mendeleyImport.js index 532416289c..4703808de2 100644 --- a/chrome/content/zotero/import/mendeley/mendeleyImport.js +++ b/chrome/content/zotero/import/mendeley/mendeleyImport.js @@ -32,6 +32,7 @@ var Zotero_Import_Mendeley = function () { this.newCollections = []; this.mendeleyAuth = null; this.newItemsOnly = false; + this.relinkOnly = false; this._tokens = null; this._db = null; @@ -92,13 +93,15 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) { skipSelect: true, ...(options.saveOptions || {}) }; + + this.newItemsOnly = this.newItemsOnly || this.relinkOnly; const libraryID = options.libraryID || Zotero.Libraries.userLibraryID; const { key: rootCollectionKey } = options.collections ? Zotero.Collections.getLibraryAndKeyFromID(options.collections[0]) : {}; - Zotero.debug(`Begining Mendeley import at ${this._started}. libraryID: ${libraryID}, linkFiles: ${this.linkFiles}, rootCollectionKey: ${rootCollectionKey}`); + Zotero.debug(`Begining Mendeley import at ${this._started}. libraryID: ${libraryID}, linkFiles: ${this.linkFiles}, rootCollectionKey: ${rootCollectionKey}, newItemsOnly: ${this.newItemsOnly}, relinkOnly: ${this.relinkOnly}`); // TODO: Get appropriate version based on schema version const mapVersion = 83; @@ -135,14 +138,17 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) { this._progressMax = 50; this._itemDone(); - const folders = this._tokens - ? await this._getFoldersAPI(mendeleyGroupID) - : await this._getFoldersDB(mendeleyGroupID); + let folderKeys = new Map(); + if(!this.relinkOnly) { + const folders = this._tokens + ? await this._getFoldersAPI(mendeleyGroupID) + : await this._getFoldersDB(mendeleyGroupID); - const collectionJSON = this._foldersToAPIJSON(folders, rootCollectionKey); - const folderKeys = this._getFolderKeys(collectionJSON); + const collectionJSON = this._foldersToAPIJSON(folders, rootCollectionKey); + folderKeys = this._getFolderKeys(collectionJSON); - await this._saveCollections(libraryID, collectionJSON, folderKeys); + await this._saveCollections(libraryID, collectionJSON, folderKeys); + } this._interruptChecker(true); // @@ -175,13 +181,13 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) { this._interruptChecker(true); - let collections = this._tokens + let collections = this.relinkOnly ? new Map() : this._tokens ? await this._getDocumentCollectionsAPI(documents, rootCollectionKey, folderKeys) : await this._getDocumentCollectionsDB(mendeleyGroupID, documents, rootCollectionKey, folderKeys); this._interruptChecker(true); - - let files = this._tokens + + let files = this.relinkOnly ? new Map() : this._tokens ? await this._getDocumentFilesAPI(documents) : await this._getDocumentFilesDB(mendeleyGroupID); @@ -1330,6 +1336,10 @@ Zotero_Import_Mendeley.prototype._saveItems = async function (libraryID, json) { await item.loadPrimaryData(); } } + + if(this.relinkOnly && !isMappedToExisting) { + continue; + } // Remove external id before save let toSave = Object.assign({}, itemJSON); diff --git a/chrome/locale/en-US/zotero/zotero.ftl b/chrome/locale/en-US/zotero/zotero.ftl index 2316d11b62..e5b6130670 100644 --- a/chrome/locale/en-US/zotero/zotero.ftl +++ b/chrome/locale/en-US/zotero/zotero.ftl @@ -61,6 +61,8 @@ import-online-intro2={ -app-name } will never see or store your { $targetApp } p import-online-form-intro = Please enter your credentials to log in to { $targetAppOnline }. This is necessary to import your { $targetApp } library into { -app-name }. import-online-wrong-credentials = Login to { $targetApp } failed. Please re-enter credentials and try again. import-online-blocked-by-plugin = The import cannot continue with { $plugin } installed. Please disable this plugin and try again. +import-online-relink-only = + .label = Relink Mendeley Desktop citations report-error = .label = Report Error… diff --git a/scss/components/_import-wizard.scss b/scss/components/_import-wizard.scss index 3794c3bd99..13f9466082 100644 --- a/scss/components/_import-wizard.scss +++ b/scss/components/_import-wizard.scss @@ -42,6 +42,10 @@ margin: 0; } + [disabled="true"] .checkbox-label { + opacity: .5; + } + #other-files { margin-left: -1px; width: 400px; diff --git a/test/tests/mendeleyImportTest.js b/test/tests/mendeleyImportTest.js index d176806ff0..8c257cf9af 100644 --- a/test/tests/mendeleyImportTest.js +++ b/test/tests/mendeleyImportTest.js @@ -402,5 +402,56 @@ describe('Zotero_Import_Mendeley', function () { assert.equal(report.getField('title'), 'Sample Report'); assert.equal(report.getRelations()['mendeleyDB:documentUUID'], '616ec6d1-8d23-4414-8b6e-7bb129677577'); }); + + it("should only correct IDs and not add new items if \"relinkOnly\" is configured", async () => { + setHTTPResponse(server, 'https://api.mendeley.com/', { + method: 'GET', + url: `documents?view=all&limit=500`, + status: 200, + headers: {}, + json: JSON.parse( + await Zotero.File.getContentsFromURLAsync('resource://zotero-unit-tests/data/mendeleyMock/items-simple-no-desktop-id.json') + ) + }); + const importer1 = getImporter(); + await importer1.translate({ + libraryID: Zotero.Libraries.userLibraryID, + collections: null, + linkFiles: false, + }); + + const report = (await Zotero.Relations + .getByPredicateAndObject('item', 'mendeleyDB:remoteDocumentUUID', '07a74c26-28d1-4d9f-a60d-3f3bc5ef76ef')) + .filter(item => item.libraryID == Zotero.Libraries.userLibraryID && !item.deleted) + .shift(); + + assert.equal(report.getField('title'), 'Sample Report'); + assert.equal(report.getRelations()['mendeleyDB:documentUUID'], '07a74c26-28d1-4d9f-a60d-3f3bc5ef76ef'); + + setHTTPResponse(server, 'https://api.mendeley.com/', { + method: 'GET', + url: `documents?view=all&limit=500`, + status: 200, + headers: {}, + json: JSON.parse( + await Zotero.File.getContentsFromURLAsync('resource://zotero-unit-tests/data/mendeleyMock/items-updated.json') + ) + }); + + const importer2 = getImporter(); + importer2.relinkOnly = true; + await importer2.translate({ + libraryID: Zotero.Libraries.userLibraryID, + collections: null, + linkFiles: false, + }); + + assert.equal(report.getField('title'), 'Sample Report'); + assert.equal(report.getRelations()['mendeleyDB:documentUUID'], '616ec6d1-8d23-4414-8b6e-7bb129677577'); + + const noNewItemHere = await Zotero.Relations.getByPredicateAndObject('item', 'mendeleyDB:documentUUID', '86e56a00-5ae5-4fe8-a977-9298a03b16d6'); + assert.lengthOf(noNewItemHere, 0); + + }); }); }); From 9b938f22369dc4a67a055190f6379c9d22647c6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adomas=20Ven=C4=8Dkauskas?= Date: Wed, 14 Dec 2022 16:17:38 +0200 Subject: [PATCH 05/11] Add a nicer interface for prompts --- chrome/content/zotero/xpcom/prompt.js | 87 +++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 chrome/content/zotero/xpcom/prompt.js diff --git a/chrome/content/zotero/xpcom/prompt.js b/chrome/content/zotero/xpcom/prompt.js new file mode 100644 index 0000000000..f6c8e2a03d --- /dev/null +++ b/chrome/content/zotero/xpcom/prompt.js @@ -0,0 +1,87 @@ +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2022 Corporation for Digital Scholarship + Vienna, Virginia, USA + http://zotero.org + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + +var { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm"); + +Zotero.Prompt = { + BUTTON_TITLE_OK: Services.prompt.BUTTON_TITLE_OK, + BUTTON_TITLE_CANCEL: Services.prompt.BUTTON_TITLE_CANCEL, + BUTTON_TITLE_YES: Services.prompt.BUTTON_TITLE_YES, + BUTTON_TITLE_NO: Services.prompt.BUTTON_TITLE_NO, + BUTTON_TITLE_SAVE: Services.prompt.BUTTON_TITLE_SAVE, + BUTTON_TITLE_DONT_SAVE: Services.prompt.BUTTON_TITLE_DONT_SAVE, + BUTTON_TITLE_REVERT: Services.prompt.BUTTON_TITLE_REVERT, + + /** + * A wrapper around XPCOM's Services.prompt.confirmEx() + * but with a friendlier interface. + * + * Button text can use special static variables from + * Zotero.Prompt + * + * @param options + * - {mozIDOMWindowProxy} window - The parent window or null. + * - {String} title - Text to appear in the title of the dialog. + * - {String} text - Text to appear in the body of the dialog. + * - {String|Number} button0 - Button 0 text + * - {String|Number} button1 - Button 1 text + * - {String|Number} button2 - Button 2 text + * - {String} checkLabel - Text to appear with the checkbox. + * - {Object} checkbox - Contains the initial checked state of the + * checkbox when this method is called and the final checked + * state after this method returns. Either {} or { value: true/false }. + * - {Number} defaultButton - The index of default button. 0 by default + * - {Boolean} delayButtons - Make the buttons initially disabled and enable them after some period + * so that the user doesn't click through the dialog without reading it. + * @returns {Number} The index of the button pressed. + */ + confirm(options = {}) { + let { window: win, title, text, button0, button1, button2, checkLabel, checkbox, defaultButton, delayButtons } = options; + if (!win) win = null; + if (!title) throw new Error('`title` is required'); + if (!text) throw new Error('`text` is required'); + if (!button0 && !button1 && !button2) { + throw new Error('At least one button is required'); + } + if (checkLabel && (!checkbox || typeof checkbox != 'object')) { + throw new Error('`checkLabel` provided without `checkbox` option'); + } + let flags = delayButtons ? Services.prompt.BUTTON_DELAY_ENABLE : 0; + if (typeof button0 == 'number') flags += Services.prompt.BUTTON_POS_0 * button0; + else if (typeof button0 == 'string') flags += Services.prompt.BUTTON_POS_0 * Services.prompt.BUTTON_TITLE_IS_STRING; + if (typeof button1 == 'number') flags += Services.prompt.BUTTON_POS_1 * button1; + else if (typeof button1 == 'string') flags += Services.prompt.BUTTON_POS_1 * Services.prompt.BUTTON_TITLE_IS_STRING; + if (typeof button2 == 'number') flags += Services.prompt.BUTTON_POS_2 * button2; + else if (typeof button2 == 'string') flags += Services.prompt.BUTTON_POS_2 * Services.prompt.BUTTON_TITLE_IS_STRING; + if (defaultButton) flags += defaultButton == 1 ? Services.prompt.BUTTON_POS_1_DEFAULT : Services.prompt.BUTTON_POS_2_DEFAULT; + return Services.prompt.confirmEx( + win, title, text, flags, + typeof button0 == 'number' ? null : button0, + typeof button1 == 'number' ? null : button1, + typeof button2 == 'number' ? null : button2, + checkLabel, checkbox + ); + } +}; \ No newline at end of file From e10fc538d044dcbcc5088ca48d184e32facf0566 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adomas=20Ven=C4=8Dkauskas?= Date: Wed, 14 Dec 2022 16:20:04 +0200 Subject: [PATCH 06/11] Add a prompt to import Mendeley DB if mendeley citation found in doc --- chrome/content/zotero/import/importWizard.js | 6 ++- chrome/content/zotero/xpcom/integration.js | 39 ++++++++++++++++++-- chrome/locale/en-US/zotero/zotero.properties | 4 ++ components/zotero-service.js | 1 + defaults/preferences/zotero.js | 1 + 5 files changed, 46 insertions(+), 5 deletions(-) diff --git a/chrome/content/zotero/import/importWizard.js b/chrome/content/zotero/import/importWizard.js index eeac19fdc8..482bc31232 100644 --- a/chrome/content/zotero/import/importWizard.js +++ b/chrome/content/zotero/import/importWizard.js @@ -61,7 +61,7 @@ const Zotero_Import_Wizard = { // eslint-disable-line no-unused-vars }, async init() { - const { mendeleyCode, libraryID } = window.arguments[0].wrappedJSObject ?? {}; + const { mendeleyCode, libraryID, pageID } = window.arguments[0].wrappedJSObject ?? {}; this.libraryID = libraryID; @@ -125,6 +125,10 @@ const Zotero_Import_Wizard = { // eslint-disable-line no-unused-vars this.wizard.shadowRoot .querySelector('.wizard-header-label').style.fontSize = '16px'; + if (pageID) { + this.wizard.goTo(pageID); + } + if (mendeleyCode && Zotero.Prefs.get("import.mendeleyUseOAuth")) { this.mendeleyCode = mendeleyCode; this.wizard.goTo('page-options'); diff --git a/chrome/content/zotero/xpcom/integration.js b/chrome/content/zotero/xpcom/integration.js index 76a79a3b07..c3561710d3 100644 --- a/chrome/content/zotero/xpcom/integration.js +++ b/chrome/content/zotero/xpcom/integration.js @@ -2655,11 +2655,42 @@ Zotero.Integration.URIMap.prototype.getZoteroItemForURIs = async function (uris) replacer = await Zotero.Relations.getByPredicateAndObject( 'item', 'mendeleyDB:documentUUID', m[1] ); - if (replacer.length && !replacer[0].deleted) { - zoteroItem = replacer[0]; - break; + if (replacer.length) { + if (!replacer[0].deleted) { + zoteroItem = replacer[0]; + break; + } } - } + // If not blocked by user having pressed skip in this session, + // or user having checked the checkbox to not be prompted about this, + // or user having imported their library with the new version of importer + else if (!(this.session.dontPromptForMendeley + || Zotero.Prefs.get('integration.dontPromptMendeleyImport') + || await Zotero.DB.valueQueryAsync("SELECT value FROM settings WHERE setting='mendeleyImport' AND key='version'") + )) { + // Prompt user to (re)import their mendeley database which might make us recognize + // these items + let checkbox = {}; + let result = Zotero.Prompt.confirm({ + title: Zotero.getString('integration.mendeleyImport.title'), + text: Zotero.getString('integration.mendeleyImport.description', [Zotero.appName]), + button0: Zotero.getString('integration.mendeleyImport.openImporter'), + button1: Zotero.getString('general.skip'), + checkLabel: Zotero.getString('general.dontAskAgain'), + checkbox + }); + if (result === 0) { + setTimeout(() => Zotero.getMainWindow().Zotero_File_Interface.showImportWizard({ pageID: 'page-mendeley-online-intro' })); + throw new Zotero.Exception.UserCancelled("Importing mendeley citations"); + } + else { + this.session.dontPromptForMendeley = true; + } + if (checkbox.value) { + Zotero.Prefs.set('integration.dontPromptMendeleyImport', true); + } + } + }; } diff --git a/chrome/locale/en-US/zotero/zotero.properties b/chrome/locale/en-US/zotero/zotero.properties index 25e339b113..7cbe8ec7ac 100644 --- a/chrome/locale/en-US/zotero/zotero.properties +++ b/chrome/locale/en-US/zotero/zotero.properties @@ -73,6 +73,7 @@ general.tryLater = Try Later general.showDirectory = Show Directory general.showInLibrary = Show in Library general.continue = Continue +general.skip = Skip general.copy = Copy general.copyToClipboard = Copy to Clipboard general.cancel = Cancel @@ -992,6 +993,9 @@ integration.exportDocument.description1 = Zotero will convert citations in the d integration.exportDocument.description2 = You should make a backup of the document before proceeding. integration.importInstructions = The Zotero citations in this document have been converted to a format that can be safely transferred between word processors. Open this document in a supported word processor and press Refresh in the Zotero plugin to continue working with the citations. integration.upgradeTemplate = The %S plugin for %S is outdated. Reinstall the plugin from Preferences → Cite → Word Processors. +integration.mendeleyImport.title = Missing Mendeley Data +integration.mendeleyImport.description = %1$S detected that the document you are citing with contains Mendeley citations. %1$S will be able to manage these citations if you import your Mendeley database. +integration.mendeleyImport.openImporter = Open Mendeley Importer... styles.install.title = Install Style styles.install.unexpectedError = An unexpected error occurred while installing "%1$S" diff --git a/components/zotero-service.js b/components/zotero-service.js index f09160fb99..ae0ccc6d67 100644 --- a/components/zotero-service.js +++ b/components/zotero-service.js @@ -53,6 +53,7 @@ const xpcomFilesAll = [ 'mimeTypeHandler', 'pdfWorker/manager', 'ipc', + 'prompt', 'profile', 'progressWindow', 'proxy', diff --git a/defaults/preferences/zotero.js b/defaults/preferences/zotero.js index 124473cea2..59793b91ea 100644 --- a/defaults/preferences/zotero.js +++ b/defaults/preferences/zotero.js @@ -135,6 +135,7 @@ pref("extensions.zotero.integration.autoRegenerate", -1); // -1 = ask; 0 = no; 1 pref("extensions.zotero.integration.useClassicAddCitationDialog", false); pref("extensions.zotero.integration.keepAddCitationDialogRaised", false); pref("extensions.zotero.integration.upgradeTemplateDelayedOn", 0); +pref("extensions.zotero.integration.dontPromptMendeleyImport", false); // Connector settings pref("extensions.zotero.httpServer.enabled", true); From a6a69605c65744dd81e2f8a806e2e35361a5ae1f Mon Sep 17 00:00:00 2001 From: Dan Stillman Date: Sat, 17 Dec 2022 02:27:28 -0700 Subject: [PATCH 07/11] Mendeley citation relinking tweaks - Show "More information" link next to relink option - Automatically check relink option when coming from integration prompt - Change done message to "[x] items were relinked" instead of "0 items were imported" --- chrome/content/zotero/import/importWizard.js | 16 ++++++++++++---- chrome/content/zotero/import/importWizard.xhtml | 1 + .../zotero/import/mendeley/mendeleyImport.js | 4 +++- chrome/content/zotero/xpcom/integration.js | 9 ++++++++- chrome/locale/en-US/zotero/zotero.dtd | 1 + chrome/locale/en-US/zotero/zotero.ftl | 5 +++++ chrome/locale/en-US/zotero/zotero.properties | 1 + scss/components/_import-wizard.scss | 8 ++++++++ 8 files changed, 39 insertions(+), 6 deletions(-) diff --git a/chrome/content/zotero/import/importWizard.js b/chrome/content/zotero/import/importWizard.js index 482bc31232..dbf7c2d9cd 100644 --- a/chrome/content/zotero/import/importWizard.js +++ b/chrome/content/zotero/import/importWizard.js @@ -61,7 +61,7 @@ const Zotero_Import_Wizard = { // eslint-disable-line no-unused-vars }, async init() { - const { mendeleyCode, libraryID, pageID } = window.arguments[0].wrappedJSObject ?? {}; + const { mendeleyCode, libraryID, pageID, relinkOnly } = window.arguments[0].wrappedJSObject ?? {}; this.libraryID = libraryID; @@ -97,10 +97,10 @@ const Zotero_Import_Wizard = { // eslint-disable-line no-unused-vars document.getElementById('import-other').checked = ev.currentTarget.value.length > 0; }); document - .querySelector('#page-done-error-mendeley > a') + .querySelector('a') .addEventListener('click', this.onURLInteract.bind(this)); document - .querySelector('#page-done-error-mendeley > a') + .querySelector('a') .addEventListener('keydown', this.onURLInteract.bind(this)); document .querySelector('#page-done-error > button') @@ -125,6 +125,11 @@ const Zotero_Import_Wizard = { // eslint-disable-line no-unused-vars this.wizard.shadowRoot .querySelector('.wizard-header-label').style.fontSize = '16px'; + if (relinkOnly) { + document.getElementById('relink-only-checkbox').checked = true; + this.onRelinkOnlyChange(); + } + if (pageID) { this.wizard.goTo(pageID); } @@ -437,9 +442,12 @@ const Zotero_Import_Wizard = { // eslint-disable-line no-unused-vars } const numItems = this.translation.newItems.length; + const numRelinked = this.translation.numRelinked; this.skipToDonePage( 'file-interface-import-complete', - ['file-interface-items-were-imported', { numItems }] + document.getElementById('relink-only-checkbox').checked + ? ['file-interface-items-were-relinked', { numRelinked }] + : ['file-interface-items-were-imported', { numItems }] ); } catch (e) { diff --git a/chrome/content/zotero/import/importWizard.xhtml b/chrome/content/zotero/import/importWizard.xhtml index 836bd172e9..1bd9bf12e1 100644 --- a/chrome/content/zotero/import/importWizard.xhtml +++ b/chrome/content/zotero/import/importWizard.xhtml @@ -56,6 +56,7 @@ >
+
diff --git a/chrome/content/zotero/import/mendeley/mendeleyImport.js b/chrome/content/zotero/import/mendeley/mendeleyImport.js index 4703808de2..90f93822e1 100644 --- a/chrome/content/zotero/import/mendeley/mendeleyImport.js +++ b/chrome/content/zotero/import/mendeley/mendeleyImport.js @@ -33,6 +33,7 @@ var Zotero_Import_Mendeley = function () { this.mendeleyAuth = null; this.newItemsOnly = false; this.relinkOnly = false; + this.numRelinked = 0; this._tokens = null; this._db = null; @@ -1319,6 +1320,7 @@ Zotero_Import_Mendeley.prototype._saveItems = async function (libraryID, json) { // Update any child items to point to the existing item's key instead of the // new generated one this._updateParentKeys('item', json, i + 1, itemJSON.key, item.key); + this.numRelinked++; // Leave item in any collections it's in itemJSON.collections = item.getCollections() @@ -1337,7 +1339,7 @@ Zotero_Import_Mendeley.prototype._saveItems = async function (libraryID, json) { } } - if(this.relinkOnly && !isMappedToExisting) { + if (this.relinkOnly && !isMappedToExisting) { continue; } diff --git a/chrome/content/zotero/xpcom/integration.js b/chrome/content/zotero/xpcom/integration.js index c3561710d3..5096004c3e 100644 --- a/chrome/content/zotero/xpcom/integration.js +++ b/chrome/content/zotero/xpcom/integration.js @@ -2680,7 +2680,14 @@ Zotero.Integration.URIMap.prototype.getZoteroItemForURIs = async function (uris) checkbox }); if (result === 0) { - setTimeout(() => Zotero.getMainWindow().Zotero_File_Interface.showImportWizard({ pageID: 'page-mendeley-online-intro' })); + setTimeout( + () => Zotero.getMainWindow().Zotero_File_Interface.showImportWizard( + { + pageID: 'page-mendeley-online-intro', + relinkOnly: true + } + ) + ); throw new Zotero.Exception.UserCancelled("Importing mendeley citations"); } else { diff --git a/chrome/locale/en-US/zotero/zotero.dtd b/chrome/locale/en-US/zotero/zotero.dtd index 5b18a80968..9d3f4563a0 100644 --- a/chrome/locale/en-US/zotero/zotero.dtd +++ b/chrome/locale/en-US/zotero/zotero.dtd @@ -15,6 +15,7 @@ + diff --git a/chrome/locale/en-US/zotero/zotero.ftl b/chrome/locale/en-US/zotero/zotero.ftl index e5b6130670..8340988b4a 100644 --- a/chrome/locale/en-US/zotero/zotero.ftl +++ b/chrome/locale/en-US/zotero/zotero.ftl @@ -50,6 +50,10 @@ file-interface-items-were-imported = { $numItems -> [one] item was imported *[other] { $numItems } items were imported } +file-interface-items-were-relinked = { $numRelinked -> + [one] item was relinked + *[other] { $numRelinked } items were relinked + } import-mendeley-encrypted = The selected Mendeley database cannot be read, likely because it is encrypted. See How do I import a Mendeley library into Zotero? for more information. @@ -63,6 +67,7 @@ import-online-wrong-credentials = Login to { $targetApp } failed. Please re-ente import-online-blocked-by-plugin = The import cannot continue with { $plugin } installed. Please disable this plugin and try again. import-online-relink-only = .label = Relink Mendeley Desktop citations +import-online-relink-kb = More Information report-error = .label = Report Error… diff --git a/chrome/locale/en-US/zotero/zotero.properties b/chrome/locale/en-US/zotero/zotero.properties index 7cbe8ec7ac..2e721cb08f 100644 --- a/chrome/locale/en-US/zotero/zotero.properties +++ b/chrome/locale/en-US/zotero/zotero.properties @@ -799,6 +799,7 @@ dragAndDrop.filesNotFound = The following files were not found and could not be fileInterface.importing = Importing… fileInterface.importComplete = Import Complete fileInterface.itemsWereImported = %1$S item was imported;%1$S items were imported +fileInterface.itemsWereRelinked = %1$S item was relinked;%1$S items were relinked fileInterface.itemsExported = Exporting items… fileInterface.import = Import fileInterface.chooseAppDatabaseToImport = Choose the %S database to import diff --git a/scss/components/_import-wizard.scss b/scss/components/_import-wizard.scss index 13f9466082..4f883669e2 100644 --- a/scss/components/_import-wizard.scss +++ b/scss/components/_import-wizard.scss @@ -92,6 +92,14 @@ } } + #page-options-relink-only { + display: flex; + + a { + margin: 4px 1em 4px; + } + } + .page-options-file-handling-description { margin-top: .6em; font-size: 11px; From 96022847d7163f1925a7d214a99c57340919704e Mon Sep 17 00:00:00 2001 From: Tom Najdek Date: Wed, 8 Mar 2023 01:40:42 +0100 Subject: [PATCH 08/11] Mendeley importer: Fix issue with empty creators (#3016) It does not appear to be possible to create a creator with no values in Mendeley, however we got reports of these causing the imports to fail. This tweak makes the importer more resilient by discarding empty/invalid creators. --- .../zotero/import/mendeley/mendeleyImport.js | 7 +- .../mendeleyMock/items-empty-creators.json | 69 +++++++++++++++++++ test/tests/mendeleyImportTest.js | 34 +++++++++ 3 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 test/tests/data/mendeleyMock/items-empty-creators.json diff --git a/chrome/content/zotero/import/mendeley/mendeleyImport.js b/chrome/content/zotero/import/mendeley/mendeleyImport.js index 90f93822e1..4c3436c08c 100644 --- a/chrome/content/zotero/import/mendeley/mendeleyImport.js +++ b/chrome/content/zotero/import/mendeley/mendeleyImport.js @@ -636,7 +636,12 @@ Zotero_Import_Mendeley.prototype._getDocumentCreatorsAPI = async function (docum const authors = (doc.authors || []).map(c => this._makeCreator('author', c.first_name, c.last_name)); const editors = (doc.editors || []).map(c => this._makeCreator('editor', c.first_name, c.last_name)); const translators = (doc.translators || []).map(c => this._makeCreator('translator', c.first_name, c.last_name)); - map.set(doc.id, [...authors, ...editors, ...translators]); + const creators = [...authors, ...editors, ...translators]; + const validCreators = creators.filter(c => c.name || c.firstName || c.lastName); + if (creators.length !== validCreators.length) { + Zotero.debug(`Discarding ${creators.length - validCreators.length} invalid creators for document ${doc.id}`); + } + map.set(doc.id, validCreators); } return map; }; diff --git a/test/tests/data/mendeleyMock/items-empty-creators.json b/test/tests/data/mendeleyMock/items-empty-creators.json new file mode 100644 index 0000000000..c93c0d99eb --- /dev/null +++ b/test/tests/data/mendeleyMock/items-empty-creators.json @@ -0,0 +1,69 @@ +[ + { + "authored": false, + "confirmed": true, + "created": "2023-03-07T13:30:46.353Z", + "file_attached": false, + "hidden": false, + "id": "b7b1023f-5072-4608-8fd5-87b045b77887", + "client_data": "{\"desktop_id\":\"9c03fca4-ee5b-435e-abdd-fb6d7d11cd02\"}", + "last_modified": "2023-03-07T13:30:46.025Z", + "private_publication": false, + "profile_id": "8dbf0832-8723-4c48-b532-20c0b7f6e01a", + "read": false, + "source": "something wrong", + "identifiers": { + "doi": "10.1111", + "pmid": "PMID: 11111111", + "arxiv": "1111.2222" + }, + "starred": false, + "title": "This one has no authors", + "type": "journal", + "folder_uuids": [ + "8d2f262d-49b3-4dfc-8968-0bb71bcd92ea" + ], + "year": 1987 + }, + { + "authored": false, + "confirmed": true, + "created": "2023-03-07T13:30:46.353Z", + "file_attached": false, + "hidden": false, + "id": "a15698a9-6214-4fac-81c5-5ac62b2d06b5", + "client_data": "{\"desktop_id\":\"fd86e48e-1931-4282-b72d-78c535b0398c\"}", + "last_modified": "2023-03-07T13:30:46.025Z", + "private_publication": false, + "profile_id": "8dbf0832-8723-4c48-b532-20c0b7f6e01a", + "read": false, + "source": "something wrong", + "identifiers": + { + "doi": "10.1111", + "pmid": "PMID: 11111111", + "arxiv": "1111.2222" + }, + "starred": false, + "title": "This one has empty authors", + "authors": + [ + { + "first_name": "", + "last_name": "" + }, + { + "first_name": "", + "last_name": "" + }, + { + + } + ], + "type": "journal", + "folder_uuids": [ + "8d2f262d-49b3-4dfc-8968-0bb71bcd92ea" + ], + "year": 1987 + } +] \ No newline at end of file diff --git a/test/tests/mendeleyImportTest.js b/test/tests/mendeleyImportTest.js index 8c257cf9af..d160c2dcb4 100644 --- a/test/tests/mendeleyImportTest.js +++ b/test/tests/mendeleyImportTest.js @@ -451,7 +451,41 @@ describe('Zotero_Import_Mendeley', function () { const noNewItemHere = await Zotero.Relations.getByPredicateAndObject('item', 'mendeleyDB:documentUUID', '86e56a00-5ae5-4fe8-a977-9298a03b16d6'); assert.lengthOf(noNewItemHere, 0); + }); + it("should handle empty creators", async () => { + setHTTPResponse(server, 'https://api.mendeley.com/', { + method: 'GET', + url: `documents?view=all&limit=500`, + status: 200, + headers: {}, + json: JSON.parse( + await Zotero.File.getContentsFromURLAsync('resource://zotero-unit-tests/data/mendeleyMock/items-empty-creators.json') + ) + }); + + const importer = getImporter(); + await importer.translate({ + libraryID: Zotero.Libraries.userLibraryID, + collections: null, + linkFiles: false, + }); + + const journalNoAuthors = (await Zotero.Relations + .getByPredicateAndObject('item', 'mendeleyDB:documentUUID', '9c03fca4-ee5b-435e-abdd-fb6d7d11cd02')) + .filter(item => item.libraryID == Zotero.Libraries.userLibraryID && !item.deleted) + .shift(); + + assert.equal(journalNoAuthors.getField('title'), 'This one has no authors'); + assert.equal(journalNoAuthors.getCreators().length, 0); + + const journalEmptyAuthors = (await Zotero.Relations + .getByPredicateAndObject('item', 'mendeleyDB:documentUUID', 'fd86e48e-1931-4282-b72d-78c535b0398c')) + .filter(item => item.libraryID == Zotero.Libraries.userLibraryID && !item.deleted) + .shift(); + + assert.equal(journalEmptyAuthors.getField('title'), 'This one has empty authors'); + assert.equal(journalEmptyAuthors.getCreators().length, 0); }); }); }); From a6042d39581ed12c7c93fee0ca6925f0af813053 Mon Sep 17 00:00:00 2001 From: Tom Najdek Date: Thu, 9 Mar 2023 22:13:19 +0100 Subject: [PATCH 09/11] Mendeley importer: Fix issue with empty tags (#3018) Also adds a test for this particular case and for importing tags in general. --- .../zotero/import/mendeley/mendeleyImport.js | 5 +++- ...mpty-creators.json => items-bad-data.json} | 28 +++++++++++++++++++ .../tests/data/mendeleyMock/items-simple.json | 25 +++++++++++++++++ test/tests/mendeleyImportTest.js | 28 +++++++++++++++++-- 4 files changed, 83 insertions(+), 3 deletions(-) rename test/tests/data/mendeleyMock/{items-empty-creators.json => items-bad-data.json} (68%) diff --git a/chrome/content/zotero/import/mendeley/mendeleyImport.js b/chrome/content/zotero/import/mendeley/mendeleyImport.js index 4c3436c08c..1509212d69 100644 --- a/chrome/content/zotero/import/mendeley/mendeleyImport.js +++ b/chrome/content/zotero/import/mendeley/mendeleyImport.js @@ -680,7 +680,10 @@ Zotero_Import_Mendeley.prototype._getDocumentTagsDB = async function (groupID) { Zotero_Import_Mendeley.prototype._getDocumentTagsAPI = async function (documents) { var map = new Map(); for (let doc of documents) { - const tags = [...(doc.tags || []).map(tag => ({ tag, type: 0 })), ...(doc.keywords || []).map(tag => ({ tag, type: 1 }))]; + const tags = [ + ...(doc.tags || []).map(tag => ({ tag, type: 0 })), + ...(doc.keywords || []).map(tag => ({ tag, type: 1 })) + ].filter(t => t.tag && t.tag.trim()); map.set(doc.id, tags); } return map; diff --git a/test/tests/data/mendeleyMock/items-empty-creators.json b/test/tests/data/mendeleyMock/items-bad-data.json similarity index 68% rename from test/tests/data/mendeleyMock/items-empty-creators.json rename to test/tests/data/mendeleyMock/items-bad-data.json index c93c0d99eb..9e9cc06dd7 100644 --- a/test/tests/data/mendeleyMock/items-empty-creators.json +++ b/test/tests/data/mendeleyMock/items-bad-data.json @@ -65,5 +65,33 @@ "8d2f262d-49b3-4dfc-8968-0bb71bcd92ea" ], "year": 1987 + }, + { + "authored": false, + "confirmed": true, + "created": "2023-03-07T13:30:46.353Z", + "file_attached": false, + "hidden": false, + "id": "49426b47-d9ff-4ab6-ba2c-54e3608f56b8", + "client_data": "{\"desktop_id\":\"c7ec2737-044a-493b-9d94-d7f67be68765\"}", + "last_modified": "2023-03-07T13:30:46.025Z", + "private_publication": false, + "profile_id": "8dbf0832-8723-4c48-b532-20c0b7f6e01a", + "read": false, + "source": "something wrong", + "identifiers": { + "doi": "10.1111", + "pmid": "PMID: 11111111", + "arxiv": "1111.2222" + }, + "starred": false, + "tags": ["", ""], + "keywords": [], + "title": "This one has empty tags and keywords", + "type": "journal", + "folder_uuids": [ + "8d2f262d-49b3-4dfc-8968-0bb71bcd92ea" + ], + "year": 1987 } ] \ No newline at end of file diff --git a/test/tests/data/mendeleyMock/items-simple.json b/test/tests/data/mendeleyMock/items-simple.json index 4e0da03e3f..085b17db10 100644 --- a/test/tests/data/mendeleyMock/items-simple.json +++ b/test/tests/data/mendeleyMock/items-simple.json @@ -87,5 +87,30 @@ "filehash": "cc22c6611277df346ff8dc7386ba3880b2bafa15" } ] + }, + { + "authored": false, + "confirmed": true, + "created": "2021-11-04T11:53:10.353Z", + "file_attached": false, + "hidden": false, + "id": "cda0c150-bc85-4312-af2c-61a29b179595", + "client_data": "{\"desktop_id\":\"4308d8ec-e8ea-43fb-9d38-4e6628f7c10a\"}", + "last_modified": "2021-11-04T11:53:10.353Z", + "private_publication": false, + "profile_id": "8dbf0832-8723-4c48-b532-20c0b7f6e01a", + "read": false, + "starred": false, + "tags": [ + "tag1", + "tag2" + ], + "keywords": [ + "keyword1", + "keyword2" + ], + "title": "Has tags", + "type": "report", + "year": 2002 } ] \ No newline at end of file diff --git a/test/tests/mendeleyImportTest.js b/test/tests/mendeleyImportTest.js index d160c2dcb4..caf66f5f73 100644 --- a/test/tests/mendeleyImportTest.js +++ b/test/tests/mendeleyImportTest.js @@ -176,6 +176,11 @@ describe('Zotero_Import_Mendeley', function () { .filter(item => item.libraryID == Zotero.Libraries.userLibraryID && !item.deleted) .shift(); + const withTags = (await Zotero.Relations + .getByPredicateAndObject('item', 'mendeleyDB:documentUUID', '4308d8ec-e8ea-43fb-9d38-4e6628f7c10a')) + .filter(item => item.libraryID == Zotero.Libraries.userLibraryID && !item.deleted) + .shift(); + assert.equal(journal.getRelations()['mendeleyDB:remoteDocumentUUID'], '7fea3cb3-f97d-3f16-8fad-f59caaa71688'); assert.equal(journal.getField('title'), 'Foo Bar'); @@ -197,6 +202,17 @@ describe('Zotero_Import_Mendeley', function () { assert.equal(journal.getField('DOI'), '10.1111'); assert.sameMembers(journal.getField('extra').split('\n'), ['PMID: 11111111', 'arXiv: 1111.2222']); + // tags + assert.equal(withTags.getTags().length, 4); + assert.sameMembers( + withTags.getTags().filter(t => t.type === 1).map(t => t.tag), + ['keyword1', 'keyword2'] + ); + assert.sameMembers( + withTags.getTags().filter(t => !t.type).map(t => t.tag), + ['tag1', 'tag2'] + ); + // attachment & annotations assert.lengthOf(withpdf.getAttachments(), 1); assert.equal(pdf.parentID, withpdf.id); @@ -453,14 +469,14 @@ describe('Zotero_Import_Mendeley', function () { assert.lengthOf(noNewItemHere, 0); }); - it("should handle empty creators", async () => { + it("should handle empty creators and tags", async () => { setHTTPResponse(server, 'https://api.mendeley.com/', { method: 'GET', url: `documents?view=all&limit=500`, status: 200, headers: {}, json: JSON.parse( - await Zotero.File.getContentsFromURLAsync('resource://zotero-unit-tests/data/mendeleyMock/items-empty-creators.json') + await Zotero.File.getContentsFromURLAsync('resource://zotero-unit-tests/data/mendeleyMock/items-bad-data.json') ) }); @@ -486,6 +502,14 @@ describe('Zotero_Import_Mendeley', function () { assert.equal(journalEmptyAuthors.getField('title'), 'This one has empty authors'); assert.equal(journalEmptyAuthors.getCreators().length, 0); + + const journalEmptyTags = (await Zotero.Relations + .getByPredicateAndObject('item', 'mendeleyDB:documentUUID', 'c7ec2737-044a-493b-9d94-d7f67be68765')) + .filter(item => item.libraryID == Zotero.Libraries.userLibraryID && !item.deleted) + .shift(); + + assert.equal(journalEmptyTags.getField('title'), 'This one has empty tags and keywords'); + assert.equal(journalEmptyTags.getTags().length, 0); }); }); }); From 5e2507041ddbc3fbfcc323943e81a7e84d7b015d Mon Sep 17 00:00:00 2001 From: Tom Najdek Date: Thu, 6 Apr 2023 16:39:24 +0200 Subject: [PATCH 10/11] Remove unused import, nicer plurals, lint --- chrome/content/zotero/import/mendeley/mendeleyImport.js | 8 +++++--- chrome/locale/en-US/zotero/zotero.ftl | 6 ++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/chrome/content/zotero/import/mendeley/mendeleyImport.js b/chrome/content/zotero/import/mendeley/mendeleyImport.js index 1509212d69..69746da5d0 100644 --- a/chrome/content/zotero/import/mendeley/mendeleyImport.js +++ b/chrome/content/zotero/import/mendeley/mendeleyImport.js @@ -11,7 +11,7 @@ Services.scriptloader.loadSubScript("chrome://zotero/content/import/mendeley/men const importerVersion = 1; const { apiTypeToDBType, apiFieldToDBField } = mendeleyOnlineMappings; -const { apiFetch, codeAuth, directAuth, get, getAll } = mendeleyAPIUtils; +const { apiFetch, codeAuth, get, getAll } = mendeleyAPIUtils; const colorMap = new Map(); colorMap.set('rgb(255, 245, 173)', '#ffd400'); @@ -140,7 +140,7 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) { this._itemDone(); let folderKeys = new Map(); - if(!this.relinkOnly) { + if (!this.relinkOnly) { const folders = this._tokens ? await this._getFoldersAPI(mendeleyGroupID) : await this._getFoldersDB(mendeleyGroupID); @@ -182,12 +182,14 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) { this._interruptChecker(true); + // eslint-disable-next-line multiline-ternary let collections = this.relinkOnly ? new Map() : this._tokens ? await this._getDocumentCollectionsAPI(documents, rootCollectionKey, folderKeys) : await this._getDocumentCollectionsDB(mendeleyGroupID, documents, rootCollectionKey, folderKeys); this._interruptChecker(true); + // eslint-disable-next-line multiline-ternary let files = this.relinkOnly ? new Map() : this._tokens ? await this._getDocumentFilesAPI(documents) : await this._getDocumentFilesDB(mendeleyGroupID); @@ -562,7 +564,7 @@ Zotero_Import_Mendeley.prototype._getDocumentsAPI = async function (groupID) { } - return (await getAll(this._tokens, 'documents', params, headers, {}, this._interruptChecker)).map(d => { + return (await getAll(this._tokens, 'documents', params, headers, {}, this._interruptChecker)).map((d) => { const processedDocument = { ...d, remoteUuid: d.id }; try { diff --git a/chrome/locale/en-US/zotero/zotero.ftl b/chrome/locale/en-US/zotero/zotero.ftl index 8340988b4a..0a66d12d89 100644 --- a/chrome/locale/en-US/zotero/zotero.ftl +++ b/chrome/locale/en-US/zotero/zotero.ftl @@ -47,11 +47,13 @@ general-error = Error file-interface-import-error = An error occurred while trying to import the selected file. Please ensure that the file is valid and try again. file-interface-import-complete = Import Complete file-interface-items-were-imported = { $numItems -> - [one] item was imported + [0] No items were imported + [one] One item was imported *[other] { $numItems } items were imported } file-interface-items-were-relinked = { $numRelinked -> - [one] item was relinked + [0] No items were relinked + [one] One item was relinked *[other] { $numRelinked } items were relinked } From d420ab34ebdb1b62b6c6651b28b15be794405740 Mon Sep 17 00:00:00 2001 From: Dan Stillman Date: Sat, 8 Apr 2023 17:16:47 -0400 Subject: [PATCH 11/11] Mendeley importer: Add `native=true` to checkbox --- chrome/content/zotero/import/importWizard.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chrome/content/zotero/import/importWizard.xhtml b/chrome/content/zotero/import/importWizard.xhtml index 1bd9bf12e1..e282fc7cfe 100644 --- a/chrome/content/zotero/import/importWizard.xhtml +++ b/chrome/content/zotero/import/importWizard.xhtml @@ -55,7 +55,7 @@ data-header-label-id="import-options" >
- +