From d4d2080a31c5baf080859d349d77edc1154303fc Mon Sep 17 00:00:00 2001 From: Tom Najdek Date: Wed, 24 Mar 2021 15:13:44 +0100 Subject: [PATCH 1/6] Mendeley online importer --- chrome/content/zotero/fileInterface.js | 111 ++++-- chrome/content/zotero/import/importWizard.js | 36 +- chrome/content/zotero/import/importWizard.xul | 14 +- .../import/mendeley/mendeleyAPIUtils.js | 51 +++ .../zotero/import/mendeley/mendeleyImport.js | 315 ++++++++++++++++-- .../import/mendeley/mendeleyOnlineMappings.js | 57 ++++ .../import/mendeley/mendeleySchemaMap.js | 1 + chrome/locale/en-US/zotero/zotero.properties | 1 + chrome/skin/default/zotero/importWizard.css | 5 + 9 files changed, 528 insertions(+), 63 deletions(-) create mode 100644 chrome/content/zotero/import/mendeley/mendeleyAPIUtils.js create mode 100644 chrome/content/zotero/import/mendeley/mendeleyOnlineMappings.js diff --git a/chrome/content/zotero/fileInterface.js b/chrome/content/zotero/fileInterface.js index 76e0415a22..b503f07aaa 100644 --- a/chrome/content/zotero/fileInterface.js +++ b/chrome/content/zotero/fileInterface.js @@ -23,7 +23,8 @@ ***** END LICENSE BLOCK ***** */ -Components.utils.import("resource://gre/modules/osfile.jsm") +Components.utils.import("resource://gre/modules/osfile.jsm"); +Components.utils.import("resource://gre/modules/Services.jsm"); import FilePicker from 'zotero/filePicker'; /****Zotero_File_Exporter**** @@ -266,7 +267,7 @@ var Zotero_File_Interface = new function() { }; - this.showImportWizard = function () { + this.showImportWizard = function (extraArgs = {}) { var libraryID = Zotero.Libraries.userLibraryID; try { let zp = Zotero.getActiveZoteroPane(); @@ -276,7 +277,8 @@ var Zotero_File_Interface = new function() { Zotero.logError(e); } var args = { - libraryID + libraryID, + ...extraArgs }; args.wrappedJSObject = args; @@ -329,30 +331,39 @@ var Zotero_File_Interface = new function() { var defaultNewCollectionPrefix = Zotero.getString("fileInterface.imported"); var translation; - // Check if the file is an SQLite database - var sample = yield Zotero.File.getSample(file.path); - if (file.path == Zotero.DataDirectory.getDatabase()) { - // Blacklist the current Zotero database, which would cause a hang - } - else if (Zotero.MIME.sniffForMIMEType(sample) == 'application/x-sqlite3') { - // Mendeley import doesn't use the real translation architecture, but we create a - // translation object with the same interface + + if (options.mendeleyOnlineToken) { translation = yield _getMendeleyTranslation(); translation.createNewCollection = createNewCollection; - defaultNewCollectionPrefix = Zotero.getString( - 'fileInterface.appImportCollection', 'Mendeley' - ); + translation.token = options.mendeleyOnlineToken; } - else if (file.path.endsWith('@www.mendeley.com.sqlite') - || file.path.endsWith('online.sqlite')) { - // Keep in sync with importWizard.js - throw new Error('Encrypted Mendeley database'); + else { + // Check if the file is an SQLite database + var sample = yield Zotero.File.getSample(file.path); + if (file.path == Zotero.DataDirectory.getDatabase()) { + // Blacklist the current Zotero database, which would cause a hang + } + else if (Zotero.MIME.sniffForMIMEType(sample) == 'application/x-sqlite3') { + // Mendeley import doesn't use the real translation architecture, but we create a + // translation object with the same interface + translation = yield _getMendeleyTranslation(); + translation.createNewCollection = createNewCollection; + defaultNewCollectionPrefix = Zotero.getString( + 'fileInterface.appImportCollection', 'Mendeley' + ); + } + else if (file.path.endsWith('@www.mendeley.com.sqlite') + || file.path.endsWith('online.sqlite')) { + // Keep in sync with importWizard.js + throw new Error('Encrypted Mendeley database'); + } + + if (!translation) { + translation = new Zotero.Translate.Import(); + } + translation.setLocation(file); } - - if (!translation) { - translation = new Zotero.Translate.Import(); - } - translation.setLocation(file); + return _finishImport({ translation, createNewCollection, @@ -592,7 +603,7 @@ var Zotero_File_Interface = new function() { eval(xmlhttp.response); } return new Zotero_Import_Mendeley(); - } + }; /** @@ -849,7 +860,57 @@ var Zotero_File_Interface = new function() { return false; } } -} + + this.authenticateMendeleyOnlinePoll = function (win) { + if (win && win[0] && win[0].location) { + const matchResult = win[0].location.toString().match(/access_token=(.*?)(?:&|$)/i); + if (matchResult) { + const mendeleyAccessToken = matchResult[1]; + Zotero.getMainWindow().setTimeout(() => this.showImportWizard({ mendeleyAccessToken }), 0); + win.close(); + return; + } + } + + if (win && !win.closed) { + Zotero.getMainWindow().setTimeout(this.authenticateMendeleyOnlinePoll.bind(this, win), 200); + } + }; + + this.authenticateMendeleyOnline = function () { + const uri = 'https://api.mendeley.com/oauth/authorize?client_id=5907&redirect_uri=https%3A%2F%2Fzotero-static.s3.amazonaws.com%2Fmendeley_oauth_redirect.html&response_type=token&state=&scope=all'; + + var win = Services.wm.getMostRecentWindow("zotero:basicViewer"); + if (win) { + win.loadURI(uri); + } + else { + const ww = Services.ww; + const arg = Components.classes["@mozilla.org/supports-string;1"] + .createInstance(Components.interfaces.nsISupportsString); + arg.data = uri; + win = ww.openWindow(null, "chrome://zotero/content/standalone/basicViewer.xul", + "basicViewer", "chrome,dialog=yes,resizable,centerscreen,menubar,scrollbars", arg); + } + + let browser; + let func = function () { + win.removeEventListener("load", func); + browser = win.document.documentElement.getElementsByTagName('browser')[0]; + browser.addEventListener("pageshow", innerFunc); + }; + let innerFunc = function () { + browser.removeEventListener("pageshow", innerFunc); + win.outerWidth = Math.max(640, Math.min(1024, win.screen.availHeight)); + win.outerHeight = Math.max(480, Math.min(768, win.screen.availWidth)); + }; + + win.addEventListener("load", func); + + // polling executed by the main window because current (wizard) window will be closed + Zotero.getMainWindow().setTimeout(this.authenticateMendeleyOnlinePoll.bind(this, win), 200); + }; +}; // Handles the display of a progress indicator Zotero_File_Interface.Progress = new function() { diff --git a/chrome/content/zotero/import/importWizard.js b/chrome/content/zotero/import/importWizard.js index 559d8e8a9e..158372f2d0 100644 --- a/chrome/content/zotero/import/importWizard.js +++ b/chrome/content/zotero/import/importWizard.js @@ -5,11 +5,12 @@ var Zotero_Import_Wizard = { _dbs: null, _file: null, _translation: null, + _mendeleyOnlineRedirectURLWithCode: null, + _mendeleyAccessToken: null, init: async function () { this._wizard = document.getElementById('import-wizard'); - var dbs = await Zotero_File_Interface.findMendeleyDatabases(); if (dbs.length) { document.getElementById('radio-import-source-mendeley').hidden = false; @@ -32,6 +33,11 @@ var Zotero_Import_Wizard = { document.getElementById('create-collection-checkbox').removeAttribute('checked'); } } + + if (args && args.mendeleyAccessToken) { + this._mendeleyAccessToken = args.mendeleyAccessToken; + this._wizard.goTo('page-options'); + } // Update labels document.getElementById('file-handling-store').label = Zotero.getString( @@ -57,6 +63,11 @@ var Zotero_Import_Wizard = { case 'radio-import-source-file': await this.chooseFile(); break; + + case 'radio-import-source-mendeley-online': + wizard.goTo('mendeley-online-explanation'); + wizard.canRewind = true; + break; case 'radio-import-source-mendeley': this._dbs = await Zotero_File_Interface.findMendeleyDatabases(); @@ -85,7 +96,19 @@ var Zotero_Import_Wizard = { throw e; } }, - + + onMendeleyOnlineShow: async function () { + document.getElementById('mendeley-online-description').textContent = Zotero.getString( + 'import.mendeleyOnline.intro', [Zotero.appName, 'Mendeley Reference Manager', 'Mendeley'] + ); + }, + + onMendeleyOnlineAdvance: function () { + if (!this._mendeleyOnlineRedirectURLWithCode) { + Zotero_File_Interface.authenticateMendeleyOnline(); + window.close(); + } + }, goToStart: function () { this._wizard.goTo('page-start'); @@ -165,7 +188,7 @@ var Zotero_Import_Wizard = { onOptionsShown: function () { - + document.getElementById('file-handling-options').hidden = !!this._mendeleyAccessToken; }, @@ -192,7 +215,7 @@ var Zotero_Import_Wizard = { onImportStart: async function () { - if (!this._file) { + if (!this._file && !this._mendeleyAccessToken) { let index = document.getElementById('file-list').selectedIndex; this._file = this._dbs[index].path; } @@ -206,7 +229,8 @@ var Zotero_Import_Wizard = { onBeforeImport: this.onBeforeImport.bind(this), addToLibraryRoot: !document.getElementById('create-collection-checkbox') .hasAttribute('checked'), - linkFiles: document.getElementById('file-handling-radio').selectedIndex == 1 + linkFiles: document.getElementById('file-handling-radio').selectedIndex == 1, + mendeleyOnlineToken: this._mendeleyAccessToken }); // Cancelled by user or due to error @@ -321,7 +345,7 @@ var Zotero_Import_Wizard = { xulElem.hidden = false; htmlElem.setAttribute('display', 'none'); } - document.getElementById('result-description') + document.getElementById('result-description'); if (showReportErrorButton) { let button = document.getElementById('result-report-error'); diff --git a/chrome/content/zotero/import/importWizard.xul b/chrome/content/zotero/import/importWizard.xul index 8721aed14f..9603f25bb6 100644 --- a/chrome/content/zotero/import/importWizard.xul +++ b/chrome/content/zotero/import/importWizard.xul @@ -21,9 +21,20 @@ onpageadvanced="Zotero_Import_Wizard.onModeChosen(); return false;"> - + + + + - { + let next = null; + let links = response.getResponseHeader('link'); + if (links) { + const matches = links.match(/<(.*?)>;\s+rel="next"/i); + + if (matches && matches.length > 1) { + next = matches[1]; + } + } + return next; +}; + +const apiFetchUrl = async (token, url, headers = {}, options = {}) => { + headers = { ...headers, Authorization: `Bearer ${token}` }; + return Zotero.HTTP.request('GET', url, { ...options, headers }); +}; + +const apiFetch = async (token, endPoint, params = {}, headers = {}, options = {}) => { + const stringParams = Object.entries(params).map(p => p.join('=')).join('&'); + const url = MENDELEY_API_URL + '/' + endPoint + '?' + stringParams; + return apiFetchUrl(token, url, headers, options); +}; + +const get = async (token, endPoint, params = {}, headers = {}, options = {}) => { + const response = await apiFetch(token, endPoint, params, headers, options); + return JSON.parse(response.responseText); +}; + +const getAll = async (token, endPoint, params = {}, headers = {}, options = {}) => { + const PER_PAGE = endPoint === 'annotations' ? 200 : 500; + const response = await apiFetch(token, endPoint, { ...params, limit: PER_PAGE }, headers, options); + var next = getNextLinkFromResponse(response); + var data = JSON.parse(response.responseText); + + while (next) { + const response = await apiFetchUrl(token, next, headers, options); //eslint-disable-line no-await-in-loop + data = [...data, ...JSON.parse(response.responseText)]; + next = getNextLinkFromResponse(response); + } + + return data; +}; + + +return { getNextLinkFromResponse, apiFetch, apiFetchUrl, get, getAll }; +})(); diff --git a/chrome/content/zotero/import/mendeley/mendeleyImport.js b/chrome/content/zotero/import/mendeley/mendeleyImport.js index 890ccb2c7c..b758359535 100644 --- a/chrome/content/zotero/import/mendeley/mendeleyImport.js +++ b/chrome/content/zotero/import/mendeley/mendeleyImport.js @@ -1,19 +1,27 @@ +/* global map:false, mendeleyOnlineMappings:false, mendeleyAPIUtils:false */ var EXPORTED_SYMBOLS = ["Zotero_Import_Mendeley"]; Components.utils.import("resource://gre/modules/Services.jsm"); Components.utils.import("resource://gre/modules/osfile.jsm"); Services.scriptloader.loadSubScript("chrome://zotero/content/include.js"); +Services.scriptloader.loadSubScript("chrome://zotero/content/import/mendeley/mendeleyOnlineMappings.js"); +Services.scriptloader.loadSubScript("chrome://zotero/content/import/mendeley/mendeleyAPIUtils.js"); + +const { apiTypeToDBType, apiFieldToDBField } = mendeleyOnlineMappings; +const { apiFetch, getAll } = mendeleyAPIUtils; var Zotero_Import_Mendeley = function () { this.createNewCollection = null; this.linkFiles = null; this.newItems = []; + this.token = null; this._db; this._file; this._itemDone; this._progress = 0; this._progressMax; + this._tmpFilesToDelete = []; }; Zotero_Import_Mendeley.prototype.setLocation = function (file) { @@ -42,6 +50,7 @@ Zotero_Import_Mendeley.prototype.setTranslator = function () {}; Zotero_Import_Mendeley.prototype.translate = async function (options = {}) { this._linkFiles = options.linkFiles; + this.timestamp = Date.now(); if (true) { Services.scriptloader.loadSubScript("chrome://zotero/content/import/mendeley/mendeleySchemaMap.js"); @@ -77,39 +86,84 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) { // Disable syncing while we're importing var resumeSync = Zotero.Sync.Runner.delayIndefinite(); - this._db = new Zotero.DBConnection(this._file); + if (this._file) { + this._db = new Zotero.DBConnection(this._file); + } try { - if (!await this._isValidDatabase()) { + if (this._file && !await this._isValidDatabase()) { throw new Error("Not a valid Mendeley database"); } - - // Collections - let folders = await this._getFolders(mendeleyGroupID); - let collectionJSON = this._foldersToAPIJSON(folders, rootCollectionKey); - let folderKeys = this._getFolderKeys(collectionJSON); + + if (!this._file && !this.token) { + throw new Error("Missing import token"); + } + + const folders = this.token + ? await this._getFoldersAPI(mendeleyGroupID) + : await this._getFoldersDB(mendeleyGroupID); + + const collectionJSON = this._foldersToAPIJSON(folders, rootCollectionKey); + const folderKeys = this._getFolderKeys(collectionJSON); await this._saveCollections(libraryID, collectionJSON, folderKeys); // // Items // - let documents = await this._getDocuments(mendeleyGroupID); + let documents = this.token + ? await this._getDocumentsAPI(mendeleyGroupID) + : await this._getDocumentsDB(mendeleyGroupID); + this._progressMax = documents.length; // Get various attributes mapped to document ids - let urls = await this._getDocumentURLs(mendeleyGroupID); - let creators = await this._getDocumentCreators(mendeleyGroupID, map.creatorTypes); - let tags = await this._getDocumentTags(mendeleyGroupID); - let collections = await this._getDocumentCollections( - mendeleyGroupID, - documents, - rootCollectionKey, - folderKeys - ); - let files = await this._getDocumentFiles(mendeleyGroupID); - let annotations = await this._getDocumentAnnotations(mendeleyGroupID); + let urls = this.token + ? await this._getDocumentURLsAPI(documents) + : await this._getDocumentURLsDB(mendeleyGroupID); + + let creators = this.token + ? await this._getDocumentCreatorsAPI(documents) + : await this._getDocumentCreatorsDB(mendeleyGroupID, map.creatorTypes); + + let tags = this.token + ? await this._getDocumentTagsAPI(documents) + : await this._getDocumentTagsDB(mendeleyGroupID); + + let collections = this.token + ? await this._getDocumentCollectionsAPI(documents, rootCollectionKey, folderKeys) + : await this._getDocumentCollectionsDB(mendeleyGroupID, documents, rootCollectionKey, folderKeys); + + let files = this.token + ? await this._getDocumentFilesAPI(documents) + : await this._getDocumentFilesDB(mendeleyGroupID); + + let annotations = this.token + ? await this._getDocumentAnnotationsAPI(mendeleyGroupID) + : await this._getDocumentAnnotationsDB(mendeleyGroupID); + for (let document of documents) { let docURLs = urls.get(document.id); let docFiles = files.get(document.id); + + if (this.token) { + // extract identifiers + ['arxiv', 'doi', 'isbn', 'issn', 'pmid', 'scopus', 'pui', 'pii', 'sgr'].forEach( + i => document[i] = (document.identifiers || {})[i] + ); + + // normalise item type from the API to match Mendeley DB + document.type = apiTypeToDBType[document.type] || document.type; + + // normalise field names from the API to match Mendeley DB + Object.keys(apiFieldToDBField).forEach((key) => { + if (key in document) { + const newKey = apiFieldToDBField[key]; + if (newKey) { + document[newKey] = document[key]; + } + delete document[key]; + } + }); + } // If there's a single PDF file, use "PDF" for the attachment title if (docFiles && docFiles.length == 1 && docFiles[0].fileURL.endsWith('.pdf')) { @@ -173,7 +227,14 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) { } finally { try { - await this._db.closeDatabase(); + if (this._file) { + await this._db.closeDatabase(); + } + if (this.token) { + await Promise.all( + this._tmpFilesToDelete.map(f => this._removeTemporaryFile(f)) + ); + } } catch (e) { Zotero.logError(e); @@ -183,6 +244,18 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) { } }; +Zotero_Import_Mendeley.prototype._removeTemporaryFile = async function (file) { + const containingDir = OS.Path.dirname(file); + try { + await Zotero.File.removeIfExists(file); + await OS.File.removeEmptyDir(containingDir); + } + catch (e) { + Zotero.logError(e); + } +}; + + Zotero_Import_Mendeley.prototype._isValidDatabase = async function () { var tables = [ 'DocumentContributors', @@ -208,7 +281,7 @@ Zotero_Import_Mendeley.prototype._isValidDatabase = async function () { // // Collections // -Zotero_Import_Mendeley.prototype._getFolders = async function (groupID) { +Zotero_Import_Mendeley.prototype._getFoldersDB = async function (groupID) { return this._db.queryAsync( `SELECT F.id, F.uuid, F.name, ` // Top-level folders can have a parentId of 0 instead of -1 (by mistake?) @@ -221,6 +294,23 @@ Zotero_Import_Mendeley.prototype._getFolders = async function (groupID) { ); }; +Zotero_Import_Mendeley.prototype._getFoldersAPI = async function (groupID) { + const params = {}; + const headers = { Accept: 'application/vnd.mendeley-folder.1+json' }; + + if (groupID && groupID !== 0) { + params.group_id = groupID; //eslint-disable-line camelcase + } + + return (await getAll(this.token, 'folders', params, headers)).map(f => ({ + id: f.id, + uuid: f.id, + name: f.name, + parentId: f.parent_id || -1, + remoteUuid: f.id + })); +}; + /** * Get flat array of collection API JSON with parentCollection set * @@ -337,13 +427,13 @@ Zotero_Import_Mendeley.prototype._findExistingCollection = async function (libra Zotero.debug(`Found existing collection ${collections[0].libraryKey} for ` + `${predicate} ${collectionJSON.relations[predicate]}`); return collections[0]; -} +}; // // Items // -Zotero_Import_Mendeley.prototype._getDocuments = async function (groupID) { +Zotero_Import_Mendeley.prototype._getDocumentsDB = async function (groupID) { return this._db.queryAsync( `SELECT D.*, RD.remoteUuid FROM Documents D ` + `JOIN RemoteDocuments RD ON (D.id=RD.documentId) ` @@ -352,12 +442,28 @@ Zotero_Import_Mendeley.prototype._getDocuments = async function (groupID) { ); }; +Zotero_Import_Mendeley.prototype._getDocumentsAPI = async function (groupID) { + const params = { view: 'all' }; + const headers = { Accept: 'application/vnd.mendeley-document-with-files-list+json' }; + + if (groupID && groupID !== 0) { + params.group_id = groupID; //eslint-disable-line camelcase + } + + + return (await getAll(this.token, 'documents', params, headers)).map(d => ({ + ...d, + uuid: d.id, + remoteUuid: d.id + })); +}; + /** * Get a Map of document ids to arrays of URLs * * @return {Map} */ -Zotero_Import_Mendeley.prototype._getDocumentURLs = async function (groupID) { +Zotero_Import_Mendeley.prototype._getDocumentURLsDB = async function (groupID) { var rows = await this._db.queryAsync( `SELECT documentId, CAST(url AS TEXT) AS url FROM DocumentUrls DU ` + `JOIN RemoteDocuments USING (documentId) ` @@ -374,13 +480,17 @@ Zotero_Import_Mendeley.prototype._getDocumentURLs = async function (groupID) { return map; }; +Zotero_Import_Mendeley.prototype._getDocumentURLsAPI = async function (documents) { + return new Map(documents.map(d => ([d.id, d.websites]))); +}; + /** * Get a Map of document ids to arrays of creator API JSON * * @param {Integer} groupID * @param {Object} creatorTypeMap - Mapping of Mendeley creator types to Zotero creator types */ -Zotero_Import_Mendeley.prototype._getDocumentCreators = async function (groupID, creatorTypeMap) { +Zotero_Import_Mendeley.prototype._getDocumentCreatorsDB = async function (groupID, creatorTypeMap) { var rows = await this._db.queryAsync( `SELECT * FROM DocumentContributors ` + `JOIN RemoteDocuments USING (documentId) ` @@ -401,10 +511,21 @@ Zotero_Import_Mendeley.prototype._getDocumentCreators = async function (groupID, return map; }; +Zotero_Import_Mendeley.prototype._getDocumentCreatorsAPI = async function (documents) { + var map = new Map(); + for (let doc of documents) { + const authors = (doc.authors || []).map(c => this._makeCreator('author', c.first_name, c.last_name)); + const editors = (doc.editors || []).map(c => this._makeCreator('editor', c.first_name, c.last_name)); + const translators = (doc.translators || []).map(c => this._makeCreator('translator', c.first_name, c.last_name)); + map.set(doc.id, [...authors, ...editors, ...translators]); + } + return map; +}; + /** * Get a Map of document ids to arrays of tag API JSON */ -Zotero_Import_Mendeley.prototype._getDocumentTags = async function (groupID) { +Zotero_Import_Mendeley.prototype._getDocumentTagsDB = async function (groupID) { var rows = await this._db.queryAsync( // Manual tags `SELECT documentId, tag, 0 AS type FROM DocumentTags ` @@ -432,10 +553,19 @@ Zotero_Import_Mendeley.prototype._getDocumentTags = async function (groupID) { return map; }; +Zotero_Import_Mendeley.prototype._getDocumentTagsAPI = async function (documents) { + var map = new Map(); + for (let doc of documents) { + const tags = [...(doc.tags || []).map(tag => ({ tag, type: 0 })), ...(doc.keywords || []).map(tag => ({ tag, type: 1 }))]; + map.set(doc.id, tags); + } + return map; +}; + /** * Get a Map of document ids to arrays of collection keys */ -Zotero_Import_Mendeley.prototype._getDocumentCollections = async function (groupID, documents, rootCollectionKey, folderKeys) { +Zotero_Import_Mendeley.prototype._getDocumentCollectionsDB = async function (groupID, documents, rootCollectionKey, folderKeys) { var rows = await this._db.queryAsync( `SELECT documentId, folderId FROM DocumentFolders DF ` + `JOIN RemoteDocuments USING (documentId) ` @@ -460,12 +590,28 @@ Zotero_Import_Mendeley.prototype._getDocumentCollections = async function (group return map; }; +Zotero_Import_Mendeley.prototype._getDocumentCollectionsAPI = async function (documents, rootCollectionKey, folderKeys) { + return new Map( + documents.map((d) => { + const keys = (d.folder_uuids || []).map((fuuid) => { + const key = folderKeys.get(fuuid); + if (!key) { + Zotero.debug(`Document folder ${fuuid} not found -- skipping`, 2); + } + return key; + }).filter(Boolean); + // Add all documents to root collection if specified + return [d.id, [...keys, ...(rootCollectionKey ? [rootCollectionKey] : [])]]; + }) + ); +}; + /** * Get a Map of document ids to arrays of file metadata * * @return {Map} */ -Zotero_Import_Mendeley.prototype._getDocumentFiles = async function (groupID) { +Zotero_Import_Mendeley.prototype._getDocumentFilesDB = async function (groupID) { var rows = await this._db.queryAsync( `SELECT documentId, hash, localUrl FROM DocumentFiles ` + `JOIN Files USING (hash) ` @@ -490,10 +636,59 @@ Zotero_Import_Mendeley.prototype._getDocumentFiles = async function (groupID) { return map; }; +Zotero_Import_Mendeley.prototype._fetchFile = async function (fileID, filePath) { + const fileDir = OS.Path.dirname(filePath); + await Zotero.File.createDirectoryIfMissingAsync(fileDir); + const xhr = await apiFetch(this.token, `files/${fileID}`, {}, {}, { responseType: 'blob', followRedirects: false }); + const uri = xhr.getResponseHeader('location'); + await Zotero.File.download(uri, filePath); + + this._progress += 1; + if (this._itemDone) { + this._itemDone(); + } +}; + +Zotero_Import_Mendeley.prototype._getDocumentFilesAPI = async function (documents) { + const map = new Map(); + + let totalSize = 0; + + Components.utils.import("resource://zotero/concurrentCaller.js"); + var caller = new ConcurrentCaller({ + numConcurrent: 6, + onError: e => Zotero.logError(e), + Promise: Zotero.Promise + }); + + for (let doc of documents) { + const files = []; + for (let file of (doc.files || [])) { + const fileName = file.file_name || 'file'; + const tmpFile = OS.Path.join(Zotero.getTempDirectory().path, `mendeley-online-import-${this.timestamp}-${file.id}`, fileName); + this._tmpFilesToDelete.push(tmpFile); + caller.add(this._fetchFile.bind(this, file.id, tmpFile)); + files.push({ + fileURL: OS.Path.toFileURI(tmpFile), + title: file.file_name || '', + contentType: file.mime_type || '', + hash: file.filehash, + }); + totalSize += file.size; + this._progressMax += 1; + } + map.set(doc.id, files); + } + // @TODO: check if enough space available totalSize + await caller.runAll(); + return map; +}; + + /** * Get a Map of document ids to arrays of annotations */ -Zotero_Import_Mendeley.prototype._getDocumentAnnotations = async function (groupID) { +Zotero_Import_Mendeley.prototype._getDocumentAnnotationsDB = async function (groupID) { var map = new Map(); // Highlights @@ -572,6 +767,65 @@ Zotero_Import_Mendeley.prototype._getDocumentAnnotations = async function (group return map; }; +Zotero_Import_Mendeley.prototype._getDocumentAnnotationsAPI = async function (groupID) { + const params = {}; + + if (groupID && groupID !== 0) { + params.group_id = groupID; //eslint-disable-line camelcase + } + + const map = new Map(); + (await getAll(this.token, 'annotations', params, { Accept: 'application/vnd.mendeley-annotation.1+json' })) + .forEach((a) => { + const rects = (a.positions || []).map(position => ({ + x1: (position.top_left || {}).x || 0, + y1: (position.top_left || {}).y || 0, + x2: (position.bottom_right || {}).x || 0, + y2: (position.bottom_right || {}).y || 0, + })); + let page = 1; + try { + // const page = ((a.positions || [])[0] || {}).page; // ??? + page = a.positions[0].page; + } + catch (e) { } + + const annotation = { + id: a.id, + color: a.color ? `#${a.color.r.toString(16)}${a.color.g.toString(16)}${a.color.b.toString(16)}` : null, + dateAdded: a.created, + dateModified: a.last_modified, + hash: a.filehash, + uuid: a.id, + page, + }; + + if (a.type === 'highlight') { + annotation.type = 'highlight'; + annotation.rects = rects; + } + + if (a.type === 'sticky_note' && rects.length > 0) { + annotation.type = 'note'; + annotation.note = a.text; + annotation.x = rects[0].x1; + annotation.y = rects[0].y1; + } + + if (a.type === 'note') { + // This is a "general note" in Mendeley. It appears to be the same thing as + // document.note thus not an annotations and can be discarded + return; + } + + if (!map.has(a.document_id)) { + map.set(a.document_id, []); + } + map.get(a.document_id).push(annotation); + }); + return map; +}; + /** * Create API JSON array with item and any child attachments or notes */ @@ -1124,7 +1378,8 @@ Zotero_Import_Mendeley.prototype._isDownloadedFile = function (path) { return parentDir.endsWith(OS.Path.join('Application Support', 'Mendeley Desktop', 'Downloaded')) || parentDir.endsWith(OS.Path.join('Local', 'Mendeley Ltd', 'Mendeley Desktop', 'Downloaded')) || parentDir.endsWith(OS.Path.join('Local', 'Mendeley Ltd.', 'Mendeley Desktop', 'Downloaded')) - || parentDir.endsWith(OS.Path.join('data', 'Mendeley Ltd.', 'Mendeley Desktop', 'Downloaded')); + || parentDir.endsWith(OS.Path.join('data', 'Mendeley Ltd.', 'Mendeley Desktop', 'Downloaded')) + || parentDir.startsWith(OS.Path.join(Zotero.getTempDirectory().path, 'mendeley-online-import')); // Mendeley Online Importer } /** diff --git a/chrome/content/zotero/import/mendeley/mendeleyOnlineMappings.js b/chrome/content/zotero/import/mendeley/mendeleyOnlineMappings.js new file mode 100644 index 0000000000..1d81ffeabe --- /dev/null +++ b/chrome/content/zotero/import/mendeley/mendeleyOnlineMappings.js @@ -0,0 +1,57 @@ +/* eslint-disable camelcase, no-unused-vars */ + +var mendeleyOnlineMappings = { + // lookup to normalise from item type presented by API to item type as stored in DB + apiTypeToDBType: { + bill: 'Bill', + book: 'Book', + book_section: 'BookSection', + case: 'Case', + computer_program: 'ComputerProgram', + conference_proceedings: 'ConferenceProceedings', + encyclopedia_article: 'EncyclopediaArticle', + film: 'Film', + generic: 'Generic', + hearing: 'Hearing', + journal: 'JournalArticle', + magazine_article: 'MagazineArticle', + newspaper_article: 'NewspaperArticle', + patent: 'Patent', + report: 'Report', + statute: 'Statute', + television_broadcast: 'TelevisionBroadcast', + thesis: 'Thesis', + web_page: 'WebPage', + working_paper: 'WorkingPaper' + }, + apiFieldToDBField: { + accessed: 'dateAccessed', + authors: false, // all author types handled separately + citation_key: 'citationKey', + created: 'added', + edition: 'edition', + editors: false, // all author types handled separately + file_attached: false, + folder_uuids: false, // collections handled separately + group_id: 'groupID', + identifiers: false, // identifiers are separately copied directly into document + keywords: false, // tags handled separately + last_modified: 'modified', + notes: 'note', + patent_application_number: 'patentApplicationNumber', + patent_legal_status: 'patentLegalStatus', + patent_owner: 'patentOwner', + private_publication: 'privatePublication', + profile_id: 'profileID', + reprint_edition: 'reprintEdition', + revision: 'revisionNumber', + series_editor: 'seriesEditor', + series_number: 'seriesNumber', + short_title: 'shortTitle', + source_type: 'sourceType', + tags: false, // tags handled separately + translators: false, // all author types handled separately + user_context: 'userContext', + websites: false // URLs handled separately + } +}; diff --git a/chrome/content/zotero/import/mendeley/mendeleySchemaMap.js b/chrome/content/zotero/import/mendeley/mendeleySchemaMap.js index af8e929e33..51ca57f8ff 100644 --- a/chrome/content/zotero/import/mendeley/mendeleySchemaMap.js +++ b/chrome/content/zotero/import/mendeley/mendeleySchemaMap.js @@ -10,6 +10,7 @@ var map = { EncyclopediaArticle: "encyclopediaArticle", Film: "film", Generic: "document", + Hearing: "hearing", JournalArticle: "journalArticle", MagazineArticle: "magazineArticle", NewspaperArticle: "newspaperArticle", diff --git a/chrome/locale/en-US/zotero/zotero.properties b/chrome/locale/en-US/zotero/zotero.properties index 9c186783ba..1c39ec23bd 100644 --- a/chrome/locale/en-US/zotero/zotero.properties +++ b/chrome/locale/en-US/zotero/zotero.properties @@ -752,6 +752,7 @@ fileInterface.OPMLFeedFilter = OPML Feed List import.fileHandling.store = Copy files to the %S storage folder import.fileHandling.link = Link to files in original location import.fileHandling.description = Linked files cannot be synced by %S. +import.mendeleyOnline.intro = On the next page you will be asked to log in to %2$S and grant %1$S access. This is necessary to import your %3$S library into %1$S. quickCopy.copyAs = Copy as %S diff --git a/chrome/skin/default/zotero/importWizard.css b/chrome/skin/default/zotero/importWizard.css index a690158aac..e759b0878e 100644 --- a/chrome/skin/default/zotero/importWizard.css +++ b/chrome/skin/default/zotero/importWizard.css @@ -59,6 +59,11 @@ listbox, #result-description, #result-description-html { line-height: 1.5; } +#mendeley-online-description { + font-size: 13px; + line-height: 1.5; +} + #result-description-html a { text-decoration: underline; } From 962222a5f5c2f108bb865fae0c7b15ce60572271 Mon Sep 17 00:00:00 2001 From: Tom Najdek Date: Fri, 23 Apr 2021 01:12:12 +0200 Subject: [PATCH 2/6] Mendeley import: Import annotations from matching files in groups All annotations in all groups are fetched and hashes are compared to detect annotations created on the same file while it was in a group library. Annotations created by other users are filtered out. --- .../zotero/import/mendeley/mendeleyImport.js | 98 ++++++++++++++++--- 1 file changed, 85 insertions(+), 13 deletions(-) diff --git a/chrome/content/zotero/import/mendeley/mendeleyImport.js b/chrome/content/zotero/import/mendeley/mendeleyImport.js index b758359535..003d84287a 100644 --- a/chrome/content/zotero/import/mendeley/mendeleyImport.js +++ b/chrome/content/zotero/import/mendeley/mendeleyImport.js @@ -8,7 +8,7 @@ Services.scriptloader.loadSubScript("chrome://zotero/content/import/mendeley/men Services.scriptloader.loadSubScript("chrome://zotero/content/import/mendeley/mendeleyAPIUtils.js"); const { apiTypeToDBType, apiFieldToDBField } = mendeleyOnlineMappings; -const { apiFetch, getAll } = mendeleyAPIUtils; +const { apiFetch, get, getAll } = mendeleyAPIUtils; var Zotero_Import_Mendeley = function () { this.createNewCollection = null; @@ -135,11 +135,46 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) { let files = this.token ? await this._getDocumentFilesAPI(documents) : await this._getDocumentFilesDB(mendeleyGroupID); - + let annotations = this.token ? await this._getDocumentAnnotationsAPI(mendeleyGroupID) : await this._getDocumentAnnotationsDB(mendeleyGroupID); + let profile = this.token + ? await this._getProfileAPI() + : await this._getProfileDB(); + + let groups = this.token + ? await this._getGroupsAPI() + : await this._getGroupsDB(); + + const fileHashLookup = new Map(); + + for (let [documentID, fileEntries] of files) { + for (let fileEntry of fileEntries) { + fileHashLookup.set(fileEntry.hash, documentID); + } + } + + + for (let group of groups) { + let groupAnnotations = this.token + ? await this._getDocumentAnnotationsAPI(group.id, profile.id) + : await this._getDocumentAnnotationsDB(group.id, profile.id); + + for (let groupAnnotationsList of groupAnnotations.values()) { + for (let groupAnnotation of groupAnnotationsList) { + if (fileHashLookup.has(groupAnnotation.hash)) { + const targetDocumentID = fileHashLookup.get(groupAnnotation.hash); + if (!annotations.has(targetDocumentID)) { + annotations.set(targetDocumentID, []); + } + annotations.get(targetDocumentID).push(groupAnnotation); + } + } + } + } + for (let document of documents) { let docURLs = urls.get(document.id); let docFiles = files.get(document.id); @@ -679,7 +714,7 @@ Zotero_Import_Mendeley.prototype._getDocumentFilesAPI = async function (document } map.set(doc.id, files); } - // @TODO: check if enough space available totalSize + // check if enough space available totalSize await caller.runAll(); return map; }; @@ -688,7 +723,7 @@ Zotero_Import_Mendeley.prototype._getDocumentFilesAPI = async function (document /** * Get a Map of document ids to arrays of annotations */ -Zotero_Import_Mendeley.prototype._getDocumentAnnotationsDB = async function (groupID) { +Zotero_Import_Mendeley.prototype._getDocumentAnnotationsDB = async function (groupID, profileID = null) { var map = new Map(); // Highlights @@ -699,8 +734,9 @@ Zotero_Import_Mendeley.prototype._getDocumentAnnotationsDB = async function (gro + `JOIN RemoteDocuments USING (documentId) ` + `JOIN FileHighlightRects FHR ON (FH.id=FHR.highlightId) ` + `WHERE groupId=? ` + + (profileID !== null ? `AND profileUuid=? ` : ``) + `ORDER BY FH.id, page, y1 DESC, x1`, - groupID + profileID !== null ? [groupID, profileID] : groupID ); var currentHighlight = null; for (let i = 0; i < rows.length; i++) { @@ -743,8 +779,9 @@ Zotero_Import_Mendeley.prototype._getDocumentAnnotationsDB = async function (gro + `FROM FileNotes ` + `JOIN RemoteDocuments USING (documentId) ` + `WHERE groupId=? ` + + (profileID !== null ? `AND profileUuid=? ` : ``) + `ORDER BY page, y, x`, - groupID + profileID !== null ? [groupID, profileID] : groupID ); for (let row of rows) { let docAnnotations = map.get(row.documentId); @@ -767,7 +804,7 @@ Zotero_Import_Mendeley.prototype._getDocumentAnnotationsDB = async function (gro return map; }; -Zotero_Import_Mendeley.prototype._getDocumentAnnotationsAPI = async function (groupID) { +Zotero_Import_Mendeley.prototype._getDocumentAnnotationsAPI = async function (groupID, profileID = null) { const params = {}; if (groupID && groupID !== 0) { @@ -777,6 +814,17 @@ Zotero_Import_Mendeley.prototype._getDocumentAnnotationsAPI = async function (gr const map = new Map(); (await getAll(this.token, 'annotations', params, { Accept: 'application/vnd.mendeley-annotation.1+json' })) .forEach((a) => { + if (profileID !== null && a.profile_id !== profileID) { + // optionally filter annotations by profile id + return; + } + + if (a.type === 'note') { + // This is a "general note" in Mendeley. It appears to be the same thing as + // document.note thus not an annotations and can be discarded + return; + } + const rects = (a.positions || []).map(position => ({ x1: (position.top_left || {}).x || 0, y1: (position.top_left || {}).y || 0, @@ -812,12 +860,6 @@ Zotero_Import_Mendeley.prototype._getDocumentAnnotationsAPI = async function (gr annotation.y = rects[0].y1; } - if (a.type === 'note') { - // This is a "general note" in Mendeley. It appears to be the same thing as - // document.note thus not an annotations and can be discarded - return; - } - if (!map.has(a.document_id)) { map.set(a.document_id, []); } @@ -826,6 +868,36 @@ Zotero_Import_Mendeley.prototype._getDocumentAnnotationsAPI = async function (gr return map; }; +Zotero_Import_Mendeley.prototype._getGroupsAPI = async function () { + const params = { type: 'all' }; + const headers = { Accept: 'application/vnd.mendeley-group-list+json' }; + + return getAll(this.token, 'groups/v2', params, headers); +}; + +Zotero_Import_Mendeley.prototype._getGroupsDB = async function () { + const rows = await this._db.queryAsync( + 'SELECT id, remoteUUid, name, isOwner FROM Groups WHERE remoteUuID != ""' + ); + return rows; +}; + + +Zotero_Import_Mendeley.prototype._getProfileAPI = async function () { + const params = { }; + const headers = { Accept: 'application/vnd.mendeley-profiles.2+json' }; + + return get(this.token, 'profiles/v2/me', params, headers); +}; + +Zotero_Import_Mendeley.prototype._getProfileDB = async function () { + const rows = await this._db.queryAsync( + 'SELECT uuid as id, firstName, lastName, displayName FROM Profiles ORDER BY ROWID LIMIT 1' + ); + + return rows[0]; +}; + /** * Create API JSON array with item and any child attachments or notes */ From d35719d0e83fbafaaf95428bda090ffee7febea7 Mon Sep 17 00:00:00 2001 From: Dan Stillman Date: Wed, 28 Apr 2021 23:45:48 -0400 Subject: [PATCH 3/6] Mendeley online import tweaks - Add "(online import)" and "(local import)" next to "Mendeley Referance Manager" and "Mendeley Desktop", respectively - Adjust size of OAuth window - Use bound parameter instead of string literal for Fx78 compatibility --- chrome/content/zotero/fileInterface.js | 4 ++-- chrome/content/zotero/import/importWizard.js | 4 ++++ chrome/content/zotero/import/importWizard.xul | 4 ++-- chrome/content/zotero/import/mendeley/mendeleyImport.js | 4 ++-- chrome/locale/en-US/zotero/zotero.properties | 2 ++ 5 files changed, 12 insertions(+), 6 deletions(-) diff --git a/chrome/content/zotero/fileInterface.js b/chrome/content/zotero/fileInterface.js index b503f07aaa..31751fd2aa 100644 --- a/chrome/content/zotero/fileInterface.js +++ b/chrome/content/zotero/fileInterface.js @@ -901,8 +901,8 @@ var Zotero_File_Interface = new function() { }; let innerFunc = function () { browser.removeEventListener("pageshow", innerFunc); - win.outerWidth = Math.max(640, Math.min(1024, win.screen.availHeight)); - win.outerHeight = Math.max(480, Math.min(768, win.screen.availWidth)); + win.outerWidth = Math.max(640, Math.min(1000, win.screen.availHeight)); + win.outerHeight = Math.max(480, Math.min(800, win.screen.availWidth)); }; win.addEventListener("load", func); diff --git a/chrome/content/zotero/import/importWizard.js b/chrome/content/zotero/import/importWizard.js index 158372f2d0..5db6b0dbc4 100644 --- a/chrome/content/zotero/import/importWizard.js +++ b/chrome/content/zotero/import/importWizard.js @@ -40,6 +40,10 @@ var Zotero_Import_Wizard = { } // Update labels + document.getElementById('radio-import-source-mendeley-online').label + = `Mendeley Reference Manager (${Zotero.getString('import.onlineImport')})`; + document.getElementById('radio-import-source-mendeley').label + = `Mendeley Desktop (${Zotero.getString('import.localImport')})`; document.getElementById('file-handling-store').label = Zotero.getString( 'import.fileHandling.store', Zotero.appName diff --git a/chrome/content/zotero/import/importWizard.xul b/chrome/content/zotero/import/importWizard.xul index 9603f25bb6..e59b2c0693 100644 --- a/chrome/content/zotero/import/importWizard.xul +++ b/chrome/content/zotero/import/importWizard.xul @@ -21,8 +21,8 @@ onpageadvanced="Zotero_Import_Wizard.onModeChosen(); return false;"> - - diff --git a/chrome/content/zotero/import/mendeley/mendeleyImport.js b/chrome/content/zotero/import/mendeley/mendeleyImport.js index 003d84287a..3a8d7704c4 100644 --- a/chrome/content/zotero/import/mendeley/mendeleyImport.js +++ b/chrome/content/zotero/import/mendeley/mendeleyImport.js @@ -877,7 +877,7 @@ Zotero_Import_Mendeley.prototype._getGroupsAPI = async function () { Zotero_Import_Mendeley.prototype._getGroupsDB = async function () { const rows = await this._db.queryAsync( - 'SELECT id, remoteUUid, name, isOwner FROM Groups WHERE remoteUuID != ""' + "SELECT id, remoteUUid, name, isOwner FROM Groups WHERE remoteUuID != ?", [''] ); return rows; }; @@ -892,7 +892,7 @@ Zotero_Import_Mendeley.prototype._getProfileAPI = async function () { Zotero_Import_Mendeley.prototype._getProfileDB = async function () { const rows = await this._db.queryAsync( - 'SELECT uuid as id, firstName, lastName, displayName FROM Profiles ORDER BY ROWID LIMIT 1' + "SELECT uuid as id, firstName, lastName, displayName FROM Profiles ORDER BY ROWID LIMIT 1" ); return rows[0]; diff --git a/chrome/locale/en-US/zotero/zotero.properties b/chrome/locale/en-US/zotero/zotero.properties index 1c39ec23bd..2275a0ff4d 100644 --- a/chrome/locale/en-US/zotero/zotero.properties +++ b/chrome/locale/en-US/zotero/zotero.properties @@ -749,6 +749,8 @@ fileInterface.exportError = An error occurred while trying to export the selecte fileInterface.importOPML = Import Feeds from OPML fileInterface.OPMLFeedFilter = OPML Feed List +import.onlineImport = online import +import.localImport = local import import.fileHandling.store = Copy files to the %S storage folder import.fileHandling.link = Link to files in original location import.fileHandling.description = Linked files cannot be synced by %S. From 3032cc6a0258b551983615b41677f6dfc8101a30 Mon Sep 17 00:00:00 2001 From: Dan Stillman Date: Wed, 28 Apr 2021 23:45:17 -0400 Subject: [PATCH 4/6] Increase default basicViewer height to 700 Though this is currently persisted, so it stops being relevant as soon as one instance of the basic viewer is resized, which maybe we don't want. --- chrome/content/zotero/standalone/basicViewer.xul | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chrome/content/zotero/standalone/basicViewer.xul b/chrome/content/zotero/standalone/basicViewer.xul index 3db5e5642f..67c7f3f8a4 100644 --- a/chrome/content/zotero/standalone/basicViewer.xul +++ b/chrome/content/zotero/standalone/basicViewer.xul @@ -44,7 +44,7 @@ onload="window.sizeToContent()" windowtype="zotero:basicViewer" title="&brandShortName;" - width="1000" height="500" + width="1000" height="700" persist="screenX screenY width height sizemode">