diff --git a/chrome/content/zotero/fileInterface.js b/chrome/content/zotero/fileInterface.js index 76e0415a22..b503f07aaa 100644 --- a/chrome/content/zotero/fileInterface.js +++ b/chrome/content/zotero/fileInterface.js @@ -23,7 +23,8 @@ ***** END LICENSE BLOCK ***** */ -Components.utils.import("resource://gre/modules/osfile.jsm") +Components.utils.import("resource://gre/modules/osfile.jsm"); +Components.utils.import("resource://gre/modules/Services.jsm"); import FilePicker from 'zotero/filePicker'; /****Zotero_File_Exporter**** @@ -266,7 +267,7 @@ var Zotero_File_Interface = new function() { }; - this.showImportWizard = function () { + this.showImportWizard = function (extraArgs = {}) { var libraryID = Zotero.Libraries.userLibraryID; try { let zp = Zotero.getActiveZoteroPane(); @@ -276,7 +277,8 @@ var Zotero_File_Interface = new function() { Zotero.logError(e); } var args = { - libraryID + libraryID, + ...extraArgs }; args.wrappedJSObject = args; @@ -329,30 +331,39 @@ var Zotero_File_Interface = new function() { var defaultNewCollectionPrefix = Zotero.getString("fileInterface.imported"); var translation; - // Check if the file is an SQLite database - var sample = yield Zotero.File.getSample(file.path); - if (file.path == Zotero.DataDirectory.getDatabase()) { - // Blacklist the current Zotero database, which would cause a hang - } - else if (Zotero.MIME.sniffForMIMEType(sample) == 'application/x-sqlite3') { - // Mendeley import doesn't use the real translation architecture, but we create a - // translation object with the same interface + + if (options.mendeleyOnlineToken) { translation = yield _getMendeleyTranslation(); translation.createNewCollection = createNewCollection; - defaultNewCollectionPrefix = Zotero.getString( - 'fileInterface.appImportCollection', 'Mendeley' - ); + translation.token = options.mendeleyOnlineToken; } - else if (file.path.endsWith('@www.mendeley.com.sqlite') - || file.path.endsWith('online.sqlite')) { - // Keep in sync with importWizard.js - throw new Error('Encrypted Mendeley database'); + else { + // Check if the file is an SQLite database + var sample = yield Zotero.File.getSample(file.path); + if (file.path == Zotero.DataDirectory.getDatabase()) { + // Blacklist the current Zotero database, which would cause a hang + } + else if (Zotero.MIME.sniffForMIMEType(sample) == 'application/x-sqlite3') { + // Mendeley import doesn't use the real translation architecture, but we create a + // translation object with the same interface + translation = yield _getMendeleyTranslation(); + translation.createNewCollection = createNewCollection; + defaultNewCollectionPrefix = Zotero.getString( + 'fileInterface.appImportCollection', 'Mendeley' + ); + } + else if (file.path.endsWith('@www.mendeley.com.sqlite') + || file.path.endsWith('online.sqlite')) { + // Keep in sync with importWizard.js + throw new Error('Encrypted Mendeley database'); + } + + if (!translation) { + translation = new Zotero.Translate.Import(); + } + translation.setLocation(file); } - - if (!translation) { - translation = new Zotero.Translate.Import(); - } - translation.setLocation(file); + return _finishImport({ translation, createNewCollection, @@ -592,7 +603,7 @@ var Zotero_File_Interface = new function() { eval(xmlhttp.response); } return new Zotero_Import_Mendeley(); - } + }; /** @@ -849,7 +860,57 @@ var Zotero_File_Interface = new function() { return false; } } -} + + this.authenticateMendeleyOnlinePoll = function (win) { + if (win && win[0] && win[0].location) { + const matchResult = win[0].location.toString().match(/access_token=(.*?)(?:&|$)/i); + if (matchResult) { + const mendeleyAccessToken = matchResult[1]; + Zotero.getMainWindow().setTimeout(() => this.showImportWizard({ mendeleyAccessToken }), 0); + win.close(); + return; + } + } + + if (win && !win.closed) { + Zotero.getMainWindow().setTimeout(this.authenticateMendeleyOnlinePoll.bind(this, win), 200); + } + }; + + this.authenticateMendeleyOnline = function () { + const uri = 'https://api.mendeley.com/oauth/authorize?client_id=5907&redirect_uri=https%3A%2F%2Fzotero-static.s3.amazonaws.com%2Fmendeley_oauth_redirect.html&response_type=token&state=&scope=all'; + + var win = Services.wm.getMostRecentWindow("zotero:basicViewer"); + if (win) { + win.loadURI(uri); + } + else { + const ww = Services.ww; + const arg = Components.classes["@mozilla.org/supports-string;1"] + .createInstance(Components.interfaces.nsISupportsString); + arg.data = uri; + win = ww.openWindow(null, "chrome://zotero/content/standalone/basicViewer.xul", + "basicViewer", "chrome,dialog=yes,resizable,centerscreen,menubar,scrollbars", arg); + } + + let browser; + let func = function () { + win.removeEventListener("load", func); + browser = win.document.documentElement.getElementsByTagName('browser')[0]; + browser.addEventListener("pageshow", innerFunc); + }; + let innerFunc = function () { + browser.removeEventListener("pageshow", innerFunc); + win.outerWidth = Math.max(640, Math.min(1024, win.screen.availHeight)); + win.outerHeight = Math.max(480, Math.min(768, win.screen.availWidth)); + }; + + win.addEventListener("load", func); + + // polling executed by the main window because current (wizard) window will be closed + Zotero.getMainWindow().setTimeout(this.authenticateMendeleyOnlinePoll.bind(this, win), 200); + }; +}; // Handles the display of a progress indicator Zotero_File_Interface.Progress = new function() { diff --git a/chrome/content/zotero/import/importWizard.js b/chrome/content/zotero/import/importWizard.js index 559d8e8a9e..158372f2d0 100644 --- a/chrome/content/zotero/import/importWizard.js +++ b/chrome/content/zotero/import/importWizard.js @@ -5,11 +5,12 @@ var Zotero_Import_Wizard = { _dbs: null, _file: null, _translation: null, + _mendeleyOnlineRedirectURLWithCode: null, + _mendeleyAccessToken: null, init: async function () { this._wizard = document.getElementById('import-wizard'); - var dbs = await Zotero_File_Interface.findMendeleyDatabases(); if (dbs.length) { document.getElementById('radio-import-source-mendeley').hidden = false; @@ -32,6 +33,11 @@ var Zotero_Import_Wizard = { document.getElementById('create-collection-checkbox').removeAttribute('checked'); } } + + if (args && args.mendeleyAccessToken) { + this._mendeleyAccessToken = args.mendeleyAccessToken; + this._wizard.goTo('page-options'); + } // Update labels document.getElementById('file-handling-store').label = Zotero.getString( @@ -57,6 +63,11 @@ var Zotero_Import_Wizard = { case 'radio-import-source-file': await this.chooseFile(); break; + + case 'radio-import-source-mendeley-online': + wizard.goTo('mendeley-online-explanation'); + wizard.canRewind = true; + break; case 'radio-import-source-mendeley': this._dbs = await Zotero_File_Interface.findMendeleyDatabases(); @@ -85,7 +96,19 @@ var Zotero_Import_Wizard = { throw e; } }, - + + onMendeleyOnlineShow: async function () { + document.getElementById('mendeley-online-description').textContent = Zotero.getString( + 'import.mendeleyOnline.intro', [Zotero.appName, 'Mendeley Reference Manager', 'Mendeley'] + ); + }, + + onMendeleyOnlineAdvance: function () { + if (!this._mendeleyOnlineRedirectURLWithCode) { + Zotero_File_Interface.authenticateMendeleyOnline(); + window.close(); + } + }, goToStart: function () { this._wizard.goTo('page-start'); @@ -165,7 +188,7 @@ var Zotero_Import_Wizard = { onOptionsShown: function () { - + document.getElementById('file-handling-options').hidden = !!this._mendeleyAccessToken; }, @@ -192,7 +215,7 @@ var Zotero_Import_Wizard = { onImportStart: async function () { - if (!this._file) { + if (!this._file && !this._mendeleyAccessToken) { let index = document.getElementById('file-list').selectedIndex; this._file = this._dbs[index].path; } @@ -206,7 +229,8 @@ var Zotero_Import_Wizard = { onBeforeImport: this.onBeforeImport.bind(this), addToLibraryRoot: !document.getElementById('create-collection-checkbox') .hasAttribute('checked'), - linkFiles: document.getElementById('file-handling-radio').selectedIndex == 1 + linkFiles: document.getElementById('file-handling-radio').selectedIndex == 1, + mendeleyOnlineToken: this._mendeleyAccessToken }); // Cancelled by user or due to error @@ -321,7 +345,7 @@ var Zotero_Import_Wizard = { xulElem.hidden = false; htmlElem.setAttribute('display', 'none'); } - document.getElementById('result-description') + document.getElementById('result-description'); if (showReportErrorButton) { let button = document.getElementById('result-report-error'); diff --git a/chrome/content/zotero/import/importWizard.xul b/chrome/content/zotero/import/importWizard.xul index 8721aed14f..9603f25bb6 100644 --- a/chrome/content/zotero/import/importWizard.xul +++ b/chrome/content/zotero/import/importWizard.xul @@ -21,9 +21,20 @@ onpageadvanced="Zotero_Import_Wizard.onModeChosen(); return false;"> - + + + + - { + let next = null; + let links = response.getResponseHeader('link'); + if (links) { + const matches = links.match(/<(.*?)>;\s+rel="next"/i); + + if (matches && matches.length > 1) { + next = matches[1]; + } + } + return next; +}; + +const apiFetchUrl = async (token, url, headers = {}, options = {}) => { + headers = { ...headers, Authorization: `Bearer ${token}` }; + return Zotero.HTTP.request('GET', url, { ...options, headers }); +}; + +const apiFetch = async (token, endPoint, params = {}, headers = {}, options = {}) => { + const stringParams = Object.entries(params).map(p => p.join('=')).join('&'); + const url = MENDELEY_API_URL + '/' + endPoint + '?' + stringParams; + return apiFetchUrl(token, url, headers, options); +}; + +const get = async (token, endPoint, params = {}, headers = {}, options = {}) => { + const response = await apiFetch(token, endPoint, params, headers, options); + return JSON.parse(response.responseText); +}; + +const getAll = async (token, endPoint, params = {}, headers = {}, options = {}) => { + const PER_PAGE = endPoint === 'annotations' ? 200 : 500; + const response = await apiFetch(token, endPoint, { ...params, limit: PER_PAGE }, headers, options); + var next = getNextLinkFromResponse(response); + var data = JSON.parse(response.responseText); + + while (next) { + const response = await apiFetchUrl(token, next, headers, options); //eslint-disable-line no-await-in-loop + data = [...data, ...JSON.parse(response.responseText)]; + next = getNextLinkFromResponse(response); + } + + return data; +}; + + +return { getNextLinkFromResponse, apiFetch, apiFetchUrl, get, getAll }; +})(); diff --git a/chrome/content/zotero/import/mendeley/mendeleyImport.js b/chrome/content/zotero/import/mendeley/mendeleyImport.js index 890ccb2c7c..b758359535 100644 --- a/chrome/content/zotero/import/mendeley/mendeleyImport.js +++ b/chrome/content/zotero/import/mendeley/mendeleyImport.js @@ -1,19 +1,27 @@ +/* global map:false, mendeleyOnlineMappings:false, mendeleyAPIUtils:false */ var EXPORTED_SYMBOLS = ["Zotero_Import_Mendeley"]; Components.utils.import("resource://gre/modules/Services.jsm"); Components.utils.import("resource://gre/modules/osfile.jsm"); Services.scriptloader.loadSubScript("chrome://zotero/content/include.js"); +Services.scriptloader.loadSubScript("chrome://zotero/content/import/mendeley/mendeleyOnlineMappings.js"); +Services.scriptloader.loadSubScript("chrome://zotero/content/import/mendeley/mendeleyAPIUtils.js"); + +const { apiTypeToDBType, apiFieldToDBField } = mendeleyOnlineMappings; +const { apiFetch, getAll } = mendeleyAPIUtils; var Zotero_Import_Mendeley = function () { this.createNewCollection = null; this.linkFiles = null; this.newItems = []; + this.token = null; this._db; this._file; this._itemDone; this._progress = 0; this._progressMax; + this._tmpFilesToDelete = []; }; Zotero_Import_Mendeley.prototype.setLocation = function (file) { @@ -42,6 +50,7 @@ Zotero_Import_Mendeley.prototype.setTranslator = function () {}; Zotero_Import_Mendeley.prototype.translate = async function (options = {}) { this._linkFiles = options.linkFiles; + this.timestamp = Date.now(); if (true) { Services.scriptloader.loadSubScript("chrome://zotero/content/import/mendeley/mendeleySchemaMap.js"); @@ -77,39 +86,84 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) { // Disable syncing while we're importing var resumeSync = Zotero.Sync.Runner.delayIndefinite(); - this._db = new Zotero.DBConnection(this._file); + if (this._file) { + this._db = new Zotero.DBConnection(this._file); + } try { - if (!await this._isValidDatabase()) { + if (this._file && !await this._isValidDatabase()) { throw new Error("Not a valid Mendeley database"); } - - // Collections - let folders = await this._getFolders(mendeleyGroupID); - let collectionJSON = this._foldersToAPIJSON(folders, rootCollectionKey); - let folderKeys = this._getFolderKeys(collectionJSON); + + if (!this._file && !this.token) { + throw new Error("Missing import token"); + } + + const folders = this.token + ? await this._getFoldersAPI(mendeleyGroupID) + : await this._getFoldersDB(mendeleyGroupID); + + const collectionJSON = this._foldersToAPIJSON(folders, rootCollectionKey); + const folderKeys = this._getFolderKeys(collectionJSON); await this._saveCollections(libraryID, collectionJSON, folderKeys); // // Items // - let documents = await this._getDocuments(mendeleyGroupID); + let documents = this.token + ? await this._getDocumentsAPI(mendeleyGroupID) + : await this._getDocumentsDB(mendeleyGroupID); + this._progressMax = documents.length; // Get various attributes mapped to document ids - let urls = await this._getDocumentURLs(mendeleyGroupID); - let creators = await this._getDocumentCreators(mendeleyGroupID, map.creatorTypes); - let tags = await this._getDocumentTags(mendeleyGroupID); - let collections = await this._getDocumentCollections( - mendeleyGroupID, - documents, - rootCollectionKey, - folderKeys - ); - let files = await this._getDocumentFiles(mendeleyGroupID); - let annotations = await this._getDocumentAnnotations(mendeleyGroupID); + let urls = this.token + ? await this._getDocumentURLsAPI(documents) + : await this._getDocumentURLsDB(mendeleyGroupID); + + let creators = this.token + ? await this._getDocumentCreatorsAPI(documents) + : await this._getDocumentCreatorsDB(mendeleyGroupID, map.creatorTypes); + + let tags = this.token + ? await this._getDocumentTagsAPI(documents) + : await this._getDocumentTagsDB(mendeleyGroupID); + + let collections = this.token + ? await this._getDocumentCollectionsAPI(documents, rootCollectionKey, folderKeys) + : await this._getDocumentCollectionsDB(mendeleyGroupID, documents, rootCollectionKey, folderKeys); + + let files = this.token + ? await this._getDocumentFilesAPI(documents) + : await this._getDocumentFilesDB(mendeleyGroupID); + + let annotations = this.token + ? await this._getDocumentAnnotationsAPI(mendeleyGroupID) + : await this._getDocumentAnnotationsDB(mendeleyGroupID); + for (let document of documents) { let docURLs = urls.get(document.id); let docFiles = files.get(document.id); + + if (this.token) { + // extract identifiers + ['arxiv', 'doi', 'isbn', 'issn', 'pmid', 'scopus', 'pui', 'pii', 'sgr'].forEach( + i => document[i] = (document.identifiers || {})[i] + ); + + // normalise item type from the API to match Mendeley DB + document.type = apiTypeToDBType[document.type] || document.type; + + // normalise field names from the API to match Mendeley DB + Object.keys(apiFieldToDBField).forEach((key) => { + if (key in document) { + const newKey = apiFieldToDBField[key]; + if (newKey) { + document[newKey] = document[key]; + } + delete document[key]; + } + }); + } // If there's a single PDF file, use "PDF" for the attachment title if (docFiles && docFiles.length == 1 && docFiles[0].fileURL.endsWith('.pdf')) { @@ -173,7 +227,14 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) { } finally { try { - await this._db.closeDatabase(); + if (this._file) { + await this._db.closeDatabase(); + } + if (this.token) { + await Promise.all( + this._tmpFilesToDelete.map(f => this._removeTemporaryFile(f)) + ); + } } catch (e) { Zotero.logError(e); @@ -183,6 +244,18 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) { } }; +Zotero_Import_Mendeley.prototype._removeTemporaryFile = async function (file) { + const containingDir = OS.Path.dirname(file); + try { + await Zotero.File.removeIfExists(file); + await OS.File.removeEmptyDir(containingDir); + } + catch (e) { + Zotero.logError(e); + } +}; + + Zotero_Import_Mendeley.prototype._isValidDatabase = async function () { var tables = [ 'DocumentContributors', @@ -208,7 +281,7 @@ Zotero_Import_Mendeley.prototype._isValidDatabase = async function () { // // Collections // -Zotero_Import_Mendeley.prototype._getFolders = async function (groupID) { +Zotero_Import_Mendeley.prototype._getFoldersDB = async function (groupID) { return this._db.queryAsync( `SELECT F.id, F.uuid, F.name, ` // Top-level folders can have a parentId of 0 instead of -1 (by mistake?) @@ -221,6 +294,23 @@ Zotero_Import_Mendeley.prototype._getFolders = async function (groupID) { ); }; +Zotero_Import_Mendeley.prototype._getFoldersAPI = async function (groupID) { + const params = {}; + const headers = { Accept: 'application/vnd.mendeley-folder.1+json' }; + + if (groupID && groupID !== 0) { + params.group_id = groupID; //eslint-disable-line camelcase + } + + return (await getAll(this.token, 'folders', params, headers)).map(f => ({ + id: f.id, + uuid: f.id, + name: f.name, + parentId: f.parent_id || -1, + remoteUuid: f.id + })); +}; + /** * Get flat array of collection API JSON with parentCollection set * @@ -337,13 +427,13 @@ Zotero_Import_Mendeley.prototype._findExistingCollection = async function (libra Zotero.debug(`Found existing collection ${collections[0].libraryKey} for ` + `${predicate} ${collectionJSON.relations[predicate]}`); return collections[0]; -} +}; // // Items // -Zotero_Import_Mendeley.prototype._getDocuments = async function (groupID) { +Zotero_Import_Mendeley.prototype._getDocumentsDB = async function (groupID) { return this._db.queryAsync( `SELECT D.*, RD.remoteUuid FROM Documents D ` + `JOIN RemoteDocuments RD ON (D.id=RD.documentId) ` @@ -352,12 +442,28 @@ Zotero_Import_Mendeley.prototype._getDocuments = async function (groupID) { ); }; +Zotero_Import_Mendeley.prototype._getDocumentsAPI = async function (groupID) { + const params = { view: 'all' }; + const headers = { Accept: 'application/vnd.mendeley-document-with-files-list+json' }; + + if (groupID && groupID !== 0) { + params.group_id = groupID; //eslint-disable-line camelcase + } + + + return (await getAll(this.token, 'documents', params, headers)).map(d => ({ + ...d, + uuid: d.id, + remoteUuid: d.id + })); +}; + /** * Get a Map of document ids to arrays of URLs * * @return {Map} */ -Zotero_Import_Mendeley.prototype._getDocumentURLs = async function (groupID) { +Zotero_Import_Mendeley.prototype._getDocumentURLsDB = async function (groupID) { var rows = await this._db.queryAsync( `SELECT documentId, CAST(url AS TEXT) AS url FROM DocumentUrls DU ` + `JOIN RemoteDocuments USING (documentId) ` @@ -374,13 +480,17 @@ Zotero_Import_Mendeley.prototype._getDocumentURLs = async function (groupID) { return map; }; +Zotero_Import_Mendeley.prototype._getDocumentURLsAPI = async function (documents) { + return new Map(documents.map(d => ([d.id, d.websites]))); +}; + /** * Get a Map of document ids to arrays of creator API JSON * * @param {Integer} groupID * @param {Object} creatorTypeMap - Mapping of Mendeley creator types to Zotero creator types */ -Zotero_Import_Mendeley.prototype._getDocumentCreators = async function (groupID, creatorTypeMap) { +Zotero_Import_Mendeley.prototype._getDocumentCreatorsDB = async function (groupID, creatorTypeMap) { var rows = await this._db.queryAsync( `SELECT * FROM DocumentContributors ` + `JOIN RemoteDocuments USING (documentId) ` @@ -401,10 +511,21 @@ Zotero_Import_Mendeley.prototype._getDocumentCreators = async function (groupID, return map; }; +Zotero_Import_Mendeley.prototype._getDocumentCreatorsAPI = async function (documents) { + var map = new Map(); + for (let doc of documents) { + const authors = (doc.authors || []).map(c => this._makeCreator('author', c.first_name, c.last_name)); + const editors = (doc.editors || []).map(c => this._makeCreator('editor', c.first_name, c.last_name)); + const translators = (doc.translators || []).map(c => this._makeCreator('translator', c.first_name, c.last_name)); + map.set(doc.id, [...authors, ...editors, ...translators]); + } + return map; +}; + /** * Get a Map of document ids to arrays of tag API JSON */ -Zotero_Import_Mendeley.prototype._getDocumentTags = async function (groupID) { +Zotero_Import_Mendeley.prototype._getDocumentTagsDB = async function (groupID) { var rows = await this._db.queryAsync( // Manual tags `SELECT documentId, tag, 0 AS type FROM DocumentTags ` @@ -432,10 +553,19 @@ Zotero_Import_Mendeley.prototype._getDocumentTags = async function (groupID) { return map; }; +Zotero_Import_Mendeley.prototype._getDocumentTagsAPI = async function (documents) { + var map = new Map(); + for (let doc of documents) { + const tags = [...(doc.tags || []).map(tag => ({ tag, type: 0 })), ...(doc.keywords || []).map(tag => ({ tag, type: 1 }))]; + map.set(doc.id, tags); + } + return map; +}; + /** * Get a Map of document ids to arrays of collection keys */ -Zotero_Import_Mendeley.prototype._getDocumentCollections = async function (groupID, documents, rootCollectionKey, folderKeys) { +Zotero_Import_Mendeley.prototype._getDocumentCollectionsDB = async function (groupID, documents, rootCollectionKey, folderKeys) { var rows = await this._db.queryAsync( `SELECT documentId, folderId FROM DocumentFolders DF ` + `JOIN RemoteDocuments USING (documentId) ` @@ -460,12 +590,28 @@ Zotero_Import_Mendeley.prototype._getDocumentCollections = async function (group return map; }; +Zotero_Import_Mendeley.prototype._getDocumentCollectionsAPI = async function (documents, rootCollectionKey, folderKeys) { + return new Map( + documents.map((d) => { + const keys = (d.folder_uuids || []).map((fuuid) => { + const key = folderKeys.get(fuuid); + if (!key) { + Zotero.debug(`Document folder ${fuuid} not found -- skipping`, 2); + } + return key; + }).filter(Boolean); + // Add all documents to root collection if specified + return [d.id, [...keys, ...(rootCollectionKey ? [rootCollectionKey] : [])]]; + }) + ); +}; + /** * Get a Map of document ids to arrays of file metadata * * @return {Map} */ -Zotero_Import_Mendeley.prototype._getDocumentFiles = async function (groupID) { +Zotero_Import_Mendeley.prototype._getDocumentFilesDB = async function (groupID) { var rows = await this._db.queryAsync( `SELECT documentId, hash, localUrl FROM DocumentFiles ` + `JOIN Files USING (hash) ` @@ -490,10 +636,59 @@ Zotero_Import_Mendeley.prototype._getDocumentFiles = async function (groupID) { return map; }; +Zotero_Import_Mendeley.prototype._fetchFile = async function (fileID, filePath) { + const fileDir = OS.Path.dirname(filePath); + await Zotero.File.createDirectoryIfMissingAsync(fileDir); + const xhr = await apiFetch(this.token, `files/${fileID}`, {}, {}, { responseType: 'blob', followRedirects: false }); + const uri = xhr.getResponseHeader('location'); + await Zotero.File.download(uri, filePath); + + this._progress += 1; + if (this._itemDone) { + this._itemDone(); + } +}; + +Zotero_Import_Mendeley.prototype._getDocumentFilesAPI = async function (documents) { + const map = new Map(); + + let totalSize = 0; + + Components.utils.import("resource://zotero/concurrentCaller.js"); + var caller = new ConcurrentCaller({ + numConcurrent: 6, + onError: e => Zotero.logError(e), + Promise: Zotero.Promise + }); + + for (let doc of documents) { + const files = []; + for (let file of (doc.files || [])) { + const fileName = file.file_name || 'file'; + const tmpFile = OS.Path.join(Zotero.getTempDirectory().path, `mendeley-online-import-${this.timestamp}-${file.id}`, fileName); + this._tmpFilesToDelete.push(tmpFile); + caller.add(this._fetchFile.bind(this, file.id, tmpFile)); + files.push({ + fileURL: OS.Path.toFileURI(tmpFile), + title: file.file_name || '', + contentType: file.mime_type || '', + hash: file.filehash, + }); + totalSize += file.size; + this._progressMax += 1; + } + map.set(doc.id, files); + } + // @TODO: check if enough space available totalSize + await caller.runAll(); + return map; +}; + + /** * Get a Map of document ids to arrays of annotations */ -Zotero_Import_Mendeley.prototype._getDocumentAnnotations = async function (groupID) { +Zotero_Import_Mendeley.prototype._getDocumentAnnotationsDB = async function (groupID) { var map = new Map(); // Highlights @@ -572,6 +767,65 @@ Zotero_Import_Mendeley.prototype._getDocumentAnnotations = async function (group return map; }; +Zotero_Import_Mendeley.prototype._getDocumentAnnotationsAPI = async function (groupID) { + const params = {}; + + if (groupID && groupID !== 0) { + params.group_id = groupID; //eslint-disable-line camelcase + } + + const map = new Map(); + (await getAll(this.token, 'annotations', params, { Accept: 'application/vnd.mendeley-annotation.1+json' })) + .forEach((a) => { + const rects = (a.positions || []).map(position => ({ + x1: (position.top_left || {}).x || 0, + y1: (position.top_left || {}).y || 0, + x2: (position.bottom_right || {}).x || 0, + y2: (position.bottom_right || {}).y || 0, + })); + let page = 1; + try { + // const page = ((a.positions || [])[0] || {}).page; // ??? + page = a.positions[0].page; + } + catch (e) { } + + const annotation = { + id: a.id, + color: a.color ? `#${a.color.r.toString(16)}${a.color.g.toString(16)}${a.color.b.toString(16)}` : null, + dateAdded: a.created, + dateModified: a.last_modified, + hash: a.filehash, + uuid: a.id, + page, + }; + + if (a.type === 'highlight') { + annotation.type = 'highlight'; + annotation.rects = rects; + } + + if (a.type === 'sticky_note' && rects.length > 0) { + annotation.type = 'note'; + annotation.note = a.text; + annotation.x = rects[0].x1; + annotation.y = rects[0].y1; + } + + if (a.type === 'note') { + // This is a "general note" in Mendeley. It appears to be the same thing as + // document.note thus not an annotations and can be discarded + return; + } + + if (!map.has(a.document_id)) { + map.set(a.document_id, []); + } + map.get(a.document_id).push(annotation); + }); + return map; +}; + /** * Create API JSON array with item and any child attachments or notes */ @@ -1124,7 +1378,8 @@ Zotero_Import_Mendeley.prototype._isDownloadedFile = function (path) { return parentDir.endsWith(OS.Path.join('Application Support', 'Mendeley Desktop', 'Downloaded')) || parentDir.endsWith(OS.Path.join('Local', 'Mendeley Ltd', 'Mendeley Desktop', 'Downloaded')) || parentDir.endsWith(OS.Path.join('Local', 'Mendeley Ltd.', 'Mendeley Desktop', 'Downloaded')) - || parentDir.endsWith(OS.Path.join('data', 'Mendeley Ltd.', 'Mendeley Desktop', 'Downloaded')); + || parentDir.endsWith(OS.Path.join('data', 'Mendeley Ltd.', 'Mendeley Desktop', 'Downloaded')) + || parentDir.startsWith(OS.Path.join(Zotero.getTempDirectory().path, 'mendeley-online-import')); // Mendeley Online Importer } /** diff --git a/chrome/content/zotero/import/mendeley/mendeleyOnlineMappings.js b/chrome/content/zotero/import/mendeley/mendeleyOnlineMappings.js new file mode 100644 index 0000000000..1d81ffeabe --- /dev/null +++ b/chrome/content/zotero/import/mendeley/mendeleyOnlineMappings.js @@ -0,0 +1,57 @@ +/* eslint-disable camelcase, no-unused-vars */ + +var mendeleyOnlineMappings = { + // lookup to normalise from item type presented by API to item type as stored in DB + apiTypeToDBType: { + bill: 'Bill', + book: 'Book', + book_section: 'BookSection', + case: 'Case', + computer_program: 'ComputerProgram', + conference_proceedings: 'ConferenceProceedings', + encyclopedia_article: 'EncyclopediaArticle', + film: 'Film', + generic: 'Generic', + hearing: 'Hearing', + journal: 'JournalArticle', + magazine_article: 'MagazineArticle', + newspaper_article: 'NewspaperArticle', + patent: 'Patent', + report: 'Report', + statute: 'Statute', + television_broadcast: 'TelevisionBroadcast', + thesis: 'Thesis', + web_page: 'WebPage', + working_paper: 'WorkingPaper' + }, + apiFieldToDBField: { + accessed: 'dateAccessed', + authors: false, // all author types handled separately + citation_key: 'citationKey', + created: 'added', + edition: 'edition', + editors: false, // all author types handled separately + file_attached: false, + folder_uuids: false, // collections handled separately + group_id: 'groupID', + identifiers: false, // identifiers are separately copied directly into document + keywords: false, // tags handled separately + last_modified: 'modified', + notes: 'note', + patent_application_number: 'patentApplicationNumber', + patent_legal_status: 'patentLegalStatus', + patent_owner: 'patentOwner', + private_publication: 'privatePublication', + profile_id: 'profileID', + reprint_edition: 'reprintEdition', + revision: 'revisionNumber', + series_editor: 'seriesEditor', + series_number: 'seriesNumber', + short_title: 'shortTitle', + source_type: 'sourceType', + tags: false, // tags handled separately + translators: false, // all author types handled separately + user_context: 'userContext', + websites: false // URLs handled separately + } +}; diff --git a/chrome/content/zotero/import/mendeley/mendeleySchemaMap.js b/chrome/content/zotero/import/mendeley/mendeleySchemaMap.js index af8e929e33..51ca57f8ff 100644 --- a/chrome/content/zotero/import/mendeley/mendeleySchemaMap.js +++ b/chrome/content/zotero/import/mendeley/mendeleySchemaMap.js @@ -10,6 +10,7 @@ var map = { EncyclopediaArticle: "encyclopediaArticle", Film: "film", Generic: "document", + Hearing: "hearing", JournalArticle: "journalArticle", MagazineArticle: "magazineArticle", NewspaperArticle: "newspaperArticle", diff --git a/chrome/locale/en-US/zotero/zotero.properties b/chrome/locale/en-US/zotero/zotero.properties index 9c186783ba..1c39ec23bd 100644 --- a/chrome/locale/en-US/zotero/zotero.properties +++ b/chrome/locale/en-US/zotero/zotero.properties @@ -752,6 +752,7 @@ fileInterface.OPMLFeedFilter = OPML Feed List import.fileHandling.store = Copy files to the %S storage folder import.fileHandling.link = Link to files in original location import.fileHandling.description = Linked files cannot be synced by %S. +import.mendeleyOnline.intro = On the next page you will be asked to log in to %2$S and grant %1$S access. This is necessary to import your %3$S library into %1$S. quickCopy.copyAs = Copy as %S diff --git a/chrome/skin/default/zotero/importWizard.css b/chrome/skin/default/zotero/importWizard.css index a690158aac..e759b0878e 100644 --- a/chrome/skin/default/zotero/importWizard.css +++ b/chrome/skin/default/zotero/importWizard.css @@ -59,6 +59,11 @@ listbox, #result-description, #result-description-html { line-height: 1.5; } +#mendeley-online-description { + font-size: 13px; + line-height: 1.5; +} + #result-description-html a { text-decoration: underline; }