Merge branch 'mendeley-online-importer'

This commit is contained in:
Dan Stillman 2021-04-29 00:48:08 -04:00
commit cdef45d6c3
12 changed files with 641 additions and 88 deletions

View file

@ -23,7 +23,8 @@
***** END LICENSE BLOCK *****
*/
Components.utils.import("resource://gre/modules/osfile.jsm")
Components.utils.import("resource://gre/modules/osfile.jsm");
Components.utils.import("resource://gre/modules/Services.jsm");
import FilePicker from 'zotero/filePicker';
/****Zotero_File_Exporter****
@ -266,7 +267,7 @@ var Zotero_File_Interface = new function() {
};
this.showImportWizard = function () {
this.showImportWizard = function (extraArgs = {}) {
var libraryID = Zotero.Libraries.userLibraryID;
try {
let zp = Zotero.getActiveZoteroPane();
@ -276,7 +277,8 @@ var Zotero_File_Interface = new function() {
Zotero.logError(e);
}
var args = {
libraryID
libraryID,
...extraArgs
};
args.wrappedJSObject = args;
@ -329,30 +331,39 @@ var Zotero_File_Interface = new function() {
var defaultNewCollectionPrefix = Zotero.getString("fileInterface.imported");
var translation;
// Check if the file is an SQLite database
var sample = yield Zotero.File.getSample(file.path);
if (file.path == Zotero.DataDirectory.getDatabase()) {
// Blacklist the current Zotero database, which would cause a hang
}
else if (Zotero.MIME.sniffForMIMEType(sample) == 'application/x-sqlite3') {
// Mendeley import doesn't use the real translation architecture, but we create a
// translation object with the same interface
if (options.mendeleyOnlineToken) {
translation = yield _getMendeleyTranslation();
translation.createNewCollection = createNewCollection;
defaultNewCollectionPrefix = Zotero.getString(
'fileInterface.appImportCollection', 'Mendeley'
);
translation.token = options.mendeleyOnlineToken;
}
else if (file.path.endsWith('@www.mendeley.com.sqlite')
|| file.path.endsWith('online.sqlite')) {
// Keep in sync with importWizard.js
throw new Error('Encrypted Mendeley database');
else {
// Check if the file is an SQLite database
var sample = yield Zotero.File.getSample(file.path);
if (file.path == Zotero.DataDirectory.getDatabase()) {
// Blacklist the current Zotero database, which would cause a hang
}
else if (Zotero.MIME.sniffForMIMEType(sample) == 'application/x-sqlite3') {
// Mendeley import doesn't use the real translation architecture, but we create a
// translation object with the same interface
translation = yield _getMendeleyTranslation();
translation.createNewCollection = createNewCollection;
defaultNewCollectionPrefix = Zotero.getString(
'fileInterface.appImportCollection', 'Mendeley'
);
}
else if (file.path.endsWith('@www.mendeley.com.sqlite')
|| file.path.endsWith('online.sqlite')) {
// Keep in sync with importWizard.js
throw new Error('Encrypted Mendeley database');
}
if (!translation) {
translation = new Zotero.Translate.Import();
}
translation.setLocation(file);
}
if (!translation) {
translation = new Zotero.Translate.Import();
}
translation.setLocation(file);
return _finishImport({
translation,
createNewCollection,
@ -592,7 +603,7 @@ var Zotero_File_Interface = new function() {
eval(xmlhttp.response);
}
return new Zotero_Import_Mendeley();
}
};
/**
@ -849,7 +860,57 @@ var Zotero_File_Interface = new function() {
return false;
}
}
}
this.authenticateMendeleyOnlinePoll = function (win) {
if (win && win[0] && win[0].location) {
const matchResult = win[0].location.toString().match(/access_token=(.*?)(?:&|$)/i);
if (matchResult) {
const mendeleyAccessToken = matchResult[1];
Zotero.getMainWindow().setTimeout(() => this.showImportWizard({ mendeleyAccessToken }), 0);
win.close();
return;
}
}
if (win && !win.closed) {
Zotero.getMainWindow().setTimeout(this.authenticateMendeleyOnlinePoll.bind(this, win), 200);
}
};
this.authenticateMendeleyOnline = function () {
const uri = 'https://api.mendeley.com/oauth/authorize?client_id=5907&redirect_uri=https%3A%2F%2Fzotero-static.s3.amazonaws.com%2Fmendeley_oauth_redirect.html&response_type=token&state=&scope=all';
var win = Services.wm.getMostRecentWindow("zotero:basicViewer");
if (win) {
win.loadURI(uri);
}
else {
const ww = Services.ww;
const arg = Components.classes["@mozilla.org/supports-string;1"]
.createInstance(Components.interfaces.nsISupportsString);
arg.data = uri;
win = ww.openWindow(null, "chrome://zotero/content/standalone/basicViewer.xul",
"basicViewer", "chrome,dialog=yes,resizable,centerscreen,menubar,scrollbars", arg);
}
let browser;
let func = function () {
win.removeEventListener("load", func);
browser = win.document.documentElement.getElementsByTagName('browser')[0];
browser.addEventListener("pageshow", innerFunc);
};
let innerFunc = function () {
browser.removeEventListener("pageshow", innerFunc);
win.outerWidth = Math.max(640, Math.min(1000, win.screen.availHeight));
win.outerHeight = Math.max(480, Math.min(800, win.screen.availWidth));
};
win.addEventListener("load", func);
// polling executed by the main window because current (wizard) window will be closed
Zotero.getMainWindow().setTimeout(this.authenticateMendeleyOnlinePoll.bind(this, win), 200);
};
};
// Handles the display of a progress indicator
Zotero_File_Interface.Progress = new function() {

View file

@ -5,11 +5,12 @@ var Zotero_Import_Wizard = {
_dbs: null,
_file: null,
_translation: null,
_mendeleyOnlineRedirectURLWithCode: null,
_mendeleyAccessToken: null,
init: async function () {
this._wizard = document.getElementById('import-wizard');
var dbs = await Zotero_File_Interface.findMendeleyDatabases();
if (dbs.length) {
document.getElementById('radio-import-source-mendeley').hidden = false;
@ -32,8 +33,17 @@ var Zotero_Import_Wizard = {
document.getElementById('create-collection-checkbox').removeAttribute('checked');
}
}
if (args && args.mendeleyAccessToken) {
this._mendeleyAccessToken = args.mendeleyAccessToken;
this._wizard.goTo('page-options');
}
// Update labels
document.getElementById('radio-import-source-mendeley-online').label
= `Mendeley Reference Manager (${Zotero.getString('import.onlineImport')})`;
document.getElementById('radio-import-source-mendeley').label
= `Mendeley Desktop (${Zotero.getString('import.localImport')})`;
document.getElementById('file-handling-store').label = Zotero.getString(
'import.fileHandling.store',
Zotero.appName
@ -57,6 +67,11 @@ var Zotero_Import_Wizard = {
case 'radio-import-source-file':
await this.chooseFile();
break;
case 'radio-import-source-mendeley-online':
wizard.goTo('mendeley-online-explanation');
wizard.canRewind = true;
break;
case 'radio-import-source-mendeley':
this._dbs = await Zotero_File_Interface.findMendeleyDatabases();
@ -85,7 +100,19 @@ var Zotero_Import_Wizard = {
throw e;
}
},
onMendeleyOnlineShow: async function () {
document.getElementById('mendeley-online-description').textContent = Zotero.getString(
'import.mendeleyOnline.intro', [Zotero.appName, 'Mendeley Reference Manager', 'Mendeley']
);
},
onMendeleyOnlineAdvance: function () {
if (!this._mendeleyOnlineRedirectURLWithCode) {
Zotero_File_Interface.authenticateMendeleyOnline();
window.close();
}
},
goToStart: function () {
this._wizard.goTo('page-start');
@ -165,7 +192,7 @@ var Zotero_Import_Wizard = {
onOptionsShown: function () {
document.getElementById('file-handling-options').hidden = !!this._mendeleyAccessToken;
},
@ -192,7 +219,7 @@ var Zotero_Import_Wizard = {
onImportStart: async function () {
if (!this._file) {
if (!this._file && !this._mendeleyAccessToken) {
let index = document.getElementById('file-list').selectedIndex;
this._file = this._dbs[index].path;
}
@ -206,7 +233,8 @@ var Zotero_Import_Wizard = {
onBeforeImport: this.onBeforeImport.bind(this),
addToLibraryRoot: !document.getElementById('create-collection-checkbox')
.hasAttribute('checked'),
linkFiles: document.getElementById('file-handling-radio').selectedIndex == 1
linkFiles: document.getElementById('file-handling-radio').selectedIndex == 1,
mendeleyOnlineToken: this._mendeleyAccessToken
});
// Cancelled by user or due to error
@ -224,13 +252,12 @@ var Zotero_Import_Wizard = {
catch (e) {
if (e.message == 'Encrypted Mendeley database') {
let url = 'https://www.zotero.org/support/kb/mendeley_import';
this._onDone(
Zotero.getString('general.error'),
// TODO: Localize
`The selected Mendeley database cannot be read, likely because it is encrypted. `
+ `See <a href="${url}" class="text-link">How do I import a Mendeley library `
+ `into Zotero?</a> for more information.`
);
let HTML_NS = 'http://www.w3.org/1999/xhtml'
let elem = document.createElementNS(HTML_NS, 'div');
elem.innerHTML = `The selected Mendeley database cannot be read, likely because it `
+ `is encrypted. See <a href="${url}" class="text-link">How do I import a `
+ `Mendeley library into Zotero?</a> for more information.`
this._onDone(Zotero.getString('general.error'), elem);
}
else {
this._onDone(
@ -310,9 +337,9 @@ var Zotero_Import_Wizard = {
var xulElem = document.getElementById('result-description');
var htmlElem = document.getElementById('result-description-html');
if (description.includes('href')) {
htmlElem.innerHTML = description;
Zotero.Utilities.Internal.updateHTMLInXUL(htmlElem);
if (description instanceof HTMLElement) {
htmlElem.appendChild(description);
Zotero.Utilities.Internal.updateHTMLInXUL(htmlElem, { callback: () => window.close() });
xulElem.hidden = true;
htmlElem.setAttribute('display', 'block');
}
@ -321,7 +348,7 @@ var Zotero_Import_Wizard = {
xulElem.hidden = false;
htmlElem.setAttribute('display', 'none');
}
document.getElementById('result-description')
document.getElementById('result-description');
if (showReportErrorButton) {
let button = document.getElementById('result-report-error');

View file

@ -1,6 +1,7 @@
<?xml version="1.0"?>
<?xml-stylesheet href="chrome://global/skin/" type="text/css"?>
<?xml-stylesheet href="chrome://zotero/skin/zotero.css" type="text/css"?>
<?xml-stylesheet href="chrome://zotero/skin/importWizard.css" type="text/css"?>
<!DOCTYPE window SYSTEM "chrome://zotero/locale/zotero.dtd">
@ -21,9 +22,20 @@
onpageadvanced="Zotero_Import_Wizard.onModeChosen(); return false;">
<radiogroup id="import-source">
<radio id="radio-import-source-file" label="&zotero.import.source.file;"/>
<radio id="radio-import-source-mendeley" label="Mendeley" hidden="true"/>
<radio id="radio-import-source-mendeley-online"/>
<radio id="radio-import-source-mendeley" hidden="true"/>
</radiogroup>
</wizardpage>
<wizardpage
next="page-options"
pageid="mendeley-online-explanation"
onpageshow="Zotero_Import_Wizard.onMendeleyOnlineShow()"
onpageadvanced="Zotero_Import_Wizard.onMendeleyOnlineAdvance(); return false;"
onpagerewound="return Zotero_Import_Wizard.goToStart()"
>
<description id="mendeley-online-description" />
</wizardpage>
<wizardpage pageid="page-file-list"
next="page-options"
@ -63,7 +75,6 @@
</radiogroup>
<description id="file-handling-description"/>
</vbox>
</wizardpage>
<wizardpage pageid="page-progress"

View file

@ -0,0 +1,51 @@
// eslint-disable-next-line no-unused-vars
var mendeleyAPIUtils = (function () {
const MENDELEY_API_URL = 'https://api.mendeley.com';
const getNextLinkFromResponse = (response) => {
let next = null;
let links = response.getResponseHeader('link');
if (links) {
const matches = links.match(/<(.*?)>;\s+rel="next"/i);
if (matches && matches.length > 1) {
next = matches[1];
}
}
return next;
};
const apiFetchUrl = async (token, url, headers = {}, options = {}) => {
headers = { ...headers, Authorization: `Bearer ${token}` };
return Zotero.HTTP.request('GET', url, { ...options, headers });
};
const apiFetch = async (token, endPoint, params = {}, headers = {}, options = {}) => {
const stringParams = Object.entries(params).map(p => p.join('=')).join('&');
const url = MENDELEY_API_URL + '/' + endPoint + '?' + stringParams;
return apiFetchUrl(token, url, headers, options);
};
const get = async (token, endPoint, params = {}, headers = {}, options = {}) => {
const response = await apiFetch(token, endPoint, params, headers, options);
return JSON.parse(response.responseText);
};
const getAll = async (token, endPoint, params = {}, headers = {}, options = {}) => {
const PER_PAGE = endPoint === 'annotations' ? 200 : 500;
const response = await apiFetch(token, endPoint, { ...params, limit: PER_PAGE }, headers, options);
var next = getNextLinkFromResponse(response);
var data = JSON.parse(response.responseText);
while (next) {
const response = await apiFetchUrl(token, next, headers, options); //eslint-disable-line no-await-in-loop
data = [...data, ...JSON.parse(response.responseText)];
next = getNextLinkFromResponse(response);
}
return data;
};
return { getNextLinkFromResponse, apiFetch, apiFetchUrl, get, getAll };
})();

View file

@ -1,19 +1,28 @@
/* global map:false, mendeleyOnlineMappings:false, mendeleyAPIUtils:false */
var EXPORTED_SYMBOLS = ["Zotero_Import_Mendeley"];
Components.utils.import("resource://gre/modules/Services.jsm");
Components.utils.import("resource://gre/modules/osfile.jsm");
Services.scriptloader.loadSubScript("chrome://zotero/content/include.js");
Services.scriptloader.loadSubScript("chrome://zotero/content/import/mendeley/mendeleyOnlineMappings.js");
Services.scriptloader.loadSubScript("chrome://zotero/content/import/mendeley/mendeleyAPIUtils.js");
const { apiTypeToDBType, apiFieldToDBField } = mendeleyOnlineMappings;
const { apiFetch, get, getAll } = mendeleyAPIUtils;
var Zotero_Import_Mendeley = function () {
this.createNewCollection = null;
this.linkFiles = null;
this.newItems = [];
this.token = null;
this._db;
this._file;
this._saveOptions = null;
this._itemDone;
this._progress = 0;
this._progressMax;
this._tmpFilesToDelete = [];
};
Zotero_Import_Mendeley.prototype.setLocation = function (file) {
@ -42,6 +51,11 @@ Zotero_Import_Mendeley.prototype.setTranslator = function () {};
Zotero_Import_Mendeley.prototype.translate = async function (options = {}) {
this._linkFiles = options.linkFiles;
this._saveOptions = {
skipSelect: true,
...(options.saveOptions || {})
};
this.timestamp = Date.now();
if (true) {
Services.scriptloader.loadSubScript("chrome://zotero/content/import/mendeley/mendeleySchemaMap.js");
@ -77,39 +91,119 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) {
// Disable syncing while we're importing
var resumeSync = Zotero.Sync.Runner.delayIndefinite();
this._db = new Zotero.DBConnection(this._file);
if (this._file) {
this._db = new Zotero.DBConnection(this._file);
}
try {
if (!await this._isValidDatabase()) {
if (this._file && !await this._isValidDatabase()) {
throw new Error("Not a valid Mendeley database");
}
// Collections
let folders = await this._getFolders(mendeleyGroupID);
let collectionJSON = this._foldersToAPIJSON(folders, rootCollectionKey);
let folderKeys = this._getFolderKeys(collectionJSON);
if (!this._file && !this.token) {
throw new Error("Missing import token");
}
const folders = this.token
? await this._getFoldersAPI(mendeleyGroupID)
: await this._getFoldersDB(mendeleyGroupID);
const collectionJSON = this._foldersToAPIJSON(folders, rootCollectionKey);
const folderKeys = this._getFolderKeys(collectionJSON);
await this._saveCollections(libraryID, collectionJSON, folderKeys);
//
// Items
//
let documents = await this._getDocuments(mendeleyGroupID);
let documents = this.token
? await this._getDocumentsAPI(mendeleyGroupID)
: await this._getDocumentsDB(mendeleyGroupID);
this._progressMax = documents.length;
// Get various attributes mapped to document ids
let urls = await this._getDocumentURLs(mendeleyGroupID);
let creators = await this._getDocumentCreators(mendeleyGroupID, map.creatorTypes);
let tags = await this._getDocumentTags(mendeleyGroupID);
let collections = await this._getDocumentCollections(
mendeleyGroupID,
documents,
rootCollectionKey,
folderKeys
);
let files = await this._getDocumentFiles(mendeleyGroupID);
let annotations = await this._getDocumentAnnotations(mendeleyGroupID);
let urls = this.token
? await this._getDocumentURLsAPI(documents)
: await this._getDocumentURLsDB(mendeleyGroupID);
let creators = this.token
? await this._getDocumentCreatorsAPI(documents)
: await this._getDocumentCreatorsDB(mendeleyGroupID, map.creatorTypes);
let tags = this.token
? await this._getDocumentTagsAPI(documents)
: await this._getDocumentTagsDB(mendeleyGroupID);
let collections = this.token
? await this._getDocumentCollectionsAPI(documents, rootCollectionKey, folderKeys)
: await this._getDocumentCollectionsDB(mendeleyGroupID, documents, rootCollectionKey, folderKeys);
let files = this.token
? await this._getDocumentFilesAPI(documents)
: await this._getDocumentFilesDB(mendeleyGroupID);
let annotations = this.token
? await this._getDocumentAnnotationsAPI(mendeleyGroupID)
: await this._getDocumentAnnotationsDB(mendeleyGroupID);
let profile = this.token
? await this._getProfileAPI()
: await this._getProfileDB();
let groups = this.token
? await this._getGroupsAPI()
: await this._getGroupsDB();
const fileHashLookup = new Map();
for (let [documentID, fileEntries] of files) {
for (let fileEntry of fileEntries) {
fileHashLookup.set(fileEntry.hash, documentID);
}
}
for (let group of groups) {
let groupAnnotations = this.token
? await this._getDocumentAnnotationsAPI(group.id, profile.id)
: await this._getDocumentAnnotationsDB(group.id, profile.id);
for (let groupAnnotationsList of groupAnnotations.values()) {
for (let groupAnnotation of groupAnnotationsList) {
if (fileHashLookup.has(groupAnnotation.hash)) {
const targetDocumentID = fileHashLookup.get(groupAnnotation.hash);
if (!annotations.has(targetDocumentID)) {
annotations.set(targetDocumentID, []);
}
annotations.get(targetDocumentID).push(groupAnnotation);
}
}
}
}
for (let document of documents) {
let docURLs = urls.get(document.id);
let docFiles = files.get(document.id);
if (this.token) {
// extract identifiers
['arxiv', 'doi', 'isbn', 'issn', 'pmid', 'scopus', 'pui', 'pii', 'sgr'].forEach(
i => document[i] = (document.identifiers || {})[i]
);
// normalise item type from the API to match Mendeley DB
document.type = apiTypeToDBType[document.type] || document.type;
// normalise field names from the API to match Mendeley DB
Object.keys(apiFieldToDBField).forEach((key) => {
if (key in document) {
const newKey = apiFieldToDBField[key];
if (newKey) {
document[newKey] = document[key];
}
delete document[key];
}
});
}
// If there's a single PDF file, use "PDF" for the attachment title
if (docFiles && docFiles.length == 1 && docFiles[0].fileURL.endsWith('.pdf')) {
@ -173,7 +267,14 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) {
}
finally {
try {
await this._db.closeDatabase();
if (this._file) {
await this._db.closeDatabase();
}
if (this.token) {
await Promise.all(
this._tmpFilesToDelete.map(f => this._removeTemporaryFile(f))
);
}
}
catch (e) {
Zotero.logError(e);
@ -183,6 +284,18 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) {
}
};
Zotero_Import_Mendeley.prototype._removeTemporaryFile = async function (file) {
const containingDir = OS.Path.dirname(file);
try {
await Zotero.File.removeIfExists(file);
await OS.File.removeEmptyDir(containingDir);
}
catch (e) {
Zotero.logError(e);
}
};
Zotero_Import_Mendeley.prototype._isValidDatabase = async function () {
var tables = [
'DocumentContributors',
@ -208,7 +321,7 @@ Zotero_Import_Mendeley.prototype._isValidDatabase = async function () {
//
// Collections
//
Zotero_Import_Mendeley.prototype._getFolders = async function (groupID) {
Zotero_Import_Mendeley.prototype._getFoldersDB = async function (groupID) {
return this._db.queryAsync(
`SELECT F.id, F.uuid, F.name, `
// Top-level folders can have a parentId of 0 instead of -1 (by mistake?)
@ -221,6 +334,23 @@ Zotero_Import_Mendeley.prototype._getFolders = async function (groupID) {
);
};
Zotero_Import_Mendeley.prototype._getFoldersAPI = async function (groupID) {
const params = {};
const headers = { Accept: 'application/vnd.mendeley-folder.1+json' };
if (groupID && groupID !== 0) {
params.group_id = groupID; //eslint-disable-line camelcase
}
return (await getAll(this.token, 'folders', params, headers)).map(f => ({
id: f.id,
uuid: f.id,
name: f.name,
parentId: f.parent_id || -1,
remoteUuid: f.id
}));
};
/**
* Get flat array of collection API JSON with parentCollection set
*
@ -306,9 +436,7 @@ Zotero_Import_Mendeley.prototype._saveCollections = async function (libraryID, j
delete toSave.remoteUUID;
collection.fromJSON(toSave);
await collection.saveTx({
skipSelect: true
});
await collection.saveTx(this._saveOptions);
}
};
@ -337,13 +465,13 @@ Zotero_Import_Mendeley.prototype._findExistingCollection = async function (libra
Zotero.debug(`Found existing collection ${collections[0].libraryKey} for `
+ `${predicate} ${collectionJSON.relations[predicate]}`);
return collections[0];
}
};
//
// Items
//
Zotero_Import_Mendeley.prototype._getDocuments = async function (groupID) {
Zotero_Import_Mendeley.prototype._getDocumentsDB = async function (groupID) {
return this._db.queryAsync(
`SELECT D.*, RD.remoteUuid FROM Documents D `
+ `JOIN RemoteDocuments RD ON (D.id=RD.documentId) `
@ -352,12 +480,28 @@ Zotero_Import_Mendeley.prototype._getDocuments = async function (groupID) {
);
};
Zotero_Import_Mendeley.prototype._getDocumentsAPI = async function (groupID) {
const params = { view: 'all' };
const headers = { Accept: 'application/vnd.mendeley-document-with-files-list+json' };
if (groupID && groupID !== 0) {
params.group_id = groupID; //eslint-disable-line camelcase
}
return (await getAll(this.token, 'documents', params, headers)).map(d => ({
...d,
uuid: d.id,
remoteUuid: d.id
}));
};
/**
* Get a Map of document ids to arrays of URLs
*
* @return {Map<Number,String[]>}
*/
Zotero_Import_Mendeley.prototype._getDocumentURLs = async function (groupID) {
Zotero_Import_Mendeley.prototype._getDocumentURLsDB = async function (groupID) {
var rows = await this._db.queryAsync(
`SELECT documentId, CAST(url AS TEXT) AS url FROM DocumentUrls DU `
+ `JOIN RemoteDocuments USING (documentId) `
@ -374,13 +518,17 @@ Zotero_Import_Mendeley.prototype._getDocumentURLs = async function (groupID) {
return map;
};
Zotero_Import_Mendeley.prototype._getDocumentURLsAPI = async function (documents) {
return new Map(documents.map(d => ([d.id, d.websites])));
};
/**
* Get a Map of document ids to arrays of creator API JSON
*
* @param {Integer} groupID
* @param {Object} creatorTypeMap - Mapping of Mendeley creator types to Zotero creator types
*/
Zotero_Import_Mendeley.prototype._getDocumentCreators = async function (groupID, creatorTypeMap) {
Zotero_Import_Mendeley.prototype._getDocumentCreatorsDB = async function (groupID, creatorTypeMap) {
var rows = await this._db.queryAsync(
`SELECT * FROM DocumentContributors `
+ `JOIN RemoteDocuments USING (documentId) `
@ -401,10 +549,21 @@ Zotero_Import_Mendeley.prototype._getDocumentCreators = async function (groupID,
return map;
};
Zotero_Import_Mendeley.prototype._getDocumentCreatorsAPI = async function (documents) {
var map = new Map();
for (let doc of documents) {
const authors = (doc.authors || []).map(c => this._makeCreator('author', c.first_name, c.last_name));
const editors = (doc.editors || []).map(c => this._makeCreator('editor', c.first_name, c.last_name));
const translators = (doc.translators || []).map(c => this._makeCreator('translator', c.first_name, c.last_name));
map.set(doc.id, [...authors, ...editors, ...translators]);
}
return map;
};
/**
* Get a Map of document ids to arrays of tag API JSON
*/
Zotero_Import_Mendeley.prototype._getDocumentTags = async function (groupID) {
Zotero_Import_Mendeley.prototype._getDocumentTagsDB = async function (groupID) {
var rows = await this._db.queryAsync(
// Manual tags
`SELECT documentId, tag, 0 AS type FROM DocumentTags `
@ -432,10 +591,19 @@ Zotero_Import_Mendeley.prototype._getDocumentTags = async function (groupID) {
return map;
};
Zotero_Import_Mendeley.prototype._getDocumentTagsAPI = async function (documents) {
var map = new Map();
for (let doc of documents) {
const tags = [...(doc.tags || []).map(tag => ({ tag, type: 0 })), ...(doc.keywords || []).map(tag => ({ tag, type: 1 }))];
map.set(doc.id, tags);
}
return map;
};
/**
* Get a Map of document ids to arrays of collection keys
*/
Zotero_Import_Mendeley.prototype._getDocumentCollections = async function (groupID, documents, rootCollectionKey, folderKeys) {
Zotero_Import_Mendeley.prototype._getDocumentCollectionsDB = async function (groupID, documents, rootCollectionKey, folderKeys) {
var rows = await this._db.queryAsync(
`SELECT documentId, folderId FROM DocumentFolders DF `
+ `JOIN RemoteDocuments USING (documentId) `
@ -460,12 +628,28 @@ Zotero_Import_Mendeley.prototype._getDocumentCollections = async function (group
return map;
};
Zotero_Import_Mendeley.prototype._getDocumentCollectionsAPI = async function (documents, rootCollectionKey, folderKeys) {
return new Map(
documents.map((d) => {
const keys = (d.folder_uuids || []).map((fuuid) => {
const key = folderKeys.get(fuuid);
if (!key) {
Zotero.debug(`Document folder ${fuuid} not found -- skipping`, 2);
}
return key;
}).filter(Boolean);
// Add all documents to root collection if specified
return [d.id, [...keys, ...(rootCollectionKey ? [rootCollectionKey] : [])]];
})
);
};
/**
* Get a Map of document ids to arrays of file metadata
*
* @return {Map<Number,Object[]>}
*/
Zotero_Import_Mendeley.prototype._getDocumentFiles = async function (groupID) {
Zotero_Import_Mendeley.prototype._getDocumentFilesDB = async function (groupID) {
var rows = await this._db.queryAsync(
`SELECT documentId, hash, localUrl FROM DocumentFiles `
+ `JOIN Files USING (hash) `
@ -490,10 +674,59 @@ Zotero_Import_Mendeley.prototype._getDocumentFiles = async function (groupID) {
return map;
};
Zotero_Import_Mendeley.prototype._fetchFile = async function (fileID, filePath) {
const fileDir = OS.Path.dirname(filePath);
await Zotero.File.createDirectoryIfMissingAsync(fileDir);
const xhr = await apiFetch(this.token, `files/${fileID}`, {}, {}, { responseType: 'blob', followRedirects: false });
const uri = xhr.getResponseHeader('location');
await Zotero.File.download(uri, filePath);
this._progress += 1;
if (this._itemDone) {
this._itemDone();
}
};
Zotero_Import_Mendeley.prototype._getDocumentFilesAPI = async function (documents) {
const map = new Map();
let totalSize = 0;
Components.utils.import("resource://zotero/concurrentCaller.js");
var caller = new ConcurrentCaller({
numConcurrent: 6,
onError: e => Zotero.logError(e),
Promise: Zotero.Promise
});
for (let doc of documents) {
const files = [];
for (let file of (doc.files || [])) {
const fileName = file.file_name || 'file';
const tmpFile = OS.Path.join(Zotero.getTempDirectory().path, `mendeley-online-import-${this.timestamp}-${file.id}`, fileName);
this._tmpFilesToDelete.push(tmpFile);
caller.add(this._fetchFile.bind(this, file.id, tmpFile));
files.push({
fileURL: OS.Path.toFileURI(tmpFile),
title: file.file_name || '',
contentType: file.mime_type || '',
hash: file.filehash,
});
totalSize += file.size;
this._progressMax += 1;
}
map.set(doc.id, files);
}
// check if enough space available totalSize
await caller.runAll();
return map;
};
/**
* Get a Map of document ids to arrays of annotations
*/
Zotero_Import_Mendeley.prototype._getDocumentAnnotations = async function (groupID) {
Zotero_Import_Mendeley.prototype._getDocumentAnnotationsDB = async function (groupID, profileID = null) {
var map = new Map();
// Highlights
@ -504,8 +737,9 @@ Zotero_Import_Mendeley.prototype._getDocumentAnnotations = async function (group
+ `JOIN RemoteDocuments USING (documentId) `
+ `JOIN FileHighlightRects FHR ON (FH.id=FHR.highlightId) `
+ `WHERE groupId=? `
+ (profileID !== null ? `AND profileUuid=? ` : ``)
+ `ORDER BY FH.id, page, y1 DESC, x1`,
groupID
profileID !== null ? [groupID, profileID] : groupID
);
var currentHighlight = null;
for (let i = 0; i < rows.length; i++) {
@ -548,8 +782,9 @@ Zotero_Import_Mendeley.prototype._getDocumentAnnotations = async function (group
+ `FROM FileNotes `
+ `JOIN RemoteDocuments USING (documentId) `
+ `WHERE groupId=? `
+ (profileID !== null ? `AND profileUuid=? ` : ``)
+ `ORDER BY page, y, x`,
groupID
profileID !== null ? [groupID, profileID] : groupID
);
for (let row of rows) {
let docAnnotations = map.get(row.documentId);
@ -572,6 +807,100 @@ Zotero_Import_Mendeley.prototype._getDocumentAnnotations = async function (group
return map;
};
Zotero_Import_Mendeley.prototype._getDocumentAnnotationsAPI = async function (groupID, profileID = null) {
const params = {};
if (groupID && groupID !== 0) {
params.group_id = groupID; //eslint-disable-line camelcase
}
const map = new Map();
(await getAll(this.token, 'annotations', params, { Accept: 'application/vnd.mendeley-annotation.1+json' }))
.forEach((a) => {
if (profileID !== null && a.profile_id !== profileID) {
// optionally filter annotations by profile id
return;
}
if (a.type === 'note') {
// This is a "general note" in Mendeley. It appears to be the same thing as
// document.note thus not an annotations and can be discarded
return;
}
const rects = (a.positions || []).map(position => ({
x1: (position.top_left || {}).x || 0,
y1: (position.top_left || {}).y || 0,
x2: (position.bottom_right || {}).x || 0,
y2: (position.bottom_right || {}).y || 0,
}));
let page = 1;
try {
// const page = ((a.positions || [])[0] || {}).page; // ???
page = a.positions[0].page;
}
catch (e) { }
const annotation = {
id: a.id,
color: a.color ? `#${a.color.r.toString(16)}${a.color.g.toString(16)}${a.color.b.toString(16)}` : null,
dateAdded: a.created,
dateModified: a.last_modified,
hash: a.filehash,
uuid: a.id,
page,
};
if (a.type === 'highlight') {
annotation.type = 'highlight';
annotation.rects = rects;
}
if (a.type === 'sticky_note' && rects.length > 0) {
annotation.type = 'note';
annotation.note = a.text;
annotation.x = rects[0].x1;
annotation.y = rects[0].y1;
}
if (!map.has(a.document_id)) {
map.set(a.document_id, []);
}
map.get(a.document_id).push(annotation);
});
return map;
};
Zotero_Import_Mendeley.prototype._getGroupsAPI = async function () {
const params = { type: 'all' };
const headers = { Accept: 'application/vnd.mendeley-group-list+json' };
return getAll(this.token, 'groups/v2', params, headers);
};
Zotero_Import_Mendeley.prototype._getGroupsDB = async function () {
const rows = await this._db.queryAsync(
"SELECT id, remoteUUid, name, isOwner FROM Groups WHERE remoteUuID != ?", ['']
);
return rows;
};
Zotero_Import_Mendeley.prototype._getProfileAPI = async function () {
const params = { };
const headers = { Accept: 'application/vnd.mendeley-profiles.2+json' };
return get(this.token, 'profiles/v2/me', params, headers);
};
Zotero_Import_Mendeley.prototype._getProfileDB = async function () {
const rows = await this._db.queryAsync(
"SELECT uuid as id, firstName, lastName, displayName FROM Profiles ORDER BY ROWID LIMIT 1"
);
return rows[0];
};
/**
* Create API JSON array with item and any child attachments or notes
*/
@ -923,8 +1252,8 @@ Zotero_Import_Mendeley.prototype._saveItems = async function (libraryID, json) {
item.fromJSON(toSave);
await item.saveTx({
skipSelect: true,
skipDateModifiedUpdate: true
skipDateModifiedUpdate: true,
...this._saveOptions
});
if (itemJSON.documentID) {
idMap.set(itemJSON.documentID, item.id);
@ -1076,9 +1405,7 @@ Zotero_Import_Mendeley.prototype._saveFilesAndAnnotations = async function (file
attachment.setRelations({
'mendeleyDB:fileHash': file.hash
});
await attachment.saveTx({
skipSelect: true
});
await attachment.saveTx(this._saveOptions);
}
}
else {
@ -1124,7 +1451,8 @@ Zotero_Import_Mendeley.prototype._isDownloadedFile = function (path) {
return parentDir.endsWith(OS.Path.join('Application Support', 'Mendeley Desktop', 'Downloaded'))
|| parentDir.endsWith(OS.Path.join('Local', 'Mendeley Ltd', 'Mendeley Desktop', 'Downloaded'))
|| parentDir.endsWith(OS.Path.join('Local', 'Mendeley Ltd.', 'Mendeley Desktop', 'Downloaded'))
|| parentDir.endsWith(OS.Path.join('data', 'Mendeley Ltd.', 'Mendeley Desktop', 'Downloaded'));
|| parentDir.endsWith(OS.Path.join('data', 'Mendeley Ltd.', 'Mendeley Desktop', 'Downloaded'))
|| parentDir.startsWith(OS.Path.join(Zotero.getTempDirectory().path, 'mendeley-online-import')); // Mendeley Online Importer
}
/**
@ -1170,7 +1498,7 @@ Zotero_Import_Mendeley.prototype._saveAnnotations = async function (annotations,
let type = 'application/pdf';
if (Zotero.MIME.sniffForMIMEType(await Zotero.File.getSample(file)) == type) {
attachmentItem.attachmentContentType = type;
await attachmentItem.saveTx();
await attachmentItem.saveTx(this._saveOptions);
}
}
@ -1269,9 +1597,7 @@ Zotero_Import_Mendeley.prototype._saveAnnotations = async function (annotations,
});
}
note.setNote('<h1>' + Zotero.getString('extractedAnnotations') + '</h1>\n' + noteStrings.join('\n'));
return note.saveTx({
skipSelect: true
});
return note.saveTx(this._saveOptions);
};

View file

@ -0,0 +1,57 @@
/* eslint-disable camelcase, no-unused-vars */
var mendeleyOnlineMappings = {
// lookup to normalise from item type presented by API to item type as stored in DB
apiTypeToDBType: {
bill: 'Bill',
book: 'Book',
book_section: 'BookSection',
case: 'Case',
computer_program: 'ComputerProgram',
conference_proceedings: 'ConferenceProceedings',
encyclopedia_article: 'EncyclopediaArticle',
film: 'Film',
generic: 'Generic',
hearing: 'Hearing',
journal: 'JournalArticle',
magazine_article: 'MagazineArticle',
newspaper_article: 'NewspaperArticle',
patent: 'Patent',
report: 'Report',
statute: 'Statute',
television_broadcast: 'TelevisionBroadcast',
thesis: 'Thesis',
web_page: 'WebPage',
working_paper: 'WorkingPaper'
},
apiFieldToDBField: {
accessed: 'dateAccessed',
authors: false, // all author types handled separately
citation_key: 'citationKey',
created: 'added',
edition: 'edition',
editors: false, // all author types handled separately
file_attached: false,
folder_uuids: false, // collections handled separately
group_id: 'groupID',
identifiers: false, // identifiers are separately copied directly into document
keywords: false, // tags handled separately
last_modified: 'modified',
notes: 'note',
patent_application_number: 'patentApplicationNumber',
patent_legal_status: 'patentLegalStatus',
patent_owner: 'patentOwner',
private_publication: 'privatePublication',
profile_id: 'profileID',
reprint_edition: 'reprintEdition',
revision: 'revisionNumber',
series_editor: 'seriesEditor',
series_number: 'seriesNumber',
short_title: 'shortTitle',
source_type: 'sourceType',
tags: false, // tags handled separately
translators: false, // all author types handled separately
user_context: 'userContext',
websites: false // URLs handled separately
}
};

View file

@ -10,6 +10,7 @@ var map = {
EncyclopediaArticle: "encyclopediaArticle",
Film: "film",
Generic: "document",
Hearing: "hearing",
JournalArticle: "journalArticle",
MagazineArticle: "magazineArticle",
NewspaperArticle: "newspaperArticle",

View file

@ -44,7 +44,7 @@
onload="window.sizeToContent()"
windowtype="zotero:basicViewer"
title="&brandShortName;"
width="1000" height="500"
width="1000" height="700"
persist="screenX screenY width height sizemode">
<script type="application/javascript" src="chrome://global/content/globalOverlay.js"/>
<script type="application/javascript" src="chrome://global/content/contentAreaUtils.js"/>

View file

@ -836,6 +836,7 @@ Zotero.Utilities.Internal = {
* force links to open in new windows, pass with
* .shiftKey = true. If not provided, the actual event will
* be used instead.
* .callback - Function to call after launching URL
*/
updateHTMLInXUL: function (elem, options) {
options = options || {};
@ -846,6 +847,9 @@ Zotero.Utilities.Internal = {
a.setAttribute('tooltiptext', href);
a.onclick = function (event) {
Zotero.launchURL(href);
if (options.callback) {
options.callback();
}
return false;
};
}

View file

@ -749,9 +749,12 @@ fileInterface.exportError = An error occurred while trying to export the selecte
fileInterface.importOPML = Import Feeds from OPML
fileInterface.OPMLFeedFilter = OPML Feed List
import.onlineImport = online import
import.localImport = local import
import.fileHandling.store = Copy files to the %S storage folder
import.fileHandling.link = Link to files in original location
import.fileHandling.description = Linked files cannot be synced by %S.
import.mendeleyOnline.intro = On the next page you will be asked to log in to %2$S and grant %1$S access. This is necessary to import your %3$S library into %1$S.
quickCopy.copyAs = Copy as %S

View file

@ -59,6 +59,11 @@ listbox, #result-description, #result-description-html {
line-height: 1.5;
}
#mendeley-online-description {
font-size: 13px;
line-height: 1.5;
}
#result-description-html a {
text-decoration: underline;
}

View file

@ -135,7 +135,14 @@ label.zotero-text-link {
text-decoration: underline;
border: 1px solid transparent;
cursor: pointer;
}
.text-link {
-moz-user-focus: normal;
color: -moz-nativehyperlinktext;
text-decoration: underline;
border: 1px solid transparent;
cursor: pointer;
}
.zotero-clicky