From 42aebccbf448f138b573013d3ffd16f50398ee73 Mon Sep 17 00:00:00 2001
From: Tom Najdek <tom@doppnet.com>
Date: Wed, 6 Jul 2022 22:47:22 +0200
Subject: [PATCH] Mendeley import: detect win close, better feedback

* Detect wizard cancel/close and interrupt import. This will still
  have to wait for current fetch (file or metadata) to complete but will
  then advance to the cleanup stage
* Advance progress bar during metadata fetch
* Add some extra logging
---
 chrome/content/zotero/import/importWizard.js  |  7 +-
 chrome/content/zotero/import/importWizard.xul |  1 +
 .../import/mendeley/mendeleyAPIUtils.js       |  4 +-
 .../zotero/import/mendeley/mendeleyImport.js  | 96 ++++++++++++++-----
 4 files changed, 84 insertions(+), 24 deletions(-)

diff --git a/chrome/content/zotero/import/importWizard.js b/chrome/content/zotero/import/importWizard.js
index 3d16905b72..b8625d06f7 100644
--- a/chrome/content/zotero/import/importWizard.js
+++ b/chrome/content/zotero/import/importWizard.js
@@ -57,7 +57,12 @@ var Zotero_Import_Wizard = {
 		
 		Zotero.Translators.init(); // async
 	},
-	
+
+	onCancel: function () {
+		if (this._translation && this._translation.interrupt) {
+			this._translation.interrupt();
+		}
+	},
 	
 	onModeChosen: async function () {
 		var wizard = this._wizard;
diff --git a/chrome/content/zotero/import/importWizard.xul b/chrome/content/zotero/import/importWizard.xul
index c4eed4c818..4fc12adaf7 100644
--- a/chrome/content/zotero/import/importWizard.xul
+++ b/chrome/content/zotero/import/importWizard.xul
@@ -10,6 +10,7 @@
 		xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul"
 		xmlns:html="http://www.w3.org/1999/xhtml"
 		title="&zotero.import;"
+		onwizardcancel="Zotero_Import_Wizard.onCancel()"
 		onload="Zotero_Import_Wizard.init()">
 	
 	<script src="../include.js"/>
diff --git a/chrome/content/zotero/import/mendeley/mendeleyAPIUtils.js b/chrome/content/zotero/import/mendeley/mendeleyAPIUtils.js
index bc3557738b..cc873e0d46 100644
--- a/chrome/content/zotero/import/mendeley/mendeleyAPIUtils.js
+++ b/chrome/content/zotero/import/mendeley/mendeleyAPIUtils.js
@@ -64,16 +64,18 @@ const get = async (tokens, endPoint, params = {}, headers = {}, options = {}) =>
 	return JSON.parse(response.responseText);
 };
 
-const getAll = async (tokens, endPoint, params = {}, headers = {}, options = {}) => {
+const getAll = async (tokens, endPoint, params = {}, headers = {}, options = {}, interruptChecker = () => {}) => {
 	const PER_PAGE = endPoint === 'annotations' ? 200 : 500;
 	const response = await apiFetch(tokens, endPoint, { ...params, limit: PER_PAGE }, headers, options);
 	var next = getNextLinkFromResponse(response);
 	var data = JSON.parse(response.responseText);
+	interruptChecker();
 	
 	while (next) {
 		const response = await apiFetchUrl(tokens, next, headers, options); //eslint-disable-line no-await-in-loop
 		data = [...data, ...JSON.parse(response.responseText)];
 		next = getNextLinkFromResponse(response);
+		interruptChecker();
 	}
 
 	return data;
diff --git a/chrome/content/zotero/import/mendeley/mendeleyImport.js b/chrome/content/zotero/import/mendeley/mendeleyImport.js
index eeb7a85d71..1e54e233c7 100644
--- a/chrome/content/zotero/import/mendeley/mendeleyImport.js
+++ b/chrome/content/zotero/import/mendeley/mendeleyImport.js
@@ -20,10 +20,27 @@ var Zotero_Import_Mendeley = function () {
 	this._db;
 	this._file;
 	this._saveOptions = null;
-	this._itemDone;
+	this._itemDone = () => {};
 	this._progress = 0;
-	this._progressMax;
+	this._progressMax = 0;
 	this._tmpFilesToDelete = [];
+	this._caller = null;
+	this._interrupted = false;
+	this._totalSize = 0;
+	this._interruptChecker = (tickProgress = false) => {
+		if (this._interrupted) {
+			throw new Error(`Mendeley Import interrupted!
+				Progress: ${this._progress} / ${this._progressMax}
+				New items created: ${this.newItems.length}
+				Total size of files to download: ${Math.round(this._totalSize / 1024)}KB
+			`);
+		}
+
+		if (tickProgress) {
+			this._progress++;
+			this._itemDone();
+		}
+	};
 };
 
 Zotero_Import_Mendeley.prototype.setLocation = function (file) {
@@ -94,7 +111,7 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) {
 	if (this._file) {
 		this._db = new Zotero.DBConnection(this._file);
 	}
-	
+
 	try {
 		if (this._file && !await this._isValidDatabase()) {
 			throw new Error("Not a valid Mendeley database");
@@ -108,14 +125,22 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) {
 			throw new Error("Missing import token");
 		}
 
+		// we don't know how long the import will be but want to show progress to give
+		// feedback that import has started so we arbitrary set progress at 2%
+		this._progress = 1;
+		this._progressMax = 50;
+		this._itemDone();
+
 		const folders = this._tokens
 			? await this._getFoldersAPI(mendeleyGroupID)
 			: await this._getFoldersDB(mendeleyGroupID);
 
 		const collectionJSON = this._foldersToAPIJSON(folders, rootCollectionKey);
 		const folderKeys = this._getFolderKeys(collectionJSON);
+
 		await this._saveCollections(libraryID, collectionJSON, folderKeys);
 		
+		this._interruptChecker(true);
 		//
 		// Items
 		//
@@ -123,40 +148,59 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) {
 			? await this._getDocumentsAPI(mendeleyGroupID)
 			: await this._getDocumentsDB(mendeleyGroupID);
 
-		this._progressMax = documents.length;
+		// Update progress to reflect items to import and remaining meta data stages
+		// We arbitrary set progress at approx 4%. We then add 8, one "tick" for each remaining meta data download.
+		this._progress = Math.max(Math.floor(0.04 * documents.length), 2);
+		this._progressMax = documents.length + this._progress + 8;
 		// Get various attributes mapped to document ids
 		let urls = this._tokens
 			? await this._getDocumentURLsAPI(documents)
 			: await this._getDocumentURLsDB(mendeleyGroupID);
 
+		this._interruptChecker(true);
+
 		let creators = this._tokens
 			? await this._getDocumentCreatorsAPI(documents)
 			: await this._getDocumentCreatorsDB(mendeleyGroupID, map.creatorTypes);
 
+		this._interruptChecker(true);
+
 		let tags = this._tokens
 			? await this._getDocumentTagsAPI(documents)
 			: await this._getDocumentTagsDB(mendeleyGroupID);
 
+		this._interruptChecker(true);
+
 		let collections = this._tokens
 			? await this._getDocumentCollectionsAPI(documents, rootCollectionKey, folderKeys)
 			: await this._getDocumentCollectionsDB(mendeleyGroupID, documents, rootCollectionKey, folderKeys);
 
+		this._interruptChecker(true);
+		
 		let files = this._tokens
 			? await this._getDocumentFilesAPI(documents)
 			: await this._getDocumentFilesDB(mendeleyGroupID);
-		
+
+		this._interruptChecker(true);
+
 		let annotations = this._tokens
 			? await this._getDocumentAnnotationsAPI(mendeleyGroupID)
 			: await this._getDocumentAnnotationsDB(mendeleyGroupID);
 
+		this._interruptChecker(true);
+
 		let profile = this._tokens
 			? await this._getProfileAPI()
 			: await this._getProfileDB();
 
+		this._interruptChecker(true);
+
 		let groups = this._tokens
 			? await this._getGroupsAPI()
 			: await this._getGroupsDB();
 
+		this._interruptChecker(true);
+
 		const fileHashLookup = new Map();
 
 		for (let [documentID, fileEntries] of files) {
@@ -165,7 +209,6 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) {
 			}
 		}
 
-
 		for (let group of groups) {
 			let groupAnnotations = this._tokens
 				? await this._getDocumentAnnotationsAPI(group.id, profile.id)
@@ -263,18 +306,17 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) {
 				);
 			}
 			this.newItems.push(Zotero.Items.get(documentIDMap.get(document.id)));
-			this._progress++;
-			if (this._itemDone) {
-				this._itemDone();
-			}
+			this._interruptChecker(true);
 		}
-	}
-	finally {
+	} catch (e) {
+		Zotero.logError(e);
+	} finally {
 		try {
 			if (this._file) {
 				await this._db.closeDatabase();
 			}
 			if (this._tokens) {
+				Zotero.debug(`Clearing ${this._tmpFilesToDelete.length} temporary files after Mendeley Import`);
 				await Promise.all(
 					this._tmpFilesToDelete.map(f => this._removeTemporaryFile(f))
 				);
@@ -288,12 +330,20 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) {
 	}
 };
 
+Zotero_Import_Mendeley.prototype.interrupt = function () {
+	this._interrupted = true;
+	
+	if (this._caller) {
+		this._caller.stop();
+	}
+};
+
 Zotero_Import_Mendeley.prototype._removeTemporaryFile = async function (file) {
 	try {
 		await Zotero.File.removeIfExists(file);
 	}
 	catch (e) {
-		Zotero.logError(e);
+		Zotero.logError("Error while removing temporary file " + file + ": " + e);
 	}
 };
 
@@ -343,7 +393,7 @@ Zotero_Import_Mendeley.prototype._getFoldersAPI = async function (groupID) {
 	if (groupID && groupID !== 0) {
 		params.group_id = groupID; //eslint-disable-line camelcase
 	}
-	return (await getAll(this._tokens, 'folders', params, headers)).map(f => ({
+	return (await getAll(this._tokens, 'folders', params, headers, {}, this._interruptChecker)).map(f => ({
 		id: f.id,
 		uuid: f.id,
 		name: f.name,
@@ -490,7 +540,7 @@ Zotero_Import_Mendeley.prototype._getDocumentsAPI = async function (groupID) {
 	}
 	
 
-	return (await getAll(this._tokens, 'documents', params, headers)).map(d => ({
+	return (await getAll(this._tokens, 'documents', params, headers, {}, this._interruptChecker)).map(d => ({
 		...d,
 		uuid: d.id,
 		remoteUuid: d.id
@@ -691,12 +741,13 @@ Zotero_Import_Mendeley.prototype._fetchFile = async function (fileID, filePath)
 Zotero_Import_Mendeley.prototype._getDocumentFilesAPI = async function (documents) {
 	const map = new Map();
 	
-	let totalSize = 0;
+	this._totalSize = 0;
 
 	Components.utils.import("resource://zotero/concurrentCaller.js");
-	var caller = new ConcurrentCaller({
+	this._caller = new ConcurrentCaller({
 		numConcurrent: 6,
 		onError: e => Zotero.logError(e),
+		logger: Zotero.debug,
 		Promise: Zotero.Promise
 	});
 
@@ -710,7 +761,7 @@ Zotero_Import_Mendeley.prototype._getDocumentFilesAPI = async function (document
 			let tmpFile = OS.Path.join(Zotero.getTempDirectory().path, `m-api-${file.id}.${ext}`);
 			
 			this._tmpFilesToDelete.push(tmpFile);
-			caller.add(this._fetchFile.bind(this, file.id, tmpFile));
+			this._caller.add(this._fetchFile.bind(this, file.id, tmpFile));
 			files.push({
 				fileURL: OS.Path.toFileURI(tmpFile),
 				title: file.file_name || '',
@@ -718,13 +769,14 @@ Zotero_Import_Mendeley.prototype._getDocumentFilesAPI = async function (document
 				hash: file.filehash,
 				fileBaseName
 			});
-			totalSize += file.size;
+			this._totalSize += file.size;
 			this._progressMax += 1;
 		}
 		map.set(doc.id, files);
 	}
 	// TODO: check if enough space available totalSize
-	await caller.runAll();
+	await this._caller.runAll();
+	this._caller = null;
 	return map;
 };
 
@@ -821,7 +873,7 @@ Zotero_Import_Mendeley.prototype._getDocumentAnnotationsAPI = async function (gr
 	}
 
 	const map = new Map();
-	(await getAll(this._tokens, 'annotations', params, { Accept: 'application/vnd.mendeley-annotation.1+json' }))
+	(await getAll(this._tokens, 'annotations', params, { Accept: 'application/vnd.mendeley-annotation.1+json' }, {}, this._interruptChecker))
 		.forEach((a) => {
 			if (profileID !== null && a.profile_id !== profileID) {
 				// optionally filter annotations by profile id
@@ -883,7 +935,7 @@ Zotero_Import_Mendeley.prototype._getGroupsAPI = async function () {
 	const params = { type: 'all' };
 	const headers = { Accept: 'application/vnd.mendeley-group-list+json' };
 	
-	return getAll(this._tokens, 'groups/v2', params, headers);
+	return getAll(this._tokens, 'groups/v2', params, headers, {}, this._interruptChecker);
 };
 
 Zotero_Import_Mendeley.prototype._getGroupsDB = async function () {