Import translation improvements

- Don't block the UI with a progress meter during imports. Instead, show
  a popup in the bottom right when the import is done that shows how
  many items were saved.
- Fix hang when importing some files
- Fix various problems with asynchronous operations/transactions
    - Use the save queue for imports instead of creating concurrent
      transactions that can time out
    - Wait for the save to finish before returning from the translate()
      promise. All save modes now use the save queue, so code that
      handled the non-save-queue process can probably be removed.
    - Serialize child attachments instead of running them concurrently.
      This might make multi-attachment saves a little slower, since they
      can't download at the same time, but it avoids problems with
      concurrent transactions. We might be able to improve this to allow
      concurrent downloads, or allow concurrent saves for a limited
      number of items (e.g., from web saving) if not for larger imports.
- Change collection handling during import, since UI is now active
  - Select the root collection at the beginning of the import
  - Assign items and collections to the root during the import instead
    of at the end
  - Don't select other collections
- Change a few ItemSaver functions to use promises and remove
  unnecessary callbacks. (This includes some connector code that needs
  to be tested.)
- Change some `parentID` variables in ItemSaver to `parentItemID` for
  clarity, since collections are now handled in more places

To-do:

- Save items in smaller batches instead of doing all in the same
  transaction
- Show progress meter in a bottom-right popup during the import
This commit is contained in:
Dan Stillman 2016-12-09 04:36:42 -05:00
parent c61a9dc5f3
commit 78b1d2ee35
7 changed files with 327 additions and 244 deletions

View file

@ -596,6 +596,92 @@ describe("Zotero.Translate", function() {
Zotero.Translators.get.restore();
});
});
describe("ItemSaver", function () {
describe("#saveCollections()", function () {
it("should add top-level collections to specified collection", function* () {
var collection = yield createDataObject('collection');
var collections = [
{
name: "Collection",
type: "collection",
children: []
}
];
var items = [
{
itemType: "book",
title: "Test"
}
];
var translation = new Zotero.Translate.Import();
translation.setString("");
translation.setTranslator(buildDummyTranslator(
"import",
"function detectImport() {}\n"
+ "function doImport() {\n"
+ " var json = JSON.parse('" + JSON.stringify(collections).replace(/['\\]/g, "\\$&") + "');\n"
+ " for (let o of json) {"
+ " var collection = new Zotero.Collection;\n"
+ " for (let field in o) { collection[field] = o[field]; }\n"
+ " collection.complete();\n"
+ " }\n"
+ " json = JSON.parse('" + JSON.stringify(items).replace(/['\\]/g, "\\$&") + "');\n"
+ " for (let o of json) {"
+ " var item = new Zotero.Item;\n"
+ " for (let field in o) { item[field] = o[field]; }\n"
+ " item.complete();\n"
+ " }\n"
+ "}"
));
yield translation.translate({
collections: [collection.id]
});
assert.lengthOf(translation.newCollections, 1);
assert.isNumber(translation.newCollections[0].id);
assert.lengthOf(translation.newItems, 1);
assert.isNumber(translation.newItems[0].id);
var childCollections = Array.from(collection.getChildCollections(true));
assert.sameMembers(childCollections, translation.newCollections.map(c => c.id));
});
});
describe("#_saveAttachment()", function () {
it("should save standalone attachment to collection", function* () {
var collection = yield createDataObject('collection');
var items = [
{
itemType: "attachment",
title: "Test",
mimeType: "text/html",
url: "http://example.com"
}
];
var translation = new Zotero.Translate.Import();
translation.setString("");
translation.setTranslator(buildDummyTranslator(
"import",
"function detectImport() {}\n"
+ "function doImport() {\n"
+ " var json = JSON.parse('" + JSON.stringify(items).replace(/['\\]/g, "\\$&") + "');\n"
+ " for (var i=0; i<json.length; i++) {"
+ " var item = new Zotero.Item;\n"
+ " for (var field in json[i]) { item[field] = json[i][field]; }\n"
+ " item.complete();\n"
+ " }\n"
+ "}"
));
yield translation.translate({
collections: [collection.id]
});
assert.lengthOf(translation.newItems, 1);
assert.isNumber(translation.newItems[0].id);
assert.ok(collection.hasItem(translation.newItems[0].id));
});
});
});
});
describe("Zotero.Translate.ItemGetter", function() {