Generalize Find Available PDF -> Find Full Text (#4397)

This commit is contained in:
Abe Jellinek 2024-07-15 11:47:07 -04:00 committed by Dan Stillman
parent 297af9b409
commit 7020d60351
11 changed files with 229 additions and 120 deletions

View file

@ -12,6 +12,11 @@
style="display: flex;">
<script src="include.js"></script>
<script src="progressQueueDialog.js"/>
<linkset>
<html:link rel="localization" href="zotero.ftl"/>
</linkset>
<vbox id="progress-queue-root" flex="1">
<label id="label" control="progress-indicator" value=""/>
<hbox align="center">

View file

@ -35,8 +35,10 @@ Zotero.Attachments = new function () {
this.BASE_PATH_PLACEHOLDER = 'attachments:';
var _findPDFQueue = [];
var _findPDFQueuePromise = null;
this.FIND_AVAILABLE_FILE_TYPES = ['application/pdf', 'application/epub+zip'];
var _findFileQueue = [];
var _findFileQueuePromise = null;
var self = this;
@ -605,7 +607,7 @@ Zotero.Attachments = new function () {
{
cookieSandbox,
referrer,
isPDF: contentType == 'application/pdf',
enforceFileType: Zotero.Attachments.FIND_AVAILABLE_FILE_TYPES.includes(contentType),
shouldDisplayCaptcha: true
}
);
@ -1090,13 +1092,13 @@ Zotero.Attachments = new function () {
* @param {Object} [options]
* @param {Object} [options.cookieSandbox]
* @param {String} [options.referrer]
* @param {Boolean} [options.isPDF] - Delete file if not PDF
* @param {Boolean} [options.enforceFileType] - Delete file if not one of SUPPORTED_FILE_TYPES
* @param {Boolean} [options.shouldDisplayCaptcha]
*/
this.downloadFile = async function (url, path, options = {}) {
Zotero.debug(`Downloading file from ${url}`);
let enforcingPDF = false;
let enforcingFileType = false;
try {
await new Zotero.Promise(function (resolve) {
var wbp = Components.classes["@mozilla.org/embedding/browser/nsWebBrowserPersist;1"]
@ -1113,9 +1115,9 @@ Zotero.Attachments = new function () {
Zotero.Utilities.Internal.saveURI(wbp, url, path, headers);
});
if (options.isPDF) {
enforcingPDF = true;
await _enforcePDF(path);
if (options.enforceFileType) {
enforcingFileType = true;
await _enforceFileType(path);
}
}
catch (e) {
@ -1127,7 +1129,7 @@ Zotero.Attachments = new function () {
}
// Custom handling for PDFs that are bot-guarded
// via a JS-redirect
if (enforcingPDF && e instanceof this.InvalidPDFException) {
if (enforcingFileType && e instanceof this.InvalidPDFException) {
if (Zotero.BrowserDownload.shouldAttemptDownloadViaBrowser(url)) {
return Zotero.BrowserDownload.downloadPDF(url, path, options);
}
@ -1137,12 +1139,12 @@ Zotero.Attachments = new function () {
};
/**
* Make sure a file is a PDF
* Make sure a file is a type we want
*/
async function _enforcePDF(path) {
async function _enforceFileType(path) {
var sample = await Zotero.File.getContentsAsync(path, null, 1000);
if (Zotero.MIME.sniffForMIMEType(sample) != 'application/pdf') {
Zotero.debug("Downloaded PDF was not a PDF", 2);
if (!Zotero.Attachments.FIND_AVAILABLE_FILE_TYPES.includes(Zotero.MIME.sniffForMIMEType(sample))) {
Zotero.debug("Downloaded file was not a supported type", 2);
if (Zotero.Debug.enabled) {
Zotero.debug(
Zotero.Utilities.ellipsize(
@ -1160,29 +1162,38 @@ Zotero.Attachments = new function () {
this.InvalidPDFException = function() {
this.message = "Downloaded PDF was not a PDF";
this.message = "Downloaded file was not a supported type (PDF or EPUB)";
this.stack = new Error().stack;
};
this.InvalidPDFException.prototype = Object.create(Error.prototype);
this.canFindPDFForItem = function (item) {
this.canFindFileForItem = function (item) {
return item.isRegularItem()
&& !item.isFeedItem
&& (!!item.getField('DOI') || !!item.getField('url') || !!item.getExtraField('DOI'))
&& item.numPDFAttachments() == 0;
&& this.FIND_AVAILABLE_FILE_TYPES.every(type => item.numFileAttachmentsWithContentType(type) == 0);
};
/**
* @deprecated Use canFindFileForItem()
*/
this.canFindPDFForItem = function (item) {
Zotero.warn('Zotero.Attachments.canFindPDFForItem() is deprecated -- use canFindFileForItem()');
return this.canFindFileForItem(item);
};
/**
* Get the PDF resolvers that can be used for a given item based on the available fields
* Get the file resolvers that can be used for a given item based on the available fields
*
* @param {Zotero.Item} item
* @param {String[]} [methods=['doi', 'url', 'oa', 'custom']]
* @param {Boolean} [automatic=false] - Only include custom resolvers with `automatic: true`
* @return {Object[]} - An array of urlResolvers (see downloadFirstAvailableFile())
*/
this.getPDFResolvers = function (item, methods, automatic) {
this.getFileResolvers = function (item, methods, automatic) {
if (!methods) {
methods = ['doi', 'url', 'oa', 'custom'];
}
@ -1242,7 +1253,7 @@ Zotero.Attachments = new function () {
}
}
catch (e) {
Zotero.debug("Error parsing custom PDF resolvers", 2);
Zotero.debug("Error parsing custom file resolvers", 2);
Zotero.debug(e, 2);
}
if (customResolvers) {
@ -1294,7 +1305,7 @@ Zotero.Attachments = new function () {
url = url.replace(/\{doi}/, doi);
resolvers.push(async function () {
Zotero.debug(`Looking for PDFs for ${doi} via ${name}`);
Zotero.debug(`Looking for files for ${doi} via ${name}`);
var req = await Zotero.HTTP.request(
method.toUpperCase(),
@ -1368,7 +1379,7 @@ Zotero.Attachments = new function () {
});
}
catch (e) {
Zotero.debug("Error parsing PDF resolver", 2);
Zotero.debug("Error parsing file resolver", 2);
Zotero.debug(e, 2);
Zotero.debug(resolver, 2);
}
@ -1382,16 +1393,25 @@ Zotero.Attachments = new function () {
/**
* Look for available PDFs for items and add as attachments
* @deprecated Use getFileResolvers()
*/
this.getPDFResolvers = function (item, methods) {
Zotero.warn('Zotero.Attachments.getPDFResolvers() is deprecated -- use getFileResolvers()');
return this.getFileResolvers(item, methods);
};
/**
* Look for available files for items and add as attachments
*
* @param {Zotero.Item[]} items
* @param {Object} [options]
* @param {String[]} [options.methods] - See getPDFResolvers()
* @param {String[]} [options.methods] - See getFileResolvers()
* @param {Number} [options.sameDomainRequestDelay=1000] - Minimum number of milliseconds
* between requests to the same domain (used in tests)
* @return {Promise}
*/
this.addAvailablePDFs = async function (items, options = {}) {
this.addAvailableFiles = async function (items, options = {}) {
const MAX_CONSECUTIVE_DOMAIN_FAILURES = 5;
const SAME_DOMAIN_REQUEST_DELAY = options.sameDomainRequestDelay || 1000;
var queue;
@ -1409,33 +1429,33 @@ Zotero.Attachments = new function () {
return domainInfo;
}
var progressQueue = Zotero.ProgressQueues.get('findPDF');
var progressQueue = Zotero.ProgressQueues.get('findFile');
if (!progressQueue) {
progressQueue = Zotero.ProgressQueues.create({
id: 'findPDF',
title: 'pane.items.menu.findAvailablePDF.multiple',
id: 'findFile',
title: 'pane.items.menu.findAvailableFile',
columns: [
'general.item',
'general.pdf'
'attachment.fullText'
]
});
progressQueue.addListener('cancel', () => queue = []);
}
queue = _findPDFQueue;
queue = _findFileQueue;
for (let item of items) {
// Skip items that aren't eligible. This is sort of weird, because it means some
// selected items just don't appear in the list, but there are several different reasons
// why items might not be eligible (non-regular items, no URL or DOI, already has a PDF)
// and listing each one seems a little unnecessary.
if (!this.canFindPDFForItem(item)) {
// why items might not be eligible (non-regular items, no URL or DOI, already has a
// full-text attachment) and listing each one seems a little unnecessary.
if (!this.canFindFileForItem(item)) {
continue;
}
let entry = {
item,
urlResolvers: this.getPDFResolvers(item, options.methods),
urlResolvers: this.getFileResolvers(item, options.methods),
domain: null,
continuation: null,
processing: false,
@ -1460,14 +1480,14 @@ Zotero.Attachments = new function () {
progressQueue.addRow(item);
}
// If no eligible items, just show a popup saying no PDFs were found
// If no eligible items, just show a popup saying no files were found
if (!queue.length) {
let progressWin = new Zotero.ProgressWindow();
let title = Zotero.getString('pane.items.menu.findAvailablePDF.multiple');
let title = Zotero.getString('pane.items.menu.findAvailableFile');
progressWin.changeHeadline(title);
let itemProgress = new progressWin.ItemProgress(
'attachmentPDF',
Zotero.getString('findPDF.noPDFsFound')
Zotero.getString('findPDF.noFilesFound')
);
progressWin.show();
itemProgress.setProgress(100);
@ -1480,12 +1500,12 @@ Zotero.Attachments = new function () {
dialog.open();
// If queue was already in progress, just wait for it to finish
if (_findPDFQueuePromise) {
return _findPDFQueuePromise;
if (_findFileQueuePromise) {
return _findFileQueuePromise;
}
var queueResolve;
_findPDFQueuePromise = new Zotero.Promise((resolve) => {
_findFileQueuePromise = new Zotero.Promise((resolve) => {
queueResolve = resolve;
});
@ -1537,7 +1557,7 @@ Zotero.Attachments = new function () {
}
// Currently filtered out above
/*if (!this.canFindPDFForItem(current.item)) {
/*if (!this.canFindFileForItem(current.item)) {
current.result = false;
progressQueue.updateRow(
current.item.id,
@ -1551,7 +1571,7 @@ Zotero.Attachments = new function () {
current.processing = true;
// Process item
this.addPDFFromURLs(
this.addFileFromURLs(
current.item,
current.urlResolvers,
{
@ -1664,7 +1684,7 @@ Zotero.Attachments = new function () {
: Zotero.ProgressQueue.ROW_FAILED,
attachment
? attachment.getField('title')
: Zotero.getString('findPDF.noPDFFound')
: Zotero.getString('findPDF.noFileFound')
);
})
.catch((e) => {
@ -1687,17 +1707,26 @@ Zotero.Attachments = new function () {
processNextItem();
});
var numPDFs = queue.reduce((accumulator, currentValue) => {
var numFiles = queue.reduce((accumulator, currentValue) => {
return accumulator + (currentValue.result ? 1 : 0);
}, 0);
dialog.setStatus(
numPDFs
? Zotero.getString('findPDF.pdfsAdded', numPDFs, numPDFs)
: Zotero.getString('findPDF.noPDFsFound')
numFiles
? { l10nId: 'find-pdf-files-added', l10nArgs: { count: numFiles } }
: Zotero.getString('findPDF.noFilesFound')
);
_findPDFQueue = [];
_findFileQueue = [];
queueResolve();
_findPDFQueuePromise = null;
_findFileQueuePromise = null;
};
/**
* @deprecated Use addAvailableFiles()
*/
this.addAvailablePDFs = function (items, options) {
Zotero.warn('Zotero.Attachments.addAvailablePDFs() is deprecated -- use addAvailableFiles()');
return this.addAvailableFiles(items, options);
};
@ -1714,14 +1743,23 @@ Zotero.Attachments = new function () {
* @param {String[]} [options.methods] - See getPDFResolvers()
* @return {Zotero.Item|false} - New Zotero.Item, or false if unsuccessful
*/
this.addAvailablePDF = async function (item, options = {}) {
Zotero.debug("Looking for available PDFs");
return this.addPDFFromURLs(item, this.getPDFResolvers(item, options.methods));
this.addAvailableFile = async function (item, options = {}) {
Zotero.debug("Looking for available files");
return this.addFileFromURLs(item, this.getFileResolvers(item, options.methods));
};
/**
* Try to add a PDF to an item from a set of URL resolvers
* @deprecated Use addAvailableFile()
*/
this.addAvailablePDF = function (item, options) {
Zotero.warn('Zotero.Attachments.addAvailablePDF() is deprecated -- use addAvailableFile()');
return this.addAvailableFile(item, options);
};
/**
* Try to add a file attachment to an item from a set of URL resolvers
*
* @param {Zotero.Item} item
* @param {(String|Object|Function)[]} urlResolvers - See downloadFirstAvailableFile()
@ -1730,19 +1768,19 @@ Zotero.Attachments = new function () {
* is started, taking the access method name as an argument
* @return {Zotero.Item|false} - New Zotero.Item, or false if unsuccessful
*/
this.addPDFFromURLs = async function (item, urlResolvers, options = {}) {
this.addFileFromURLs = async function (item, urlResolvers, options = {}) {
var fileBaseName = this.getFileBaseNameFromItem(item);
var tmpDir;
var tmpFile;
var attachmentItem = false;
try {
tmpDir = (await this.createTemporaryStorageDirectory()).path;
tmpFile = OS.Path.join(tmpDir, fileBaseName + '.pdf');
let { title, url, props } = await this.downloadFirstAvailableFile(
tmpFile = OS.Path.join(tmpDir, fileBaseName + '.tmp');
let { title, mimeType, url, props } = await this.downloadFirstAvailableFile(
urlResolvers,
tmpFile,
{
isPDF: true,
enforceFileType: true,
shouldDisplayCaptcha: true,
onAccessMethodStart: options.onAccessMethodStart,
onBeforeRequest: options.onBeforeRequest,
@ -1750,13 +1788,21 @@ Zotero.Attachments = new function () {
}
);
if (url) {
if (!mimeType) {
mimeType = await Zotero.MIME.getMIMETypeFromFile(tmpFile);
}
if (!this.FIND_AVAILABLE_FILE_TYPES.includes(mimeType)) {
throw new Error(`Resolved file is unsupported type ${mimeType}`);
}
let filename = fileBaseName + '.' + (Zotero.MIME.getPrimaryExtension(mimeType) || 'dat');
await IOUtils.move(tmpFile, PathUtils.join(tmpDir, filename));
attachmentItem = await this.createURLAttachmentFromTemporaryStorageDirectory({
directory: tmpDir,
libraryID: item.libraryID,
filename: PathUtils.filename(tmpFile),
title: title || _getPDFTitleFromVersion(props.articleVersion),
filename,
title: title || _getTitleFromVersion(props.articleVersion),
url,
contentType: 'application/pdf',
contentType: mimeType,
parentItemID: item.id
});
}
@ -1775,7 +1821,16 @@ Zotero.Attachments = new function () {
};
function _getPDFTitleFromVersion(version) {
/**
* @deprecated Use addFileFromURLs()
*/
this.addPDFFromURLs = function (item, urlResolvers, options) {
Zotero.warn('Zotero.Attachments.addPDFFromURLs() is deprecated -- use addFileFromURLs()');
return this.addFileFromURLs(item, urlResolvers, options);
};
function _getTitleFromVersion(version) {
var str;
switch (version) {
@ -1805,7 +1860,7 @@ Zotero.Attachments = new function () {
*
* @param {(String|Object|Function)[]} urlResolvers - An array of URLs, objects, or functions
* that return arrays of objects. Objects should contain 'url' and/or 'pageURL' (the latter
* being a webpage that might contain a translatable PDF link), 'accessMethod' (which will
* being a webpage that might contain a translatable file link), 'accessMethod' (which will
* be displayed in the save popup), and an optional 'articleVersion' ('submittedVersion',
* 'acceptedVersion', or 'publishedVersion'). Functions that return promises are waited for,
* and functions aren't called unless a file hasn't yet been found from an earlier entry.
@ -1815,8 +1870,8 @@ Zotero.Attachments = new function () {
* @param {Function} [options.onAfterRequest] - Function that runs after a request
* @param {Function} [options.onRequestError] - Function that runs when a request fails.
* Return true to retry request and false to skip.
* @return {Object|false} - Object with successful 'title' (when available from translator), 'url', and 'props'
* from the associated urlResolver, or false if no file could be downloaded
* @return {Object|false} - Object with successful 'title' and 'mimeType' (when available from translator), 'url',
* and 'props' from the associated urlResolver, or false if no file could be downloaded
*/
this.downloadFirstAvailableFile = async function (urlResolvers, path, options) {
const maxURLs = 6;
@ -1900,7 +1955,7 @@ Zotero.Attachments = new function () {
// Ignore URLs we've already tried
if (url && isTriedURL(url)) {
Zotero.debug(`PDF at ${url} was already tried -- skipping`);
Zotero.debug(`File at ${url} was already tried -- skipping`);
url = null;
}
if (pageURL && isTriedURL(pageURL)) {
@ -1946,9 +2001,10 @@ Zotero.Attachments = new function () {
if (pageURL) {
url = null;
let title = null;
let mimeType = null;
let responseURL;
try {
Zotero.debug(`Looking for PDF on ${pageURL}`);
Zotero.debug(`Looking for file on ${pageURL}`);
let redirects = 0;
let nextURL = pageURL;
@ -2060,7 +2116,7 @@ Zotero.Attachments = new function () {
// use redirects plus cookies for IP-based authentication [1]. The downside
// is that we might follow the same set of redirects more than once, but we
// won't process the final page multiple times, and if a publisher URL does
// redirect that's hopefully a decent indication that a PDF will be found
// redirect that's hopefully a decent indication that a file will be found
// the first time around.
//
// [1] https://forums.zotero.org/discussion/81182
@ -2072,31 +2128,31 @@ Zotero.Attachments = new function () {
continue;
}
// If DOI resolves directly to a PDF, save it to disk
if (contentType && contentType.startsWith('application/pdf')) {
Zotero.debug("URL resolves directly to PDF");
// If DOI resolves directly to a file, save it to disk
if (contentType && this.FIND_AVAILABLE_FILE_TYPES.some(type => contentType.startsWith(type))) {
Zotero.debug("URL resolves directly to file");
await Zotero.File.putContentsAsync(path, blob);
await _enforcePDF(path);
await _enforceFileType(path);
return { url: responseURL, props: urlResolver };
}
// Otherwise translate the Document we parsed above
else if (doc) {
({ title, url } = await Zotero.Utilities.Internal.getPDFFromDocument(doc));
({ title, mimeType, url } = await Zotero.Utilities.Internal.getFileFromDocument(doc));
}
}
catch (e) {
Zotero.debug(`Error getting PDF from ${pageURL}: ${e}\n\n${e.stack}`);
Zotero.debug(`Error getting file from ${pageURL}: ${e}\n\n${e.stack}`);
continue;
}
if (!url) {
Zotero.debug(`No PDF found on ${responseURL || pageURL}`);
Zotero.debug(`No file found on ${responseURL || pageURL}`);
continue;
}
if (isTriedURL(url)) {
Zotero.debug(`PDF at ${url} was already tried -- skipping`);
Zotero.debug(`File at ${url} was already tried -- skipping`);
continue;
}
// Don't try this PDF URL again
// Don't try this file URL again
addTriedURL(url);
// Use the page we loaded as the referrer
@ -2108,7 +2164,7 @@ Zotero.Attachments = new function () {
await beforeRequest(url);
await this.downloadFile(url, path, downloadOptions);
afterRequest(url);
return { title, url, props: urlResolver };
return { title, mimeType, url, props: urlResolver };
}
catch (e) {
Zotero.debug(`Error downloading ${url}: ${e}\n\n${e.stack}`);
@ -2128,7 +2184,7 @@ Zotero.Attachments = new function () {
* @deprecated Use Zotero.Utilities.cleanURL instead
*/
this.cleanAttachmentURI = function (uri, tryHttp) {
Zotero.debug("Zotero.Attachments.cleanAttachmentURI() is deprecated -- use Zotero.Utilities.cleanURL");
Zotero.warn("Zotero.Attachments.cleanAttachmentURI() is deprecated -- use Zotero.Utilities.cleanURL");
return Zotero.Utilities.cleanURL(uri, tryHttp);
}

View file

@ -60,7 +60,12 @@ Zotero.ProgressQueueDialog = function (progressQueue) {
if (_progressWindow) {
let label = _progressWindow.document.getElementById("label");
if (label) {
label.value = msg;
if (typeof msg === 'object' && 'l10nId' in msg) {
_progressWindow.document.l10n.setAttributes(label, msg.l10nId, msg.l10nArgs);
}
else {
label.value = msg;
}
}
}
};

View file

@ -305,7 +305,7 @@ Zotero.Translate.ItemSaver.prototype = {
// No translated, no OA, just potential custom, so create a status line
if (!jsonAttachment) {
jsonAttachment = this._makeJSONAttachment(
jsonItem.id, Zotero.getString('findPDF.searchingForAvailablePDFs')
jsonItem.id, Zotero.getString('findPDF.searchingForAvailableFiles')
);
}
}
@ -330,7 +330,7 @@ Zotero.Translate.ItemSaver.prototype = {
let attachment;
try {
attachment = await Zotero.Attachments.addPDFFromURLs(
attachment = await Zotero.Attachments.addFileFromURLs(
item,
resolvers,
{

View file

@ -1336,7 +1336,7 @@ Zotero.Utilities.Internal = {
* @param {doc} Document
* @return {{ title: string, url: string } | false} - PDF attachment title and URL, or false if none found
*/
getPDFFromDocument: async function (doc) {
getFileFromDocument: async function (doc) {
let translate = new Zotero.Translate.Web();
translate.setDocument(doc);
var translators = await translate.getTranslators();
@ -1354,14 +1354,24 @@ Zotero.Utilities.Internal = {
return false;
}
for (let attachment of newItems[0].attachments) {
if (attachment.mimeType == 'application/pdf') {
return { title: attachment.title, url: attachment.url };
if (Zotero.Attachments.FIND_AVAILABLE_FILE_TYPES.includes(attachment.mimeType)) {
return {
title: attachment.title,
mimeType: attachment.mimeType,
url: attachment.url
};
}
}
return false;
},
getPDFFromDocument(doc) {
Zotero.debug('Zotero.Utilities.Internal.getPDFFromDocument() is deprecated -- use getFileFromDocument()');
return this.getFileFromDocument(doc);
},
/**
* Hyphenate an ISBN based on the registrant table available from
* https://www.isbn-international.org/range_file_generation

View file

@ -3499,7 +3499,7 @@ var ZoteroPane = new function()
'createNoteFromAnnotations',
'addAttachments',
'sep2',
'findPDF',
'findFile',
'sep3',
'toggleRead',
'addToCollection',
@ -3659,7 +3659,7 @@ var ZoteroPane = new function()
}
if (items.some(item => item.isRegularItem())) {
show.add(m.findPDF);
show.add(m.findFile);
show.add(m.sep3);
}
}
@ -3747,11 +3747,11 @@ var ZoteroPane = new function()
menuitem.setAttribute('label', Zotero.getString(str));
}
if (Zotero.Attachments.canFindPDFForItem(item)) {
show.add(m.findPDF);
if (Zotero.Attachments.canFindFileForItem(item)) {
show.add(m.findFile);
show.add(m.sep3);
if (!collectionTreeRow.filesEditable) {
disable.add(m.findPDF);
disable.add(m.findFile);
}
}
@ -3939,7 +3939,7 @@ var ZoteroPane = new function()
}
// Set labels, plural if necessary
menu.childNodes[m.findPDF].setAttribute('label', Zotero.getString('pane.items.menu.findAvailablePDF' + multiple));
menu.childNodes[m.findFile].setAttribute('label', Zotero.getString('pane.items.menu.findAvailableFile'));
menu.childNodes[m.moveToTrash].setAttribute('label', Zotero.getString('pane.items.menu.moveToTrash' + multiple));
menu.childNodes[m.deleteFromLibrary].setAttribute('label', Zotero.getString('pane.items.menu.delete'));
menu.childNodes[m.exportItems].setAttribute('label', Zotero.getString(`pane.items.menu.export${noteExport ? 'Note' : ''}` + multiple));
@ -4428,12 +4428,12 @@ var ZoteroPane = new function()
};
this.findPDFForSelectedItems = async function () {
this.findFilesForSelectedItems = async function () {
if (!this.canEdit()) {
this.displayCannotEditLibraryMessage();
return;
}
await Zotero.Attachments.addAvailablePDFs(this.getSelectedItems());
await Zotero.Attachments.addAvailableFiles(this.getSelectedItems());
};

View file

@ -958,7 +958,7 @@
</menupopup>
</menu>
<menuseparator/>
<menuitem class="menuitem-iconic zotero-menuitem-find-pdf" oncommand="ZoteroPane.findPDFForSelectedItems()"/>
<menuitem class="menuitem-iconic zotero-menuitem-find-pdf" oncommand="ZoteroPane.findFilesForSelectedItems()"/>
<menuseparator/>
<menuitem class="menuitem-iconic zotero-menuitem-toggle-read-item" oncommand="ZoteroPane_Local.toggleSelectedItemsRead();"/>
<menu class="menu-iconic zotero-menuitem-add-to-collection">

View file

@ -651,3 +651,8 @@ advanced-search-operators-menu =
advanced-search-condition-input =
.aria-label = Value
.label = { $label }
find-pdf-files-added = { $count ->
[one] { $count } file added
*[other] { $count } files added
}

View file

@ -343,8 +343,7 @@ pane.items.removeRecursive = Are you sure you want to remove the selected item f
pane.items.removeRecursive.multiple = Are you sure you want to remove the selected items from this collection and all subcollections?
pane.items.menu.addNoteFromAnnotations = Add Note from Annotations
pane.items.menu.createNoteFromAnnotations = Create Note from Annotations
pane.items.menu.findAvailablePDF = Find Available PDF
pane.items.menu.findAvailablePDF.multiple = Find Available PDFs
pane.items.menu.findAvailableFile = Find Full Text
pane.items.menu.addToCollection = Add to Collection
pane.items.menu.remove = Remove Item from Collection…
pane.items.menu.remove.multiple = Remove Items from Collection…
@ -674,13 +673,12 @@ ingester.importFile.intoNewCollection = Import into new collection
ingester.lookup.performing = Performing Lookup…
ingester.lookup.error = An error occurred while performing lookup for this item.
findPDF.searchingForAvailablePDFs = Searching for available PDFs…
findPDF.searchingForAvailableFiles = Searching for available files…
findPDF.checkingItems = Checking %1$S item;Checking %1$S items
findPDF.pdfsAdded = %1$S PDF added;%1$S PDFs added
findPDF.openAccessPDF = Open-Access PDF
findPDF.pdfWithMethod = PDF (%S)
findPDF.noPDFsFound = No PDFs found
findPDF.noPDFFound = No PDF found
findPDF.noFilesFound = No files found
findPDF.noFileFound = No file found
attachment.fullText = Full Text
attachment.acceptedVersion = Accepted Version

View file

@ -5,7 +5,7 @@ $item-type-icons: (
attachments-snapshot: "attachment-snapshot",
attachments-epub: "attachment-epub",
attach-note: "note",
find-pdf: "attachment-pdf",
find-file: "attachment-pdf",
convert-to-book-section: "book-section",
convert-to-book: "book",
);

View file

@ -609,7 +609,7 @@ describe("Zotero.Attachments", function() {
});
});
describe("Find Available PDF", function () {
describe("Find Full Text", function () {
var doiPrefix = 'https://doi.org/';
var doi1 = '10.1111/abcd';
var doi2 = '10.2222/bcde';
@ -627,6 +627,7 @@ describe("Zotero.Attachments", function() {
var pageURL8 = 'http://website2/article8';
var pageURL9 = 'http://website/article9';
var pageURL10 = 'http://website/refresh';
var pageURL11 = 'http://website/book';
var httpd;
var port = 16213;
@ -634,6 +635,9 @@ describe("Zotero.Attachments", function() {
var pdfPath = OS.Path.join(getTestDataDirectory().path, 'test.pdf');
var pdfURL = `${baseURL}article1/pdf`;
var pdfSize;
var epubPath = OS.Path.join(getTestDataDirectory().path, 'stub.epub');
var epubURL = `${baseURL}article11/epub`;
var epubSize;
var requestStub;
var requestStubCallTimes = [];
var return429 = true;
@ -713,6 +717,7 @@ describe("Zotero.Attachments", function() {
// DOI 6 redirects to page 8, which is on a different domain and has a PDF
[doiPrefix + doi6, pageURL8, true],
[pageURL8, pageURL8, true],
[pageURL11, epubURL, false],
// Redirect loop
['http://website/redirect_loop1', 'http://website/redirect_loop2', false],
@ -857,6 +862,7 @@ describe("Zotero.Attachments", function() {
});
pdfSize = await OS.File.stat(pdfPath).size;
epubSize = await OS.File.stat(epubPath).size;
Zotero.Prefs.clear('findPDFs.resolvers');
});
@ -864,8 +870,12 @@ describe("Zotero.Attachments", function() {
beforeEach(async function () {
({ httpd } = await startHTTPServer(port));
httpd.registerFile(
pdfURL.substr(baseURL.length - 1),
Zotero.File.pathToFile(OS.Path.join(getTestDataDirectory().path, 'test.pdf'))
pdfURL.substring(baseURL.length - 1),
Zotero.File.pathToFile(pdfPath)
);
httpd.registerFile(
epubURL.substring(baseURL.length - 1),
Zotero.File.pathToFile(epubPath)
);
// Generate a page with a relative PDF URL
@ -897,7 +907,7 @@ describe("Zotero.Attachments", function() {
Zotero.Prefs.clear('findPDFs.resolvers');
// Close progress dialog after each run
var queue = Zotero.ProgressQueues.get('findPDF');
var queue = Zotero.ProgressQueues.get('findFile');
if (queue) {
queue.getDialog().close();
}
@ -913,7 +923,7 @@ describe("Zotero.Attachments", function() {
item.setField('title', 'Test');
item.setField('DOI', doi);
await item.saveTx();
var attachment = await Zotero.Attachments.addAvailablePDF(item);
var attachment = await Zotero.Attachments.addAvailableFile(item);
assert.equal(requestStub.callCount, 2);
assert.isTrue(requestStub.getCall(0).calledWith('GET', 'https://doi.org/' + doi));
@ -931,7 +941,7 @@ describe("Zotero.Attachments", function() {
item.setField('title', 'Test');
item.setField('DOI', doi);
await item.saveTx();
var attachment = await Zotero.Attachments.addAvailablePDF(item);
var attachment = await Zotero.Attachments.addAvailableFile(item);
assert.equal(requestStub.callCount, 1);
assert.isTrue(requestStub.calledWith('GET', 'https://doi.org/' + doi));
@ -949,7 +959,7 @@ describe("Zotero.Attachments", function() {
item.setField('title', 'Test');
item.setField('extra', 'DOI: ' + doi);
await item.saveTx();
var attachment = await Zotero.Attachments.addAvailablePDF(item);
var attachment = await Zotero.Attachments.addAvailableFile(item);
assert.equal(requestStub.callCount, 2);
assert.isTrue(requestStub.getCall(0).calledWith('GET', 'https://doi.org/' + doi));
@ -967,7 +977,7 @@ describe("Zotero.Attachments", function() {
item.setField('title', 'Test');
item.setField('url', url);
await item.saveTx();
var attachment = await Zotero.Attachments.addAvailablePDF(item);
var attachment = await Zotero.Attachments.addAvailableFile(item);
assert.equal(requestStub.callCount, 1);
assert.isTrue(requestStub.calledWith('GET', url));
@ -979,13 +989,33 @@ describe("Zotero.Attachments", function() {
assert.equal(await OS.File.stat(attachment.getFilePath()).size, pdfSize);
});
it("should add an EPUB from a URL with a redirect", async function () {
var url = pageURL11;
var item = createUnsavedDataObject('item', { itemType: 'book' });
item.setField('title', 'Test');
item.setField('url', url);
await item.saveTx();
var attachment = await Zotero.Attachments.addAvailableFile(item);
assert.equal(requestStub.callCount, 2);
var call = requestStub.getCall(0);
assert.isTrue(call.calledWith('GET', url));
call = requestStub.getCall(1);
assert.isTrue(call.calledWith('GET', epubURL));
assert.ok(attachment);
var json = attachment.toJSON();
assert.equal(json.url, epubURL);
assert.equal(json.contentType, 'application/epub+zip');
assert.equal(json.filename, 'Test.epub');
assert.equal(await OS.File.stat(attachment.getFilePath()).size, epubSize);
});
it("should add an OA PDF from a direct URL", async function () {
var doi = doi2;
var item = createUnsavedDataObject('item', { itemType: 'journalArticle' });
item.setField('title', 'Test');
item.setField('DOI', doi);
await item.saveTx();
var attachment = await Zotero.Attachments.addAvailablePDF(item);
var attachment = await Zotero.Attachments.addAvailableFile(item);
assert.equal(requestStub.callCount, 3);
var call1 = requestStub.getCall(0);
@ -1009,7 +1039,7 @@ describe("Zotero.Attachments", function() {
item.setField('title', 'Test');
item.setField('DOI', doi);
await item.saveTx();
var attachment = await Zotero.Attachments.addAvailablePDF(item);
var attachment = await Zotero.Attachments.addAvailableFile(item);
assert.equal(requestStub.callCount, 4);
// Check the DOI (and get nothing)
@ -1039,7 +1069,7 @@ describe("Zotero.Attachments", function() {
item.setField('DOI', doi);
item.setField('url', pageURL4);
await item.saveTx();
var attachment = await Zotero.Attachments.addAvailablePDF(item);
var attachment = await Zotero.Attachments.addAvailableFile(item);
assert.equal(requestStub.callCount, 3);
var call = requestStub.getCall(0);
@ -1065,7 +1095,7 @@ describe("Zotero.Attachments", function() {
item2.setField('url', url2);
await item2.saveTx();
var attachments = await Zotero.Attachments.addAvailablePDFs([item1, item2]);
var attachments = await Zotero.Attachments.addAvailableFiles([item1, item2]);
assert.equal(requestStub.callCount, 2);
assert.isAbove(requestStubCallTimes[1] - requestStubCallTimes[0], 998);
@ -1096,7 +1126,7 @@ describe("Zotero.Attachments", function() {
item3.setField('url', url3);
await item3.saveTx();
var attachments = await Zotero.Attachments.addAvailablePDFs([item1, item2, item3]);
var attachments = await Zotero.Attachments.addAvailableFiles([item1, item2, item3]);
assert.equal(requestStub.callCount, 6);
assert.equal(requestStub.getCall(0).args[1], doiPrefix + doi1);
@ -1130,7 +1160,7 @@ describe("Zotero.Attachments", function() {
item2.setField('url', url2);
await item2.saveTx();
var attachments = await Zotero.Attachments.addAvailablePDFs([item1, item2]);
var attachments = await Zotero.Attachments.addAvailableFiles([item1, item2]);
assert.equal(requestStub.callCount, 3);
assert.equal(requestStub.getCall(0).args[1], pageURL9);
@ -1148,7 +1178,7 @@ describe("Zotero.Attachments", function() {
item.setField('title', 'Test');
item.setField('url', url);
await item.saveTx();
var attachment = await Zotero.Attachments.addAvailablePDF(item);
var attachment = await Zotero.Attachments.addAvailableFile(item);
assert.equal(requestStub.callCount, 2);
assert.equal(requestStub.getCall(0).args[1], pageURL10)
@ -1165,7 +1195,7 @@ describe("Zotero.Attachments", function() {
var item = createUnsavedDataObject('item', { itemType: 'journalArticle' });
item.setField('url', 'http://website/redirect_loop1');
await item.saveTx();
var attachment = await Zotero.Attachments.addAvailablePDF(item);
var attachment = await Zotero.Attachments.addAvailableFile(item);
assert.isFalse(attachment);
assert.equal(requestStub.callCount, 7);
});
@ -1174,7 +1204,7 @@ describe("Zotero.Attachments", function() {
var item = createUnsavedDataObject('item', { itemType: 'journalArticle' });
item.setField('url', 'http://website/too_many_redirects1');
await item.saveTx();
var attachment = await Zotero.Attachments.addAvailablePDF(item);
var attachment = await Zotero.Attachments.addAvailableFile(item);
assert.isFalse(attachment);
assert.equal(requestStub.callCount, 10);
});
@ -1196,7 +1226,7 @@ describe("Zotero.Attachments", function() {
}];
Zotero.Prefs.set('findPDFs.resolvers', JSON.stringify(resolvers));
var attachment = await Zotero.Attachments.addAvailablePDF(item);
var attachment = await Zotero.Attachments.addAvailableFile(item);
assert.equal(requestStub.callCount, 4);
var call = requestStub.getCall(0);
@ -1234,7 +1264,7 @@ describe("Zotero.Attachments", function() {
}];
Zotero.Prefs.set('findPDFs.resolvers', JSON.stringify(resolvers));
var attachment = await Zotero.Attachments.addAvailablePDF(item);
var attachment = await Zotero.Attachments.addAvailableFile(item);
assert.equal(requestStub.callCount, 4);
var call = requestStub.getCall(0);
@ -1270,7 +1300,7 @@ describe("Zotero.Attachments", function() {
}];
Zotero.Prefs.set('findPDFs.resolvers', JSON.stringify(resolvers));
var attachment = await Zotero.Attachments.addAvailablePDF(item);
var attachment = await Zotero.Attachments.addAvailableFile(item);
assert.equal(requestStub.callCount, 4);
var call = requestStub.getCall(0);
@ -1310,7 +1340,7 @@ describe("Zotero.Attachments", function() {
}];
Zotero.Prefs.set('findPDFs.resolvers', JSON.stringify(resolvers));
var attachment = await Zotero.Attachments.addAvailablePDF(item);
var attachment = await Zotero.Attachments.addAvailableFile(item);
assert.equal(requestStub.callCount, 5);
var call = requestStub.getCall(0);