From ca36096bcf2401b59334fd58d27ada32d4184179 Mon Sep 17 00:00:00 2001 From: Aurimas Vinckevicius Date: Thu, 6 Nov 2014 22:08:56 -0600 Subject: [PATCH 01/17] Add FeedReader --- chrome/content/zotero/xpcom/feedReader.js | 530 ++++++++++++++++++++++ components/zotero-service.js | 1 + 2 files changed, 531 insertions(+) create mode 100644 chrome/content/zotero/xpcom/feedReader.js diff --git a/chrome/content/zotero/xpcom/feedReader.js b/chrome/content/zotero/xpcom/feedReader.js new file mode 100644 index 0000000000..c6cf624f17 --- /dev/null +++ b/chrome/content/zotero/xpcom/feedReader.js @@ -0,0 +1,530 @@ +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2015 Center for History and New Media + George Mason University, Fairfax, Virginia, USA + http://zotero.org + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + + +/** + * Sample feeds: + * + * http://cyber.law.harvard.edu/rss/examples/rss2sample.xml + * http://feeds.feedburner.com/acs/acbcct + * http://www.cell.com/molecular-cell/current.rss + * http://ieeexplore.ieee.org/search/searchresult.jsp?searchField%3DSearch_All%26queryText%3Dwater&searchOrigin=saved_searches&rssFeed=true&rssFeedName=water + * http://www.sciencemag.org/rss/current.xml + * http://rss.sciencedirect.com/publication/science/20925212 + * http://www.ncbi.nlm.nih.gov/entrez/eutils/erss.cgi?rss_guid=1fmfIeN4X5Q8HemTZD5Rj6iu6-FQVCn7xc7_IPIIQtS1XiD9bf + * http://export.arxiv.org/rss/astro-ph + */ + +/** + * class Zotero.FeedReader + * Asynchronously reads an ATOM/RSS feed + * + * @param {String} url URL of the feed + * + * @property {Zotero.Promise} feedProperties An object + * representing feed properties + * @property {Zotero.Promise*} itemIterator Returns an iterator + * for feed items. The iterator returns FeedItem promises that have to be + * resolved before requesting the next promise. When all items are exhausted. + * the promise resolves to null. + * @method {void} terminate Stops retrieving/parsing the feed. Data parsed up + * to this point is still available. + */ +Zotero.FeedReader = new function() { + let ios = Components.classes["@mozilla.org/network/io-service;1"] + .getService(Components.interfaces.nsIIOService); + + /***************************** + * Item processing functions * + *****************************/ + + /** + * Determine item type based on item data + */ + function guessItemType(item) { + // Default to journalArticle + item.itemType = 'journalArticle'; + + if (item.ISSN) { + return; // journalArticle + } + + if (item.ISBN) { + item.itemType = 'bookSection'; + return; + } + + if (item.publicationType) { + let type = item.publicationType.toLowerCase(); + if (type.indexOf('conference') != -1) { + item.itemType = 'conferencePaper'; + return; + } + if (type.indexOf('journal') != -1) { + item.itemType = 'journalArticle'; + return; + } + if (type.indexOf('book') != -1) { + item.itemType = 'bookSection'; + return; + } + } + }; + + /* + * Fetch creators from given field of a feed entry + */ + function processCreators(feedEntry, field, role) { + let names = [], + nameStr; + try { + let personArr = feedEntry[field]; // Seems like this part can throw if there is no author data in the feed + for (let i=0; i 1 + // If only one comma and first part has more than one space, + // it's probably not lastName, firstName + || (commas == 1 && name.split(/\s*,/)[0].indexOf(' ') != -1) + ) + ) { + // Probably multiple authors listed in a single field + nameStr = name; + break; // For clarity. personArr.length == 1 anyway + } else { + names.push(name); + } + } + } catch(e) { + if (e.result != Components.results.NS_ERROR_FAILURE) throw e + + if (field != 'authors') return []; + + // ieeexplore places these in "authors"... sigh + nameStr = getFeedField(feedEntry, null, 'authors'); + if (nameStr) nameStr = Zotero.Utilities.trimInternal(nameStr); + if (!nameStr) return []; + } + + if (nameStr) { + names = nameStr.split(/\s(?:and|&)\s|\s*[,;]\s*/); + } + + let creators = []; + for (let i=0; i { + let items = feed.items; + if (items && items.length) { + for (let i=0; i { + // Make sure the last promise gets resolved to null + let lastItem = this._feedItems[this._feedItems.length - 1]; + lastItem.resolve(null); + }); + + // Set up asynchronous feed processor + let feedProcessor = Components.classes["@mozilla.org/feed-processor;1"] + .createInstance(Components.interfaces.nsIFeedProcessor); + + let feedUrl = ios.newURI(url, null, null); + feedProcessor.parseAsync(null, feedUrl); + + feedProcessor.listener = { + /* + * MDN suggests that we could use nsIFeedProgressListener to handle the feed + * as it gets loaded, but this is actually not implemented (as of 32.0.3), + * so we have to load the whole feed and handle it in handleResult. + */ + handleResult: (result) => { + if (!result.doc) { + this.terminate("No Feed"); + return; + } + + let newFeed = result.doc.QueryInterface(Components.interfaces.nsIFeed); + this._feed.resolve(newFeed); + } + }; + + Zotero.debug("FeedReader: Fetching feed from " + feedUrl.spec); + + this._channel = ios.newChannelFromURI(feedUrl); + this._channel.asyncOpen(feedProcessor, null); // Sends an HTTP request + } + + Zotero.defineProperty(FeedReader.prototype, 'feedProperties', { + get: function() this._feedProperties + }); + + /* + * Feed item iterator + * Each iteration returns a _promise_ for an item. The promise _MUST_ be + * resolved before requesting the next item. + * The last item will always be resolved to `null`, unless the feed processing + * is terminated ahead of time, in which case it will be rejected with the reason + * for termination. + */ + Zotero.defineProperty(FeedReader.prototype, 'itemIterator', { + get: function() { + let items = this._feedItems; + return new function() { + let i = 0; + this.next = function() { + let item = items[i++]; + return { + value: item ? item.promise : null, + done: i >= items.length + }; + }; + } + } + }); + + /* + * Terminate feed processing at any given time + * @param {String} status Reason for terminating processing + */ + FeedReader.prototype.terminate = function(status) { + Zotero.debug("FeedReader: Terminating feed reader (" + status + ")"); + + // Reject feed promise if not resolved yet + if (this._feed.promise.isPending()) { + this._feed.reject(status); + } + + // Reject feed item promise if not resolved yet + let lastItem = this._feedItems[this._feedItems.length - 1]; + if (lastItem.promise.isPending()) { + lastItem.reject(status); + } + + // Close feed connection + if (channel.isPending) { + channel.cancel(Components.results.NS_BINDING_ABORTED); + } + }; + + return FeedReader; +}; \ No newline at end of file diff --git a/components/zotero-service.js b/components/zotero-service.js index 86ed2c06aa..6fe1af8979 100644 --- a/components/zotero-service.js +++ b/components/zotero-service.js @@ -85,6 +85,7 @@ const xpcomFilesLocal = [ 'data/tags', 'db', 'duplicates', + 'feedReader', 'fulltext', 'id', 'integration', From 4c94b05023567a2bfa1b1dae480ce87618d03b48 Mon Sep 17 00:00:00 2001 From: Aurimas Vinckevicius Date: Thu, 6 Nov 2014 22:06:30 -0600 Subject: [PATCH 02/17] New feed data methods --- chrome/content/zotero/xpcom/data/feed.js | 111 +++++++++++++++++- chrome/content/zotero/xpcom/data/feeds.js | 36 ++++++ chrome/content/zotero/xpcom/data/libraries.js | 2 +- 3 files changed, 147 insertions(+), 2 deletions(-) diff --git a/chrome/content/zotero/xpcom/data/feed.js b/chrome/content/zotero/xpcom/data/feed.js index e8e9a4cdf2..41633c15fb 100644 --- a/chrome/content/zotero/xpcom/data/feed.js +++ b/chrome/content/zotero/xpcom/data/feed.js @@ -270,4 +270,113 @@ Zotero.Feed.prototype._finalizeSave = Zotero.Promise.coroutine(function* (env) { Zotero.Feed.prototype._finalizeErase = Zotero.Promise.method(function(env) { Zotero.Feeds.unregister(this.libraryID); return Zotero.Feed._super.prototype._finalizeErase.apply(this, arguments); -}); \ No newline at end of file +}); + +Zotero.Feed.prototype.getExpiredFeedItemIDs = Zotero.Promise.coroutine(function* () { + let sql = "SELECT itemID AS id FROM feedItems " + + "WHERE readTimestamp IS NOT NULL " + + "AND (julianday(readTimestamp, 'utc') + (?) - julianday('now', 'utc')) > 0"; + let expiredIDs = yield Zotero.DB.queryAsync(sql, [{int: this.cleanupAfter}]); + return expiredIDs.map(row => row.id); +}); + +Zotero.Feed.prototype._updateFeed = Zotero.Promise.coroutine(function* () { + let errorMessage = ''; + try { + // Clear expired items + if (this.cleanupAfter) { + let expiredItems = yield this.getExpiredFeedItemIDs(); + Zotero.debug("Cleaning up read feed items..."); + if (expiredItems.length) { + Zotero.debug(expiredItems.join(', ')); + yield Zotero.FeedItems.erase(expiredItems); + } else { + Zotero.debug("No expired feed items"); + } + } + } catch(e) { + Zotero.debug("Error clearing expired feed items."); + Zotero.debug(e); + } + + try { + let fr = new Zotero.FeedReader(this.url); + let itemIterator = fr.createItemIterator(); + let item, toAdd = [], processedGUIDs = []; + while (item = yield itemIterator.next().value) { + if (item.dateModified && this.lastUpdate + && item.dateModified < this.lastUpdate + ) { + Zotero.debug("Item modification date before last update date (" + this._feedLastCheck + ")"); + Zotero.debug(item); + // We can stop now + fr.terminate(); + break; + } + + if (processedGUIDs.indexOf(item.guid) != -1) { + Zotero.debug("Feed item " + item.guid + " already processed from feed."); + continue; + } + processedGUIDs.push(item.guid); + + Zotero.debug("New feed item retrieved:"); + Zotero.debug(item); + + let feedItem = yield Zotero.FeedItems.getAsyncByGUID(item.guid); + if (!feedItem) { + feedItem = new Zotero.FeedItem(); + feedItem.guid = item.guid; + feedItem.setCollections([this.id]); + } else { + Zotero.debug("Feed item " + item.guid + " already in library."); + if (item.dateModified && feedItem.dateModified + && feedItem.dateModified == item.dateModified + ) { + Zotero.debug("Modification date has not changed. Skipping update."); + continue; + } + Zotero.debug("Updating metadata"); + yield feedItem.loadItemData(); + yield feedItem.loadCreators(); + feedItem.isRead = false; + } + + // Delete invalid data + delete item.guid; + + feedItem.fromJSON(item); + toAdd.push(feedItem); + } + + // Save in reverse order + let savePromises = new Array(toAdd.length); + for (let i=toAdd.length-1; i>=0; i--) { + yield toAdd[i].save({skipEditCheck: true, setDateModified: true}); + } + + this.lastUpdate = Zotero.Date.dateToSQL(new Date(), true); + } catch(e) { + Zotero.debug("Error processing feed from " + this.url); + Zotero.debug(e); + errorMessage = e.message || 'Error processing feed'; + } + + this.lastCheck = Zotero.Date.dateToSQL(new Date(), true); + this.lastCheckError = errorMessage || null; + yield this.save({skipEditCheck: true}); +}); + +Zotero.Feed.prototype.updateFeed = function() { + return this._updateFeed() + .finally(function() { + Zotero.Feeds.scheduleNextFeedCheck(); + }); +} + +Zotero.Feed.prototype.erase = Zotero.Promise.coroutine(function* () { + yield this.loadChildItems(); + let childItemIDs = this.getChildItems(true, true); + yield Zotero.FeedItems.erase(childItemIDs); + return Zotero.Feed._super.prototype.erase.call(this); // Don't tell it to delete child items. They're already gone +}) diff --git a/chrome/content/zotero/xpcom/data/feeds.js b/chrome/content/zotero/xpcom/data/feeds.js index 5a3e7f3480..94309b9d4b 100644 --- a/chrome/content/zotero/xpcom/data/feeds.js +++ b/chrome/content/zotero/xpcom/data/feeds.js @@ -110,4 +110,40 @@ Zotero.Feeds = new function() { return !!Object.keys(this._cache.urlByLibraryID).length } + + this.scheduleNextFeedCheck = Zotero.Promise.coroutine(function* () { + Zotero.debug("Scheduling next feed update."); + let sql = "SELECT ( CASE " + + "WHEN lastCheck IS NULL THEN 0 " + + "ELSE julianday(lastCheck, 'utc') + (refreshInterval/1440.0) - julianday('now', 'utc') " + + "END ) * 1440 AS nextCheck " + + "FROM feeds WHERE refreshInterval IS NOT NULL " + + "ORDER BY nextCheck ASC LIMIT 1"; + var nextCheck = yield Zotero.DB.valueQueryAsync(sql); + + if (this._nextFeedCheck) { + this._nextFeedCheck.cancel(); + this._nextFeedCheck = null; + } + + if (nextCheck !== false) { + nextCheck = nextCheck > 0 ? Math.ceil(nextCheck * 60000) : 0; + Zotero.debug("Next feed check in " + nextCheck/60000 + " minutes"); + this._nextFeedCheck = Zotero.Promise.delay(nextCheck).cancellable(); + Zotero.Promise.all([this._nextFeedCheck, globalFeedCheckDelay]) + .then(() => { + globalFeedCheckDelay = Zotero.Promise.delay(60000); // Don't perform auto-updates more than once per minute + return this.updateFeeds() + }) + .catch(e => { + if (e instanceof Zotero.Promise.CancellationError) { + Zotero.debug('Next update check cancelled'); + return; + } + throw e; + }); + } else { + Zotero.debug("No feeds with auto-update."); + } + }); } diff --git a/chrome/content/zotero/xpcom/data/libraries.js b/chrome/content/zotero/xpcom/data/libraries.js index 39cc8ea1c2..d0a5177ee6 100644 --- a/chrome/content/zotero/xpcom/data/libraries.js +++ b/chrome/content/zotero/xpcom/data/libraries.js @@ -298,7 +298,7 @@ Zotero.Libraries = new function () { this._ensureExists(libraryID); return Zotero.Libraries.get(libraryID).filesEditable; }; - + /** * @deprecated * From 2e56e2f659c1254a80665ea9df0c62d139c0eadc Mon Sep 17 00:00:00 2001 From: Aurimas Vinckevicius Date: Thu, 6 Nov 2014 22:12:12 -0600 Subject: [PATCH 03/17] Add feed button and Add Feed dialog --- chrome/content/zotero/feedSettings.js | 187 ++++++++++++++++++ chrome/content/zotero/feedSettings.xul | 50 +++++ chrome/content/zotero/zoteroPane.js | 13 ++ chrome/content/zotero/zoteroPane.xul | 2 + chrome/locale/en-US/zotero/zotero.dtd | 13 ++ chrome/skin/default/zotero/overlay.css | 5 + .../skin/default/zotero/toolbar-feed-add.png | Bin 0 -> 763 bytes chrome/skin/default/zotero/zotero.css | 8 + 8 files changed, 278 insertions(+) create mode 100644 chrome/content/zotero/feedSettings.js create mode 100644 chrome/content/zotero/feedSettings.xul create mode 100644 chrome/skin/default/zotero/toolbar-feed-add.png diff --git a/chrome/content/zotero/feedSettings.js b/chrome/content/zotero/feedSettings.js new file mode 100644 index 0000000000..41734c115b --- /dev/null +++ b/chrome/content/zotero/feedSettings.js @@ -0,0 +1,187 @@ +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2015 Center for History and New Media + George Mason University, Fairfax, Virginia, USA + http://zotero.org + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + +////////////////////////////////////////////////////////////////////////////// +// +// Zotero_Feed_Settings +// +////////////////////////////////////////////////////////////////////////////// + +var Zotero_Feed_Settings = new function() { + let urlIsValid = true, + data = null, + feedReader = null, + urlTainted = false; + + let cleanURL = function(url) { + url = url.trim(); + if (!url) return; + + let ios = Components.classes["@mozilla.org/network/io-service;1"] + .getService(Components.interfaces.nsIIOService); + + let cleanUrl; + try { + let uri = ios.newURI(url, null, null); + if (uri.scheme != 'http' && uri.scheme != 'https') { + Zotero.debug(uri.scheme + " is not a supported protocol for feeds."); + } + + cleanUrl = uri.spec; + } catch (e) { + if (e.result == Components.results.NS_ERROR_MALFORMED_URI) { + // Assume it's a URL missing "http://" part + try { + cleanUrl = ios.newURI('http://' + url, null, null).spec; + } catch (e) {} + } + throw e; + } + + if (!cleanUrl) return; + + if (/^https?:\/\/[^\/\s]+\/\S/.test(cleanUrl)) return cleanUrl; + }; + + this.init = function() { + this.toggleAdvancedOptions(false); + + data = window.arguments[0]; + + if (data.url) { + document.getElementById('feed-url').value = data.url; + } + + if (!data.url) { + this.invalidateUrl(); + } else { + // Do not allow to change URL for existing feed + document.getElementById('feed-url').readOnly = true; + } + + if (data.title) { + document.getElementById('feed-title').value = data.title; + } + + let ttl; + if (data.ttl !== undefined) { + ttl = Math.floor(data.ttl / 60); + } else { + ttl = 1; + } + document.getElementById('feed-ttl').value = ttl; + + let cleanAfter = data.cleanAfter; + if (cleanAfter === undefined) cleanAfter = 2; + document.getElementById('feed-cleanAfter').value = cleanAfter; + + if (data.url && !data.urlIsValid) { + this.validateUrl(); + } + }; + + this.invalidateUrl = function() { + urlTainted = true; + if (feedReader) { + feedReader.terminate(); + feedReader = null; + } + + if (!urlIsValid) return; + + urlIsValid = false; + document.getElementById('feed-title').disabled = true; + document.getElementById('feed-ttl').disabled = true; + document.getElementById('feed-cleanAfter').disabled = true; + document.documentElement.getButton('accept').disabled = true; + }; + + this.validateUrl = function() { + if (feedReader) { + feedReader.terminate(); + feedReader = null; + } + + let url = cleanURL(document.getElementById('feed-url').value); + urlTainted = false; + if (!url) return; + + let fr = feedReader = new Zotero.FeedReader(url); + fr.feedProperties + .then( feed => { + if (feedReader !== fr || urlTainted) return; + + let title = document.getElementById('feed-title'); + if (!data.url && feed.title) { + title.value = feed.title; + } + + let ttl = document.getElementById('feed-ttl'); + if (!data.url && feed.ttl) { + ttl.value = Math.floor(feed.ttl / 60) || 1; + } + + document.getElementById('feed-url').value = url; + + urlIsValid = true; + title.disabled = false; + ttl.disabled = false; + document.getElementById('feed-cleanAfter').disabled = false; + document.documentElement.getButton('accept').disabled = false; + }) + .catch( e => { + Zotero.debug(e); + }) + .finally( () => { + if (feedReader === fr) feedReader = null; + }); + }; + + this.accept = function() { + data.url = document.getElementById('feed-url').value; + data.title = document.getElementById('feed-title').value; + data.ttl = document.getElementById('feed-ttl').value * 60; + data.cleanAfter = document.getElementById('feed-cleanAfter').value * 1; + return true; + }; + + this.cancel = function() { + data.cancelled = true; + return true; + }; + + /* + * Show/hide advanced options + * @param {Boolean} [show] If set, indicates whether the advanced + * options should be shown or not. If omitted, the options toggle + */ + this.toggleAdvancedOptions = function(show) { + var opts = document.getElementById("advanced-options-togglable"); + opts.hidden = show !== undefined ? !show : !opts.hidden; + document.getElementById("advanced-options") + .setAttribute("state", opts.hidden ? "closed" : "open"); + window.sizeToContent(); + }; +} \ No newline at end of file diff --git a/chrome/content/zotero/feedSettings.xul b/chrome/content/zotero/feedSettings.xul new file mode 100644 index 0000000000..45ebf8014b --- /dev/null +++ b/chrome/content/zotero/feedSettings.xul @@ -0,0 +1,50 @@ + + + + %zoteroDTD; +]> + + +