mirrors/zotero
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

- make proquest work better behind proxies

Browse source 
- improved frame support
This commit is contained in:
Simon Kornblith 2006-08-24 18:00:48 +00:00
parent c5ec34d6ae
commit 0e63958f96
3 changed files with 46 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
chrome/chromeFiles/content/scholar/ingester/browser.js
View file

@ -88,6 +88,17 @@ Scholar_Ingester_Interface.scrapeThisPage = function(saveLocation) {
}
}
Scholar_Ingester_Interface.searchFrames = function(rootDoc, searchDoc) {
for each(var frame in rootDoc.frames) {
if(frame.document == searchDoc ||
(frame.document.frames && searchFrames(frame, searchDoc))) {
return true;
}
}
return false;
}
/*
* An event handler called when a new document is loaded. Creates a new document
* object, and updates the status of the capture icon
@ -118,9 +129,13 @@ Scholar_Ingester_Interface.contentLoad = function(event) {
var data = Scholar_Ingester_Interface._getData(browser);
// if there's already a scrapable page in the browser window, and it's
// still there, return
// still there, ensure it is actually part of the page, then return
if(data.translators && data.translators.length && data.document.location) {
return;
if(Scholar_Ingester_Interface.searchFrames(rootDoc, data.document)) {
return;
} else {
data.document = null;
}
}
// get translators

4
chrome/chromeFiles/content/scholar/xpcom/utilities.js
View file

@ -259,12 +259,12 @@ Scholar.Utilities.Ingester._protocolRe = new RegExp();
Scholar.Utilities.Ingester._protocolRe.compile("^(?:(?:http|https|ftp):|[^:]*/)", "i");
Scholar.Utilities.Ingester.prototype.processDocuments = function(urls, processor, done, exception) {
if(this.translate.locationIsProxied) {
for(i in urls) {
for(var i in urls) {
if(this.translate.locationIsProxied) {
urls[i] = Scholar.Ingester.ProxyMonitor.properToProxy(urls[i]);
}
// check for a protocol colon
if(!Scholar.Utilities.Ingester._protocolRe.test(uris[i])) {
if(!Scholar.Utilities.Ingester._protocolRe.test(urls[i])) {
throw("invalid URL in processDocuments");
}
}

43
scrapers.sql
View file

@ -1,4 +1,4 @@
-- 54
-- 55
-- Set the following timestamp to the most recent scraper update date
REPLACE INTO "version" VALUES ('repository', STRFTIME('%s', '2006-08-15 15:42:00'));
@ -1100,15 +1100,25 @@ function doWeb(doc, url) {
}
}');
REPLACE INTO "translators" VALUES ('a77690cf-c5d1-8fc4-110f-d1fc765dcf88', '2006-06-26 16:01:00', 4, 'ProQuest', 'Simon Kornblith', '^http://proquest\.umi\.com/pqdweb\?((?:.*\&)?did=.*&Fmt=[0-9]|(?:.*\&)Fmt=[0-9].*&did=|(?:.*\&)searchInterface=)',
REPLACE INTO "translators" VALUES ('a77690cf-c5d1-8fc4-110f-d1fc765dcf88', '2006-06-26 16:01:00', 4, 'ProQuest', 'Simon Kornblith', '^http://[^/]+/pqdweb\?((?:.*\&)?did=.*&Fmt=[0-9]|(?:.*\&)Fmt=[0-9].*&did=|(?:.*\&)searchInterface=)',
'function detectWeb(doc, url) {
if(doc.title == "Results") {
return "multiple";
} else {
return "magazineArticle";
var namespace = doc.documentElement.namespaceURI;
var nsResolver = namespace ? function(prefix) {
if (prefix == ''x'') return namespace; else return null;
} : null;
if(doc.evaluate(''//img[substring(@src, string-length(@src)-32) = "/images/common/logo_proquest.gif" or substring(@src, string-length(@src)-38) = "/images/common/logo_proquest_small.gif"]'',
doc, nsResolver, XPathResult.ANY_TYPE, null)) {
if(doc.title == "Results") {
return "multiple";
} else {
return "magazineArticle";
}
}
}',
'function scrape(doc) {
Scholar.Utilities.debug(doc.getElementsByTagName("body")[0].innerHTML);
var namespace = doc.documentElement.namespaceURI;
var nsResolver = namespace ? function(prefix) {
if (prefix == ''x'') return namespace; else return null;
@ -1128,7 +1138,7 @@ REPLACE INTO "translators" VALUES ('a77690cf-c5d1-8fc4-110f-d1fc765dcf88', '2006
// there are sometimes additional tags representing higlighting
var author = Scholar.Utilities.getNodeString(doc, elmt, ''.//text()'', nsResolver);
if(author) {
newItem.creators.push(Scholar.Utilities.cleanAuthor(author, "author", true));
newItem.creators.push(Scholar.Utilities.cleanAuthor(author, "author"));
}
}
@ -1275,7 +1285,7 @@ function doWeb(doc, url) {
// Require link to match this
var tagRegexp = new RegExp();
tagRegexp.compile(''^http://[^/]+/pqdweb\\?((?:.*&)?did=.*&Fmt=[12]|(?:.*&)Fmt=[12].*&did=)'');
tagRegexp.compile(''^http://[^/]+/pqdweb\\?((?:.*&)?did=.*&Fmt=[12][^0-9]|(?:.*&)Fmt=[12][^0-9].*&did=)'');
var tableRows = doc.evaluate(''//tr[@class="rowUnMarked"]'',
doc, nsResolver, XPathResult.ANY_TYPE, null);
@ -1301,22 +1311,23 @@ function doWeb(doc, url) {
return true;
}
var uris = new Array();
var urls = new Array();
for(var i in items) {
uris.push(i);
urls.push(i);
}
Scholar.Utilities.processDocuments(uris, function(doc) { scrape(doc) },
Scholar.Utilities.processDocuments(urls, function(doc) { scrape(doc) },
function() { Scholar.done(); }, null);
Scholar.wait();
} else {
var fmtCheck = /(?:\&|\?)Fmt=([0-9]+)/i
var m = fmtCheck.exec(doc.location.href);
if(m && (m[1] == "1" || m[1] == "2" || m[1] == "3")) {
if(doc.evaluate(''/html/body/span[@class="textMedium"]/table/tbody/tr/td[@class="headerBlack"]/strong//text()'',
doc, nsResolver, XPathResult.ANY_TYPE, null).iterateNext()) {
scrape(doc);
} else if(m) {
Scholar.Utilities.loadDocument(doc.location.href.replace("Fmt="+m[1], "Fmt=1"), function(doc) { scrape(doc); Scholar.done(); }, null);
} else {
var newURL = doc.location.href.replace(/RQT=[0-9]+/i, "RQT=309");
newURL = newURL.replace(/Fmt=[0-9]+/i, "Fmt=1");
Scholar.Utilities.loadDocument(newURL, function(doc) { scrape(doc); Scholar.done(); }, null);
Scholar.wait();
}
}