mirrors/zotero
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Add a generic scraper that will scrape any website, although it may not always find very much information. It looks at META tags, both Dublin Core and otherwise.

Browse source 
When tags are ready, we can pull out META keywords.
This commit is contained in:
Simon Kornblith 2006-06-26 20:44:45 +00:00
parent 031d78eeee
commit f1cc809f76
1 changed files with 31 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

49
scrapers.sql
View file

@ -1,7 +1,7 @@
-- 26
-- 27
-- Set the following timestamp to the most recent scraper update date
REPLACE INTO "version" VALUES ('repository', STRFTIME('%s', '2006-06-26 16:01:00'));
REPLACE INTO "version" VALUES ('repository', STRFTIME('%s', '2006-06-26 16:41:00'));
REPLACE INTO "scrapers" VALUES('96b9f483-c44d-5784-cdad-ce21b984fe01', '2006-06-26 16:01:00', 'Amazon.com Scraper', 'Simon Kornblith', '^http://www\.amazon\.com/(?:gp/(?:product|search)/|exec/obidos/search-handle-url/)',
'if(doc.title.indexOf("search") >= 0) {
@ -794,7 +794,6 @@ if(newUri) {
var m = urlRe.exec(urls[0]);
var clearUrl = m[0]+"?clear_saves=1";
var postUrl = m[0];
var exportUrl = m[1]+"++export/1,-1,-1,B/export";
var actionUrl = m[2]+m[3];
var postString = "";
@ -2325,40 +2324,54 @@ utilities.HTTPUtilities.doGet(newUri, null, function(text) {
wait();');
REPLACE INTO "scrapers" VALUES('951c027d-74ac-47d4-a107-9c3069ab7b48', '2006-06-26 16:01:00', 'Scraper for Dublin Core expressed as HTML META elements', 'Simon Kornblith',
REPLACE INTO "scrapers" VALUES('951c027d-74ac-47d4-a107-9c3069ab7b48', '2006-06-26 16:41:00', 'Generic Scraper', 'Simon Kornblith', '',
'return "website";',
'var metaTags = doc.getElementsByTagName("meta");
if(metaTags) {
for(var i=0; i<metaTags.length; i++) {
var tag = metaTags[i].getAttribute("name");
var value = metaTags[i].getAttribute("content");
if(tag && value && tag.substr(0, 3).toLowerCase() == "dc.") {
return true;
}
}
}
return false;', 'var prefixRDF = ''http://www.w3.org/1999/02/22-rdf-syntax-ns#'';
'var prefixRDF = ''http://www.w3.org/1999/02/22-rdf-syntax-ns#'';
var prefixDC = ''http://purl.org/dc/elements/1.1/'';
var prefixDCMI = ''http://purl.org/dc/dcmitype/'';
var prefixDummy = ''http://chnm.gmu.edu/firefox-scholar/'';
var uri = doc.location.href;
// Eventually, we can grab a last modified date from the Last-Modified header,
// but Piggy Bank will never be able to manage that
var metaTags = doc.getElementsByTagName("meta");
var foundCreator = false; // Multiple creators encoded two different ways can screw us up
var foundTitle = false; // Can always figure this out on our own
for(var i=0; i<metaTags.length; i++) {
var tag = metaTags[i].getAttribute("name");
var value = metaTags[i].getAttribute("content");
if(tag && value && tag.substr(0, 3).toLowerCase() == "dc.") {
var suffix = tag.substr(3);
if(suffix == "creator") {
if(suffix == "creator" && !foundCreator) {
// Everyone uses different methods of encoding the DC creator; clean them
value = utilities.cleanAuthor(value);
var foundCreator = true;
}
if(suffix == "title") {
foundTitle = true;
}
model.addStatement(uri, prefixDC + suffix, value, true);
} else if(tag && value && (tag == "author" || tag == "author-personal")) {
value = utilities.cleanAuthor(value);
var foundCreator = true;
model.addStatement(uri, prefixDC + "creator", value, true);
} else if(tag && value && tag == "author-corporate") {
var foundCreator = true;
model.addStatement(uri, prefixDC + "creator", value, true);
} else if(tag && value && tag == "title") {
var foundTitle = true;
model.addStatement(uri, prefixDC + "title", value, true);
}
}');
}
if(!foundTitle) {
model.addStatement(uri, prefixDC + "title", doc.title, true);
}
model.addStatement(uri, prefixRDF + "type", prefixDummy + "website", false);');
REPLACE INTO "scrapers" VALUES('3e684d82-73a3-9a34-095f-19b112d88bbf', '2006-06-26 16:01:00', 'Google Books Scraper', 'Simon Kornblith', '^http://books\.google\.com/books\?(.*vid=.*\&id=.*|.*q=.*)',
'var re = new RegExp(''^http://books\\.google\\.com/books\\?vid=([^&]+).*\\&id=([^&]+)'', ''i'');