From 076ee0fad23b71b47c5f1901b6e737a03be43b10 Mon Sep 17 00:00:00 2001 From: Simon Kornblith Date: Thu, 8 Jun 2006 01:26:40 +0000 Subject: [PATCH] Add PubMed scraper, fix a few other small bugs --- .../content/scholar/xpcom/ingester.js | 6 +- scrapers.sql | 126 +++++++++++++++--- 2 files changed, 110 insertions(+), 22 deletions(-) diff --git a/chrome/chromeFiles/content/scholar/xpcom/ingester.js b/chrome/chromeFiles/content/scholar/xpcom/ingester.js index 627f137d0e..da176d14ed 100644 --- a/chrome/chromeFiles/content/scholar/xpcom/ingester.js +++ b/chrome/chromeFiles/content/scholar/xpcom/ingester.js @@ -322,7 +322,7 @@ Scholar.Ingester.HTTPUtilities.prototype.doGet = function(url, onStatus, onDone) var xmlhttp = new this.window.XMLHttpRequest(); xmlhttp.open('GET', url, true); - xmlhttp.overrideMimeType("text/xml"); + xmlhttp.overrideMimeType("text/plain"); var me = this; xmlhttp.onreadystatechange = function() { @@ -335,7 +335,7 @@ Scholar.Ingester.HTTPUtilities.prototype.doPost = function(url, body, onStatus, var xmlhttp = new this.window.XMLHttpRequest(); xmlhttp.open('POST', url, true); - xmlhttp.overrideMimeType("text/xml"); + xmlhttp.overrideMimeType("text/plain"); var me = this; xmlhttp.onreadystatechange = function() { @@ -348,7 +348,7 @@ Scholar.Ingester.HTTPUtilities.prototype.doOptions = function(url, body, onStatu var xmlhttp = new this.window.XMLHttpRequest(); xmlhttp.open('OPTIONS', url, true); - xmlhttp.overrideMimeType("text/xml"); + xmlhttp.overrideMimeType("text/plain"); var me = this; xmlhttp.onreadystatechange = function() { diff --git a/scrapers.sql b/scrapers.sql index 5e33237bed..e46491e5ba 100644 --- a/scrapers.sql +++ b/scrapers.sql @@ -1,4 +1,4 @@ --- 4 +-- 5 DELETE FROM scrapers; INSERT INTO "scrapers" VALUES(1, NULL, NULL, 20060603002000, 'Amazon.com Scraper', 'Simon Kornblith', '^http://www\.amazon\.com/gp/product/', NULL, 'var prefixRDF = ''http://www.w3.org/1999/02/22-rdf-syntax-ns#''; var prefixDC = ''http://purl.org/dc/elements/1.1/''; @@ -783,7 +783,7 @@ for (var i = 0; i < elmts.length; i++) { if(firstChar == "v") { rdfUri = prefixDummy + "volume"; } else if(firstChar == "i") { - rdfUri = prefixDummy + "issue"; + rdfUri = prefixDummy + "number"; } else if(firstChar == "p") { rdfUri = prefixDummy + "pages"; var pagesRegexp = /p(\w+)\((\w+)\)/; @@ -1312,7 +1312,7 @@ utilities.loadDocument(newUri, browser, function(newBrowser) { wait();'); -INSERT INTO "scrapers" VALUES(18, NULL, NULL, 20060603002000, 'Project MUSE Scraper', 'Simon Kornblith', '^http://muse.jhu.edu/journals/[^/]+/[^/]+/[^/]+.html$', NULL, 'var prefixRDF = ''http://www.w3.org/1999/02/22-rdf-syntax-ns#''; +INSERT INTO "scrapers" VALUES(18, NULL, NULL, 20060603002000, 'Project MUSE Scraper', 'Simon Kornblith', '^http://muse\.jhu\.edu/journals/[^/]+/[^/]+/[^/]+\.html$', NULL, 'var prefixRDF = ''http://www.w3.org/1999/02/22-rdf-syntax-ns#''; var prefixDC = ''http://purl.org/dc/elements/1.1/''; var prefixDCMI = ''http://purl.org/dc/dcmitype/''; var prefixDummy = ''http://chnm.gmu.edu/firefox-scholar/''; @@ -1322,10 +1322,6 @@ var nsResolver = namespace ? function(prefix) { if (prefix == ''x'') return namespace; else return null; } : null; -function numbersOnly(text) { - return text.replace(/[^0-9]/g, ""); -} - var uri = doc.location.href; var elmts = utilities.gatherElementsOnXPath(doc, doc, ''//comment()'', nsResolver); @@ -1341,24 +1337,29 @@ for(i in elmts) { // expose DOM/XPath to sandboxed scripts var newDOM = new XML(headerData); -function mapDOM(path, rdfUri) { - if(newDOM.header[path]) { - model.addStatement(uri, rdfUri, newDOM[path].text(), true); +function mapRDF(text, rdfUri) { + if(text) { + model.addStatement(uri, rdfUri, text, true); } } -mapDOM("journal", prefixDummy + "publication"); -mapDOM("issn", prefixDummy + "publication", numbersOnly); -mapDOM("volume", prefixDummy + "volume"); -mapDOM("issue", prefixDummy + "issue"); -mapDOM("year", prefixDummy + "year"); -mapDOM("pubdate", prefixDC + "date"); -mapDOM("doctitle", prefixDC + "title"); +mapRDF(newDOM.journal.text(), prefixDummy + "publication"); +mapRDF(newDOM.volume.text(), prefixDummy + "volume"); +mapRDF(newDOM.issue.text(), prefixDummy + "number"); +mapRDF(newDOM.year.text(), prefixDummy + "year"); +mapRDF(newDOM.pubdate.text(), prefixDC + "date"); +mapRDF(newDOM.doctitle.text(), prefixDC + "title"); + +// Do ISSN +var issn = newDOM.issn.text(); +if(issn) { + model.addStatement(uri, prefixDC + "identifier", "ISSN "+issn.replace(/[^0-9]/g, ""), true); +} // Do pages var fpage = newDOM.fpage.text(); var lpage = newDOM.lpage.text(); -if(fpage) { +if(fpage != "") { var pages = fpage; if(lpage) { pages += "-"+lpage; @@ -1374,4 +1375,91 @@ for(i in elmts) { model.addStatement(uri, prefixDC + "creator", fname+" "+surname, true); } -model.addStatement(uri, prefixRDF + "type", prefixDummy + "journal", false);'); \ No newline at end of file +model.addStatement(uri, prefixRDF + "type", prefixDummy + "journal", false);'); + +INSERT INTO "scrapers" VALUES(19, NULL, NULL, 20060603002000, 'PubMed Scraper', 'Simon Kornblith', '^http://www\.ncbi\.nlm\.nih\.gov/entrez/query\.fcgi\?(?:.*db=PubMed.*list_uids=[0-9]|.*list_uids=[0-9].*db=PubMed)', NULL, 'var prefixRDF = ''http://www.w3.org/1999/02/22-rdf-syntax-ns#''; +var prefixDC = ''http://purl.org/dc/elements/1.1/''; +var prefixDCMI = ''http://purl.org/dc/dcmitype/''; +var prefixDummy = ''http://chnm.gmu.edu/firefox-scholar/''; + +function mapRDF(text, rdfUri) { + if(text != "") { + model.addStatement(uri, rdfUri, text, true); + } +} + +function stringTrimmer(x) { + var x = x.replace(/^[^\w(]+/, ""); + return x.replace(/[^\w)]+$/, ""); +} + +var uri = doc.location.href; +var newUri = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=PubMed&retmode=xml&rettype=citation&id="; +var idRegexp = /[\?\&]list_uids=([0-9\,]+)/; +var m = idRegexp.exec(uri); +newUri += m[1]; + +utilities.HTTPUtilities.doGet(newUri, null, function(text) { + // Remove xml parse instruction and doctype + text = text.replace(/]*>/, "").replace(/<\?xml[^>]*\?>/, ""); + + var xml = new XML(text); + + for(var i=0; i