Adding new translators from Frank.
This commit is contained in:
Matt Burton
parent
ba381943e1
commit
88d987c6ff
2 changed files with 281 additions and 0 deletions
160
translators/Japan Times Online.js
Normal file
160
translators/Japan Times Online.js
Normal file
|
|
@ -0,0 +1,160 @@
|
||||||
|
{
|
||||||
|
"translatorID":"b56d756e-934e-4b46-bc58-d61dccc9f32f",
|
||||||
|
"translatorType":4,
|
||||||
|
"label":"Japan Times Online",
|
||||||
|
"creator":"Frank Bennett",
|
||||||
|
"target":"^http://(?:www|search)\\.japantimes\\.co\\.jp/(?:cgi-bin|gsearch|features|entertainment|sports|life|news)",
|
||||||
|
"minVersion":"2.0b7",
|
||||||
|
"maxVersion":"",
|
||||||
|
"priority":100,
|
||||||
|
"inRepository":true,
|
||||||
|
"lastUpdated":"2009-01-23 02:17:09"
|
||||||
|
}
|
||||||
|
|
||||||
|
// #################################
|
||||||
|
// #### Local utility functions ####
|
||||||
|
// #################################
|
||||||
|
|
||||||
|
var itemRe = new RegExp('^http://search\.japantimes\.co\.jp/cgi-bin/[a-z]{2}[0-9]{8}[a-z0-9]{2}\.html');
|
||||||
|
|
||||||
|
var getResolver = function (doc) {
|
||||||
|
var namespace, resolver;
|
||||||
|
namespace = doc.documentElement.namespaceURI;
|
||||||
|
if (namespace) {
|
||||||
|
resolver = function(prefix) {
|
||||||
|
if (prefix == 'x') {
|
||||||
|
return namespace;
|
||||||
|
} else {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
} else {
|
||||||
|
resolver = null;
|
||||||
|
}
|
||||||
|
return resolver;
|
||||||
|
};
|
||||||
|
|
||||||
|
var getTagContent = function (txt, attribute, value) {
|
||||||
|
var ret, m, rex;
|
||||||
|
ret = false;
|
||||||
|
rex = RegExp("<[^>]*" + attribute + "=\"" + value + "\"[^>]*>([^<]*)<");
|
||||||
|
m = rex.exec(txt);
|
||||||
|
if (m) {
|
||||||
|
ret = m[1];
|
||||||
|
}
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
|
var getTagsWithAttributeAndContent = function (txt, tag, attribute) {
|
||||||
|
var ret, pos, len, lst, m, tagsrex, attribrex;
|
||||||
|
ret = {};
|
||||||
|
tagsrex = RegExp("(<" + tag + "(?: [^>]*>|>)|</" + tag+ ">)");
|
||||||
|
attribrex = RegExp(' ' + attribute + '="([^"]+)"');
|
||||||
|
lst = txt.split(tagsrex);
|
||||||
|
if (lst.length > 1) {
|
||||||
|
len = lst.length;
|
||||||
|
for (pos=1; pos < len; pos += 4) {
|
||||||
|
if (pos < (len - 2) && lst[pos + 2] == ("</" + tag + ">")) {
|
||||||
|
m = lst[pos].match(attribrex);
|
||||||
|
if (m) {
|
||||||
|
if (!itemRe.exec(m[1])) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
var title = lst[pos + 1];
|
||||||
|
title = title.replace(/\|.*/, "").replace(/<[^>]+>/g, "");;
|
||||||
|
ret[m[1]] = Zotero.Utilities.unescapeHTML(title);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
|
// #########################
|
||||||
|
// ##### API functions #####
|
||||||
|
// #########################
|
||||||
|
|
||||||
|
var detectWeb = function (doc, url) {
|
||||||
|
if (itemRe.test(doc.location.href)) {
|
||||||
|
return "newspaperArticle";
|
||||||
|
} else {
|
||||||
|
return "multiple";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var doWeb = function (doc, url) {
|
||||||
|
var type, nsResolver, availableItems, xpath, found, nodes, headline, pos, myurl, m, items;
|
||||||
|
nsResolver = getResolver(doc);
|
||||||
|
type = detectWeb(doc, url);
|
||||||
|
if (type === "multiple") {
|
||||||
|
availableItems = {};
|
||||||
|
if (url.match(/\/gsearch\//)) {
|
||||||
|
//
|
||||||
|
// For Google SafeSearch. Thanks, guys, it was an entertaining afternoon.
|
||||||
|
//
|
||||||
|
xpath = '//iframe[@name="googleSearchFrame"]';
|
||||||
|
var iframe = doc.evaluate(xpath, doc, nsResolver, XPathResult.ANY_TYPE, null).iterateNext();
|
||||||
|
var address = iframe.src;
|
||||||
|
var page = Zotero.Utilities.retrieveSource(address);
|
||||||
|
availableItems = getTagsWithAttributeAndContent(page, "a", "href");
|
||||||
|
} else {
|
||||||
|
xpath = '//a[contains(@href, "cgi-bin")]';
|
||||||
|
nodes = doc.evaluate(xpath, doc, nsResolver, XPathResult.ANY_TYPE, null);
|
||||||
|
found = nodes.iterateNext();
|
||||||
|
while (found) {
|
||||||
|
if (!itemRe.test(found)) {
|
||||||
|
found = nodes.iterateNext();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
headline = found.text;
|
||||||
|
//
|
||||||
|
// Some headlines have a weird structure that yields two
|
||||||
|
// entries, the second of which is blank. Nothing is lost
|
||||||
|
// by this construct.
|
||||||
|
//
|
||||||
|
if (!headline.replace("\n", "")) {
|
||||||
|
found = nodes.iterateNext();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
headline = headline.replace("\u00a0", " ", "g").replace("\n", " ", "g");
|
||||||
|
headline = headline.replace(/^\s+/, "").replace(/\s+$/, "").replace(/\s+/g, " ");
|
||||||
|
availableItems[found.href] = headline;
|
||||||
|
found = nodes.iterateNext();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (availableItems.__count__) {
|
||||||
|
items = Zotero.selectItems(availableItems);
|
||||||
|
for (myurl in items) {
|
||||||
|
if (items.hasOwnProperty(myurl)) {
|
||||||
|
scrapeAndParse(myurl);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if (type === "newspaperArticle") {
|
||||||
|
scrapeAndParse(url);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// ############################
|
||||||
|
// ##### Scraper function #####
|
||||||
|
// ############################
|
||||||
|
|
||||||
|
var scrapeAndParse = function (url) {
|
||||||
|
var item, mytxt, m, val;
|
||||||
|
item = new Zotero.Item("newspaperArticle");
|
||||||
|
|
||||||
|
mytxt = Zotero.Utilities.retrieveSource(url);
|
||||||
|
|
||||||
|
item.publicationTitle = "Japan Times Online";
|
||||||
|
item.url = url;
|
||||||
|
val = getTagContent(mytxt, "id", "date");
|
||||||
|
if (val) {
|
||||||
|
item.date = val;
|
||||||
|
}
|
||||||
|
val = getTagContent(mytxt, "id", "headline");
|
||||||
|
if (val) {
|
||||||
|
item.title = val;
|
||||||
|
}
|
||||||
|
item.attachments.push({title:"Japan Times Online snapshot", mimeType:"text/html", url:url});
|
||||||
|
item.complete();
|
||||||
|
};
|
||||||
121
translators/Mainichi Daily News.js
Normal file
121
translators/Mainichi Daily News.js
Normal file
|
|
@ -0,0 +1,121 @@
|
||||||
|
{
|
||||||
|
"translatorID":"b56f856e-934e-4b46-bc58-d61dccc9f32f",
|
||||||
|
"translatorType":4,
|
||||||
|
"label":"Mainichi Daily News",
|
||||||
|
"creator":"Frank Bennett",
|
||||||
|
"target":"^http://(?:search\\.)*mdn\\.mainichi\\.jp/(?:$|result\?|mdnnews/|perspectives/|features/|arts/|travel/)",
|
||||||
|
"minVersion":"2.0b7",
|
||||||
|
"maxVersion":"",
|
||||||
|
"priority":100,
|
||||||
|
"inRepository":true,
|
||||||
|
"lastUpdated":"2009-01-23 02:17:09"
|
||||||
|
}
|
||||||
|
|
||||||
|
// #################################
|
||||||
|
// #### Local utility functions ####
|
||||||
|
// #################################
|
||||||
|
|
||||||
|
var itemRe = new RegExp('.*/([0-9]{8})[a-z]{1}[0-9]{1}[a-z]{1}[0-9]{2}[a-z]{1}[0-9]{1}[a-z]{2}[0-9]{6}c\.html');
|
||||||
|
|
||||||
|
var getResolver = function (doc) {
|
||||||
|
var namespace, resolver;
|
||||||
|
namespace = doc.documentElement.namespaceURI;
|
||||||
|
if (namespace) {
|
||||||
|
resolver = function(prefix) {
|
||||||
|
if (prefix == 'x') {
|
||||||
|
return namespace;
|
||||||
|
} else {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
} else {
|
||||||
|
resolver = null;
|
||||||
|
}
|
||||||
|
return resolver;
|
||||||
|
};
|
||||||
|
|
||||||
|
var cleanUp = function (str) {
|
||||||
|
var ret;
|
||||||
|
ret = str.replace("\u00a0", " ", "g").replace("\n", " ", "g");
|
||||||
|
ret = ret.replace(/^\s+/, "").replace(/\s+$/, "").replace(/\s+/g, " ");
|
||||||
|
ret = ret.replace(/\|.*/, "").replace(/<[^>]+>/g, "");;
|
||||||
|
ret = Zotero.Utilities.unescapeHTML(ret);
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// #########################
|
||||||
|
// ##### API functions #####
|
||||||
|
// #########################
|
||||||
|
|
||||||
|
var detectWeb = function (doc, url) {
|
||||||
|
if (itemRe.test(doc.location.href)) {
|
||||||
|
return "newspaperArticle";
|
||||||
|
} else {
|
||||||
|
return "multiple";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var doWeb = function (doc, url) {
|
||||||
|
var type, nsResolver, availableItems, xpath, found, nodes, headline, pos, myurl, m, items, title;
|
||||||
|
nsResolver = getResolver(doc);
|
||||||
|
type = detectWeb(doc, url);
|
||||||
|
if (type === "multiple") {
|
||||||
|
availableItems = {};
|
||||||
|
if (url.match(/^http:\/\/search\.mdn\.mainichi\.jp\/result\?/)){
|
||||||
|
xpath = '//div[@class="ResultTitle"]/a[contains(@href, "mdn.mainichi.jp")]';
|
||||||
|
} else {
|
||||||
|
xpath = '//h2[@class="NewsTitle"]/a[@href]|//ul[@class="Mark"]/li/a[@href]';
|
||||||
|
}
|
||||||
|
nodes = doc.evaluate(xpath, doc, nsResolver, XPathResult.ANY_TYPE, null);
|
||||||
|
found = nodes.iterateNext();
|
||||||
|
while (found) {
|
||||||
|
if (!itemRe.test(found.href)) {
|
||||||
|
found = nodes.iterateNext();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
headline = found.textContent;
|
||||||
|
headline = cleanUp(headline);
|
||||||
|
availableItems[found.href] = headline;
|
||||||
|
found = nodes.iterateNext();
|
||||||
|
}
|
||||||
|
if (availableItems.__count__) {
|
||||||
|
items = Zotero.selectItems(availableItems);
|
||||||
|
for (myurl in items) {
|
||||||
|
if (items.hasOwnProperty(myurl)) {
|
||||||
|
scrapeAndParse(myurl, availableItems[myurl]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if (type === "newspaperArticle") {
|
||||||
|
xpath = '//h2[@class="NewsTitle"]';
|
||||||
|
nodes = doc.evaluate(xpath, doc, nsResolver, XPathResult.ANY_TYPE, null);
|
||||||
|
title = nodes.iterateNext();
|
||||||
|
if (title) {
|
||||||
|
title = cleanUp(title.textContent);
|
||||||
|
scrapeAndParse(url, title);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// ############################
|
||||||
|
// ##### Scraper function #####
|
||||||
|
// ############################
|
||||||
|
|
||||||
|
var scrapeAndParse = function (url, title) {
|
||||||
|
var item, mytxt, m, val;
|
||||||
|
item = new Zotero.Item("newspaperArticle");
|
||||||
|
item.title = title;
|
||||||
|
item.publicationTitle = "Mainichi Daily News";
|
||||||
|
item.edition = "online edition";
|
||||||
|
item.url = url;
|
||||||
|
m = itemRe.exec(url);
|
||||||
|
if (m) {
|
||||||
|
var year = m[1].slice(0,4);
|
||||||
|
var month = m[1].slice(4,6);
|
||||||
|
var day = m[1].slice(6,8);
|
||||||
|
item.date = [year, month, day].join("-");
|
||||||
|
}
|
||||||
|
item.attachments.push({title:"Mainichi Daily News snapshot", mimeType:"text/html", url:url});
|
||||||
|
item.complete();
|
||||||
|
};
|
||||||
Loading…
Reference in a new issue