mirrors/zotero
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Made the schema update system moderately less convoluted

Browse source 
- Broke schema functions into separate object and got rid of DB_VERSION config constant in favor of a toVersion variable in the _migrateSchema command (which isn't technically necessary either, since the version number at the top of schema.sql is now always compared to the DB version at startup) but will help reduce the chance that someone will update the schema file without adding migration steps)

- Removed Amazon scraper from schema.sql, as it will be loaded with the rest of the scrapers
This commit is contained in:
Dan Stillman 2006-06-07 01:02:59 +00:00
parent 098e90fea9
commit 448fde9ff1
5 changed files with 206 additions and 250 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

181
chrome/chromeFiles/content/scholar/xpcom/db.js
View file

@ -15,11 +15,11 @@ Scholar.DB = new function(){
this.statementQuery = statementQuery;
this.getColumns = getColumns;
this.getColumnHash = getColumnHash;
this.updateSchema = updateSchema;
this.beginTransaction = beginTransaction;
this.commitTransaction = commitTransaction;
this.rollbackTransaction = rollbackTransaction;
this.transactionInProgress = transactionInProgress;
this.tableExists = tableExists;
/////////////////////////////////////////////////////////////////
//
@ -246,6 +246,11 @@ Scholar.DB = new function(){
}
function tableExists(table){
return _getDBConnection().tableExists(table);
}
function getColumns(table){
var db = _getDBConnection();
@ -277,30 +282,6 @@ Scholar.DB = new function(){
return hash;
}
/*
* Checks if the DB schema exists and is up-to-date, updating if necessary
*/
function updateSchema(){
var DBVersion = _getDBVersion();
if (DBVersion > SCHOLAR_CONFIG['DB_VERSION']){
throw("Scholar DB version is newer than config version");
}
else if (DBVersion < SCHOLAR_CONFIG['DB_VERSION']){
if (!DBVersion){
Scholar.debug('Database does not exist -- creating\n');
return _initializeSchema();
}
return _migrateSchema(DBVersion);
}
else if (SCHOLAR_CONFIG['DB_REBUILD']){
if (confirm('Erase all data and recreate database from schema?')){
return _initializeSchema();
}
}
}
/////////////////////////////////////////////////////////////////
@ -333,154 +314,4 @@ Scholar.DB = new function(){
return _connection;
}
/*
* Retrieve the DB schema version
*/
function _getDBVersion(){
if (_getDBConnection().tableExists('version')){
return valueQuery("SELECT version FROM version;");
}
return false;
}
/*
* Load in SQL schema
*
* Returns an _array_ of SQL statements for feeding into query()
*/
function _getSchemaSQL(){
// We pull the schema from an external file so we only have to process
// it when necessary
var file = Components.classes["@mozilla.org/extensions/manager;1"]
.getService(Components.interfaces.nsIExtensionManager)
.getInstallLocation(SCHOLAR_CONFIG['GUID'])
.getItemLocation(SCHOLAR_CONFIG['GUID']);
file.append('schema.sql');
// Open an input stream from file
var istream = Components.classes["@mozilla.org/network/file-input-stream;1"]
.createInstance(Components.interfaces.nsIFileInputStream);
istream.init(file, 0x01, 0444, 0);
istream.QueryInterface(Components.interfaces.nsILineInputStream);
var line = {}, sql = '', hasmore;
// Fetch the schema version from the first line of the file
istream.readLine(line);
var schemaVersion = line.value.match(/-- ([0-9]+)/)[1];
do {
hasmore = istream.readLine(line);
sql += line.value + "\n";
} while(hasmore);
istream.close();
if (schemaVersion!=SCHOLAR_CONFIG['DB_VERSION']){
throw("Scholar config version does not match schema version");
}
return sql;
}
/*
* Retrieve the version attribute of the schema SQL XML
*/
function _getSchemaSQLVersion(){
var file = Components.classes["@mozilla.org/extensions/manager;1"]
.getService(Components.interfaces.nsIExtensionManager)
.getInstallLocation(SCHOLAR_CONFIG['GUID'])
.getItemLocation(SCHOLAR_CONFIG['GUID']);
file.append('schema.sql');
// Open an input stream from file
var istream = Components.classes["@mozilla.org/network/file-input-stream;1"]
.createInstance(Components.interfaces.nsIFileInputStream);
istream.init(file, 0x01, 0444, 0);
istream.QueryInterface(Components.interfaces.nsILineInputStream);
var line = {};
// Fetch the schema version from the first line of the file
istream.readLine(line);
var schemaVersion = line.value.match(/-- ([0-9]+)/)[1];
istream.close();
return schemaVersion;
}
/*
* Create new DB schema
*/
function _initializeSchema(){
try {
beginTransaction();
var sql = _getSchemaSQL();
query(sql);
query("INSERT INTO version VALUES (" + SCHOLAR_CONFIG['DB_VERSION'] + ")");
commitTransaction();
}
catch(e){
alert(e);
rollbackTransaction();
}
}
/*
* Migrate schema from an older version, preserving data
*/
function _migrateSchema(fromVersion){
var toVersion = SCHOLAR_CONFIG['DB_VERSION'];
var schemaVersion = _getSchemaSQLVersion();
if (toVersion!=schemaVersion){
throw("Scholar config version does not match schema version");
}
Scholar.debug('Updating DB from version ' + fromVersion + ' to ' + toVersion + '\n');
// Step through version changes until we reach the current version
//
// Each block performs the changes necessary to move from the
// previous revision to that one.
//
// N.B. Be sure to call _updateDBVersion(i) at the end of each block and
// update SCHOLAR_CONFIG['DB_VERSION'] to the target version
for (var i=parseInt(fromVersion) + 1; i<=toVersion; i++){
if (i==9){
Scholar.DB.query("DROP TABLE IF EXISTS objectCreators; "
+ "DROP TABLE IF EXISTS objectData; DROP TABLE IF EXISTS objectKeywords; "
+ "DROP TABLE IF EXISTS objectTypeFields; DROP TABLE IF EXISTS objectTypes; "
+ "DROP TABLE IF EXISTS objects; DROP TABLE IF EXISTS treeOrder;");
_updateDBVersion(i);
}
// For now, just wipe and recreate
if (i==13){
Scholar.DB.query("DROP TABLE IF EXISTS folders; "
+ "DROP TABLE IF EXISTS treeStructure;");
_initializeSchema();
}
if (i==14){
// do stuff
// _updateDBVersion(i);
}
}
}
/*
* Update the DB schema version tag of an existing database
*/
function _updateDBVersion(version){
return query("UPDATE version SET version=" + version);
}
}

195
chrome/chromeFiles/content/scholar/xpcom/schema.js Normal file
View file

@ -0,0 +1,195 @@
Scholar.Schema = new function(){
var _dbVersion;
var _schemaVersion;
this.updateSchema = updateSchema;
/*
* Checks if the DB schema exists and is up-to-date, updating if necessary
*/
function updateSchema(){
var dbVersion = _getDBVersion();
var schemaVersion = _getSchemaSQLVersion();
if (dbVersion > schemaVersion){
throw("Scholar DB version is newer than schema version");
}
else if (dbVersion < schemaVersion){
if (!dbVersion){
Scholar.debug('Database does not exist -- creating\n');
return _initializeSchema();
}
return _migrateSchema(dbVersion);
}
else if (SCHOLAR_CONFIG['DB_REBUILD']){
if (confirm('Erase all data and recreate database from schema?')){
return _initializeSchema();
}
}
}
/////////////////////////////////////////////////////////////////
//
// Private methods
//
/////////////////////////////////////////////////////////////////
/*
* Retrieve the DB schema version
*/
function _getDBVersion(){
if (_dbVersion){
return _dbVersion;
}
if (Scholar.DB.tableExists('version')){
var dbVersion = Scholar.DB.valueQuery("SELECT version FROM version;");
_dbVersion = dbVersion;
return dbVersion;
}
return false;
}
/*
* Retrieve the version attribute of the schema SQL XML
*/
function _getSchemaSQLVersion(){
if (_schemaVersion){
return _schemaVersion;
}
var file = Components.classes["@mozilla.org/extensions/manager;1"]
.getService(Components.interfaces.nsIExtensionManager)
.getInstallLocation(SCHOLAR_CONFIG['GUID'])
.getItemLocation(SCHOLAR_CONFIG['GUID']);
file.append('schema.sql');
// Open an input stream from file
var istream = Components.classes["@mozilla.org/network/file-input-stream;1"]
.createInstance(Components.interfaces.nsIFileInputStream);
istream.init(file, 0x01, 0444, 0);
istream.QueryInterface(Components.interfaces.nsILineInputStream);
var line = {};
// Fetch the schema version from the first line of the file
istream.readLine(line);
var schemaVersion = line.value.match(/-- ([0-9]+)/)[1];
istream.close();
_schemaVersion = schemaVersion;
return schemaVersion;
}
/*
* Load in SQL schema
*
* Returns an _array_ of SQL statements for feeding into query()
*/
function _getSchemaSQL(){
// We pull the schema from an external file so we only have to process
// it when necessary
var file = Components.classes["@mozilla.org/extensions/manager;1"]
.getService(Components.interfaces.nsIExtensionManager)
.getInstallLocation(SCHOLAR_CONFIG['GUID'])
.getItemLocation(SCHOLAR_CONFIG['GUID']);
file.append('schema.sql');
// Open an input stream from file
var istream = Components.classes["@mozilla.org/network/file-input-stream;1"]
.createInstance(Components.interfaces.nsIFileInputStream);
istream.init(file, 0x01, 0444, 0);
istream.QueryInterface(Components.interfaces.nsILineInputStream);
var line = {}, sql = '', hasmore;
// Skip the first line, which contains the schema version
istream.readLine(line);
//var schemaVersion = line.value.match(/-- ([0-9]+)/)[1];
do {
hasmore = istream.readLine(line);
sql += line.value + "\n";
} while(hasmore);
istream.close();
return sql;
}
/*
* Create new DB schema
*/
function _initializeSchema(){
try {
beginTransaction();
var sql = _getSchemaSQL();
query(sql);
query("INSERT INTO version VALUES (" + _getSchemaSQLVersion() + ")");
commitTransaction();
}
catch(e){
alert(e);
rollbackTransaction();
}
}
/*
* Migrate schema from an older version, preserving data
*/
function _migrateSchema(fromVersion){
//
// Change this value to match the schema version
//
var toVersion = 13;
if (toVersion != _getSchemaSQLVersion()){
throw('Schema version does not match version in _migrateSchema()');
}
Scholar.debug('Updating DB from version ' + fromVersion + ' to ' + toVersion + '\n');
Scholar.DB.beginTransaction();
// Step through version changes until we reach the current version
//
// Each block performs the changes necessary to move from the
// previous revision to that one.
for (var i=parseInt(fromVersion) + 1; i<=toVersion; i++){
if (i==9){
Scholar.DB.query("DROP TABLE IF EXISTS objectCreators; "
+ "DROP TABLE IF EXISTS objectData; DROP TABLE IF EXISTS objectKeywords; "
+ "DROP TABLE IF EXISTS objectTypeFields; DROP TABLE IF EXISTS objectTypes; "
+ "DROP TABLE IF EXISTS objects; DROP TABLE IF EXISTS treeOrder;");
}
// For now, just wipe and recreate
if (i==13){
Scholar.DB.query("DROP TABLE IF EXISTS folders; "
+ "DROP TABLE IF EXISTS treeStructure;");
_initializeSchema();
}
if (i==14){
// do stuff
}
}
_updateDBVersion(i-1);
Scholar.DB.commitTransaction();
}
/*
* Update the DB schema version tag of an existing database
*/
function _updateDBVersion(version){
return Scholar.DB.query("UPDATE version SET version=" + version);
}
}

3
chrome/chromeFiles/content/scholar/xpcom/scholar.js
View file

@ -1,7 +1,6 @@
const SCHOLAR_CONFIG = {
GUID: 'scholar@chnm.gmu.edu',
DB_FILE: 'scholar.sqlite',
DB_VERSION: 13, // must match version at top of schema.sql
DB_REBUILD: false, // erase DB and recreate from schema
DEBUG_LOGGING: true,
DEBUG_TO_CONSOLE: true // dump debug messages to console rather than (much slower) Debug Logger
@ -34,7 +33,7 @@ var Scholar = new function(){
return false;
}
Scholar.DB.updateSchema();
Scholar.Schema.updateSchema();
// Load in the localization stringbundle for use by getString(name)
var src = 'chrome://scholar/locale/scholar.properties';

5
components/chnmIScholarService.js
View file

@ -18,11 +18,14 @@ Cc["@mozilla.org/moz/jssubscript-loader;1"]
.getService(Ci.mozIJSSubScriptLoader)
.loadSubScript("chrome://scholar/content/xpcom/scholar.js");
Cc["@mozilla.org/moz/jssubscript-loader;1"]
.getService(Ci.mozIJSSubScriptLoader)
.loadSubScript("chrome://scholar/content/xpcom/db.js");
Cc["@mozilla.org/moz/jssubscript-loader;1"]
.getService(Ci.mozIJSSubScriptLoader)
.loadSubScript("chrome://scholar/content/xpcom/schema.js");
Cc["@mozilla.org/moz/jssubscript-loader;1"]
.getService(Ci.mozIJSSubScriptLoader)
.loadSubScript("chrome://scholar/content/xpcom/data_access.js");

72
schema.sql
View file

@ -251,75 +251,3 @@
INSERT INTO collectionItems VALUES (6856, 13, 1);
INSERT INTO collectionItems VALUES (7373, 15, 0);
INSERT INTO collectionItems VALUES (1241, 12, 0);
INSERT INTO "scrapers" VALUES(1, NULL, NULL, NULL, 'Amazon.com Scraper', 'Simon Kornblith', '^http://www.amazon.com/gp/product/', NULL, 'var prefixRDF = ''http://www.w3.org/1999/02/22-rdf-syntax-ns#'';
var prefixDC = ''http://purl.org/dc/elements/1.1/'';
var prefixDCMI = ''http://purl.org/dc/dcmitype/'';
var prefixDummy = ''http://chnm.gmu.edu/firefox-scholar/'';
var namespace = doc.documentElement.namespaceURI;
var nsResolver = namespace ? function(prefix) {
if (prefix == ''x'') return namespace; else return null;
} : null;
var getNode = function(doc, contextNode, xpath, nsResolver) {
return doc.evaluate(xpath, contextNode, nsResolver, XPathResult.ANY_TYPE,null).iterateNext();
}
var cleanString = function(s) {
s = utilities.trimString(s);
return s.replace(/ +/g, " ");
}
var uri = doc.location.href;
model.addStatement(uri, prefixRDF + "type", prefixDCMI + "text", false);
// Retrieve authors
var xpath = ''/html/body/table/tbody/tr/td[2]/form/div[@class="buying"]/a'';
var elmts = utilities.gatherElementsOnXPath(doc, doc, xpath, nsResolver);
for (var i = 0; i < elmts.length; i++) {
var elmt = elmts[i];
model.addStatement(uri, prefixDC + ''creator'', cleanString(getNode(doc, elmt, ''./text()[1]'', nsResolver).nodeValue), false); // Use your own type here
}
// Retrieve data from "Product Details" box
var xpath = ''/html/body/table/tbody/tr/td[2]/table/tbody/tr/td[@class="bucket"]/div[@class="content"]/ul/li'';
var elmts = utilities.gatherElementsOnXPath(doc, doc, xpath, nsResolver);
for (var i = 0; i < elmts.length; i++) {
var elmt = elmts[i];
var attribute = cleanString(getNode(doc, elmt, ''./B[1]/text()[1]'', nsResolver).nodeValue);
if(getNode(doc, elmt, ''./text()[1]'', nsResolver)) {
var value = cleanString(getNode(doc, elmt, ''./text()[1]'', nsResolver).nodeValue);
if(attribute == "Publisher:") {
if(value.lastIndexOf("(") != -1) {
var date = value.substring(value.lastIndexOf("(")+1, value.length-1);
value = value.substring(0, value.lastIndexOf("(")-1);
}
if(value.lastIndexOf(";") != -1) {
var edition = value.substring(value.lastIndexOf(";")+2, value.length);
value = value.substring(0, value.lastIndexOf(";"));
}
model.addStatement(uri, prefixDC + ''publisher'', value);
model.addStatement(uri, prefixDC + ''date'', date);
model.addStatement(uri, prefixDC + ''hasVersion'', edition);
} else if(attribute == "Language:") {
model.addStatement(uri, prefixDC + ''language'', value);
} else if(attribute == "ISBN:") {
model.addStatement(uri, prefixDC + ''identifier'', ''ISBN ''+value);
} else if(value.substring(value.indexOf(" ")+1, value.length) == "pages") {
model.addStatement(uri, prefixDummy + ''pages'', value.substring(0, value.indexOf(" ")));
model.addStatement(uri, prefixDC + ''medium'', attribute.substring(0, attribute.indexOf(":")));
}
}
}
var xpath = ''/html/body/table/tbody/tr/td[2]/form/div[@class="buying"]/b[@class="sans"]'';
var elmts = utilities.gatherElementsOnXPath(doc, doc, xpath, nsResolver);
var title = cleanString(getNode(doc, elmts[0], ''./text()[1]'', nsResolver).nodeValue);
if(title.lastIndexOf("(") != -1 && title.lastIndexOf(")") == title.length-1) {
title = title.substring(0, title.lastIndexOf("(")-1);
}
model.addStatement(uri, prefixDC + ''title'', title);');