mirrors/zotero
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

This isn't quite done (I'm discussing changing the scrapers schema with Simon to better handle scraper updates) but in the interest of getting the scrapers in for testing, I'll commit this now.

Browse source 
Integrated the scrapers with the schema update mechanism. Changed a bunch of schema methods to handle both schema.sql and scrapers.sql (or others, if need be) and altered the version table to track mu
ltiple versions for different files. This theoretically should detect that the version table has changed and force a reinitialization of the DB--let me know if there are problems.
This commit is contained in:
Dan Stillman 2006-06-07 15:27:21 +00:00
parent 882a96ee40
commit 393807b152
3 changed files with 113 additions and 56 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

157
chrome/chromeFiles/content/scholar/xpcom/schema.js
View file

@ -1,6 +1,6 @@
Scholar.Schema = new function(){
var _dbVersion;
var _schemaVersion;
var _dbVersions = [];
var _schemaVersions = [];
this.updateSchema = updateSchema;
@ -11,21 +11,31 @@ Scholar.Schema = new function(){
var dbVersion = _getDBVersion();
var schemaVersion = _getSchemaSQLVersion();
if (dbVersion > schemaVersion){
throw("Scholar DB version is newer than schema version");
if (dbVersion == schemaVersion){
if (SCHOLAR_CONFIG['DB_REBUILD']){
if (confirm('Erase all data and recreate database from schema?')){
_initializeSchema();
return;
}
}
_updateScrapers();
return;
}
// If DB version is less than schema file, create or update
else if (dbVersion < schemaVersion){
if (!dbVersion){
Scholar.debug('Database does not exist -- creating\n');
return _initializeSchema();
_initializeSchema();
return;
}
return _migrateSchema(dbVersion);
}
else if (SCHOLAR_CONFIG['DB_REBUILD']){
if (confirm('Erase all data and recreate database from schema?')){
return _initializeSchema();
_migrateSchema(dbVersion);
_updateScrapers();
return;
}
else {
throw("Scholar DB version is newer than schema version");
}
}
@ -39,14 +49,32 @@ Scholar.Schema = new function(){
/*
* Retrieve the DB schema version
*/
function _getDBVersion(){
if (_dbVersion){
return _dbVersion;
function _getDBVersion(schema){
// Default to schema.sql
if (!schema){
schema = 'schema';
}
if (_dbVersions[schema]){
return _dbVersions[schema];
}
if (Scholar.DB.tableExists('version')){
var dbVersion = Scholar.DB.valueQuery("SELECT version FROM version;");
_dbVersion = dbVersion;
try {
var dbVersion = Scholar.DB.valueQuery("SELECT version FROM "
+ " version WHERE schema='" + schema + "'");
}
// DEBUG: this is temporary to handle version table schema change
catch(e){
if (e=='no such column: schema'){
Scholar.debug(e, 1);
return false;
}
// If some other problem, bail
throw(e);
}
_dbVersions[schema] = dbVersion;
return dbVersion;
}
return false;
@ -54,18 +82,25 @@ Scholar.Schema = new function(){
/*
* Retrieve the version attribute of the schema SQL XML
* Retrieve the version from the top line of the schema SQL file
*/
function _getSchemaSQLVersion(){
if (_schemaVersion){
return _schemaVersion;
function _getSchemaSQLVersion(schema){
// Default to schema.sql
if (!schema){
schema = 'schema';
}
var schemaFile = schema + '.sql';
if (_schemaVersions[schema]){
return _schemaVersions[schema];
}
var file = Components.classes["@mozilla.org/extensions/manager;1"]
.getService(Components.interfaces.nsIExtensionManager)
.getInstallLocation(SCHOLAR_CONFIG['GUID'])
.getItemLocation(SCHOLAR_CONFIG['GUID']);
file.append('schema.sql');
file.append(schemaFile);
// Open an input stream from file
var istream = Components.classes["@mozilla.org/network/file-input-stream;1"]
@ -80,7 +115,7 @@ Scholar.Schema = new function(){
var schemaVersion = line.value.match(/-- ([0-9]+)/)[1];
istream.close();
_schemaVersion = schemaVersion;
_schemaVersions[schema] = schemaVersion;
return schemaVersion;
}
@ -90,14 +125,21 @@ Scholar.Schema = new function(){
*
* Returns an _array_ of SQL statements for feeding into query()
*/
function _getSchemaSQL(){
function _getSchemaSQL(schema){
// Default to schema.sql
if (!schema){
schema = 'schema';
}
var schemaFile = schema + '.sql';
// We pull the schema from an external file so we only have to process
// it when necessary
var file = Components.classes["@mozilla.org/extensions/manager;1"]
.getService(Components.interfaces.nsIExtensionManager)
.getInstallLocation(SCHOLAR_CONFIG['GUID'])
.getItemLocation(SCHOLAR_CONFIG['GUID']);
file.append('schema.sql');
file.append(schemaFile);
// Open an input stream from file
var istream = Components.classes["@mozilla.org/network/file-input-stream;1"]
@ -128,9 +170,12 @@ Scholar.Schema = new function(){
function _initializeSchema(){
try {
Scholar.DB.beginTransaction();
var sql = _getSchemaSQL();
Scholar.DB.query(sql);
Scholar.DB.query("INSERT INTO version VALUES (" + _getSchemaSQLVersion() + ")");
Scholar.DB.query(_getSchemaSQL());
Scholar.DB.query("INSERT INTO version VALUES ('schema', "
+ _getSchemaSQLVersion() + ")");
Scholar.DB.query(_getSchemaSQL('scrapers'));
Scholar.DB.query("INSERT INTO version VALUES ('scrapers', "
+ _getSchemaSQLVersion('scrapers') + ")");
Scholar.DB.commitTransaction();
}
catch(e){
@ -140,6 +185,38 @@ Scholar.Schema = new function(){
}
/*
* Update a DB schema version tag in an existing database
*/
function _updateDBVersion(schema, version){
return Scholar.DB.query("UPDATE version SET version=" + version
+ " WHERE schema='" + schema + "'");
}
/*
* Update the scrapers in the DB to the latest bundled versions
*/
function _updateScrapers(){
var dbVersion = _getDBVersion('scrapers');
var schemaVersion = _getSchemaSQLVersion('scrapers');
if (dbVersion == schemaVersion){
return;
}
else if (dbVersion < schemaVersion){
Scholar.DB.beginTransaction();
Scholar.DB.query(_getSchemaSQL('scrapers'));
_updateDBVersion('scrapers', schemaVersion);
Scholar.DB.commitTransaction();
return;
}
else {
throw("Scraper set in DB is newer than schema version");
}
}
/*
* Migrate schema from an older version, preserving data
*/
@ -147,7 +224,7 @@ Scholar.Schema = new function(){
//
// Change this value to match the schema version
//
var toVersion = 13;
var toVersion = 15;
if (toVersion != _getSchemaSQLVersion()){
throw('Schema version does not match version in _migrateSchema()');
@ -162,34 +239,12 @@ Scholar.Schema = new function(){
// Each block performs the changes necessary to move from the
// previous revision to that one.
for (var i=parseInt(fromVersion) + 1; i<=toVersion; i++){
if (i==9){
Scholar.DB.query("DROP TABLE IF EXISTS objectCreators; "
+ "DROP TABLE IF EXISTS objectData; DROP TABLE IF EXISTS objectKeywords; "
+ "DROP TABLE IF EXISTS objectTypeFields; DROP TABLE IF EXISTS objectTypes; "
+ "DROP TABLE IF EXISTS objects; DROP TABLE IF EXISTS treeOrder;");
}
// For now, just wipe and recreate
if (i==13){
Scholar.DB.query("DROP TABLE IF EXISTS folders; "
+ "DROP TABLE IF EXISTS treeStructure;");
_initializeSchema();
}
if (i==14){
if (i==15){
// do stuff
}
}
_updateDBVersion(i-1);
_updateDBVersion('schema', i-1);
Scholar.DB.commitTransaction();
}
/*
* Update the DB schema version tag of an existing database
*/
function _updateDBVersion(version){
return Scholar.DB.query("UPDATE version SET version=" + version);
}
}

7
schema.sql
View file

@ -1,9 +1,12 @@
-- 13
-- 15
DROP TABLE IF EXISTS version;
CREATE TABLE version (
version INTEGER PRIMARY KEY
schema TEXT PRIMARY KEY,
version INT NOT NULL
);
DROP INDEX IF EXISTS schema;
CREATE INDEX schema ON version(schema);
DROP TABLE IF EXISTS items;
CREATE TABLE items (

3
scrapers.sql
View file

@ -1,4 +1,4 @@
BEGIN TRANSACTION;
-- 1
DELETE FROM scrapers;
INSERT INTO "scrapers" VALUES(1, NULL, NULL, 20060603002000, 'Amazon.com Scraper', 'Simon Kornblith', '^http://www\.amazon\.com/gp/product/', NULL, 'var prefixRDF = ''http://www.w3.org/1999/02/22-rdf-syntax-ns#'';
var prefixDC = ''http://purl.org/dc/elements/1.1/'';
@ -1052,4 +1052,3 @@ utilities.loadDocument(newUri, browser, function(newBrowser) {
}, function() {})
wait();');
COMMIT;