This isn't quite done (I'm discussing changing the scrapers schema with Simon to better handle scraper updates) but in the interest of getting the scrapers in for testing, I'll commit this now.

Integrated the scrapers with the schema update mechanism. Changed a bunch of schema methods to handle both schema.sql and scrapers.sql (or others, if need be) and altered the version table to track mu
ltiple versions for different files. This theoretically should detect that the version table has changed and force a reinitialization of the DB--let me know if there are problems.

chrome/chromeFiles/content/scholar/xpcom/schema.js

@@ -1,6 +1,6 @@
 Scholar.Schema = new function(){
-	var _dbVersion;
+	var _dbVersions = [];
-	var _schemaVersion;
+	var _schemaVersions = [];
 	
 	this.updateSchema = updateSchema;
 	
@@ -11,21 +11,31 @@ Scholar.Schema = new function(){
 		var dbVersion = _getDBVersion();
 		var schemaVersion = _getSchemaSQLVersion();
 		
-		if (dbVersion > schemaVersion){
+		if (dbVersion == schemaVersion){
-			throw("Scholar DB version is newer than schema version");
+			if (SCHOLAR_CONFIG['DB_REBUILD']){
+				if (confirm('Erase all data and recreate database from schema?')){
+					_initializeSchema();
+					return;
+				}
+			}
+			
+			_updateScrapers();
+			return;
 		}
+		// If DB version is less than schema file, create or update
 		else if (dbVersion < schemaVersion){
 			if (!dbVersion){
 				Scholar.debug('Database does not exist -- creating\n');
-				return _initializeSchema();
+				_initializeSchema();
+				return;
 			}
 			
-			return _migrateSchema(dbVersion);
+			_migrateSchema(dbVersion);
+			_updateScrapers();
+			return;
 		}
-		else if (SCHOLAR_CONFIG['DB_REBUILD']){
+		else {
-			if (confirm('Erase all data and recreate database from schema?')){
+			throw("Scholar DB version is newer than schema version");
-				return _initializeSchema();
-			}
 		}
 	}
 	
@@ -39,14 +49,32 @@ Scholar.Schema = new function(){
 	/*
 	 * Retrieve the DB schema version
 	 */
-	function _getDBVersion(){
+	function _getDBVersion(schema){
-		if (_dbVersion){
+		// Default to schema.sql
-			return _dbVersion;
+		if (!schema){
+			schema = 'schema';
+		}
+		
+		if (_dbVersions[schema]){
+			return _dbVersions[schema];
 		}
 		
 		if (Scholar.DB.tableExists('version')){
-			var dbVersion = Scholar.DB.valueQuery("SELECT version FROM version;");
+			try {
-			_dbVersion = dbVersion;
+				var dbVersion = Scholar.DB.valueQuery("SELECT version FROM "
+					+ " version WHERE schema='" + schema + "'");
+			}
+			// DEBUG: this is temporary to handle version table schema change
+			catch(e){
+				if (e=='no such column: schema'){
+					Scholar.debug(e, 1);
+					return false;
+				}
+				
+				// If some other problem, bail
+				throw(e);
+			}
+			_dbVersions[schema] = dbVersion;
 			return dbVersion;
 		}
 		return false;
@@ -54,18 +82,25 @@ Scholar.Schema = new function(){
 	
 	
 	/*
-	 * Retrieve the version attribute of the schema SQL XML
+	 * Retrieve the version from the top line of the schema SQL file
 	 */
-	function _getSchemaSQLVersion(){
+	function _getSchemaSQLVersion(schema){
-		if (_schemaVersion){
+		// Default to schema.sql
-			return _schemaVersion;
+		if (!schema){
+			schema = 'schema';
+		}
+		
+		var schemaFile = schema + '.sql';
+		
+		if (_schemaVersions[schema]){
+			return _schemaVersions[schema];
 		}
 		
 		var file = Components.classes["@mozilla.org/extensions/manager;1"]
                     .getService(Components.interfaces.nsIExtensionManager)
                     .getInstallLocation(SCHOLAR_CONFIG['GUID'])
                     .getItemLocation(SCHOLAR_CONFIG['GUID']); 
-		file.append('schema.sql');
+		file.append(schemaFile);
 		
 		// Open an input stream from file
 		var istream = Components.classes["@mozilla.org/network/file-input-stream;1"]
@@ -80,7 +115,7 @@ Scholar.Schema = new function(){
 		var schemaVersion = line.value.match(/-- ([0-9]+)/)[1];
 		istream.close();
 		
-		_schemaVersion = schemaVersion;
+		_schemaVersions[schema] = schemaVersion;
 		return schemaVersion;
 	}
 	
@@ -90,14 +125,21 @@ Scholar.Schema = new function(){
 	 *
 	 * Returns an _array_ of SQL statements for feeding into query()
 	 */
-	function _getSchemaSQL(){
+	function _getSchemaSQL(schema){
+		// Default to schema.sql
+		if (!schema){
+			schema = 'schema';
+		}
+		
+		var schemaFile = schema + '.sql';
+		
 		// We pull the schema from an external file so we only have to process
 		// it when necessary
 		var file = Components.classes["@mozilla.org/extensions/manager;1"]
                     .getService(Components.interfaces.nsIExtensionManager)
                     .getInstallLocation(SCHOLAR_CONFIG['GUID'])
                     .getItemLocation(SCHOLAR_CONFIG['GUID']); 
-		file.append('schema.sql');
+		file.append(schemaFile);
 		
 		// Open an input stream from file
 		var istream = Components.classes["@mozilla.org/network/file-input-stream;1"]
@@ -128,9 +170,12 @@ Scholar.Schema = new function(){
 	function _initializeSchema(){
 		try {
 			Scholar.DB.beginTransaction();
-			var sql = _getSchemaSQL();
+			Scholar.DB.query(_getSchemaSQL());
-			Scholar.DB.query(sql);
+			Scholar.DB.query("INSERT INTO version VALUES ('schema', "
-			Scholar.DB.query("INSERT INTO version VALUES (" + _getSchemaSQLVersion() + ")");
+				+ _getSchemaSQLVersion() + ")");
+			Scholar.DB.query(_getSchemaSQL('scrapers'));
+			Scholar.DB.query("INSERT INTO version VALUES ('scrapers', "
+				+ _getSchemaSQLVersion('scrapers') + ")");
 			Scholar.DB.commitTransaction();
 		}
 		catch(e){
@@ -140,6 +185,38 @@ Scholar.Schema = new function(){
 	}
 	
 	
+	/*
+	 * Update a DB schema version tag in an existing database
+	 */
+	function _updateDBVersion(schema, version){
+		return Scholar.DB.query("UPDATE version SET version=" + version
+			+ " WHERE schema='" + schema + "'");
+	}
+	
+	
+	/*
+	 * Update the scrapers in the DB to the latest bundled versions
+	 */
+	function _updateScrapers(){
+		var dbVersion = _getDBVersion('scrapers');
+		var schemaVersion = _getSchemaSQLVersion('scrapers');
+		
+		if (dbVersion == schemaVersion){
+			return;
+		}
+		else if (dbVersion < schemaVersion){
+			Scholar.DB.beginTransaction();
+			Scholar.DB.query(_getSchemaSQL('scrapers'));
+			_updateDBVersion('scrapers', schemaVersion);
+			Scholar.DB.commitTransaction();
+			return;
+		}
+		else {
+			throw("Scraper set in DB is newer than schema version");
+		}
+	}
+	
+	
 	/*
 	 * Migrate schema from an older version, preserving data
 	 */
@@ -147,7 +224,7 @@ Scholar.Schema = new function(){
 		//
 		// Change this value to match the schema version
 		//
-		var toVersion = 13;
+		var toVersion = 15;
 		
 		if (toVersion != _getSchemaSQLVersion()){
 			throw('Schema version does not match version in _migrateSchema()');
@@ -162,34 +239,12 @@ Scholar.Schema = new function(){
 		// Each block performs the changes necessary to move from the
 		// previous revision to that one.
 		for (var i=parseInt(fromVersion) + 1; i<=toVersion; i++){
-			if (i==9){
+			if (i==15){
-				Scholar.DB.query("DROP TABLE IF EXISTS objectCreators; "
-					+ "DROP TABLE IF EXISTS objectData; DROP TABLE IF EXISTS objectKeywords; "
-					+ "DROP TABLE IF EXISTS objectTypeFields; DROP TABLE IF EXISTS objectTypes; "
-					+ "DROP TABLE IF EXISTS objects; DROP TABLE IF EXISTS treeOrder;");
-			}
-			
-			// For now, just wipe and recreate
-			if (i==13){
-				Scholar.DB.query("DROP TABLE IF EXISTS folders; "
-					+ "DROP TABLE IF EXISTS treeStructure;");
-				_initializeSchema();
-			}
-			
-			if (i==14){
 				// do stuff
 			}
 		}
 		
-		_updateDBVersion(i-1);
+		_updateDBVersion('schema', i-1);
 		Scholar.DB.commitTransaction();
 	}
-	
-	
-	/*
-	 * Update the DB schema version tag of an existing database
-	 */
-	function _updateDBVersion(version){
-		return Scholar.DB.query("UPDATE version SET version=" + version);
-	}
 }

schema.sql

@@ -1,9 +1,12 @@
--- 13
+-- 15
 
     DROP TABLE IF EXISTS version;
     CREATE TABLE version (
-        version INTEGER PRIMARY KEY
+        schema TEXT PRIMARY KEY,
+        version INT NOT NULL
     );
+    DROP INDEX IF EXISTS schema;
+    CREATE INDEX schema ON version(schema);
     
     DROP TABLE IF EXISTS items;
     CREATE TABLE items (

scrapers.sql

@@ -1,4 +1,4 @@
-BEGIN TRANSACTION;
+-- 1
 DELETE FROM scrapers;
 INSERT INTO "scrapers" VALUES(1, NULL, NULL, 20060603002000, 'Amazon.com Scraper', 'Simon Kornblith', '^http://www\.amazon\.com/gp/product/', NULL, 'var prefixRDF = ''http://www.w3.org/1999/02/22-rdf-syntax-ns#'';
 var prefixDC = ''http://purl.org/dc/elements/1.1/'';
@@ -1051,5 +1051,4 @@ utilities.loadDocument(newUri, browser, function(newBrowser) {
 	done();
 }, function() {})
 
-wait();');
+wait();');
-COMMIT;