f07ff9ac2a
If you have attachments to the old terminology, feel free to file a complaint. Changed interface code too, since David is gone (or at the very least has more important things to do with his remaining time)
438 lines
12 KiB
JavaScript
438 lines
12 KiB
JavaScript
Scholar.Schema = new function(){
|
|
var _dbVersions = [];
|
|
var _schemaVersions = [];
|
|
var _repositoryTimer;
|
|
|
|
this.updateSchema = updateSchema;
|
|
this.updateScrapersRemote = updateScrapersRemote;
|
|
this.stopRepositoryTimer = stopRepositoryTimer;
|
|
|
|
/*
|
|
* Checks if the DB schema exists and is up-to-date, updating if necessary
|
|
*/
|
|
function updateSchema(){
|
|
var dbVersion = _getDBVersion();
|
|
var schemaVersion = _getSchemaSQLVersion();
|
|
|
|
if (dbVersion == schemaVersion){
|
|
if (SCHOLAR_CONFIG['DB_REBUILD']){
|
|
if (confirm('Erase all data and recreate database from schema?')){
|
|
_initializeSchema();
|
|
return;
|
|
}
|
|
}
|
|
|
|
_updateScrapersLocal();
|
|
return;
|
|
}
|
|
// If DB version is less than schema file, create or update
|
|
else if (dbVersion < schemaVersion){
|
|
if (!dbVersion){
|
|
Scholar.debug('Database does not exist -- creating\n');
|
|
_initializeSchema();
|
|
return;
|
|
}
|
|
|
|
_migrateSchema(dbVersion);
|
|
_updateScrapersLocal();
|
|
return;
|
|
}
|
|
else {
|
|
throw("Scholar DB version is newer than schema version");
|
|
}
|
|
}
|
|
|
|
|
|
/**
|
|
* Send XMLHTTP request for updated scrapers to the central repository
|
|
*
|
|
* _force_ forces a repository query regardless of how long it's been
|
|
* since the last check
|
|
**/
|
|
function updateScrapersRemote(force){
|
|
if (!force){
|
|
// Check user preference for automatic updates
|
|
if (!Scholar.Prefs.get('automaticScraperUpdates')){
|
|
Scholar.debug('Automatic scraper updating disabled -- not checking repository', 4);
|
|
return false;
|
|
}
|
|
|
|
// Determine the earliest local time that we'd query the repository again
|
|
var nextCheck = new Date();
|
|
nextCheck.setTime((parseInt(_getDBVersion('lastcheck'))
|
|
+ SCHOLAR_CONFIG['REPOSITORY_CHECK_INTERVAL']) * 1000); // JS uses ms
|
|
var now = new Date();
|
|
|
|
// If enough time hasn't passed, don't update
|
|
if (now < nextCheck){
|
|
Scholar.debug('Not enough time since last update -- not checking repository', 4);
|
|
// Set the repository timer to the remaining time
|
|
_setRepositoryTimer(Math.round((nextCheck.getTime() - now.getTime()) / 1000));
|
|
return false;
|
|
}
|
|
}
|
|
|
|
// If transaction already in progress, delay by a few seconds
|
|
if (Scholar.DB.transactionInProgress()){
|
|
Scholar.debug('Transaction in progress -- delaying repository check', 4)
|
|
_setRepositoryTimer(30);
|
|
return false;
|
|
}
|
|
|
|
// Get the last timestamp we got from the server
|
|
var lastUpdated = _getDBVersion('repository');
|
|
|
|
var url = SCHOLAR_CONFIG['REPOSITORY_URL'] + '/updated?'
|
|
+ (lastUpdated ? 'last=' + lastUpdated + '&' : '')
|
|
+ 'version=' + Scholar.version;
|
|
|
|
Scholar.debug('Checking repository for updates (' + url + ')');
|
|
var get = Scholar.Utilities.HTTP.doGet(url, _updateScrapersRemoteCallback);
|
|
|
|
// TODO: instead, add an observer to start and stop timer on online state change
|
|
if (!get){
|
|
Scholar.debug('Browser is offline -- skipping check');
|
|
_setRepositoryTimer(SCHOLAR_CONFIG['REPOSITORY_RETRY_INTERVAL']);
|
|
}
|
|
}
|
|
|
|
|
|
function stopRepositoryTimer(){
|
|
if (_repositoryTimer){
|
|
Scholar.debug('Stopping repository check timer');
|
|
_repositoryTimer.cancel();
|
|
}
|
|
}
|
|
|
|
|
|
|
|
/////////////////////////////////////////////////////////////////
|
|
//
|
|
// Private methods
|
|
//
|
|
/////////////////////////////////////////////////////////////////
|
|
|
|
/*
|
|
* Retrieve the DB schema version
|
|
*/
|
|
function _getDBVersion(schema){
|
|
// Default to schema.sql
|
|
if (!schema){
|
|
schema = 'schema';
|
|
}
|
|
|
|
if (_dbVersions[schema]){
|
|
return _dbVersions[schema];
|
|
}
|
|
|
|
if (Scholar.DB.tableExists('version')){
|
|
try {
|
|
var dbVersion = Scholar.DB.valueQuery("SELECT version FROM "
|
|
+ "version WHERE schema='" + schema + "'");
|
|
}
|
|
// DEBUG: this is temporary to handle version table schema change
|
|
catch(e){
|
|
if (e=='no such column: schema'){
|
|
Scholar.debug(e, 1);
|
|
return false;
|
|
}
|
|
|
|
// If some other problem, bail
|
|
throw(e);
|
|
}
|
|
_dbVersions[schema] = dbVersion;
|
|
return dbVersion;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
|
|
/*
|
|
* Retrieve the version from the top line of the schema SQL file
|
|
*/
|
|
function _getSchemaSQLVersion(schema){
|
|
// Default to schema.sql
|
|
if (!schema){
|
|
schema = 'schema';
|
|
}
|
|
|
|
var schemaFile = schema + '.sql';
|
|
|
|
if (_schemaVersions[schema]){
|
|
return _schemaVersions[schema];
|
|
}
|
|
|
|
var file = Components.classes["@mozilla.org/extensions/manager;1"]
|
|
.getService(Components.interfaces.nsIExtensionManager)
|
|
.getInstallLocation(SCHOLAR_CONFIG['GUID'])
|
|
.getItemLocation(SCHOLAR_CONFIG['GUID']);
|
|
file.append(schemaFile);
|
|
|
|
// Open an input stream from file
|
|
var istream = Components.classes["@mozilla.org/network/file-input-stream;1"]
|
|
.createInstance(Components.interfaces.nsIFileInputStream);
|
|
istream.init(file, 0x01, 0444, 0);
|
|
istream.QueryInterface(Components.interfaces.nsILineInputStream);
|
|
|
|
var line = {};
|
|
|
|
// Fetch the schema version from the first line of the file
|
|
istream.readLine(line);
|
|
var schemaVersion = line.value.match(/-- ([0-9]+)/)[1];
|
|
istream.close();
|
|
|
|
_schemaVersions[schema] = schemaVersion;
|
|
return schemaVersion;
|
|
}
|
|
|
|
|
|
/*
|
|
* Load in SQL schema
|
|
*
|
|
* Returns an _array_ of SQL statements for feeding into query()
|
|
*/
|
|
function _getSchemaSQL(schema){
|
|
// Default to schema.sql
|
|
if (!schema){
|
|
schema = 'schema';
|
|
}
|
|
|
|
var schemaFile = schema + '.sql';
|
|
|
|
// We pull the schema from an external file so we only have to process
|
|
// it when necessary
|
|
var file = Components.classes["@mozilla.org/extensions/manager;1"]
|
|
.getService(Components.interfaces.nsIExtensionManager)
|
|
.getInstallLocation(SCHOLAR_CONFIG['GUID'])
|
|
.getItemLocation(SCHOLAR_CONFIG['GUID']);
|
|
file.append(schemaFile);
|
|
|
|
// Open an input stream from file
|
|
var istream = Components.classes["@mozilla.org/network/file-input-stream;1"]
|
|
.createInstance(Components.interfaces.nsIFileInputStream);
|
|
istream.init(file, 0x01, 0444, 0);
|
|
istream.QueryInterface(Components.interfaces.nsILineInputStream);
|
|
|
|
var line = {}, sql = '', hasmore;
|
|
|
|
// Skip the first line, which contains the schema version
|
|
istream.readLine(line);
|
|
//var schemaVersion = line.value.match(/-- ([0-9]+)/)[1];
|
|
|
|
do {
|
|
hasmore = istream.readLine(line);
|
|
sql += line.value + "\n";
|
|
} while(hasmore);
|
|
|
|
istream.close();
|
|
|
|
return sql;
|
|
}
|
|
|
|
|
|
/*
|
|
* Create new DB schema
|
|
*/
|
|
function _initializeSchema(){
|
|
try {
|
|
Scholar.DB.beginTransaction();
|
|
Scholar.DB.query(_getSchemaSQL());
|
|
_updateDBVersion('schema', _getSchemaSQLVersion());
|
|
Scholar.DB.query(_getSchemaSQL('scrapers'));
|
|
_updateDBVersion('scrapers', _getSchemaSQLVersion('scrapers'));
|
|
Scholar.DB.commitTransaction();
|
|
}
|
|
catch(e){
|
|
Scholar.debug(e, 1);
|
|
alert('Error initializing Scholar database'); // TODO: localize
|
|
Scholar.DB.rollbackTransaction();
|
|
}
|
|
}
|
|
|
|
|
|
/*
|
|
* Update a DB schema version tag in an existing database
|
|
*/
|
|
function _updateDBVersion(schema, version){
|
|
_dbVersions[schema] = version;
|
|
var sql = "REPLACE INTO version (schema,version) VALUES (?,?)";
|
|
return Scholar.DB.query(sql, [{'string':schema},{'int':version}]);
|
|
}
|
|
|
|
|
|
/*
|
|
* Update the scrapers in the DB to the latest bundled versions
|
|
*/
|
|
function _updateScrapersLocal(){
|
|
var dbVersion = _getDBVersion('scrapers');
|
|
var schemaVersion = _getSchemaSQLVersion('scrapers');
|
|
|
|
if (dbVersion == schemaVersion){
|
|
return;
|
|
}
|
|
else if (dbVersion < schemaVersion){
|
|
Scholar.DB.beginTransaction();
|
|
Scholar.DB.query(_getSchemaSQL('scrapers'));
|
|
_updateDBVersion('scrapers', schemaVersion);
|
|
Scholar.DB.commitTransaction();
|
|
return;
|
|
}
|
|
else {
|
|
throw("Scraper set in DB is newer than schema version");
|
|
}
|
|
}
|
|
|
|
|
|
/**
|
|
* Process the response from the repository
|
|
**/
|
|
function _updateScrapersRemoteCallback(xmlhttp){
|
|
if (!xmlhttp.responseXML){
|
|
if (!xmlhttp.noNetwork){
|
|
Scholar.debug('Invalid response from repository', 2);
|
|
}
|
|
_setRepositoryTimer(SCHOLAR_CONFIG['REPOSITORY_RETRY_INTERVAL']);
|
|
return false;
|
|
}
|
|
|
|
var currentTime = xmlhttp.responseXML.
|
|
getElementsByTagName('currentTime')[0].firstChild.nodeValue;
|
|
var updates = xmlhttp.responseXML.getElementsByTagName('translator');
|
|
|
|
Scholar.DB.beginTransaction();
|
|
|
|
// Store the timestamp provided by the server
|
|
_updateDBVersion('repository', currentTime);
|
|
|
|
// And the local timestamp of the update time
|
|
var d = new Date();
|
|
_updateDBVersion('lastcheck', Math.round(d.getTime()/1000)); // JS uses ms
|
|
|
|
if (!updates.length){
|
|
Scholar.debug('All scrapers are up-to-date');
|
|
Scholar.DB.commitTransaction();
|
|
_setRepositoryTimer(SCHOLAR_CONFIG['REPOSITORY_CHECK_INTERVAL']);
|
|
return false;
|
|
}
|
|
|
|
for (var i=0, len=updates.length; i<len; i++){
|
|
try {
|
|
_scraperXMLToDBQuery(updates[i]);
|
|
}
|
|
catch (e) {
|
|
Scholar.debug(e, 1);
|
|
Scholar.DB.rollbackTransaction();
|
|
var breakout = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (!breakout){
|
|
Scholar.DB.commitTransaction();
|
|
_setRepositoryTimer(SCHOLAR_CONFIG['REPOSITORY_CHECK_INTERVAL']);
|
|
}
|
|
}
|
|
|
|
|
|
/**
|
|
* Set the interval between repository queries
|
|
*
|
|
* We add an additional two seconds to avoid race conditions
|
|
**/
|
|
function _setRepositoryTimer(interval){
|
|
if (!interval){
|
|
interval = SCHOLAR_CONFIG['REPOSITORY_CHECK_INTERVAL'];
|
|
}
|
|
|
|
var fudge = 2; // two seconds
|
|
var displayInterval = interval + fudge;
|
|
var interval = (interval + fudge) * 1000; // convert to ms
|
|
|
|
if (!_repositoryTimer || _repositoryTimer.delay!=interval){
|
|
Scholar.debug('Setting repository check interval to ' + displayInterval + ' seconds');
|
|
_repositoryTimer = Components.classes["@mozilla.org/timer;1"].
|
|
createInstance(Components.interfaces.nsITimer);
|
|
_repositoryTimer.initWithCallback({
|
|
// implements nsITimerCallback
|
|
notify: function(timer){
|
|
Scholar.Schema.updateScrapersRemote();
|
|
}
|
|
}, interval, Components.interfaces.nsITimer.TYPE_REPEATING_SLACK);
|
|
}
|
|
}
|
|
|
|
|
|
/**
|
|
* Traverse an XML scraper node from the repository and
|
|
* update the local scrapers table with the scraper data
|
|
**/
|
|
function _scraperXMLToDBQuery(xmlnode){
|
|
var sqlValues = [
|
|
{'string':xmlnode.getAttribute('id')},
|
|
{'string':xmlnode.getAttribute('lastUpdated')},
|
|
{'string':xmlnode.getAttribute('type')},
|
|
{'string':xmlnode.getElementsByTagName('label')[0].firstChild.nodeValue},
|
|
{'string':xmlnode.getElementsByTagName('creator')[0].firstChild.nodeValue},
|
|
// target
|
|
(xmlnode.getElementsByTagName('target').item(0) &&
|
|
xmlnode.getElementsByTagName('target')[0].firstChild)
|
|
? {'string':xmlnode.getElementsByTagName('target')[0].firstChild.nodeValue}
|
|
: {'null':true},
|
|
// detectCode can not exist or be empty
|
|
(xmlnode.getElementsByTagName('detectCode').item(0) &&
|
|
xmlnode.getElementsByTagName('detectCode')[0].firstChild)
|
|
? {'string':xmlnode.getElementsByTagName('detectCode')[0].firstChild.nodeValue}
|
|
: {'null':true},
|
|
{'string':xmlnode.getElementsByTagName('code')[0].firstChild.nodeValue}
|
|
]
|
|
|
|
var sql = "REPLACE INTO translators VALUES (?,?,?,?,?,?,?,?)";
|
|
return Scholar.DB.query(sql, sqlValues);
|
|
}
|
|
|
|
|
|
/*
|
|
* Migrate schema from an older version, preserving data
|
|
*/
|
|
function _migrateSchema(fromVersion){
|
|
//
|
|
// Change this value to match the schema version
|
|
//
|
|
var toVersion = 39;
|
|
|
|
if (toVersion != _getSchemaSQLVersion()){
|
|
throw('Schema version does not match version in _migrateSchema()');
|
|
}
|
|
|
|
Scholar.debug('Updating DB from version ' + fromVersion + ' to ' + toVersion + '\n');
|
|
|
|
Scholar.DB.beginTransaction();
|
|
|
|
// Step through version changes until we reach the current version
|
|
//
|
|
// Each block performs the changes necessary to move from the
|
|
// previous revision to that one.
|
|
for (var i=fromVersion + 1; i<=toVersion; i++){
|
|
if (i==30){
|
|
// Remove old SQLite DB
|
|
var file = Scholar.getProfileDirectory();
|
|
file.append('scholar.sqlite');
|
|
if (file.exists()){
|
|
file.remove(null);
|
|
}
|
|
}
|
|
|
|
if (i==39){
|
|
// Clear storage directory
|
|
var file = Scholar.getStorageDirectory();
|
|
if (file.exists()){
|
|
file.remove(true);
|
|
}
|
|
_initializeSchema();
|
|
}
|
|
}
|
|
|
|
_updateDBVersion('schema', i-1);
|
|
Scholar.DB.commitTransaction();
|
|
}
|
|
}
|