Fix several possible DB upgrade errors from 4.0 to 5.0

- We were updating global schema before migrating userdata, but a 4 → 5
  upgrade involved a system.sql version bump, which wiped out itemTypes,
  causing 'annotation' to not exist after the upgrade. This moves global
  schema updates after userdata migration and bumps the global schema
  version to repair DBs that were already upgraded and broken.

- A system.sql bump without a global schema update would result in empty
  tables. This moves the global-schema-related tables to userdata.sql.

- The DB integrity check before userdata updates added in 5b9e6497a
  could fail when coming from an older DB, because the checks assume
  current schema. An integrity check is now done after a userdata update.
  (We were already skipping the new table/index reconciliation stuff. If
  old DBs are discovered to have problems that would cause a migration
  step to fail, we'll fix those explicitly in the steps.)

Also:

- Make sure `version` is `versionNumber` in the `fields` table. It was
  changed with a system.sql bump in 5.0, but hard-coded fields were later
  removed from system.sql in favor of schema.json, meaning that anyone who
  upgraded from 4.0 after that would never have `version` removed and so
  would have both fields (one from before and one from schema.json).
This commit is contained in:
Dan Stillman 2021-08-14 05:04:16 -04:00
parent 4021492797
commit 0f96c20f3c
5 changed files with 195 additions and 165 deletions

View file

@ -424,6 +424,9 @@ Zotero.DBConnection.prototype.getNextName = Zotero.Promise.coroutine(function* (
/**
* @param {Function} func - Generator function that yields promises,
* generally from queryAsync() and similar
* @param {Object} [options]
* @param {Boolean} [options.disableForeignKeys] - Disable foreign key constraints before
* transaction and re-enable after. (`PRAGMA foreign_keys=0|1` is a no-op during a transaction.)
* @return {Promise} - Promise for result of generator function
*/
Zotero.DBConnection.prototype.executeTransaction = Zotero.Promise.coroutine(function* (func, options) {
@ -464,6 +467,11 @@ Zotero.DBConnection.prototype.executeTransaction = Zotero.Promise.coroutine(func
this._callbacks.begin[i](id);
}
}
if (options.disableForeignKeys) {
yield this.queryAsync("PRAGMA foreign_keys = 0");
}
var conn = this._getConnection(options) || (yield this._getConnectionAsync(options));
var result = yield conn.executeTransaction(func);
Zotero.debug(`Committed DB transaction ${id}`, 4);
@ -538,6 +546,10 @@ Zotero.DBConnection.prototype.executeTransaction = Zotero.Promise.coroutine(func
throw e;
}
finally {
if (options.disableForeignKeys) {
yield this.queryAsync("PRAGMA foreign_keys = 1");
}
// Reset options back to their previous values
if (options) {
for (let option in options) {

View file

@ -139,7 +139,7 @@ Zotero.Schema = new function(){
}
// Check if DB is coming from the DB Repair Tool and should be checked
var integrityCheck = await this.integrityCheckRequired();
var integrityCheckRequired = await this.integrityCheckRequired();
// Check whether bundled global schema file is newer than DB
var bundledGlobalSchema = await _readGlobalSchemaFromFile();
@ -156,7 +156,7 @@ Zotero.Schema = new function(){
await Zotero.DB.backupDatabase(userdata, true);
}
// Automatic backup
else if (integrityCheck || bundledGlobalSchemaVersionCompare === 1) {
else if (integrityCheckRequired || bundledGlobalSchemaVersionCompare === 1) {
await Zotero.DB.backupDatabase(false, true);
}
@ -169,6 +169,42 @@ Zotero.Schema = new function(){
var updated;
await Zotero.DB.queryAsync("PRAGMA foreign_keys = false");
try {
updated = await Zotero.DB.executeTransaction(async function (conn) {
var updated = await _updateSchema('system');
// Update custom tables if they exist so that changes are in
// place before user data migration
if (Zotero.DB.tableExists('customItemTypes')) {
await _updateCustomTables();
}
// Auto-repair databases flagged for repair or coming from the DB Repair Tool
//
// If we need to run migration steps, skip the check until after the update, since
// the integrity check is expecting to run on the current data model.
var integrityCheckDone = false;
var toVersion = await _getSchemaSQLVersion('userdata');
if (integrityCheckRequired && userdata >= toVersion) {
await this.integrityCheck(true);
integrityCheckDone = true;
}
updated = await _migrateUserDataSchema(userdata, options);
await _updateSchema('triggers');
// Populate combined tables for custom types and fields -- this is likely temporary
//
// We do this again in case custom fields were changed during user data migration
await _updateCustomTables();
// If we updated the DB, also do an integrity check for good measure
if (updated && !integrityCheckDone) {
await this.integrityCheck(true);
}
return updated;
}.bind(this));
// If bundled global schema file is newer than DB, apply it
if (bundledGlobalSchemaVersionCompare === 1) {
await Zotero.DB.executeTransaction(async function () {
@ -188,35 +224,6 @@ Zotero.Schema = new function(){
}
await _loadGlobalSchema(data, bundledGlobalSchema.version);
}
updated = await Zotero.DB.executeTransaction(async function (conn) {
var updated = await _updateSchema('system');
// Update custom tables if they exist so that changes are in
// place before user data migration
if (Zotero.DB.tableExists('customItemTypes')) {
await _updateCustomTables();
}
// Auto-repair databases flagged for repair or coming from the DB Repair Tool
if (integrityCheck) {
// If we need to run migration steps, don't reconcile tables, since it might
// create tables that aren't expected to exist yet
let toVersion = await _getSchemaSQLVersion('userdata');
await this.integrityCheck(true, { skipReconcile: userdata < toVersion });
options.skipIntegrityCheck = true;
}
updated = await _migrateUserDataSchema(userdata, options);
await _updateSchema('triggers');
// Populate combined tables for custom types and fields -- this is likely temporary
//
// We do this again in case custom fields were changed during user data migration
await _updateCustomTables();
return updated;
}.bind(this));
}
finally {
await Zotero.DB.queryAsync("PRAGMA foreign_keys = true");
@ -410,7 +417,7 @@ Zotero.Schema = new function(){
/**
* Update the item-type/field/creator mapping tables based on the passed schema
*/
async function _updateGlobalSchema(data) {
async function _updateGlobalSchema(data, options) {
Zotero.debug("Updating global schema to version " + data.version);
Zotero.DB.requireTransaction();
@ -571,7 +578,7 @@ Zotero.Schema = new function(){
var bundledVersion = (await _readGlobalSchemaFromFile()).version;
await _loadGlobalSchema(data, bundledVersion);
await _reloadSchema();
await _reloadSchema(options);
// Mark that we need to migrate Extra values to any newly available fields in
// Zotero.Schema.migrateExtraFields()
await Zotero.DB.queryAsync(
@ -585,7 +592,7 @@ Zotero.Schema = new function(){
this._updateGlobalSchemaForTest = async function (schema) {
await Zotero.DB.executeTransaction(async function () {
await _updateGlobalSchema(schema);
}.bind(this));
}.bind(this), { disableForeignKeys: true });
};
@ -768,7 +775,7 @@ Zotero.Schema = new function(){
}
yield _reloadSchema();
});
}, { disableForeignKeys: true });
var s = new Zotero.Search;
s.name = "Overdue NSF Reviewers";
@ -817,39 +824,44 @@ Zotero.Schema = new function(){
}
yield _reloadSchema();
}.bind(this));
}.bind(this), { disableForeignKeys: true });
ps.alert(null, "Zotero Item Type Removed", "The 'NSF Reviewer' item type has been uninstalled.");
}
});
var _reloadSchema = Zotero.Promise.coroutine(function* () {
yield _updateCustomTables();
yield Zotero.ItemTypes.init();
yield Zotero.ItemFields.init();
yield Zotero.CreatorTypes.init();
yield Zotero.SearchConditions.init();
async function _reloadSchema(options) {
await _updateCustomTables(options);
await Zotero.ItemTypes.init();
await Zotero.ItemFields.init();
await Zotero.CreatorTypes.init();
await Zotero.SearchConditions.init();
// Update item type menus in every open window
Zotero.Schema.schemaUpdatePromise.then(function () {
var wm = Services.wm;
var enumerator = wm.getEnumerator("navigator:browser");
var enumerator = Services.wm.getEnumerator("navigator:browser");
while (enumerator.hasMoreElements()) {
let win = enumerator.getNext();
win.ZoteroPane.buildItemTypeSubMenu();
win.document.getElementById('zotero-editpane-item-box').buildItemTypeMenu();
}
});
});
}
var _updateCustomTables = async function () {
var _updateCustomTables = async function (options) {
Zotero.debug("Updating custom tables");
Zotero.DB.requireTransaction();
if (!options?.foreignKeyChecksAllowed) {
if (await Zotero.DB.valueQueryAsync("PRAGMA foreign_keys")) {
throw new Error("Foreign key checks must be disabled before updating custom tables");
}
}
await Zotero.DB.queryAsync("DELETE FROM itemTypesCombined");
await Zotero.DB.queryAsync("DELETE FROM fieldsCombined WHERE fieldID NOT IN (SELECT fieldID FROM itemData)");
await Zotero.DB.queryAsync("DELETE FROM fieldsCombined");
await Zotero.DB.queryAsync("DELETE FROM itemTypeFieldsCombined");
await Zotero.DB.queryAsync("DELETE FROM baseFieldMappingsCombined");
@ -860,7 +872,7 @@ Zotero.Schema = new function(){
+ "SELECT customItemTypeID + " + offset + " AS itemTypeID, typeName, display, 1 AS custom FROM customItemTypes"
);
await Zotero.DB.queryAsync(
"INSERT OR IGNORE INTO fieldsCombined "
"INSERT INTO fieldsCombined "
+ "SELECT fieldID, fieldName, NULL AS label, fieldFormatID, 0 AS custom FROM fields UNION "
+ "SELECT customFieldID + " + offset + " AS fieldID, fieldName, label, NULL, 1 AS custom FROM customFields"
);
@ -1776,7 +1788,7 @@ Zotero.Schema = new function(){
* deleted
*/
this.integrityCheck = Zotero.Promise.coroutine(function* (fix, options = {}) {
Zotero.debug("Checking database integrity");
Zotero.debug("Checking database schema integrity");
// Just as a sanity check, make sure combined field tables are populated,
// so that we don't try to wipe out all data
@ -2205,7 +2217,7 @@ Zotero.Schema = new function(){
});
var schema = yield _readGlobalSchemaFromFile();
yield _updateGlobalSchema(schema);
yield _updateGlobalSchema(schema, { foreignKeyChecksAllowed: true });
yield _getSchemaSQLVersion('system').then(function (version) {
return _updateDBVersion('system', version);
@ -2567,11 +2579,6 @@ Zotero.Schema = new function(){
return false;
}
if (!options.skipIntegrityCheck) {
// Check integrity, but don't create missing tables
yield Zotero.Schema.integrityCheck(true, { skipReconcile: true });
}
Zotero.debug('Updating user data tables from version ' + fromVersion + ' to ' + toVersion);
if (options.onBeforeUpdate) {
@ -3282,6 +3289,20 @@ Zotero.Schema = new function(){
yield Zotero.DB.queryAsync("UPDATE itemAnnotations SET color='#000000' WHERE color='#000'");
}
else if (i == 117) {
let versionFieldID = yield Zotero.DB.valueQueryAsync("SELECT fieldID FROM fields WHERE fieldName='version'");
if (versionFieldID) {
let versionNumberFieldID = yield Zotero.DB.valueQueryAsync("SELECT fieldID FROM fields WHERE fieldName='versionNumber'");
if (versionNumberFieldID) {
yield Zotero.DB.queryAsync("UPDATE itemData SET fieldID=? WHERE fieldID=?", [versionNumberFieldID, versionFieldID]);
yield Zotero.DB.queryAsync("DELETE FROM fields WHERE fieldID=?", versionFieldID);
}
else {
yield Zotero.DB.queryAsync("UPDATE fields SET fieldName=? WHERE fieldName=?", ['versionNumber', 'version']);
}
}
}
// If breaking compatibility or doing anything dangerous, clear minorUpdateFrom
}

@ -1 +1 @@
Subproject commit b5b3f51217a99b3c41585d468ee1dc0837233d13
Subproject commit 97e0a8efa2cb2cf6c9853ceca334ec56180a9df0

View file

@ -23,25 +23,6 @@
-- This file creates system tables that can be safely wiped and reinitialized
-- at any time, as long as existing ids are preserved.
-- Valid item types ("book," "journalArticle," etc.)
DROP TABLE IF EXISTS itemTypes;
CREATE TABLE itemTypes (
itemTypeID INTEGER PRIMARY KEY,
typeName TEXT,
templateItemTypeID INT,
display INT DEFAULT 1 -- 0 == hide, 1 == display, 2 == primary
);
-- Populated at startup from itemTypes and customItemTypes
DROP TABLE IF EXISTS itemTypesCombined;
CREATE TABLE itemTypesCombined (
itemTypeID INT NOT NULL,
typeName TEXT NOT NULL,
display INT DEFAULT 1 NOT NULL,
custom INT NOT NULL,
PRIMARY KEY (itemTypeID)
);
-- Describes various types of fields and their format restrictions,
-- and indicates whether data should be stored as strings or integers
--
@ -53,77 +34,6 @@ CREATE TABLE fieldFormats (
isInteger INT
);
-- Field types for item metadata
DROP TABLE IF EXISTS fields;
CREATE TABLE fields (
fieldID INTEGER PRIMARY KEY,
fieldName TEXT,
fieldFormatID INT,
FOREIGN KEY (fieldFormatID) REFERENCES fieldFormats(fieldFormatID)
);
-- Populated at startup from fields and customFields
DROP TABLE IF EXISTS fieldsCombined;
CREATE TABLE fieldsCombined (
fieldID INT NOT NULL,
fieldName TEXT NOT NULL,
label TEXT,
fieldFormatID INT,
custom INT NOT NULL,
PRIMARY KEY (fieldID)
);
-- Defines valid fields for each itemType, their display order, and their default visibility
DROP TABLE IF EXISTS itemTypeFields;
CREATE TABLE itemTypeFields (
itemTypeID INT,
fieldID INT,
hide INT,
orderIndex INT,
PRIMARY KEY (itemTypeID, orderIndex),
UNIQUE (itemTypeID, fieldID),
FOREIGN KEY (itemTypeID) REFERENCES itemTypes(itemTypeID),
FOREIGN KEY (fieldID) REFERENCES fields(fieldID)
);
CREATE INDEX itemTypeFields_fieldID ON itemTypeFields(fieldID);
-- Populated at startup from itemTypeFields and customItemTypeFields
DROP TABLE IF EXISTS itemTypeFieldsCombined;
CREATE TABLE itemTypeFieldsCombined (
itemTypeID INT NOT NULL,
fieldID INT NOT NULL,
hide INT,
orderIndex INT NOT NULL,
PRIMARY KEY (itemTypeID, orderIndex),
UNIQUE (itemTypeID, fieldID)
);
CREATE INDEX itemTypeFieldsCombined_fieldID ON itemTypeFieldsCombined(fieldID);
-- Maps base fields to type-specific fields (e.g. publisher to label in audioRecording)
DROP TABLE IF EXISTS baseFieldMappings;
CREATE TABLE baseFieldMappings (
itemTypeID INT,
baseFieldID INT,
fieldID INT,
PRIMARY KEY (itemTypeID, baseFieldID, fieldID),
FOREIGN KEY (itemTypeID) REFERENCES itemTypes(itemTypeID),
FOREIGN KEY (baseFieldID) REFERENCES fields(fieldID),
FOREIGN KEY (fieldID) REFERENCES fields(fieldID)
);
CREATE INDEX baseFieldMappings_baseFieldID ON baseFieldMappings(baseFieldID);
CREATE INDEX baseFieldMappings_fieldID ON baseFieldMappings(fieldID);
-- Populated at startup from baseFieldMappings and customBaseFieldMappings
DROP TABLE IF EXISTS baseFieldMappingsCombined;
CREATE TABLE baseFieldMappingsCombined (
itemTypeID INT,
baseFieldID INT,
fieldID INT,
PRIMARY KEY (itemTypeID, baseFieldID, fieldID)
);
CREATE INDEX baseFieldMappingsCombined_baseFieldID ON baseFieldMappingsCombined(baseFieldID);
CREATE INDEX baseFieldMappingsCombined_fieldID ON baseFieldMappingsCombined(fieldID);
DROP TABLE IF EXISTS charsets;
CREATE TABLE charsets (
charsetID INTEGER PRIMARY KEY,
@ -147,24 +57,6 @@ CREATE TABLE fileTypeMimeTypes (
);
CREATE INDEX fileTypeMimeTypes_mimeType ON fileTypeMimeTypes(mimeType);
-- Defines the possible creator types (contributor, editor, author)
DROP TABLE IF EXISTS creatorTypes;
CREATE TABLE creatorTypes (
creatorTypeID INTEGER PRIMARY KEY,
creatorType TEXT
);
DROP TABLE IF EXISTS itemTypeCreatorTypes;
CREATE TABLE itemTypeCreatorTypes (
itemTypeID INT,
creatorTypeID INT,
primaryField INT,
PRIMARY KEY (itemTypeID, creatorTypeID),
FOREIGN KEY (itemTypeID) REFERENCES itemTypes(itemTypeID),
FOREIGN KEY (creatorTypeID) REFERENCES creatorTypes(creatorTypeID)
);
CREATE INDEX itemTypeCreatorTypes_creatorTypeID ON itemTypeCreatorTypes(creatorTypeID);
DROP TABLE IF EXISTS syncObjectTypes;
CREATE TABLE syncObjectTypes (
syncObjectTypeID INTEGER PRIMARY KEY,

View file

@ -1,4 +1,4 @@
-- 116
-- 117
-- Copyright (c) 2009 Center for History and New Media
-- George Mason University, Fairfax, Virginia, USA
@ -23,6 +23,111 @@
-- This file creates tables containing user-specific data for new users --
-- any changes made here must be mirrored in transition steps in schema.js::_migrateSchema()
--
-- Tables populated by global schema
--
-- Valid item types ("book," "journalArticle," etc.)
CREATE TABLE itemTypes (
itemTypeID INTEGER PRIMARY KEY,
typeName TEXT,
templateItemTypeID INT,
display INT DEFAULT 1 -- 0 == hide, 1 == display, 2 == primary
);
-- Populated at startup from itemTypes and customItemTypes
CREATE TABLE itemTypesCombined (
itemTypeID INT NOT NULL,
typeName TEXT NOT NULL,
display INT DEFAULT 1 NOT NULL,
custom INT NOT NULL,
PRIMARY KEY (itemTypeID)
);
-- Field types for item metadata
CREATE TABLE fields (
fieldID INTEGER PRIMARY KEY,
fieldName TEXT,
fieldFormatID INT,
FOREIGN KEY (fieldFormatID) REFERENCES fieldFormats(fieldFormatID)
);
-- Populated at startup from fields and customFields
CREATE TABLE fieldsCombined (
fieldID INT NOT NULL,
fieldName TEXT NOT NULL,
label TEXT,
fieldFormatID INT,
custom INT NOT NULL,
PRIMARY KEY (fieldID)
);
-- Defines valid fields for each itemType, their display order, and their default visibility
CREATE TABLE itemTypeFields (
itemTypeID INT,
fieldID INT,
hide INT,
orderIndex INT,
PRIMARY KEY (itemTypeID, orderIndex),
UNIQUE (itemTypeID, fieldID),
FOREIGN KEY (itemTypeID) REFERENCES itemTypes(itemTypeID),
FOREIGN KEY (fieldID) REFERENCES fields(fieldID)
);
CREATE INDEX itemTypeFields_fieldID ON itemTypeFields(fieldID);
-- Populated at startup from itemTypeFields and customItemTypeFields
CREATE TABLE itemTypeFieldsCombined (
itemTypeID INT NOT NULL,
fieldID INT NOT NULL,
hide INT,
orderIndex INT NOT NULL,
PRIMARY KEY (itemTypeID, orderIndex),
UNIQUE (itemTypeID, fieldID)
);
CREATE INDEX itemTypeFieldsCombined_fieldID ON itemTypeFieldsCombined(fieldID);
-- Maps base fields to type-specific fields (e.g. publisher to label in audioRecording)
CREATE TABLE baseFieldMappings (
itemTypeID INT,
baseFieldID INT,
fieldID INT,
PRIMARY KEY (itemTypeID, baseFieldID, fieldID),
FOREIGN KEY (itemTypeID) REFERENCES itemTypes(itemTypeID),
FOREIGN KEY (baseFieldID) REFERENCES fields(fieldID),
FOREIGN KEY (fieldID) REFERENCES fields(fieldID)
);
CREATE INDEX baseFieldMappings_baseFieldID ON baseFieldMappings(baseFieldID);
CREATE INDEX baseFieldMappings_fieldID ON baseFieldMappings(fieldID);
-- Populated at startup from baseFieldMappings and customBaseFieldMappings
CREATE TABLE baseFieldMappingsCombined (
itemTypeID INT,
baseFieldID INT,
fieldID INT,
PRIMARY KEY (itemTypeID, baseFieldID, fieldID)
);
CREATE INDEX baseFieldMappingsCombined_baseFieldID ON baseFieldMappingsCombined(baseFieldID);
CREATE INDEX baseFieldMappingsCombined_fieldID ON baseFieldMappingsCombined(fieldID);
-- Defines the possible creator types (contributor, editor, author)
CREATE TABLE creatorTypes (
creatorTypeID INTEGER PRIMARY KEY,
creatorType TEXT
);
CREATE TABLE itemTypeCreatorTypes (
itemTypeID INT,
creatorTypeID INT,
primaryField INT,
PRIMARY KEY (itemTypeID, creatorTypeID),
FOREIGN KEY (itemTypeID) REFERENCES itemTypes(itemTypeID),
FOREIGN KEY (creatorTypeID) REFERENCES creatorTypes(creatorTypeID)
);
CREATE INDEX itemTypeCreatorTypes_creatorTypeID ON itemTypeCreatorTypes(creatorTypeID);
--
-- End of tables populated by global schema
--
CREATE TABLE version (
schema TEXT PRIMARY KEY,