Add -g flag to runtests.sh to generate test data

Add functions to generate sample data for various formats
* Zotero Web API JSON (Zotero.Item::toJSON)
* CiteProc-JS JSON
* Export translator JSON
* Direct serialization of Zotero.Item fields
Add a way to load sample data into DB from JSON
Add tests for loading sample data into DB
Add tests for automatically generated data
This will help us make sure that field mappings and data formats don't change
This commit is contained in:
Aurimas Vinckevicius 2015-03-23 23:52:36 -05:00
parent 9d5d8b525a
commit 2ebce91ecf
5 changed files with 593 additions and 24 deletions

View file

@ -1,3 +1,8 @@
// Useful "constants"
var sqlDateTimeRe = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/;
var isoDateTimeRe = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$/;
var zoteroObjectKeyRe = /^[23456789ABCDEFGHIJKMNPQRSTUVWXZ]{8}$/; // based on Zotero.Utilities::generateObjectKey()
/**
* Waits for a DOM event on the specified node. Returns a promise
* resolved with the event.
@ -144,8 +149,8 @@ function installPDFTools() {
}
/**
* Returns a promise for the nsIFile corresponding to the test data
* directory (i.e., test/tests/data)
* Returns the nsIFile corresponding to the test data directory
* (i.e., test/tests/data)
*/
function getTestDataDirectory() {
Components.utils.import("resource://gre/modules/Services.jsm");
@ -167,4 +172,310 @@ function resetDB() {
}).then(function() {
return Zotero.Schema.schemaUpdatePromise;
});
}
/**
* Equivalent to JSON.stringify, except that object properties are stringified
* in a sorted order.
*/
function stableStringify(obj, level, label) {
if (!level) level = 0;
let indent = '\t'.repeat(level);
if (label) label = JSON.stringify('' + label) + ': ';
else label = '';
if (typeof obj == 'function' || obj === undefined) return null;
if (typeof obj != 'object' || obj === null) return indent + label + JSON.stringify(obj);
if (Array.isArray(obj)) {
let str = indent + label + '[';
for (let i=0; i<obj.length; i++) {
let json = stableStringify(obj[i], level + 1);
if (json === null) json = indent + '\tnull'; // function
str += '\n' + json + (i < obj.length-1 ? ',' : '');
}
return str + (obj.length ? '\n' + indent : '') + ']';
}
let keys = Object.keys(obj).sort(),
empty = true,
str = indent + label + '{';
for (let i=0; i<keys.length; i++) {
let json = stableStringify(obj[keys[i]], level + 1, keys[i]);
if (json === null) continue; // function
empty = false;
str += '\n' + json + (i < keys.length-1 ? ',' : '');
}
return str + (!empty ? '\n' + indent : '') + '}';
}
/**
* Loads specified sample data from file
*/
function loadSampleData(dataName) {
let data = Zotero.File.getContentsFromURL('resource://zotero-unit-tests/data/' + dataName + '.js');
return JSON.parse(data);
}
/**
* Generates sample item data that is stored in data/sampleItemData.js
*/
function generateAllTypesAndFieldsData() {
let data = {};
let itemTypes = Zotero.ItemTypes.getTypes();
// For most fields, use the field name as the value, but this doesn't
// work well for some fields that expect values in certain formats
let specialValues = {
date: '1999-12-31',
filingDate: '2000-01-02',
accessDate: '1997-06-13 23:59:58',
number: 3,
numPages: 4,
issue: 5,
volume: 6,
numberOfVolumes: 7,
edition: 8,
seriesNumber: 9,
ISBN: '978-1-234-56789-7',
ISSN: '1234-5679',
url: 'http://www.example.com',
pages: '1-10',
DOI: '10.1234/example.doi',
runningTime: '1:22:33',
language: 'en-US'
};
// Item types that should not be included in sample data
let excludeItemTypes = ['note', 'attachment'];
for (let i = 0; i < itemTypes.length; i++) {
if (excludeItemTypes.indexOf(itemTypes[i].name) != -1) continue;
let itemFields = data[itemTypes[i].name] = {
itemType: itemTypes[i].name
};
let fields = Zotero.ItemFields.getItemTypeFields(itemTypes[i].id);
for (let j = 0; j < fields.length; j++) {
let field = fields[j];
field = Zotero.ItemFields.getBaseIDFromTypeAndField(itemTypes[i].id, field) || field;
let name = Zotero.ItemFields.getName(field),
value;
// Use field name as field value
if (specialValues[name]) {
value = specialValues[name];
} else {
value = name.charAt(0).toUpperCase() + name.substr(1);
// Make it look nice (sentence case)
value = value.replace(/([a-z])([A-Z])/g, '$1 $2')
.replace(/ [A-Z](?![A-Z])/g, m => m.toLowerCase()); // not all-caps words
}
itemFields[name] = value;
}
let creatorTypes = Zotero.CreatorTypes.getTypesForItemType(itemTypes[i].id),
creators = itemFields.creators = [];
for (let j = 0; j < creatorTypes.length; j++) {
let typeName = creatorTypes[j].name;
creators.push({
creatorType: typeName,
firstName: typeName + 'First',
lastName: typeName + 'Last'
});
}
}
return data;
}
/**
* Populates the database with sample items
* The field values should be in the form exactly as they would appear in Zotero
*/
function populateDBWithSampleData(data) {
Zotero.DB.beginTransaction();
for (let itemName in data) {
let item = data[itemName];
let zItem = new Zotero.Item(item.itemType);
for (let itemField in item) {
if (itemField == 'itemType') continue;
if (itemField == 'creators') {
let creators = item[itemField];
for (let i=0; i<creators.length; i++) {
let creator = new Zotero.Creator();
creator.firstName = creators[i].firstName;
creator.lastName = creators[i].lastName;
creator = Zotero.Creators.get(creator.save());
zItem.setCreator(i, creator, creators[i].creatorType);
}
continue;
}
if (itemField == 'tags') {
// Must save item first
continue;
}
zItem.setField(itemField, item[itemField]);
}
item.id = zItem.save();
if (item.tags && item.tags.length) {
zItem = Zotero.Items.get(item.id);
for (let i=0; i<item.tags.length; i++) {
zItem.addTag(item.tags[i].tag, item.tags[i].type);
}
}
}
Zotero.DB.commitTransaction();
return data;
}
function generateItemJSONData(options, currentData) {
let items = populateDBWithSampleData(loadSampleData('allTypesAndFields')),
jsonData = {};
for (let itemName in items) {
let zItem = Zotero.Items.get(items[itemName].id);
jsonData[itemName] = zItem.toJSON(options);
// Adjut accessDate so that it doesn't depend on computer time zone
// Effectively, assume that current time zone is UTC
if (jsonData[itemName].accessDate) {
let date = Zotero.Date.isoToDate(jsonData[itemName].accessDate);
date.setUTCMinutes(date.getUTCMinutes() - date.getTimezoneOffset());
jsonData[itemName].accessDate = Zotero.Date.dateToISO(date);
}
// Don't replace some fields that _always_ change (e.g. item keys)
// as long as it follows expected format
// This makes it easier to generate more meaningful diffs
if (!currentData || !currentData[itemName]) continue;
for (let field in jsonData[itemName]) {
let oldVal = currentData[itemName][field];
if (!oldVal) continue;
let val = jsonData[itemName][field];
switch (field) {
case 'dateAdded':
case 'dateModified':
if (!isoDateTimeRe.test(oldVal) || !isoDateTimeRe.test(val)) continue;
break;
case 'key':
if (!zoteroObjectKeyRe.test(oldVal) || !zoteroObjectKeyRe.test(val)) continue;
break;
default:
continue;
}
jsonData[itemName][field] = oldVal;
}
}
return jsonData;
}
function generateCiteProcJSExportData(currentData) {
let items = populateDBWithSampleData(loadSampleData('allTypesAndFields')),
cslExportData = {};
for (let itemName in items) {
let zItem = Zotero.Items.get(items[itemName].id);
cslExportData[itemName] = Zotero.Cite.System.prototype.retrieveItem(zItem);
// Don't replace id as long as it follows expected format
if (!currentData || !currentData[itemName]) continue;
// For simplicity, be more lenient than for item key
let idRe = /^http:\/\/zotero\.org\/users\/local\/\w{8}\/items\/\w{8}$/;
for (let field in cslExportData[itemName]) {
let oldVal = currentData[itemName][field];
if (!oldVal) continue;
let val = cslExportData[itemName][field];
switch (field) {
case 'id':
if (!idRe.test(oldVal) || !idRe.test(val)) continue;
break;
default:
continue;
}
cslExportData[itemName][field] = oldVal;
}
}
return cslExportData;
}
function generateTranslatorExportData(legacy, currentData) {
let items = populateDBWithSampleData(loadSampleData('allTypesAndFields')),
translatorExportData = {};
let itemGetter = new Zotero.Translate.ItemGetter();
itemGetter.legacy = !!legacy;
for (let itemName in items) {
let zItem = Zotero.Items.get(items[itemName].id);
itemGetter._itemsLeft = [zItem];
translatorExportData[itemName] = itemGetter.nextItem();
// Adjut ISO accessDate so that it doesn't depend on computer time zone
// Effectively, assume that current time zone is UTC
if (!legacy && translatorExportData[itemName].accessDate) {
let date = Zotero.Date.isoToDate(translatorExportData[itemName].accessDate);
date.setUTCMinutes(date.getUTCMinutes() - date.getTimezoneOffset());
translatorExportData[itemName].accessDate = Zotero.Date.dateToISO(date);
}
// Don't replace some fields that _always_ change (e.g. item keys)
if (!currentData || !currentData[itemName]) continue;
// For simplicity, be more lenient than for item key
let uriRe = /^http:\/\/zotero\.org\/users\/local\/\w{8}\/items\/\w{8}$/;
let itemIDRe = /^\d+$/;
for (let field in translatorExportData[itemName]) {
let oldVal = currentData[itemName][field];
if (!oldVal) continue;
let val = translatorExportData[itemName][field];
switch (field) {
case 'uri':
if (!uriRe.test(oldVal) || !uriRe.test(val)) continue;
break;
case 'itemID':
if (!itemIDRe.test(oldVal) || !itemIDRe.test(val)) continue;
break;
case 'key':
if (!zoteroObjectKeyRe.test(oldVal) || !zoteroObjectKeyRe.test(val)) continue;
break;
case 'dateAdded':
case 'dateModified':
if (legacy) {
if (!sqlDateTimeRe.test(oldVal) || !sqlDateTimeRe.test(val)) continue;
} else {
if (!isoDateTimeRe.test(oldVal) || !isoDateTimeRe.test(val)) continue;
}
break;
default:
continue;
}
translatorExportData[itemName][field] = oldVal;
}
}
return translatorExportData;
}