Add -g flag to runtests.sh to generate test data
Add functions to generate sample data for various formats * Zotero Web API JSON (Zotero.Item::toJSON) * CiteProc-JS JSON * Export translator JSON * Direct serialization of Zotero.Item fields Add a way to load sample data into DB from JSON Add tests for loading sample data into DB Add tests for automatically generated data This will help us make sure that field mappings and data formats don't change
This commit is contained in:
Aurimas Vinckevicius
parent
9d5d8b525a
commit
2ebce91ecf
5 changed files with 593 additions and 24 deletions
|
|
@ -1,3 +1,8 @@
|
|||
// Useful "constants"
|
||||
var sqlDateTimeRe = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/;
|
||||
var isoDateTimeRe = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$/;
|
||||
var zoteroObjectKeyRe = /^[23456789ABCDEFGHIJKMNPQRSTUVWXZ]{8}$/; // based on Zotero.Utilities::generateObjectKey()
|
||||
|
||||
/**
|
||||
* Waits for a DOM event on the specified node. Returns a promise
|
||||
* resolved with the event.
|
||||
|
|
@ -144,8 +149,8 @@ function installPDFTools() {
|
|||
}
|
||||
|
||||
/**
|
||||
* Returns a promise for the nsIFile corresponding to the test data
|
||||
* directory (i.e., test/tests/data)
|
||||
* Returns the nsIFile corresponding to the test data directory
|
||||
* (i.e., test/tests/data)
|
||||
*/
|
||||
function getTestDataDirectory() {
|
||||
Components.utils.import("resource://gre/modules/Services.jsm");
|
||||
|
|
@ -167,4 +172,310 @@ function resetDB() {
|
|||
}).then(function() {
|
||||
return Zotero.Schema.schemaUpdatePromise;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Equivalent to JSON.stringify, except that object properties are stringified
|
||||
* in a sorted order.
|
||||
*/
|
||||
function stableStringify(obj, level, label) {
|
||||
if (!level) level = 0;
|
||||
let indent = '\t'.repeat(level);
|
||||
|
||||
if (label) label = JSON.stringify('' + label) + ': ';
|
||||
else label = '';
|
||||
|
||||
if (typeof obj == 'function' || obj === undefined) return null;
|
||||
|
||||
if (typeof obj != 'object' || obj === null) return indent + label + JSON.stringify(obj);
|
||||
|
||||
if (Array.isArray(obj)) {
|
||||
let str = indent + label + '[';
|
||||
for (let i=0; i<obj.length; i++) {
|
||||
let json = stableStringify(obj[i], level + 1);
|
||||
if (json === null) json = indent + '\tnull'; // function
|
||||
str += '\n' + json + (i < obj.length-1 ? ',' : '');
|
||||
}
|
||||
return str + (obj.length ? '\n' + indent : '') + ']';
|
||||
}
|
||||
|
||||
let keys = Object.keys(obj).sort(),
|
||||
empty = true,
|
||||
str = indent + label + '{';
|
||||
for (let i=0; i<keys.length; i++) {
|
||||
let json = stableStringify(obj[keys[i]], level + 1, keys[i]);
|
||||
if (json === null) continue; // function
|
||||
|
||||
empty = false;
|
||||
str += '\n' + json + (i < keys.length-1 ? ',' : '');
|
||||
}
|
||||
|
||||
return str + (!empty ? '\n' + indent : '') + '}';
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads specified sample data from file
|
||||
*/
|
||||
function loadSampleData(dataName) {
|
||||
let data = Zotero.File.getContentsFromURL('resource://zotero-unit-tests/data/' + dataName + '.js');
|
||||
return JSON.parse(data);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates sample item data that is stored in data/sampleItemData.js
|
||||
*/
|
||||
function generateAllTypesAndFieldsData() {
|
||||
let data = {};
|
||||
let itemTypes = Zotero.ItemTypes.getTypes();
|
||||
// For most fields, use the field name as the value, but this doesn't
|
||||
// work well for some fields that expect values in certain formats
|
||||
let specialValues = {
|
||||
date: '1999-12-31',
|
||||
filingDate: '2000-01-02',
|
||||
accessDate: '1997-06-13 23:59:58',
|
||||
number: 3,
|
||||
numPages: 4,
|
||||
issue: 5,
|
||||
volume: 6,
|
||||
numberOfVolumes: 7,
|
||||
edition: 8,
|
||||
seriesNumber: 9,
|
||||
ISBN: '978-1-234-56789-7',
|
||||
ISSN: '1234-5679',
|
||||
url: 'http://www.example.com',
|
||||
pages: '1-10',
|
||||
DOI: '10.1234/example.doi',
|
||||
runningTime: '1:22:33',
|
||||
language: 'en-US'
|
||||
};
|
||||
|
||||
// Item types that should not be included in sample data
|
||||
let excludeItemTypes = ['note', 'attachment'];
|
||||
|
||||
for (let i = 0; i < itemTypes.length; i++) {
|
||||
if (excludeItemTypes.indexOf(itemTypes[i].name) != -1) continue;
|
||||
|
||||
let itemFields = data[itemTypes[i].name] = {
|
||||
itemType: itemTypes[i].name
|
||||
};
|
||||
|
||||
let fields = Zotero.ItemFields.getItemTypeFields(itemTypes[i].id);
|
||||
for (let j = 0; j < fields.length; j++) {
|
||||
let field = fields[j];
|
||||
field = Zotero.ItemFields.getBaseIDFromTypeAndField(itemTypes[i].id, field) || field;
|
||||
|
||||
let name = Zotero.ItemFields.getName(field),
|
||||
value;
|
||||
|
||||
// Use field name as field value
|
||||
if (specialValues[name]) {
|
||||
value = specialValues[name];
|
||||
} else {
|
||||
value = name.charAt(0).toUpperCase() + name.substr(1);
|
||||
// Make it look nice (sentence case)
|
||||
value = value.replace(/([a-z])([A-Z])/g, '$1 $2')
|
||||
.replace(/ [A-Z](?![A-Z])/g, m => m.toLowerCase()); // not all-caps words
|
||||
}
|
||||
|
||||
itemFields[name] = value;
|
||||
}
|
||||
|
||||
let creatorTypes = Zotero.CreatorTypes.getTypesForItemType(itemTypes[i].id),
|
||||
creators = itemFields.creators = [];
|
||||
for (let j = 0; j < creatorTypes.length; j++) {
|
||||
let typeName = creatorTypes[j].name;
|
||||
creators.push({
|
||||
creatorType: typeName,
|
||||
firstName: typeName + 'First',
|
||||
lastName: typeName + 'Last'
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Populates the database with sample items
|
||||
* The field values should be in the form exactly as they would appear in Zotero
|
||||
*/
|
||||
function populateDBWithSampleData(data) {
|
||||
Zotero.DB.beginTransaction();
|
||||
|
||||
for (let itemName in data) {
|
||||
let item = data[itemName];
|
||||
let zItem = new Zotero.Item(item.itemType);
|
||||
for (let itemField in item) {
|
||||
if (itemField == 'itemType') continue;
|
||||
|
||||
if (itemField == 'creators') {
|
||||
let creators = item[itemField];
|
||||
for (let i=0; i<creators.length; i++) {
|
||||
let creator = new Zotero.Creator();
|
||||
creator.firstName = creators[i].firstName;
|
||||
creator.lastName = creators[i].lastName;
|
||||
creator = Zotero.Creators.get(creator.save());
|
||||
|
||||
zItem.setCreator(i, creator, creators[i].creatorType);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (itemField == 'tags') {
|
||||
// Must save item first
|
||||
continue;
|
||||
}
|
||||
|
||||
zItem.setField(itemField, item[itemField]);
|
||||
}
|
||||
item.id = zItem.save();
|
||||
|
||||
if (item.tags && item.tags.length) {
|
||||
zItem = Zotero.Items.get(item.id);
|
||||
for (let i=0; i<item.tags.length; i++) {
|
||||
zItem.addTag(item.tags[i].tag, item.tags[i].type);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Zotero.DB.commitTransaction();
|
||||
return data;
|
||||
}
|
||||
|
||||
function generateItemJSONData(options, currentData) {
|
||||
let items = populateDBWithSampleData(loadSampleData('allTypesAndFields')),
|
||||
jsonData = {};
|
||||
|
||||
for (let itemName in items) {
|
||||
let zItem = Zotero.Items.get(items[itemName].id);
|
||||
jsonData[itemName] = zItem.toJSON(options);
|
||||
|
||||
// Adjut accessDate so that it doesn't depend on computer time zone
|
||||
// Effectively, assume that current time zone is UTC
|
||||
if (jsonData[itemName].accessDate) {
|
||||
let date = Zotero.Date.isoToDate(jsonData[itemName].accessDate);
|
||||
date.setUTCMinutes(date.getUTCMinutes() - date.getTimezoneOffset());
|
||||
jsonData[itemName].accessDate = Zotero.Date.dateToISO(date);
|
||||
}
|
||||
|
||||
// Don't replace some fields that _always_ change (e.g. item keys)
|
||||
// as long as it follows expected format
|
||||
// This makes it easier to generate more meaningful diffs
|
||||
if (!currentData || !currentData[itemName]) continue;
|
||||
|
||||
for (let field in jsonData[itemName]) {
|
||||
let oldVal = currentData[itemName][field];
|
||||
if (!oldVal) continue;
|
||||
|
||||
let val = jsonData[itemName][field];
|
||||
switch (field) {
|
||||
case 'dateAdded':
|
||||
case 'dateModified':
|
||||
if (!isoDateTimeRe.test(oldVal) || !isoDateTimeRe.test(val)) continue;
|
||||
break;
|
||||
case 'key':
|
||||
if (!zoteroObjectKeyRe.test(oldVal) || !zoteroObjectKeyRe.test(val)) continue;
|
||||
break;
|
||||
default:
|
||||
continue;
|
||||
}
|
||||
|
||||
jsonData[itemName][field] = oldVal;
|
||||
}
|
||||
}
|
||||
|
||||
return jsonData;
|
||||
}
|
||||
|
||||
function generateCiteProcJSExportData(currentData) {
|
||||
let items = populateDBWithSampleData(loadSampleData('allTypesAndFields')),
|
||||
cslExportData = {};
|
||||
|
||||
for (let itemName in items) {
|
||||
let zItem = Zotero.Items.get(items[itemName].id);
|
||||
cslExportData[itemName] = Zotero.Cite.System.prototype.retrieveItem(zItem);
|
||||
|
||||
// Don't replace id as long as it follows expected format
|
||||
if (!currentData || !currentData[itemName]) continue;
|
||||
|
||||
// For simplicity, be more lenient than for item key
|
||||
let idRe = /^http:\/\/zotero\.org\/users\/local\/\w{8}\/items\/\w{8}$/;
|
||||
for (let field in cslExportData[itemName]) {
|
||||
let oldVal = currentData[itemName][field];
|
||||
if (!oldVal) continue;
|
||||
|
||||
let val = cslExportData[itemName][field];
|
||||
switch (field) {
|
||||
case 'id':
|
||||
if (!idRe.test(oldVal) || !idRe.test(val)) continue;
|
||||
break;
|
||||
default:
|
||||
continue;
|
||||
}
|
||||
|
||||
cslExportData[itemName][field] = oldVal;
|
||||
}
|
||||
}
|
||||
|
||||
return cslExportData;
|
||||
}
|
||||
|
||||
function generateTranslatorExportData(legacy, currentData) {
|
||||
let items = populateDBWithSampleData(loadSampleData('allTypesAndFields')),
|
||||
translatorExportData = {};
|
||||
|
||||
let itemGetter = new Zotero.Translate.ItemGetter();
|
||||
itemGetter.legacy = !!legacy;
|
||||
|
||||
for (let itemName in items) {
|
||||
let zItem = Zotero.Items.get(items[itemName].id);
|
||||
itemGetter._itemsLeft = [zItem];
|
||||
translatorExportData[itemName] = itemGetter.nextItem();
|
||||
|
||||
// Adjut ISO accessDate so that it doesn't depend on computer time zone
|
||||
// Effectively, assume that current time zone is UTC
|
||||
if (!legacy && translatorExportData[itemName].accessDate) {
|
||||
let date = Zotero.Date.isoToDate(translatorExportData[itemName].accessDate);
|
||||
date.setUTCMinutes(date.getUTCMinutes() - date.getTimezoneOffset());
|
||||
translatorExportData[itemName].accessDate = Zotero.Date.dateToISO(date);
|
||||
}
|
||||
|
||||
// Don't replace some fields that _always_ change (e.g. item keys)
|
||||
if (!currentData || !currentData[itemName]) continue;
|
||||
|
||||
// For simplicity, be more lenient than for item key
|
||||
let uriRe = /^http:\/\/zotero\.org\/users\/local\/\w{8}\/items\/\w{8}$/;
|
||||
let itemIDRe = /^\d+$/;
|
||||
for (let field in translatorExportData[itemName]) {
|
||||
let oldVal = currentData[itemName][field];
|
||||
if (!oldVal) continue;
|
||||
|
||||
let val = translatorExportData[itemName][field];
|
||||
switch (field) {
|
||||
case 'uri':
|
||||
if (!uriRe.test(oldVal) || !uriRe.test(val)) continue;
|
||||
break;
|
||||
case 'itemID':
|
||||
if (!itemIDRe.test(oldVal) || !itemIDRe.test(val)) continue;
|
||||
break;
|
||||
case 'key':
|
||||
if (!zoteroObjectKeyRe.test(oldVal) || !zoteroObjectKeyRe.test(val)) continue;
|
||||
break;
|
||||
case 'dateAdded':
|
||||
case 'dateModified':
|
||||
if (legacy) {
|
||||
if (!sqlDateTimeRe.test(oldVal) || !sqlDateTimeRe.test(val)) continue;
|
||||
} else {
|
||||
if (!isoDateTimeRe.test(oldVal) || !isoDateTimeRe.test(val)) continue;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
continue;
|
||||
}
|
||||
|
||||
translatorExportData[itemName][field] = oldVal;
|
||||
}
|
||||
}
|
||||
|
||||
return translatorExportData;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue